From ff87f40854b31792fc13e303b8b85e87d2a0698d Mon Sep 17 00:00:00 2001 From: Dmitriy Kovalenko Date: Wed, 5 Aug 2026 20:07:04 -0700 Subject: [PATCH] fix: Reduce amount of rescans in giant /Users/neogoose like folders --- Makefile | 21 +- crates/fff-core/Cargo.toml | 3 + crates/fff-core/build.rs | 9 + crates/fff-core/src/constants.rs | 15 +- crates/fff-core/src/file_picker.rs | 4 + crates/fff-core/src/ignore.rs | 84 +- crates/fff-core/src/lib.rs | 6 + crates/fff-core/src/rescan_stats.rs | 215 ++ crates/fff-core/src/rescan_throttle.rs | 124 + crates/fff-core/src/shared.rs | 61 +- crates/fff-core/src/simd_path.rs | 14 +- .../src/watcher/background_watcher.rs | 111 +- crates/fff-core/src/watcher/mod.rs | 4 + crates/fff-core/src/watcher/rescan_tests.rs | 621 +++++ crates/fff-core/tests/rescan_regression.rs | 343 +++ crates/fff-nvim/Cargo.toml | 2 + crates/fff-nvim/src/bin/rescan_probe.rs | 158 ++ packages/fff-bun/src/fff-api.ts | 15 +- packages/fff-node/src/fff-api.ts | 15 +- packages/fff-node/src/ffi.ts | 70 +- packages/pi-fff/src/aux-finders.ts | 12 +- packages/pi-fff/src/index.ts | 2245 ++++++++--------- packages/pi-fff/src/query.ts | 6 +- packages/pi-fff/test/aux-finders.test.ts | 4 +- packages/shared/fff-api.ts | 15 +- 25 files changed, 2864 insertions(+), 1313 deletions(-) create mode 100644 crates/fff-core/src/rescan_stats.rs create mode 100644 crates/fff-core/src/rescan_throttle.rs create mode 100644 crates/fff-core/src/watcher/rescan_tests.rs create mode 100644 crates/fff-core/tests/rescan_regression.rs create mode 100644 crates/fff-nvim/src/bin/rescan_probe.rs diff --git a/Makefile b/Makefile index 92e45407e..e29cda330 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL := bash # string rather than the literal `-o` / `pipefail` tokens. .SHELLFLAGS := -o pipefail -euc -.PHONY: build build-c-lib install uninstall test test-rust test-c-smoke test-c-api test-lua test-lua-snap test-version test-bun test-node prepare-bun prepare-bun-packaged prepare-node set-npm-version header test-stress test-stress-seeded test-stress-random test-stress-regressions test-stress-repos test-node-stress sync-js-api sync-js-api-check bump-homebrew-formula bump-install-mcp-sh test-bun-compile +.PHONY: build build-c-lib install uninstall test test-rust test-rescan test-rescan-known-defects rescan-probe test-c-smoke test-c-api test-lua test-lua-snap test-version test-bun test-node prepare-bun prepare-bun-packaged prepare-node set-npm-version header test-stress test-stress-seeded test-stress-random test-stress-regressions test-stress-repos test-node-stress sync-js-api sync-js-api-check bump-homebrew-formula bump-install-mcp-sh test-bun-compile all: format test lint @@ -92,6 +92,25 @@ test-setup: test-rust: cargo test --workspace --no-default-features --features zlob --exclude fff-nvim +# Watcher rescan harness: asserts that editing, build output, git activity and +# preview reads all stay on the incremental path instead of re-walking the tree. +test-rescan: + cargo test -p fff-search --no-default-features --features zlob \ + --lib --test rescan_regression -- rescan + +# Live probe for watcher rescan requests and their causes. +# Usage: make rescan-probe DIR=~/some/repo [SECONDS=120] +rescan-probe: + cargo run --release -p fff-nvim --bin rescan_probe \ + --no-default-features --features zlob,rescan-stats -- \ + $(or $(DIR),.) $(if $(SECONDS),--seconds $(SECONDS),) + +# The same harness, restricted to cases that currently fail on purpose. Each +# `#[ignore]` reason names the code that causes the unnecessary rescan. +test-rescan-known-defects: + cargo test --no-fail-fast -p fff-search --no-default-features --features zlob \ + --lib --test rescan_regression -- --ignored --nocapture + CC ?= cc CFLAGS ?= -O0 -g -Wall -Wextra -std=c99 TARGET_DIR ?= target/release diff --git a/crates/fff-core/Cargo.toml b/crates/fff-core/Cargo.toml index 24416b074..9671904a7 100644 --- a/crates/fff-core/Cargo.toml +++ b/crates/fff-core/Cargo.toml @@ -41,6 +41,9 @@ harness = false default = ["ripgrep"] # Enable C FFI exports ffi = [] +# Count full rescans and their causes. Always on in debug builds; enable this +# to keep the accounting in a release build (used by the rescan_probe binary). +rescan-stats = [] # Enables POC definition classification for grep result matched lines definitions = [] # Pure-Rust filesystem walker + glob matcher (ignore + globset crates). diff --git a/crates/fff-core/build.rs b/crates/fff-core/build.rs index 1e4dfaa98..b9e1c40c8 100644 --- a/crates/fff-core/build.rs +++ b/crates/fff-core/build.rs @@ -3,6 +3,15 @@ fn main() { // used by tests/fuzz_git_watcher_stress.rs println!("cargo::rustc-check-cfg=cfg(stress)"); + // Full-rescan accounting. Debug builds get it for free; a release build has + // to opt in with `--features rescan-stats` (what the rescan_probe needs). + println!("cargo::rustc-check-cfg=cfg(rescan_stats)"); + if std::env::var("DEBUG").is_ok_and(|debug| debug != "false") + || std::env::var("CARGO_FEATURE_RESCAN_STATS").is_ok() + { + println!("cargo::rustc-cfg=rescan_stats"); + } + // When the `zlob` feature is enabled (Zig-compiled C library): // On Windows MSVC, explicitly link the C runtime libraries. // Zig-compiled static libraries don't emit /DEFAULTLIB directives for the diff --git a/crates/fff-core/src/constants.rs b/crates/fff-core/src/constants.rs index 95cdc2b06..31042078d 100644 --- a/crates/fff-core/src/constants.rs +++ b/crates/fff-core/src/constants.rs @@ -12,16 +12,25 @@ pub const MAX_INDEXABLE_FILE_SIZE: usize = 2 * 1024 * 1024; pub const MAX_CACHED_CONTENT_BYTES: u64 = 512 * 1024 * 1024; /// Files below one page waste the remainder when mmapped, so the cache skips -/// them and falls back to chunked reads. Unused on Windows (no content cache). +/// them and falls back to chunked reads. Unused on Windows (no content cache) #[cfg(all(not(target_os = "windows"), target_arch = "aarch64"))] pub const MMAP_THRESHOLD: u64 = 16 * 1024; #[cfg(all(not(target_os = "windows"), not(target_arch = "aarch64")))] pub const MMAP_THRESHOLD: u64 = 4 * 1024; -/// Capacity reserved for files the watcher discovers after the initial scan; -/// exceeding it forces a full rescan. +/// Watcher overflow capacity reserved after the initial scan pub const MAX_OVERFLOW_FILES: usize = 1024; +/// Minimum delay between watcher-initiated rescans. +pub const RESCAN_MIN_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30); + +/// Rescan delay for large indexes. +pub const RESCAN_MIN_INTERVAL_LARGE_INDEX: std::time::Duration = + std::time::Duration::from_secs(5 * 60); + +/// Live-file count at which [`RESCAN_MIN_INTERVAL_LARGE_INDEX`] takes over. +pub const LARGE_INDEX_FILE_COUNT: usize = 1_000_000; + /// Fresh-mmap threshold: files at or above this size get mmapped directly on /// cache miss instead of chunked reads into Vec. Empirically tuned per-platform. /// Only referenced on Unix; Windows uses the `std::fs::read` fallback so this diff --git a/crates/fff-core/src/file_picker.rs b/crates/fff-core/src/file_picker.rs index 36b93269c..bbb62096b 100644 --- a/crates/fff-core/src/file_picker.rs +++ b/crates/fff-core/src/file_picker.rs @@ -629,6 +629,10 @@ impl FilePicker { &self.base_path } + pub fn has_git_repo(&self) -> bool { + self.sync_data.git_workdir.is_some() + } + /// Ignore rules the walker assembled during the last scan (zlob backend /// only). The background watcher uses these to filter events without /// libgit2. `None` when the backend doesn't surface rules or no ignore diff --git a/crates/fff-core/src/ignore.rs b/crates/fff-core/src/ignore.rs index ac29f523a..b24da252c 100644 --- a/crates/fff-core/src/ignore.rs +++ b/crates/fff-core/src/ignore.rs @@ -3,28 +3,53 @@ use std::path::Path; /// Directories excluded when walking a non-git root. Entries are `cfg`-gated /// so a single iteration covers standard + platform-specific overrides. pub(crate) const IGNORED_DIRS: &[&str] = &[ + // various dev tools that can be meet in the developer app "node_modules", "__pycache__", "venv", ".venv", - // Rust (glob-only patterns for non_git_repo_overrides; is_non_code_directory - // matches the "target" component separately). "target/debug", "target/release", "target/rust-analyzer", "target/criterion", + // Language package caches in non-git roots. + "go/pkg/mod", + ".cargo/registry", + ".rustup/toolchains", + ".gradle/caches", + ".m2/repository", + ".npm/_cacache", + ".pub-cache", + #[cfg(not(target_os = "windows"))] + ".local/state", // this contains tons of logs which generate too much watcher noise #[cfg(target_os = "macos")] "Library/Application Support", #[cfg(target_os = "macos")] "Library/Caches", - // App-group sandbox storage — used by iMessage, Photos, Notes, Calendar, - // Electron apps, etc. for SQLite-WAL, LevelDB, protobuf files. These are - // almost entirely extension-less binary files (~80k on a typical $HOME) - // that never need to appear in a fuzzy or grep search. #[cfg(target_os = "macos")] - "Library/Group Containers", + "Library/Containers", // sandboxed apps data #[cfg(target_os = "macos")] - "Library/Containers", + "Library/Group Containers", // random application data and networking + #[cfg(target_os = "macos")] + "Library/pnpm", + #[cfg(target_os = "macos")] + "Library/Metadata", + #[cfg(target_os = "macos")] + "Library/Developer/CoreSimulator", + #[cfg(target_os = "macos")] + "Library/Android", + #[cfg(target_os = "macos")] + "Library/Logs", + #[cfg(target_os = "macos")] + "Library/Daemon Containers", + #[cfg(target_os = "macos")] + "Library/Trial", + #[cfg(target_os = "macos")] + "Library/Preferences", + #[cfg(target_os = "macos")] + "Library/Messages", + #[cfg(target_os = "macos")] + "Library/IdentityServices", #[cfg(target_os = "windows")] "bin/Debug", #[cfg(target_os = "windows")] @@ -57,6 +82,10 @@ pub(crate) fn non_git_repo_overrides(base_path: &Path) -> Option bool { let path_str = path.as_os_str().to_str().unwrap_or(""); IGNORED_DIRS.iter().any(|&dir| { + // Entries are gitignore patterns for the walkers; here they are matched + // as substrings, so a leading `*` wildcard has to come off first. + let dir = dir.strip_prefix('*').unwrap_or(dir); + #[cfg(target_os = "windows")] let dir = dir.replace('/', std::path::MAIN_SEPARATOR_STR); #[cfg(target_os = "windows")] @@ -66,3 +95,42 @@ pub(crate) fn is_non_code_directory(path: &Path) -> bool { path_str.contains(dir) }) } + +#[cfg(all(test, target_os = "macos"))] +mod tests { + use super::*; + + #[test] + fn home_machine_state_is_excluded_but_source_trees_are_not() { + // Representative machine state from a home index. + for rel in [ + "Library/pnpm/store/v3/files/00/abcdef", + "Library/Preferences/com.apple.finder.plist", + "Library/Messages/prewarm.db-shm", + "Library/IdentityServices/TetraDB-identityservicesd.db-wal", + "Library/Developer/CoreSimulator/Devices/X/data/f", + "go/pkg/mod/github.com/x/y@v1/main.go", + ".cargo/registry/src/index.crates.io-1/serde-1.0/src/lib.rs", + "Library/Android/sdk/platforms/android-34/data/x", + ".local/state/nvim/fff+123+456.log", + ] { + assert!( + is_non_code_directory(Path::new(rel)), + "{rel} must not reach the index" + ); + } + + // Source trees under $HOME stay searchable. + for rel in [ + "dev/chromium/third_party/blink/renderer/core/dom/node.cc", + "dev/fff.nvim/crates/fff-core/src/lib.rs", + "Documents/notes/todo.md", + "dev/myproj/pkg/mod/thing.go", + ] { + assert!( + !is_non_code_directory(Path::new(rel)), + "{rel} must stay searchable" + ); + } + } +} diff --git a/crates/fff-core/src/lib.rs b/crates/fff-core/src/lib.rs index ff4bdfc30..a6c514a49 100644 --- a/crates/fff-core/src/lib.rs +++ b/crates/fff-core/src/lib.rs @@ -135,6 +135,12 @@ pub use types::*; pub mod constants; +/// Watcher rescan request accounting. +pub mod rescan_stats; +pub use rescan_stats::{RESCAN_STATS_ENABLED, RescanReason, RescanStats}; + +mod rescan_throttle; + // ================================== // these are public only for benchmarks, no backward compatibility guaranteed #[doc(hidden)] diff --git a/crates/fff-core/src/rescan_stats.rs b/crates/fff-core/src/rescan_stats.rs new file mode 100644 index 000000000..b61f03cf6 --- /dev/null +++ b/crates/fff-core/src/rescan_stats.rs @@ -0,0 +1,215 @@ +#[cfg(rescan_stats)] +use std::sync::atomic::{AtomicUsize, Ordering}; + +/// Whether rescan accounting is compiled in. +pub const RESCAN_STATS_ENABLED: bool = cfg!(rescan_stats); + +/// Cause recorded for a filesystem rescan request. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum RescanReason { + /// Requested through the public API (refresh, directory change). + Explicit, + /// The kernel dropped events and asked us to re-read the subtree. + KernelEventLoss, + /// A `.gitignore`/`.ignore` changed, so the cached ignore rules are stale. + IgnoreFileChanged, + /// A single debounce batch touched more paths than we apply incrementally. + EventBatchOverflow, + /// The picker refused an incremental insert/update. + IndexUpdateRejected, + /// The post-scan overflow region ran out of slots. + OverflowCapacity, +} + +impl RescanReason { + pub const ALL: [RescanReason; 6] = [ + RescanReason::Explicit, + RescanReason::KernelEventLoss, + RescanReason::IgnoreFileChanged, + RescanReason::EventBatchOverflow, + RescanReason::IndexUpdateRejected, + RescanReason::OverflowCapacity, + ]; + + pub const fn as_str(self) -> &'static str { + match self { + RescanReason::Explicit => "explicit", + RescanReason::KernelEventLoss => "kernel_event_loss", + RescanReason::IgnoreFileChanged => "ignore_file_changed", + RescanReason::EventBatchOverflow => "event_batch_overflow", + RescanReason::IndexUpdateRejected => "index_update_rejected", + RescanReason::OverflowCapacity => "overflow_capacity", + } + } + + const fn slot(self) -> usize { + match self { + RescanReason::Explicit => 0, + RescanReason::KernelEventLoss => 1, + RescanReason::IgnoreFileChanged => 2, + RescanReason::EventBatchOverflow => 3, + RescanReason::IndexUpdateRejected => 4, + RescanReason::OverflowCapacity => 5, + } + } +} + +impl std::fmt::Display for RescanReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Snapshot of rescan requests grouped by reason. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct RescanStats { + pub total: usize, + /// Requests suppressed during the cooldown. + pub throttled: usize, + counts: [usize; RescanReason::ALL.len()], + throttled_counts: [usize; RescanReason::ALL.len()], +} + +impl RescanStats { + pub fn count(&self, reason: RescanReason) -> usize { + self.counts[reason.slot()] + } + + pub fn count_throttled(&self, reason: RescanReason) -> usize { + self.throttled_counts[reason.slot()] + } + + /// Admitted requests originating from watcher fallbacks. + pub fn watcher_triggered(&self) -> usize { + self.total - self.count(RescanReason::Explicit) + } + + /// Per-reason delta against an earlier snapshot. + pub fn since(&self, earlier: &RescanStats) -> RescanStats { + let mut counts = [0usize; RescanReason::ALL.len()]; + let mut throttled_counts = [0usize; RescanReason::ALL.len()]; + for slot in 0..RescanReason::ALL.len() { + counts[slot] = self.counts[slot].saturating_sub(earlier.counts[slot]); + throttled_counts[slot] = + self.throttled_counts[slot].saturating_sub(earlier.throttled_counts[slot]); + } + + RescanStats { + total: self.total.saturating_sub(earlier.total), + throttled: self.throttled.saturating_sub(earlier.throttled), + counts, + throttled_counts, + } + } +} + +impl std::fmt::Display for RescanStats { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} rescan(s)", self.total)?; + let mut first = true; + for reason in RescanReason::ALL { + let count = self.count(reason); + if count == 0 { + continue; + } + f.write_str(if first { " [" } else { ", " })?; + write!(f, "{reason}={count}")?; + first = false; + } + if !first { + f.write_str("]")?; + } + if self.throttled > 0 { + write!(f, ", {} throttled", self.throttled)?; + } + Ok(()) + } +} + +#[cfg(rescan_stats)] +#[derive(Default)] +pub(crate) struct RescanCounters { + counters: [AtomicUsize; RescanReason::ALL.len()], + throttled: [AtomicUsize; RescanReason::ALL.len()], +} + +#[cfg(rescan_stats)] +impl RescanCounters { + pub(crate) fn record(&self, reason: RescanReason) { + self.counters[reason.slot()].fetch_add(1, Ordering::Relaxed); + } + + pub(crate) fn record_throttled(&self, reason: RescanReason) { + self.throttled[reason.slot()].fetch_add(1, Ordering::Relaxed); + } + + pub(crate) fn snapshot(&self) -> RescanStats { + let mut stats = RescanStats::default(); + for reason in RescanReason::ALL { + let count = self.counters[reason.slot()].load(Ordering::Relaxed); + stats.counts[reason.slot()] = count; + stats.total += count; + + let throttled = self.throttled[reason.slot()].load(Ordering::Relaxed); + stats.throttled_counts[reason.slot()] = throttled; + stats.throttled += throttled; + } + stats + } + + pub(crate) fn reset(&self) { + for counter in self.counters.iter().chain(self.throttled.iter()) { + counter.store(0, Ordering::Relaxed); + } + } +} + +// Release builds retain the API without counter storage. +#[cfg(not(rescan_stats))] +#[derive(Default)] +pub(crate) struct RescanCounters; + +#[cfg(not(rescan_stats))] +impl RescanCounters { + pub(crate) fn record(&self, _reason: RescanReason) {} + + pub(crate) fn record_throttled(&self, _reason: RescanReason) {} + + pub(crate) fn snapshot(&self) -> RescanStats { + RescanStats::default() + } + + pub(crate) fn reset(&self) {} +} + +#[cfg(all(test, rescan_stats))] +mod tests { + use super::*; + + #[test] + fn counters_attribute_and_diff_per_reason() { + let counters = RescanCounters::default(); + counters.record(RescanReason::Explicit); + let baseline = counters.snapshot(); + + counters.record(RescanReason::IgnoreFileChanged); + counters.record(RescanReason::IgnoreFileChanged); + counters.record(RescanReason::OverflowCapacity); + + let stats = counters.snapshot(); + assert_eq!(stats.total, 4); + assert_eq!(stats.watcher_triggered(), 3); + + let delta = stats.since(&baseline); + assert_eq!(delta.total, 3); + assert_eq!(delta.count(RescanReason::Explicit), 0); + assert_eq!(delta.count(RescanReason::IgnoreFileChanged), 2); + assert_eq!( + delta.to_string(), + "3 rescan(s) [ignore_file_changed=2, overflow_capacity=1]" + ); + + counters.reset(); + assert_eq!(counters.snapshot(), RescanStats::default()); + } +} diff --git a/crates/fff-core/src/rescan_throttle.rs b/crates/fff-core/src/rescan_throttle.rs new file mode 100644 index 000000000..2b8cb7b48 --- /dev/null +++ b/crates/fff-core/src/rescan_throttle.rs @@ -0,0 +1,124 @@ +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Instant; + +use crate::constants::{ + LARGE_INDEX_FILE_COUNT, RESCAN_MIN_INTERVAL, RESCAN_MIN_INTERVAL_LARGE_INDEX, +}; + +const NEVER: u64 = u64::MAX; + +// Drops watcher rescan requests inside the cooldown after the last scan. +// A slightly stale index is fine: the next admitted event rescans everything. +pub(crate) struct RescanThrottle { + epoch: Instant, + last_admitted: AtomicU64, +} + +impl Default for RescanThrottle { + fn default() -> Self { + Self { + epoch: Instant::now(), + last_admitted: AtomicU64::new(NEVER), + } + } +} + +impl RescanThrottle { + /// Returns `true` if a rescan may start now and records it as the last scan + pub(crate) fn admit(&self, live_files: usize, has_git_repo: bool) -> bool { + let min_interval = if !has_git_repo && live_files >= LARGE_INDEX_FILE_COUNT { + RESCAN_MIN_INTERVAL_LARGE_INDEX + } else { + RESCAN_MIN_INTERVAL + }; + + let min_ms = min_interval.as_millis() as u64; + let now = self.elapsed_ms(); + + loop { + let last = self.last_admitted.load(Ordering::Acquire); + if last != NEVER && now.saturating_sub(last) < min_ms { + return false; + } + // CAS so two concurrent requests cannot both start a walk. + if self + .last_admitted + .compare_exchange(last, now, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + return true; + } + } + } + + /// Records an explicit (unthrottled) scan so watcher requests right after + /// it are dropped: the index is already fresh. + pub(crate) fn note_explicit_scan(&self) { + self.last_admitted + .store(self.elapsed_ms(), Ordering::Release); + } + + fn elapsed_ms(&self) -> u64 { + self.epoch.elapsed().as_millis() as u64 + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + fn throttle_at(ms_ago: u64) -> RescanThrottle { + let now = Instant::now(); + RescanThrottle { + epoch: now + .checked_sub(Duration::from_millis(ms_ago)) + .expect("monotonic clock older than the rewind"), + last_admitted: AtomicU64::new(0), + } + } + + #[test] + fn first_request_is_always_admitted() { + let throttle = RescanThrottle::default(); + assert!(throttle.admit(100, true)); + } + + #[test] + fn requests_inside_the_cooldown_are_dropped() { + let throttle = throttle_at(1_000); + assert!(!throttle.admit(100, false)); + assert!(!throttle.admit(100, false)); + } + + #[test] + fn a_large_index_outside_a_git_repo_uses_the_slower_cadence() { + // A minute is past the normal cooldown but not the large-index one. + let throttle = throttle_at(60_000); + assert!(throttle.admit(100, false)); + + let throttle = throttle_at(60_000); + assert!(!throttle.admit(LARGE_INDEX_FILE_COUNT, false)); + } + + #[test] + fn a_git_repo_keeps_the_normal_cadence_at_any_size() { + let throttle = throttle_at(60_000); + assert!(throttle.admit(LARGE_INDEX_FILE_COUNT, true)); + } + + #[test] + fn cooldown_expiry_admits_again() { + let throttle = throttle_at(RESCAN_MIN_INTERVAL.as_millis() as u64 + 1); + assert!(throttle.admit(100, false)); + // Admission rearms the cooldown. + assert!(!throttle.admit(100, false)); + } + + #[test] + fn explicit_scan_rearms_the_cooldown() { + let throttle = RescanThrottle::default(); + throttle.note_explicit_scan(); + assert!(!throttle.admit(100, false)); + } +} diff --git a/crates/fff-core/src/shared.rs b/crates/fff-core/src/shared.rs index 71cda38dd..06883ea8f 100644 --- a/crates/fff-core/src/shared.rs +++ b/crates/fff-core/src/shared.rs @@ -8,6 +8,8 @@ use crate::file_picker::FilePicker; use crate::frecency::FrecencyTracker; use crate::git::GitStatusCache; use crate::query_tracker::QueryTracker; +use crate::rescan_stats::{RescanCounters, RescanReason, RescanStats}; +use crate::rescan_throttle::RescanThrottle; use crate::scan::ScanJob; use crate::watch::{WatchEvent, WatchId, WatchOptions, WatchRegistry}; use git2::Repository; @@ -77,6 +79,8 @@ pub struct SharedPickerInner { /// Watch subscriptions live outside the picker lock so delivery and /// (un)subscribing never contend with searches. watchers: Arc, + rescans: RescanCounters, + rescan_throttle: RescanThrottle, } impl Default for SharedPickerInner { @@ -84,6 +88,8 @@ impl Default for SharedPickerInner { Self { picker: parking_lot::RwLock::new(None), watchers: Arc::new(WatchRegistry::default()), + rescans: RescanCounters::default(), + rescan_throttle: RescanThrottle::default(), } } } @@ -199,6 +205,39 @@ impl SharedFilePicker { /// Performs a safe async rescan. Guarantees only single active rescan per picker. /// If many rescans requested the last one guaranteed to be finished. pub fn trigger_full_rescan_async(&self, shared_frecency: &SharedFrecency) -> Result<(), Error> { + self.trigger_full_rescan_with_reason(shared_frecency, RescanReason::Explicit) + .map(|_| ()) + } + + /// Returns admitted and throttled rescan requests by reason. + /// Counters start at picker creation or the last reset. + pub fn rescan_stats(&self) -> RescanStats { + self.0.rescans.snapshot() + } + + pub fn reset_rescan_stats(&self) { + self.0.rescans.reset(); + } + + /// Returns `Ok(true)` when a rescan was started (or queued behind an + /// active scan) and `Ok(false)` when the request was throttled — the + /// caller must then fall back to incremental event processing. + pub(crate) fn trigger_full_rescan_with_reason( + &self, + shared_frecency: &SharedFrecency, + reason: RescanReason, + ) -> Result { + // for giant folders we have no other choice other than throttling rescans + // if user is running application in millions of files with a ton of rescan events + // we drop / throttle some of requests to avoid constant burst of IO + if reason == RescanReason::Explicit { + self.0.rescan_throttle.note_explicit_scan(); + } else if !self.check_rescan_throttle(reason) { + return Ok(false); + } + + self.0.rescans.record(reason); + match ScanJob::new_rescan(self, shared_frecency)? { Some(job) => { job.spawn(); @@ -219,7 +258,27 @@ impl SharedFilePicker { } } } - Ok(()) + Ok(true) + } + + fn check_rescan_throttle(&self, reason: RescanReason) -> bool { + let (live_files, has_git) = self + .read() + .ok() + .and_then(|guard| { + guard + .as_ref() + .map(|picker| (picker.live_file_count(), picker.has_git_repo())) + }) + .unwrap_or((0, false)); + + if self.0.rescan_throttle.admit(live_files, has_git) { + return true; + } + + self.0.rescans.record_throttled(reason); + tracing::debug!(%reason, live_files, "Rescan throttled, skipping"); + false } /// Subscribe to filesystem changes matching `pattern`. diff --git a/crates/fff-core/src/simd_path.rs b/crates/fff-core/src/simd_path.rs index 881f10962..0795e042e 100644 --- a/crates/fff-core/src/simd_path.rs +++ b/crates/fff-core/src/simd_path.rs @@ -415,7 +415,10 @@ mod tests { #[test] fn test_chunked_string_full_path() { - let (store, strings, _files) = build_test_store(&["src/components/Button.tsx"]); + let (store, strings, _files) = build_test_store(&[ + "src/components/Button.tsx", + "src/components/Button.test.tsx", + ]); let arena = store.as_arena_ptr(); let cs = &strings[0]; @@ -423,6 +426,15 @@ mod tests { assert_eq!(cs.read_to_buf(arena, &mut buf), "src/components/Button.tsx"); assert_eq!(cs.byte_len, 25); assert_eq!(cs.filename_offset, 15); + + let cs = &strings[1]; + let mut buf = [0u8; 512]; + assert_eq!( + cs.read_to_buf(arena, &mut buf), + "src/components/Button.test.tsx" + ); + assert_eq!(cs.byte_len, 30); + assert_eq!(cs.filename_offset, 15); } #[test] diff --git a/crates/fff-core/src/watcher/background_watcher.rs b/crates/fff-core/src/watcher/background_watcher.rs index 73568a877..b6d6d7140 100644 --- a/crates/fff-core/src/watcher/background_watcher.rs +++ b/crates/fff-core/src/watcher/background_watcher.rs @@ -2,6 +2,7 @@ use crate::constants::MAX_OVERFLOW_FILES; use crate::error::Error; use crate::file_picker::FFFMode; use crate::git_status_worker::GitStatusWorker; +use crate::rescan_stats::RescanReason; use crate::shared::{SharedFilePicker, SharedFrecency}; use crate::sort_buffer::sort_with_buffer; use crate::watch::{RawWatchEvent, WatchEventKind}; @@ -324,7 +325,7 @@ impl Drop for BackgroundWatcher { } #[tracing::instrument(name = "fs_events", skip(events, shared_picker, shared_frecency, git_status_worker), level = Level::DEBUG)] -fn handle_debounced_events( +pub(crate) fn handle_debounced_events( mode: FFFMode, events: Vec, base_path: &Path, @@ -342,8 +343,8 @@ fn handle_debounced_events( .ok() .and_then(|g| g.as_ref().and_then(|p| p.ignore_rules())); let filter = IgnoreFilter::new(base_path, walker_rules, repo.as_ref()); - let mut need_full_rescan = false; let mut need_full_git_rescan = false; + let mut batch_overflow_attempted = false; let mut paths_to_remove = Vec::new(); let mut dirs_to_remove: Vec = Vec::new(); let mut paths_to_add_or_modify = Vec::new(); @@ -353,6 +354,21 @@ fn handle_debounced_events( let watch_registry = shared_picker.watch_registry(); let need_events_propagation = watch_registry.is_active(); + let try_trigger_full_rescan = |reason: RescanReason| -> bool { + match shared_picker.trigger_full_rescan_with_reason(shared_frecency, reason) { + Ok(true) => { + warn!(%reason, "Triggering full rescan"); + watch_registry.dispatch_rescan(base_path); + true + } + Ok(false) => false, + Err(e) => { + error!(%reason, "Failed to trigger full rescan: {:?}", e); + false + } + } + }; + for debounced_event in &events { // It is very important to not react to the access errors because we inevitably // gonna trigger the sync by our own preview or other unnecessary noise @@ -370,22 +386,19 @@ fn handle_debounced_events( // When macOS FSEvents (or other backends) overflow their event buffer, the kernel // drops individual events and emits a rescan flag telling us to re-scan the subtree if debounced_event.event.need_rescan() { - if debounced_event.event.paths.len() < 16 // this should be usually one event + let small_and_known = debounced_event.event.paths.len() < 16 // this should be usually one event && debounced_event .paths .iter() // but we are smart enough and not falling into the paths - .all(|p| !p.is_dir() && !filter.is_ignored(p)) - { - break; + .all(|p| !p.is_dir() && !filter.is_ignored(p)); + + if !small_and_known && try_trigger_full_rescan(RescanReason::KernelEventLoss) { + return Vec::new(); } - warn!( - "Received rescan event for paths {:?}, triggering full rescan", - debounced_event.event.paths - ); - need_full_rescan = true; - break; + // Small batches and throttled rescans fall through: the listed + // paths are still applied incrementally below. } tracing::debug!(event = ?debounced_event.event, "Processing FS event"); @@ -394,13 +407,24 @@ fn handle_debounced_events( path.file_name().and_then(|f| f.to_str()), Some(".ignore") | Some(".gitignore") ) { + if path + .parent() + .is_some_and(|parent| filter.is_ignored(parent)) + { + continue; + } + info!( "Detected change in ignore definition file: {}", path.display() ); - need_full_rescan = true; - break; + if try_trigger_full_rescan(RescanReason::IgnoreFileChanged) { + return Vec::new(); + } + + // Throttled: fall through so the ignore file itself stays + // indexed; the stale rules heal on the next admitted rescan. } if is_dotgit_change_affecting_status(path, &repo) { @@ -462,29 +486,18 @@ fn handle_debounced_events( } affected_paths_count += debounced_event.event.paths.len(); - if affected_paths_count > MAX_OVERFLOW_FILES { + if !batch_overflow_attempted && affected_paths_count > MAX_OVERFLOW_FILES * 4 { + batch_overflow_attempted = true; warn!( ?affected_paths_count, - max = MAX_OVERFLOW_FILES, + max = MAX_OVERFLOW_FILES * 4, "Too many affected paths in a single batch, triggering full rescan", ); - need_full_rescan = true; - break; - } - - if need_full_rescan { - break; - } - } - - if need_full_rescan { - info!(?affected_paths_count, "Triggering full rescan"); - watch_registry.dispatch_rescan(base_path); - if let Err(e) = shared_picker.trigger_full_rescan_async(shared_frecency) { - error!("Failed to trigger full rescan: {:?}", e); + if try_trigger_full_rescan(RescanReason::EventBatchOverflow) { + return Vec::new(); + } } - return Vec::new(); } // It's important to get the allocated sort @@ -511,7 +524,7 @@ fn handle_debounced_events( } let mut files_to_update_git_status = Vec::new(); - let mut need_full_rescan = false; + let mut index_update_rejected = false; let mut overflow_count = 0; let mut removed_from_dirs = Vec::new(); let mut watch_events = ahash::AHashMap::new(); @@ -572,6 +585,13 @@ fn handle_debounced_events( files_to_update_git_status.reserve(paths_to_add_or_modify.len()); for path in &paths_to_add_or_modify { + if picker.get_overflow_files().len() >= MAX_OVERFLOW_FILES + && picker.get_file_by_path(path).is_none() + { + index_update_rejected = true; + break; + } + let existed = need_events_propagation && picker.get_file_by_path(path).is_some(); if picker.handle_create_or_modify(path).is_some() { @@ -586,7 +606,7 @@ fn handle_debounced_events( watch_events.insert(path.to_path_buf(), kind); } } else { - need_full_rescan = true; + index_update_rejected = true; } } @@ -598,13 +618,22 @@ fn handle_debounced_events( overflow_count, "File index changes applied", ); - if need_full_rescan || overflow_count > MAX_OVERFLOW_FILES { - info!("Watcher faced limit of index overflow. Triggering rescan"); - watch_registry.dispatch_rescan(base_path); - if let Err(e) = shared_picker.trigger_full_rescan_async(shared_frecency) { - error!("Failed to trigger full rescan: {:?}", e); - } - } else if need_events_propagation { + let rescan_started = if index_update_rejected || overflow_count > MAX_OVERFLOW_FILES { + let reason = if index_update_rejected { + RescanReason::IndexUpdateRejected + } else { + RescanReason::OverflowCapacity + }; + + info!(%reason, "Watcher faced limit of index overflow. Triggering rescan"); + try_trigger_full_rescan(reason) + } else { + false + }; + + // When the rescan is throttled the incrementally applied changes are + // still the freshest state we have — propagate them to subscribers. + if !rescan_started && need_events_propagation { watch_registry.dispatch( base_path, watch_events @@ -664,7 +693,7 @@ fn handle_debounced_events( // do not try to update the paths if we anyway going to rescan everything from scratch // no repo => no consumer thread, so don't accumulate paths nobody will drain - if !need_full_rescan && repo.is_some() { + if !index_update_rejected && repo.is_some() { if need_full_git_rescan { // A full git rescan re-reads every tracked path (including ones that just // went clean after a commit), so it already subsumes the per-path update. diff --git a/crates/fff-core/src/watcher/mod.rs b/crates/fff-core/src/watcher/mod.rs index 4a1b390b1..ba2e51ab8 100644 --- a/crates/fff-core/src/watcher/mod.rs +++ b/crates/fff-core/src/watcher/mod.rs @@ -3,3 +3,7 @@ pub use background_watcher::*; mod watch; pub use watch::*; + +// The harness reads rescan counters, which release builds compile out. +#[cfg(all(test, rescan_stats))] +mod rescan_tests; diff --git a/crates/fff-core/src/watcher/rescan_tests.rs b/crates/fff-core/src/watcher/rescan_tests.rs new file mode 100644 index 000000000..1a6c4ef66 --- /dev/null +++ b/crates/fff-core/src/watcher/rescan_tests.rs @@ -0,0 +1,621 @@ +use std::path::PathBuf; +use std::process::Command; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use notify::Event; +use notify::EventKind; +use notify::event::{ + AccessKind, AccessMode, CreateKind, DataChange, Flag, ModifyKind, RemoveKind, RenameMode, +}; +use notify_debouncer_full::DebouncedEvent; +use tempfile::TempDir; + +use super::handle_debounced_events; +use crate::constants::MAX_OVERFLOW_FILES; +use crate::file_picker::{FFFMode, FilePicker, FilePickerOptions}; +use crate::git_status_worker::GitStatusWorker; +use crate::rescan_stats::{RescanReason, RescanStats}; +use crate::shared::{SharedFilePicker, SharedFrecency}; + +#[test] +fn saving_an_indexed_file_stays_incremental() { + let f = Fixture::new(); + f.write("src/main.rs", "fn main() {}"); + f.index(); + + f.write("src/main.rs", "fn main() { println!(); }"); + let delta = f.feed([modify(f.path("src/main.rs"))]); + + f.assert_no_rescan(&delta, "saving a tracked file"); +} + +#[test] +fn editor_atomic_save_stays_incremental() { + let f = Fixture::new(); + f.write("src/main.rs", "fn main() {}"); + f.index(); + + // write-to-temp + rename-over-target, the way vim/VSCode/IntelliJ save. + f.write("src/main.rs", "fn main() { println!(); }"); + let target = f.path("src/main.rs"); + let temp = f.path("src/.main.rs.swp"); + let delta = f.feed([ + DebouncedEvent::new( + Event::new(EventKind::Create(CreateKind::File)).add_path(temp.clone()), + Instant::now(), + ), + DebouncedEvent::new( + Event::new(EventKind::Modify(ModifyKind::Name(RenameMode::From))) + .add_path(temp.clone()), + Instant::now(), + ), + DebouncedEvent::new( + Event::new(EventKind::Modify(ModifyKind::Name(RenameMode::To))) + .add_path(target.clone()), + Instant::now(), + ), + DebouncedEvent::new( + Event::new(EventKind::Remove(RemoveKind::File)).add_path(temp), + Instant::now(), + ), + ]); + + f.assert_no_rescan(&delta, "an atomic editor save"); + assert!(f.is_indexed("src/main.rs"), "target must stay indexed"); +} + +#[test] +fn creating_and_deleting_files_stays_incremental() { + let f = Fixture::new(); + f.write("src/main.rs", "fn main() {}"); + f.index(); + + f.write("src/added.rs", "pub fn added() {}"); + let created = f.feed([create(f.path("src/added.rs"))]); + f.assert_no_rescan(&created, "creating a file"); + assert!(f.is_indexed("src/added.rs")); + + f.remove("src/added.rs"); + let removed = f.feed([remove_file(f.path("src/added.rs"))]); + f.assert_no_rescan(&removed, "deleting a file"); + assert!(!f.is_indexed("src/added.rs")); +} + +#[test] +fn deleting_a_directory_stays_incremental() { + let f = Fixture::new(); + f.write("src/main.rs", "fn main() {}"); + f.write("src/nested/a.rs", ""); + f.write("src/nested/b.rs", ""); + f.index(); + + std::fs::remove_dir_all(f.path("src/nested")).unwrap(); + let delta = f.feed([DebouncedEvent::new( + Event::new(EventKind::Remove(RemoveKind::Folder)).add_path(f.path("src/nested")), + Instant::now(), + )]); + + f.assert_no_rescan(&delta, "deleting a directory"); + assert!(!f.is_indexed("src/nested/a.rs")); + assert!(f.is_indexed("src/main.rs")); +} + +#[test] +fn read_only_access_events_are_ignored() { + let f = Fixture::new(); + f.write("src/main.rs", "fn main() {}"); + f.index(); + + // fff's own preview + grep reads generate these; reacting to them would + // make the picker rescan whenever the user scrolls the result list. + let path = f.path("src/main.rs"); + let delta = f.feed([ + DebouncedEvent::new( + Event::new(EventKind::Access(AccessKind::Read)).add_path(path.clone()), + Instant::now(), + ), + DebouncedEvent::new( + Event::new(EventKind::Access(AccessKind::Open(AccessMode::Read))) + .add_path(path.clone()), + Instant::now(), + ), + DebouncedEvent::new( + Event::new(EventKind::Access(AccessKind::Close(AccessMode::Read))).add_path(path), + Instant::now(), + ), + ]); + + f.assert_no_rescan(&delta, "read-only access events"); +} + +#[test] +fn recreating_the_same_paths_does_not_consume_overflow_capacity() { + let f = Fixture::new(); + f.write("src/main.rs", "fn main() {}"); + f.index(); + + // Recreated paths must reuse their overflow slots. + for _ in 0..8 { + for i in 0..200 { + let rel = format!("gen/out{i}.rs"); + f.write(&rel, "generated"); + f.feed([create(f.path(&rel))]); + } + for i in 0..200 { + let rel = format!("gen/out{i}.rs"); + f.remove(&rel); + f.feed([remove_file(f.path(&rel))]); + } + } + + let delta = f.all_rescans(); + f.assert_no_rescan(&delta, "1600 create/delete cycles over 200 stable paths"); + assert!( + f.overflow_len() <= 200, + "each path must claim one overflow slot at most, got {}", + f.overflow_len() + ); +} + +#[test] +fn writes_inside_a_gitignored_directory_stay_incremental() { + let f = Fixture::with_git(); + f.write(".gitignore", "target/\nnode_modules/\n"); + f.write("src/main.rs", "fn main() {}"); + f.index(); + + let mut events = Vec::new(); + for i in 0..64 { + let rel = format!("target/debug/artifact{i}.o"); + f.write(&rel, "binary"); + events.push(create(f.path(&rel))); + } + + let delta = f.feed(events); + f.assert_no_rescan(&delta, "build output written into an ignored directory"); +} + +#[test] +fn ignored_event_batch_above_index_capacity_stays_incremental() { + let f = Fixture::with_git(); + f.write(".gitignore", "node_modules/\n"); + f.write("src/main.rs", "fn main() {}"); + f.index(); + + let events = (0..MAX_OVERFLOW_FILES + 1) + .map(|i| { + let rel = format!("node_modules/pkg/file{i}.js"); + f.write(&rel, ""); + create(f.path(&rel)) + }) + .collect::>(); + + let delta = f.feed(events); + f.assert_no_rescan(&delta, "ignored events above the index capacity"); + assert_eq!(f.overflow_len(), 0); +} + +#[test] +fn repeated_edits_above_index_capacity_stay_incremental() { + let f = Fixture::new(); + f.write("src/main.rs", "fn main() {}"); + f.index(); + + let path = f.path("src/main.rs"); + let events = (0..MAX_OVERFLOW_FILES + 1) + .map(|_| modify(path.clone())) + .collect::>(); + + let delta = f.feed(events); + f.assert_no_rescan(&delta, "repeated edits above the index capacity"); + assert_eq!(f.overflow_len(), 0); +} + +#[test] +fn ignore_file_inside_an_ignored_directory_stays_incremental() { + let f = Fixture::with_git(); + f.write(".gitignore", "node_modules/\n"); + f.write("src/main.rs", "fn main() {}"); + f.index(); + + let ignore_files = + ["left-pad", "lodash", "typescript"].map(|pkg| format!("node_modules/{pkg}/.gitignore")); + + for rel in &ignore_files { + f.write(rel, "dist\n"); + } + let delta = f.feed(ignore_files.iter().map(|rel| create(f.path(rel)))); + f.assert_no_rescan(&delta, "creating ignored .gitignore files"); + + for rel in &ignore_files { + f.write(rel, "build\n"); + } + let delta = f.feed(ignore_files.iter().map(|rel| modify(f.path(rel)))); + f.assert_no_rescan(&delta, "modifying ignored .gitignore files"); + + for rel in &ignore_files { + f.remove(rel); + } + let delta = f.feed(ignore_files.iter().map(|rel| remove_file(f.path(rel)))); + f.assert_no_rescan(&delta, "removing ignored .gitignore files"); +} + +#[test] +fn ignore_file_inside_an_indexed_directory_triggers_a_rescan() { + let f = Fixture::with_git(); + f.write("src/.gitignore", ".gitignore\ngenerated/\n"); + f.write("src/main.rs", "fn main() {}"); + f.index(); + + f.write("src/.gitignore", ".gitignore\ngenerated/\nbuild/\n"); + let delta = f.feed([modify(f.path("src/.gitignore"))]); + + assert_eq!(delta.count(RescanReason::IgnoreFileChanged), 1); +} + +#[test] +fn git_internal_churn_stays_incremental() { + let f = Fixture::with_git(); + f.write("src/main.rs", "fn main() {}"); + f.index(); + + let git_dir = f.path(".git"); + let delta = f.feed([ + create(git_dir.join("index.lock")), + modify(git_dir.join("index")), + remove_file(git_dir.join("index.lock")), + modify(git_dir.join("HEAD")), + modify(git_dir.join("logs/HEAD")), + modify(git_dir.join("COMMIT_EDITMSG")), + modify(git_dir.join("refs/heads/main")), + ]); + + f.assert_no_rescan(&delta, "git writing its own metadata"); +} + +#[test] +fn changing_the_root_ignore_file_triggers_a_rescan() { + let f = Fixture::with_git(); + f.write(".gitignore", "target/\n"); + f.write("src/main.rs", "fn main() {}"); + f.index(); + + f.write(".gitignore", "target/\nsrc/\n"); + let delta = f.feed([modify(f.path(".gitignore"))]); + + assert_eq!( + delta.count(RescanReason::IgnoreFileChanged), + 1, + "the indexed set depends on the root ignore rules, got {delta}" + ); +} + +#[test] +fn kernel_event_loss_on_a_directory_triggers_a_rescan() { + let f = Fixture::new(); + f.write("src/main.rs", "fn main() {}"); + f.index(); + + let delta = f.feed([DebouncedEvent::new( + Event::new(EventKind::Modify(ModifyKind::Any)) + .add_path(f.path("src")) + .set_flag(Flag::Rescan), + Instant::now(), + )]); + + assert_eq!( + delta.count(RescanReason::KernelEventLoss), + 1, + "a dropped-events flag over a directory means unknown subtree state, got {delta}" + ); +} + +#[test] +fn new_files_above_index_capacity_trigger_a_rescan() { + let f = Fixture::new(); + f.write("src/main.rs", "fn main() {}"); + f.index(); + + let events = (0..MAX_OVERFLOW_FILES + 1) + .map(|i| { + let rel = format!("src/bulk{i}.rs"); + f.write(&rel, ""); + create(f.path(&rel)) + }) + .collect::>(); + + let delta = f.feed(events); + assert_eq!( + delta.count(RescanReason::IndexUpdateRejected), + 1, + "new files above the overflow region cannot be applied incrementally, got {delta}" + ); +} + +#[test] +fn batch_at_the_overflow_boundary_stays_incremental() { + let f = Fixture::new(); + f.write("src/main.rs", "fn main() {}"); + f.index(); + + let events = (0..MAX_OVERFLOW_FILES) + .map(|i| { + let rel = format!("src/bulk{i}.rs"); + f.write(&rel, ""); + create(f.path(&rel)) + }) + .collect::>(); + + let delta = f.feed(events); + f.assert_no_rescan(&delta, "a batch exactly at the overflow limit"); +} + +#[test] +fn event_batch_at_four_times_index_capacity_stays_incremental() { + let f = Fixture::new(); + f.write("src/main.rs", "fn main() {}"); + f.index(); + + let path = f.path("src/main.rs"); + let events = (0..MAX_OVERFLOW_FILES * 4) + .map(|_| modify(path.clone())) + .collect::>(); + + let delta = f.feed(events); + f.assert_no_rescan(&delta, "an event batch exactly at the event limit"); +} + +#[test] +fn event_batch_above_four_times_index_capacity_triggers_a_rescan() { + let f = Fixture::new(); + f.write("src/main.rs", "fn main() {}"); + f.index(); + + let path = f.path("src/main.rs"); + let events = (0..MAX_OVERFLOW_FILES * 4 + 1) + .map(|_| modify(path.clone())) + .collect::>(); + + let delta = f.feed(events); + assert_eq!( + delta.count(RescanReason::EventBatchOverflow), + 1, + "an event batch above four times the index capacity must rescan, got {delta}" + ); +} + +#[test] +fn repeated_triggers_inside_the_cooldown_collapse_to_one_rescan() { + let f = Fixture::with_git(); + f.write(".gitignore", "target/\n"); + f.write("src/main.rs", "fn main() {}"); + f.index(); + + // Repeated batches during the cooldown must share one walk. + for round in 0..50 { + f.write(".gitignore", &format!("target/\n# round {round}\n")); + f.feed([modify(f.path(".gitignore"))]); + } + + let stats = f.all_rescans(); + assert_eq!( + stats.total, 1, + "50 triggers inside the cooldown must collapse to a single walk, got {stats}" + ); + assert_eq!( + stats.throttled, 49, + "every suppressed request must be accounted for, got {stats}" + ); +} + +#[test] +fn an_explicit_request_is_never_throttled() { + let f = Fixture::with_git(); + f.write(".gitignore", "target/\n"); + f.write("src/main.rs", "fn main() {}"); + f.index(); + + // Burn the cooldown with a watcher trigger, then confirm a user-initiated + // refresh still goes through. + f.write(".gitignore", "target/\nsrc/\n"); + f.feed([modify(f.path(".gitignore"))]); + + for _ in 0..3 { + f.picker.trigger_full_rescan_async(&f.frecency).unwrap(); + } + + let stats = f.all_rescans(); + assert_eq!( + stats.count(RescanReason::Explicit), + 3, + "explicit refreshes must bypass the throttle, got {stats}" + ); + assert_eq!(stats.count_throttled(RescanReason::Explicit), 0); +} + +#[test] +fn events_after_a_suppressed_kernel_rescan_are_still_applied() { + let f = Fixture::new(); + f.write("src/main.rs", "fn main() {}"); + f.index(); + + f.write("src/added.rs", "pub fn added() {}"); + let delta = f.feed([ + DebouncedEvent::new( + Event::new(EventKind::Modify(ModifyKind::Data(DataChange::Content))) + .add_path(f.path("src/main.rs")) + .set_flag(Flag::Rescan), + Instant::now(), + ), + create(f.path("src/added.rs")), + ]); + + f.assert_no_rescan(&delta, "a dropped-events flag over a single tracked file"); + assert!( + f.is_indexed("src/added.rs"), + "suppressing the rescan must not drop the rest of the batch" + ); +} + +#[test] +fn a_throttled_ignore_file_event_is_still_applied_incrementally() { + let f = Fixture::with_git(); + f.write(".gitignore", "target/\n"); + f.write("src/main.rs", "fn main() {}"); + f.index(); + + // Burn the cooldown: deleting .gitignore admits a full rescan. + f.remove(".gitignore"); + let delta = f.feed([remove_file(f.path(".gitignore"))]); + assert_eq!(delta.count(RescanReason::IgnoreFileChanged), 1); + f.picker.wait_for_indexing_complete(Duration::from_secs(10)); + + // Recreating it inside the cooldown throttles the rescan, but the file + // itself must re-enter the index via the incremental fallback. + f.write(".gitignore", "target/\n__ignored_x/\n"); + let delta = f.feed([create(f.path(".gitignore"))]); + assert_eq!(delta.total, 0, "the rescan must be throttled, got {delta}"); + assert_eq!(delta.count_throttled(RescanReason::IgnoreFileChanged), 1); + assert!( + f.is_indexed(".gitignore"), + "a throttled ignore-file event must still index the file itself" + ); +} + +struct Fixture { + base: PathBuf, + picker: SharedFilePicker, + frecency: SharedFrecency, + git_workdir: Option, + git_worker: Arc, + // Dropped last so background work started by a triggered rescan still + // sees the tree it was asked to walk. + _tmp: TempDir, +} + +impl Fixture { + fn new() -> Self { + Self::build(false) + } + + fn with_git() -> Self { + Self::build(true) + } + + fn build(git: bool) -> Self { + let tmp = tempfile::tempdir().unwrap(); + let base = crate::path_utils::canonicalize(tmp.path()).unwrap(); + let git_workdir = git.then(|| { + let status = Command::new("git") + .args(["init", "-b", "main"]) + .current_dir(&base) + .output() + .expect("git init"); + assert!(status.status.success(), "git init failed"); + base.clone() + }); + + Self { + base, + picker: SharedFilePicker::default(), + frecency: SharedFrecency::noop(), + git_workdir, + git_worker: GitStatusWorker::new(), + _tmp: tmp, + } + } + + fn index(&self) { + let mut picker = FilePicker::new(FilePickerOptions { + base_path: self.base.to_string_lossy().into_owned(), + watch: false, + ..Default::default() + }) + .unwrap(); + picker.collect_files().unwrap(); + self.picker.rebase_watches(&self.base); + *self.picker.write().unwrap() = Some(picker); + } + + fn feed(&self, events: impl IntoIterator) -> RescanStats { + let before = self.picker.rescan_stats(); + handle_debounced_events( + FFFMode::Neovim, + events.into_iter().collect(), + &self.base, + &self.git_workdir, + &self.picker, + &self.frecency, + &self.git_worker, + ); + + self.picker.rescan_stats().since(&before) + } + + fn assert_no_rescan(&self, delta: &RescanStats, what: &str) { + assert_eq!(delta.total, 0, "{what} must not trigger a rescan: {delta}"); + } + + fn path(&self, rel: &str) -> PathBuf { + self.base.join(rel) + } + + fn write(&self, rel: &str, contents: &str) { + let path = self.path(rel); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, contents).unwrap(); + } + + fn remove(&self, rel: &str) { + std::fs::remove_file(self.path(rel)).unwrap(); + } + + fn is_indexed(&self, rel: &str) -> bool { + let guard = self.picker.read().unwrap(); + guard + .as_ref() + .and_then(|p| p.get_file_by_path(self.path(rel))) + .is_some_and(|file| !file.is_deleted()) + } + + fn all_rescans(&self) -> RescanStats { + self.picker.rescan_stats() + } + + fn overflow_len(&self) -> usize { + let guard = self.picker.read().unwrap(); + guard + .as_ref() + .map(|p| p.get_overflow_files().len()) + .unwrap_or(0) + } +} + +impl Drop for Fixture { + fn drop(&mut self) { + // A test that intentionally triggers a rescan leaves a walk running on + // the background pool; let it finish before the tree disappears. + self.picker + .wait_for_indexing_complete(Duration::from_secs(10)); + } +} + +fn event(kind: EventKind, path: PathBuf) -> DebouncedEvent { + DebouncedEvent::new(Event::new(kind).add_path(path), Instant::now()) +} + +fn create(path: PathBuf) -> DebouncedEvent { + event(EventKind::Create(CreateKind::File), path) +} + +fn modify(path: PathBuf) -> DebouncedEvent { + event( + EventKind::Modify(ModifyKind::Data(DataChange::Content)), + path, + ) +} + +fn remove_file(path: PathBuf) -> DebouncedEvent { + event(EventKind::Remove(RemoveKind::File), path) +} diff --git a/crates/fff-core/tests/rescan_regression.rs b/crates/fff-core/tests/rescan_regression.rs new file mode 100644 index 000000000..a1cc5d4e0 --- /dev/null +++ b/crates/fff-core/tests/rescan_regression.rs @@ -0,0 +1,343 @@ +#![cfg(rescan_stats)] + +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::Duration; + +use fff_search::file_picker::{FFFMode, FilePicker}; +use fff_search::{FilePickerOptions, RescanStats, SharedFilePicker, SharedFrecency}; +use tempfile::TempDir; + +const SETTLE: Duration = Duration::from_millis(600); + +#[test] +fn saving_source_files_does_not_rescan() { + let repo = WatchedRepo::new(|base| { + write(base, ".gitignore", "target/\n"); + for i in 0..20 { + write(base, &format!("src/mod{i}.rs"), "pub fn f() {}"); + } + }); + + for round in 0..10 { + for i in 0..20 { + repo.write( + &format!("src/mod{i}.rs"), + &format!("pub fn f() {{ let _ = {round}; }}"), + ); + } + repo.settle(); + } + + repo.assert_quiet("200 file saves"); +} + +#[test] +fn build_output_in_ignored_directories_does_not_rescan() { + let repo = WatchedRepo::new(|base| { + write(base, ".gitignore", "target/\nnode_modules/\ndist/\n"); + write(base, "src/main.rs", "fn main() {}"); + }); + + for round in 0..4 { + for i in 0..150 { + repo.write(&format!("target/debug/deps/unit-{round}-{i}.o"), "binary"); + repo.write(&format!("dist/chunk-{round}-{i}.js"), "bundled"); + } + repo.settle(); + } + + repo.assert_quiet("1200 build artifacts written into ignored directories"); +} + +#[test] +fn adding_source_files_and_directories_does_not_rescan() { + let repo = WatchedRepo::new(|base| { + write(base, ".gitignore", "target/\n"); + write(base, "src/main.rs", "fn main() {}"); + }); + + for i in 0..40 { + repo.write(&format!("src/feature{i}/mod.rs"), "pub mod inner;"); + repo.write(&format!("src/feature{i}/inner.rs"), "pub fn go() {}"); + } + repo.settle(); + + assert!( + repo.wait_indexed("src/feature39/inner.rs"), + "watcher must index files in newly created directories" + ); + repo.assert_quiet("40 new directories with 80 files"); +} + +#[test] +fn recreating_generated_files_does_not_rescan() { + let repo = WatchedRepo::new(|base| { + write(base, ".gitignore", "target/\n"); + write(base, "src/main.rs", "fn main() {}"); + }); + + // Recreated paths must reuse their overflow slots. + for round in 0..12 { + for i in 0..40 { + repo.write(&format!("src/generated/api{i}.rs"), "pub struct A;"); + } + repo.settle(); + for i in 0..40 { + repo.remove(&format!("src/generated/api{i}.rs")); + } + repo.settle(); + assert!( + repo.overflow_len() <= 64, + "round {round}: regenerating the same paths grew the overflow region to {}", + repo.overflow_len() + ); + } + + repo.assert_quiet("12 codegen cycles over 40 stable paths"); +} + +#[test] +fn git_workflow_does_not_rescan() { + let repo = WatchedRepo::new(|base| { + write(base, ".gitignore", "target/\n"); + write(base, "src/main.rs", "fn main() {}"); + write(base, "src/lib.rs", "pub mod thing;"); + git(base, &["init", "-b", "main"]); + git(base, &["add", "-A"]); + git(base, &["commit", "-m", "initial"]); + }); + + repo.write("src/main.rs", "fn main() { println!(\"hi\"); }"); + repo.settle(); + repo.git(&["add", "-A"]); + repo.settle(); + repo.git(&["commit", "-m", "second"]); + repo.settle(); + repo.git(&["checkout", "-b", "feature"]); + repo.settle(); + repo.write("src/feature.rs", "pub fn feature() {}"); + repo.git(&["add", "-A"]); + repo.git(&["commit", "-m", "feature"]); + repo.settle(); + repo.git(&["checkout", "main"]); + repo.settle(); + repo.git(&["merge", "feature"]); + repo.settle(); + + repo.assert_quiet("a commit / branch / merge cycle"); +} + +#[test] +fn reading_files_does_not_rescan() { + let repo = WatchedRepo::new(|base| { + write(base, ".gitignore", "target/\n"); + for i in 0..50 { + write(base, &format!("src/mod{i}.rs"), "pub fn f() {}"); + } + }); + + // Preview rendering and grep open every file in the result list. Reacting + // to those reads would make the picker rescan while the user scrolls. + for _ in 0..5 { + for i in 0..50 { + let _ = std::fs::read(repo.path(&format!("src/mod{i}.rs"))).unwrap(); + } + } + repo.settle(); + + repo.assert_quiet("reading every indexed file"); +} + +#[test] +fn npm_install_style_churn_does_not_rescan() { + let repo = WatchedRepo::new(|base| { + write(base, ".gitignore", "node_modules/\n"); + write(base, "src/index.ts", "export const a = 1;"); + }); + + for pkg in 0..100 { + repo.write(&format!("node_modules/pkg{pkg}/package.json"), "{}"); + repo.write( + &format!("node_modules/pkg{pkg}/index.js"), + "module.exports={}", + ); + repo.write(&format!("node_modules/pkg{pkg}/.gitignore"), "dist\n"); + } + repo.settle(); + repo.settle(); + + repo.assert_quiet("an npm install into an ignored node_modules"); +} + +#[test] +fn a_churning_root_is_capped_at_one_rescan_per_cooldown() { + let repo = WatchedRepo::new(|base| { + write(base, "src/main.rs", "fn main() {}"); + }); + + // Root ignore changes force watcher rescan requests. + for round in 0..25 { + repo.write(".gitignore", &format!("target/\n# round {round}\n")); + std::thread::sleep(Duration::from_millis(120)); + } + repo.settle(); + + let stats = repo.rescans(); + assert!( + stats.total <= 1, + "a churning root must not exceed one walk per cooldown, got {stats}" + ); + assert!( + stats.throttled > 0, + "the suppressed triggers must be recorded, got {stats}" + ); +} + +struct WatchedRepo { + base: PathBuf, + picker: SharedFilePicker, + _frecency: SharedFrecency, + _tmp: TempDir, +} + +impl WatchedRepo { + fn new(setup: impl FnOnce(&Path)) -> Self { + let tmp = tempfile::tempdir().unwrap(); + let base = fff_search::path_utils::canonicalize(tmp.path()).unwrap(); + setup(&base); + + let picker = SharedFilePicker::default(); + let frecency = SharedFrecency::noop(); + FilePicker::new_with_shared_state( + picker.clone(), + frecency.clone(), + FilePickerOptions { + base_path: base.to_string_lossy().into_owned(), + enable_mmap_cache: false, + mode: FFFMode::Neovim, + watch: true, + ..Default::default() + }, + ) + .expect("failed to create file picker"); + + assert!( + picker.wait_for_scan(Duration::from_secs(60)), + "timed out waiting for the initial scan" + ); + assert!( + picker.wait_for_watcher(Duration::from_secs(60)), + "timed out waiting for the watcher" + ); + + let repo = Self { + base, + picker, + _frecency: frecency, + _tmp: tmp, + }; + repo.settle(); + repo.picker.reset_rescan_stats(); + repo + } + + fn settle(&self) { + std::thread::sleep(SETTLE); + assert!( + self.picker + .wait_for_indexing_complete(Duration::from_secs(60)), + "timed out waiting for background indexing to finish" + ); + } + + fn assert_quiet(&self, workload: &str) { + let stats = self.rescans(); + assert_eq!( + stats.watcher_triggered(), + 0, + "{workload} must be absorbed incrementally, but the watcher fell back to {stats}" + ); + } + + fn rescans(&self) -> RescanStats { + self.picker.rescan_stats() + } + + fn path(&self, rel: &str) -> PathBuf { + self.base.join(rel) + } + + fn write(&self, rel: &str, contents: &str) { + write(&self.base, rel, contents); + } + + fn remove(&self, rel: &str) { + std::fs::remove_file(self.path(rel)).unwrap(); + } + + fn git(&self, args: &[&str]) { + git(&self.base, args); + } + + fn wait_indexed(&self, rel: &str) -> bool { + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while std::time::Instant::now() < deadline { + if self.is_indexed(rel) { + return true; + } + std::thread::sleep(Duration::from_millis(50)); + } + false + } + + fn is_indexed(&self, rel: &str) -> bool { + let guard = self.picker.read().unwrap(); + guard + .as_ref() + .and_then(|p| p.get_file_by_path(self.path(rel))) + .is_some_and(|file| !file.is_deleted()) + } + + fn overflow_len(&self) -> usize { + let guard = self.picker.read().unwrap(); + guard + .as_ref() + .map(|p| p.get_overflow_files().len()) + .unwrap_or(0) + } +} + +impl Drop for WatchedRepo { + fn drop(&mut self) { + // Stop the watcher before the tree disappears, otherwise a late batch + // races the tempdir removal. + if let Ok(mut guard) = self.picker.write() { + guard.take(); + } + } +} + +fn write(base: &Path, rel: &str, contents: &str) { + let path = base.join(rel); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, contents).unwrap(); +} + +fn git(dir: &Path, args: &[&str]) { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .env("GIT_AUTHOR_NAME", "test") + .env("GIT_AUTHOR_EMAIL", "test@test.com") + .env("GIT_COMMITTER_NAME", "test") + .env("GIT_COMMITTER_EMAIL", "test@test.com") + .output() + .unwrap_or_else(|e| panic!("git {args:?} failed to spawn: {e}")); + + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); +} diff --git a/crates/fff-nvim/Cargo.toml b/crates/fff-nvim/Cargo.toml index 37394d746..c53d8df83 100644 --- a/crates/fff-nvim/Cargo.toml +++ b/crates/fff-nvim/Cargo.toml @@ -15,6 +15,8 @@ crate-type = ["cdylib", "rlib"] default = ["ripgrep"] ripgrep = ["fff/ripgrep", "fff-query-parser/ripgrep", "dep:ignore"] zlob = ["fff/zlob", "fff-query-parser/zlob", "dep:zlob"] +# Keep full-rescan accounting in a release build; required by rescan_probe. +rescan-stats = ["fff/rescan-stats"] [dependencies] # Workspace dependencies diff --git a/crates/fff-nvim/src/bin/rescan_probe.rs b/crates/fff-nvim/src/bin/rescan_probe.rs new file mode 100644 index 000000000..556fd6be3 --- /dev/null +++ b/crates/fff-nvim/src/bin/rescan_probe.rs @@ -0,0 +1,158 @@ +use fff::file_picker::FilePicker; +use fff::{ + FFFMode, FilePickerOptions, RESCAN_STATS_ENABLED, RescanReason, RescanStats, SharedFilePicker, + SharedFrecency, +}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; + +const POLL: Duration = Duration::from_millis(250); + +fn main() -> Result<(), Box> { + let (base_path, run_for) = parse_args()?; + + if !RESCAN_STATS_ENABLED { + return Err( + "this build has rescan accounting compiled out; rebuild with \ + `--features rescan-stats` (or drop `--release`)" + .into(), + ); + } + + let picker = SharedFilePicker::default(); + let frecency = SharedFrecency::noop(); + + println!("indexing {base_path} ..."); + let started = Instant::now(); + FilePicker::new_with_shared_state( + picker.clone(), + frecency.clone(), + FilePickerOptions { + base_path: base_path.clone(), + enable_mmap_cache: false, + mode: FFFMode::default(), + watch: true, + ..Default::default() + }, + )?; + + if !picker.wait_for_scan(Duration::from_secs(600)) { + return Err("timed out waiting for the initial scan".into()); + } + if !picker.wait_for_watcher(Duration::from_secs(600)) { + return Err("timed out waiting for the watcher".into()); + } + + println!( + "indexed {} files in {:.2}s; watching for rescan requests.\n", + live_files(&picker), + started.elapsed().as_secs_f64() + ); + picker.reset_rescan_stats(); + + let running = Arc::new(AtomicBool::new(true)); + let stop = Arc::clone(&running); + ctrlc::set_handler(move || stop.store(false, Ordering::SeqCst))?; + + let watching_since = Instant::now(); + let mut last = RescanStats::default(); + + while running.load(Ordering::SeqCst) { + std::thread::sleep(POLL); + + let stats = picker.rescan_stats(); + let delta = stats.since(&last); + if delta.total > 0 || delta.throttled > 0 { + let now = watching_since.elapsed().as_secs_f64(); + let files = live_files(&picker); + let overflow = overflow_files(&picker); + + for reason in RescanReason::ALL { + for _ in 0..delta.count(reason) { + println!( + "[{now:>8.2}s] request {reason:<21} files={files} overflow={overflow}" + ); + } + let suppressed = delta.count_throttled(reason); + if suppressed > 0 { + println!("[{now:>8.2}s] throttled {reason:<21} x{suppressed}"); + } + } + last = stats; + } + + if run_for.is_some_and(|limit| watching_since.elapsed() >= limit) { + break; + } + } + + let elapsed = watching_since.elapsed(); + let stats = picker.rescan_stats(); + println!("\n{:.1}s watched", elapsed.as_secs_f64()); + println!("{stats}"); + if stats.watcher_triggered() > 0 { + println!( + "{:.1} watcher rescan requests/minute", + stats.watcher_triggered() as f64 / elapsed.as_secs_f64().max(1.0) * 60.0 + ); + } else { + println!("no full rescans: every change was applied incrementally"); + } + if stats.throttled > 0 { + println!( + "{} additional request(s) were throttled; {} total requests observed", + stats.throttled, + stats.total + stats.throttled + ); + } + + if let Ok(mut guard) = picker.write() { + guard.take(); + } + + Ok(()) +} + +fn parse_args() -> Result<(String, Option), Box> { + let mut base_path = None; + let mut run_for = None; + let mut args = std::env::args().skip(1); + + while let Some(arg) = args.next() { + match arg.as_str() { + "--seconds" | "-s" => { + let value = args.next().ok_or("--seconds needs a value")?; + run_for = Some(Duration::from_secs(value.parse()?)); + } + "--help" | "-h" => { + println!("usage: rescan_probe [path] [--seconds N]"); + std::process::exit(0); + } + other => base_path = Some(other.to_string()), + } + } + + let base_path = match base_path { + Some(path) => path, + None => std::env::current_dir()?.to_string_lossy().into_owned(), + }; + + Ok((base_path, run_for)) +} + +fn live_files(picker: &SharedFilePicker) -> usize { + picker + .read() + .ok() + .and_then(|g| g.as_ref().map(|p| p.live_file_count())) + .unwrap_or(0) +} + +fn overflow_files(picker: &SharedFilePicker) -> usize { + picker + .read() + .ok() + .and_then(|g| g.as_ref().map(|p| p.get_overflow_files().len())) + .unwrap_or(0) +} diff --git a/packages/fff-bun/src/fff-api.ts b/packages/fff-bun/src/fff-api.ts index d22c6de5c..2d0038aab 100644 --- a/packages/fff-bun/src/fff-api.ts +++ b/packages/fff-bun/src/fff-api.ts @@ -573,16 +573,10 @@ export interface FileFinderApi { glob(pattern: string, options?: GlobOptions): Result; /** Fuzzy directory search. */ - directorySearch( - query: string, - options?: DirSearchOptions, - ): Result; + directorySearch(query: string, options?: DirSearchOptions): Result; /** Fuzzy search over files and directories interleaved by score. */ - mixedSearch( - query: string, - options?: SearchOptions, - ): Result; + mixedSearch(query: string, options?: SearchOptions): Result; /** Content search (live grep). */ grep(query: string, options?: GrepOptions): Result; @@ -647,10 +641,7 @@ export interface FileFinderApi { * Events are debounced and submitted in batches per 100-ms window at most 128 events. * Gitignored and other ignored files are never triggering watcher. */ - watch( - callback: WatchBatchCallback, - options?: WatchOptions, - ): Result; + watch(callback: WatchBatchCallback, options?: WatchOptions): Result; watch( pattern: string, callback: WatchBatchCallback, diff --git a/packages/fff-node/src/fff-api.ts b/packages/fff-node/src/fff-api.ts index d22c6de5c..2d0038aab 100644 --- a/packages/fff-node/src/fff-api.ts +++ b/packages/fff-node/src/fff-api.ts @@ -573,16 +573,10 @@ export interface FileFinderApi { glob(pattern: string, options?: GlobOptions): Result; /** Fuzzy directory search. */ - directorySearch( - query: string, - options?: DirSearchOptions, - ): Result; + directorySearch(query: string, options?: DirSearchOptions): Result; /** Fuzzy search over files and directories interleaved by score. */ - mixedSearch( - query: string, - options?: SearchOptions, - ): Result; + mixedSearch(query: string, options?: SearchOptions): Result; /** Content search (live grep). */ grep(query: string, options?: GrepOptions): Result; @@ -647,10 +641,7 @@ export interface FileFinderApi { * Events are debounced and submitted in batches per 100-ms window at most 128 events. * Gitignored and other ignored files are never triggering watcher. */ - watch( - callback: WatchBatchCallback, - options?: WatchOptions, - ): Result; + watch(callback: WatchBatchCallback, options?: WatchOptions): Result; watch( pattern: string, callback: WatchBatchCallback, diff --git a/packages/fff-node/src/ffi.ts b/packages/fff-node/src/ffi.ts index e2315600b..61080f421 100644 --- a/packages/fff-node/src/ffi.ts +++ b/packages/fff-node/src/ffi.ts @@ -246,11 +246,7 @@ function readResultEnvelope( paramsValue: unknown[], ): { rawPtr: JsExternal; struct: FffResultRaw } | Result { loadLibrary(); - const { rawPtr, struct: structData } = callRaw( - funcName, - paramsType, - paramsValue, - ); + const { rawPtr, struct: structData } = callRaw(funcName, paramsType, paramsValue); if (structData.success === 0) { const errorStr = readCString(structData.error); @@ -328,8 +324,7 @@ function callJsonResult( if (isNullPointer(handlePtr)) return { ok: true, value: undefined as T }; const jsonStr = readCString(handlePtr); freeString(handlePtr); - if (jsonStr === null || jsonStr === "") - return { ok: true, value: undefined as T }; + if (jsonStr === null || jsonStr === "") return { ok: true, value: undefined as T }; try { return { ok: true, value: snakeToCamel(JSON.parse(jsonStr)) as T }; } catch { @@ -849,16 +844,10 @@ function readGrepMatchFromRaw(raw: FffGrepMatchRaw): GrepMatch { match.fuzzyScore = raw.fuzzy_score; } if (raw.context_before_count > 0) { - match.contextBefore = readCStringArray( - raw.context_before, - raw.context_before_count, - ); + match.contextBefore = readCStringArray(raw.context_before, raw.context_before_count); } if (raw.context_after_count > 0) { - match.contextAfter = readCStringArray( - raw.context_after, - raw.context_after_count, - ); + match.contextAfter = readCStringArray(raw.context_after, raw.context_after_count); } if (raw.is_definition !== 0) { match.isDefinition = true; @@ -927,8 +916,7 @@ function parseGrepResult(rawPtr: JsExternal): Result { totalFilesSearched: gr.total_files_searched, totalFiles: gr.total_files, filteredFileCount: gr.filtered_file_count, - nextCursor: - gr.next_file_offset > 0 ? createGrepCursor(gr.next_file_offset) : null, + nextCursor: gr.next_file_offset > 0 ? createGrepCursor(gr.next_file_offset) : null, }; if (regexFallbackError) { grepResult.regexFallbackError = regexFallbackError; @@ -1280,14 +1268,7 @@ export function ffiGlob( DataType.U32, // page_index DataType.U32, // page_size ], - paramsValue: [ - handle, - pattern, - currentFile, - maxThreads, - pageIndex, - pageSize, - ], + paramsValue: [handle, pattern, currentFile, maxThreads, pageIndex, pageSize], freeResultMemory: false, }) as JsExternal; @@ -1319,14 +1300,7 @@ export function ffiSearchDirectories( DataType.U32, // page_index DataType.U32, // page_size ], - paramsValue: [ - handle, - query, - currentFile ?? "", - maxThreads, - pageIndex, - pageSize, - ], + paramsValue: [handle, query, currentFile ?? "", maxThreads, pageIndex, pageSize], freeResultMemory: false, }) as JsExternal; @@ -1545,11 +1519,7 @@ export function ffiGetScanProgress(handle: NativeHandle): Result<{ isWarmupComplete: boolean; }> { loadLibrary(); - const res = readResultEnvelope( - "fff_get_scan_progress", - [DataType.External], - [handle], - ); + const res = readResultEnvelope("fff_get_scan_progress", [DataType.External], [handle]); if ("ok" in res) return res; const handlePtr = res.struct.handle; @@ -1584,10 +1554,7 @@ export function ffiGetScanProgress(handle: NativeHandle): Result<{ /** * Wait for a tree scan to complete. */ -export function ffiWaitForScan( - handle: NativeHandle, - timeoutMs: number, -): Result { +export function ffiWaitForScan(handle: NativeHandle, timeoutMs: number): Result { return callBoolResult( "fff_wait_for_scan", [DataType.External, DataType.U64], @@ -1598,10 +1565,7 @@ export function ffiWaitForScan( /** * Restart index in new path. */ -export function ffiRestartIndex( - handle: NativeHandle, - newPath: string, -): Result { +export function ffiRestartIndex(handle: NativeHandle, newPath: string): Result { return callVoidResult( "fff_restart_index", [DataType.External, DataType.String], @@ -1772,8 +1736,7 @@ function ensureWatchTrampoline(): JsExternal { // fff watcher uses a single cross-boundary FFI callback to deliver all events which we then manually // mapping to the user's javascript functions function ensureWatchCallbackRegistered(handle: NativeHandle): Result { - if (watchInstances.has(handle as unknown)) - return { ok: true, value: undefined }; + if (watchInstances.has(handle as unknown)) return { ok: true, value: undefined }; const trampoline = ensureWatchTrampoline(); const registered = callVoidResult( "fff_set_watch_callback", @@ -1785,11 +1748,7 @@ function ensureWatchCallbackRegistered(handle: NativeHandle): Result { } function releaseWatchTrampolineIfIdle(): void { - if ( - watchHandlers.size > 0 || - watchInstances.size > 0 || - watchTrampoline === null - ) + if (watchHandlers.size > 0 || watchInstances.size > 0 || watchTrampoline === null) return; freePointer({ paramsType: [WATCH_TRAMPOLINE_TYPE], @@ -1835,10 +1794,7 @@ export function ffiWatch( * this returns the callback can never run again (a late native tail batch * misses the map lookup and is dropped). */ -export function ffiUnwatch( - handle: NativeHandle, - watchId: number, -): Result { +export function ffiUnwatch(handle: NativeHandle, watchId: number): Result { const result = callBoolResult( "fff_unwatch", [DataType.External, DataType.U64], diff --git a/packages/pi-fff/src/aux-finders.ts b/packages/pi-fff/src/aux-finders.ts index 9c24ced2d..dd637f002 100644 --- a/packages/pi-fff/src/aux-finders.ts +++ b/packages/pi-fff/src/aux-finders.ts @@ -84,8 +84,7 @@ export class AuxFinderPool { private async create(root: string): Promise { if (this.entries.length >= MAX_AUX) { let oldest = this.entries[0]; - for (const e of this.entries) - if (e.lastUsed < oldest.lastUsed) oldest = e; + for (const e of this.entries) if (e.lastUsed < oldest.lastUsed) oldest = e; if (!oldest.finder.isDestroyed) oldest.finder.destroy(); this.entries = this.entries.filter((e) => e !== oldest); } @@ -101,9 +100,7 @@ export class AuxFinderPool { enableFsRootScanning: this.opts.enableFsRootScanning, }); if (!result.ok) - throw new Error( - `Failed to create aux file finder for ${root}: ${result.error}`, - ); + throw new Error(`Failed to create aux file finder for ${root}: ${result.error}`); await result.value.waitForScan(SCAN_TIMEOUT_MS); const entry: AuxPicker = { @@ -125,9 +122,7 @@ export class AuxFinderPool { // remainder usable as a fuzzy path constraint relative to that root. Glob and // nonexistent segments both go into the suffix: we walk up to the nearest // existing ancestor so partially-wrong paths still resolve to a search root. -export function resolveAuxRoot( - absPath: string, -): { root: string; suffix: string } | null { +export function resolveAuxRoot(absPath: string): { root: string; suffix: string } | null { const trimmed = path.normalize(absPath.trim()).replace(/\/+$/, "") || "/"; if (!path.isAbsolute(trimmed)) return null; if (trimmed === path.sep) return { root: path.sep, suffix: "" }; @@ -187,7 +182,6 @@ export function routePathConstraint( return resolveAuxRoot(candidate); } - export function rootCovers(root: string, target: string): boolean { if (root === target) return true; const prefix = root.endsWith(path.sep) ? root : root + path.sep; diff --git a/packages/pi-fff/src/index.ts b/packages/pi-fff/src/index.ts index 96eaf7eb5..f17cd4ed5 100644 --- a/packages/pi-fff/src/index.ts +++ b/packages/pi-fff/src/index.ts @@ -1,1148 +1,1097 @@ -/** - * pi-fff: FFF-powered file search extension for pi - * - * Overrides built-in `find` and `grep` tools with FFF and adds FFF-backed - * @-mention autocomplete suggestions to the interactive editor. - */ - -import nodePath from "node:path"; -import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; -import { - type AutocompleteItem, - type AutocompleteProvider, - Text, -} from "@earendil-works/pi-tui"; -import type { - FileFinderApi, - GrepCursor, - GrepMode, - GrepResult, - MixedItem, - SearchResult, -} from "@ff-labs/fff-node"; -import { Type } from "@sinclair/typebox"; -import { AuxFinderPool, routePathConstraint } from "./aux-finders"; -import { buildQuery } from "./query"; -import { loadSdk, SCAN_TIMEOUT_MS } from "./sdk"; - -export { SCAN_TIMEOUT_MS } from "./sdk"; - -// --------------------------------------------------------------------------- -// Constants -// --------------------------------------------------------------------------- - -const DEFAULT_GREP_LIMIT = 20; -const DEFAULT_FIND_LIMIT = 30; -const GREP_MAX_LINE_LENGTH = 500; -const MENTION_MAX_RESULTS = 20; - -// If we exceed 10 seconds for indexed grep - something is definitely off -const GREP_TIME_BUDGET_MS = 10_000; - -type FffMode = "tools-and-ui" | "tools-only" | "override"; - -const VALID_MODES: FffMode[] = ["tools-and-ui", "tools-only", "override"]; - -interface ToolNames { - grep: string; - find: string; - multiGrep: string; -} - -const FFF_TOOL_NAMES: ToolNames = { - grep: "ffgrep", - find: "fffind", - multiGrep: "fff-multi-grep", -}; -const OVERRIDE_TOOL_NAMES: ToolNames = { - grep: "grep", - find: "find", - multiGrep: "multi_grep", -}; - -function resolveToolNames(mode: FffMode): ToolNames { - return mode === "override" ? OVERRIDE_TOOL_NAMES : FFF_TOOL_NAMES; -} - -// --------------------------------------------------------------------------- -// Cursor store — simple bounded Map for pagination cursors -// --------------------------------------------------------------------------- - -const cursorCache = new Map(); -let cursorCounter = 0; - -function storeCursor(cursor: GrepCursor): string { - const id = `fff_c${++cursorCounter}`; - cursorCache.set(id, cursor); - if (cursorCache.size > 200) { - const first = cursorCache.keys().next().value; - if (first) cursorCache.delete(first); - } - return id; -} - -function getCursor(id: string): GrepCursor | undefined { - return cursorCache.get(id); -} - -// Find pagination uses a page-index cursor: native `fileSearch` takes -// pageIndex/pageSize, so the cursor is just the next page index paired with -// the query+limit that produced it. Stored tokens are opaque IDs to the agent. -interface FindCursor { - query: string; - pattern: string; - pageSize: number; - nextPageIndex: number; - auxRoot?: string; -} - -const findCursorCache = new Map(); -let findCursorCounter = 0; - -function storeFindCursor(cursor: FindCursor): string { - const id = `${++findCursorCounter}`; - findCursorCache.set(id, cursor); - if (findCursorCache.size > 200) { - const first = findCursorCache.keys().next().value; - if (first) findCursorCache.delete(first); - } - return id; -} - -function getFindCursor(id: string): FindCursor | undefined { - return findCursorCache.get(id); -} - -// --------------------------------------------------------------------------- -// Output formatting helpers -// --------------------------------------------------------------------------- - -function truncateLine(line: string, max = GREP_MAX_LINE_LENGTH): string { - const trimmed = line.trim(); - return trimmed.length <= max ? trimmed : `${trimmed.slice(0, max)}...`; -} - -const HOT_FRECENCY = 25; -const WARM_FRECENCY = 20; - -// Shared annotation helper for both find-output paths and grep-output file -// headers. Returns at most ONE tag so output stays scannable. Priority: -// git-dirty (most actionable — file is changing right now) beats frecency -// (historically often-touched). Keeping one function ensures the two tools -// never drift in how they surface git/frecency signal. -export function fffFileAnnotation(item: { - gitStatus?: string; - totalFrecencyScore?: number; - accessFrecencyScore?: number; -}): string { - const git = item.gitStatus; - if (git && git !== "clean" && git !== "unknown" && git !== "") { - return ` [${git} in git]`; - } - - const frecency = item.totalFrecencyScore ?? item.accessFrecencyScore ?? 0; - if (frecency >= HOT_FRECENCY) return " [VERY often touched file]"; - if (frecency >= WARM_FRECENCY) return " [often touched file]"; - - return ""; -} - -// fff-core native definition classifier (byte-level scanner in Rust) is enabled -// via GrepOptions.classifyDefinitions. Each GrepMatch carries isDefinition for -// downstream consumers; pi-fff does NOT use it to re-sort. -// -// Ordering policy: NO CUSTOM SORTING. The engine already returns items in -// frecency order (most-accessed files first). pi-fff only groups consecutive -// matches into per-file blocks and preserves whatever order the engine -// provided — inside a file we keep matches in source-line order because the -// engine emits them that way. - -function formatGrepOutput(result: GrepResult): string { - if (result.items.length === 0) return "No matches found"; - - // Build file-grouped output in the order files first appear in the result. - // This preserves native frecency ordering across files without re-sorting. - const lines: string[] = []; - let currentFile = ""; - let shown = 0; - - for (const match of result.items) { - if (match.relativePath !== currentFile) { - if (lines.length > 0) lines.push(""); - currentFile = match.relativePath; - lines.push(`${currentFile}${fffFileAnnotation(match)}`); - } - - match.contextBefore?.forEach((line: string, i: number) => { - const lineNum = match.lineNumber - match.contextBefore!.length + i; - lines.push(` ${lineNum}- ${truncateLine(line)}`); - }); - - lines.push(` ${match.lineNumber}: ${truncateLine(match.lineContent)}`); - shown++; - - match.contextAfter?.forEach((line: string, i: number) => { - const lineNum = match.lineNumber + 1 + i; - lines.push(` ${lineNum}- ${truncateLine(line)}`); - }); - } - - return lines.join("\n"); -} - -// Weak-match threshold is derived from the query length, matching the -// scoring formula in crates/fff-core/src/score.rs: a perfect match scores -// `len * 16`, so we treat anything below 50% of that as scattered fuzzy noise. -// When the top score is weak, trim output to a small sample instead of dumping -// the full limit worth of noise into the agent's context. -const FIND_WEAK_SAMPLE_SIZE = 5; - -function weakScoreThreshold(pattern: string): number { - const perfect = pattern.length * 12; - return Math.floor((perfect * 50) / 100); -} - -interface FormattedFind { - output: string; - weak: boolean; - shownCount: number; -} - -function formatFindOutput( - result: SearchResult, - limit: number, - pattern: string, -): FormattedFind { - if (result.items.length === 0) { - return { - output: "No files found matching pattern", - weak: false, - shownCount: 0, - }; - } - - // NO CUSTOM SORTING — trust native frecency order from the engine. - const reordered = result.items.map((item) => ({ item })); - - // Peek at the top native score to decide whether results are scattered - // fuzzy noise (query length-scaled threshold from score.rs). - const topScore = result.scores[0]?.total ?? 0; - const weak = topScore < weakScoreThreshold(pattern); - const effective = weak ? Math.min(FIND_WEAK_SAMPLE_SIZE, limit) : limit; - const shown = reordered.slice(0, effective); - - return { - output: shown - .map((p) => `${p.item.relativePath}${fffFileAnnotation(p.item)}`) - .join("\n"), - weak, - shownCount: shown.length, - }; -} - -// --------------------------------------------------------------------------- -// Mention autocomplete helpers -// --------------------------------------------------------------------------- - -function extractAtPrefix(textBeforeCursor: string): string | null { - const match = textBeforeCursor.match(/(?:^|[ \t])(@(?:"[^"]*|[^\s]*))$/); - return match?.[1] ?? null; -} - -function buildAtCompletionValue(path: string): string { - return path.includes(" ") ? `@"${path}"` : `@${path}`; -} - -function createFffMentionProvider( - getItems: (query: string, signal: AbortSignal) => Promise, -): AutocompleteProvider { - return { - async getSuggestions(lines, cursorLine, cursorCol, options) { - const currentLine = lines[cursorLine] || ""; - const prefix = extractAtPrefix(currentLine.slice(0, cursorCol)); - if (!prefix || options.signal.aborted) return null; - - const query = prefix.startsWith('@"') ? prefix.slice(2) : prefix.slice(1); - const items = await getItems(query, options.signal); - return options.signal.aborted || items.length === 0 - ? null - : { items, prefix }; - }, - applyCompletion(_lines, cursorLine, cursorCol, item, prefix) { - const currentLine = _lines[cursorLine] || ""; - const before = currentLine.slice(0, cursorCol - prefix.length); - const after = currentLine.slice(cursorCol); - const newLine = before + item.value + after; - const newCursorCol = cursorCol - prefix.length + item.value.length; - return { - lines: [ - ..._lines.slice(0, cursorLine), - newLine, - ..._lines.slice(cursorLine + 1), - ], - cursorLine, - cursorCol: newCursorCol, - }; - }, - }; -} - -// --------------------------------------------------------------------------- -// Extension -// --------------------------------------------------------------------------- - -export default function fffExtension(pi: ExtensionAPI) { - let mainFinder: FileFinderApi | null = null; - let finderCwd: string | null = null; - // Concurrent ensureFinder() callers share the same in-flight promise so - // FileFinder.create() (which takes native DB locks) runs at most once per - // base path at a time — otherwise parallel tool calls would race and - // deadlock at the native layer (issue #403). - let finderPromise: Promise | null = null; - let activeCwd = process.cwd(); - - // Mode resolution: flag > env > default - let currentMode: FffMode = - (pi.getFlag("fff-mode") as FffMode) ?? - (process.env.PI_FFF_MODE as FffMode) ?? - "tools-and-ui"; - - const toolNames = resolveToolNames(currentMode); - - // DB path resolution: flag > env > undefined (use fff-node defaults) - const frecencyDbPath = - (pi.getFlag("fff-frecency-db") as string | undefined) ?? - process.env.FFF_FRECENCY_DB ?? - undefined; - const historyDbPath = - (pi.getFlag("fff-history-db") as string | undefined) ?? - process.env.FFF_HISTORY_DB ?? - undefined; - - // Root scanning opt-in: flag (boolean) > env ("1"/"true") > false. - // FFF refuses to init at / unless this is set. Home dir scanning is on by - // default for pi — launching pi from $HOME is a normal flow. - function resolveBoolOpt(flagName: string, envName: string): boolean { - const flag = pi.getFlag(flagName); - if (typeof flag === "boolean") return flag; - if (typeof flag === "string") return flag === "true" || flag === "1"; - const env = process.env[envName]; - return env === "1" || env === "true"; - } - const enableFsRootScanning = resolveBoolOpt( - "fff-enable-root-scan", - "FFF_ENABLE_ROOT_SCAN", - ); - - function getMode(): FffMode { - return currentMode; - } - - function setMode(mode: FffMode): void { - currentMode = mode; - } - - function shouldEnableMentions(): boolean { - return currentMode !== "tools-only"; - } - - let auxPool = new AuxFinderPool({ - enableFsRootScanning, - }); - - // in case cwd changes we need to figure this out - function ensureFinder(cwd: string): Promise { - if (mainFinder && !mainFinder.isDestroyed && finderCwd === cwd) - return Promise.resolve(mainFinder); - - if (finderPromise) return finderPromise; - - finderPromise = (async () => { - if (mainFinder && !mainFinder.isDestroyed) { - mainFinder.destroy(); - mainFinder = null; - finderCwd = null; - } - - const { FileFinder } = await loadSdk(); - const result = FileFinder.create({ - basePath: cwd, - frecencyDbPath, - historyDbPath, - aiMode: true, - enableHomeDirScanning: true, - enableFsRootScanning, - }); - - if (!result.ok) - throw new Error(`Failed to create FFF file finder: ${result.error}`); - - mainFinder = result.value; - finderCwd = cwd; - await mainFinder.waitForScan(SCAN_TIMEOUT_MS); - return mainFinder; - })().finally(() => { - finderPromise = null; - }); - - return finderPromise; - } - - function destroyFinder() { - if (mainFinder && !mainFinder.isDestroyed) { - mainFinder.destroy(); - mainFinder = null; - finderCwd = null; - } - - if (auxPool) { - auxPool.destroy(); - } - } - - async function resolveFinderForPath( - pathParam: string | undefined, - pattern: string, - exclude: string | string[] | undefined, - ): Promise<{ finder: FileFinderApi; query: string; root: string } | null> { - const route = routePathConstraint(pathParam, activeCwd); - if (!route) return null; - const aux = await auxPool.acquire(route.root); - // A broader covering picker may have been reused; rebase the suffix so the - // constraint stays relative to the picker's actual root. - const rebase = nodePath - .relative(aux.root, route.root) - .replaceAll(nodePath.sep, "/"); - const suffix = [rebase, route.suffix].filter(Boolean).join("/"); - const query = buildQuery(suffix || undefined, pattern, exclude, aux.root); - return { finder: aux.finder, query, root: aux.root }; - } - - async function getMentionItems( - query: string, - signal: AbortSignal, - ): Promise { - if (signal.aborted) return []; - const f = await ensureFinder(activeCwd); - if (signal.aborted) return []; - - const result = f.mixedSearch(query, { pageSize: MENTION_MAX_RESULTS }); - if (!result.ok) return []; - - return result.value.items - .slice(0, MENTION_MAX_RESULTS) - .map((mixed: MixedItem) => { - if (mixed.type === "directory") { - return { - value: buildAtCompletionValue(mixed.item.relativePath), - label: mixed.item.dirName, - description: mixed.item.relativePath, - }; - } - return { - value: buildAtCompletionValue(mixed.item.relativePath), - label: mixed.item.fileName, - description: mixed.item.relativePath, - }; - }); - } - - function registerAutocompleteProvider(ctx: { - ui: { - addAutocompleteProvider?: ( - factory: (current: AutocompleteProvider) => AutocompleteProvider, - ) => void; - }; - }) { - // pi forks (e.g. omp) may not expose addAutocompleteProvider; skip UI wiring - // and let tools continue to work instead of failing session_start. - if (typeof ctx.ui.addAutocompleteProvider !== "function") return; - - ctx.ui.addAutocompleteProvider((current) => { - const mentionProvider = createFffMentionProvider(getMentionItems); - - return { - async getSuggestions(lines, cursorLine, cursorCol, options) { - if (shouldEnableMentions()) { - try { - const mentionResult = await mentionProvider.getSuggestions( - lines, - cursorLine, - cursorCol, - options, - ); - if (mentionResult) return mentionResult; - } catch { - // Delegate when FFF lookup is unavailable. - } - } - - return current.getSuggestions(lines, cursorLine, cursorCol, options); - }, - applyCompletion(lines, cursorLine, cursorCol, item, prefix) { - return current.applyCompletion( - lines, - cursorLine, - cursorCol, - item, - prefix, - ); - }, - shouldTriggerFileCompletion(lines, cursorLine, cursorCol) { - return ( - current.shouldTriggerFileCompletion?.( - lines, - cursorLine, - cursorCol, - ) ?? true - ); - }, - }; - }); - } - - // --- Flags / lifecycle --- - - pi.registerFlag("fff-mode", { - description: "FFF mode: tools-and-ui | tools-only | override", - type: "string", - }); - - pi.registerFlag("fff-frecency-db", { - description: - "Path to the frecency database (overrides FFF_FRECENCY_DB env)", - type: "string", - }); - - pi.registerFlag("fff-history-db", { - description: - "Path to the query history database (overrides FFF_HISTORY_DB env)", - type: "string", - }); - - pi.registerFlag("fff-enable-root-scan", { - description: - "Allow indexing when launched from the filesystem root (also: FFF_ENABLE_ROOT_SCAN env)", - type: "boolean", - }); - - pi.on("session_start", async (_event, ctx) => { - try { - activeCwd = ctx.cwd; - - // Restore persisted mode from session entries. This handles session - // resume after process restart where env vars are lost, and ensures - // the env var is set for the next /reload in the same session. - const entries = ctx.sessionManager?.getEntries(); - if (entries) { - const modeEntry = [...entries] - .reverse() - .find( - (e: { type: string; customType?: string }) => - e.type === "custom" && e.customType === "fff-mode", - ); - if ( - modeEntry && - typeof (modeEntry as any).data?.mode === "string" && - VALID_MODES.includes((modeEntry as any).data.mode as FffMode) - ) { - const restored = (modeEntry as any).data.mode as FffMode; - if (restored !== currentMode) { - currentMode = restored; - } - } - } - - registerAutocompleteProvider(ctx); - await ensureFinder(activeCwd); - } catch (e: unknown) { - ctx.ui.notify( - `FFF init failed: ${e instanceof Error ? e.message : String(e)}`, - "error", - ); - } - }); - - pi.on("session_shutdown", async () => { - destroyFinder(); - }); - - // --- Shared render helpers --- - - const renderTextResult = ( - result: { content?: { type: string; text?: string }[] }, - options: { expanded?: boolean }, - theme: any, - context: any, - maxLines = 15, - ) => { - const text = - (context.lastComponent as Text | undefined) ?? new Text("", 0, 0); - const output = - result.content?.find((c) => c.type === "text")?.text?.trim() ?? ""; - if (!output) { - text.setText(theme.fg("muted", "No output")); - return text; - } - - const lines = output.split("\n"); - const displayLines = lines.slice( - 0, - options.expanded ? lines.length : maxLines, - ); - let content = `\n${displayLines.map((line: string) => theme.fg("toolOutput", line)).join("\n")}`; - if (lines.length > displayLines.length) { - content += theme.fg( - "muted", - `\n... (${lines.length - displayLines.length} more lines)`, - ); - } - text.setText(content); - return text; - }; - - // --- grep tool --- - - const grepSchema = Type.Object({ - pattern: Type.String({ - description: "Search pattern (literal text or regex)", - }), - path: Type.Optional( - Type.String({ - description: - "Path constraint. Directory prefix (src/ or src/foo/), bare filename with extension (main.rs), or glob (*.ts, src/**/*.cc, {src,lib}/**). Applied to the full repo-relative path. Absolute, ~/, and ../ paths outside the workspace are also supported and searched with a separate index.", - }), - ), - exclude: Type.Optional( - Type.Union([Type.String(), Type.Array(Type.String())], { - description: - "Exclude paths (comma/space-separated or array). Same syntax as path: directory prefix ('test/'), filename with extension ('config.json'), or glob ('*.min.js', '**/*.{rs,go}'). A leading '!' is optional and ignored — both 'test/' and '!test/' work. Example: 'test/,*.min.js,!vendor/'.", - }), - ), - caseSensitive: Type.Optional( - Type.Boolean({ - description: - "Force case-sensitive matching. Default uses smart-case (case-insensitive when pattern is all lowercase).", - }), - ), - context: Type.Optional( - Type.Number({ description: "Context lines before+after each match" }), - ), - limit: Type.Optional( - Type.Number({ - description: `Max matches (default ${DEFAULT_GREP_LIMIT})`, - }), - ), - cursor: Type.Optional( - Type.String({ description: "Pagination cursor from previous result" }), - ), - }); - - pi.registerTool({ - name: toolNames.grep, - label: toolNames.grep, - description: `Grep file contents. Smart-case, auto-detects regex vs literal, git-aware. Results are ranked by frecency (most-accessed files first); matches within a file stay in source order. Default limit ${DEFAULT_GREP_LIMIT}.`, - promptSnippet: "Grep contents", - promptGuidelines: [ - `${toolNames.grep}: prefer bare identifiers as patterns. Literal queries are most efficient.`, - `${toolNames.grep}: use path for include ('src/', '*.ts') and exclude for noise ('test/,*.min.js').`, - `${toolNames.grep}: caseSensitive: true when you need exact case (smart-case otherwise).`, - `${toolNames.grep}: after 1-2 greps, read the top match instead of more greps.`, - ], - parameters: grepSchema, - - async execute(_toolCallId, params, signal) { - if (signal?.aborted) throw new Error("Operation aborted"); - - const pattern = params.pattern; - const aux = await resolveFinderForPath( - params.path, - pattern, - params.exclude, - ); - - const picker = aux ? aux.finder : await ensureFinder(activeCwd); - const effectiveLimit = Math.max(1, params.limit ?? DEFAULT_GREP_LIMIT); - const query = aux - ? aux.query - : buildQuery(params.path, pattern, params.exclude, activeCwd); - - // Auto-detect: regex if the pattern has regex metacharacters AND parses - // as a valid regex, otherwise plain literal. The fuzzy fallback below - // only kicks in for plain mode — regex queries are intentional. - const hasRegexSyntax = - pattern !== pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); - - let mode: GrepMode = hasRegexSyntax ? "regex" : "plain"; - if (mode === "regex") { - try { - new RegExp(pattern); - } catch { - mode = "plain"; - } - } - - // Guard: the agent keeps calling grep with '.*' or similar wildcard-only regex - // to try to read a whole file. That's not what grep is for — return a terse error - // steering them to a real pattern, preventing dozens of wasted retries. - const p = pattern.trim(); - const isWildcardOnly = - hasRegexSyntax && - /^(?:[.^$]*(?:[.][*+?]|\*|\+)[.^$]*|[.^$\s]*|\.\*\??|\.\*[+?]?|\.\+\??|\.|\*|\?)$/.test( - p, - ); - - if (isWildcardOnly) { - return { - content: [ - { - type: "text", - text: `Pattern '${params.pattern}' matches everything — grep needs a concrete substring or identifier. Example: \`pattern: 'MyClass'\` or \`pattern: 'export function'\`.`, - }, - ], - details: { totalMatched: 0, totalFiles: 0 }, - }; - } - - // caseSensitive override flips smartCase off; omitting it keeps smart-case - // (case-insensitive when pattern is all lowercase). - const smartCase = params.caseSensitive !== true; - - const grepResult = picker.grep(query, { - mode, - smartCase, - maxMatchesPerFile: Math.min(effectiveLimit, 50), - cursor: (params.cursor ? getCursor(params.cursor) : null) ?? null, - beforeContext: params.context ?? 0, - afterContext: params.context ?? 0, - classifyDefinitions: true, - timeBudgetMs: GREP_TIME_BUDGET_MS, - }); - - if (!grepResult.ok) throw new Error(grepResult.error); - - let result = grepResult.value; - let fuzzyNotice: string | null = null; - - // if we hit the timeout do not run the fuzzy fallback - // cause it will only consumer more time - if ( - result.items.length === 0 && - !result.nextCursor && - !params.cursor && - mode !== "regex" - ) { - // When the caller pinned a specific file (path has an extension), the - // fuzzy fallback broadens across the whole picker — the file may just - // be misnamed. For directory constraints (or no path), we keep the - // constrained query so the fallback does not leak matches from - // excluded / out-of-scope directories. - const lastSeg = params.path?.split(/[\\/]/).pop() ?? ""; - const pathTargetsFile = /\.[a-zA-Z][a-zA-Z0-9]{0,9}$/.test(lastSeg); - const fuzzyQuery = pathTargetsFile ? pattern : query; - const fuzzy = picker.grep(fuzzyQuery, { - mode: "fuzzy", - smartCase, - maxMatchesPerFile: Math.min(effectiveLimit, 50), - cursor: null, - beforeContext: 0, - afterContext: 0, - classifyDefinitions: true, - timeBudgetMs: GREP_TIME_BUDGET_MS, - }); - - if (fuzzy.ok && fuzzy.value.items.length > 0) { - fuzzyNotice = `0 exact matches. Maybe you meant this?`; - result = fuzzy.value; - } - } - - let output = formatGrepOutput(result); - const notices: string[] = []; - if (result.regexFallbackError) { - notices.push( - `Invalid regex: ${result.regexFallbackError}, used literal match`, - ); - } - if (result.nextCursor) { - notices.push( - `Continue with cursor="${storeCursor(result.nextCursor)}"`, - ); - } - - if (notices.length > 0) output += `\n\n[${notices.join(". ")}]`; - if (fuzzyNotice) output = `[${fuzzyNotice}]\n${output}`; - - return { - content: [{ type: "text", text: output }], - details: { - totalMatched: result.totalMatched, - totalFiles: result.totalFiles, - }, - }; - }, - - renderCall(args, theme, context) { - const text = - (context.lastComponent as Text | undefined) ?? new Text("", 0, 0); - const pattern = args?.pattern ?? ""; - const path = args?.path ?? "."; - let content = - theme.fg("toolTitle", theme.bold(toolNames.grep)) + - " " + - theme.fg("accent", `/${pattern}/`) + - theme.fg("toolOutput", ` in ${path}`); - if (args?.limit !== undefined) - content += theme.fg("toolOutput", ` limit ${args.limit}`); - if (args?.cursor) content += theme.fg("muted", ` (page)`); - text.setText(content); - return text; - }, - - renderResult(result, options, theme, context) { - return renderTextResult(result, options, theme, context, 15); - }, - }); - - // --- find tool --- - - const findSchema = Type.Object({ - pattern: Type.String({ - description: - "Fuzzy filename search and glob search. Frecency-ranked, git-aware. Multi-word = narrower (AND) not bound to order, use for multi word related concept search. Prefer this over ls/find/bash as the first exploration step whenever the user names a concept, feature, or symbol — it surfaces the relevant files in one call. Only use ls/read on a directory when you specifically need the alphabetical layout of an unknown repo, or when a concept search returned nothing.", - }), - path: Type.Optional( - Type.String({ - description: - "Path constraint. Directory prefix (src/ or src/foo/), bare filename with extension (main.rs), or glob (*.ts, src/**/*.cc, {src,lib}/**). Applied to the full repo-relative path. Absolute, ~/, and ../ paths outside the workspace are also supported and searched with a separate index.", - }), - ), - exclude: Type.Optional( - Type.Union([Type.String(), Type.Array(Type.String())], { - description: - "Exclude paths (comma/space-separated or array). Same syntax as path: directory prefix ('test/'), filename with extension ('config.json'), or glob ('*.min.js', '**/*.{rs,go}'). A leading '!' is optional and ignored — both 'test/' and '!test/' work. Example: 'test/,*.min.js,!vendor/'.", - }), - ), - limit: Type.Optional( - Type.Number({ - description: `Max results per page (default ${DEFAULT_FIND_LIMIT})`, - }), - ), - cursor: Type.Optional( - Type.String({ description: "Pagination cursor from previous result" }), - ), - }); - - pi.registerTool({ - name: toolNames.find, - label: toolNames.find, - description: `Fuzzy path search and glob search. Matches against the whole repo-relative path, not just the filename. Frecency-ranked, git-aware. Multi-word = narrower (AND). Default limit ${DEFAULT_FIND_LIMIT}.`, - promptSnippet: "Find files by path or glob", - promptGuidelines: [ - `${toolNames.find}: matches the WHOLE path, not just the filename — \`profile\` hits \`chrome/browser/profiles/x.cc\` too.`, - `${toolNames.find}: keep queries to 1-2 terms; extra words narrow.`, - `${toolNames.find}: use for paths, not content. Use ${toolNames.grep} for content.`, - `${toolNames.find}: for exact path matches use a glob in \`path\` — e.g. path: '**/profile.h' for exact filename, or path: 'src/**/profile.h' scoped to a subtree. Bare patterns are fuzzy.`, - `${toolNames.find}: to list everything inside a directory, pass path: 'dir/**' with an empty or wildcard pattern instead of using pattern alone.`, - `${toolNames.find}: use exclude: 'test/,*.min.js' to cut noise in large repos.`, - ], - parameters: findSchema, - - async execute(_toolCallId, params, signal) { - if (signal?.aborted) throw new Error("Operation aborted"); - - // if resumed we use the same picker as before - const resumed = params.cursor ? getFindCursor(params.cursor) : undefined; - const aux = resumed - ? resumed.auxRoot - ? { - finder: (await auxPool.acquire(resumed.auxRoot, { exact: true })) - .finder, - root: resumed.auxRoot, - } - : null - : await resolveFinderForPath( - params.path, - params.pattern, - params.exclude, - ); - - const picker = aux ? aux.finder : await ensureFinder(activeCwd); - const effectiveLimit = resumed - ? resumed.pageSize - : Math.max(1, params.limit ?? DEFAULT_FIND_LIMIT); - - const query = resumed - ? resumed.query - : aux && "query" in aux - ? (aux as { query: string }).query - : buildQuery(params.path, params.pattern, params.exclude, activeCwd); - - const pattern = resumed ? resumed.pattern : params.pattern; - const pageIndex = resumed?.nextPageIndex ?? 0; - const auxRoot = resumed?.auxRoot ?? aux?.root; - - const searchResult = picker.fileSearch(query, { - pageIndex, - pageSize: effectiveLimit, - }); - if (!searchResult.ok) throw new Error(searchResult.error); - - const result = searchResult.value; - const formatted = formatFindOutput(result, effectiveLimit, pattern); - let output = formatted.output; - - // Infer hasMore: native fileSearch fills pageSize when more results - // exist, so if we got a full page AND totalMatched exceeds what we've - // shown so far there's another page to fetch. - const shownSoFar = pageIndex * effectiveLimit + result.items.length; - const hasMore = - result.items.length >= effectiveLimit && - result.totalMatched > shownSoFar; - - const notices: string[] = []; - if (formatted.weak && formatted.shownCount > 0) - notices.push( - `Query "${pattern}" produced only weak scattered fuzzy matches. Output capped at ${formatted.shownCount}/${result.totalMatched}.`, - ); - - if (!formatted.weak && hasMore) { - const remaining = result.totalMatched - shownSoFar; - const cursorId = storeFindCursor({ - query, - pattern, - pageSize: effectiveLimit, - nextPageIndex: pageIndex + 1, - auxRoot, - }); - notices.push( - `${remaining} more match${remaining === 1 ? "" : "es"} available. cursor="${cursorId}" to continue`, - ); - } - - if (notices.length > 0) output += `\n\n[${notices.join(". ")}]`; - return { - content: [{ type: "text", text: output }], - details: { - totalMatched: result.totalMatched, - totalFiles: result.totalFiles, - pageIndex, - hasMore, - }, - }; - }, - - renderCall(args, theme, context) { - const text = - (context.lastComponent as Text | undefined) ?? new Text("", 0, 0); - const pattern = args?.pattern ?? ""; - const path = args?.path ?? "."; - let content = - theme.fg("toolTitle", theme.bold(toolNames.find)) + - " " + - theme.fg("accent", pattern) + - theme.fg("toolOutput", ` in ${path}`); - if (args?.limit !== undefined) - content += theme.fg("toolOutput", ` (limit ${args.limit})`); - if (args?.cursor) content += theme.fg("muted", ` (page)`); - text.setText(content); - return text; - }, - - renderResult(result, options, theme, context) { - return renderTextResult(result, options, theme, context, 20); - }, - }); - - // --- multi_grep tool --- - // My latest tests are showing that the multi grep tool is only harmful, trying to get rid of it - const enableMultiGrep = process.env.PI_FFF_MULTIGREP === "1"; - - if (enableMultiGrep) { - const multiGrepSchema = Type.Object({ - patterns: Type.Array(Type.String(), { - description: - "Literal patterns (OR). Include snake_case/camelCase/PascalCase variants.", - }), - constraints: Type.Optional( - Type.String({ description: "File filter, e.g. '*.{ts,tsx} !test/'" }), - ), - context: Type.Optional( - Type.Number({ description: "Context lines before+after" }), - ), - limit: Type.Optional( - Type.Number({ - description: `Max matches (default ${DEFAULT_GREP_LIMIT})`, - }), - ), - cursor: Type.Optional(Type.String({ description: "Pagination cursor" })), - }); - - pi.registerTool({ - name: toolNames.multiGrep, - label: toolNames.multiGrep, - description: - "Search file contents for ANY of multiple literal patterns (OR, SIMD Aho-Corasick). Faster than regex alternation.", - promptSnippet: "Multi-pattern OR content search", - promptGuidelines: [ - `${toolNames.multiGrep}: use when searching for several identifiers at once.`, - `${toolNames.multiGrep}: include all naming-convention variants (snake/camel/Pascal).`, - `${toolNames.multiGrep}: patterns are literal. Use constraints for file filters.`, - ], - parameters: multiGrepSchema, - - async execute(_toolCallId, params, signal) { - if (signal?.aborted) throw new Error("Operation aborted"); - if (!params.patterns?.length) - throw new Error("patterns array must have at least 1 element"); - - const f = await ensureFinder(activeCwd); - const effectiveLimit = Math.max(1, params.limit ?? DEFAULT_GREP_LIMIT); - - const grepResult = f.multiGrep({ - patterns: params.patterns, - constraints: params.constraints, - maxMatchesPerFile: Math.min(effectiveLimit, 50), - smartCase: true, - cursor: (params.cursor ? getCursor(params.cursor) : null) ?? null, - beforeContext: params.context ?? 0, - afterContext: params.context ?? 0, - }); - - if (!grepResult.ok) throw new Error(grepResult.error); - - const result = grepResult.value; - let output = formatGrepOutput(result); - - const notices: string[] = []; - if (result.items.length >= effectiveLimit) - notices.push(`${effectiveLimit}+ matches (refine patterns)`); - if (result.nextCursor) - notices.push( - `More available. cursor="${storeCursor(result.nextCursor)}" to continue`, - ); - - if (notices.length > 0) output += `\n\n[${notices.join(". ")}]`; - - return { - content: [{ type: "text", text: output }], - details: { - totalMatched: result.totalMatched, - totalFiles: result.totalFiles, - patterns: params.patterns, - }, - }; - }, - - renderCall(args, theme, context) { - const text = - (context.lastComponent as Text | undefined) ?? new Text("", 0, 0); - const patterns = args?.patterns ?? []; - const constraints = args?.constraints; - let content = - theme.fg("toolTitle", theme.bold(toolNames.multiGrep)) + - " " + - theme.fg("accent", patterns.map((p: string) => `"${p}"`).join(", ")); - if (constraints) content += theme.fg("toolOutput", ` (${constraints})`); - if (args?.cursor) content += theme.fg("muted", ` (page)`); - text.setText(content); - return text; - }, - - renderResult(result, options, theme, context) { - return renderTextResult(result, options, theme, context, 15); - }, - }); - } // end if (enableMultiGrep) - - // --- commands --- - - pi.registerCommand("fff-mode", { - description: - "Show or set FFF mode: /fff-mode [tools-and-ui | tools-only | override]", - handler: async (args, ctx) => { - const arg = (args || "").trim(); - - // No args - show current mode - if (!arg) { - const mode = getMode(); - const flag = pi.getFlag("fff-mode") ?? "unset"; - ctx.ui.notify(`Current mode: '${mode}' (flag: ${flag})`, "info"); - return; - } - - // Validate and set mode - if (!VALID_MODES.includes(arg as FffMode)) { - ctx.ui.notify( - `Usage: /fff-mode [${VALID_MODES.join(" | ")}]`, - "warning", - ); - return; - } - - const newMode = arg as FffMode; - const oldMode = getMode(); - setMode(newMode); - - pi.appendEntry("fff-mode", { mode: newMode }); - - const note = - (oldMode === "override") !== (newMode === "override") - ? " (tool name change requires /reload)" - : ""; - ctx.ui.notify(`Mode changed: '${oldMode}' → '${newMode}'${note}`, "info"); - }, - }); - - pi.registerCommand("fff-health", { - description: "Show FFF file finder health and status", - handler: async (_args, ctx) => { - if (!mainFinder || mainFinder.isDestroyed) { - ctx.ui.notify("FFF not initialized", "warning"); - return; - } - - const health = mainFinder.healthCheck(); - if (!health.ok) { - ctx.ui.notify(`Health check failed: ${health.error}`, "error"); - return; - } - - const lines = [ - `FFF v${health.value.version}`, - `Mode: ${getMode()}`, - `Git: ${health.value.git.repositoryFound ? `yes (${health.value.git.workdir ?? "unknown"})` : "no"}`, - `Picker: ${health.value.filePicker.initialized ? `${health.value.filePicker.indexedFiles ?? 0} files` : "not initialized"}`, - `Frecency: ${health.value.frecency.initialized ? "active" : "disabled"}`, - `Query tracker: ${health.value.queryTracker.initialized ? "active" : "disabled"}`, - ]; - - const progress = mainFinder.getScanProgress(); - if (progress.ok) { - lines.push( - `Scanning: ${progress.value.isScanning ? "yes" : "no"} (${progress.value.scannedFilesCount} files)`, - ); - } - - ctx.ui.notify(lines.join("\n"), "info"); - }, - }); - - pi.registerCommand("fff-rescan", { - description: "Trigger FFF to rescan files", - handler: async (_args, ctx) => { - if (!mainFinder || mainFinder.isDestroyed) { - ctx.ui.notify("FFF not initialized", "warning"); - return; - } - - const result = mainFinder.scanFiles(); - if (!result.ok) { - ctx.ui.notify(`Rescan failed: ${result.error}`, "error"); - return; - } - - ctx.ui.notify("FFF rescan triggered", "info"); - }, - }); -} +/** + * pi-fff: FFF-powered file search extension for pi + * + * Overrides built-in `find` and `grep` tools with FFF and adds FFF-backed + * @-mention autocomplete suggestions to the interactive editor. + */ + +import nodePath from "node:path"; +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { + type AutocompleteItem, + type AutocompleteProvider, + Text, +} from "@earendil-works/pi-tui"; +import type { + FileFinderApi, + GrepCursor, + GrepMode, + GrepResult, + MixedItem, + SearchResult, +} from "@ff-labs/fff-node"; +import { Type } from "@sinclair/typebox"; +import { AuxFinderPool, routePathConstraint } from "./aux-finders"; +import { buildQuery } from "./query"; +import { loadSdk, SCAN_TIMEOUT_MS } from "./sdk"; + +export { SCAN_TIMEOUT_MS } from "./sdk"; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const DEFAULT_GREP_LIMIT = 20; +const DEFAULT_FIND_LIMIT = 30; +const GREP_MAX_LINE_LENGTH = 500; +const MENTION_MAX_RESULTS = 20; + +// If we exceed 10 seconds for indexed grep - something is definitely off +const GREP_TIME_BUDGET_MS = 10_000; + +type FffMode = "tools-and-ui" | "tools-only" | "override"; + +const VALID_MODES: FffMode[] = ["tools-and-ui", "tools-only", "override"]; + +interface ToolNames { + grep: string; + find: string; + multiGrep: string; +} + +const FFF_TOOL_NAMES: ToolNames = { + grep: "ffgrep", + find: "fffind", + multiGrep: "fff-multi-grep", +}; +const OVERRIDE_TOOL_NAMES: ToolNames = { + grep: "grep", + find: "find", + multiGrep: "multi_grep", +}; + +function resolveToolNames(mode: FffMode): ToolNames { + return mode === "override" ? OVERRIDE_TOOL_NAMES : FFF_TOOL_NAMES; +} + +// --------------------------------------------------------------------------- +// Cursor store — simple bounded Map for pagination cursors +// --------------------------------------------------------------------------- + +const cursorCache = new Map(); +let cursorCounter = 0; + +function storeCursor(cursor: GrepCursor): string { + const id = `fff_c${++cursorCounter}`; + cursorCache.set(id, cursor); + if (cursorCache.size > 200) { + const first = cursorCache.keys().next().value; + if (first) cursorCache.delete(first); + } + return id; +} + +function getCursor(id: string): GrepCursor | undefined { + return cursorCache.get(id); +} + +// Find pagination uses a page-index cursor: native `fileSearch` takes +// pageIndex/pageSize, so the cursor is just the next page index paired with +// the query+limit that produced it. Stored tokens are opaque IDs to the agent. +interface FindCursor { + query: string; + pattern: string; + pageSize: number; + nextPageIndex: number; + auxRoot?: string; +} + +const findCursorCache = new Map(); +let findCursorCounter = 0; + +function storeFindCursor(cursor: FindCursor): string { + const id = `${++findCursorCounter}`; + findCursorCache.set(id, cursor); + if (findCursorCache.size > 200) { + const first = findCursorCache.keys().next().value; + if (first) findCursorCache.delete(first); + } + return id; +} + +function getFindCursor(id: string): FindCursor | undefined { + return findCursorCache.get(id); +} + +// --------------------------------------------------------------------------- +// Output formatting helpers +// --------------------------------------------------------------------------- + +function truncateLine(line: string, max = GREP_MAX_LINE_LENGTH): string { + const trimmed = line.trim(); + return trimmed.length <= max ? trimmed : `${trimmed.slice(0, max)}...`; +} + +const HOT_FRECENCY = 25; +const WARM_FRECENCY = 20; + +// Shared annotation helper for both find-output paths and grep-output file +// headers. Returns at most ONE tag so output stays scannable. Priority: +// git-dirty (most actionable — file is changing right now) beats frecency +// (historically often-touched). Keeping one function ensures the two tools +// never drift in how they surface git/frecency signal. +export function fffFileAnnotation(item: { + gitStatus?: string; + totalFrecencyScore?: number; + accessFrecencyScore?: number; +}): string { + const git = item.gitStatus; + if (git && git !== "clean" && git !== "unknown" && git !== "") { + return ` [${git} in git]`; + } + + const frecency = item.totalFrecencyScore ?? item.accessFrecencyScore ?? 0; + if (frecency >= HOT_FRECENCY) return " [VERY often touched file]"; + if (frecency >= WARM_FRECENCY) return " [often touched file]"; + + return ""; +} + +// fff-core native definition classifier (byte-level scanner in Rust) is enabled +// via GrepOptions.classifyDefinitions. Each GrepMatch carries isDefinition for +// downstream consumers; pi-fff does NOT use it to re-sort. +// +// Ordering policy: NO CUSTOM SORTING. The engine already returns items in +// frecency order (most-accessed files first). pi-fff only groups consecutive +// matches into per-file blocks and preserves whatever order the engine +// provided — inside a file we keep matches in source-line order because the +// engine emits them that way. + +function formatGrepOutput(result: GrepResult): string { + if (result.items.length === 0) return "No matches found"; + + // Build file-grouped output in the order files first appear in the result. + // This preserves native frecency ordering across files without re-sorting. + const lines: string[] = []; + let currentFile = ""; + let shown = 0; + + for (const match of result.items) { + if (match.relativePath !== currentFile) { + if (lines.length > 0) lines.push(""); + currentFile = match.relativePath; + lines.push(`${currentFile}${fffFileAnnotation(match)}`); + } + + match.contextBefore?.forEach((line: string, i: number) => { + const lineNum = match.lineNumber - match.contextBefore!.length + i; + lines.push(` ${lineNum}- ${truncateLine(line)}`); + }); + + lines.push(` ${match.lineNumber}: ${truncateLine(match.lineContent)}`); + shown++; + + match.contextAfter?.forEach((line: string, i: number) => { + const lineNum = match.lineNumber + 1 + i; + lines.push(` ${lineNum}- ${truncateLine(line)}`); + }); + } + + return lines.join("\n"); +} + +// Weak-match threshold is derived from the query length, matching the +// scoring formula in crates/fff-core/src/score.rs: a perfect match scores +// `len * 16`, so we treat anything below 50% of that as scattered fuzzy noise. +// When the top score is weak, trim output to a small sample instead of dumping +// the full limit worth of noise into the agent's context. +const FIND_WEAK_SAMPLE_SIZE = 5; + +function weakScoreThreshold(pattern: string): number { + const perfect = pattern.length * 12; + return Math.floor((perfect * 50) / 100); +} + +interface FormattedFind { + output: string; + weak: boolean; + shownCount: number; +} + +function formatFindOutput( + result: SearchResult, + limit: number, + pattern: string, +): FormattedFind { + if (result.items.length === 0) { + return { + output: "No files found matching pattern", + weak: false, + shownCount: 0, + }; + } + + // NO CUSTOM SORTING — trust native frecency order from the engine. + const reordered = result.items.map((item) => ({ item })); + + // Peek at the top native score to decide whether results are scattered + // fuzzy noise (query length-scaled threshold from score.rs). + const topScore = result.scores[0]?.total ?? 0; + const weak = topScore < weakScoreThreshold(pattern); + const effective = weak ? Math.min(FIND_WEAK_SAMPLE_SIZE, limit) : limit; + const shown = reordered.slice(0, effective); + + return { + output: shown + .map((p) => `${p.item.relativePath}${fffFileAnnotation(p.item)}`) + .join("\n"), + weak, + shownCount: shown.length, + }; +} + +// --------------------------------------------------------------------------- +// Mention autocomplete helpers +// --------------------------------------------------------------------------- + +function extractAtPrefix(textBeforeCursor: string): string | null { + const match = textBeforeCursor.match(/(?:^|[ \t])(@(?:"[^"]*|[^\s]*))$/); + return match?.[1] ?? null; +} + +function buildAtCompletionValue(path: string): string { + return path.includes(" ") ? `@"${path}"` : `@${path}`; +} + +function createFffMentionProvider( + getItems: (query: string, signal: AbortSignal) => Promise, +): AutocompleteProvider { + return { + async getSuggestions(lines, cursorLine, cursorCol, options) { + const currentLine = lines[cursorLine] || ""; + const prefix = extractAtPrefix(currentLine.slice(0, cursorCol)); + if (!prefix || options.signal.aborted) return null; + + const query = prefix.startsWith('@"') ? prefix.slice(2) : prefix.slice(1); + const items = await getItems(query, options.signal); + return options.signal.aborted || items.length === 0 ? null : { items, prefix }; + }, + applyCompletion(_lines, cursorLine, cursorCol, item, prefix) { + const currentLine = _lines[cursorLine] || ""; + const before = currentLine.slice(0, cursorCol - prefix.length); + const after = currentLine.slice(cursorCol); + const newLine = before + item.value + after; + const newCursorCol = cursorCol - prefix.length + item.value.length; + return { + lines: [..._lines.slice(0, cursorLine), newLine, ..._lines.slice(cursorLine + 1)], + cursorLine, + cursorCol: newCursorCol, + }; + }, + }; +} + +// --------------------------------------------------------------------------- +// Extension +// --------------------------------------------------------------------------- + +export default function fffExtension(pi: ExtensionAPI) { + let mainFinder: FileFinderApi | null = null; + let finderCwd: string | null = null; + // Concurrent ensureFinder() callers share the same in-flight promise so + // FileFinder.create() (which takes native DB locks) runs at most once per + // base path at a time — otherwise parallel tool calls would race and + // deadlock at the native layer (issue #403). + let finderPromise: Promise | null = null; + let activeCwd = process.cwd(); + + // Mode resolution: flag > env > default + let currentMode: FffMode = + (pi.getFlag("fff-mode") as FffMode) ?? + (process.env.PI_FFF_MODE as FffMode) ?? + "tools-and-ui"; + + const toolNames = resolveToolNames(currentMode); + + // DB path resolution: flag > env > undefined (use fff-node defaults) + const frecencyDbPath = + (pi.getFlag("fff-frecency-db") as string | undefined) ?? + process.env.FFF_FRECENCY_DB ?? + undefined; + const historyDbPath = + (pi.getFlag("fff-history-db") as string | undefined) ?? + process.env.FFF_HISTORY_DB ?? + undefined; + + // Root scanning opt-in: flag (boolean) > env ("1"/"true") > false. + // FFF refuses to init at / unless this is set. Home dir scanning is on by + // default for pi — launching pi from $HOME is a normal flow. + function resolveBoolOpt(flagName: string, envName: string): boolean { + const flag = pi.getFlag(flagName); + if (typeof flag === "boolean") return flag; + if (typeof flag === "string") return flag === "true" || flag === "1"; + const env = process.env[envName]; + return env === "1" || env === "true"; + } + const enableFsRootScanning = resolveBoolOpt( + "fff-enable-root-scan", + "FFF_ENABLE_ROOT_SCAN", + ); + + function getMode(): FffMode { + return currentMode; + } + + function setMode(mode: FffMode): void { + currentMode = mode; + } + + function shouldEnableMentions(): boolean { + return currentMode !== "tools-only"; + } + + let auxPool = new AuxFinderPool({ + enableFsRootScanning, + }); + + // in case cwd changes we need to figure this out + function ensureFinder(cwd: string): Promise { + if (mainFinder && !mainFinder.isDestroyed && finderCwd === cwd) + return Promise.resolve(mainFinder); + + if (finderPromise) return finderPromise; + + finderPromise = (async () => { + if (mainFinder && !mainFinder.isDestroyed) { + mainFinder.destroy(); + mainFinder = null; + finderCwd = null; + } + + const { FileFinder } = await loadSdk(); + const result = FileFinder.create({ + basePath: cwd, + frecencyDbPath, + historyDbPath, + aiMode: true, + enableHomeDirScanning: true, + enableFsRootScanning, + }); + + if (!result.ok) + throw new Error(`Failed to create FFF file finder: ${result.error}`); + + mainFinder = result.value; + finderCwd = cwd; + await mainFinder.waitForScan(SCAN_TIMEOUT_MS); + return mainFinder; + })().finally(() => { + finderPromise = null; + }); + + return finderPromise; + } + + function destroyFinder() { + if (mainFinder && !mainFinder.isDestroyed) { + mainFinder.destroy(); + mainFinder = null; + finderCwd = null; + } + + if (auxPool) { + auxPool.destroy(); + } + } + + async function resolveFinderForPath( + pathParam: string | undefined, + pattern: string, + exclude: string | string[] | undefined, + ): Promise<{ finder: FileFinderApi; query: string; root: string } | null> { + const route = routePathConstraint(pathParam, activeCwd); + if (!route) return null; + const aux = await auxPool.acquire(route.root); + // A broader covering picker may have been reused; rebase the suffix so the + // constraint stays relative to the picker's actual root. + const rebase = nodePath.relative(aux.root, route.root).replaceAll(nodePath.sep, "/"); + const suffix = [rebase, route.suffix].filter(Boolean).join("/"); + const query = buildQuery(suffix || undefined, pattern, exclude, aux.root); + return { finder: aux.finder, query, root: aux.root }; + } + + async function getMentionItems( + query: string, + signal: AbortSignal, + ): Promise { + if (signal.aborted) return []; + const f = await ensureFinder(activeCwd); + if (signal.aborted) return []; + + const result = f.mixedSearch(query, { pageSize: MENTION_MAX_RESULTS }); + if (!result.ok) return []; + + return result.value.items.slice(0, MENTION_MAX_RESULTS).map((mixed: MixedItem) => { + if (mixed.type === "directory") { + return { + value: buildAtCompletionValue(mixed.item.relativePath), + label: mixed.item.dirName, + description: mixed.item.relativePath, + }; + } + return { + value: buildAtCompletionValue(mixed.item.relativePath), + label: mixed.item.fileName, + description: mixed.item.relativePath, + }; + }); + } + + function registerAutocompleteProvider(ctx: { + ui: { + addAutocompleteProvider?: ( + factory: (current: AutocompleteProvider) => AutocompleteProvider, + ) => void; + }; + }) { + // pi forks (e.g. omp) may not expose addAutocompleteProvider; skip UI wiring + // and let tools continue to work instead of failing session_start. + if (typeof ctx.ui.addAutocompleteProvider !== "function") return; + + ctx.ui.addAutocompleteProvider((current) => { + const mentionProvider = createFffMentionProvider(getMentionItems); + + return { + async getSuggestions(lines, cursorLine, cursorCol, options) { + if (shouldEnableMentions()) { + try { + const mentionResult = await mentionProvider.getSuggestions( + lines, + cursorLine, + cursorCol, + options, + ); + if (mentionResult) return mentionResult; + } catch { + // Delegate when FFF lookup is unavailable. + } + } + + return current.getSuggestions(lines, cursorLine, cursorCol, options); + }, + applyCompletion(lines, cursorLine, cursorCol, item, prefix) { + return current.applyCompletion(lines, cursorLine, cursorCol, item, prefix); + }, + shouldTriggerFileCompletion(lines, cursorLine, cursorCol) { + return ( + current.shouldTriggerFileCompletion?.(lines, cursorLine, cursorCol) ?? true + ); + }, + }; + }); + } + + // --- Flags / lifecycle --- + + pi.registerFlag("fff-mode", { + description: "FFF mode: tools-and-ui | tools-only | override", + type: "string", + }); + + pi.registerFlag("fff-frecency-db", { + description: "Path to the frecency database (overrides FFF_FRECENCY_DB env)", + type: "string", + }); + + pi.registerFlag("fff-history-db", { + description: "Path to the query history database (overrides FFF_HISTORY_DB env)", + type: "string", + }); + + pi.registerFlag("fff-enable-root-scan", { + description: + "Allow indexing when launched from the filesystem root (also: FFF_ENABLE_ROOT_SCAN env)", + type: "boolean", + }); + + pi.on("session_start", async (_event, ctx) => { + try { + activeCwd = ctx.cwd; + + // Restore persisted mode from session entries. This handles session + // resume after process restart where env vars are lost, and ensures + // the env var is set for the next /reload in the same session. + const entries = ctx.sessionManager?.getEntries(); + if (entries) { + const modeEntry = [...entries] + .reverse() + .find( + (e: { type: string; customType?: string }) => + e.type === "custom" && e.customType === "fff-mode", + ); + if ( + modeEntry && + typeof (modeEntry as any).data?.mode === "string" && + VALID_MODES.includes((modeEntry as any).data.mode as FffMode) + ) { + const restored = (modeEntry as any).data.mode as FffMode; + if (restored !== currentMode) { + currentMode = restored; + } + } + } + + registerAutocompleteProvider(ctx); + await ensureFinder(activeCwd); + } catch (e: unknown) { + ctx.ui.notify( + `FFF init failed: ${e instanceof Error ? e.message : String(e)}`, + "error", + ); + } + }); + + pi.on("session_shutdown", async () => { + destroyFinder(); + }); + + // --- Shared render helpers --- + + const renderTextResult = ( + result: { content?: { type: string; text?: string }[] }, + options: { expanded?: boolean }, + theme: any, + context: any, + maxLines = 15, + ) => { + const text = (context.lastComponent as Text | undefined) ?? new Text("", 0, 0); + const output = result.content?.find((c) => c.type === "text")?.text?.trim() ?? ""; + if (!output) { + text.setText(theme.fg("muted", "No output")); + return text; + } + + const lines = output.split("\n"); + const displayLines = lines.slice(0, options.expanded ? lines.length : maxLines); + let content = `\n${displayLines.map((line: string) => theme.fg("toolOutput", line)).join("\n")}`; + if (lines.length > displayLines.length) { + content += theme.fg( + "muted", + `\n... (${lines.length - displayLines.length} more lines)`, + ); + } + text.setText(content); + return text; + }; + + // --- grep tool --- + + const grepSchema = Type.Object({ + pattern: Type.String({ + description: "Search pattern (literal text or regex)", + }), + path: Type.Optional( + Type.String({ + description: + "Path constraint. Directory prefix (src/ or src/foo/), bare filename with extension (main.rs), or glob (*.ts, src/**/*.cc, {src,lib}/**). Applied to the full repo-relative path. Absolute, ~/, and ../ paths outside the workspace are also supported and searched with a separate index.", + }), + ), + exclude: Type.Optional( + Type.Union([Type.String(), Type.Array(Type.String())], { + description: + "Exclude paths (comma/space-separated or array). Same syntax as path: directory prefix ('test/'), filename with extension ('config.json'), or glob ('*.min.js', '**/*.{rs,go}'). A leading '!' is optional and ignored — both 'test/' and '!test/' work. Example: 'test/,*.min.js,!vendor/'.", + }), + ), + caseSensitive: Type.Optional( + Type.Boolean({ + description: + "Force case-sensitive matching. Default uses smart-case (case-insensitive when pattern is all lowercase).", + }), + ), + context: Type.Optional( + Type.Number({ description: "Context lines before+after each match" }), + ), + limit: Type.Optional( + Type.Number({ + description: `Max matches (default ${DEFAULT_GREP_LIMIT})`, + }), + ), + cursor: Type.Optional( + Type.String({ description: "Pagination cursor from previous result" }), + ), + }); + + pi.registerTool({ + name: toolNames.grep, + label: toolNames.grep, + description: `Grep file contents. Smart-case, auto-detects regex vs literal, git-aware. Results are ranked by frecency (most-accessed files first); matches within a file stay in source order. Default limit ${DEFAULT_GREP_LIMIT}.`, + promptSnippet: "Grep contents", + promptGuidelines: [ + `${toolNames.grep}: prefer bare identifiers as patterns. Literal queries are most efficient.`, + `${toolNames.grep}: use path for include ('src/', '*.ts') and exclude for noise ('test/,*.min.js').`, + `${toolNames.grep}: caseSensitive: true when you need exact case (smart-case otherwise).`, + `${toolNames.grep}: after 1-2 greps, read the top match instead of more greps.`, + ], + parameters: grepSchema, + + async execute(_toolCallId, params, signal) { + if (signal?.aborted) throw new Error("Operation aborted"); + + const pattern = params.pattern; + const aux = await resolveFinderForPath(params.path, pattern, params.exclude); + + const picker = aux ? aux.finder : await ensureFinder(activeCwd); + const effectiveLimit = Math.max(1, params.limit ?? DEFAULT_GREP_LIMIT); + const query = aux + ? aux.query + : buildQuery(params.path, pattern, params.exclude, activeCwd); + + // Auto-detect: regex if the pattern has regex metacharacters AND parses + // as a valid regex, otherwise plain literal. The fuzzy fallback below + // only kicks in for plain mode — regex queries are intentional. + const hasRegexSyntax = pattern !== pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + + let mode: GrepMode = hasRegexSyntax ? "regex" : "plain"; + if (mode === "regex") { + try { + new RegExp(pattern); + } catch { + mode = "plain"; + } + } + + // Guard: the agent keeps calling grep with '.*' or similar wildcard-only regex + // to try to read a whole file. That's not what grep is for — return a terse error + // steering them to a real pattern, preventing dozens of wasted retries. + const p = pattern.trim(); + const isWildcardOnly = + hasRegexSyntax && + /^(?:[.^$]*(?:[.][*+?]|\*|\+)[.^$]*|[.^$\s]*|\.\*\??|\.\*[+?]?|\.\+\??|\.|\*|\?)$/.test( + p, + ); + + if (isWildcardOnly) { + return { + content: [ + { + type: "text", + text: `Pattern '${params.pattern}' matches everything — grep needs a concrete substring or identifier. Example: \`pattern: 'MyClass'\` or \`pattern: 'export function'\`.`, + }, + ], + details: { totalMatched: 0, totalFiles: 0 }, + }; + } + + // caseSensitive override flips smartCase off; omitting it keeps smart-case + // (case-insensitive when pattern is all lowercase). + const smartCase = params.caseSensitive !== true; + + const grepResult = picker.grep(query, { + mode, + smartCase, + maxMatchesPerFile: Math.min(effectiveLimit, 50), + cursor: (params.cursor ? getCursor(params.cursor) : null) ?? null, + beforeContext: params.context ?? 0, + afterContext: params.context ?? 0, + classifyDefinitions: true, + timeBudgetMs: GREP_TIME_BUDGET_MS, + }); + + if (!grepResult.ok) throw new Error(grepResult.error); + + let result = grepResult.value; + let fuzzyNotice: string | null = null; + + // if we hit the timeout do not run the fuzzy fallback + // cause it will only consumer more time + if ( + result.items.length === 0 && + !result.nextCursor && + !params.cursor && + mode !== "regex" + ) { + // When the caller pinned a specific file (path has an extension), the + // fuzzy fallback broadens across the whole picker — the file may just + // be misnamed. For directory constraints (or no path), we keep the + // constrained query so the fallback does not leak matches from + // excluded / out-of-scope directories. + const lastSeg = params.path?.split(/[\\/]/).pop() ?? ""; + const pathTargetsFile = /\.[a-zA-Z][a-zA-Z0-9]{0,9}$/.test(lastSeg); + const fuzzyQuery = pathTargetsFile ? pattern : query; + const fuzzy = picker.grep(fuzzyQuery, { + mode: "fuzzy", + smartCase, + maxMatchesPerFile: Math.min(effectiveLimit, 50), + cursor: null, + beforeContext: 0, + afterContext: 0, + classifyDefinitions: true, + timeBudgetMs: GREP_TIME_BUDGET_MS, + }); + + if (fuzzy.ok && fuzzy.value.items.length > 0) { + fuzzyNotice = `0 exact matches. Maybe you meant this?`; + result = fuzzy.value; + } + } + + let output = formatGrepOutput(result); + const notices: string[] = []; + if (result.regexFallbackError) { + notices.push(`Invalid regex: ${result.regexFallbackError}, used literal match`); + } + if (result.nextCursor) { + notices.push(`Continue with cursor="${storeCursor(result.nextCursor)}"`); + } + + if (notices.length > 0) output += `\n\n[${notices.join(". ")}]`; + if (fuzzyNotice) output = `[${fuzzyNotice}]\n${output}`; + + return { + content: [{ type: "text", text: output }], + details: { + totalMatched: result.totalMatched, + totalFiles: result.totalFiles, + }, + }; + }, + + renderCall(args, theme, context) { + const text = (context.lastComponent as Text | undefined) ?? new Text("", 0, 0); + const pattern = args?.pattern ?? ""; + const path = args?.path ?? "."; + let content = + theme.fg("toolTitle", theme.bold(toolNames.grep)) + + " " + + theme.fg("accent", `/${pattern}/`) + + theme.fg("toolOutput", ` in ${path}`); + if (args?.limit !== undefined) + content += theme.fg("toolOutput", ` limit ${args.limit}`); + if (args?.cursor) content += theme.fg("muted", ` (page)`); + text.setText(content); + return text; + }, + + renderResult(result, options, theme, context) { + return renderTextResult(result, options, theme, context, 15); + }, + }); + + // --- find tool --- + + const findSchema = Type.Object({ + pattern: Type.String({ + description: + "Fuzzy filename search and glob search. Frecency-ranked, git-aware. Multi-word = narrower (AND) not bound to order, use for multi word related concept search. Prefer this over ls/find/bash as the first exploration step whenever the user names a concept, feature, or symbol — it surfaces the relevant files in one call. Only use ls/read on a directory when you specifically need the alphabetical layout of an unknown repo, or when a concept search returned nothing.", + }), + path: Type.Optional( + Type.String({ + description: + "Path constraint. Directory prefix (src/ or src/foo/), bare filename with extension (main.rs), or glob (*.ts, src/**/*.cc, {src,lib}/**). Applied to the full repo-relative path. Absolute, ~/, and ../ paths outside the workspace are also supported and searched with a separate index.", + }), + ), + exclude: Type.Optional( + Type.Union([Type.String(), Type.Array(Type.String())], { + description: + "Exclude paths (comma/space-separated or array). Same syntax as path: directory prefix ('test/'), filename with extension ('config.json'), or glob ('*.min.js', '**/*.{rs,go}'). A leading '!' is optional and ignored — both 'test/' and '!test/' work. Example: 'test/,*.min.js,!vendor/'.", + }), + ), + limit: Type.Optional( + Type.Number({ + description: `Max results per page (default ${DEFAULT_FIND_LIMIT})`, + }), + ), + cursor: Type.Optional( + Type.String({ description: "Pagination cursor from previous result" }), + ), + }); + + pi.registerTool({ + name: toolNames.find, + label: toolNames.find, + description: `Fuzzy path search and glob search. Matches against the whole repo-relative path, not just the filename. Frecency-ranked, git-aware. Multi-word = narrower (AND). Default limit ${DEFAULT_FIND_LIMIT}.`, + promptSnippet: "Find files by path or glob", + promptGuidelines: [ + `${toolNames.find}: matches the WHOLE path, not just the filename — \`profile\` hits \`chrome/browser/profiles/x.cc\` too.`, + `${toolNames.find}: keep queries to 1-2 terms; extra words narrow.`, + `${toolNames.find}: use for paths, not content. Use ${toolNames.grep} for content.`, + `${toolNames.find}: for exact path matches use a glob in \`path\` — e.g. path: '**/profile.h' for exact filename, or path: 'src/**/profile.h' scoped to a subtree. Bare patterns are fuzzy.`, + `${toolNames.find}: to list everything inside a directory, pass path: 'dir/**' with an empty or wildcard pattern instead of using pattern alone.`, + `${toolNames.find}: use exclude: 'test/,*.min.js' to cut noise in large repos.`, + ], + parameters: findSchema, + + async execute(_toolCallId, params, signal) { + if (signal?.aborted) throw new Error("Operation aborted"); + + // if resumed we use the same picker as before + const resumed = params.cursor ? getFindCursor(params.cursor) : undefined; + const aux = resumed + ? resumed.auxRoot + ? { + finder: (await auxPool.acquire(resumed.auxRoot, { exact: true })).finder, + root: resumed.auxRoot, + } + : null + : await resolveFinderForPath(params.path, params.pattern, params.exclude); + + const picker = aux ? aux.finder : await ensureFinder(activeCwd); + const effectiveLimit = resumed + ? resumed.pageSize + : Math.max(1, params.limit ?? DEFAULT_FIND_LIMIT); + + const query = resumed + ? resumed.query + : aux && "query" in aux + ? (aux as { query: string }).query + : buildQuery(params.path, params.pattern, params.exclude, activeCwd); + + const pattern = resumed ? resumed.pattern : params.pattern; + const pageIndex = resumed?.nextPageIndex ?? 0; + const auxRoot = resumed?.auxRoot ?? aux?.root; + + const searchResult = picker.fileSearch(query, { + pageIndex, + pageSize: effectiveLimit, + }); + if (!searchResult.ok) throw new Error(searchResult.error); + + const result = searchResult.value; + const formatted = formatFindOutput(result, effectiveLimit, pattern); + let output = formatted.output; + + // Infer hasMore: native fileSearch fills pageSize when more results + // exist, so if we got a full page AND totalMatched exceeds what we've + // shown so far there's another page to fetch. + const shownSoFar = pageIndex * effectiveLimit + result.items.length; + const hasMore = + result.items.length >= effectiveLimit && result.totalMatched > shownSoFar; + + const notices: string[] = []; + if (formatted.weak && formatted.shownCount > 0) + notices.push( + `Query "${pattern}" produced only weak scattered fuzzy matches. Output capped at ${formatted.shownCount}/${result.totalMatched}.`, + ); + + if (!formatted.weak && hasMore) { + const remaining = result.totalMatched - shownSoFar; + const cursorId = storeFindCursor({ + query, + pattern, + pageSize: effectiveLimit, + nextPageIndex: pageIndex + 1, + auxRoot, + }); + notices.push( + `${remaining} more match${remaining === 1 ? "" : "es"} available. cursor="${cursorId}" to continue`, + ); + } + + if (notices.length > 0) output += `\n\n[${notices.join(". ")}]`; + return { + content: [{ type: "text", text: output }], + details: { + totalMatched: result.totalMatched, + totalFiles: result.totalFiles, + pageIndex, + hasMore, + }, + }; + }, + + renderCall(args, theme, context) { + const text = (context.lastComponent as Text | undefined) ?? new Text("", 0, 0); + const pattern = args?.pattern ?? ""; + const path = args?.path ?? "."; + let content = + theme.fg("toolTitle", theme.bold(toolNames.find)) + + " " + + theme.fg("accent", pattern) + + theme.fg("toolOutput", ` in ${path}`); + if (args?.limit !== undefined) + content += theme.fg("toolOutput", ` (limit ${args.limit})`); + if (args?.cursor) content += theme.fg("muted", ` (page)`); + text.setText(content); + return text; + }, + + renderResult(result, options, theme, context) { + return renderTextResult(result, options, theme, context, 20); + }, + }); + + // --- multi_grep tool --- + // My latest tests are showing that the multi grep tool is only harmful, trying to get rid of it + const enableMultiGrep = process.env.PI_FFF_MULTIGREP === "1"; + + if (enableMultiGrep) { + const multiGrepSchema = Type.Object({ + patterns: Type.Array(Type.String(), { + description: + "Literal patterns (OR). Include snake_case/camelCase/PascalCase variants.", + }), + constraints: Type.Optional( + Type.String({ description: "File filter, e.g. '*.{ts,tsx} !test/'" }), + ), + context: Type.Optional(Type.Number({ description: "Context lines before+after" })), + limit: Type.Optional( + Type.Number({ + description: `Max matches (default ${DEFAULT_GREP_LIMIT})`, + }), + ), + cursor: Type.Optional(Type.String({ description: "Pagination cursor" })), + }); + + pi.registerTool({ + name: toolNames.multiGrep, + label: toolNames.multiGrep, + description: + "Search file contents for ANY of multiple literal patterns (OR, SIMD Aho-Corasick). Faster than regex alternation.", + promptSnippet: "Multi-pattern OR content search", + promptGuidelines: [ + `${toolNames.multiGrep}: use when searching for several identifiers at once.`, + `${toolNames.multiGrep}: include all naming-convention variants (snake/camel/Pascal).`, + `${toolNames.multiGrep}: patterns are literal. Use constraints for file filters.`, + ], + parameters: multiGrepSchema, + + async execute(_toolCallId, params, signal) { + if (signal?.aborted) throw new Error("Operation aborted"); + if (!params.patterns?.length) + throw new Error("patterns array must have at least 1 element"); + + const f = await ensureFinder(activeCwd); + const effectiveLimit = Math.max(1, params.limit ?? DEFAULT_GREP_LIMIT); + + const grepResult = f.multiGrep({ + patterns: params.patterns, + constraints: params.constraints, + maxMatchesPerFile: Math.min(effectiveLimit, 50), + smartCase: true, + cursor: (params.cursor ? getCursor(params.cursor) : null) ?? null, + beforeContext: params.context ?? 0, + afterContext: params.context ?? 0, + }); + + if (!grepResult.ok) throw new Error(grepResult.error); + + const result = grepResult.value; + let output = formatGrepOutput(result); + + const notices: string[] = []; + if (result.items.length >= effectiveLimit) + notices.push(`${effectiveLimit}+ matches (refine patterns)`); + if (result.nextCursor) + notices.push( + `More available. cursor="${storeCursor(result.nextCursor)}" to continue`, + ); + + if (notices.length > 0) output += `\n\n[${notices.join(". ")}]`; + + return { + content: [{ type: "text", text: output }], + details: { + totalMatched: result.totalMatched, + totalFiles: result.totalFiles, + patterns: params.patterns, + }, + }; + }, + + renderCall(args, theme, context) { + const text = (context.lastComponent as Text | undefined) ?? new Text("", 0, 0); + const patterns = args?.patterns ?? []; + const constraints = args?.constraints; + let content = + theme.fg("toolTitle", theme.bold(toolNames.multiGrep)) + + " " + + theme.fg("accent", patterns.map((p: string) => `"${p}"`).join(", ")); + if (constraints) content += theme.fg("toolOutput", ` (${constraints})`); + if (args?.cursor) content += theme.fg("muted", ` (page)`); + text.setText(content); + return text; + }, + + renderResult(result, options, theme, context) { + return renderTextResult(result, options, theme, context, 15); + }, + }); + } // end if (enableMultiGrep) + + // --- commands --- + + pi.registerCommand("fff-mode", { + description: "Show or set FFF mode: /fff-mode [tools-and-ui | tools-only | override]", + handler: async (args, ctx) => { + const arg = (args || "").trim(); + + // No args - show current mode + if (!arg) { + const mode = getMode(); + const flag = pi.getFlag("fff-mode") ?? "unset"; + ctx.ui.notify(`Current mode: '${mode}' (flag: ${flag})`, "info"); + return; + } + + // Validate and set mode + if (!VALID_MODES.includes(arg as FffMode)) { + ctx.ui.notify(`Usage: /fff-mode [${VALID_MODES.join(" | ")}]`, "warning"); + return; + } + + const newMode = arg as FffMode; + const oldMode = getMode(); + setMode(newMode); + + pi.appendEntry("fff-mode", { mode: newMode }); + + const note = + (oldMode === "override") !== (newMode === "override") + ? " (tool name change requires /reload)" + : ""; + ctx.ui.notify(`Mode changed: '${oldMode}' → '${newMode}'${note}`, "info"); + }, + }); + + pi.registerCommand("fff-health", { + description: "Show FFF file finder health and status", + handler: async (_args, ctx) => { + if (!mainFinder || mainFinder.isDestroyed) { + ctx.ui.notify("FFF not initialized", "warning"); + return; + } + + const health = mainFinder.healthCheck(); + if (!health.ok) { + ctx.ui.notify(`Health check failed: ${health.error}`, "error"); + return; + } + + const lines = [ + `FFF v${health.value.version}`, + `Mode: ${getMode()}`, + `Git: ${health.value.git.repositoryFound ? `yes (${health.value.git.workdir ?? "unknown"})` : "no"}`, + `Picker: ${health.value.filePicker.initialized ? `${health.value.filePicker.indexedFiles ?? 0} files` : "not initialized"}`, + `Frecency: ${health.value.frecency.initialized ? "active" : "disabled"}`, + `Query tracker: ${health.value.queryTracker.initialized ? "active" : "disabled"}`, + ]; + + const progress = mainFinder.getScanProgress(); + if (progress.ok) { + lines.push( + `Scanning: ${progress.value.isScanning ? "yes" : "no"} (${progress.value.scannedFilesCount} files)`, + ); + } + + ctx.ui.notify(lines.join("\n"), "info"); + }, + }); + + pi.registerCommand("fff-rescan", { + description: "Trigger FFF to rescan files", + handler: async (_args, ctx) => { + if (!mainFinder || mainFinder.isDestroyed) { + ctx.ui.notify("FFF not initialized", "warning"); + return; + } + + const result = mainFinder.scanFiles(); + if (!result.ok) { + ctx.ui.notify(`Rescan failed: ${result.error}`, "error"); + return; + } + + ctx.ui.notify("FFF rescan triggered", "info"); + }, + }); +} diff --git a/packages/pi-fff/src/query.ts b/packages/pi-fff/src/query.ts index 38ccb6dc0..501d8333d 100644 --- a/packages/pi-fff/src/query.ts +++ b/packages/pi-fff/src/query.ts @@ -10,11 +10,7 @@ export function normalizePathConstraint( if (path.isAbsolute(trimmed)) { const relative = path.relative(cwd, trimmed).replaceAll(path.sep, "/"); if (relative === "") return null; - if ( - relative.startsWith("../") || - relative === ".." || - path.isAbsolute(relative) - ) { + if (relative.startsWith("../") || relative === ".." || path.isAbsolute(relative)) { throw new Error( `Path constraint must be relative to the workspace: ${pathConstraint}`, ); diff --git a/packages/pi-fff/test/aux-finders.test.ts b/packages/pi-fff/test/aux-finders.test.ts index 7237aa030..53263ca54 100644 --- a/packages/pi-fff/test/aux-finders.test.ts +++ b/packages/pi-fff/test/aux-finders.test.ts @@ -98,9 +98,7 @@ describe("routePathConstraint", () => { }); test("returns null when .. resolves back inside the workspace", () => { - expect( - routePathConstraint("../workspace/src", workspace), - ).toBeNull(); + expect(routePathConstraint("../workspace/src", workspace)).toBeNull(); }); }); }); diff --git a/packages/shared/fff-api.ts b/packages/shared/fff-api.ts index 01b2201d8..dcd9f2ac2 100644 --- a/packages/shared/fff-api.ts +++ b/packages/shared/fff-api.ts @@ -567,16 +567,10 @@ export interface FileFinderApi { glob(pattern: string, options?: GlobOptions): Result; /** Fuzzy directory search. */ - directorySearch( - query: string, - options?: DirSearchOptions, - ): Result; + directorySearch(query: string, options?: DirSearchOptions): Result; /** Fuzzy search over files and directories interleaved by score. */ - mixedSearch( - query: string, - options?: SearchOptions, - ): Result; + mixedSearch(query: string, options?: SearchOptions): Result; /** Content search (live grep). */ grep(query: string, options?: GrepOptions): Result; @@ -641,10 +635,7 @@ export interface FileFinderApi { * Events are debounced and submitted in batches per 100-ms window at most 128 events. * Gitignored and other ignored files are never triggering watcher. */ - watch( - callback: WatchBatchCallback, - options?: WatchOptions, - ): Result; + watch(callback: WatchBatchCallback, options?: WatchOptions): Result; watch( pattern: string, callback: WatchBatchCallback,