more efficient way to track subdirs

Author: dmtrKovalenko

crates/fff-core/src/file_picker.rs

@@ -729,19 +729,6 @@ impl FilePicker {
         &self.sync_data.dirs
     }
 
-    /// Whether the absolute `path` is a directory known to the index
-    /// (including dirs that were empty at scan time).
-    pub(crate) fn has_indexed_dir(&self, path: &Path) -> bool {
-        let Ok(rel) = path.strip_prefix(&self.base_path) else {
-            return false;
-        };
-        let mut rel = crate::path_utils::to_canonical_slashes(&rel.to_string_lossy()).into_owned();
-        if !rel.is_empty() && !rel.ends_with('/') {
-            rel.push('/');
-        }
-        self.sync_data.find_dir_index(&rel).is_some()
-    }
-
     /// Actual heap bytes used: (chunked_path_store, 0, 0).
     /// The second element is 0 because leaked overflow stores aren't tracked.
     pub fn arena_bytes(&self) -> (usize, usize, usize) {
@@ -2047,9 +2034,7 @@ impl FileSync {
         )?;
         let ignore_rules = ignore_rules.map(Arc::new);
 
-        // Sort files by (dir_part, filename) — grouping them into contiguous
-        // per-dir runs — and the walked dirs by the same '/'-terminated form,
-        // so the build pass below can merge both in a single sweep.
+        // group walked dirs and files with a dir part to the same order
         BACKGROUND_THREAD_POOL.install(|| {
             rayon::join(
                 || {
@@ -2200,35 +2185,45 @@ fn populates_dirs_files_chunked_storage<'a>(
     let mut dirs: Vec<DirItem> = Vec::with_capacity(walked_dirs.len() + 1);
     let mut dir_iter = walked_dirs.iter().peekable();
 
+    // Root-level files sort first and their "" parent is never a walker dir.
+    if pairs
+        .first()
+        .is_some_and(|(f, _)| f.path.filename_offset == 0)
+    {
+        push_dir_item(&mut dirs, chunk_storage, "");
+    }
+
+    // Detects contiguous same-dir runs (pairs are sorted by dir) so the
+    // merge below runs once per directory, not once per file.
     let mut prev_dir: &'a str = "";
-    let mut prev_dir_valid = false;
     let mut current_dir_idx: u32 = 0;
 
     for (file, rel) in pairs.iter_mut() {
         let rel: &'a str = rel;
         let dir_part: &'a str = &rel[..file.path.filename_offset as usize];
 
-        if !prev_dir_valid || prev_dir != dir_part {
+        if prev_dir != dir_part {
             // Flush walked dirs up to and including this file's parent,
             // keeping the table sorted for the find_dir_index binary search.
-            let mut matched = false;
             while let Some(dir) = dir_iter.peek()
-                && dir.as_str() <= dir_part
+                && dir.as_str() < dir_part
             {
-                matched = dir.as_str() == dir_part;
                 push_dir_item(&mut dirs, chunk_storage, dir);
                 dir_iter.next();
             }
 
-            // Root-level files ("" dir part) and parents the walker reported
-            // with a non-dir kind (e.g. followed symlinks) aren't in the list.
-            if !matched {
-                push_dir_item(&mut dirs, chunk_storage, dir_part);
+            match dir_iter.peek() {
+                Some(dir) if dir.as_str() == dir_part => {
+                    push_dir_item(&mut dirs, chunk_storage, dir);
+                    dir_iter.next();
+                }
+                // Parents the walker reported with a non-dir kind
+                // (e.g. followed symlinks) aren't in the list.
+                _ => push_dir_item(&mut dirs, chunk_storage, dir_part),
             }
-            current_dir_idx = (dirs.len() - 1) as u32;
 
+            current_dir_idx = (dirs.len() - 1) as u32;
             prev_dir = dir_part;
-            prev_dir_valid = true;
         }
 
         file.path = chunk_storage.add_file_immediate(rel, file.path.filename_offset);

crates/fff-core/src/walk/mod.rs

@@ -20,9 +20,7 @@ pub(crate) use ripgrep::walk_collect_files;
 
 pub(crate) struct WalkOutput {
     pub(crate) pairs: Vec<(FileItem, String)>,
-    /// Every non-ignored directory the walk visited, as '/'-canonical
-    /// '/'-terminated paths relative to the base (base itself excluded).
-    /// This is the source of the picker's dir table; unsorted (parallel walk).
+    /// Every non-ignored directory the walk visited, relative, ending with /
     pub(crate) dirs: Vec<String>,
     pub(crate) ignore_rules: Option<WalkIgnoreRules>,
 }

crates/fff-core/src/walk/ripgrep.rs

@@ -34,11 +34,12 @@ pub(crate) fn walk_collect_files(
 
     let walker = walk_builder.build_parallel();
 
-    let pairs = parking_lot::Mutex::new(Vec::<(FileItem, String)>::new());
-    let dirs = parking_lot::Mutex::new(Vec::<String>::new());
+    // Single lock for both collections: every entry is either a file or a
+    // dir, so this keeps one mutex acquisition per entry.
+    let collected =
+        parking_lot::Mutex::new((Vec::<(FileItem, String)>::new(), Vec::<String>::new()));
     walker.run(|| {
-        let pairs = &pairs;
-        let dirs = &dirs;
+        let collected = &collected;
         let counter = Arc::clone(synced_files_count);
         let base_path = base_path.to_path_buf();
 
@@ -60,7 +61,7 @@ pub(crate) fn walk_collect_files(
                 let (file_item, rel_path) =
                     FileItem::new_from_walk(path, &base_path, None, metadata.as_ref());
 
-                pairs.lock().push((file_item, rel_path));
+                collected.lock().0.push((file_item, rel_path));
                 counter.fetch_add(1, Ordering::Relaxed);
             } else if entry.depth() > 0 && entry.file_type().is_some_and(|ft| ft.is_dir()) {
                 let path = entry.path();
@@ -70,16 +71,17 @@ pub(crate) fn walk_collect_files(
                     let mut rel = crate::path_utils::to_canonical_slashes(&rel.to_string_lossy())
                         .into_owned();
                     rel.push('/');
-                    dirs.lock().push(rel);
+                    collected.lock().1.push(rel);
                 }
             }
             ignore::WalkState::Continue
         })
     });
 
+    let (pairs, dirs) = collected.into_inner();
     Ok(WalkOutput {
-        pairs: pairs.into_inner(),
-        dirs: dirs.into_inner(),
+        pairs,
+        dirs,
         ignore_rules: None,
     })
 }

crates/fff-core/src/walk/zlob.rs

@@ -46,19 +46,19 @@ pub(crate) fn walk_collect_files(
         tracing::warn!(?e, "zlob extra_ignore rejected; walking without it");
     }
 
-    let pairs = Mutex::new(Vec::new());
-    let dirs = Mutex::new(Vec::new());
+    // Single lock for both collections: every entry is either a file or a
+    // dir, so this keeps one mutex acquisition per entry.
+    let collected = Mutex::new((Vec::new(), Vec::new()));
 
     let outcome = match builder.run(|entry| {
         if !entry.is_file() {
-            // `.git` and ignored dirs are pruned by zlob itself (GITIGNORE
-            // flag); empty rel path is the walk root, covered separately.
+            // unlike ripgrep walker zlob doesnt show .git files
             if entry.is_dir() {
                 let rel_bytes = entry.relative_path_bytes();
                 if !rel_bytes.is_empty() {
                     let mut rel = String::from_utf8_lossy(rel_bytes).into_owned();
                     rel.push('/');
-                    dirs.lock().push(rel);
+                    collected.lock().1.push(rel);
                 }
             }
 
@@ -84,9 +84,9 @@ pub(crate) fn walk_collect_files(
         let rel_str = String::from_utf8_lossy(rel_bytes).into_owned();
         let item = FileItem::new_raw(basename_offset, size, modified, None, is_binary);
 
-        let mut guard = pairs.lock();
-        guard.push((item, rel_str));
-        let n = guard.len();
+        let mut guard = collected.lock();
+        guard.0.push((item, rel_str));
+        let n = guard.0.len();
         drop(guard);
 
         if n % PROGRESS_STEP == 0 {
@@ -104,7 +104,7 @@ pub(crate) fn walk_collect_files(
         }
     };
 
-    let pairs = pairs.into_inner();
+    let (pairs, dirs) = collected.into_inner();
     // Always report the exact final total regardless of the last step.
     synced_files_count.store(pairs.len(), Ordering::Relaxed);
 
@@ -117,7 +117,7 @@ pub(crate) fn walk_collect_files(
 
     Ok(WalkOutput {
         pairs,
-        dirs: dirs.into_inner(),
+        dirs,
         ignore_rules,
     })
 }