Improve pattern search exclusions and ranked overflow results

CHANGELOG.md

@@ -8,6 +8,10 @@ Status of the `main` branch. Changes prior to the next official version change w
     the actual semantics (configurations are automatically migrated)
   - Fix: glob matching bare `*` and `?` in non-`**` patterns matched across `/`, contradicting documented behaviour #1732
 
+* Tools:
+  - `search_for_pattern`: support additive exclusion globs and ordered priority globs, and preserve
+    representative prioritized matches with omitted-result counts when responses exceed the output limit.
+
 * Language Servers: 
   - Allow language server priorities to be configured in `serena_config.yml` (for auto-detection during 
     project creation) 

src/serena/project.py

@@ -403,6 +403,7 @@ def search_project_files_for_pattern(
         context_lines_after: int = 0,
         paths_include_glob: str | None = None,
         paths_exclude_glob: str | None = None,
+        paths_exclude_globs: list[str] | None = None,
         multiline: bool = True,
         code_files_only: bool = True,
     ) -> list[MatchedConsecutiveLines]:
@@ -415,6 +416,7 @@ def search_project_files_for_pattern(
         :param context_lines_after: Number of lines of context to include after each match
         :param paths_include_glob: Glob pattern to filter which files to include in the search
         :param paths_exclude_glob: Glob pattern to filter which files to exclude from the search. Takes precedence over paths_include_glob.
+        :param paths_exclude_globs: Additional glob patterns to exclude. Additive with paths_exclude_glob.
         :param multiline: Whether to compile the regex with the DOTALL flag (``.`` matches newlines).
         :return: List of matched consecutive lines with context
         """
@@ -426,6 +428,7 @@ def search_project_files_for_pattern(
             context_lines_after=context_lines_after,
             paths_include_glob=paths_include_glob,
             paths_exclude_glob=paths_exclude_glob,
+            paths_exclude_globs=paths_exclude_globs,
             multiline=multiline,
         )
 

src/serena/tools/file_tools.py

@@ -6,6 +6,7 @@
   * editing at the file level
 """
 
+import json
 import os
 from collections import defaultdict
 from fnmatch import fnmatch
@@ -17,6 +18,7 @@
 from serena.util.text_utils import (
     ContentReplacer,
     GlobMatcher,
+    MatchedConsecutiveLines,
     MultiFileContentReplacer,
     ReplacementOccurrence,
 )
@@ -566,6 +568,8 @@ def apply(
         context_lines_after: int = 0,
         paths_include_glob: str = "",
         paths_exclude_glob: str = "",
+        paths_exclude_globs: list[str] | None = None,
+        paths_priority_globs: list[str] | None = None,
         relative_path: str = "",
         restrict_search_to_code_files: bool = False,
         multiline: bool = True,
@@ -580,6 +584,9 @@ def apply(
         :param context_lines_after: number of context lines to include after each match.
         :param paths_include_glob: optional glob (relative to project root, e.g. ``"src/**/*.ts"``) restricting which files are searched.
         :param paths_exclude_glob: optional glob to exclude files; takes precedence over `paths_include_glob`.
+        :param paths_exclude_globs: additional optional globs to exclude files; additive with `paths_exclude_glob`.
+        :param paths_priority_globs: optional ordered globs that rank matching paths before unmatched paths.
+            The first matching glob determines a path's priority.
         :param relative_path: restricts the search to this file or subdirectory of the project root
         :param restrict_search_to_code_files: whether to search only files containing analyzable code symbols
             (useful when looking for class/method definitions); otherwise also search non-code files.
@@ -599,10 +606,31 @@ def apply(
             context_lines_after=context_lines_after,
             paths_include_glob=paths_include_glob.strip(),
             paths_exclude_glob=paths_exclude_glob.strip(),
+            paths_exclude_globs=[glob.strip() for glob in paths_exclude_globs or [] if glob.strip()],
             multiline=multiline,
             code_files_only=restrict_search_to_code_files,
         )
 
+        # normalize the public path contract before ranking and serialization
+        for match in matches:
+            assert match.source_file_path is not None
+            match.source_file_path = GlobMatcher.normalize_path(match.source_file_path)
+
+        # rank prioritized matches deterministically while preserving source order when no priorities are supplied
+        priority_matchers = [GlobMatcher(glob.strip()) for glob in paths_priority_globs or [] if glob.strip()]
+        if priority_matchers:
+
+            def priority(indexed_match: tuple[int, MatchedConsecutiveLines]) -> tuple[int, str, int, int]:
+                original_index, match = indexed_match
+                assert match.source_file_path is not None
+                priority_index = next(
+                    (index for index, matcher in enumerate(priority_matchers) if matcher.matches(match.source_file_path)),
+                    len(priority_matchers),
+                )
+                return priority_index, match.source_file_path, match.matched_lines[0].line_number, original_index
+
+            matches = [match for _, match in sorted(enumerate(matches), key=priority)]
+
         # group matches by file
         file_to_matches: dict[str, list[str]] = defaultdict(list)
         for match in matches:
@@ -616,9 +644,79 @@ def apply(
             first = match.matched_lines[0]
             match_lines_by_file[match.source_file_path].append({"line": first.line_number, "text": first.line_content.strip()})
 
+        result = self._to_json(file_to_matches)
+
         # shortened result closures, from least to most aggressive shortening
         _TEXT_TRUNCATE = 60
 
+        def make_ranked_excerpt() -> str:
+            """Largest ranked prefix of whole match entries that fits the configured answer budget."""
+            effective_max_answer_chars = (
+                self.agent.serena_config.default_max_tool_answer_chars if max_answer_chars == -1 else max_answer_chars
+            )
+            too_long_message = (
+                f"The answer is too long ({len(result)} characters). You can adjust your query or raise the max_answer_chars parameter."
+            )
+            excerpt_budget = effective_max_answer_chars - len(too_long_message) - 1
+            excerpt_header = "Ranked match excerpt:\n"
+
+            # precompute exact serialized prefix sizes and file counts in one pass
+            serialized_mapping_lengths = [2]  # opening and closing braces
+            shown_file_counts = [0]
+            shown_paths: set[str] = set()
+            for match in matches:
+                assert match.source_file_path is not None
+                path = match.source_file_path
+                serialized_match_length = len(json.dumps(match.to_display_string(), ensure_ascii=False))
+                serialized_mapping_length = serialized_mapping_lengths[-1]
+                if path in shown_paths:
+                    serialized_mapping_length += 2 + serialized_match_length
+                else:
+                    if shown_paths:
+                        serialized_mapping_length += 2
+                    serialized_mapping_length += len(json.dumps(path, ensure_ascii=False)) + 3 + serialized_match_length + 1
+                    shown_paths.add(path)
+                serialized_mapping_lengths.append(serialized_mapping_length)
+                shown_file_counts.append(len(shown_paths))
+
+            # precompute counts of files represented by each omitted suffix
+            omitted_file_counts = [0] * (len(matches) + 1)
+            omitted_paths: set[str] = set()
+            for index in range(len(matches) - 1, -1, -1):
+                path = matches[index].source_file_path
+                assert path is not None
+                omitted_paths.add(path)
+                omitted_file_counts[index] = len(omitted_paths)
+
+            def footer(shown_match_count: int) -> str:
+                return (
+                    f"Showing {shown_match_count} of {len(matches)} matches across "
+                    f"{shown_file_counts[shown_match_count]} of {len(file_to_matches)} files; "
+                    f"omitted {len(matches) - shown_match_count} matches across "
+                    f"{omitted_file_counts[shown_match_count]} files; refine the query for omitted results."
+                )
+
+            # select the largest fitting prefix without serializing candidate mappings
+            shown_match_count = 0
+            for shown_match_count in range(1, len(matches) + 1):
+                excerpt_length = len(excerpt_header) + serialized_mapping_lengths[shown_match_count] + 1 + len(footer(shown_match_count))
+                if excerpt_length <= excerpt_budget:
+                    continue
+                shown_match_count -= 1
+                break
+            else:
+                shown_match_count = len(matches)
+
+            if shown_match_count == 0:
+                return result
+
+            # render the selected prefix once
+            excerpt: dict[str, list[str]] = defaultdict(list)
+            for match in matches[:shown_match_count]:
+                assert match.source_file_path is not None
+                excerpt[match.source_file_path].append(match.to_display_string())
+            return f"{excerpt_header}{self._to_json(excerpt)}\n{footer(shown_match_count)}"
+
         def render_first_lines(truncate: bool) -> str:
             """Render each match's first line, either in full or truncated to a fixed length."""
 
@@ -660,11 +758,11 @@ def make_per_file_counts() -> str:
         def make_summary() -> str:
             return f"Found {len(matches)} matches in {len(match_lines_by_file)} files."
 
-        result = self._to_json(file_to_matches)
         return self._limit_length(
             result,
             max_answer_chars,
             shortened_result_factories=[
+                make_ranked_excerpt,
                 make_first_lines_full,
                 make_first_lines_truncated,
                 make_line_numbers_only,

src/serena/util/file_proxy.py

@@ -97,22 +97,29 @@ def __iter__(self) -> Iterator[FileProxy]:
     def from_local_project_paths(cls, relative_paths: list[str], project: "Project") -> Self:
         return cls([LocalProjectFileProxy(path, project) for path in relative_paths])
 
-    def filter_glob(self, paths_include_glob: str | None = None, paths_exclude_glob: str | None = None) -> "FileCollection":
+    def filter_glob(
+        self,
+        paths_include_glob: str | None = None,
+        paths_exclude_glob: str | None = None,
+        paths_exclude_globs: list[str] | None = None,
+    ) -> "FileCollection":
         """
         Filters the collection based on the given patterns.
         Note: Filtering is applied only to local project files. Other files are always retained.
 
         :param paths_include_glob: optional glob pattern to include files from the list
         :param paths_exclude_glob: optional glob pattern to exclude files from the list
+        :param paths_exclude_globs: additional optional glob patterns to exclude; additive with paths_exclude_glob
         :return: the filtered collection
         """
         from serena.util.text_utils import GlobMatcher
 
-        if paths_include_glob is None and paths_exclude_glob is None:
+        if paths_include_glob is None and paths_exclude_glob is None and not paths_exclude_globs:
             return self
 
         include_glob_matcher = GlobMatcher(paths_include_glob) if paths_include_glob else None
-        exclude_glob_matcher = GlobMatcher(paths_exclude_glob) if paths_exclude_glob else None
+        exclude_globs = [glob for glob in [paths_exclude_glob, *(paths_exclude_globs or [])] if glob]
+        exclude_glob_matchers = [(glob, GlobMatcher(glob)) for glob in exclude_globs]
 
         filtered_files = []
         for f in self._file_proxies:
@@ -122,10 +129,10 @@ def filter_glob(self, paths_include_glob: str | None = None, paths_exclude_glob:
                     if not include_glob_matcher.matches(path):
                         log.debug(f"Skipping {path}: does not match include pattern {paths_include_glob}")
                         continue
-                if exclude_glob_matcher:
-                    if exclude_glob_matcher.matches(path):
-                        log.debug(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}")
-                        continue
+                matching_exclude_glob = next((glob for glob, matcher in exclude_glob_matchers if matcher.matches(path)), None)
+                if matching_exclude_glob:
+                    log.debug(f"Skipping {path}: matches exclude pattern {matching_exclude_glob}")
+                    continue
             filtered_files.append(f)
 
         return FileCollection(filtered_files)

src/serena/util/text_utils.py

@@ -203,14 +203,19 @@ def __init__(self, expr: str):
         """
         :param expr: a glob pattern which may make use of brace expansion (e.g., "src/**/*.{js,jsx,ts,tsx}")
         """
-        expr = expr.replace("\\", "/")  # normalise backslashes to forward slashes
+        expr = self.normalize_path(expr)
         self._glob_expr = expr
         self._glob_patterns = self._expand_braces(expr)
         self._regex_patterns = [re.compile(self._translate_glob_to_regex(p)) for p in self._glob_patterns]
 
     def _tostring_includes(self) -> list[str]:
         return ["_glob_expr"]
 
+    @staticmethod
+    def normalize_path(path: str) -> str:
+        """Normalize a project-relative path for glob matching and public search output."""
+        return path.replace("\\", "/")
+
     @staticmethod
     def _expand_braces(pattern: str) -> list[str]:
         """
@@ -289,7 +294,7 @@ def _translate_glob_to_regex(pattern: str) -> str:
         return f"(?s:{inner})\\Z"
 
     def matches(self, path: str) -> bool:
-        path = path.replace("\\", "/")  # normalise backslashes to forward slashes
+        path = self.normalize_path(path)
         return any(regex.match(path) for regex in self._regex_patterns)
 
 
@@ -300,6 +305,7 @@ def search_files(
     context_lines_after: int = 0,
     paths_include_glob: str | None = None,
     paths_exclude_glob: str | None = None,
+    paths_exclude_globs: list[str] | None = None,
     multiline: bool = True,
 ) -> list[MatchedConsecutiveLines]:
     """
@@ -311,22 +317,28 @@ def search_files(
     :param context_lines_after: number of context lines to include after matches
     :param paths_include_glob: optional glob pattern to include files from the list
     :param paths_exclude_glob: optional glob pattern to exclude files from the list
+    :param paths_exclude_globs: additional optional glob patterns to exclude; additive with paths_exclude_glob
     :param multiline: whether to apply multi-line matching, enabling the flags re.DOTALL and re.MULTILINE (default: True)
     :return: list of MatchedConsecutiveLines objects
     """
     # apply glob filter
-    file_collection = file_collection.filter_glob(paths_include_glob=paths_include_glob, paths_exclude_glob=paths_exclude_glob)
+    file_collection = file_collection.filter_glob(
+        paths_include_glob=paths_include_glob,
+        paths_exclude_glob=paths_exclude_glob,
+        paths_exclude_globs=paths_exclude_globs,
+    )
     log.info(f"Processing {len(file_collection)} files.")
 
     def process_single_file(file_proxy: FileProxy) -> dict[str, Any]:
         """Process a single file - this function will be parallelized."""
         relative_path = file_proxy.get_relative_path()
+        normalized_relative_path = GlobMatcher.normalize_path(relative_path)
         try:
             file_content = file_proxy.get_contents()
             search_results = search_text(
                 pattern,
                 content=file_content,
-                source_file_path=relative_path,
+                source_file_path=normalized_relative_path,
                 context_lines_before=context_lines_before,
                 context_lines_after=context_lines_after,
                 multiline=multiline,