From 4d007248e0bde189a613fc4da1c03cffb37bc37b Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Wed, 6 May 2026 08:40:12 -0400 Subject: [PATCH] perf(meta-phases): let the model decide when to browse the repo, not the prompt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The meta_semantic / meta_mechanical / meta_systemic phases were burning 15–40 minutes apiece in production, even though Kimi K2.6 is fast per-turn. Profiling traced this back to the prompts themselves: each phase had an "Investigation Protocol" section that *mandated* browsing the actual source files, and a "Quality Gate" that *rejected* diff-only answers ("If your rationale says 'visible in the diff' or 'based on the patches', you have not investigated enough"). That language turned every phase into a 20–40 turn opencode session of file reads, when the diff patches plus the intake/anatomy summary already contain almost everything these phases need to produce dimensions. The model wasn't slow — it was being instructed to do expensive exploration before it was allowed to answer. Replaced the mandate with neutral framing: - "Working with Context": diff + intake/anatomy is usually enough. Repo access is available; use it sparingly, only when the diff genuinely doesn't show what you need. - "Quality Bar": dimensions must be specific (named files, line ranges). The diff already carries those; a dimension grounded in the diff is good. Vague dimensions are still rejected. Each lens keeps its specific bullets (semantic = logic/contracts, mechanical = signatures/imports, systemic = patterns/coverage) and its target_files requirement. The output schema is unchanged. What changes is the model is no longer penalized for producing a correct answer from the diff in 2 turns instead of 30. Also bumped the in-code default model openrouter/moonshotai/kimi-k2.5 → k2.6 to match what's actually deployed in production. Local dev was silently running on an older model than prod. Co-Authored-By: Claude Opus 4.7 (1M context) --- .env.example | 4 +- src/pr_af/config.py | 6 +- src/pr_af/reasoners/harnesses.py | 100 ++++++++++++++++--------------- 3 files changed, 56 insertions(+), 54 deletions(-) diff --git a/.env.example b/.env.example index db1e1b0..7a5536a 100644 --- a/.env.example +++ b/.env.example @@ -5,8 +5,8 @@ OPENROUTER_API_KEY= GH_TOKEN= # Model configuration (optional — defaults shown) -# PR_AF_MODEL=openrouter/moonshotai/kimi-k2.5 -# PR_AF_AI_MODEL=openrouter/moonshotai/kimi-k2.5 +# PR_AF_MODEL=openrouter/moonshotai/kimi-k2.6 +# PR_AF_AI_MODEL=openrouter/moonshotai/kimi-k2.6 # PR_AF_PROVIDER=opencode # # PR_AF_MODEL applies to all tiers (budget/mid/premium) unless individually diff --git a/src/pr_af/config.py b/src/pr_af/config.py index 5215e8b..019e1e9 100644 --- a/src/pr_af/config.py +++ b/src/pr_af/config.py @@ -90,7 +90,7 @@ def _default_tier_map(provider: str = "opencode") -> dict[str, str]: # opencode / default — OpenRouter model IDs ai_model = os.getenv( "PR_AF_AI_MODEL", - os.getenv("AI_MODEL", os.getenv("PR_AF_MODEL", "openrouter/moonshotai/kimi-k2.5")), + os.getenv("AI_MODEL", os.getenv("PR_AF_MODEL", "openrouter/moonshotai/kimi-k2.6")), ) return { "budget": os.getenv("PR_AF_MODEL_BUDGET", ai_model), @@ -337,12 +337,12 @@ class AIIntegrationConfig(BaseModel): default_factory=lambda: os.getenv("PR_AF_PROVIDER", os.getenv("HARNESS_PROVIDER", "opencode")) ) harness_model: str = Field( - default_factory=lambda: os.getenv("PR_AF_MODEL", os.getenv("HARNESS_MODEL", "openrouter/moonshotai/kimi-k2.5")) + default_factory=lambda: os.getenv("PR_AF_MODEL", os.getenv("HARNESS_MODEL", "openrouter/moonshotai/kimi-k2.6")) ) ai_model: str = Field( default_factory=lambda: os.getenv( "PR_AF_AI_MODEL", - os.getenv("AI_MODEL", os.getenv("PR_AF_MODEL", "openrouter/moonshotai/kimi-k2.5")), + os.getenv("AI_MODEL", os.getenv("PR_AF_MODEL", "openrouter/moonshotai/kimi-k2.6")), ) ) max_turns: int = Field(default_factory=lambda: int(os.getenv("PR_AF_MAX_TURNS", "50"))) diff --git a/src/pr_af/reasoners/harnesses.py b/src/pr_af/reasoners/harnesses.py index d7229b6..fc7f36e 100644 --- a/src/pr_af/reasoners/harnesses.py +++ b/src/pr_af/reasoners/harnesses.py @@ -581,16 +581,13 @@ async def meta_semantic( f"Are there silent swallows or unhandled paths?\n" f"- **Data flow**: Does data pass through the same transformations? Are there type " f"coercions, format changes, or encoding differences?\n\n" - f"## Investigation Protocol\n\n" - f"You have full access to the repository. The context below gives you a starting " - f"point — PR summary, anatomy, and diff patches.\n\n" - f"- START by reading the context to understand WHAT changed.\n" - f"- THEN browse the actual source files to understand HOW the changed code fits into " - f"the broader codebase.\n" - f"- Read the changed functions. Then find their callers. Trace how data flows through " - f"them. Check what error paths exist.\n" - f"- ADAPT your investigation based on what you discover — if you find a concerning " - f"pattern, dig deeper in adjacent files and call paths.\n\n" + f"## Working with Context\n\n" + f"The diff patches plus the intake/anatomy summary below are usually enough to draft " + f"solid semantic dimensions — they contain the actual code that changed plus the " + f"structural analysis. Repository access is available if you need it (cwd is the repo), " + f"but use it sparingly: dig in only when the diff genuinely doesn't show what you need " + f"(e.g., to confirm an unchanged caller's signature or trace an error path that crosses " + f"the diff boundary).\n\n" f"## What NOT to Include\n\n" f"Do NOT generate dimensions about:\n" f"- Code style, naming, formatting (that's Systemic)\n" @@ -605,12 +602,16 @@ async def meta_semantic( f"CRITICAL: target_files MUST contain at least one actual file path from the repository " f"(e.g. 'sdk/python/agentfield/agent.py'). Dimensions with empty target_files are " f"discarded — the reviewer cannot review without knowing which files to examine.\n" - f"The review_prompt must include specific file paths and line ranges discovered during " - f"your repository investigation, plus the exact verification steps the reviewer should run.\n\n" - f"## Quality Gate\n\n" - f"Do NOT generate dimensions based solely on diff text. Every dimension must be informed " - f"by what you discovered in the actual codebase. If your rationale says 'visible in the " - f"diff' or 'based on the patches', you have not investigated enough.\n\n" + f"The review_prompt must include specific file paths and line ranges (from the diff " + f"or, if you needed to look further, from the repo), plus the exact verification " + f"steps the reviewer should run.\n\n" + f"## Quality Bar\n\n" + f"Each dimension must be SPECIFIC: named files, function names, line ranges where " + f"applicable. The diff patches contain real file paths and line numbers — use them. " + f"A dimension grounded in the diff is good; a vague dimension is not. Reach for the " + f"repository only when the diff is genuinely insufficient to name what to review " + f"(e.g., to find an unchanged caller's signature). Each tool call is real wall-clock " + f"time — extra exploration rarely improves the dimensions you'd produce.\n\n" f"Depth '{depth}' means: quick=1-2 dimensions, standard=2-3, deep=3-5\n" f"If the PR has no semantic risk, return ZERO dimensions. Do not pad.\n\n" f"Also provide a rationale explaining your dimension choices and a confidence " @@ -671,17 +672,13 @@ async def meta_mechanical( f"dependencies? Are optional dependencies guarded?\n" f"- **Runtime mechanics**: Will this code actually execute without AttributeError, " f"TypeError, KeyError, ImportError? Trace the exact runtime behavior.\n\n" - f"## Investigation Protocol\n\n" - f"You have full access to the repository. The context below gives you a starting " - f"point — PR summary, anatomy, and diff patches.\n\n" - f"- START by reading the context to understand WHAT changed.\n" - f"- THEN browse the actual source files to understand HOW the changed code fits into " - f"the broader codebase.\n" - f"- Read the actual function signatures that changed. Then search for all callers of " - f"those functions. Check whether callers pass the right arguments and whether import " - f"chains still resolve correctly.\n" - f"- ADAPT your investigation based on what you discover — if you find one caller or " - f"dependency break, keep tracing until you understand blast radius.\n\n" + f"## Working with Context\n\n" + f"The diff patches plus the intake/anatomy summary below are usually enough to draft " + f"solid mechanical dimensions — they contain the changed signatures, decorator changes, " + f"and import edits. Repository access is available if you need it (cwd is the repo), " + f"but use it sparingly: dig in only when the diff genuinely doesn't show what you need " + f"(e.g., to find unchanged callers of a renamed function or confirm a base-class " + f"contract you don't have in the patch).\n\n" f"## What NOT to Include\n\n" f"Do NOT generate dimensions about:\n" f"- Whether the logic is correct (that's Semantic)\n" @@ -697,12 +694,16 @@ async def meta_mechanical( f"CRITICAL: target_files MUST contain at least one actual file path from the repository " f"(e.g. 'sdk/python/agentfield/agent.py'). Dimensions with empty target_files are " f"discarded — the reviewer cannot review without knowing which files to examine.\n" - f"The review_prompt must include specific file paths and line ranges discovered during " - f"your repository investigation, plus the exact call sites/import chains to verify.\n\n" - f"## Quality Gate\n\n" - f"Do NOT generate dimensions based solely on diff text. Every dimension must be informed " - f"by what you discovered in the actual codebase. If your rationale says 'visible in the " - f"diff' or 'based on the patches', you have not investigated enough.\n\n" + f"The review_prompt must include specific file paths and line ranges (from the diff " + f"or, if you needed to look further, from the repo), plus the exact call sites or " + f"import chains to verify.\n\n" + f"## Quality Bar\n\n" + f"Each dimension must be SPECIFIC: named files, function names, line ranges where " + f"applicable. The diff patches contain real file paths and line numbers — use them. " + f"A dimension grounded in the diff is good; a vague dimension is not. Reach for the " + f"repository only when the diff is genuinely insufficient to name what to review " + f"(e.g., to find an unchanged caller's signature). Each tool call is real wall-clock " + f"time — extra exploration rarely improves the dimensions you'd produce.\n\n" f"Depth '{depth}' means: quick=1-2 dimensions, standard=2-3, deep=3-5\n" f"If the PR has no mechanical risk, return ZERO dimensions. Do not pad.\n\n" f"Also provide a rationale explaining your dimension choices and a confidence " @@ -763,16 +764,13 @@ async def meta_systemic( f"alternatives? Is the dependency well-maintained?\n" f"- **Migration completeness**: If this is part of a larger migration, is it " f"complete or does it leave the codebase in a mixed state?\n\n" - f"## Investigation Protocol\n\n" - f"You have full access to the repository. The context below gives you a starting " - f"point — PR summary, anatomy, and diff patches.\n\n" - f"- START by reading the context to understand WHAT changed.\n" - f"- THEN browse the actual source files to understand HOW the changed code fits into " - f"the broader codebase.\n" - f"- Browse similar files in the same directories to understand existing patterns and " - f"compare the changed code against those patterns.\n" - f"- ADAPT your investigation based on what you discover — if the change deviates from " - f"an established architecture pattern, trace where else that pattern is enforced.\n\n" + f"## Working with Context\n\n" + f"The diff patches plus the intake/anatomy summary below are usually enough to draft " + f"solid systemic dimensions — pattern deviation, complexity changes, missing tests, " + f"and migration completeness are mostly visible from the diff itself. Repository " + f"access is available if you need it (cwd is the repo), but use it sparingly: dig in " + f"only to compare the change against ONE or TWO sibling files when pattern fit is the " + f"whole question, not as a default mode of operation.\n\n" f"## What NOT to Include\n\n" f"Do NOT generate dimensions about:\n" f"- Whether the logic produces correct results (that's Semantic)\n" @@ -788,12 +786,16 @@ async def meta_systemic( f"CRITICAL: target_files MUST contain at least one actual file path from the repository " f"(e.g. 'sdk/python/agentfield/agent.py'). Dimensions with empty target_files are " f"discarded — the reviewer cannot review without knowing which files to examine.\n" - f"The review_prompt must include specific file paths and line ranges discovered during " - f"your repository investigation, plus the pattern comparisons the reviewer should validate.\n\n" - f"## Quality Gate\n\n" - f"Do NOT generate dimensions based solely on diff text. Every dimension must be informed " - f"by what you discovered in the actual codebase. If your rationale says 'visible in the " - f"diff' or 'based on the patches', you have not investigated enough.\n\n" + f"The review_prompt must include specific file paths and line ranges (from the diff " + f"or, if you needed to look further, from the repo), plus the pattern comparisons " + f"the reviewer should validate.\n\n" + f"## Quality Bar\n\n" + f"Each dimension must be SPECIFIC: named files, function names, line ranges where " + f"applicable. The diff patches contain real file paths and line numbers — use them. " + f"A dimension grounded in the diff is good; a vague dimension is not. Reach for the " + f"repository only when the diff is genuinely insufficient to name what to review " + f"(e.g., to find an unchanged caller's signature). Each tool call is real wall-clock " + f"time — extra exploration rarely improves the dimensions you'd produce.\n\n" f"Depth '{depth}' means: quick=0-1 dimensions, standard=1-2, deep=2-3\n" f"Systemic concerns are LOWER priority than Semantic and Mechanical. " f"If the PR is a focused bugfix with no architectural impact, return ZERO dimensions.\n\n"