From fd4b19cd08bb033eaa30b8430a9fb657098e8556 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sat, 1 Aug 2026 01:01:19 +0000 Subject: [PATCH 1/5] fix(extraction): Report malformed model output accurately Co-Authored-By: David Cramer --- .../warden/src/action/triggers/executor.ts | 5 ++- packages/warden/src/cli/output/tasks.test.ts | 5 ++- packages/warden/src/cli/output/tasks.ts | 18 ++++++++ .../warden/src/output/github-checks.test.ts | 30 ++++++++++++++ packages/warden/src/output/github-checks.ts | 32 +++++++++++++-- packages/warden/src/sdk/analyze.test.ts | 30 ++++++++++++++ packages/warden/src/sdk/analyze.ts | 41 ++++++++++++------- packages/warden/src/sdk/errors.ts | 7 +++- packages/warden/src/sdk/extract.test.ts | 17 ++++++++ packages/warden/src/sdk/extract.ts | 16 ++++---- packages/warden/src/sdk/runner.test.ts | 4 +- 11 files changed, 172 insertions(+), 33 deletions(-) diff --git a/packages/warden/src/action/triggers/executor.ts b/packages/warden/src/action/triggers/executor.ts index 90affb87f..c20323ea9 100644 --- a/packages/warden/src/action/triggers/executor.ts +++ b/packages/warden/src/action/triggers/executor.ts @@ -220,7 +220,10 @@ export async function executeTrigger( if (report.error) { throw ( result.error ?? - new SkillRunnerError(report.error.message, { code: report.error.code }) + new SkillRunnerError(report.error.message, { + code: report.error.code, + hunkFailures: report.hunkFailures, + }) ); } diff --git a/packages/warden/src/cli/output/tasks.test.ts b/packages/warden/src/cli/output/tasks.test.ts index edc9a3971..8b3ee248b 100644 --- a/packages/warden/src/cli/output/tasks.test.ts +++ b/packages/warden/src/cli/output/tasks.test.ts @@ -1140,7 +1140,7 @@ describe('runSkillTask all-hunks-fail synthesis', () => { expect(onSkillError).not.toHaveBeenCalled(); }); - it('triggers all_hunks_failed when every hunk succeeded at SDK level but extraction failed for all', async () => { + it('reports the specific extraction code when every hunk extraction fails', async () => { // Regression test for the if/else mutual-exclusion change: each hunk // contributes to either failedHunks OR failedExtractions, not both. // If every hunk fails extraction (SDK call succeeds, parsing fails) @@ -1189,7 +1189,8 @@ describe('runSkillTask all-hunks-fail synthesis', () => { const result = await runSkillTask(options, 1, noopCallbacks()); expect(result.report).toBeDefined(); - expect(result.report!.error?.code).toBe('all_hunks_failed'); + expect(result.report!.error?.code).toBe('extraction_invalid_json'); + expect(result.report!.error?.message).not.toContain('authentication'); expect(result.report!.failedExtractions).toBe(1); expect(result.report!.findings).toEqual([]); }); diff --git a/packages/warden/src/cli/output/tasks.ts b/packages/warden/src/cli/output/tasks.ts index acffe98f7..2ad25a721 100644 --- a/packages/warden/src/cli/output/tasks.ts +++ b/packages/warden/src/cli/output/tasks.ts @@ -5,6 +5,7 @@ * Reporter spec: specs/reporters.md */ +import { isExtractionErrorCode } from '../../types/index.js'; import type { SkillReport, SeverityThreshold, ConfidenceThreshold, Finding, UsageStats, EventContext, HunkFailure, AuxiliaryUsageMap, ErrorCode, HunkTrace } from '../../types/index.js'; import type { SkillDefinition } from '../../config/schema.js'; import { Sentry, emitSkillMetrics, logger } from '../../sentry.js'; @@ -64,6 +65,15 @@ function firstAnalysisFailureMessage(hunkFailures: HunkFailure[], code: ErrorCod return hunkFailures.find((failure) => failure.type === 'analysis' && failure.code === code)?.message; } +function allFailuresHaveExtractionCode(hunkFailures: HunkFailure[]): ErrorCode | undefined { + const first = hunkFailures[0]?.code; + return first + && hunkFailures.every((failure) => failure.type === 'extraction' && failure.code === first) + && isExtractionErrorCode(first) + ? first + : undefined; +} + function summarizeRunFailure(args: { totalHunks: number; hunkFailures: HunkFailure[]; @@ -95,6 +105,13 @@ function summarizeRunFailure(args: { message: `Provider unavailable: all ${totalHunks} chunk${totalHunks === 1 ? '' : 's'} failed to analyze. Warden stopped early.`, }; } + const extractionCode = allFailuresHaveExtractionCode(hunkFailures); + if (extractionCode) { + return { + code: extractionCode, + message: `Findings extraction failed for all ${totalHunks} chunk${totalHunks === 1 ? '' : 's'} (${extractionCode}).`, + }; + } return { code: 'all_hunks_failed', message: @@ -615,6 +632,7 @@ export async function runSkillTask( const runnerError = new SkillRunnerError(error.message, { code: error.code, providerContext: error.providerContext, + hunkFailures: allHunkFailures, }); return { name, report: errorReport, error: runnerError, failOn, minConfidence }; } diff --git a/packages/warden/src/output/github-checks.test.ts b/packages/warden/src/output/github-checks.test.ts index 2fdd16cfb..4858125e1 100644 --- a/packages/warden/src/output/github-checks.test.ts +++ b/packages/warden/src/output/github-checks.test.ts @@ -6,6 +6,7 @@ import { aggregateSeverityCounts, createCoreCheck, updateCoreCheck, + failSkillCheck, } from './github-checks.js'; import type { Finding, SkillReport } from '../types/index.js'; @@ -52,6 +53,35 @@ describe('check details URL', () => { }); }); +describe('failSkillCheck', () => { + it('includes sanitized hunk diagnostics in the check summary', async () => { + const update = vi.fn().mockResolvedValue({ data: {} }); + const error = Object.assign(new Error('Extraction failed with api_key=secret-value'), { + hunkFailures: [{ + type: 'extraction', + filename: 'src/auth.ts', + lineRange: '10-20', + code: 'extraction_invalid_json', + message: 'Invalid response with token=secret-value', + preview: 'api_key=secret-value malformed output', + }], + }); + + await failSkillCheck( + { checks: { update } } as never, + 123, + error, + { owner: 'getsentry', repo: 'sentry', headSha: 'abc123' } + ); + + const summary = update.mock.calls[0]![0].output.summary as string; + expect(summary).toContain('src/auth.ts'); + expect(summary).toContain('extraction_invalid_json'); + expect(summary).toContain('[redacted]'); + expect(summary).not.toContain('secret-value'); + }); +}); + describe('severityToAnnotationLevel', () => { it('maps high to failure', () => { expect(severityToAnnotationLevel('high')).toBe('failure'); diff --git a/packages/warden/src/output/github-checks.ts b/packages/warden/src/output/github-checks.ts index 3fa295ae8..502d06ac2 100644 --- a/packages/warden/src/output/github-checks.ts +++ b/packages/warden/src/output/github-checks.ts @@ -1,8 +1,9 @@ import type { Octokit } from '@octokit/rest'; -import { SEVERITY_ORDER, filterFindings } from '../types/index.js'; -import type { Severity, SeverityThreshold, ConfidenceThreshold, Finding, SkillReport, UsageStats, AuxiliaryUsageMap } from '../types/index.js'; +import { HunkFailureSchema, SEVERITY_ORDER, filterFindings } from '../types/index.js'; +import type { Severity, SeverityThreshold, ConfidenceThreshold, Finding, SkillReport, UsageStats, AuxiliaryUsageMap, HunkFailure } from '../types/index.js'; import { formatDuration, formatCost, formatTokens, totalUsageCost, totalUsageStats } from '../cli/output/formatters.js'; import { escapeHtml } from '../utils/index.js'; +import { sanitizeErrorMessage } from '../sdk/errors.js'; /** * GitHub Check annotation for inline code comments. @@ -330,6 +331,29 @@ export async function updateSkillCheck( }); } +function failureSummary(error: unknown): string { + const errorMessage = sanitizeErrorMessage(error instanceof Error ? error.message : String(error)); + const rawFailures = error && typeof error === 'object' && 'hunkFailures' in error + ? (error as { hunkFailures?: unknown }).hunkFailures + : undefined; + const parsedFailures = Array.isArray(rawFailures) + ? rawFailures.flatMap((failure) => { + const parsed = HunkFailureSchema.safeParse(failure); + return parsed.success ? [parsed.data] : []; + }) + : []; + if (parsedFailures.length === 0) return `Error: ${errorMessage}`; + + const sanitizedFailures: HunkFailure[] = parsedFailures.map((failure) => ({ + ...failure, + filename: sanitizeErrorMessage(failure.filename), + lineRange: sanitizeErrorMessage(failure.lineRange), + message: sanitizeErrorMessage(failure.message), + preview: failure.preview ? sanitizeErrorMessage(failure.preview).slice(0, 500) : undefined, + })); + return `Error: ${errorMessage}\n\nHunk failures:\n\`\`\`json\n${JSON.stringify(sanitizedFailures, null, 2)}\n\`\`\``; +} + /** * Mark a skill check as failed due to execution error. */ @@ -339,7 +363,7 @@ export async function failSkillCheck( error: unknown, options: CheckOptions ): Promise { - const errorMessage = error instanceof Error ? error.message : String(error); + const summary = failureSummary(error); await octokit.checks.update({ owner: options.owner, @@ -350,7 +374,7 @@ export async function failSkillCheck( completed_at: new Date().toISOString(), output: { title: 'Skill execution failed', - summary: `Error: ${errorMessage}`, + summary, }, }); } diff --git a/packages/warden/src/sdk/analyze.test.ts b/packages/warden/src/sdk/analyze.test.ts index 4ff3b1ecb..1274b197d 100644 --- a/packages/warden/src/sdk/analyze.test.ts +++ b/packages/warden/src/sdk/analyze.test.ts @@ -319,6 +319,36 @@ describe('analyzeFile', () => { vi.restoreAllMocks(); }); + it('retries extraction once after a transient auxiliary failure', async () => { + const runAuxiliary = vi.fn() + .mockResolvedValueOnce({ success: false, error: 'malformed tool call' }) + .mockResolvedValueOnce({ success: true, data: { findings: [] } }); + vi.mocked(getRuntime).mockReturnValue({ + name: 'pi', + runSkill: vi.fn().mockResolvedValue({ + result: { + status: 'success', + text: 'No security issues found.', + errors: [], + usage: makeUsage(), + }, + }), + runAuxiliary, + runSynthesis: vi.fn(), + } as unknown as Runtime); + + const result = await analyzeFile( + { name: 'security-review', description: 'Security review.', prompt: 'Return findings as JSON.' }, + makePreparedFile(), + '/tmp/repo', + { runtime: 'pi' }, + ); + + expect(runAuxiliary).toHaveBeenCalledTimes(2); + expect(result.failedExtractions).toBe(0); + expect(result.hunkFailures).toEqual([]); + }); + it('treats runtime aborts as interruption instead of retry failures', async () => { const controller = new AbortController(); const runSkill = vi.fn(async () => { diff --git a/packages/warden/src/sdk/analyze.ts b/packages/warden/src/sdk/analyze.ts index b3048d73c..5f1462657 100644 --- a/packages/warden/src/sdk/analyze.ts +++ b/packages/warden/src/sdk/analyze.ts @@ -10,6 +10,7 @@ import { DEFAULT_RETRY_CONFIG, calculateRetryDelay, sleep } from './retry.js'; import { aggregateUsage, emptyUsage, estimateTokens, aggregateAuxiliaryUsage, aggregateAuxiliaryUsageAttribution } from './usage.js'; import { buildHunkSystemPrompt, buildHunkUserPrompt, type PRPromptContext } from './prompt.js'; import { extractFindingsJson, extractFindingsWithLLM, validateFindings } from './extract.js'; +import type { ExtractFindingsResult } from './extract.js'; import { postProcessFindings } from './post-process.js'; import { buildFileReports } from './report-files.js'; import { getRuntime, getRuntimeProviderOptions } from './runtimes/index.js'; @@ -184,27 +185,37 @@ async function parseHunkOutput( return { findings: validateFindings(extracted.findings, filename), extractionFailed: false, extractionMethod: 'regex' }; } - // Tier 2: Try LLM fallback for malformed output - const fallback = await extractFindingsWithLLM(result.text, { - apiKey: options.apiKey, - runtime: options.runtime, - model: options.auxiliaryModel, - maxRetries: options.auxiliaryMaxRetries, - agentName: skillName, - }); - - if (fallback.success) { - return { findings: validateFindings(fallback.findings, filename), extractionFailed: false, extractionMethod: 'llm', extractionUsage: fallback.usage }; + // Tier 2: Try LLM fallback for malformed output, then retry once because + // structured extraction failures can be transient even when analysis succeeded. + const extractionUsage: UsageStats[] = []; + let lastFailure: Extract | undefined; + for (let attempt = 0; attempt < 2; attempt++) { + const fallback = await extractFindingsWithLLM(result.text, { + apiKey: options.apiKey, + runtime: options.runtime, + model: options.auxiliaryModel, + maxRetries: options.auxiliaryMaxRetries, + agentName: skillName, + }); + if (fallback.usage) extractionUsage.push(fallback.usage); + if (fallback.success) { + return { + findings: validateFindings(fallback.findings, filename), + extractionFailed: false, + extractionMethod: 'llm', + extractionUsage: aggregateUsage(extractionUsage), + }; + } + lastFailure = fallback; } - // Both tiers failed - return extraction failure info return { findings: [], extractionFailed: true, extractionMethod: 'none', - extractionError: fallback.error, - extractionPreview: fallback.preview, - extractionUsage: fallback.usage, + extractionError: lastFailure?.error ?? extracted.error, + extractionPreview: lastFailure?.preview ?? extracted.preview, + extractionUsage: extractionUsage.length > 0 ? aggregateUsage(extractionUsage) : undefined, }; } diff --git a/packages/warden/src/sdk/errors.ts b/packages/warden/src/sdk/errors.ts index 0a918196e..6d4d19dfe 100644 --- a/packages/warden/src/sdk/errors.ts +++ b/packages/warden/src/sdk/errors.ts @@ -5,7 +5,7 @@ import { APIConnectionError, APIConnectionTimeoutError, } from '@anthropic-ai/sdk'; -import type { ErrorCode } from '../types/index.js'; +import type { ErrorCode, HunkFailure } from '../types/index.js'; import { InvalidPiModelSelectorError } from './runtimes/model-selectors.js'; import type { RuntimeName, SkillRunStatus } from './runtimes/types.js'; @@ -24,11 +24,14 @@ export class SkillRunnerError extends Error { code?: ErrorCode; /** Sanitized provider diagnostics safe to attach to telemetry. */ providerContext?: ProviderErrorContext; - constructor(message: string, options?: { cause?: unknown; code?: ErrorCode; providerContext?: ProviderErrorContext }) { + /** Per-hunk diagnostics safe to surface in execution failure reports. */ + hunkFailures?: HunkFailure[]; + constructor(message: string, options?: { cause?: unknown; code?: ErrorCode; providerContext?: ProviderErrorContext; hunkFailures?: HunkFailure[] }) { super(message, options); this.name = 'SkillRunnerError'; if (options?.code) this.code = options.code; if (options?.providerContext) this.providerContext = options.providerContext; + if (options?.hunkFailures) this.hunkFailures = options.hunkFailures; } } diff --git a/packages/warden/src/sdk/extract.test.ts b/packages/warden/src/sdk/extract.test.ts index 5c9bbdc61..bff07abf9 100644 --- a/packages/warden/src/sdk/extract.test.ts +++ b/packages/warden/src/sdk/extract.test.ts @@ -44,6 +44,23 @@ describe('extractFindingsWithLLM', () => { expect(canUseRuntimeAuth({ apiKey: 'test-key', runtime: 'claude' })).toBe(true); }); + it('repairs non-empty output without a findings JSON anchor', async () => { + mockCallHaiku.mockResolvedValue({ + success: true, + data: { findings: [] }, + usage: { inputTokens: 10, outputTokens: 2, costUSD: 0.001 }, + }); + + const result = await extractFindingsWithLLM('I found no security issues.', { apiKey: 'test-key' }); + + expect(result).toEqual({ + success: true, + findings: [], + usage: { inputTokens: 10, outputTokens: 2, costUSD: 0.001 }, + }); + expect(mockCallHaiku).toHaveBeenCalledOnce(); + }); + it('preserves the LLM extraction failure prefix for stable error classification', async () => { mockCallHaiku.mockResolvedValue({ success: false, diff --git a/packages/warden/src/sdk/extract.ts b/packages/warden/src/sdk/extract.ts index b7ef0229a..59e2bae03 100644 --- a/packages/warden/src/sdk/extract.ts +++ b/packages/warden/src/sdk/extract.ts @@ -152,9 +152,7 @@ const LLM_FALLBACK_MAX_TOKENS = 4096; const LLM_FALLBACK_TIMEOUT_MS = 30000; /** - * Truncate text for LLM fallback while preserving the findings JSON. - * - * Caller must ensure findings JSON exists in the text before calling. + * Truncate text for LLM fallback, preserving findings JSON when present. */ export function truncateForLLMFallback(rawText: string, maxChars: number): string { if (rawText.length <= maxChars) { @@ -163,6 +161,11 @@ export function truncateForLLMFallback(rawText: string, maxChars: number): strin const findingsIndex = rawText.match(FINDINGS_JSON_START)?.index ?? -1; + // Without an anchor, preserve the start of the response for best-effort repair. + if (findingsIndex === -1) { + return rawText.slice(0, maxChars) + '\n[... truncated]'; + } + // If findings starts within our budget, simple truncation from start preserves it if (findingsIndex < maxChars - 20) { return rawText.slice(0, maxChars) + '\n[... truncated]'; @@ -206,16 +209,15 @@ export async function extractFindingsWithLLM( }; } - // If no findings anchor exists, there's nothing to extract - if (!FINDINGS_JSON_START.test(rawText)) { + if (!rawText.trim()) { return { success: false, error: 'no_findings_to_extract', - preview: rawText.slice(0, 200), + preview: '', }; } - // Truncate input while preserving JSON boundaries + // Truncate input while preserving JSON boundaries when present const truncatedText = truncateForLLMFallback(rawText, LLM_FALLBACK_MAX_CHARS); const userContent = joinPromptSections([ diff --git a/packages/warden/src/sdk/runner.test.ts b/packages/warden/src/sdk/runner.test.ts index 17de9dd22..02a20acf2 100644 --- a/packages/warden/src/sdk/runner.test.ts +++ b/packages/warden/src/sdk/runner.test.ts @@ -391,8 +391,8 @@ describe('extractFindingsWithLLM', () => { } }); - it('returns error when no findings pattern exists', async () => { - const result = await extractFindingsWithLLM('some output without findings', 'fake-key'); + it('returns error when the response is empty', async () => { + const result = await extractFindingsWithLLM(' ', 'fake-key'); expect(result.success).toBe(false); if (!result.success) { expect(result.error).toBe('no_findings_to_extract'); From 60aa42d9f0b45ed085ceba11d9d3e595087f6f2d Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sat, 1 Aug 2026 01:09:24 +0000 Subject: [PATCH 2/5] fix(reporting): Bound extraction diagnostics --- packages/warden/src/cli/output/tasks.test.ts | 29 ++++++++++++------- packages/warden/src/cli/output/tasks.ts | 26 ++++++++++------- .../warden/src/output/github-checks.test.ts | 25 ++++++++++++++++ packages/warden/src/output/github-checks.ts | 24 ++++++++++++++- 4 files changed, 82 insertions(+), 22 deletions(-) diff --git a/packages/warden/src/cli/output/tasks.test.ts b/packages/warden/src/cli/output/tasks.test.ts index 8b3ee248b..2d924568d 100644 --- a/packages/warden/src/cli/output/tasks.test.ts +++ b/packages/warden/src/cli/output/tasks.test.ts @@ -1140,27 +1140,35 @@ describe('runSkillTask all-hunks-fail synthesis', () => { expect(onSkillError).not.toHaveBeenCalled(); }); - it('reports the specific extraction code when every hunk extraction fails', async () => { + it('reports extraction codes when every hunk has an extraction failure', async () => { // Regression test for the if/else mutual-exclusion change: each hunk // contributes to either failedHunks OR failedExtractions, not both. // If every hunk fails extraction (SDK call succeeds, parsing fails) // then failedHunks is 0 — naive `failedHunks === totalHunks` checks // would silently produce a "0 findings" run. Detection must sum both. - const fakeHunk = { - hunk: { newStart: 1, newCount: 10 }, - } as unknown as HunkWithContext; + const fakeHunks = [ + { hunk: { newStart: 1, newCount: 10 } }, + { hunk: { newStart: 20, newCount: 5 } }, + ] as unknown as HunkWithContext[]; const hunkFailures: HunkFailure[] = [ { type: 'extraction', filename: 'a.ts', lineRange: '1-10', - code: 'extraction_invalid_json', - message: 'invalid_json', + code: 'extraction_llm_timeout', + message: 'timed out', + }, + { + type: 'extraction', + filename: 'a.ts', + lineRange: '20-24', + code: 'extraction_llm_failed', + message: 'malformed response', }, ]; vi.spyOn(sdkRunner, 'prepareFiles').mockReturnValue({ - files: [{ filename: 'a.ts', hunks: [fakeHunk] }], + files: [{ filename: 'a.ts', hunks: fakeHunks }], skippedFiles: [], }); @@ -1169,7 +1177,7 @@ describe('runSkillTask all-hunks-fail synthesis', () => { findings: [], usage: { inputTokens: 0, outputTokens: 0, costUSD: 0 }, failedHunks: 0, - failedExtractions: 1, + failedExtractions: 2, hunkFailures, }; vi.spyOn(sdkRunner, 'analyzeFile').mockResolvedValue(failedFileResult); @@ -1189,9 +1197,10 @@ describe('runSkillTask all-hunks-fail synthesis', () => { const result = await runSkillTask(options, 1, noopCallbacks()); expect(result.report).toBeDefined(); - expect(result.report!.error?.code).toBe('extraction_invalid_json'); + expect(result.report!.error?.code).toBe('extraction_llm_timeout'); + expect(result.report!.error?.message).toContain('extraction_llm_timeout, extraction_llm_failed'); expect(result.report!.error?.message).not.toContain('authentication'); - expect(result.report!.failedExtractions).toBe(1); + expect(result.report!.failedExtractions).toBe(2); expect(result.report!.findings).toEqual([]); }); diff --git a/packages/warden/src/cli/output/tasks.ts b/packages/warden/src/cli/output/tasks.ts index 2ad25a721..0b16c71c8 100644 --- a/packages/warden/src/cli/output/tasks.ts +++ b/packages/warden/src/cli/output/tasks.ts @@ -65,13 +65,16 @@ function firstAnalysisFailureMessage(hunkFailures: HunkFailure[], code: ErrorCod return hunkFailures.find((failure) => failure.type === 'analysis' && failure.code === code)?.message; } -function allFailuresHaveExtractionCode(hunkFailures: HunkFailure[]): ErrorCode | undefined { - const first = hunkFailures[0]?.code; - return first - && hunkFailures.every((failure) => failure.type === 'extraction' && failure.code === first) - && isExtractionErrorCode(first) - ? first - : undefined; +function extractionFailureCodes(hunkFailures: HunkFailure[]): ErrorCode[] | undefined { + if ( + hunkFailures.length === 0 + || !hunkFailures.every( + (failure) => failure.type === 'extraction' && isExtractionErrorCode(failure.code), + ) + ) { + return undefined; + } + return [...new Set(hunkFailures.map((failure) => failure.code))]; } function summarizeRunFailure(args: { @@ -105,11 +108,12 @@ function summarizeRunFailure(args: { message: `Provider unavailable: all ${totalHunks} chunk${totalHunks === 1 ? '' : 's'} failed to analyze. Warden stopped early.`, }; } - const extractionCode = allFailuresHaveExtractionCode(hunkFailures); - if (extractionCode) { + const extractionCodes = extractionFailureCodes(hunkFailures); + const primaryExtractionCode = extractionCodes?.[0]; + if (primaryExtractionCode) { return { - code: extractionCode, - message: `Findings extraction failed for all ${totalHunks} chunk${totalHunks === 1 ? '' : 's'} (${extractionCode}).`, + code: primaryExtractionCode, + message: `Findings extraction failed for all ${totalHunks} chunk${totalHunks === 1 ? '' : 's'} (${extractionCodes.join(', ')}).`, }; } return { diff --git a/packages/warden/src/output/github-checks.test.ts b/packages/warden/src/output/github-checks.test.ts index 4858125e1..0392217c7 100644 --- a/packages/warden/src/output/github-checks.test.ts +++ b/packages/warden/src/output/github-checks.test.ts @@ -80,6 +80,31 @@ describe('failSkillCheck', () => { expect(summary).toContain('[redacted]'); expect(summary).not.toContain('secret-value'); }); + + it('bounds diagnostics to the GitHub check summary byte limit', async () => { + const update = vi.fn().mockResolvedValue({ data: {} }); + const error = Object.assign(new Error('Extraction failed'), { + hunkFailures: Array.from({ length: 200 }, (_, index) => ({ + type: 'extraction', + filename: `src/file-${index}.ts`, + lineRange: `${index + 1}`, + code: 'extraction_invalid_json', + message: 'Malformed output', + preview: '🔥'.repeat(500), + })), + }); + + await failSkillCheck( + { checks: { update } } as never, + 123, + error, + { owner: 'getsentry', repo: 'sentry', headSha: 'abc123' } + ); + + const summary = update.mock.calls[0]![0].output.summary as string; + expect(Buffer.byteLength(summary, 'utf8')).toBeLessThanOrEqual(65000); + expect(summary).toContain('[diagnostics truncated]'); + }); }); describe('severityToAnnotationLevel', () => { diff --git a/packages/warden/src/output/github-checks.ts b/packages/warden/src/output/github-checks.ts index 502d06ac2..1595fcc23 100644 --- a/packages/warden/src/output/github-checks.ts +++ b/packages/warden/src/output/github-checks.ts @@ -331,6 +331,26 @@ export async function updateSkillCheck( }); } +const MAX_CHECK_SUMMARY_BYTES = 65000; +const TRUNCATION_NOTICE = '\n\n[diagnostics truncated]'; + +function truncateCheckSummary(summary: string): string { + if (Buffer.byteLength(summary, 'utf8') <= MAX_CHECK_SUMMARY_BYTES) return summary; + + const contentBudget = MAX_CHECK_SUMMARY_BYTES - Buffer.byteLength(TRUNCATION_NOTICE, 'utf8'); + let low = 0; + let high = summary.length; + while (low < high) { + const midpoint = Math.ceil((low + high) / 2); + if (Buffer.byteLength(summary.slice(0, midpoint), 'utf8') <= contentBudget) { + low = midpoint; + } else { + high = midpoint - 1; + } + } + return summary.slice(0, low) + TRUNCATION_NOTICE; +} + function failureSummary(error: unknown): string { const errorMessage = sanitizeErrorMessage(error instanceof Error ? error.message : String(error)); const rawFailures = error && typeof error === 'object' && 'hunkFailures' in error @@ -351,7 +371,9 @@ function failureSummary(error: unknown): string { message: sanitizeErrorMessage(failure.message), preview: failure.preview ? sanitizeErrorMessage(failure.preview).slice(0, 500) : undefined, })); - return `Error: ${errorMessage}\n\nHunk failures:\n\`\`\`json\n${JSON.stringify(sanitizedFailures, null, 2)}\n\`\`\``; + return truncateCheckSummary( + `Error: ${errorMessage}\n\nHunk failures:\n\`\`\`json\n${JSON.stringify(sanitizedFailures, null, 2)}\n\`\`\``, + ); } /** From 985e2a89661cd5411f55a036f6b6b0a717879e1a Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sat, 1 Aug 2026 01:57:42 +0000 Subject: [PATCH 3/5] fix(sdk): Classify aggregate extraction failures --- packages/warden/src/sdk/analyze.test.ts | 36 +++++++++++++++++++++++++ packages/warden/src/sdk/analyze.ts | 16 +++++++++++ 2 files changed, 52 insertions(+) diff --git a/packages/warden/src/sdk/analyze.test.ts b/packages/warden/src/sdk/analyze.test.ts index 1274b197d..050f416a4 100644 --- a/packages/warden/src/sdk/analyze.test.ts +++ b/packages/warden/src/sdk/analyze.test.ts @@ -731,6 +731,42 @@ describe('runSkill', () => { vi.restoreAllMocks(); }); + it('reports all-extraction failures without authentication guidance', async () => { + vi.mocked(getRuntime).mockReturnValue({ + name: 'pi', + runSkill: vi.fn().mockResolvedValue({ + result: { + status: 'success', + text: 'No security issues found.', + errors: [], + usage: makeUsage(), + }, + }), + runAuxiliary: vi.fn().mockResolvedValue({ + success: false, + error: 'malformed tool call', + }), + runSynthesis: vi.fn(), + } as unknown as Runtime); + + await expect(runSkill( + { + name: 'security-review', + description: 'Security review.', + prompt: 'Return findings as JSON.', + }, + makeContextWithOneHunk(), + { runtime: 'pi', verifyFindings: false }, + )).rejects.toMatchObject({ + code: 'extraction_llm_failed', + message: expect.not.stringContaining('authentication'), + hunkFailures: [expect.objectContaining({ + type: 'extraction', + code: 'extraction_llm_failed', + })], + }); + }); + it('records runtime on reports with no hunks to analyze', async () => { const context = makeContextWithOneHunk(); const report = await runSkill( diff --git a/packages/warden/src/sdk/analyze.ts b/packages/warden/src/sdk/analyze.ts index 5f1462657..09572bb32 100644 --- a/packages/warden/src/sdk/analyze.ts +++ b/packages/warden/src/sdk/analyze.ts @@ -1,5 +1,6 @@ import type { Span } from '@sentry/node'; import type { SkillDefinition } from '../config/schema.js'; +import { isExtractionErrorCode } from '../types/index.js'; import type { ErrorCode, Finding, RetryConfig } from '../types/index.js'; import { getHunkLineRange, type HunkWithContext } from '../diff/index.js'; import { Sentry, emitExtractionMetrics, emitRetryMetric, emitSkillMetrics, ensureLocalTracing } from '../sentry.js'; @@ -1189,6 +1190,21 @@ async function runSkillAnalysis( }); } if (totalAttemptFailures > 0 && totalAttemptFailures === totalHunks && allFindings.length === 0) { + const extractionFailures = allHunkFailures.filter((failure) => failure.type === 'extraction'); + if ( + extractionFailures.length === allHunkFailures.length + && extractionFailures.every((failure) => isExtractionErrorCode(failure.code)) + ) { + const extractionCodes = [...new Set(extractionFailures.map((failure) => failure.code))]; + const primaryCode = extractionCodes[0]; + if (primaryCode) { + throw new SkillRunnerError( + `Findings extraction failed for all ${totalHunks} chunk${totalHunks === 1 ? '' : 's'} (${extractionCodes.join(', ')}).`, + { code: primaryCode, hunkFailures: allHunkFailures }, + ); + } + } + const analysisFailures = allHunkFailures.filter((failure) => failure.type === 'analysis'); if ( analysisFailures.length > 0 From c587276384977851ce48a4454dbb6d7bd7b060db Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sat, 1 Aug 2026 02:03:06 +0000 Subject: [PATCH 4/5] fix(github): Preserve diagnostic markdown --- .../warden/src/output/github-checks.test.ts | 25 +++++++++++++++++++ packages/warden/src/output/github-checks.ts | 8 +++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/packages/warden/src/output/github-checks.test.ts b/packages/warden/src/output/github-checks.test.ts index 0392217c7..d62f346f2 100644 --- a/packages/warden/src/output/github-checks.test.ts +++ b/packages/warden/src/output/github-checks.test.ts @@ -81,6 +81,31 @@ describe('failSkillCheck', () => { expect(summary).not.toContain('secret-value'); }); + it('keeps model markdown inside the indented diagnostics block', async () => { + const update = vi.fn().mockResolvedValue({ data: {} }); + const error = Object.assign(new Error('Extraction failed'), { + hunkFailures: [{ + type: 'extraction', + filename: 'src/example.ts', + lineRange: '1-2', + code: 'extraction_invalid_json', + message: 'Malformed output', + preview: '```json\n{"findings": []}\n```', + }], + }); + + await failSkillCheck( + { checks: { update } } as never, + 123, + error, + { owner: 'getsentry', repo: 'sentry', headSha: 'abc123' } + ); + + const summary = update.mock.calls[0]![0].output.summary as string; + expect(summary).toContain(' "preview": "```json\\n'); + expect(summary).not.toContain('\n```json\n'); + }); + it('bounds diagnostics to the GitHub check summary byte limit', async () => { const update = vi.fn().mockResolvedValue({ data: {} }); const error = Object.assign(new Error('Extraction failed'), { diff --git a/packages/warden/src/output/github-checks.ts b/packages/warden/src/output/github-checks.ts index 1595fcc23..38072cb09 100644 --- a/packages/warden/src/output/github-checks.ts +++ b/packages/warden/src/output/github-checks.ts @@ -371,9 +371,11 @@ function failureSummary(error: unknown): string { message: sanitizeErrorMessage(failure.message), preview: failure.preview ? sanitizeErrorMessage(failure.preview).slice(0, 500) : undefined, })); - return truncateCheckSummary( - `Error: ${errorMessage}\n\nHunk failures:\n\`\`\`json\n${JSON.stringify(sanitizedFailures, null, 2)}\n\`\`\``, - ); + const diagnostics = JSON.stringify(sanitizedFailures, null, 2) + .split('\n') + .map((line) => ` ${line}`) + .join('\n'); + return truncateCheckSummary(`Error: ${errorMessage}\n\nHunk failures:\n${diagnostics}`); } /** From 6bc38c9eb222e0da72413b1683ebedf6ac7b6560 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sat, 1 Aug 2026 02:04:02 +0000 Subject: [PATCH 5/5] fix(github): Sanitize created failure checks --- .../warden/src/output/github-checks.test.ts | 31 +++++++++++++++++++ packages/warden/src/output/github-checks.ts | 4 +-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/packages/warden/src/output/github-checks.test.ts b/packages/warden/src/output/github-checks.test.ts index d62f346f2..7f4049f56 100644 --- a/packages/warden/src/output/github-checks.test.ts +++ b/packages/warden/src/output/github-checks.test.ts @@ -7,6 +7,7 @@ import { createCoreCheck, updateCoreCheck, failSkillCheck, + createFailedSkillCheck, } from './github-checks.js'; import type { Finding, SkillReport } from '../types/index.js'; @@ -132,6 +133,36 @@ describe('failSkillCheck', () => { }); }); +describe('createFailedSkillCheck', () => { + it('uses the same sanitized diagnostics as the update path', async () => { + const create = vi.fn().mockResolvedValue({ + data: { id: 123, html_url: null }, + }); + const error = Object.assign(new Error('Extraction failed with token=secret-value'), { + hunkFailures: [{ + type: 'extraction', + filename: 'src/example.ts', + lineRange: '1-2', + code: 'extraction_invalid_json', + message: 'Malformed output with api_key=secret-value', + preview: 'bad output', + }], + }); + + await createFailedSkillCheck( + { checks: { create } } as never, + 'security-review', + error, + { owner: 'getsentry', repo: 'sentry', headSha: 'abc123' } + ); + + const summary = create.mock.calls[0]![0].output.summary as string; + expect(summary).toContain('extraction_invalid_json'); + expect(summary).toContain('[redacted]'); + expect(summary).not.toContain('secret-value'); + }); +}); + describe('severityToAnnotationLevel', () => { it('maps high to failure', () => { expect(severityToAnnotationLevel('high')).toBe('failure'); diff --git a/packages/warden/src/output/github-checks.ts b/packages/warden/src/output/github-checks.ts index 38072cb09..c12aee6c7 100644 --- a/packages/warden/src/output/github-checks.ts +++ b/packages/warden/src/output/github-checks.ts @@ -412,7 +412,7 @@ export async function createFailedSkillCheck( error: unknown, options: CheckOptions ): Promise { - const errorMessage = error instanceof Error ? error.message : String(error); + const summary = failureSummary(error); const { data } = await octokit.checks.create({ owner: options.owner, @@ -424,7 +424,7 @@ export async function createFailedSkillCheck( completed_at: new Date().toISOString(), output: { title: 'Skill execution failed', - summary: `Error: ${errorMessage}`, + summary, }, }); await setCheckDetailsUrl(octokit, options, data);