diff --git a/packages/warden/src/action/triggers/executor.ts b/packages/warden/src/action/triggers/executor.ts index 90affb87f..c20323ea9 100644 --- a/packages/warden/src/action/triggers/executor.ts +++ b/packages/warden/src/action/triggers/executor.ts @@ -220,7 +220,10 @@ export async function executeTrigger( if (report.error) { throw ( result.error ?? - new SkillRunnerError(report.error.message, { code: report.error.code }) + new SkillRunnerError(report.error.message, { + code: report.error.code, + hunkFailures: report.hunkFailures, + }) ); } diff --git a/packages/warden/src/cli/output/tasks.test.ts b/packages/warden/src/cli/output/tasks.test.ts index edc9a3971..2d924568d 100644 --- a/packages/warden/src/cli/output/tasks.test.ts +++ b/packages/warden/src/cli/output/tasks.test.ts @@ -1140,27 +1140,35 @@ describe('runSkillTask all-hunks-fail synthesis', () => { expect(onSkillError).not.toHaveBeenCalled(); }); - it('triggers all_hunks_failed when every hunk succeeded at SDK level but extraction failed for all', async () => { + it('reports extraction codes when every hunk has an extraction failure', async () => { // Regression test for the if/else mutual-exclusion change: each hunk // contributes to either failedHunks OR failedExtractions, not both. // If every hunk fails extraction (SDK call succeeds, parsing fails) // then failedHunks is 0 — naive `failedHunks === totalHunks` checks // would silently produce a "0 findings" run. Detection must sum both. - const fakeHunk = { - hunk: { newStart: 1, newCount: 10 }, - } as unknown as HunkWithContext; + const fakeHunks = [ + { hunk: { newStart: 1, newCount: 10 } }, + { hunk: { newStart: 20, newCount: 5 } }, + ] as unknown as HunkWithContext[]; const hunkFailures: HunkFailure[] = [ { type: 'extraction', filename: 'a.ts', lineRange: '1-10', - code: 'extraction_invalid_json', - message: 'invalid_json', + code: 'extraction_llm_timeout', + message: 'timed out', + }, + { + type: 'extraction', + filename: 'a.ts', + lineRange: '20-24', + code: 'extraction_llm_failed', + message: 'malformed response', }, ]; vi.spyOn(sdkRunner, 'prepareFiles').mockReturnValue({ - files: [{ filename: 'a.ts', hunks: [fakeHunk] }], + files: [{ filename: 'a.ts', hunks: fakeHunks }], skippedFiles: [], }); @@ -1169,7 +1177,7 @@ describe('runSkillTask all-hunks-fail synthesis', () => { findings: [], usage: { inputTokens: 0, outputTokens: 0, costUSD: 0 }, failedHunks: 0, - failedExtractions: 1, + failedExtractions: 2, hunkFailures, }; vi.spyOn(sdkRunner, 'analyzeFile').mockResolvedValue(failedFileResult); @@ -1189,8 +1197,10 @@ describe('runSkillTask all-hunks-fail synthesis', () => { const result = await runSkillTask(options, 1, noopCallbacks()); expect(result.report).toBeDefined(); - expect(result.report!.error?.code).toBe('all_hunks_failed'); - expect(result.report!.failedExtractions).toBe(1); + expect(result.report!.error?.code).toBe('extraction_llm_timeout'); + expect(result.report!.error?.message).toContain('extraction_llm_timeout, extraction_llm_failed'); + expect(result.report!.error?.message).not.toContain('authentication'); + expect(result.report!.failedExtractions).toBe(2); expect(result.report!.findings).toEqual([]); }); diff --git a/packages/warden/src/cli/output/tasks.ts b/packages/warden/src/cli/output/tasks.ts index acffe98f7..0b16c71c8 100644 --- a/packages/warden/src/cli/output/tasks.ts +++ b/packages/warden/src/cli/output/tasks.ts @@ -5,6 +5,7 @@ * Reporter spec: specs/reporters.md */ +import { isExtractionErrorCode } from '../../types/index.js'; import type { SkillReport, SeverityThreshold, ConfidenceThreshold, Finding, UsageStats, EventContext, HunkFailure, AuxiliaryUsageMap, ErrorCode, HunkTrace } from '../../types/index.js'; import type { SkillDefinition } from '../../config/schema.js'; import { Sentry, emitSkillMetrics, logger } from '../../sentry.js'; @@ -64,6 +65,18 @@ function firstAnalysisFailureMessage(hunkFailures: HunkFailure[], code: ErrorCod return hunkFailures.find((failure) => failure.type === 'analysis' && failure.code === code)?.message; } +function extractionFailureCodes(hunkFailures: HunkFailure[]): ErrorCode[] | undefined { + if ( + hunkFailures.length === 0 + || !hunkFailures.every( + (failure) => failure.type === 'extraction' && isExtractionErrorCode(failure.code), + ) + ) { + return undefined; + } + return [...new Set(hunkFailures.map((failure) => failure.code))]; +} + function summarizeRunFailure(args: { totalHunks: number; hunkFailures: HunkFailure[]; @@ -95,6 +108,14 @@ function summarizeRunFailure(args: { message: `Provider unavailable: all ${totalHunks} chunk${totalHunks === 1 ? '' : 's'} failed to analyze. Warden stopped early.`, }; } + const extractionCodes = extractionFailureCodes(hunkFailures); + const primaryExtractionCode = extractionCodes?.[0]; + if (primaryExtractionCode) { + return { + code: primaryExtractionCode, + message: `Findings extraction failed for all ${totalHunks} chunk${totalHunks === 1 ? '' : 's'} (${extractionCodes.join(', ')}).`, + }; + } return { code: 'all_hunks_failed', message: @@ -615,6 +636,7 @@ export async function runSkillTask( const runnerError = new SkillRunnerError(error.message, { code: error.code, providerContext: error.providerContext, + hunkFailures: allHunkFailures, }); return { name, report: errorReport, error: runnerError, failOn, minConfidence }; } diff --git a/packages/warden/src/output/github-checks.test.ts b/packages/warden/src/output/github-checks.test.ts index 2fdd16cfb..7f4049f56 100644 --- a/packages/warden/src/output/github-checks.test.ts +++ b/packages/warden/src/output/github-checks.test.ts @@ -6,6 +6,8 @@ import { aggregateSeverityCounts, createCoreCheck, updateCoreCheck, + failSkillCheck, + createFailedSkillCheck, } from './github-checks.js'; import type { Finding, SkillReport } from '../types/index.js'; @@ -52,6 +54,115 @@ describe('check details URL', () => { }); }); +describe('failSkillCheck', () => { + it('includes sanitized hunk diagnostics in the check summary', async () => { + const update = vi.fn().mockResolvedValue({ data: {} }); + const error = Object.assign(new Error('Extraction failed with api_key=secret-value'), { + hunkFailures: [{ + type: 'extraction', + filename: 'src/auth.ts', + lineRange: '10-20', + code: 'extraction_invalid_json', + message: 'Invalid response with token=secret-value', + preview: 'api_key=secret-value malformed output', + }], + }); + + await failSkillCheck( + { checks: { update } } as never, + 123, + error, + { owner: 'getsentry', repo: 'sentry', headSha: 'abc123' } + ); + + const summary = update.mock.calls[0]![0].output.summary as string; + expect(summary).toContain('src/auth.ts'); + expect(summary).toContain('extraction_invalid_json'); + expect(summary).toContain('[redacted]'); + expect(summary).not.toContain('secret-value'); + }); + + it('keeps model markdown inside the indented diagnostics block', async () => { + const update = vi.fn().mockResolvedValue({ data: {} }); + const error = Object.assign(new Error('Extraction failed'), { + hunkFailures: [{ + type: 'extraction', + filename: 'src/example.ts', + lineRange: '1-2', + code: 'extraction_invalid_json', + message: 'Malformed output', + preview: '```json\n{"findings": []}\n```', + }], + }); + + await failSkillCheck( + { checks: { update } } as never, + 123, + error, + { owner: 'getsentry', repo: 'sentry', headSha: 'abc123' } + ); + + const summary = update.mock.calls[0]![0].output.summary as string; + expect(summary).toContain(' "preview": "```json\\n'); + expect(summary).not.toContain('\n```json\n'); + }); + + it('bounds diagnostics to the GitHub check summary byte limit', async () => { + const update = vi.fn().mockResolvedValue({ data: {} }); + const error = Object.assign(new Error('Extraction failed'), { + hunkFailures: Array.from({ length: 200 }, (_, index) => ({ + type: 'extraction', + filename: `src/file-${index}.ts`, + lineRange: `${index + 1}`, + code: 'extraction_invalid_json', + message: 'Malformed output', + preview: '🔥'.repeat(500), + })), + }); + + await failSkillCheck( + { checks: { update } } as never, + 123, + error, + { owner: 'getsentry', repo: 'sentry', headSha: 'abc123' } + ); + + const summary = update.mock.calls[0]![0].output.summary as string; + expect(Buffer.byteLength(summary, 'utf8')).toBeLessThanOrEqual(65000); + expect(summary).toContain('[diagnostics truncated]'); + }); +}); + +describe('createFailedSkillCheck', () => { + it('uses the same sanitized diagnostics as the update path', async () => { + const create = vi.fn().mockResolvedValue({ + data: { id: 123, html_url: null }, + }); + const error = Object.assign(new Error('Extraction failed with token=secret-value'), { + hunkFailures: [{ + type: 'extraction', + filename: 'src/example.ts', + lineRange: '1-2', + code: 'extraction_invalid_json', + message: 'Malformed output with api_key=secret-value', + preview: 'bad output', + }], + }); + + await createFailedSkillCheck( + { checks: { create } } as never, + 'security-review', + error, + { owner: 'getsentry', repo: 'sentry', headSha: 'abc123' } + ); + + const summary = create.mock.calls[0]![0].output.summary as string; + expect(summary).toContain('extraction_invalid_json'); + expect(summary).toContain('[redacted]'); + expect(summary).not.toContain('secret-value'); + }); +}); + describe('severityToAnnotationLevel', () => { it('maps high to failure', () => { expect(severityToAnnotationLevel('high')).toBe('failure'); diff --git a/packages/warden/src/output/github-checks.ts b/packages/warden/src/output/github-checks.ts index 3fa295ae8..c12aee6c7 100644 --- a/packages/warden/src/output/github-checks.ts +++ b/packages/warden/src/output/github-checks.ts @@ -1,8 +1,9 @@ import type { Octokit } from '@octokit/rest'; -import { SEVERITY_ORDER, filterFindings } from '../types/index.js'; -import type { Severity, SeverityThreshold, ConfidenceThreshold, Finding, SkillReport, UsageStats, AuxiliaryUsageMap } from '../types/index.js'; +import { HunkFailureSchema, SEVERITY_ORDER, filterFindings } from '../types/index.js'; +import type { Severity, SeverityThreshold, ConfidenceThreshold, Finding, SkillReport, UsageStats, AuxiliaryUsageMap, HunkFailure } from '../types/index.js'; import { formatDuration, formatCost, formatTokens, totalUsageCost, totalUsageStats } from '../cli/output/formatters.js'; import { escapeHtml } from '../utils/index.js'; +import { sanitizeErrorMessage } from '../sdk/errors.js'; /** * GitHub Check annotation for inline code comments. @@ -330,6 +331,53 @@ export async function updateSkillCheck( }); } +const MAX_CHECK_SUMMARY_BYTES = 65000; +const TRUNCATION_NOTICE = '\n\n[diagnostics truncated]'; + +function truncateCheckSummary(summary: string): string { + if (Buffer.byteLength(summary, 'utf8') <= MAX_CHECK_SUMMARY_BYTES) return summary; + + const contentBudget = MAX_CHECK_SUMMARY_BYTES - Buffer.byteLength(TRUNCATION_NOTICE, 'utf8'); + let low = 0; + let high = summary.length; + while (low < high) { + const midpoint = Math.ceil((low + high) / 2); + if (Buffer.byteLength(summary.slice(0, midpoint), 'utf8') <= contentBudget) { + low = midpoint; + } else { + high = midpoint - 1; + } + } + return summary.slice(0, low) + TRUNCATION_NOTICE; +} + +function failureSummary(error: unknown): string { + const errorMessage = sanitizeErrorMessage(error instanceof Error ? error.message : String(error)); + const rawFailures = error && typeof error === 'object' && 'hunkFailures' in error + ? (error as { hunkFailures?: unknown }).hunkFailures + : undefined; + const parsedFailures = Array.isArray(rawFailures) + ? rawFailures.flatMap((failure) => { + const parsed = HunkFailureSchema.safeParse(failure); + return parsed.success ? [parsed.data] : []; + }) + : []; + if (parsedFailures.length === 0) return `Error: ${errorMessage}`; + + const sanitizedFailures: HunkFailure[] = parsedFailures.map((failure) => ({ + ...failure, + filename: sanitizeErrorMessage(failure.filename), + lineRange: sanitizeErrorMessage(failure.lineRange), + message: sanitizeErrorMessage(failure.message), + preview: failure.preview ? sanitizeErrorMessage(failure.preview).slice(0, 500) : undefined, + })); + const diagnostics = JSON.stringify(sanitizedFailures, null, 2) + .split('\n') + .map((line) => ` ${line}`) + .join('\n'); + return truncateCheckSummary(`Error: ${errorMessage}\n\nHunk failures:\n${diagnostics}`); +} + /** * Mark a skill check as failed due to execution error. */ @@ -339,7 +387,7 @@ export async function failSkillCheck( error: unknown, options: CheckOptions ): Promise { - const errorMessage = error instanceof Error ? error.message : String(error); + const summary = failureSummary(error); await octokit.checks.update({ owner: options.owner, @@ -350,7 +398,7 @@ export async function failSkillCheck( completed_at: new Date().toISOString(), output: { title: 'Skill execution failed', - summary: `Error: ${errorMessage}`, + summary, }, }); } @@ -364,7 +412,7 @@ export async function createFailedSkillCheck( error: unknown, options: CheckOptions ): Promise { - const errorMessage = error instanceof Error ? error.message : String(error); + const summary = failureSummary(error); const { data } = await octokit.checks.create({ owner: options.owner, @@ -376,7 +424,7 @@ export async function createFailedSkillCheck( completed_at: new Date().toISOString(), output: { title: 'Skill execution failed', - summary: `Error: ${errorMessage}`, + summary, }, }); await setCheckDetailsUrl(octokit, options, data); diff --git a/packages/warden/src/sdk/analyze.test.ts b/packages/warden/src/sdk/analyze.test.ts index 4ff3b1ecb..050f416a4 100644 --- a/packages/warden/src/sdk/analyze.test.ts +++ b/packages/warden/src/sdk/analyze.test.ts @@ -319,6 +319,36 @@ describe('analyzeFile', () => { vi.restoreAllMocks(); }); + it('retries extraction once after a transient auxiliary failure', async () => { + const runAuxiliary = vi.fn() + .mockResolvedValueOnce({ success: false, error: 'malformed tool call' }) + .mockResolvedValueOnce({ success: true, data: { findings: [] } }); + vi.mocked(getRuntime).mockReturnValue({ + name: 'pi', + runSkill: vi.fn().mockResolvedValue({ + result: { + status: 'success', + text: 'No security issues found.', + errors: [], + usage: makeUsage(), + }, + }), + runAuxiliary, + runSynthesis: vi.fn(), + } as unknown as Runtime); + + const result = await analyzeFile( + { name: 'security-review', description: 'Security review.', prompt: 'Return findings as JSON.' }, + makePreparedFile(), + '/tmp/repo', + { runtime: 'pi' }, + ); + + expect(runAuxiliary).toHaveBeenCalledTimes(2); + expect(result.failedExtractions).toBe(0); + expect(result.hunkFailures).toEqual([]); + }); + it('treats runtime aborts as interruption instead of retry failures', async () => { const controller = new AbortController(); const runSkill = vi.fn(async () => { @@ -701,6 +731,42 @@ describe('runSkill', () => { vi.restoreAllMocks(); }); + it('reports all-extraction failures without authentication guidance', async () => { + vi.mocked(getRuntime).mockReturnValue({ + name: 'pi', + runSkill: vi.fn().mockResolvedValue({ + result: { + status: 'success', + text: 'No security issues found.', + errors: [], + usage: makeUsage(), + }, + }), + runAuxiliary: vi.fn().mockResolvedValue({ + success: false, + error: 'malformed tool call', + }), + runSynthesis: vi.fn(), + } as unknown as Runtime); + + await expect(runSkill( + { + name: 'security-review', + description: 'Security review.', + prompt: 'Return findings as JSON.', + }, + makeContextWithOneHunk(), + { runtime: 'pi', verifyFindings: false }, + )).rejects.toMatchObject({ + code: 'extraction_llm_failed', + message: expect.not.stringContaining('authentication'), + hunkFailures: [expect.objectContaining({ + type: 'extraction', + code: 'extraction_llm_failed', + })], + }); + }); + it('records runtime on reports with no hunks to analyze', async () => { const context = makeContextWithOneHunk(); const report = await runSkill( diff --git a/packages/warden/src/sdk/analyze.ts b/packages/warden/src/sdk/analyze.ts index b3048d73c..09572bb32 100644 --- a/packages/warden/src/sdk/analyze.ts +++ b/packages/warden/src/sdk/analyze.ts @@ -1,5 +1,6 @@ import type { Span } from '@sentry/node'; import type { SkillDefinition } from '../config/schema.js'; +import { isExtractionErrorCode } from '../types/index.js'; import type { ErrorCode, Finding, RetryConfig } from '../types/index.js'; import { getHunkLineRange, type HunkWithContext } from '../diff/index.js'; import { Sentry, emitExtractionMetrics, emitRetryMetric, emitSkillMetrics, ensureLocalTracing } from '../sentry.js'; @@ -10,6 +11,7 @@ import { DEFAULT_RETRY_CONFIG, calculateRetryDelay, sleep } from './retry.js'; import { aggregateUsage, emptyUsage, estimateTokens, aggregateAuxiliaryUsage, aggregateAuxiliaryUsageAttribution } from './usage.js'; import { buildHunkSystemPrompt, buildHunkUserPrompt, type PRPromptContext } from './prompt.js'; import { extractFindingsJson, extractFindingsWithLLM, validateFindings } from './extract.js'; +import type { ExtractFindingsResult } from './extract.js'; import { postProcessFindings } from './post-process.js'; import { buildFileReports } from './report-files.js'; import { getRuntime, getRuntimeProviderOptions } from './runtimes/index.js'; @@ -184,27 +186,37 @@ async function parseHunkOutput( return { findings: validateFindings(extracted.findings, filename), extractionFailed: false, extractionMethod: 'regex' }; } - // Tier 2: Try LLM fallback for malformed output - const fallback = await extractFindingsWithLLM(result.text, { - apiKey: options.apiKey, - runtime: options.runtime, - model: options.auxiliaryModel, - maxRetries: options.auxiliaryMaxRetries, - agentName: skillName, - }); - - if (fallback.success) { - return { findings: validateFindings(fallback.findings, filename), extractionFailed: false, extractionMethod: 'llm', extractionUsage: fallback.usage }; + // Tier 2: Try LLM fallback for malformed output, then retry once because + // structured extraction failures can be transient even when analysis succeeded. + const extractionUsage: UsageStats[] = []; + let lastFailure: Extract | undefined; + for (let attempt = 0; attempt < 2; attempt++) { + const fallback = await extractFindingsWithLLM(result.text, { + apiKey: options.apiKey, + runtime: options.runtime, + model: options.auxiliaryModel, + maxRetries: options.auxiliaryMaxRetries, + agentName: skillName, + }); + if (fallback.usage) extractionUsage.push(fallback.usage); + if (fallback.success) { + return { + findings: validateFindings(fallback.findings, filename), + extractionFailed: false, + extractionMethod: 'llm', + extractionUsage: aggregateUsage(extractionUsage), + }; + } + lastFailure = fallback; } - // Both tiers failed - return extraction failure info return { findings: [], extractionFailed: true, extractionMethod: 'none', - extractionError: fallback.error, - extractionPreview: fallback.preview, - extractionUsage: fallback.usage, + extractionError: lastFailure?.error ?? extracted.error, + extractionPreview: lastFailure?.preview ?? extracted.preview, + extractionUsage: extractionUsage.length > 0 ? aggregateUsage(extractionUsage) : undefined, }; } @@ -1178,6 +1190,21 @@ async function runSkillAnalysis( }); } if (totalAttemptFailures > 0 && totalAttemptFailures === totalHunks && allFindings.length === 0) { + const extractionFailures = allHunkFailures.filter((failure) => failure.type === 'extraction'); + if ( + extractionFailures.length === allHunkFailures.length + && extractionFailures.every((failure) => isExtractionErrorCode(failure.code)) + ) { + const extractionCodes = [...new Set(extractionFailures.map((failure) => failure.code))]; + const primaryCode = extractionCodes[0]; + if (primaryCode) { + throw new SkillRunnerError( + `Findings extraction failed for all ${totalHunks} chunk${totalHunks === 1 ? '' : 's'} (${extractionCodes.join(', ')}).`, + { code: primaryCode, hunkFailures: allHunkFailures }, + ); + } + } + const analysisFailures = allHunkFailures.filter((failure) => failure.type === 'analysis'); if ( analysisFailures.length > 0 diff --git a/packages/warden/src/sdk/errors.ts b/packages/warden/src/sdk/errors.ts index 0a918196e..6d4d19dfe 100644 --- a/packages/warden/src/sdk/errors.ts +++ b/packages/warden/src/sdk/errors.ts @@ -5,7 +5,7 @@ import { APIConnectionError, APIConnectionTimeoutError, } from '@anthropic-ai/sdk'; -import type { ErrorCode } from '../types/index.js'; +import type { ErrorCode, HunkFailure } from '../types/index.js'; import { InvalidPiModelSelectorError } from './runtimes/model-selectors.js'; import type { RuntimeName, SkillRunStatus } from './runtimes/types.js'; @@ -24,11 +24,14 @@ export class SkillRunnerError extends Error { code?: ErrorCode; /** Sanitized provider diagnostics safe to attach to telemetry. */ providerContext?: ProviderErrorContext; - constructor(message: string, options?: { cause?: unknown; code?: ErrorCode; providerContext?: ProviderErrorContext }) { + /** Per-hunk diagnostics safe to surface in execution failure reports. */ + hunkFailures?: HunkFailure[]; + constructor(message: string, options?: { cause?: unknown; code?: ErrorCode; providerContext?: ProviderErrorContext; hunkFailures?: HunkFailure[] }) { super(message, options); this.name = 'SkillRunnerError'; if (options?.code) this.code = options.code; if (options?.providerContext) this.providerContext = options.providerContext; + if (options?.hunkFailures) this.hunkFailures = options.hunkFailures; } } diff --git a/packages/warden/src/sdk/extract.test.ts b/packages/warden/src/sdk/extract.test.ts index 5c9bbdc61..bff07abf9 100644 --- a/packages/warden/src/sdk/extract.test.ts +++ b/packages/warden/src/sdk/extract.test.ts @@ -44,6 +44,23 @@ describe('extractFindingsWithLLM', () => { expect(canUseRuntimeAuth({ apiKey: 'test-key', runtime: 'claude' })).toBe(true); }); + it('repairs non-empty output without a findings JSON anchor', async () => { + mockCallHaiku.mockResolvedValue({ + success: true, + data: { findings: [] }, + usage: { inputTokens: 10, outputTokens: 2, costUSD: 0.001 }, + }); + + const result = await extractFindingsWithLLM('I found no security issues.', { apiKey: 'test-key' }); + + expect(result).toEqual({ + success: true, + findings: [], + usage: { inputTokens: 10, outputTokens: 2, costUSD: 0.001 }, + }); + expect(mockCallHaiku).toHaveBeenCalledOnce(); + }); + it('preserves the LLM extraction failure prefix for stable error classification', async () => { mockCallHaiku.mockResolvedValue({ success: false, diff --git a/packages/warden/src/sdk/extract.ts b/packages/warden/src/sdk/extract.ts index b7ef0229a..59e2bae03 100644 --- a/packages/warden/src/sdk/extract.ts +++ b/packages/warden/src/sdk/extract.ts @@ -152,9 +152,7 @@ const LLM_FALLBACK_MAX_TOKENS = 4096; const LLM_FALLBACK_TIMEOUT_MS = 30000; /** - * Truncate text for LLM fallback while preserving the findings JSON. - * - * Caller must ensure findings JSON exists in the text before calling. + * Truncate text for LLM fallback, preserving findings JSON when present. */ export function truncateForLLMFallback(rawText: string, maxChars: number): string { if (rawText.length <= maxChars) { @@ -163,6 +161,11 @@ export function truncateForLLMFallback(rawText: string, maxChars: number): strin const findingsIndex = rawText.match(FINDINGS_JSON_START)?.index ?? -1; + // Without an anchor, preserve the start of the response for best-effort repair. + if (findingsIndex === -1) { + return rawText.slice(0, maxChars) + '\n[... truncated]'; + } + // If findings starts within our budget, simple truncation from start preserves it if (findingsIndex < maxChars - 20) { return rawText.slice(0, maxChars) + '\n[... truncated]'; @@ -206,16 +209,15 @@ export async function extractFindingsWithLLM( }; } - // If no findings anchor exists, there's nothing to extract - if (!FINDINGS_JSON_START.test(rawText)) { + if (!rawText.trim()) { return { success: false, error: 'no_findings_to_extract', - preview: rawText.slice(0, 200), + preview: '', }; } - // Truncate input while preserving JSON boundaries + // Truncate input while preserving JSON boundaries when present const truncatedText = truncateForLLMFallback(rawText, LLM_FALLBACK_MAX_CHARS); const userContent = joinPromptSections([ diff --git a/packages/warden/src/sdk/runner.test.ts b/packages/warden/src/sdk/runner.test.ts index 17de9dd22..02a20acf2 100644 --- a/packages/warden/src/sdk/runner.test.ts +++ b/packages/warden/src/sdk/runner.test.ts @@ -391,8 +391,8 @@ describe('extractFindingsWithLLM', () => { } }); - it('returns error when no findings pattern exists', async () => { - const result = await extractFindingsWithLLM('some output without findings', 'fake-key'); + it('returns error when the response is empty', async () => { + const result = await extractFindingsWithLLM(' ', 'fake-key'); expect(result.success).toBe(false); if (!result.success) { expect(result.error).toBe('no_findings_to_extract');