diff --git a/.gitignore b/.gitignore index 85d13be..5693c93 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ config.json .DS_Store *.log docs/superpowers/ +.claude/ diff --git a/README.md b/README.md index ddee6ad..be9f3fe 100644 --- a/README.md +++ b/README.md @@ -161,13 +161,13 @@ signal, not a verdict. | Rule | Fires when | Meaning | |---|---|---| | Low cache hit ratio | cost ≥ $1 **and** cache-read < 50% of input-side tokens | Context rebuilt instead of served from cache (reads are ~10× cheaper). | -| Premium model, short session | used `fable-5` **and** < 20 messages | A short session rarely needs the most expensive model. Est. saving = `fableCost × 0.7`. | +| Premium model, short session | used a top-tier model (e.g. `fable-5` / `mythos-5`) **and** < 20 messages | A short session rarely needs the most expensive model. Est. saving = `premiumCost × 0.7`. | | Subagent-heavy | cost ≥ $5 **and** subagents > 60% of cost | Fan-out overhead — check the delegation earned its cost. | Only the premium-model rule estimates dollars. Known limits: a low cache ratio is -often not your fault (5-minute cache TTL, `/clear`, idle gaps); the Sonnet-saving -estimate assumes the task would have succeeded on Sonnet; the cutoffs will produce -some false positives. +often not your fault (5-minute cache TTL, `/clear`, idle gaps); the saving +estimate assumes the task would have succeeded on a cheaper model; the cutoffs +will produce some false positives. ## How costs are computed diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 03a088f..3d46c33 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -95,7 +95,7 @@ tuned): | Rule | Fires when | |---|---| | Low cache hit ratio | cost ≥ $1 and cache-read < 50% of input-side tokens | -| Premium model, short session | used `fable-5` and < 20 messages (est. saving = `fableCost × 0.7`) | +| Premium model, short session | used a top-tier model (`fable-5` / `mythos-5`) and < 20 messages (est. saving = `premiumCost × 0.7`) | | Subagent-heavy | cost ≥ $5 and subagents > 60% of cost | Only flagged sessions are returned, sorted by cost, capped at 25. It is a "look diff --git a/extension/package-lock.json b/extension/package-lock.json index f9c67fa..a37618d 100644 --- a/extension/package-lock.json +++ b/extension/package-lock.json @@ -1,12 +1,12 @@ { "name": "cccost-dashboard-vscode", - "version": "2.0.0", + "version": "2.0.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cccost-dashboard-vscode", - "version": "2.0.0", + "version": "2.0.2", "license": "MIT", "devDependencies": { "@vscode/vsce": "^3.2.1", diff --git a/extension/package.json b/extension/package.json index 3c12ce4..8bd1e4a 100644 --- a/extension/package.json +++ b/extension/package.json @@ -2,7 +2,7 @@ "name": "cccost-dashboard-vscode", "displayName": "cccost — Cost Dashboard for Claude Code", "description": "Unofficial. Local, zero-setup cost dashboard for Claude Code — per project, model, and prompt, with an efficiency advisor. Reads ~/.claude/projects/ logs. Not affiliated with Anthropic. Desktop only.", - "version": "2.0.0", + "version": "2.0.2", "publisher": "turja", "icon": "icon.png", "galleryBanner": { diff --git a/lib/core.js b/lib/core.js index 557c8ca..79d7096 100644 --- a/lib/core.js +++ b/lib/core.js @@ -9,6 +9,12 @@ const PRICING = [ { match: 'haiku', rates: { input: 1, output: 5, write5m: 1.25, write1h: 2, read: 0.1 } }, ]; +// The most expensive input tier in the table. The advisor flags short sessions +// that used a top-tier model (any model priced at this rate); matching against +// the pricing table rather than a hard-coded model name keeps it correct as +// models change and covers every model in the tier, not just one. +const MAX_INPUT_RATE = Math.max(...PRICING.map((p) => p.rates.input)); + // Local calendar date (YYYY-MM-DD) of a timestamp, in the machine's timezone. // Daily/monthly buckets are attributed by local date so billing months follow // wall-clock time, not UTC. @@ -24,6 +30,105 @@ function getRates(model) { return entry ? entry.rates : null; } +// Known, literal error-message substrings from Claude Code's own tool +// implementations. First match wins. Anything else (e.g. a bare shell "Exit +// code N") is classified 'other' rather than guessed — a wrong guess here +// would misdirect a real fix (see waste-detection review history). +const ERROR_REASONS = [ + { match: "doesn't want to proceed", reason: 'user-rejected' }, + { match: 'has not been read yet', reason: 'edit-before-read' }, + { match: 'String to replace not found', reason: 'edit-string-not-found' }, + { match: 'has been modified since read', reason: 'stale-read' }, + { match: 'File does not exist', reason: 'file-not-found' }, + { match: 'denied by the Claude Code auto mode classifier', reason: 'auto-mode-denied' }, + { match: 'is temporarily unavailable', reason: 'model-unavailable' }, + { match: 'shell cwd recovered', reason: 'cwd-deleted' }, +]; + +// tool_result.content is either a plain string or an array of {type:'text'} +// blocks — normalize to one string for substring matching. +function textOf(content) { + if (typeof content === 'string') return content; + if (Array.isArray(content)) return content.map((b) => (b && b.text) || '').join(' '); + return ''; +} + +function classifyErrorReason(content) { + const text = textOf(content); + const entry = ERROR_REASONS.find((r) => text.includes(r.match)); + return entry ? entry.reason : 'other'; +} + +// Errored-call detail: how many distinct samples to keep per (tool, reason) +// and how long each command/message may be before truncation. +const SAMPLE_MAX = 3; +const SAMPLE_TRUNC = 200; + +// Mask credentials before a command or error message is ever stored, since a +// Bash command or its output can carry tokens/passwords. Targeted patterns +// only — mask the value, keep the surrounding text so the sample stays useful. +function redactSecrets(input) { + if (!input) return ''; + return input + // No leading \b: env-var style names (DB_PASSWORD, STRIPE_SECRET_KEY, + // ANTHROPIC_API_KEY) glue the keyword to a prefix with '_', and '_' is a + // word char so \b never matches there. The require an immediate '='/':' + // after the keyword is what keeps this from over-matching prose. + .replace(/(password|passwd|pwd|secret|token|api[_-]?key|access[_-]?key|auth[_-]?token|credential|bearer)(\s*[=:]\s*)("[^"]*"|'[^']*'|[^\s"']+)/gi, + (_, k, sep) => `${k}${sep}«redacted»`) + .replace(/(--(?:password|token|secret|api[_-]?key|access[_-]?key|auth)[a-z-]*)([=\s]+)("[^"]*"|'[^']*'|[^\s"']+)/gi, + (_, flag, sep) => `${flag}${sep}«redacted»`) + // Glued single-dash password flags (mysql/psql: -pSECRET, no space). Over-matches + // an unrelated single-dash long flag typed the same way (e.g. -parallel) — for a + // security mask that's the safe side of the tradeoff, not a correctness bug. + .replace(/(\s-p)([^\s"']+)/g, (_, flag) => `${flag}«redacted»`) + .replace(/\b(Authorization\s*:\s*)[^"'\n]+/gi, (_, p) => `${p}«redacted»`) + .replace(/\bBearer\s+\S+/gi, 'Bearer «redacted»') + .replace(/([a-z][a-z0-9+.-]*:\/\/[^\s:/@]+:)[^\s@]+(@)/gi, (_, a, b) => `${a}«redacted»${b}`) + .replace(/\bAKIA[0-9A-Z]{16}\b/g, '«redacted»') + // Body allows '-'/'_' too, so hyphenated keys (sk-ant-api03-...) redact in full + // instead of stopping at the first internal hyphen. + .replace(/\b(?:sk|pk|ghp|gho|ghs|xox[baprs])[-_][A-Za-z0-9_-]{10,}\b/g, '«redacted»'); +} + +function truncate(s, n) { + return s.length > n ? `${s.slice(0, n)}…` : s; +} + +// Add a sample to an array with cap + de-dup (used on first capture and merge). +function pushSample(arr, sample) { + if (arr.length >= SAMPLE_MAX) return; + if (arr.some((x) => x.target === sample.target && x.text === sample.text)) return; + arr.push(sample); +} + +// Record what a confirmed errored call actually was: the command/file that ran +// and the error message, both redacted + truncated. Keyed by tool+reason so the +// UI can show a few concrete examples per failure kind. +function addErrorSample(samples, tool, reason, target, text) { + const key = `${tool} ${reason}`; + const arr = samples[key] || (samples[key] = []); + pushSample(arr, { + tool, + reason, + target: truncate(redactSecrets(target || ''), SAMPLE_TRUNC), + text: truncate(redactSecrets(text || ''), SAMPLE_TRUNC), + }); +} + +// The identifier that answers "which one" for a tool call: the shell command +// for Bash, else the file path or search pattern from the input. +function toolTarget(block) { + const input = block.input || {}; + if (block.name === 'Bash') return input.command || ''; + if (input.file_path) return input.file_path; + if (input.notebook_path) return input.notebook_path; + if (input.pattern) return input.pattern; + if (input.url) return input.url; + if (input.query) return input.query; + return ''; +} + function emptyTokens() { return { input: 0, output: 0, cacheWrite5m: 0, cacheWrite1h: 0, cacheRead: 0 }; } @@ -32,6 +137,59 @@ function addTokens(target, src) { for (const k of Object.keys(target)) target[k] += src[k]; } +// Zeroed waste aggregate. Raw maps are kept so files can be merged per key. +function emptyWaste() { + return { + toolCallCount: 0, + erroredToolCalls: 0, + erroredByTool: {}, + erroredByReason: {}, + errorSamples: {}, + redundantReads: 0, + duplicateFiles: {}, + daily: {}, + }; +} + +// Bump a per-day waste counter (redundantReads/erroredToolCalls), keyed by the +// local date of the event that caused it. Timestampless events (malformed or +// missing `timestamp` field) are not attributable to a day and are skipped. +function bumpDailyWaste(daily, timestamp, field) { + if (!timestamp) return; + const date = localDate(timestamp); + if (!daily[date]) daily[date] = { erroredToolCalls: 0, redundantReads: 0 }; + daily[date][field]++; +} + +// Sum src's waste into target (scalars added, maps merged per key). Missing src +// (a session with no tool blocks) is a no-op. +function mergeWasteInto(target, src) { + if (!src) return target; + target.toolCallCount += src.toolCallCount; + target.erroredToolCalls += src.erroredToolCalls; + target.redundantReads += src.redundantReads; + for (const [k, v] of Object.entries(src.erroredByTool)) { + target.erroredByTool[k] = (target.erroredByTool[k] || 0) + v; + } + for (const [k, v] of Object.entries(src.erroredByReason)) { + target.erroredByReason[k] = (target.erroredByReason[k] || 0) + v; + } + if (!target.errorSamples) target.errorSamples = {}; + for (const [k, arr] of Object.entries(src.errorSamples || {})) { + const dst = target.errorSamples[k] || (target.errorSamples[k] = []); + for (const s of arr) pushSample(dst, s); + } + for (const [k, v] of Object.entries(src.duplicateFiles)) { + target.duplicateFiles[k] = (target.duplicateFiles[k] || 0) + v; + } + for (const [date, d] of Object.entries(src.daily)) { + if (!target.daily[date]) target.daily[date] = { erroredToolCalls: 0, redundantReads: 0 }; + target.daily[date].erroredToolCalls += d.erroredToolCalls; + target.daily[date].redundantReads += d.redundantReads; + } + return target; +} + function sumTokens(t) { return t.input + t.output + t.cacheWrite5m + t.cacheWrite1h + t.cacheRead; } @@ -71,6 +229,12 @@ function parseSession(text, { sessionId, project }) { const byMsgId = new Map(); let malformedLines = 0; let cwd = null; + const waste = emptyWaste(); + const toolNameById = new Map(); // tool_use_id -> {name, target}, to attribute errors + const cleanReads = new Set(); // file paths read whole (no offset/limit) and not edited/written since + const seenToolUse = new Set(); // tool_use ids already counted (streaming writes the line repeatedly) + const seenErrors = new Set(); // tool_use ids whose errored result is already counted + const pendingErrors = []; // {toolUseId, name} errors not yet confirmed as retried for (const line of text.split('\n')) { if (!line.trim()) continue; @@ -82,6 +246,67 @@ function parseSession(text, { sessionId, project }) { continue; } if (!cwd && obj.cwd) cwd = obj.cwd; + + // Tool events share this single pass. tool_use lives on assistant messages, + // tool_result on user messages (no usage) — so scan here, before the + // usage-only filter below. Only array content carries tool blocks. + const content = obj.message && obj.message.content; + if (Array.isArray(content)) { + for (const block of content) { + if (obj.type === 'assistant' && block.type === 'tool_use') { + if (block.id && seenToolUse.has(block.id)) continue; // streaming re-write, already counted + const blockTarget = toolTarget(block); + if (block.id) { seenToolUse.add(block.id); toolNameById.set(block.id, { name: block.name, target: blockTarget }); } + waste.toolCallCount++; + const fp = block.input && block.input.file_path; + const notebookPath = block.input && block.input.notebook_path; + const isRanged = block.input && (block.input.offset != null || block.input.limit != null); + if (fp && block.name === 'Read' && !isRanged) { + if (cleanReads.has(fp)) { + waste.redundantReads++; + waste.duplicateFiles[fp] = (waste.duplicateFiles[fp] || 0) + 1; + bumpDailyWaste(waste.daily, obj.timestamp, 'redundantReads'); + } else { + cleanReads.add(fp); + } + } else if (fp && (block.name === 'Edit' || block.name === 'Write' || block.name === 'MultiEdit')) { + cleanReads.delete(fp); // a re-read after an edit is legitimate + } else if (notebookPath && block.name === 'NotebookEdit') { + cleanReads.delete(notebookPath); + } else if (block.name === 'Bash') { + cleanReads.clear(); // a shell command may have mutated any file + } + // A retry: this call matches an earlier unresolved error for the same + // tool AND target, so that earlier failure did cost extra quota. Count + // it now. Matching by tool name alone would let any unrelated later + // call of the same tool falsely "confirm" a prior different call's error. + // Require a non-empty target too: tools toolTarget can't identify + // (e.g. TodoWrite) would otherwise all collide on '' and falsely match. + const retryIdx = blockTarget + ? pendingErrors.findIndex((e) => e.name === block.name && e.target === blockTarget) + : -1; + if (retryIdx !== -1) { + const { name, reason, target, text } = pendingErrors[retryIdx]; + pendingErrors.splice(retryIdx, 1); + waste.erroredToolCalls++; + waste.erroredByTool[name] = (waste.erroredByTool[name] || 0) + 1; + waste.erroredByReason[reason] = (waste.erroredByReason[reason] || 0) + 1; + addErrorSample(waste.errorSamples, name, reason, target, text); + bumpDailyWaste(waste.daily, obj.timestamp, 'erroredToolCalls'); + } + } else if (obj.type === 'user' && block.type === 'tool_result' && block.is_error === true) { + if (block.tool_use_id && seenErrors.has(block.tool_use_id)) continue; // dedup streamed result + if (block.tool_use_id) seenErrors.add(block.tool_use_id); + const info = toolNameById.get(block.tool_use_id); + const name = (info && info.name) || 'unknown'; + const target = (info && info.target) || ''; + const reason = classifyErrorReason(block.content); + // waste counted (and the sample kept) only if the same tool is retried later + pendingErrors.push({ toolUseId: block.tool_use_id, name, reason, target, text: textOf(block.content) }); + } + } + } + if (obj.type !== 'assistant' || !obj.message || !obj.message.usage) continue; const msg = obj.message; const id = msg.id || obj.uuid; @@ -132,6 +357,7 @@ function parseSession(text, { sessionId, project }) { costUSD, models, daily, + waste, malformedLines, unknownModelMessages, }; @@ -282,8 +508,10 @@ function mergeSessionAggregates(aggregates) { for (const [date, d] of Object.entries(first.daily)) { merged.daily[date] = { ...d }; } + merged.waste = mergeWasteInto(emptyWaste(), first.waste); for (const s of rest) { merged.project = merged.project || s.project; + mergeWasteInto(merged.waste, s.waste); merged.messages += s.messages; merged.costUSD += s.costUSD; merged.malformedLines += s.malformedLines; @@ -321,23 +549,38 @@ function advisorFor(s) { const t = s.tokens; const denom = t.input + t.cacheWrite5m + t.cacheWrite1h + t.cacheRead; - if (s.costUSD >= 1 && denom > 0 && t.cacheRead / denom < 0.5) { - reasons.push(`Low cache hit ratio (${Math.round((t.cacheRead / denom) * 100)}%) — context likely rebuilt repeatedly`); + // Precision guard: a low cache ratio on a tiny/idle session is noise; only on a + // non-trivial input volume (>=200k tokens) is it real context-rebuild cost. + if (s.costUSD >= 1 && denom >= 200000 && t.cacheRead / denom < 0.5) { + reasons.push({ + rule: 'low-cache-hit', + text: `Low cache hit ratio (${Math.round((t.cacheRead / denom) * 100)}%) — context likely rebuilt repeatedly`, + action: 'Run /clear between unrelated tasks so context is served from cache instead of rebuilt.', + }); } - let fableCost = 0; + let premiumCost = 0; for (const [model, m] of Object.entries(s.models)) { - if (model.includes('fable-5')) fableCost += m.costUSD; + const rates = getRates(model); + if (rates && rates.input === MAX_INPUT_RATE) premiumCost += m.costUSD; } - if (fableCost > 0 && s.messages < 20) { - const save = fableCost * 0.7; + if (premiumCost > 0 && s.messages < 20) { + const save = premiumCost * 0.7; estSavingUSD += save; - reasons.push(`fable-5 on a short session — sonnet likely sufficient (est. save $${save.toFixed(2)})`); + reasons.push({ + rule: 'premium-model-short-session', + text: `Premium model on a short session — a cheaper model likely sufficient (est. save $${save.toFixed(2)})`, + action: 'Route short or simple tasks to a cheaper model (Sonnet) via /model.', + }); } const sub = s.subagentCostUSD || 0; if (s.costUSD >= 5 && sub / s.costUSD > 0.6) { - reasons.push(`${Math.round((sub / s.costUSD) * 100)}% of cost from subagents ($${sub.toFixed(2)}) — check delegation value`); + reasons.push({ + rule: 'subagent-heavy', + text: `${Math.round((sub / s.costUSD) * 100)}% of cost from subagents ($${sub.toFixed(2)}) — check delegation value`, + action: 'Check the fan-out earned its cost — try fewer subagents or a single-agent pass.', + }); } if (!reasons.length) return null; @@ -381,6 +624,13 @@ function buildResponse(sessionAggregates, config) { const monthlyMap = new Map(); const sessions = []; const advisor = []; + const wasteTotals = { erroredToolCalls: 0, redundantReads: 0 }; + const erroredByToolMap = new Map(); + const erroredByReasonMap = new Map(); + const errorSampleMap = new Map(); + const dupFilesMap = new Map(); + const wasteByProject = new Map(); + const wasteDailyMap = new Map(); for (const s of sessionAggregates) { summary.malformedLines += s.malformedLines; @@ -422,6 +672,34 @@ function buildResponse(sessionAggregates, config) { m.tokens += d.tokens; } + const w = s.waste || emptyWaste(); + wasteTotals.erroredToolCalls += w.erroredToolCalls; + wasteTotals.redundantReads += w.redundantReads; + for (const [name, c] of Object.entries(w.erroredByTool)) { + erroredByToolMap.set(name, (erroredByToolMap.get(name) || 0) + c); + } + for (const [reason, c] of Object.entries(w.erroredByReason || {})) { + erroredByReasonMap.set(reason, (erroredByReasonMap.get(reason) || 0) + c); + } + for (const [k, arr] of Object.entries(w.errorSamples || {})) { + let dst = errorSampleMap.get(k); + if (!dst) errorSampleMap.set(k, (dst = [])); + for (const s of arr) pushSample(dst, s); + } + for (const [fp, c] of Object.entries(w.duplicateFiles)) { + dupFilesMap.set(fp, (dupFilesMap.get(fp) || 0) + c); + } + let wp = wasteByProject.get(s.project); + if (!wp) wasteByProject.set(s.project, (wp = { project: s.project, erroredToolCalls: 0, redundantReads: 0 })); + wp.erroredToolCalls += w.erroredToolCalls; + wp.redundantReads += w.redundantReads; + for (const [date, d] of Object.entries(w.daily || {})) { + let wd = wasteDailyMap.get(date); + if (!wd) wasteDailyMap.set(date, (wd = { date, erroredToolCalls: 0, redundantReads: 0 })); + wd.erroredToolCalls += d.erroredToolCalls; + wd.redundantReads += d.redundantReads; + } + const flagged = advisorFor(s); if (flagged) advisor.push(flagged); } @@ -432,8 +710,30 @@ function buildResponse(sessionAggregates, config) { const monthly = [...monthlyMap.values()].sort((a, b) => (a.month < b.month ? -1 : 1)); + const waste = { + erroredToolCalls: wasteTotals.erroredToolCalls, + redundantReads: wasteTotals.redundantReads, + duplicateFileCount: dupFilesMap.size, + erroredByTool: [...erroredByToolMap.entries()] + .map(([name, count]) => ({ name, count })) + .sort((a, b) => b.count - a.count), + erroredByReason: [...erroredByReasonMap.entries()] + .map(([reason, count]) => ({ reason, count })) + .sort((a, b) => b.count - a.count), + errorSamples: [...errorSampleMap.values()].flat().slice(0, 30), + byProject: [...wasteByProject.values()] + .filter((p) => p.erroredToolCalls + p.redundantReads > 0) + .sort((a, b) => (b.erroredToolCalls + b.redundantReads) - (a.erroredToolCalls + a.redundantReads)), + topDuplicateFiles: [...dupFilesMap.entries()] + .map(([path, extraReads]) => ({ path, extraReads })) + .sort((a, b) => b.extraReads - a.extraReads) + .slice(0, 25), + trend: [...wasteDailyMap.values()].sort((a, b) => (a.date < b.date ? -1 : 1)), + }; + const roi = { subscriptionUSDPerMonth: cfg.subscriptionUSDPerMonth, + configured: !!(config && config.subscriptionUSDPerMonth != null), months: monthly.map((m) => ({ month: m.month, valueUSD: m.costUSD, @@ -449,6 +749,7 @@ function buildResponse(sessionAggregates, config) { daily: [...dailyMap.values()].sort((a, b) => (a.date < b.date ? -1 : 1)), monthly, roi, + waste, advisor: advisor.slice(0, 25), sessions, }; @@ -456,5 +757,6 @@ function buildResponse(sessionAggregates, config) { module.exports = { getRates, parseSession, buildResponse, mergeSessionAggregates, sumTokens, - buildReport, DEFAULT_CONFIG, parseTurns, attributeSubagentTurns, + buildReport, DEFAULT_CONFIG, parseTurns, attributeSubagentTurns, classifyErrorReason, + redactSecrets, }; diff --git a/package.json b/package.json index 16a4816..bf2e5a5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cccost-dashboard", - "version": "2.0.0", + "version": "2.0.2", "description": "Local, zero-setup cost dashboard for Claude Code — per project, client, model, and prompt, with an efficiency advisor.", "author": "Simanta Deb Turja", "license": "MIT", diff --git a/server.js b/server.js index b0fcdab..c180023 100755 --- a/server.js +++ b/server.js @@ -7,6 +7,9 @@ const { buildResponse, buildReport, parseTurns, attributeSubagentTurns } = requi const { createStore, loadConfig, sessionKeyFor } = require('./lib/scan'); const PORT = process.env.PORT || 3456; +// Loopback-only: this dashboard has no auth, so binding wider would expose +// project paths, prompts, and error samples to anyone on the LAN. +const HOST = '127.0.0.1'; const DIST_DIR = path.join(__dirname, 'web', 'dist'); const INDEX_HTML = path.join(DIST_DIR, 'index.html'); const MIME = { @@ -116,7 +119,7 @@ function start() { } throw err; }); - server.listen(PORT, () => { + server.listen(PORT, HOST, () => { console.log(`Dashboard: http://localhost:${PORT} (${store.size} session files)`); }); } @@ -125,4 +128,4 @@ if (require.main === module) { start(); } -module.exports = { sessionKeyFor, resolveAssetPath }; +module.exports = { sessionKeyFor, resolveAssetPath, server, HOST }; diff --git a/test/core.test.js b/test/core.test.js index 4e06b0f..6dea3c6 100644 --- a/test/core.test.js +++ b/test/core.test.js @@ -6,7 +6,8 @@ const { test } = require('node:test'); const assert = require('node:assert'); const { parseSession, buildResponse, mergeSessionAggregates, getRates, - buildReport, DEFAULT_CONFIG, parseTurns, attributeSubagentTurns, + buildReport, DEFAULT_CONFIG, parseTurns, attributeSubagentTurns, classifyErrorReason, + redactSecrets, } = require('../lib/core'); const opusLine = JSON.stringify({ @@ -218,6 +219,20 @@ test('buildResponse roi: multiple = monthly value / subscription, ascending', () assert.strictEqual(r2.roi.months[1].multiple, 35 / 100); }); +test('buildResponse roi: configured flag reflects whether the user set a plan price', () => { + const sessions = [acme1, acme2, globex, personal]; + // no config → $200 default, flagged as NOT user-configured + const rDefault = buildResponse(sessions); + assert.strictEqual(rDefault.roi.subscriptionUSDPerMonth, 200); + assert.strictEqual(rDefault.roi.configured, false); + // config object without the key → still a default, not configured + assert.strictEqual(buildResponse(sessions, {}).roi.configured, false); + // explicit plan price → configured; the Pro ($20) case that was silently 10x off + const rSet = buildResponse(sessions, { subscriptionUSDPerMonth: 20 }); + assert.strictEqual(rSet.roi.configured, true); + assert.strictEqual(rSet.roi.months[1].multiple, 35 / 20); +}); + test('mergeSessionAggregates sums subagentCostUSD from non-main files', () => { const main = Object.assign(parseSession(opusLine, { sessionId: 'sess-1', project: 'p' }), { isMain: true }); const sub1 = Object.assign(parseSession(fableLine, { sessionId: 'sess-1', project: 'p' }), { isMain: false }); @@ -244,25 +259,45 @@ function fakeSession(o) { } const advisorById = (r) => Object.fromEntries(r.advisor.map((a) => [a.sessionId, a])); -test('advisor rule 1: low cache ratio fires below 0.5, not at threshold', () => { - const fire = fakeSession({ sessionId: 'r1-fire', costUSD: 2, tokens: { ...emptyTok(), input: 1000, cacheRead: 100 } }); - const nofire = fakeSession({ sessionId: 'r1-thresh', costUSD: 2, tokens: { ...emptyTok(), input: 100, cacheRead: 100 } }); - const nocost = fakeSession({ sessionId: 'r1-cost', costUSD: 0.5, tokens: { ...emptyTok(), input: 1000, cacheRead: 100 } }); - const a = advisorById(buildResponse([fire, nofire, nocost])); - assert.deepStrictEqual(a['r1-fire'].reasons, ['Low cache hit ratio (9%) — context likely rebuilt repeatedly']); +test('advisor rule 1: low cache ratio fires below 0.5 with non-trivial volume, not at threshold', () => { + // denom = input + cacheWrite5m + cacheWrite1h + cacheRead; guard requires denom >= 200000. + const fire = fakeSession({ sessionId: 'r1-fire', costUSD: 2, tokens: { ...emptyTok(), input: 190000, cacheRead: 10000 } }); + const thresh = fakeSession({ sessionId: 'r1-thresh', costUSD: 2, tokens: { ...emptyTok(), input: 100000, cacheRead: 100000 } }); + const nocost = fakeSession({ sessionId: 'r1-cost', costUSD: 0.5, tokens: { ...emptyTok(), input: 190000, cacheRead: 10000 } }); + const small = fakeSession({ sessionId: 'r1-small', costUSD: 2, tokens: { ...emptyTok(), input: 19000, cacheRead: 1000 } }); + const a = advisorById(buildResponse([fire, thresh, nocost, small])); + assert.deepStrictEqual(a['r1-fire'].reasons, [{ + rule: 'low-cache-hit', + text: 'Low cache hit ratio (5%) — context likely rebuilt repeatedly', + action: 'Run /clear between unrelated tasks so context is served from cache instead of rebuilt.', + }]); assert.strictEqual(a['r1-fire'].estSavingUSD, 0); - assert.ok(!a['r1-thresh']); - assert.ok(!a['r1-cost']); + assert.ok(!a['r1-thresh']); // ratio == 0.5, not below + assert.ok(!a['r1-cost']); // cost < 1 + assert.ok(!a['r1-small']); // same 5% ratio but denom < 200000 — precision guard }); -test('advisor rule 2: fable-5 on short session, est saving = fableCost * 0.7', () => { - const models = { 'claude-fable-5': { costUSD: 5, messages: 5, tokens: emptyTok() } }; - const fire = fakeSession({ sessionId: 'r2-fire', costUSD: 5, messages: 10, models }); - const nofire = fakeSession({ sessionId: 'r2-long', costUSD: 5, messages: 20, models }); - const a = advisorById(buildResponse([fire, nofire])); - assert.deepStrictEqual(a['r2-fire'].reasons, ['fable-5 on a short session — sonnet likely sufficient (est. save $3.50)']); - assert.ok(Math.abs(a['r2-fire'].estSavingUSD - 3.5) < 1e-9); +test('advisor rule 2: any top-tier model on a short session flags; est saving = premiumCost * 0.7', () => { + const MSG = { + rule: 'premium-model-short-session', + text: 'Premium model on a short session — a cheaper model likely sufficient (est. save $3.50)', + action: 'Route short or simple tasks to a cheaper model (Sonnet) via /model.', + }; + const fableModels = { 'claude-fable-5': { costUSD: 5, messages: 5, tokens: emptyTok() } }; + const mythosModels = { 'claude-mythos-5': { costUSD: 5, messages: 5, tokens: emptyTok() } }; + const opusModels = { 'claude-opus-4-8': { costUSD: 5, messages: 5, tokens: emptyTok() } }; + const fable = fakeSession({ sessionId: 'r2-fable', costUSD: 5, messages: 10, models: fableModels }); + const mythos = fakeSession({ sessionId: 'r2-mythos', costUSD: 5, messages: 10, models: mythosModels }); + const long = fakeSession({ sessionId: 'r2-long', costUSD: 5, messages: 20, models: fableModels }); + const opus = fakeSession({ sessionId: 'r2-opus', costUSD: 5, messages: 10, models: opusModels }); + const a = advisorById(buildResponse([fable, mythos, long, opus])); + // both top-tier models (fable-5 and mythos-5 share the max input rate) fire + assert.deepStrictEqual(a['r2-fable'].reasons, [MSG]); + assert.deepStrictEqual(a['r2-mythos'].reasons, [MSG]); + assert.ok(Math.abs(a['r2-fable'].estSavingUSD - 3.5) < 1e-9); + // a longer session doesn't fire; a cheaper tier (opus) doesn't fire this rule assert.ok(!a['r2-long']); + assert.ok(!a['r2-opus']); }); test('advisor rule 3: subagent-heavy fires above 0.6 and cost >= 5, not at threshold', () => { @@ -270,7 +305,11 @@ test('advisor rule 3: subagent-heavy fires above 0.6 and cost >= 5, not at thres const nofire = fakeSession({ sessionId: 'r3-thresh', costUSD: 5, subagentCostUSD: 3 }); const nocost = fakeSession({ sessionId: 'r3-cost', costUSD: 4, subagentCostUSD: 4 }); const a = advisorById(buildResponse([fire, nofire, nocost])); - assert.deepStrictEqual(a['r3-fire'].reasons, ['67% of cost from subagents ($4.00) — check delegation value']); + assert.deepStrictEqual(a['r3-fire'].reasons, [{ + rule: 'subagent-heavy', + text: '67% of cost from subagents ($4.00) — check delegation value', + action: 'Check the fan-out earned its cost — try fewer subagents or a single-agent pass.', + }]); assert.ok(!a['r3-thresh']); assert.ok(!a['r3-cost']); }); @@ -278,7 +317,8 @@ test('advisor rule 3: subagent-heavy fires above 0.6 and cost >= 5, not at thres test('advisor sorts by cost desc and caps at 25', () => { const many = []; for (let i = 1; i <= 30; i++) { - many.push(fakeSession({ sessionId: `s${i}`, costUSD: i, tokens: { ...emptyTok(), input: 1000 } })); + // input >= 200000 with cacheRead 0 → passes the rule-1 precision guard so each session flags + many.push(fakeSession({ sessionId: `s${i}`, costUSD: i, tokens: { ...emptyTok(), input: 200000 } })); } const r = buildResponse(many); assert.strictEqual(r.advisor.length, 25); @@ -371,3 +411,381 @@ test('attributeSubagentTurns: window attribution incl. before-first and after-la assert.ok(Math.abs(turns[1].costUSD - 30) < 1e-9); assert.ok(turns[1].models.includes('claude-fable-5')); }); + +// ---- waste: tool_use / tool_result parsing + duplicate reads ---- +const asstTools = (o) => JSON.stringify({ + type: 'assistant', + timestamp: o.ts || '2026-07-01T10:00:00.000Z', + cwd: o.cwd, + message: { id: o.id, model: 'claude-opus-4-8', usage: { input_tokens: 1 }, content: o.blocks }, +}); +const userResults = (o) => JSON.stringify({ + type: 'user', + timestamp: o.ts || '2026-07-01T10:01:00.000Z', + message: { content: o.blocks }, +}); + +test('waste: counts tool_use calls and attributes errored tool_result to its tool name, only when retried', () => { + const text = [ + asstTools({ id: 'm1', cwd: '/w/proj', blocks: [ + { type: 'tool_use', id: 't1', name: 'Bash', input: { command: 'ls' } }, + { type: 'tool_use', id: 't2', name: 'Read', input: { file_path: '/w/a.js' } }, + { type: 'tool_use', id: 't3', name: 'Bash', input: { command: 'boom' } }, + ] }), + userResults({ blocks: [ + { type: 'tool_result', tool_use_id: 't3', is_error: true }, + { type: 'tool_result', tool_use_id: 't2' }, // success — ignored + ] }), + asstTools({ id: 'm2', blocks: [{ type: 'tool_use', id: 't4', name: 'Bash', input: { command: 'boom' } }] }), // retry of t3 + ].join('\n'); + const s = parseSession(text, { sessionId: 'w1', project: 'p' }); + assert.strictEqual(s.waste.toolCallCount, 4); + assert.strictEqual(s.waste.erroredToolCalls, 1); + assert.deepStrictEqual(s.waste.erroredByTool, { Bash: 1 }); + assert.strictEqual(s.waste.redundantReads, 0); +}); + +test('waste: an unrelated later call of the same tool does not falsely confirm a different call\'s error', () => { + const text = [ + asstTools({ id: 'm1', blocks: [{ type: 'tool_use', id: 't1', name: 'Bash', input: { command: 'rm -rf /tmp/A' } }] }), + userResults({ blocks: [{ type: 'tool_result', tool_use_id: 't1', is_error: true, content: 'File does not exist' }] }), + // A different, unrelated Bash command — not a retry of t1 — that also happens to error. + asstTools({ id: 'm2', blocks: [{ type: 'tool_use', id: 't2', name: 'Bash', input: { command: 'curl https://x' } }] }), + userResults({ blocks: [{ type: 'tool_result', tool_use_id: 't2', is_error: true, content: 'connection refused' }] }), + ].join('\n'); + const s = parseSession(text, { sessionId: 'w-falseretry', project: 'p' }); + assert.strictEqual(s.waste.erroredToolCalls, 0); + assert.deepStrictEqual(s.waste.errorSamples, {}); +}); + +test('waste: two different WebFetch errors never falsely confirm each other (no shared empty target)', () => { + const text = [ + asstTools({ id: 'm1', blocks: [{ type: 'tool_use', id: 't1', name: 'WebFetch', input: { url: 'https://a.example' } }] }), + userResults({ blocks: [{ type: 'tool_result', tool_use_id: 't1', is_error: true }] }), + asstTools({ id: 'm2', blocks: [{ type: 'tool_use', id: 't2', name: 'WebFetch', input: { url: 'https://totally-different.example' } }] }), + userResults({ blocks: [{ type: 'tool_result', tool_use_id: 't2', is_error: true }] }), + ].join('\n'); + const s = parseSession(text, { sessionId: 'w-webfetch-nomatch', project: 'p' }); + assert.strictEqual(s.waste.erroredToolCalls, 0); +}); + +test('waste: a genuine WebFetch retry of the SAME url is still counted (url is now a target)', () => { + const text = [ + asstTools({ id: 'm1', blocks: [{ type: 'tool_use', id: 't1', name: 'WebFetch', input: { url: 'https://a.example' } }] }), + userResults({ blocks: [{ type: 'tool_result', tool_use_id: 't1', is_error: true }] }), + asstTools({ id: 'm2', blocks: [{ type: 'tool_use', id: 't2', name: 'WebFetch', input: { url: 'https://a.example' } }] }), // retry + ].join('\n'); + const s = parseSession(text, { sessionId: 'w-webfetch-match', project: 'p' }); + assert.strictEqual(s.waste.erroredToolCalls, 1); +}); + +test('waste: an errored tool call with no later retry of the same tool is not counted', () => { + const text = [ + asstTools({ id: 'm1', blocks: [{ type: 'tool_use', id: 't1', name: 'Bash', input: { command: 'git diff --exit-code' } }] }), + userResults({ blocks: [{ type: 'tool_result', tool_use_id: 't1', is_error: true }] }), + ].join('\n'); + const s = parseSession(text, { sessionId: 'w-nonretry', project: 'p' }); + assert.strictEqual(s.waste.erroredToolCalls, 0); + assert.deepStrictEqual(s.waste.erroredByTool, {}); +}); + +test('waste: streaming re-writes of the same tool_use/result are counted once', () => { + const line = asstTools({ id: 'm1', blocks: [{ type: 'tool_use', id: 't1', name: 'Bash', input: { command: 'boom' } }] }); + const res = userResults({ blocks: [{ type: 'tool_result', tool_use_id: 't1', is_error: true }] }); + const retry = asstTools({ id: 'm2', blocks: [{ type: 'tool_use', id: 't2', name: 'Bash', input: { command: 'boom' } }] }); + const s = parseSession([line, line, res, res, retry, retry].join('\n'), { sessionId: 'w-dup', project: 'p' }); + assert.strictEqual(s.waste.toolCallCount, 2); + assert.strictEqual(s.waste.erroredToolCalls, 1); +}); + +test('waste: duplicate Read is redundant; a Read after an Edit is not', () => { + const dup = parseSession([ + asstTools({ id: 'm1', blocks: [{ type: 'tool_use', id: 'r1', name: 'Read', input: { file_path: '/w/a.js' } }] }), + asstTools({ id: 'm2', blocks: [{ type: 'tool_use', id: 'r2', name: 'Read', input: { file_path: '/w/a.js' } }] }), + ].join('\n'), { sessionId: 'w2', project: 'p' }); + assert.strictEqual(dup.waste.redundantReads, 1); + assert.deepStrictEqual(dup.waste.duplicateFiles, { '/w/a.js': 1 }); + + const reset = parseSession([ + asstTools({ id: 'm1', blocks: [{ type: 'tool_use', id: 'r1', name: 'Read', input: { file_path: '/w/a.js' } }] }), + asstTools({ id: 'm2', blocks: [{ type: 'tool_use', id: 'e1', name: 'Edit', input: { file_path: '/w/a.js' } }] }), + asstTools({ id: 'm3', blocks: [{ type: 'tool_use', id: 'r2', name: 'Read', input: { file_path: '/w/a.js' } }] }), + ].join('\n'), { sessionId: 'w3', project: 'p' }); + assert.strictEqual(reset.waste.redundantReads, 0); + assert.deepStrictEqual(reset.waste.duplicateFiles, {}); +}); + +test('waste: a Read after a MultiEdit is not redundant (the file genuinely changed)', () => { + const s = parseSession([ + asstTools({ id: 'm1', blocks: [{ type: 'tool_use', id: 'r1', name: 'Read', input: { file_path: '/w/a.js' } }] }), + asstTools({ id: 'm2', blocks: [{ type: 'tool_use', id: 'e1', name: 'MultiEdit', input: { file_path: '/w/a.js' } }] }), + asstTools({ id: 'm3', blocks: [{ type: 'tool_use', id: 'r2', name: 'Read', input: { file_path: '/w/a.js' } }] }), + ].join('\n'), { sessionId: 'w-multiedit-reset', project: 'p' }); + assert.strictEqual(s.waste.redundantReads, 0); +}); + +test('waste: a Read after a NotebookEdit on the same notebook path is not redundant', () => { + const s = parseSession([ + asstTools({ id: 'm1', blocks: [{ type: 'tool_use', id: 'r1', name: 'Read', input: { file_path: '/w/nb.ipynb' } }] }), + asstTools({ id: 'm2', blocks: [{ type: 'tool_use', id: 'e1', name: 'NotebookEdit', input: { notebook_path: '/w/nb.ipynb' } }] }), + asstTools({ id: 'm3', blocks: [{ type: 'tool_use', id: 'r2', name: 'Read', input: { file_path: '/w/nb.ipynb' } }] }), + ].join('\n'), { sessionId: 'w-notebookedit-reset', project: 'p' }); + assert.strictEqual(s.waste.redundantReads, 0); +}); + +test('waste: reads with offset/limit (pagination) are never flagged redundant', () => { + const s = parseSession([ + asstTools({ id: 'm1', blocks: [{ type: 'tool_use', id: 'r1', name: 'Read', input: { file_path: '/w/big.log', offset: 0, limit: 2000 } }] }), + asstTools({ id: 'm2', blocks: [{ type: 'tool_use', id: 'r2', name: 'Read', input: { file_path: '/w/big.log', offset: 2000, limit: 2000 } }] }), + asstTools({ id: 'm3', blocks: [{ type: 'tool_use', id: 'r3', name: 'Read', input: { file_path: '/w/big.log', offset: 0, limit: 2000 } }] }), // exact repeat of r1's range + ].join('\n'), { sessionId: 'w-paginated', project: 'p' }); + assert.strictEqual(s.waste.redundantReads, 0); + assert.deepStrictEqual(s.waste.duplicateFiles, {}); +}); + +test('waste: a Bash call between two whole-file reads clears redundancy (file may have been mutated)', () => { + const s = parseSession([ + asstTools({ id: 'm1', blocks: [{ type: 'tool_use', id: 'r1', name: 'Read', input: { file_path: '/w/a.js' } }] }), + asstTools({ id: 'm2', blocks: [{ type: 'tool_use', id: 'b1', name: 'Bash', input: { command: 'sed -i s/x/y/ /w/a.js' } }] }), + asstTools({ id: 'm3', blocks: [{ type: 'tool_use', id: 'r2', name: 'Read', input: { file_path: '/w/a.js' } }] }), + ].join('\n'), { sessionId: 'w-bash-reset', project: 'p' }); + assert.strictEqual(s.waste.redundantReads, 0); +}); + +test('waste: sessions with no tool blocks get a zeroed waste object', () => { + const s = parseSession(opusLine, { sessionId: 'w-none', project: 'p' }); + assert.deepStrictEqual(s.waste, { + toolCallCount: 0, erroredToolCalls: 0, erroredByTool: {}, erroredByReason: {}, + errorSamples: {}, redundantReads: 0, duplicateFiles: {}, daily: {}, + }); +}); + +test('waste: a retried errored call keeps a sample with its command and error text', () => { + const text = [ + asstTools({ id: 'm1', blocks: [{ type: 'tool_use', id: 't1', name: 'Bash', input: { command: 'npm run boom' } }] }), + userResults({ blocks: [{ type: 'tool_result', tool_use_id: 't1', is_error: true, content: 'File does not exist' }] }), + asstTools({ id: 'm2', blocks: [{ type: 'tool_use', id: 't2', name: 'Bash', input: { command: 'npm run boom' } }] }), // retry + ].join('\n'); + const s = parseSession(text, { sessionId: 'w-sample', project: 'p' }); + assert.deepStrictEqual(s.waste.errorSamples, { + 'Bash file-not-found': [{ tool: 'Bash', reason: 'file-not-found', target: 'npm run boom', text: 'File does not exist' }], + }); +}); + +test('waste: no sample is kept for an errored call that is never retried', () => { + const text = [ + asstTools({ id: 'm1', blocks: [{ type: 'tool_use', id: 't1', name: 'Bash', input: { command: 'ls /nope' } }] }), + userResults({ blocks: [{ type: 'tool_result', tool_use_id: 't1', is_error: true, content: 'File does not exist' }] }), + ].join('\n'); + const s = parseSession(text, { sessionId: 'w-nosample', project: 'p' }); + assert.deepStrictEqual(s.waste.errorSamples, {}); +}); + +test('waste: error samples redact credentials and cap at 3 distinct per tool+reason', () => { + const lines = []; + // 4 distinct failing commands of the same tool+reason; only 3 should be kept. + // The secret is masked; the trailing tag keeps each command distinct. + for (let i = 0; i < 4; i++) { + lines.push(asstTools({ id: `a${i}`, blocks: [{ type: 'tool_use', id: `t${i}`, name: 'Bash', input: { command: `curl --token deadbeef${i} tag${i}` } }] })); + lines.push(userResults({ blocks: [{ type: 'tool_result', tool_use_id: `t${i}`, is_error: true, content: 'boom' }] })); + lines.push(asstTools({ id: `r${i}`, blocks: [{ type: 'tool_use', id: `x${i}`, name: 'Bash', input: { command: `curl --token deadbeef${i} tag${i}` } }] })); + } + const s = parseSession(lines.join('\n'), { sessionId: 'w-cap', project: 'p' }); + const arr = s.waste.errorSamples['Bash other']; + assert.strictEqual(arr.length, 3); + for (const smp of arr) { + assert.ok(!/deadbeef/.test(smp.target), `token leaked: ${smp.target}`); + assert.ok(smp.target.includes('«redacted»')); + } +}); + +test('redactSecrets masks values but keeps surrounding text', () => { + assert.strictEqual(redactSecrets('export API_KEY=abc123def'), 'export API_KEY=«redacted»'); + assert.strictEqual(redactSecrets('curl --token deadbeef https://x'), 'curl --token «redacted» https://x'); + assert.strictEqual(redactSecrets('-H "Authorization: Bearer xyz"'), '-H "Authorization: «redacted»"'); + assert.strictEqual(redactSecrets('psql postgres://u:p4ss@db'), 'psql postgres://u:«redacted»@db'); + assert.strictEqual(redactSecrets('plain command with no secret'), 'plain command with no secret'); +}); + +test('redactSecrets catches env-var style names where the keyword is glued to a prefix by _', () => { + assert.strictEqual(redactSecrets('export DB_PASSWORD=hunter22222'), 'export DB_PASSWORD=«redacted»'); + assert.strictEqual(redactSecrets('export STRIPE_SECRET_KEY=sk_live_abcdefgh'), 'export STRIPE_SECRET_KEY=«redacted»'); + assert.strictEqual(redactSecrets('export MY_TOKEN=abcdefghijklmnop'), 'export MY_TOKEN=«redacted»'); + assert.strictEqual( + redactSecrets('export ANTHROPIC_API_KEY=sk-ant-api03-abcdefghijklmnopqrstuvwxyz1234567890'), + 'export ANTHROPIC_API_KEY=«redacted»', + ); +}); + +test('redactSecrets masks glued mysql/psql-style -p (no space, no keyword)', () => { + assert.strictEqual(redactSecrets('mysql -uroot -phunter2 db'), 'mysql -uroot -p«redacted» db'); +}); + +test('redactSecrets keeps a bare quote intact when the secret sits inside an already-quoted arg', () => { + assert.strictEqual( + redactSecrets('curl -H "x-api-key: sk-ant-api03-abcdefghijklmnopqrstuvwxyz1234567890"'), + 'curl -H "x-api-key: «redacted»"', + ); +}); + +test('buildResponse flattens error samples across sessions', () => { + const s = parseSession([ + asstTools({ id: 'm1', cwd: '/w/proj', blocks: [{ type: 'tool_use', id: 't1', name: 'Bash', input: { command: 'boom' } }] }), + userResults({ blocks: [{ type: 'tool_result', tool_use_id: 't1', is_error: true, content: 'File does not exist' }] }), + asstTools({ id: 'm2', blocks: [{ type: 'tool_use', id: 't2', name: 'Bash', input: { command: 'boom' } }] }), + ].join('\n'), { sessionId: 'w-br', project: 'p' }); + const r = buildResponse([s]); + assert.deepStrictEqual(r.waste.errorSamples, [ + { tool: 'Bash', reason: 'file-not-found', target: 'boom', text: 'File does not exist' }, + ]); +}); + +test('mergeSessionAggregates merges waste across a main and a subagent file', () => { + const main = parseSession([ + asstTools({ id: 'm1', cwd: '/w/proj', blocks: [ + { type: 'tool_use', id: 't1', name: 'Read', input: { file_path: '/w/a.js' } }, + { type: 'tool_use', id: 't2', name: 'Read', input: { file_path: '/w/a.js' } }, // redundant + { type: 'tool_use', id: 't3', name: 'Bash', input: { command: 'boom' } }, + ] }), + userResults({ blocks: [{ type: 'tool_result', tool_use_id: 't3', is_error: true }] }), + asstTools({ id: 'm2', blocks: [{ type: 'tool_use', id: 't4', name: 'Bash', input: { command: 'boom' } }] }), // retry of t3 + ].join('\n'), { sessionId: 's', project: 'p' }); + const sub = parseSession([ + asstTools({ id: 's1', blocks: [{ type: 'tool_use', id: 'u1', name: 'Bash', input: { command: 'boom' } }] }), + userResults({ blocks: [{ type: 'tool_result', tool_use_id: 'u1', is_error: true }] }), + asstTools({ id: 's2', blocks: [{ type: 'tool_use', id: 'u2', name: 'Bash', input: { command: 'boom' } }] }), // retry of u1 + ].join('\n'), { sessionId: 's', project: 'p' }); + const merged = mergeSessionAggregates([main, sub]); + assert.strictEqual(merged.waste.toolCallCount, 6); + assert.strictEqual(merged.waste.erroredToolCalls, 2); + assert.deepStrictEqual(merged.waste.erroredByTool, { Bash: 2 }); + assert.strictEqual(merged.waste.redundantReads, 1); + assert.deepStrictEqual(merged.waste.duplicateFiles, { '/w/a.js': 1 }); +}); + +test('buildResponse aggregates waste and byProject across sessions in different projects', () => { + const projA = parseSession([ + asstTools({ id: 'a1', cwd: '/w/alpha', blocks: [ + { type: 'tool_use', id: 't1', name: 'Read', input: { file_path: '/w/alpha/x.js' } }, + { type: 'tool_use', id: 't2', name: 'Read', input: { file_path: '/w/alpha/x.js' } }, // redundant + { type: 'tool_use', id: 't3', name: 'Bash', input: { command: 'boom' } }, + ] }), + userResults({ blocks: [{ type: 'tool_result', tool_use_id: 't3', is_error: true }] }), + asstTools({ id: 'a2', blocks: [{ type: 'tool_use', id: 't4', name: 'Bash', input: { command: 'boom' } }] }), // retry of t3 + ].join('\n'), { sessionId: 'A', project: 'p' }); + const projB = parseSession([ + asstTools({ id: 'b1', cwd: '/w/beta', blocks: [ + { type: 'tool_use', id: 'u1', name: 'Edit', input: { file_path: '/w/beta/y.js' } }, + { type: 'tool_use', id: 'u2', name: 'Bash', input: { command: 'boom' } }, + ] }), + userResults({ blocks: [ + { type: 'tool_result', tool_use_id: 'u1', is_error: true }, + { type: 'tool_result', tool_use_id: 'u2', is_error: true }, + ] }), + asstTools({ id: 'b2', blocks: [ + { type: 'tool_use', id: 'u3', name: 'Edit', input: { file_path: '/w/beta/y.js' } }, // retry of u1 + { type: 'tool_use', id: 'u4', name: 'Bash', input: { command: 'boom' } }, // retry of u2 + ] }), + ].join('\n'), { sessionId: 'B', project: 'p' }); + + const r = buildResponse([projA, projB]); + assert.strictEqual(r.waste.erroredToolCalls, 3); // 1 + 2 + assert.strictEqual(r.waste.redundantReads, 1); + assert.strictEqual(r.waste.duplicateFileCount, 1); + assert.deepStrictEqual(r.waste.erroredByTool, [{ name: 'Bash', count: 2 }, { name: 'Edit', count: 1 }]); + assert.deepStrictEqual(r.waste.topDuplicateFiles, [{ path: '/w/alpha/x.js', extraReads: 1 }]); + // byProject sorted by (errored + redundant) desc: beta has 2, alpha has 1+1=2 → tie, both present + assert.strictEqual(r.waste.byProject.length, 2); + const beta = r.waste.byProject.find((p) => p.project === '/w/beta'); + const alpha = r.waste.byProject.find((p) => p.project === '/w/alpha'); + assert.deepStrictEqual(beta, { project: '/w/beta', erroredToolCalls: 2, redundantReads: 0 }); + assert.deepStrictEqual(alpha, { project: '/w/alpha', erroredToolCalls: 1, redundantReads: 1 }); +}); + +test('buildResponse aggregates waste into a per-day trend, merged across projects', () => { + const projA = parseSession([ + asstTools({ id: 'a1', ts: '2026-07-01T10:00:00.000Z', cwd: '/w/alpha', blocks: [ + { type: 'tool_use', id: 't1', name: 'Read', input: { file_path: '/w/alpha/x.js' } }, + { type: 'tool_use', id: 't2', name: 'Read', input: { file_path: '/w/alpha/x.js' } }, // redundant, day 1 + ] }), + ].join('\n'), { sessionId: 'A', project: 'p' }); + const projB = parseSession([ + asstTools({ id: 'b1', ts: '2026-07-01T11:00:00.000Z', cwd: '/w/beta', blocks: [ + { type: 'tool_use', id: 'u1', name: 'Bash', input: { command: 'boom' } }, + ] }), + userResults({ ts: '2026-07-01T11:01:00.000Z', blocks: [{ type: 'tool_result', tool_use_id: 'u1', is_error: true }] }), + asstTools({ id: 'b2', ts: '2026-07-02T09:00:00.000Z', blocks: [ + { type: 'tool_use', id: 'u2', name: 'Bash', input: { command: 'boom' } }, // retry, day 2 — waste attributed to retry's day + ] }), + ].join('\n'), { sessionId: 'B', project: 'p' }); + + const r = buildResponse([projA, projB]); + assert.deepStrictEqual(r.waste.trend, [ + { date: '2026-07-01', erroredToolCalls: 0, redundantReads: 1 }, + { date: '2026-07-02', erroredToolCalls: 1, redundantReads: 0 }, + ]); +}); + +// ---- waste: error-reason classification ---- +// Verbatim (truncated) strings pulled from real Claude Code session logs, so +// the matcher is proven against actual message shapes, not invented ones. +test('classifyErrorReason: known Claude Code error strings map to specific reasons', () => { + assert.strictEqual( + classifyErrorReason('File has not been read yet. Read it first before writing to it.'), + 'edit-before-read', + ); + assert.strictEqual( + classifyErrorReason("The user doesn't want to proceed with this tool use. The tool use was rejected"), + 'user-rejected', + ); + assert.strictEqual( + classifyErrorReason('String to replace not found in file.\nString: def foo():'), + 'edit-string-not-found', + ); + assert.strictEqual( + classifyErrorReason('File has been modified since read, either by the user or by a linter.'), + 'stale-read', + ); + assert.strictEqual( + classifyErrorReason('File does not exist. Note: your current working directory is /Users/x/proj.'), + 'file-not-found', + ); + assert.strictEqual( + classifyErrorReason('Permission for this action was denied by the Claude Code auto mode classifier. Reason: ...'), + 'auto-mode-denied', + ); + assert.strictEqual( + classifyErrorReason('claude-opus-4-8[1m] is temporarily unavailable, so auto mode cannot determine the safety'), + 'model-unavailable', + ); + assert.strictEqual( + classifyErrorReason('Working directory "/x" was deleted; shell cwd recovered to "/Users/x"'), + 'cwd-deleted', + ); +}); + +test('classifyErrorReason: unrecognized text (e.g. a bare shell exit code) is "other", not guessed', () => { + assert.strictEqual(classifyErrorReason('Exit code 1\nsome command output here'), 'other'); + assert.strictEqual(classifyErrorReason(''), 'other'); + assert.strictEqual(classifyErrorReason(undefined), 'other'); +}); + +test('classifyErrorReason: handles array-of-text-block content shape, not just plain strings', () => { + const content = [{ type: 'text', text: 'File does not exist. Note: cwd is /x' }]; + assert.strictEqual(classifyErrorReason(content), 'file-not-found'); +}); + +test('waste: erroredByReason is attributed at retry time and aggregated in buildResponse', () => { + const s = parseSession([ + asstTools({ id: 'm1', blocks: [{ type: 'tool_use', id: 't1', name: 'Edit', input: { file_path: '/w/a.js' } }] }), + userResults({ blocks: [{ + type: 'tool_result', tool_use_id: 't1', is_error: true, + content: 'String to replace not found in file.', + }] }), + asstTools({ id: 'm2', blocks: [{ type: 'tool_use', id: 't2', name: 'Edit', input: { file_path: '/w/a.js' } }] }), // retry + ].join('\n'), { sessionId: 'w-reason', project: 'p' }); + assert.deepStrictEqual(s.waste.erroredByReason, { 'edit-string-not-found': 1 }); + + const r = buildResponse([s]); + assert.deepStrictEqual(r.waste.erroredByReason, [{ reason: 'edit-string-not-found', count: 1 }]); +}); diff --git a/test/server.test.js b/test/server.test.js index f198fd8..28eba33 100644 --- a/test/server.test.js +++ b/test/server.test.js @@ -3,7 +3,7 @@ const { test } = require('node:test'); const assert = require('node:assert'); const path = require('node:path'); // Requiring server.js does not start a listener (guarded by require.main). -const { sessionKeyFor, resolveAssetPath } = require('../server'); +const { sessionKeyFor, resolveAssetPath, server, HOST } = require('../server'); test('sessionKeyFor: main file vs nested subagent file share one key', () => { const main = sessionKeyFor('proj-dir', 'abc123.jsonl'); @@ -23,3 +23,11 @@ test('resolveAssetPath: serves under assets, rejects traversal', () => { assert.strictEqual(resolveAssetPath('/assets/../secret'), null); assert.strictEqual(resolveAssetPath('/assets/../../../etc/passwd'), null); }); + +test('server binds to loopback only, not every interface', () => new Promise((resolve) => { + assert.strictEqual(HOST, '127.0.0.1'); + server.listen(0, HOST, () => { + assert.strictEqual(server.address().address, '127.0.0.1'); + server.close(resolve); + }); +})); diff --git a/web/package-lock.json b/web/package-lock.json index 75f4da9..7a01a08 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -1,12 +1,12 @@ { "name": "claude-code-cost-dashboard-web", - "version": "1.0.0", + "version": "2.0.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "claude-code-cost-dashboard-web", - "version": "1.0.0", + "version": "2.0.1", "dependencies": { "react": "^19.2.7", "react-dom": "^19.2.7" diff --git a/web/package.json b/web/package.json index 27ff6f0..a3848d3 100644 --- a/web/package.json +++ b/web/package.json @@ -1,7 +1,7 @@ { "name": "claude-code-cost-dashboard-web", "private": true, - "version": "2.0.0", + "version": "2.0.1", "license": "MIT", "type": "module", "engines": { diff --git a/web/src/App.jsx b/web/src/App.jsx index cafd9a1..fd0c008 100644 --- a/web/src/App.jsx +++ b/web/src/App.jsx @@ -8,9 +8,10 @@ import ProjectsTable from './components/ProjectsTable.jsx'; import ModelsTable from './components/ModelsTable.jsx'; import AdvisorTable from './components/AdvisorTable.jsx'; import SessionsTable from './components/SessionsTable.jsx'; +import WasteTable from './components/WasteTable.jsx'; import Diagnostics from './components/Diagnostics.jsx'; -const TABS = ['overview', 'breakdown', 'advisor', 'sessions']; +const TABS = ['overview', 'breakdown', 'advisor', 'waste', 'sessions']; function currentTab() { const h = window.location.hash.slice(1); @@ -91,6 +92,13 @@ export default function App() { )} + {tab === 'waste' && ( + <> +

Repeated waste across sessions

+ + + )} + {tab === 'sessions' && ( <>

Sessions

diff --git a/web/src/components/AdvisorTable.jsx b/web/src/components/AdvisorTable.jsx index d19d87e..795df92 100644 --- a/web/src/components/AdvisorTable.jsx +++ b/web/src/components/AdvisorTable.jsx @@ -13,7 +13,7 @@ export default function AdvisorTable({ rows }) { Session Date Cost - Est. saving + Est. capacity Reasons @@ -27,7 +27,10 @@ export default function AdvisorTable({ rows }) { {a.estSavingUSD ? fmtUSD(a.estSavingUSD) : '—'} {a.reasons.map((r, i) => ( -
{r}
+
+
{r.rule} {r.text}
+
{r.action}
+
))} diff --git a/web/src/components/TabNav.jsx b/web/src/components/TabNav.jsx index 57fee99..da50463 100644 --- a/web/src/components/TabNav.jsx +++ b/web/src/components/TabNav.jsx @@ -2,6 +2,7 @@ const LABELS = { overview: 'Overview', breakdown: 'Breakdown', advisor: 'Advisor', + waste: 'Waste', sessions: 'Sessions', }; diff --git a/web/src/components/Tiles.jsx b/web/src/components/Tiles.jsx index e72c92f..f20f728 100644 --- a/web/src/components/Tiles.jsx +++ b/web/src/components/Tiles.jsx @@ -37,6 +37,7 @@ export default function Tiles({ summary, roi }) {
{monthName(last.month)} value vs ${roi.subscriptionUSDPerMonth}/mo plan + {!roi.configured && ' (default)'} {last.multiple.toFixed(1)} @@ -65,6 +66,8 @@ export default function Tiles({ summary, roi }) {
{monthName(last.month)} returned {fmtUSD(last.valueUSD)} of API-equivalent value. + {!roi.configured && + ` Multiple assumes a $${roi.subscriptionUSDPerMonth}/mo plan — set subscriptionUSDPerMonth in config.json to match yours.`}
)} diff --git a/web/src/components/WasteTable.jsx b/web/src/components/WasteTable.jsx new file mode 100644 index 0000000..01115d6 --- /dev/null +++ b/web/src/components/WasteTable.jsx @@ -0,0 +1,195 @@ +import { shortProject } from '../format.js'; +import WasteTrend from './WasteTrend.jsx'; + +// Human labels for the reason slugs classifyErrorReason() (lib/core.js) emits. +// 'other' covers everything not confidently matched (e.g. a bare shell exit +// code) — left uncategorized rather than guessed. +const REASON_LABELS = { + 'edit-before-read': 'Edited a file before reading it', + 'edit-string-not-found': 'Edit: replacement text not found', + 'stale-read': 'File changed since it was read', + 'file-not-found': 'File does not exist', + 'user-rejected': 'Rejected by user', + 'auto-mode-denied': 'Blocked by auto mode', + 'model-unavailable': 'Model temporarily unavailable', + 'cwd-deleted': 'Working directory was deleted', + other: 'Other (e.g. shell exit code)', +}; + +// A one-liner shown in both states so a first-time viewer knows what "waste" +// means and why the counts matter before reading any number. +function Intro() { + return ( + <> +

+ Tool calls that failed and were retried, and files{' '} + re-read when the answer was already in context — quota you + paid for but didn't need. Shown as counts, not dollars: token usage is + logged per message, not per tool call. +

+

+ How it's measured: an errored tool call is a tool result the log marks + as an error, counted only when the same tool is called again afterward + (a retry); a redundant read is the same file read whole again with no + edit or shell command in between (partial reads and re-reads after a + change are expected, so they aren't counted). +

+ + ); +} + +// Exact counts only — token usage is per-message, not per-tool-block, so there +// is no honest per-tool dollar figure to show here. +export default function WasteTable({ rows }) { + if (!rows || (!rows.erroredToolCalls && !rows.redundantReads)) { + return ( + <> + +
No repeated waste detected — tool calls mostly succeeded and context was reused.
+ + ); + } + + return ( + <> + +

+ {rows.erroredToolCalls.toLocaleString('en-US')} errored tool calls ·{' '} + {rows.redundantReads.toLocaleString('en-US')} redundant file reads across{' '} + {rows.duplicateFileCount.toLocaleString('en-US')} file + {rows.duplicateFileCount === 1 ? '' : 's'}. +

+ + + + {rows.erroredByTool.length > 0 && ( + <> +

Errored tool calls by tool

+
+ + + + + + + + + {rows.erroredByTool.map((t) => ( + + + + + ))} + +
ToolErrors
{t.name}{t.count}
+
+ + )} + + {rows.erroredByReason.length > 0 && ( + <> +

Errored tool calls by reason

+
+ + + + + + + + + {rows.erroredByReason.map((r) => ( + + + + + ))} + +
ReasonErrors
{REASON_LABELS[r.reason] || r.reason}{r.count}
+
+ + )} + + {rows.errorSamples && rows.errorSamples.length > 0 && ( + <> +

What the errors were

+

+ A few concrete examples per failure kind — the command or file that + failed and the message it returned. Recognizable credentials (keyed + flags, tokens, auth headers) are redacted — review before sharing. +

+
+ + + + + + + + + + {rows.errorSamples.map((e, i) => ( + + + + + + ))} + +
ToolCommand / targetWhat went wrong
{e.tool}{e.target || '—'}{REASON_LABELS[e.reason] || e.reason}{e.text ? `: ${e.text}` : ''}
+
+ + )} + + {rows.topDuplicateFiles.length > 0 && ( + <> +

Most re-read files

+
+ + + + + + + + + {rows.topDuplicateFiles.map((f) => ( + + + + + ))} + +
FileRedundant reads
{shortProject(f.path)}{f.extraReads}
+
+ + )} + + {rows.byProject.length > 0 && ( + <> +

By project

+
+ + + + + + + + + + {rows.byProject.map((p) => ( + + + + + + ))} + +
ProjectErrored callsRedundant reads
{shortProject(p.project)}{p.erroredToolCalls}{p.redundantReads}
+
+ + )} + + ); +} diff --git a/web/src/components/WasteTrend.jsx b/web/src/components/WasteTrend.jsx new file mode 100644 index 0000000..afcfbb1 --- /dev/null +++ b/web/src/components/WasteTrend.jsx @@ -0,0 +1,74 @@ +const W = 1080; +const H = 160; +const PAD = { t: 10, r: 6, b: 22, l: 34 }; +const GAP = 2; +const DAYS = 30; + +const pad2 = (n) => String(n).padStart(2, '0'); +const dateKey = (d) => `${d.getFullYear()}-${pad2(d.getMonth() + 1)}-${pad2(d.getDate())}`; +const parse = (s) => { const [y, m, dd] = s.split('-').map(Number); return new Date(y, m - 1, dd); }; + +// Zero-filled last-30-days series so gaps read as empty bars, not a squeezed timeline. +function buildSeries(trend) { + if (!trend.length) return []; + const newest = parse(trend[trend.length - 1].date); + const byKey = new Map(trend.map((d) => [d.date, d])); + const out = []; + for (let i = DAYS - 1; i >= 0; i--) { + const cur = new Date(newest); + cur.setDate(newest.getDate() - i); + const key = dateKey(cur); + const d = byKey.get(key) || { erroredToolCalls: 0, redundantReads: 0 }; + out.push({ key, label: key.slice(5), erroredToolCalls: d.erroredToolCalls, redundantReads: d.redundantReads }); + } + return out; +} + +// Stacked daily bars: errored tool calls (retried) below, redundant reads above. +// A quick way to see whether waste is trending up or down, not just the total. +export default function WasteTrend({ trend }) { + const rows = buildSeries(trend || []); + if (!rows.length) return null; + + const max = Math.max(...rows.map((d) => d.erroredToolCalls + d.redundantReads), 1); + const plotW = W - PAD.l - PAD.r; + const plotH = H - PAD.t - PAD.b; + const band = plotW / rows.length; + const barW = Math.max(band - GAP, 1); + const labelStep = Math.max(1, Math.ceil(rows.length / 10)); + + return ( +
+
+ Waste trend — last {DAYS} days + + errored calls redundant reads + +
+ + + {rows.map((d, i) => { + const x = PAD.l + i * band + (band - barW) / 2; + const errH = (plotH * d.erroredToolCalls) / max; + const redH = (plotH * d.redundantReads) / max; + const errY = H - PAD.b - errH; + const redY = errY - redH; + return ( + + {d.erroredToolCalls > 0 && } + {d.redundantReads > 0 && } + {(d.erroredToolCalls > 0 || d.redundantReads > 0) && ( + {`${d.key}: ${d.erroredToolCalls} errored, ${d.redundantReads} redundant`} + )} + {i % labelStep === 0 && ( + + {d.label} + + )} + + ); + })} + +
+ ); +} diff --git a/web/src/styles.css b/web/src/styles.css index 2f5605d..ddccb39 100644 --- a/web/src/styles.css +++ b/web/src/styles.css @@ -16,6 +16,7 @@ --accent-chart: #5b8def; --accent-soft: rgba(91, 141, 239, 0.12); --good: #1f8a55; + --danger: #c1443c; --sans: -apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, sans-serif; --mono: ui-monospace, "SF Mono", "JetBrains Mono", Menlo, Consolas, monospace; @@ -38,6 +39,7 @@ --accent-chart: #5b8def; --accent-soft: rgba(91, 141, 239, 0.16); --good: #47b37e; + --danger: #e0776f; } } @@ -121,6 +123,30 @@ body { background: var(--border); } +.waste-intro { + color: var(--muted); + font-size: 13px; + line-height: 1.6; + margin: 4px 0 8px; + max-width: 60ch; +} + +.waste-method { + color: var(--muted); + font-size: 12px; + line-height: 1.6; + margin: 0 0 12px; + max-width: 60ch; + opacity: 0.85; +} + +.waste-summary { + color: var(--muted); + font-size: 13px; + line-height: 1.5; + margin: 4px 0 0; +} + /* ---- Hero / meter panel ---- */ .hero { background: var(--surface); @@ -228,6 +254,21 @@ tr.detail > td { background: var(--surface-2); padding: 0; } .share-cell { color: var(--muted); } +.advisor-action { color: var(--muted); font-size: 12px; margin-top: 2px; } + +.advisor-rule { + display: inline-block; + font-size: 10px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.02em; + color: var(--muted); + background: var(--surface-2); + border-radius: 3px; + padding: 1px 5px; + margin-right: 4px; +} + /* ---- Prompt timeline ---- */ .timeline-label { font: 600 11px/1 var(--sans); @@ -325,6 +366,13 @@ tr.detail > td { background: var(--surface-2); padding: 0; } .chart-wrap .grid-line { stroke: var(--border); stroke-width: 1; } .chart-wrap .bar { fill: var(--accent-chart); transition: fill 0.1s; } .chart-wrap .bar.is-hover { fill: var(--accent); } +.chart-wrap .bar-error { fill: var(--danger); } +.chart-wrap .bar-redundant { fill: var(--accent-chart); } +.chart-legend { color: var(--muted); font-size: 12px; display: inline-flex; align-items: center; gap: 5px; } +.chart-legend .dot { width: 8px; height: 8px; border-radius: 50%; display: inline-block; margin-left: 10px; } +.chart-legend .dot:first-child { margin-left: 0; } +.chart-legend .dot-error { background: var(--danger); } +.chart-legend .dot-redundant { background: var(--accent-chart); } .tooltip { position: absolute; pointer-events: none;