diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 1796db0..7c1c368 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -6,7 +6,7 @@ "plugins": [ { "name": "docs-assist", - "version": "0.9.6", + "version": "0.9.7", "description": "A documentation coach for Claude Code. Guides subject matter experts through contributing their knowledge: you bring the expertise, the plugin handles the writing. Also provides content audits, style enforcement, and docs-as-code workflows.", "author": { "name": "Edward Angert" diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 05c7f9c..f8c533f 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "docs-assist", - "version": "0.9.6", + "version": "0.9.7", "description": "A documentation coach for Claude Code. Guides subject matter experts through contributing their knowledge: you bring the expertise, the plugin handles the writing. Also provides content audits, style enforcement, and docs-as-code workflows.", "author": { "name": "Edward Angert" diff --git a/CHANGELOG.md b/CHANGELOG.md index f801867..05b0a65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,21 @@ All notable changes to this project are documented here. The format is based on Keep a Changelog, and the project follows Semantic Versioning. +## 0.9.7 - 2026-07-20 + +`check-facts.mjs` verifies the curated `.docs-assist/reference.yml` registry deterministically; this release extends the same approach, cold, to the whole doc set. + +### Added + +- `check-claims.mjs` and `claim-briefs.mjs`: a deterministic, dependency-free pair that mechanizes the first half of `claim-verification.md`'s method across the whole doc set. `check-claims.mjs` extracts identifier-shaped claims (CLI commands/flags, config/env keys, function or class names, file paths, version requirements) from every doc and resolves each against the code with `git grep`/`git ls-files`, no agent involved; `claim-briefs.mjs` turns what's left (described-behavior and numeric claims a lookup can't settle) into one self-contained brief per doc for the `doc-auditor` fan-out. + `/docs-assist:audit` and `claim-verification.md` now point at it as the recommended first pass before tracing claims by hand. + `/docs-assist:setup-hooks` gained a CI claim check (`assets/ci/github/check-claims.yml`), the same sticky-PR-comment pattern as the reference-registry check, non-strict by default since a "missing" result can also mean the claim's target is real but gitignored. + Born from a real cross-project run: extracted and mechanically resolved 453 candidate claims from a 14-doc corpus in seconds, fanned the 192 that needed judgment out to 14 parallel agents, and found 6 real drifted or inconsistent claims a lint pass alone would have missed. + +### Version Policy + +- Bumped to 0.9.7 on the maintainer's explicit call, the same policy as 0.9.6: agent-driven work caps at 0.9.5 by default, and a version bump past that is always the maintainer's decision, not the agent's. + ## 0.9.6 - 2026-07-19 This release asks whether the docs actually work, not just whether they read well. diff --git a/assets/ci/check-claims.mjs b/assets/ci/check-claims.mjs new file mode 100644 index 0000000..f348aba --- /dev/null +++ b/assets/ci/check-claims.mjs @@ -0,0 +1,269 @@ +#!/usr/bin/env node +// Docs Assist claim checker. +// +// Deterministic, dependency-free: extracts checkable claims (CLI commands and +// flags, config/env keys, function or class names, file paths, version +// requirements, numeric or described behavior, external links) from every doc +// in scope, then resolves the identifier-shaped ones (paths, flags, names, +// keys) against the code with `git grep` and `git ls-files`. This is the +// mechanical half of claim-verification.md's method, applied to the whole +// corpus instead of one claim at a time; `check-facts.mjs` does the same job +// for the curated `.docs-assist/reference.yml` registry, this does it for +// everything else, extraction included. +// +// What it cannot settle stays unsettled on purpose: described behavior +// ("retries three times", "commits atomically") and numeric/runtime claims +// need a reader, not a lookup. Those are written to claims-needs-judgment.json, +// grouped by doc, for claim-briefs.mjs to turn into one agent brief per doc. +// +// Usage: node check-claims.mjs [docsDir] [outDir] +// Env: +// DOCS_DIR docs directory (default: docs_dir from +// .docs-assist/config.yml, else "docs") +// CHECK_CLAIMS_OUT output directory (default: ".docs-assist/claims") +// CHECK_CLAIMS_STRICT "1" exits nonzero when any claim resolves "missing" +// GITHUB_STEP_SUMMARY when set, the report is appended there too +// +// A claim that resolves "missing" means grep found nothing anywhere in the +// tracked tree, not that the doc is necessarily wrong: a generated-artifact +// filename or a doc-only placeholder token can look identical to real drift +// from a regex's point of view. Those are the two cases this script actively +// tries to rule out before calling something missing (see checkFilePath and +// checkConfigKey below); anything left over after that is worth a human or +// agent second look, not an auto-fix. + +import { readFileSync, existsSync, appendFileSync, mkdirSync, writeFileSync } from 'node:fs'; +import { execFileSync } from 'node:child_process'; + +function sh(args) { + try { return execFileSync(args[0], args.slice(1), { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }).trim(); } + catch (e) { return e.stdout ? String(e.stdout).trim() : ''; } +} + +function docsDir() { + if (process.argv[2]) return process.argv[2]; + if (process.env.DOCS_DIR) return process.env.DOCS_DIR; + try { + const m = readFileSync('.docs-assist/config.yml', 'utf8').match(/^docs_dir:\s*(\S+)/m); + if (m) return m[1]; + } catch { /* no config */ } + return 'docs'; +} + +function mdFiles(dir) { + const out = []; + const entries = sh(['git', 'ls-files', '--', dir]); + for (const f of entries.split('\n')) if (/\.mdx?$/.test(f)) out.push(f); + return out; +} + +const DOCS_DIR = docsDir(); +const OUT_DIR = process.argv[3] || process.env.CHECK_CLAIMS_OUT || '.docs-assist/claims'; +mkdirSync(OUT_DIR, { recursive: true }); + +const docs = mdFiles(DOCS_DIR); +for (const extra of ['README.md', 'AGENTS.md', 'CLAUDE.md', 'llms.txt']) { + if (existsSync(extra)) docs.push(extra); +} + +function report(text) { + console.log(text); + if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, text + '\n'); +} + +if (!docs.length) { + report(`## Claim check\n\nNo docs found under \`${DOCS_DIR}\`. Nothing to check.\n`); + process.exit(0); +} + +// (category, pattern): pattern's group 0 is the matched claim text. +const PATTERNS = [ + ['file-path', /`[\w./-]+\.(?:py|js|mjs|cjs|ts|tsx|jsx|go|rb|rs|java|kt|c|h|cpp|hpp|md|mdx|yml|yaml|json|jsonc|toml|ini|sh|Dockerfile)`/g], + ['bare-path', /`(?:src|lib|scripts|docs|deploy|config|bin)\/[\w./-]+`/g], + ['cli-command', /`[\w.-]+\s+(?:run|exec)\s+[\w.-]+(?:\s+[\w.<>-]+)*`/g], + ['cli-flag', /`--[\w-]+(?:[= ][^`]*)?`|`-[a-zA-Z]\b[^`]*`/g], + ['identifier-call', /`[A-Za-z_][\w.]*\(\)?`/g], + ['env-or-config-key', /`[A-Z][A-Z0-9_]{2,}`/g], + ['yaml-key', /`[a-z][a-z0-9_]*:\s*[\w./"'-]*`/g], + ['version-requirement', /\b(?:Python|Node(?:\.js)?|Go|Ruby|Rust|npm|Docker|Kubernetes)\s+v?[\d.]+\+?\b/gi], + ['external-link', /https?:\/\/[^\s)\]]+/g], + ['numeric-behavior', /\b\d+(?:\.\d+)?\s*(?:x|%|seconds?|minutes?|hours?|days?|bits?|bytes?|retries|times?)\b/gi], + ['described-behavior', /\b(?:always|never|must|defaults? to|retries?|returns?|raises?|throws?|rejects?|refuses?|guarantees?|atomically|silently|automatically)\b/gi], + ['issue-reference', /#\d{3,}\b/g], +]; + +// Categories a lookup can settle without judgment. +const MECHANICAL = new Set(['file-path', 'bare-path', 'cli-flag', 'cli-command', 'identifier-call', 'env-or-config-key', 'yaml-key']); + +function extractFromDoc(doc) { + const claims = []; + const lines = readFileSync(doc, 'utf8').split('\n'); + let inFence = false; + lines.forEach((raw, i) => { + const line = raw.trim(); + if (/^(```|~~~)/.test(line)) { inFence = !inFence; return; } + if (inFence) return; + const spans = []; + for (const [category, pattern] of PATTERNS) { + pattern.lastIndex = 0; + let m; + while ((m = pattern.exec(raw))) { + const span = [m.index, m.index + m[0].length]; + if (spans.some(([s, e]) => span[0] < e && span[1] > s)) continue; + spans.push(span); + claims.push({ doc, line: i + 1, category, matched: m[0], context: line }); + } + } + }); + return claims; +} + +const allClaims = docs.flatMap(extractFromDoc); + +// --- Mechanical resolution --------------------------------------------- + +// All tracked paths, for basename lookups. `git ls-files -- '**/foo'` looks +// like it should do this, but `**` needs `:(glob)` pathspec magic that isn't +// on by default, so it silently matches nothing; filtering the full listing +// in JS sidesteps the pathspec-magic footgun entirely. +const allTrackedFiles = sh(['git', 'ls-files']).split('\n').filter(Boolean); + +const gitGrepCache = new Map(); +function gitGrepHits(pattern) { + if (gitGrepCache.has(pattern)) return gitGrepCache.get(pattern); + // -e marks the pattern explicitly: without it, a pattern starting with + // "-" (any CLI flag) is misparsed as a git-grep option and dumps usage. + const out = sh(['git', 'grep', '-n', '-F', '-w', '-e', pattern, '--', ':!*.md', ':!*.mdx']); + const hits = out ? out.split('\n').filter(Boolean) : []; + gitGrepCache.set(pattern, hits); + return hits; +} + +// A literal (non-word-boundary) substring search, for filenames and flags +// that aren't standalone identifiers (e.g. "validation-summary.md", "--foo=bar"). +function gitGrepLiteral(pattern) { + const key = `lit:${pattern}`; + if (gitGrepCache.has(key)) return gitGrepCache.get(key); + const out = sh(['git', 'grep', '-n', '-F', '-e', pattern, '--', ':!*.md', ':!*.mdx']); + const hits = out ? out.split('\n').filter(Boolean) : []; + gitGrepCache.set(key, hits); + return hits; +} + +function checkFilePath(matched, context) { + const pathStr = matched.replace(/`/g, ''); + if (existsSync(pathStr)) return ['confirmed', `exists at ${pathStr}`]; + const base = pathStr.split('/').pop(); + const byBasename = allTrackedFiles.find((f) => f === base || f.endsWith(`/${base}`)); + if (byBasename) return ['confirmed', `found at ${byBasename}`]; + // Not on disk doesn't settle it: a runtime-generated artifact (a report the + // tool writes) is often named as a string literal in source without ever + // being checked in. Confirm the name is real that way before calling it + // missing, and only report "missing" outright when neither check backs it. + const literalHits = gitGrepLiteral(pathStr.length > 3 ? pathStr : base); + if (literalHits.length) return ['confirmed', `named as a generated-artifact filename in source: ${literalHits[0]}`]; + const generativeWords = ['writes', 'written', 'generates', 'generated', 'creates', 'created', 'produces', 'produced', 'output', 'outputs', 'saves', 'saved']; + if (generativeWords.some((w) => context.toLowerCase().includes(w))) { + return ['needs-judgment', `no file found for ${pathStr}, but context suggests a generated artifact: verify by running the tool, not by grep`]; + } + return ['missing', `no file named ${pathStr} found on disk, by basename, or as a string literal in tracked source`]; +} + +function checkIdentifierCall(matched) { + let name = matched.replace(/`/g, '').replace(/\(.*$/, ''); + name = name.split('.').pop(); + if (!name) return ['needs-judgment', 'empty identifier after normalization']; + const hits = gitGrepHits(name); + if (hits.length) return ['confirmed', `'${name}' found: ${hits[0]}`]; + return ['missing', `'${name}' not found anywhere in tracked non-doc source`]; +} + +function checkCliFlag(matched) { + const flag = matched.replace(/`/g, '').split(/[ =]/)[0]; + if (!flag.startsWith('-')) return ['needs-judgment', 'not a real flag token']; + const bare = flag.replace(/^-+/, ''); + if (bare.includes('_')) { + return ['needs-judgment', 'underscore in a dashed flag suggests a placeholder, not a literal flag']; + } + let hits = gitGrepLiteral(flag); + if (hits.length) return ['confirmed', `${flag} found as a literal: ${hits[0]}`]; + // Frameworks that derive a CLI flag from a parameter/field name (typer, + // click, clap, cobra) often have no literal string for the dashed form at + // all; check the underscore-normalized identifier too before giving up. + const param = bare.replace(/-/g, '_'); + hits = gitGrepHits(param); + if (hits.length) return ['confirmed', `${flag} inferred from parameter/field '${param}': ${hits[0]}`]; + return ['missing', `${flag} not found as a literal or as parameter/field '${param}'`]; +} + +function checkConfigKey(matched) { + const key = matched.replace(/`/g, '').split(':')[0].trim(); + const hits = gitGrepHits(key); + if (hits.length) return ['confirmed', `'${key}' found: ${hits[0]}`]; + // SCREAMING_SNAKE backtick tokens are also used as doc-only placeholders + // for a CLI positional arg (`DATASET_URL` standing in for ``); + // a doc using it as `` elsewhere confirms that reading, not drift. + const placeholderHits = gitGrepLiteral(`<${key}>`); + if (placeholderHits.length) { + return ['needs-judgment', `'${key}' not found in code, but used as a <${key}> placeholder elsewhere: likely doc shorthand, not a real key`]; + } + return ['missing', `'${key}' not found anywhere in tracked non-doc source`]; +} + +function checkCliCommand(matched) { + const tokens = matched.replace(/`/g, '').split(/\s+/).filter((t) => !/^<.*>$/.test(t)); + const last = tokens.at(-1); + if (!last) return ['needs-judgment', 'could not extract a command token']; + const hits = gitGrepHits(last) || gitGrepHits(last.replace(/-/g, '_')); + if (hits.length) return ['confirmed', `'${last}' found: ${hits[0]}`]; + return ['missing', `'${last}' not found anywhere in tracked non-doc source`]; +} + +const CHECKERS = { + 'file-path': (c) => checkFilePath(c.matched, c.context), + 'bare-path': (c) => checkFilePath(c.matched, c.context), + 'identifier-call': (c) => checkIdentifierCall(c.matched), + 'cli-flag': (c) => checkCliFlag(c.matched), + 'cli-command': (c) => checkCliCommand(c.matched), + 'env-or-config-key': (c) => checkConfigKey(c.matched), + 'yaml-key': (c) => checkConfigKey(c.matched), +}; + +let confirmed = 0, missing = 0, resolved = 0; +for (const c of allClaims) { + const checker = CHECKERS[c.category]; + if (!checker) { c.status = 'needs-judgment'; c.evidence = ''; continue; } + const [status, evidence] = checker(c); + c.status = status; + c.evidence = evidence; + resolved++; + if (status === 'confirmed') confirmed++; + else if (status === 'missing') missing++; +} + +writeFileSync(`${OUT_DIR}/claims.json`, JSON.stringify(allClaims, null, 2) + '\n'); + +const needsJudgment = allClaims.filter((c) => c.status === 'needs-judgment'); +const byDoc = {}; +for (const c of needsJudgment) (byDoc[c.doc] ??= []).push(c); +writeFileSync(`${OUT_DIR}/claims-needs-judgment.json`, JSON.stringify(byDoc, null, 2) + '\n'); + +const missingClaims = allClaims.filter((c) => c.status === 'missing'); + +let out = `## Claim check\n\n`; +out += `${docs.length} docs, ${allClaims.length} candidate claims (${resolved} mechanically checkable: ${confirmed} confirmed, ${missing} missing, ${resolved - confirmed - missing} demoted to judgment). `; +out += `${needsJudgment.length} require a reader, grouped by doc in \`${OUT_DIR}/claims-needs-judgment.json\`: hand those to \`claim-briefs.mjs\`.\n\n`; +if (missingClaims.length) { + out += `### Missing (code doesn't back the claim anymore)\n\n`; + out += `| Doc | Line | Category | Matched | Evidence |\n| --- | ---: | --- | --- | --- |\n`; + for (const c of missingClaims) { + out += `| \`${c.doc}\` | ${c.line} | ${c.category} | \`${c.matched}\` | ${c.evidence} |\n`; + } + out += '\n'; +} else { + out += `No claims resolved "missing" this run.\n\n`; +} +out += `Full claim set: \`${OUT_DIR}/claims.json\`.\n`; + +report(out); +if (missingClaims.length && process.env.CHECK_CLAIMS_STRICT === '1') process.exit(1); diff --git a/assets/ci/claim-briefs.mjs b/assets/ci/claim-briefs.mjs new file mode 100644 index 0000000..71b6ffe Binary files /dev/null and b/assets/ci/claim-briefs.mjs differ diff --git a/assets/ci/github/check-claims.yml b/assets/ci/github/check-claims.yml new file mode 100644 index 0000000..836b09a --- /dev/null +++ b/assets/ci/github/check-claims.yml @@ -0,0 +1,66 @@ +# Docs Assist claim check. +# +# Installed by /docs-assist:setup-hooks (ci). Runs the deterministic +# check-claims detector on every pull request: extracts checkable claims +# (CLI flags, config/env keys, function or class names, file paths, version +# requirements) from every doc under docs_dir plus README/AGENTS/CLAUDE.md, +# and resolves each one against the code with `git grep` / `git ls-files`. +# +# Cheap by design: no agent, no network calls beyond GitHub itself, no +# tokens. It catches a flag, path, or identifier a doc still references +# after the code moved, renamed, or removed it. It does not settle +# described-behavior or numeric claims ("retries three times"); those need +# a reader, which is what claim-briefs.mjs and the agent fan-out in +# claim-verification.md are for; this workflow only runs the mechanical half. +# Set CHECK_CLAIMS_STRICT: "1" to fail the check instead of only reporting; +# consider starting non-strict, since a "missing" result can also mean the +# claim's target is intentionally untracked (gitignored) rather than gone. +# +# The result is posted as a single sticky PR comment, updated in place on +# every push, so re-runs never pile up new comments. + +name: Claim check + +on: + pull_request: + +permissions: + contents: read + pull-requests: write + +jobs: + check-claims: + name: Check doc claims against code + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-node@v4 + with: + node-version: "20" + - name: Check claims + run: node scripts/check-claims.mjs > check-claims-report.md + env: + CHECK_CLAIMS_STRICT: "0" + # continue-on-error: fork PRs get a read-only token, so the comment + # post fails there; the report still lands in the job summary above. + - name: Post sticky PR comment + if: always() && github.event_name == 'pull_request' + continue-on-error: true + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + if (!fs.existsSync('check-claims-report.md')) return; + const marker = ''; + const body = marker + '\n' + fs.readFileSync('check-claims-report.md', 'utf8'); + const { data: comments } = await github.rest.issues.listComments({ + ...context.repo, issue_number: context.issue.number, per_page: 100, + }); + const existing = comments.find((c) => c.body && c.body.startsWith(marker)); + if (existing) { + await github.rest.issues.updateComment({ ...context.repo, comment_id: existing.id, body }); + } else { + await github.rest.issues.createComment({ ...context.repo, issue_number: context.issue.number, body }); + } diff --git a/commands/audit.md b/commands/audit.md index f7be25e..a5fba1e 100644 --- a/commands/audit.md +++ b/commands/audit.md @@ -96,7 +96,7 @@ For each document, evaluate: #### Content Issues -- **Trace claims to the code.** This is not optional for a full-set or directory audit, and it is the highest-value part of the audit, not a nice-to-have layered on top of the mechanical checks: for every doc, walk each command, flag, config key, default value, endpoint, version requirement, and described behavior out to the actual source and confirm it still matches. See `${CLAUDE_PLUGIN_ROOT}/skills/docs-assist/reference/claim-verification.md` for the method, what counts as a claim, and how to classify what you find (matches, drifted, missing, needs `/docs-assist:verify`). A pass that runs the mechanical linters, gets them clean, and stops there has finished the cheaper half of the audit and skipped the half a reader depends on. +- **Trace claims to the code.** This is not optional for a full-set or directory audit, and it is the highest-value part of the audit, not a nice-to-have layered on top of the mechanical checks: for every doc, walk each command, flag, config key, default value, endpoint, version requirement, and described behavior out to the actual source and confirm it still matches. Run `node ${CLAUDE_PLUGIN_ROOT}/assets/ci/check-claims.mjs` first: it resolves the identifier-shaped claims (paths, flags, function/class names, config keys) deterministically across the whole set in one pass, so the trace below spends judgment only on what a lookup can't settle (described behavior, numeric assertions). See `${CLAUDE_PLUGIN_ROOT}/skills/docs-assist/reference/claim-verification.md` for the full method, what counts as a claim, and how to classify what you find (matches, drifted, missing, needs `/docs-assist:verify`). A pass that runs the mechanical linters, gets them clean, and stops there has finished the cheaper half of the audit and skipped the half a reader depends on. - Outdated information (check dates and version references; for a full-set audit in a git repo, `node ${CLAUDE_PLUGIN_ROOT}/assets/ci/docs-decay.mjs` ranks every doc by staleness risk in one deterministic pass, prioritizing which docs get the claim trace above first) - Unverified claims: docs whose `sme-attested` frontmatter ledger is large or old. Surface the specific claims so a reviewer can verify and delete entries (the ledger exists to shrink; see `${CLAUDE_PLUGIN_ROOT}/skills/docs-assist/reference/frontmatter-spec.md`) - Incomplete instructions (missing steps). Reading can only catch so much here: for a load-bearing procedural doc, recommend `/docs-assist:verify`, which executes the steps in an isolated workspace and finds the break a read-through misses diff --git a/commands/setup-hooks.md b/commands/setup-hooks.md index 25aed7a..c3e22cb 100644 --- a/commands/setup-hooks.md +++ b/commands/setup-hooks.md @@ -1,6 +1,6 @@ --- -description: Install opt-in documentation hooks (git pre-commit lint, in-session doc lint, CI docs-impact check, CI reference-registry check). Default off, nothing is installed without your choice -argument-hint: [pre-commit | claude-code | ci | ci-facts | all] +description: Install opt-in documentation hooks (git pre-commit lint, in-session doc lint, CI docs-impact check, CI reference-registry check, CI claim check). Default off, nothing is installed without your choice +argument-hint: [pre-commit | claude-code | ci | ci-facts | ci-claims | all] --- # Set Up Documentation Hooks @@ -13,6 +13,7 @@ Install hooks that keep documentation in shape automatically. Hooks are opt-in: - **Claude Code post-edit lint** (`${CLAUDE_PLUGIN_ROOT}/assets/hooks/claude-code-hooks.json`): a `PostToolUse` hook that lints a Markdown file right after Claude writes or edits it, so style issues surface in the session. Requires `jq` and `npx`. - **CI docs-impact check** (`${CLAUDE_PLUGIN_ROOT}/assets/ci/docs-impact.mjs` and `${CLAUDE_PLUGIN_ROOT}/assets/ci/github/docs-impact.yml`): a deterministic detector that runs on every pull request and reports when a diff rides the change types that ripple into docs: moved or renamed docs, changed headings, changed code terms the docs mention, or a large source change with no docs touched. It costs no agent tokens; it tells reviewers when `/docs-assist:update` is worth running, and can be made blocking with `DOCS_IMPACT_STRICT`. - **CI reference-registry check** (`${CLAUDE_PLUGIN_ROOT}/assets/ci/check-facts.mjs` and `${CLAUDE_PLUGIN_ROOT}/assets/ci/github/check-facts.yml`): a deterministic detector that runs on every pull request and verifies the mechanical parts of `.docs-assist/reference.yml`: every `fact` entry's `source` still contains the referenced identifier, and every `pointer` entry's target file and heading anchor still resolve. Only offer this when the project has a `reference.yml` with `fact` or `pointer` entries; skip it otherwise, since there's nothing for it to check. Can be made blocking with `CHECK_FACTS_STRICT`. +- **CI claim check** (`${CLAUDE_PLUGIN_ROOT}/assets/ci/check-claims.mjs` and `${CLAUDE_PLUGIN_ROOT}/assets/ci/github/check-claims.yml`): a deterministic detector that runs on every pull request, extracting checkable claims (CLI flags, config/env keys, function or class names, file paths, version requirements) from every doc and resolving each against the code with `git grep`. Unlike the reference-registry check, this needs no curated `.docs-assist/reference.yml` entries first; it scans the whole doc set cold. It only settles claims a lookup can settle: described-behavior and numeric claims still need `claim-verification.md`'s agent trace. Can be made blocking with `CHECK_CLAIMS_STRICT`, though non-strict is the safer default since a "missing" result can also mean the claim's target is intentionally untracked (gitignored) rather than gone. ## Process @@ -64,7 +65,17 @@ When chosen: - Explain the tuning knob: `CHECK_FACTS_STRICT` (fail the check instead of reporting). - Tell the user this only checks that a `fact`'s source and a `pointer`'s target still exist; it does not compare a fact's value against its source, since that needs understanding the source language. That deeper check stays something `doc-auditor` and drafting do. -### 7. Confirm and Explain +### 7. Install the CI Claim Check + +When chosen: + +- Copy `${CLAUDE_PLUGIN_ROOT}/assets/ci/check-claims.mjs` to `scripts/check-claims.mjs` and `${CLAUDE_PLUGIN_ROOT}/assets/ci/github/check-claims.yml` to `.github/workflows/check-claims.yml`. +- If either destination exists, show a diff and confirm. Never silently overwrite. +- Explain the tuning knobs: `CHECK_CLAIMS_STRICT` (fail the check instead of reporting) and `DOCS_DIR` (defaults to `docs_dir` from `.docs-assist/config.yml`). +- Tell the user what this catches and what it doesn't: it resolves identifier-shaped claims (paths, flags, function/class names, config keys) against `git grep`/`git ls-files`, so it's fast and needs no agent, but it demotes anything about described or numeric behavior to a `claims-needs-judgment.json` file rather than guessing. `${CLAUDE_PLUGIN_ROOT}/assets/ci/claim-briefs.mjs` turns that into one brief per doc for a manual or agent-driven follow-up trace; this workflow does not run that step automatically. +- A "missing" result isn't automatically a doc bug: a claim naming a file that's real but gitignored (a vendored asset, a build output) will also show as missing, since the check only sees `git`-tracked content. Say so if the user is deciding whether to flip on `CHECK_CLAIMS_STRICT`. + +### 8. Confirm and Explain After installing, tell the user: diff --git a/skills/docs-assist/reference/claim-verification.md b/skills/docs-assist/reference/claim-verification.md index 8efc4e4..e1029bd 100644 --- a/skills/docs-assist/reference/claim-verification.md +++ b/skills/docs-assist/reference/claim-verification.md @@ -15,7 +15,9 @@ A doc with no such claims (pure narrative, a concept overview with no specifics) ## How to Trace One -For each claim in the doc under audit: +Before tracing by hand, run `node ${CLAUDE_PLUGIN_ROOT}/assets/ci/check-claims.mjs` (no setup needed; it auto-discovers docs from `docs_dir` plus README/AGENTS/CLAUDE.md). It's deterministic and dependency-free, the same philosophy as `check-facts.mjs` and `docs-decay.mjs`: it extracts every identifier-shaped claim (file paths, CLI flags, function or class names, config/env keys) across the whole doc set in one pass and resolves each against the code with `git grep`, instead of a reader re-deriving the same mechanical lookups one claim at a time. What it confirms or flags "missing" needs no further tracing; what it can't settle (described behavior, numeric assertions, runtime claims) it writes to `claims-needs-judgment.json`, grouped by doc, which is exactly what the fan-out below is for. A "missing" result still deserves a look before you report it: the target can be real but gitignored (a vendored asset, a build artifact) rather than actually gone, since the check only sees tracked content. + +For each claim the script leaves unresolved, or when working a doc by hand without it: 1. Find where the code would prove or disprove it: grep for the flag name, the config key, the constant, the function. Read the actual definition, not just a usage site. 2. Compare what you find to what the doc states: @@ -27,7 +29,7 @@ For each claim in the doc under audit: ## Scope It Like Everything Else -- **Full-set audit**: trace claims across the whole set. This is exactly the per-doc work the fan-out threshold exists for (see the Notes in `${CLAUDE_PLUGIN_ROOT}/commands/audit.md`): hand this method to each `doc-auditor` slice in its brief, not only the mechanical checklist. A slice that only runs the mechanical checks and skips this is an incomplete slice, not a fast one. +- **Full-set audit**: run `check-claims.mjs` once for the whole set first, then fan out only what's left. Run `node ${CLAUDE_PLUGIN_ROOT}/assets/ci/claim-briefs.mjs` against its output to generate one self-contained brief per doc from `claims-needs-judgment.json` (the claim rows, the classification method, and instructions to report only Drifted/Missing/worth-a-second-look), and hand each brief to a `doc-auditor` slice as its claim-tracing task: this is exactly the per-doc work the fan-out threshold exists for (see the Notes in `${CLAUDE_PLUGIN_ROOT}/commands/audit.md`). A slice that only runs the mechanical checklist and skips this is an incomplete slice, not a fast one. - **Change-based audit or update**: `impact-analysis.md`'s "Command, flag, endpoint, or config key changed" row already covers the diff-driven version of this: something changed, follow it to what it touches. This method is what to run when there is no diff, against the docs set as it already stands, which is the common case for "review our existing docs" rather than "review this PR." - **Health check**: too deep for the fast scorecard. Health's Freshness dimension uses `docs-decay.mjs`'s churn heuristic to rank which docs are worth this trace, not to perform the trace itself. Point the full audit, or `/docs-assist:verify` for procedural claims, at what the ranking surfaces.