diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ebbc115..f6d32c05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,36 @@ jobs: - name: Run tests run: make test + # Surfaces the enforcement corpus as its own check. The `test` job already + # runs these assertions, but a green suite of ~4,700 tests is not evidence + # that a gate can fail -- and a gate that never fails is worthless. This job + # makes the red direction visible in the pipeline: known-bad fixtures must be + # rejected, known-good fixtures must stay clean. + gates: + name: Enforcement Gates (seeded violations must fail) + runs-on: ubuntu-latest + permissions: + # This job runs pull-request code (`make install`, `make test-gates`), so + # it gets the least privilege that still lets it check out and read. + contents: read + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + # Keep the job token out of .git/config so the test corpus this job + # executes cannot read it back. The initial fetch is unaffected. + persist-credentials: false + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install test dependencies + run: make install + + - name: Known-bad must be red, known-good must stay green + run: make test-gates + coverage: name: Coverage (≥90%) runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index fdbe9e3f..78a44107 100644 --- a/.gitignore +++ b/.gitignore @@ -569,6 +569,10 @@ coverage.xml **/.cache/decisions.jsonl.1 **/.cache/decisions.jsonl.lock +# doc-index cache (doc_index.py) — per-project structural index for JIT +# retrieval, read-once-per-file. Rebuilds automatically on content change. +**/.cache/doc-index/ + # Superpowers brainstorming/planning specs (local-only) docs/superpowers/ diff --git a/Makefile b/Makefile index d503a512..97ca668b 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,7 @@ help: @echo "" @echo "Available targets:" @echo " make test - Run all tests in parallel (-n 6)" + @echo " make test-gates - Run the enforcement gate corpus (seeded violations must fail)" @echo " make test-verbose - Run tests with verbose output" @echo " make test-quick - Run fast tests only (skip slow integration tests)" @echo " make test-coverage - Run tests with coverage report" @@ -149,6 +150,14 @@ test: check-pytest @echo "Running Constructor Studio tests with pipx..." $(PYTEST_PIPX) tests/ -n 6 -v --tb=short +# Runs the enforcement corpus on its own so the pipeline shows, as a named +# check, that these gates fail on a seeded violation. The assertions already +# live in the suite; a green `test` job does not evidence the red direction +# unless you know the tests exist and go and read them. +test-gates: check-pytest + @echo "Running enforcement gate corpus (known-bad must be red, known-good must be green)..." + $(PYTEST_PIPX) tests/test_enforcement_empty_codebase.py tests/test_enforcement_empty_scan.py -v --tb=short + # Run tests with verbose output test-verbose: check-pytest @echo "Running Constructor Studio tests (verbose) with pipx..." diff --git a/architecture/features/eval-harness.md b/architecture/features/eval-harness.md index 46ee0e47..4c96f91d 100644 --- a/architecture/features/eval-harness.md +++ b/architecture/features/eval-harness.md @@ -13,6 +13,7 @@ - [Run Eval Suite](#run-eval-suite) - [Score Structural Compliance](#score-structural-compliance) - [Score Rule Compliance (Advisory Judge)](#score-rule-compliance-advisory-judge) + - [Assess Semantic Coverage](#assess-semantic-coverage) - [4. States (CDSL)](#4-states-cdsl) - [Eval Report Lifecycle](#eval-report-lifecycle) - [5. Definitions of Done](#5-definitions-of-done) @@ -209,6 +210,40 @@ or unreadable gold file is *excluded* from calibration rather than counted as a - [x] - `p1` - A deterministic reference-stub `JudgeFn` for tests and calibration wiring (not a model) - `inst-judge-stub` - [x] - `p1` - The `Calibration` result model (accuracy, consistency, coverage) - `inst-judge-calibrate` +### Assess Semantic Coverage + +- [x] `p1` - **ID**: `cpt-studio-algo-eval-semantic` + +The semantic-coverage engine: it asks whether a marked block *implements the requirement it +cites*, which marker density cannot. A deterministic stdlib token-overlap pre-filter scores +every block against its requirement text and surfaces the weak links; only those go to a +pluggable `SemanticJudgeFn` (the model call lives out-of-tree). It is advisory throughout — +no verdict here touches an exit code — and honest: a block with no retrievable requirement, or +too little text to compare, is reported `unjudgeable`, never a silent "covered". Files a human +declared `excluded` in the coverage report are skipped; files flagged `whole_file_claims` are +prioritised. Every model verdict carries an evidence check: a quote absent from the code is not +trusted. + +**Steps**: +1. [x] - `p1` - Tokenise code and requirement (camelCase + Unicode word-boundary split, stopwords dropped) and score their overlap against the requirement set, or `None` below the token floor - `inst-semantic-tokenize` +2. [x] - `p1` - Rank every pairing weakest-first, marking below-threshold blocks as weak links and below-floor blocks as unjudgeable, and marking every block in a whole-file-claim file a weak link regardless of overlap (always judged) - `inst-semantic-prefilter` +3. [x] - `p1` - Judge only the weak links through the injected `SemanticJudgeFn`, degrading a raising or malformed reply to unjudgeable and evidence-checking every quote - `inst-semantic-finding` +4. [x] - `p1` - Assess pairings end-to-end — scope, rank, judge weak links, and report unjudgeable coverage gaps — advisory throughout - `inst-semantic-report` + +**Supporting**: +- [x] - `p1` - Module imports, verdict constants, and the provisional calibration constants + stopword set - `inst-semantic-imports` +- [x] - `p1` - The pairing, ranked (with a whole-file-claim `forced` flag), request, reply data model and the `SemanticJudgeFn` seam — the seam is called synchronously and unbounded, so a judge that can hang must enforce its own timeout - `inst-semantic-datamodel` +- [x] - `p1` - Blank character spans in place (spaces, newlines preserved) so a verbatim quote of the untouched text still matches - `inst-semantic-blank` +- [x] - `p1` - Derive two code views in one grammar-aware `tokenize` pass — an overlap view (comments and all strings blanked, so prose cannot inflate the score) and an evidence view (comments and docstrings blanked, inline strings kept) — falling back to a plain comment strip for a non-tokenisable fragment - `inst-semantic-strip` +- [x] - `p1` - Read the frozen coverage-report contract (`excluded[]` / `whole_file_claims[]`, path-separator normalised, `excluded` taking precedence), degrading to an empty scope - `inst-semantic-scope` +- [x] - `p1` - Build the deterministic judge prompt (bounded fields) and parse a reply to a verdict - `inst-semantic-prompt` +- [x] - `p1` - The evidence-present hallucination guard: a quote must occur in the evidence view (comments/docstrings stripped), so a quote matching only a comment is not evidence - `inst-semantic-evidence` +- [x] - `p1` - The `SemanticGap` coverage-gap record (block_id, path, start_line, reason), located like a finding so a report consumer can point at every gap - `inst-semantic-gap` +- [x] - `p1` - The reference stub judge (deterministic overlap buckets) and its best-evidence-line quote - `inst-semantic-stub` +- [x] - `p1` - The requirement resolver adapter over `get_content_scoped` - `inst-semantic-resolve` +- [x] - `p1` - Read a `[gold]` verdict label for calibration, or `None` when absent/malformed - `inst-semantic-gold` +- [x] - `p1` - Calibrate the judge over gold-backed pairings: report accuracy, consistency, and effective sample size per case, excluding unscoreable / crashed / strict-tie cases — accuracy over cases with a majority, consistency over cases with ≥2 surviving runs — `None` when nothing is measurable - `inst-semantic-calibrate` + ## 4. States (CDSL) ### Eval Report Lifecycle @@ -253,6 +288,7 @@ them (subscripting when the flag is absent raises `KeyError`): | Eval Harness | `skills/studio/scripts/studio/utils/eval_harness.py` | Scenario/run loading, scorer seam, runner, report, regression diff | | Structural Scorer | `skills/studio/scripts/studio/utils/eval_structural.py` | Deterministic structural checks over a run's manifest + phase frontmatter | | Advisory Judge | `skills/studio/scripts/studio/utils/eval_judge.py` | Advisory rule-compliance judge (pluggable model seam) + gold-set calibration | +| Semantic Coverage | `skills/studio/scripts/studio/utils/eval_semantic.py` | Token-overlap pre-filter + advisory judge seam over marked-block vs requirement, with honesty + evidence guards | ## 7. Acceptance Criteria @@ -267,3 +303,7 @@ them (subscripting when the flag is absent raises `KeyError`): - [x] `p1` - Judge calibration reports accuracy against a human gold set and run-to-run consistency, kept separate from structural compliance; judge coverage is derived from which scenarios carry a gold set - [x] `p1` - The run evidence is hard-capped: the total (headers, truncation markers, separators and the omission line included) never exceeds the evidence budget. A trimmed phase is still judged; when whole phases are omitted to fit, the request is marked incomplete and the judge returns `UNKNOWN` without a model call — a verdict is never certified from evidence with entire phases unseen - [x] `p1` - In default (non-JSON) mode the human summary prints the scorer coverage line, explains that advisory-only UNKNOWNs come from an unwired judge (never counted against the gate), and prints the calibration metrics under `--calibrate`; the JSON payload is unchanged +- [x] `p1` - Semantic coverage ranks marked blocks by deterministic token-overlap against their requirement and judges only the surfaced weak links — a strong-overlap block triggers no model call, except a block in a `whole_file_claims` file, which is always judged regardless of overlap (its structural coverage rests on a whole-file scope marker, so a high lexical score there must not buy a free pass) +- [x] `p1` - A block with no retrievable requirement, or with too little text to compare, is reported `unjudgeable`, never a silent `covered`; with no `SemanticJudgeFn` wired every weak link is `unjudgeable` and nothing gates +- [x] `p1` - A model `evidence_quote` absent from the cited code sets `evidence_ok=false`; a raising or malformed judge reply degrades that one finding to `unjudgeable` without sinking the assessment +- [x] `p1` - Files a human declared in the coverage report's `excluded[]` are skipped; a report lacking the `excluded[]` / `whole_file_claims[]` fields yields an empty scope (skip nothing, prioritise nothing), never an error diff --git a/architecture/features/traceability-validation.md b/architecture/features/traceability-validation.md index b7494576..25afc650 100644 --- a/architecture/features/traceability-validation.md +++ b/architecture/features/traceability-validation.md @@ -21,6 +21,7 @@ - [List ID Kinds](#list-id-kinds) - [Validate TOC](#validate-toc) - [TOC Utilities](#toc-utilities) + - [Document Index](#document-index) - [Markdown Parsing Utilities](#markdown-parsing-utilities) - [Fixing Prompt Enrichment](#fixing-prompt-enrichment) - [Headings Contract Validation](#headings-contract-validation) @@ -398,11 +399,12 @@ Catches structural and traceability issues that AI agents miss or hallucinate 2. [x] - `p1` - Generate expected TOC from headings - `inst-toc-generate-expected` 3. [x] - `p1` - Compare existing vs expected: check anchor validity, heading coverage, staleness - `inst-toc-compare` 4. [x] - `p1` - **IF** mismatch, record error with diff details - `inst-toc-if-mismatch` -4. [x] - `p1` - **RETURN** JSON: `{status, files_checked, errors}` - `inst-toc-return` +4. [x] - `p1` - **RETURN** JSON: `{status, files_validated, error_count, warning_count, results}`, each `results[]` entry `{file, status, error_count, warning_count}` plus `errors`/`warnings` arrays when `--verbose` or non-empty - `inst-toc-return` **Supporting**: - [x] - `p1` - Imports and module setup for validate-toc command - `inst-toc-imports` -- [x] - `p1` - Human-friendly formatter for validate-toc output - `inst-toc-format` +- [x] - `p1` - Validate a single file, never raising: a missing file or a read failure (permission denied, binary/non-UTF-8 content, a TOCTOU race) is reported as its own ERROR result rather than aborting the whole batch and discarding results already collected for earlier files - `inst-toc-validate-one` +- [x] - `p1` - Human-friendly formatter for validate-toc output: a WARN-only file prints its warnings the same way a FAIL file prints its errors, not just the bare status - `inst-toc-format` ### TOC Utilities @@ -416,6 +418,9 @@ Catches structural and traceability issues that AI agents miss or hallucinate 4. [x] - `p1` - Insert/update TOC using heading-based insertion (`## Table of Contents`) for kit file generator - `inst-toc-util-insert-heading` 5. [x] - `p1` - Process file: strip manual TOC, insert marker-based TOC, write if changed - `inst-toc-util-process-file` 6. [x] - `p1` - Validate TOC: check existence, anchor validity, completeness, staleness - `inst-toc-util-validate` +7. [x] - `p1` - Parse headings with line numbers, fence-aware and front-matter-aware (skips a leading YAML block so a `#`-prefixed front-matter line is never mistaken for a heading); shared by doc-index and the JIT-retrieval readiness checks, which need section boundaries the plain heading list doesn't carry -- `parse_headings` itself now delegates here, stripping the line number, so both share one fence/heading-match implementation - `inst-toc-util-parse-headings-lines` +8. [x] - `p1` - Collect JIT-retrieval readiness warnings for a document: gathers all four signals below over *every* heading level, independent of whatever level cap the caller configured for TOC-completeness checking - `inst-toc-jit-readiness-collect` +9. [x] - `p1` - Compute the four JIT-retrieval readiness signals -- duplicate heading titles (compared case-insensitively, with internal whitespace collapsed and Unicode-normalized, though the original text is still shown in the warning), heading depth jumps, oversized sections (`--max-section-lines`, default 300, validated against non-finite/non-positive input independent of the CLI's own argparse guard), and a missing top-of-file description/frontmatter block -- all warning-only, never errors (see constructorfabric/studio#104) - `inst-toc-jit-readiness` **Supporting**: - [x] - `p1` - Imports, constants, fence tracking, GitHub anchor slug generation - `inst-toc-util-datamodel` @@ -437,6 +442,43 @@ Catches structural and traceability issues that AI agents miss or hallucinate - [x] - `p1` - Heading-based TOC new-insert branch: inject `## Table of Contents` before first heading when absent - `inst-toc-util-insert-heading-new` - [x] - `p1` - TOC validate init: build heading list and expected TOC string before comparison checks - `inst-toc-util-validate-init` +### Document Index + +- [x] `p1` - **ID**: `cpt-studio-algo-traceability-validation-doc-index` + +**Input**: Markdown file path + +**Output**: `cfs doc-index`'s JSON is `{file, cache_hit, total_lines, section_count, sections, section_level, retrieval_section_count, retrieval_sections}` -- every heading's own line range in `sections[]` (`{level, heading, line_start, line_end, summary}`), plus a coarser "one chunk per real section" grouping in `retrieval_sections[]` at an inferred heading level, each with a content hash and a summary slot. The underlying cached index dict additionally carries `schema_version`, `path`, and `etag`. + +A cached, read-once-per-file structural index for Markdown JIT retrieval (see +constructorfabric/studio#104): parsing a file's headings/section boundaries +happens once, not once per query, until its stat fingerprint (`mtime_ns` + +size) changes. +The cache-validity fingerprint is deliberately metadata-only (`mtime` + file +size via `Path.stat()`), never a content hash — the point of the cache is to +avoid reading the file at all on a hit, and a content hash would defeat that +by requiring the read it's meant to save. A build reads the content and +takes that fingerprint bracketed by a stat snapshot on each side, so the +fingerprint saved is provably the one that matches what was actually parsed +even if a write lands in the narrow window during the read. + +1. [x] - `p1` - Build a fresh structural index: parse headings + line ranges from current content, compute the stat-based fingerprint, stamp the current schema version - `inst-doc-index-build` +2. [x] - `p1` - Load a cached index for a file, validated against current stat metadata (no content read on a hit) and against the required-field shape at the current schema version; returns `None` if missing, stale, corrupt, or an incomplete/outdated shape - `inst-doc-index-load` +3. [x] - `p1` - Persist an index to its cache location atomically (temp file + `os.replace`, so a concurrent reader never observes a torn write); no-ops silently outside a Studio-adapted project - `inst-doc-index-save` +4. [x] - `p1` - Return the cached index or build-and-cache a fresh one; reports cache hit/miss for benchmarking - `inst-doc-index-get-or-build` +5. [x] - `p1` - Attach a one-line, LLM-authored summary to a cached section by its `line_start`, for a future per-section-summary caller - `inst-doc-index-annotate` +6. [x] - `p1` - Infer which heading level represents one retrievable section: the most-recurring level wins over a level that appears only once (however shallow), since PDF-conversion heading levels don't reliably encode true nesting depth — a fixed level assumption silently produces a degenerate mega-section on such documents - `inst-doc-index-infer-level` +7. [x] - `p1` - Group headings at exactly the inferred level into retrieval sections (off-level headings stay inside whichever section they fall under, never split one apart); hash each section's own text for section-granularity staleness detection - `inst-doc-index-retrieval-sections` +8. [x] - `p1` - Diff the current file against its last cached build at section granularity: which retrieval sections are unchanged vs. changed, or whether the section count itself changed (a structural change, matched by position not heading text, since duplicate titles are real) - `inst-doc-index-diff-stale` + +**Supporting**: +- [x] - `p1` - Stat-based cache-validity fingerprint (`mtime_ns` + size); resolved from the file's own path, never a content hash - `inst-doc-index-etag` +- [x] - `p1` - Resolve the cache file location within the Studio directory owning the indexed file, resolved from the file's own path (not the process's working directory) - `inst-doc-index-cache-path` +- [x] - `p1` - `cfs doc-index` CLI wrapper: parse arguments, build the JSON output payload, reporting a clean error for a missing or unreadable file - `inst-doc-index-cmd` +- [x] - `p1` - Human-friendly formatter for `cfs doc-index` output - `inst-doc-index-cmd-format` +- [x] - `p1` - Read a file's content bracketed by an etag snapshot on each side, retrying on mismatch: closes the window where a write between the read and the fingerprint could save stale headings under a fresh-looking etag - `inst-doc-index-stable-read` +- [x] - `p1` - Re-parse a file's current content into retrieval sections for staleness comparison, and build the `(heading, line_start)` identity pair that disambiguates a duplicate heading title in a diff result - `inst-doc-index-diff-stale-helpers` + ### Markdown Parsing Utilities - [x] `p1` - **ID**: `cpt-studio-algo-traceability-validation-parsing-utils` diff --git a/skills/studio/scripts/studio/cli.py b/skills/studio/scripts/studio/cli.py index dab43452..633509c2 100644 --- a/skills/studio/scripts/studio/cli.py +++ b/skills/studio/scripts/studio/cli.py @@ -134,6 +134,10 @@ def _cmd_eval(argv: List[str]) -> int: from .commands.eval import cmd_eval return cmd_eval(argv) +def _cmd_doc_index(argv: List[str]) -> int: + from .commands.doc_index import cmd_doc_index + return cmd_doc_index(argv) + # ============================================================================= # ADAPTER COMMAND # ============================================================================= @@ -216,6 +220,7 @@ def _cmd_map(argv: List[str]) -> int: "resolve-vars": "Resolve template variables to absolute paths", "toc": "Generate/update Table of Contents", "chunk-input": "Chunk oversized workflow input into line-bounded Markdown files", + "doc-index": "Build/reuse a cached heading index for a Markdown file (read once, not per query)", "pdsl": "Validate PDSL prompt blocks", "workspace-init": "Initialize multi-repo workspace", "workspace-add": "Add a source to workspace config", @@ -232,7 +237,7 @@ def _cmd_map(argv: List[str]) -> int: ("Validation", ["validate", "validate-kits", "validate-toc", "spec-coverage", "check-language"]), ("Search & Navigation", ["list-ids", "list-id-kinds", "get-content", "where-defined", "where-used"]), ("Kit Management", ["kit"]), - ("Utility", ["toc", "chunk-input", "pdsl"]), + ("Utility", ["toc", "chunk-input", "doc-index", "pdsl"]), ("Workspace", ["workspace-init", "workspace-add", "workspace-info", "workspace-sync"]), ("Delegation", ["delegate"]), ("Diagnostics", ["doctor"]), @@ -263,6 +268,7 @@ def _cmd_map(argv: List[str]) -> int: "validate-toc": "_cmd_validate_toc", "spec-coverage": "_cmd_spec_coverage", "chunk-input": "_cmd_chunk_input", + "doc-index": "_cmd_doc_index", "workspace-init": "_cmd_workspace_init", "workspace-add": "_cmd_workspace_add", "workspace-info": "_cmd_workspace_info", @@ -296,6 +302,7 @@ def _cmd_map(argv: List[str]) -> int: _cmd_validate_toc, _cmd_spec_coverage, _cmd_chunk_input, + _cmd_doc_index, _cmd_workspace_init, _cmd_workspace_add, _cmd_workspace_info, diff --git a/skills/studio/scripts/studio/commands/doc_index.py b/skills/studio/scripts/studio/commands/doc_index.py new file mode 100644 index 00000000..ba512b50 --- /dev/null +++ b/skills/studio/scripts/studio/commands/doc_index.py @@ -0,0 +1,92 @@ +"""Studio doc-index command — build/reuse a cached structural index for a +Markdown file, so heading-based JIT retrieval reads a file's structure once, +not once per query. + +Thin CLI wrapper around ``studio.utils.doc_index``. +""" + +import argparse +import logging +from pathlib import Path +from typing import List + +from ..utils.doc_index import get_or_build_doc_index +from ..utils.ui import ui + +logger = logging.getLogger(__name__) + + +# @cpt-begin:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-cmd +def cmd_doc_index(argv: List[str]) -> int: + """Build (or reuse the cached) structural index for a Markdown file.""" + p = argparse.ArgumentParser( + prog="cfs doc-index", + description=( + "Build or reuse a cached heading/section index for a Markdown file, " + "so navigation reads the file's structure once, not once per query. " + "section_level is inferred from the most frequently repeated heading " + "level (ties prefer the shallower level); a level used only once is " + "never chosen." + ), + ) + p.add_argument("file", help="Markdown file path") + p.add_argument( + "--rebuild", + action="store_true", + help="Force a fresh build even if a valid cached index exists", + ) + args = p.parse_args(argv) + + filepath = Path(args.file).resolve() + if not filepath.is_file(): + ui.result( + {"file": str(filepath), "status": "ERROR", "message": "File not found"}, + human_fn=lambda d: ui.error(f"{d['file']}: {d['message']}"), + ) + return 2 + + try: + index = get_or_build_doc_index(filepath, force_rebuild=args.rebuild) + except (OSError, UnicodeDecodeError) as exc: + logger.warning("doc-index: cannot read %s: %s", filepath, exc) + ui.result( + {"file": str(filepath), "status": "ERROR", "message": f"Cannot read file: {exc}"}, + human_fn=lambda d: ui.error(f"{d['file']}: {d['message']}"), + ) + return 2 + + output = { + "file": str(filepath), + "cache_hit": index["cache_hit"], + "total_lines": index["total_lines"], + "section_count": len(index["sections"]), + "sections": index["sections"], + "section_level": index["section_level"], + "retrieval_section_count": len(index["retrieval_sections"]), + "retrieval_sections": index["retrieval_sections"], + } + ui.result(output, human_fn=_human_doc_index) + return 0 +# @cpt-end:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-cmd + + +# @cpt-begin:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-cmd-format +def _human_doc_index(data: dict) -> None: + ui.header("Doc Index") + ui.substep(data["file"]) + hit = "cache hit — reused existing index" if data["cache_hit"] else "cache miss — built fresh index" + ui.substep(hit) + ui.substep(f"{data['section_count']} section(s), {data['total_lines']} total lines") + for s in data["sections"]: + summary = f" — {s['summary']}" if s.get("summary") else "" + ui.substep(f" H{s['level']} [{s['line_start']}-{s['line_end']}] {s['heading']}{summary}") + ui.blank() + + level = data["section_level"] + ui.step(f"Retrieval sections (level {level}, {data['retrieval_section_count']} section(s))" if level is not None + else "Retrieval sections (no headings — none inferred)") + for s in data["retrieval_sections"]: + summary = f" — {s['summary']}" if s.get("summary") else "" + ui.substep(f" [{s['line_start']}-{s['line_end']}] {s['heading']} ({s['hash'][:12]}){summary}") + ui.blank() +# @cpt-end:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-cmd-format diff --git a/skills/studio/scripts/studio/commands/validate_toc.py b/skills/studio/scripts/studio/commands/validate_toc.py index 0a7d719e..d1e49ebd 100644 --- a/skills/studio/scripts/studio/commands/validate_toc.py +++ b/skills/studio/scripts/studio/commands/validate_toc.py @@ -14,10 +14,58 @@ from pathlib import Path from typing import List -from ..utils.toc import add_toc_max_level_argument, validate_toc +from ..utils import error_codes as EC +from ..utils.toc import DEFAULT_MAX_SECTION_LINES, add_toc_max_level_argument, validate_toc from ..utils.ui import ui # @cpt-end:cpt-studio-algo-traceability-validation-validate-toc:p1:inst-toc-imports +# @cpt-begin:cpt-studio-algo-traceability-validation-validate-toc:p1:inst-toc-validate-one +def _validate_one_file(filepath: Path, args: argparse.Namespace) -> dict: + """Validate a single file, returning its result dict. Never raises -- + a missing file or a read failure (permission denied, binary/non-UTF-8 + content, a TOCTOU race) is reported as an ERROR result instead, so one + bad file in a batch can't abort validation of the rest. + """ + if not filepath.is_file(): + return { + "file": str(filepath), + "status": "ERROR", + "message": "File not found", + "code": EC.FILE_LOAD_ERROR, + } + + try: + content = filepath.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + return { + "file": str(filepath), + "status": "ERROR", + "message": f"Could not read file: {exc}", + "code": EC.FILE_READ_ERROR, + } + + report = validate_toc( + content, + artifact_path=filepath, + max_heading_level=args.max_level, + max_section_lines=args.max_section_lines, + ) + errors = report.get("errors", []) + warnings = report.get("warnings", []) + file_result: dict = { + "file": str(filepath), + "status": "FAIL" if errors else ("WARN" if warnings else "PASS"), + "error_count": len(errors), + "warning_count": len(warnings), + } + if args.verbose or errors: + file_result["errors"] = errors + if args.verbose or warnings: + file_result["warnings"] = warnings + return file_result +# @cpt-end:cpt-studio-algo-traceability-validation-validate-toc:p1:inst-toc-validate-one + + def cmd_validate_toc(argv: List[str]) -> int: """Validate Table of Contents in markdown files.""" # @cpt-begin:cpt-studio-algo-traceability-validation-validate-toc:p1:inst-toc-parse-args @@ -31,6 +79,12 @@ def cmd_validate_toc(argv: List[str]) -> int: help="Markdown file path(s) to validate", ) add_toc_max_level_argument(p) + p.add_argument( + "--max-section-lines", + type=int, + default=DEFAULT_MAX_SECTION_LINES, + help=f"Warn when a section exceeds this many lines (default: {DEFAULT_MAX_SECTION_LINES})", + ) p.add_argument( "--verbose", action="store_true", @@ -40,49 +94,19 @@ def cmd_validate_toc(argv: List[str]) -> int: # @cpt-end:cpt-studio-algo-traceability-validation-validate-toc:p1:inst-toc-parse-args # @cpt-begin:cpt-studio-algo-traceability-validation-validate-toc:p1:inst-toc-resolve-files - results = [] - total_errors = 0 - total_warnings = 0 files_to_validate = [Path(f).resolve() for f in args.files] # @cpt-end:cpt-studio-algo-traceability-validation-validate-toc:p1:inst-toc-resolve-files # @cpt-begin:cpt-studio-algo-traceability-validation-validate-toc:p1:inst-toc-foreach-file - for filepath in files_to_validate: - - if not filepath.is_file(): - results.append({ - "file": str(filepath), - "status": "ERROR", - "message": "File not found", - }) + results = [_validate_one_file(filepath, args) for filepath in files_to_validate] + total_errors = 0 + total_warnings = 0 + for file_result in results: + if file_result["status"] == "ERROR": total_errors += 1 - continue - - content = filepath.read_text(encoding="utf-8") - report = validate_toc( - content, - artifact_path=filepath, - max_heading_level=args.max_level, - ) - - errors = report.get("errors", []) - warnings = report.get("warnings", []) - total_errors += len(errors) - total_warnings += len(warnings) - - file_result: dict = { - "file": str(filepath), - "status": "FAIL" if errors else ("WARN" if warnings else "PASS"), - "error_count": len(errors), - "warning_count": len(warnings), - } - - if args.verbose or errors: - file_result["errors"] = errors - if args.verbose or warnings: - file_result["warnings"] = warnings - - results.append(file_result) + else: + total_errors += file_result["error_count"] + total_warnings += file_result["warning_count"] # @cpt-end:cpt-studio-algo-traceability-validation-validate-toc:p1:inst-toc-foreach-file # @cpt-begin:cpt-studio-algo-traceability-validation-validate-toc:p1:inst-toc-return @@ -123,6 +147,10 @@ def _human_validate_toc(data: dict) -> None: ui.substep(f" ✗ {e}") for w in r.get("warnings", []): ui.substep(f" ⚠ {w}") + elif status == "WARN": + ui.warn(f"{path}: {warns} warning(s)") + for w in r.get("warnings", []): + ui.substep(f" ⚠ {w}") else: ui.substep(f"{path}: {status}") overall = data.get("status", "") diff --git a/skills/studio/scripts/studio/utils/doc_index.py b/skills/studio/scripts/studio/utils/doc_index.py new file mode 100644 index 00000000..f239b260 --- /dev/null +++ b/skills/studio/scripts/studio/utils/doc_index.py @@ -0,0 +1,563 @@ +"""Cached, read-once-per-file document index for Markdown JIT retrieval. + +Builds a structural index (headings + section line ranges) for a Markdown +file exactly once, persists it keyed by an etag of the file's own state, and +reuses that cached index on every subsequent call against the same file -- +until the file actually changes. This is the "read once per file, not once +per query" mechanism: parsing/etag work never repeats across queries, and +optional per-section summaries (written by an LLM caller, not by this +module) accumulate in the same cached artifact instead of being +re-derived each time. + +Scope: Markdown only. PDF/DOCX conversion is a separate concern (Layer 1); +this module operates purely on already-plain-text content (Layer 2). + +Schema contract: adding a new top-level key to the index dict is always +additive and does not require bumping ``_SCHEMA_VERSION`` -- an older +build of this module simply never wrote that key, which +``_has_schema_current_index``'s required-field check already treats as +"predates the current schema" and rebuilds. ``_SCHEMA_VERSION`` exists for +the other kind of change: an existing key's *meaning* or *shape* changing +incompatibly (e.g. what ``retrieval_sections`` entries contain), which a +field-presence check alone can't detect since the field is still there, +just holding something a new reader would misinterpret. Bump +``_SCHEMA_VERSION`` for that kind of change; a plain new field needs only +listing in ``_REQUIRED_INDEX_FIELDS`` if a consumer reads it unconditionally. + +See constructorfabric/studio#104. + +@cpt-algo:cpt-studio-algo-traceability-validation-doc-index:p1 +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import time +from collections import Counter +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from .toc import parse_headings_with_lines + +logger = logging.getLogger(__name__) + +_CACHE_SUBDIR = ".cache" +_INDEX_CACHE_DIR = "doc-index" + +#: Bumped whenever the index's own shape changes incompatibly. Checked +#: alongside the etag so a future schema change invalidates an +#: old-format cache instead of silently returning old-shape data past a +#: matching etag. +_SCHEMA_VERSION = 1 + + +# @cpt-begin:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-etag +def _compute_etag(path: Path) -> str: + """Compute a cheap cache-validity fingerprint from filesystem metadata. + + Deliberately *not* a content hash: ``Path.stat()`` is metadata-only (no + file read), which is what lets a cache *hit* stay free of a full read -- + the whole point of a read-once-per-file index. mtime + size changes on + a same-size, same-line-count text swap too, since a write ordinarily + advances mtime -- a byte-count/line-count-only fingerprint would miss + that edit outright, and computing either requires reading the entire + file this check exists to avoid reading. + + Known, accepted limitation: on a filesystem with coarse mtime + resolution (e.g. some FAT32/older-HFS+/NFS configurations), two + same-size edits landing within one mtime tick can share an identical + etag, and a cache hit would then return the first edit's stale data. + Trading that narrow, filesystem-dependent risk for never reading the + file on a cache hit is this module's whole reason to exist; closing it + fully would mean a content hash, which defeats the point. + """ + st = path.stat() + return f"{st.st_mtime_ns}:{st.st_size}" +# @cpt-end:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-etag + + +# @cpt-begin:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-cache-path +def _index_cache_path(path: Path) -> Optional[Path]: + """Resolve ``/.cache/doc-index/.json`` for a file. + + Resolved from ``path`` itself (not the process's current working + directory), so indexing a file outside the caller's cwd still resolves + -- and always resolves -- the Studio directory that actually owns it. + + Returns ``None`` when no Studio directory can be found (e.g. outside a + Studio-adapted project) -- callers should fall back to an uncached build. + """ + from .files import find_studio_directory + + try: + studio_dir = find_studio_directory(path.resolve().parent) + except OSError as exc: + # A file whose parent can't be stat'd (permissions, a race) is not a + # reason to fail the caller -- just an uncached build, like "no + # Studio directory found". Warning, not debug: this is a genuine + # anomaly (unlike the ordinary, unlogged "no Studio directory" + # case below), and should be visible at the CLI's default log + # level rather than indistinguishable from a routine cache miss. + logger.warning("doc-index cache path lookup failed for %s: %s", path, exc) + studio_dir = None + if studio_dir is None: + return None + + slug = hashlib.sha256(str(path.resolve()).encode("utf-8")).hexdigest()[:16] + return studio_dir / _CACHE_SUBDIR / _INDEX_CACHE_DIR / f"{slug}.json" +# @cpt-end:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-cache-path + + +# @cpt-begin:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-infer-level +def infer_section_level(headings_with_lines: List[Tuple[int, str, int]]) -> Optional[int]: + """Infer which heading level represents one retrievable section. + + PDF-to-Markdown conversion assigns heading levels by font-size/style + heuristics, not semantic depth -- a document's real top-level chapters + can land on any level. A real document converted during this feature's + own development put all 8 of its actual chapters on H5, while a single + stray H3 subsection appeared once in the middle; a fixed-level + assumption (e.g. "H1-H3 is the chapter level") silently turned the back + half of that real document into one fake 6,601-line "section" bounded + by that one stray heading (see constructorfabric/studio#104). + + Heuristic: a document's real recurring structure shows up as the + heading level used *most often* -- real chapters repeat throughout a + document precisely because they're structure, not noise. A level used + only once is excluded as a candidate outright: a single occurrence + can't be "the" recurring section boundary by definition, and treating + it as one produces exactly the degenerate failure above. Ties (and the + all-singletons fallback) prefer the shallowest level, on the + conservative assumption that a coarser grouping beats fragmenting a + document into many tiny sections. + + Returns ``None`` for a headingless document. + """ + if not headings_with_lines: + return None + counts = Counter(level for level, _text, _line in headings_with_lines) + recurring = {level: count for level, count in counts.items() if count >= 2} + if not recurring: + return min(counts) + max_count = max(recurring.values()) + return min(level for level, count in recurring.items() if count == max_count) +# @cpt-end:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-infer-level + + +# @cpt-begin:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-retrieval-sections +def _build_retrieval_sections( + headings_with_lines: List[Tuple[int, str, int]], + lines: List[str], + section_level: Optional[int], +) -> List[Dict[str, Any]]: + """Group headings at exactly ``section_level`` into retrieval sections. + + Deliberately an *exact* level match, not "level <= section_level": the + same unreliable level-assignment this whole mechanism exists to work + around means a stray heading numerically shallower than the real + chapter level (like the H3 in the docstring above, sitting inside what + is structurally an H5 chapter) is not a trustworthy higher-level + boundary -- it's noise. Content under an off-level heading stays inside + whichever ``section_level`` section it falls under, rather than + splitting a real section apart. + + Each section's ``hash`` is a SHA-256 of its own text slice, with each + line's trailing whitespace stripped before hashing -- a harmless + "trim trailing whitespace on save" edit (a common editor/IDE default) + changes no meaningful content and must not look like a real edit to + :func:`diff_stale_sections`, which is the whole point of hashing at + section granularity in the first place. The per-section granularity + :func:`diff_stale_sections` needs to tell "this one section changed" + from "the whole file changed", which a whole-file fingerprint + structurally cannot do. + """ + if section_level is None: + return [] + line_count = len(lines) + marks = [(text, line_start) for level, text, line_start in headings_with_lines if level == section_level] + sections: List[Dict[str, Any]] = [] + for i, (text, line_start) in enumerate(marks): + line_end = marks[i + 1][1] - 1 if i + 1 < len(marks) else line_count + hash_text = "\n".join(line.rstrip() for line in lines[line_start - 1:line_end]) + sections.append({ + "heading": text, + "line_start": line_start, + "line_end": line_end, + "hash": hashlib.sha256(hash_text.encode("utf-8")).hexdigest(), + "summary": None, + }) + return sections +# @cpt-end:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-retrieval-sections + + +_MAX_READ_ATTEMPTS = 3 + + +# @cpt-begin:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-stable-read +def _read_with_stable_etag(path: Path) -> Tuple[str, str]: + """Read a file's content together with an etag proven to match it. + + A write landing between reading the content and computing the etag + could otherwise save headings parsed from the *old* content stamped + with the *new* file's etag -- :func:`load_doc_index` would then treat + that stale index as valid until a later edit changes the etag again, + since nothing about the fingerprint itself would look wrong. + + Fixed by bracketing the read with a stat snapshot on each side: if they + match, the file didn't change during the read, so the etag genuinely + describes the content just read. If they don't, retry. After + ``_MAX_READ_ATTEMPTS`` under sustained contention, return the last read + anyway, stamped with its own trailing etag -- the safe direction to + fail in, since a file still being rewritten that fast will simply look + stale again on the very next check, never silently wrong. + """ + etag_after = _compute_etag(path) + for _ in range(_MAX_READ_ATTEMPTS): + etag_before = etag_after + content = path.read_text(encoding="utf-8") + etag_after = _compute_etag(path) + if etag_before == etag_after: + return content, etag_after + return content, etag_after +# @cpt-end:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-stable-read + + +# @cpt-begin:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-build +def build_doc_index(path: Path) -> Dict[str, Any]: + """Build a fresh structural index for a Markdown file. + + Purely deterministic -- headings, section line ranges, and an etag. + Contains no LLM-generated content; per-section ``summary`` fields start + as ``None`` and are filled in later via :func:`annotate_section_summary`. + + ``sections`` lists *every* heading, any level (unchanged from before -- + still what :func:`annotate_section_summary` matches against by + ``line_start``). ``retrieval_sections`` is the coarser, inferred + "one chunk per real chapter" grouping a future TF-IDF/cascade/OKF + caller should read against instead -- see :func:`infer_section_level` + for why a fixed heading level can't be assumed. + """ + canonical_path = path.resolve() + content, etag = _read_with_stable_etag(canonical_path) + lines = content.split("\n") + line_count = len(lines) + + headings = parse_headings_with_lines(lines) + sections: List[Dict[str, Any]] = [] + for i, (level, text, line_start) in enumerate(headings): + line_end = headings[i + 1][2] - 1 if i + 1 < len(headings) else line_count + sections.append({ + "level": level, + "heading": text, + "line_start": line_start, + "line_end": line_end, + "summary": None, + }) + + section_level = infer_section_level(headings) + + return { + "schema_version": _SCHEMA_VERSION, + "path": str(canonical_path), + "etag": etag, + "built_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "total_lines": line_count, + "sections": sections, + "section_level": section_level, + "retrieval_sections": _build_retrieval_sections(headings, lines, section_level), + } +# @cpt-end:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-build + + +def _read_cache_file(cache_path: Path) -> Optional[Dict[str, Any]]: + """Read and parse a cache file, or ``None`` if missing/corrupt. + + No staleness check -- just "can this be read as JSON at all". Shared by + :func:`load_doc_index` (which layers the etag check on top) and + :func:`diff_stale_sections` (which deliberately reads a cache the + whole-file etag already considers stale, to compare it section by + section instead of discarding it outright). + """ + try: + return json.loads(cache_path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as exc: + # Reached only once the caller has already confirmed the cache file + # exists, so a failure here is real corruption or a permissions + # problem, not a routine cache miss -- warning, not debug, so it's + # visible at the CLI's default log level instead of masquerading + # as an ordinary first-time build. + logger.warning("doc-index cache unreadable at %s: %s", cache_path, exc) + return None + + +# @cpt-begin:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-load +_REQUIRED_INDEX_FIELDS = ("total_lines", "sections", "section_level", "retrieval_sections") + + +def _has_schema_current_index(cached: Dict[str, Any]) -> bool: + """``True`` only if ``cached`` carries every field a consumer + (``commands/doc_index.py``, :func:`annotate_section_summary`) reads by + subscript, at the schema version this module currently writes -- + treated the same as a stale/corrupt cache otherwise, so a partially + written, hand-edited, or pre-schema-bump cache triggers a clean rebuild + instead of a ``KeyError`` deep in a consumer. + """ + if cached.get("schema_version") != _SCHEMA_VERSION: + return False + return all(field in cached for field in _REQUIRED_INDEX_FIELDS) + + +def load_doc_index(path: Path) -> Optional[Dict[str, Any]]: + """Load a cached index for ``path``, or ``None`` if missing/stale/absent. + + Staleness is detected from cheap ``Path.stat()`` metadata alone -- this + never reads the file's content, so a cache *hit* stays free of a full + read (the property the whole cache exists to provide). Only a stale or + absent cache falls through to :func:`build_doc_index`, which does the + one real read. + + A matching etag alone isn't enough: a cache written by an older version + of this module (before ``section_level``/``retrieval_sections`` + existed) can have a matching etag if the file hasn't changed since, but + a caller reading those fields on it would hit a ``KeyError`` rather + than a clean rebuild. Treated the same as a stale cache -- rebuilt, + not crashed on. + """ + cache_path = _index_cache_path(path) + if cache_path is None or not cache_path.is_file(): + return None + + cached = _read_cache_file(cache_path) + if cached is None: + return None + + canonical_path = path.resolve() + try: + current_etag = _compute_etag(canonical_path) + except OSError as exc: + # The cache file was just confirmed to exist, so a stat() failure + # on the *source* file here means it vanished or became unreadable + # since -- a real anomaly, not a routine miss. + logger.warning("doc-index staleness check failed for %s: %s", path, exc) + return None + + if cached.get("etag") != current_etag: + return None + if not _has_schema_current_index(cached): + logger.debug("doc-index cache for %s is malformed or predates the current schema; rebuilding", path) + return None + return cached +# @cpt-end:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-load + + +# @cpt-begin:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-save +def save_doc_index(path: Path, index: Dict[str, Any]) -> None: + """Persist an index to its cache location. No-ops outside a Studio project. + + Written atomically (temp file + ``os.replace``): a reader racing a + concurrent writer sees either the old complete file or the new complete + one, never a torn/partial write. + """ + cache_path = _index_cache_path(path) + if cache_path is None: + return + cache_path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = cache_path.with_name(f"{cache_path.name}.{os.getpid()}.tmp") + tmp_path.write_text(json.dumps(index, indent=2), encoding="utf-8") + os.replace(tmp_path, cache_path) +# @cpt-end:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-save + + +# @cpt-begin:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-get-or-build +def get_or_build_doc_index(path: Path, *, force_rebuild: bool = False) -> Dict[str, Any]: + """Return the cached index for ``path``, building and caching it if needed. + + This is the "read once per file" entrypoint: the first call for a given + file (or the first call after it changes) pays the parse cost and writes + the cache; every subsequent call against an unchanged file returns the + cached result directly. ``index["cache_hit"]`` reports which happened, + for benchmarking. + """ + if not force_rebuild: + cached = load_doc_index(path) + if cached is not None: + cached["cache_hit"] = True + return cached + + fresh = build_doc_index(path) + save_doc_index(path, fresh) + fresh["cache_hit"] = False + return fresh +# @cpt-end:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-get-or-build + + +# @cpt-begin:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-diff-stale-helpers +def _compute_fresh_retrieval_sections(path: Path) -> Optional[List[Dict[str, Any]]]: + """Re-parse a file's current content into retrieval sections, for + comparison against a cached build. ``None`` on a read failure (e.g. the + file was deleted after it was cached).""" + canonical_path = path.resolve() + try: + content = canonical_path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + # Deliberately still debug, unlike this module's other fallback + # logs: this one path has a genuinely expected trigger ("the file + # was deleted after it was cached", per this function's own + # contract) alongside the anomalous ones, so promoting it would + # make a normal outcome noisy rather than making a real anomaly + # visible. + logger.debug("doc-index section diff failed for %s: %s", path, exc) + return None + + lines = content.split("\n") + headings = parse_headings_with_lines(lines) + section_level = infer_section_level(headings) + return _build_retrieval_sections(headings, lines, section_level) + + +def _position_entry(section: Dict[str, Any]) -> Dict[str, Any]: + """The (heading, line_start) pair identifying one retrieval section in + a :func:`diff_stale_sections` result -- ``line_start`` is what actually + disambiguates two sections sharing a duplicate heading title.""" + return {"heading": section["heading"], "line_start": section["line_start"]} +# @cpt-end:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-diff-stale-helpers + + +# @cpt-begin:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-diff-stale +def diff_stale_sections(path: Path) -> Optional[Dict[str, Any]]: + """Compare the current file against its last cached build at *section* + granularity, not just "is the whole file's cache stale". + + This is what makes a real partial rebuild possible: :func:`load_doc_index` + answers "did anything change" (whole-file, via the etag); this answers + "which retrieval sections actually changed", so a caller doing expensive + per-section work (e.g. an LLM re-summarizing one section) can skip the + ones that didn't. + + Returns ``None`` when there's nothing to diff against -- never built, no + Studio directory, or the cached build predates ``retrieval_sections`` + (an older index format) -- callers should treat that as "everything is + new" and do a full build instead. + + Otherwise returns ``{"structural_change": bool, "unchanged": [...], + "changed": [...]}``, where each entry is ``{"heading": str, "line_start": + int}`` -- the *current* (fresh) position, in document order. Sections + are matched by *position*, not heading text: duplicate heading titles + are real (see the ``toc-heading-duplicate`` check), so heading text + alone can't tell two same-named sections apart -- ``line_start`` is + what a caller should actually use to address "this specific section" + afterwards (e.g. to call :func:`annotate_section_summary`), with the + heading text included only for human-readable logging. When the + section *count* itself differs, ``structural_change`` is ``True`` and + ``changed``/``unchanged`` aren't populated -- a position-based diff + across a changed count can't be safely narrowed to "which ones + changed" without guessing, so the caller should fall back to a full + rebuild rather than have this function guess for it. + """ + cache_path = _index_cache_path(path) + if cache_path is None or not cache_path.is_file(): + return None + + cached = _read_cache_file(cache_path) + if cached is None or "retrieval_sections" not in cached: + return None + + fresh_sections = _compute_fresh_retrieval_sections(path) + if fresh_sections is None: + return None + + old_sections = cached["retrieval_sections"] + if len(old_sections) != len(fresh_sections): + return { + "structural_change": True, + "unchanged": [], + "changed": [_position_entry(s) for s in fresh_sections], + } + + unchanged: List[Dict[str, Any]] = [] + changed: List[Dict[str, Any]] = [] + for old, new in zip(old_sections, fresh_sections, strict=True): + (unchanged if old["hash"] == new["hash"] else changed).append(_position_entry(new)) + return {"structural_change": False, "unchanged": unchanged, "changed": changed} +# @cpt-end:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-diff-stale + + +def _with_cache_lock(cache_path: Path, fn): + """Run ``fn()`` -- a read-modify-write cycle against ``cache_path`` -- + under an exclusive lock on a sibling ``.lock`` file, serializing + concurrent callers so two overlapping read-modify-write cycles (e.g. + two :func:`annotate_section_summary` calls for different sections of + the same document, running from separate processes) can't each load + the same base index, mutate their own part, and have whichever writes + last silently discard the other's update. Mirrors + :func:`studio.utils.decision_log._append_locked`'s exact fallback: an + exclusive ``fcntl`` lock where available (POSIX), otherwise runs + ``fn()`` unlocked on platforms without it (e.g. Windows) -- atomicity + of each individual write is already guaranteed by :func:`save_doc_index` + regardless; only the cross-call serialization is best-effort there. + """ + try: + import fcntl # pylint: disable=import-outside-toplevel + except ImportError: + return fn() + lock_path = cache_path.with_name(f"{cache_path.name}.lock") + lock_path.parent.mkdir(parents=True, exist_ok=True) + with open(lock_path, "a", encoding="utf-8") as lock_fh: + fcntl.flock(lock_fh.fileno(), fcntl.LOCK_EX) + return fn() + + +# @cpt-begin:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-annotate +def annotate_section_summary(path: Path, line_start: int, summary: str) -> bool: + """Attach a one-line summary to a cached section, keyed by its line_start. + + Summaries are written by an LLM caller during a one-time enrichment + pass, never generated inside this module. Returns ``False`` when no + valid (non-stale) cached index exists or no section matches + ``line_start`` -- callers should build the index first. + + Updates the matching entry in both ``sections`` (any heading level) and + ``retrieval_sections`` (the coarser grouping) when both have a section + starting at ``line_start`` -- a retriever reading ``retrieval_sections`` + needs the summary to show up there too, not just in the finer-grained + list. A ``line_start`` that only matches ``sections`` (an off-level + heading that isn't itself a retrieval section's start) updates only + that list, which is correct: there is no corresponding retrieval + section to update. + + The read-modify-write cycle (load, mutate one section, save) runs + under :func:`_with_cache_lock`, so two concurrent calls annotating + different sections of the same document don't race and silently drop + one side's update. + """ + cache_path = _index_cache_path(path) + if cache_path is None: + return False + + def _read_modify_write() -> bool: + index = load_doc_index(path) + if index is None: + return False + + matched = False + for section in index["sections"]: + if section["line_start"] == line_start: + section["summary"] = summary + matched = True + break + if not matched: + return False + + for retrieval_section in index.get("retrieval_sections", []): + if retrieval_section["line_start"] == line_start: + retrieval_section["summary"] = summary + break + + save_doc_index(path, index) + return True + + return _with_cache_lock(cache_path, _read_modify_write) +# @cpt-end:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-annotate diff --git a/skills/studio/scripts/studio/utils/error_codes.py b/skills/studio/scripts/studio/utils/error_codes.py index 3a45a4ad..8800c40a 100644 --- a/skills/studio/scripts/studio/utils/error_codes.py +++ b/skills/studio/scripts/studio/utils/error_codes.py @@ -111,6 +111,12 @@ TOC_HEADING_NOT_IN_TOC = "toc-heading-not-in-toc" TOC_STALE = "toc-stale" +# JIT-retrieval readiness signals (warning-only) — see constructorfabric/studio#104 +TOC_HEADING_DUPLICATE = "toc-heading-duplicate" +TOC_HEADING_DEPTH_JUMP = "toc-heading-depth-jump" +TOC_SECTION_TOO_LONG = "toc-section-too-long" +TOC_MISSING_DESCRIPTION = "toc-missing-description" + # --------------------------------------------------------------------------- # File errors # --------------------------------------------------------------------------- diff --git a/skills/studio/scripts/studio/utils/eval_semantic.py b/skills/studio/scripts/studio/utils/eval_semantic.py new file mode 100644 index 00000000..73572d7d --- /dev/null +++ b/skills/studio/scripts/studio/utils/eval_semantic.py @@ -0,0 +1,773 @@ +"""Semantic coverage — *does a marked block implement the requirement it cites?* + +Structural ``spec-coverage`` scores marker **density**: a file can read 100% covered while the +code inside its markers does the wrong thing. This engine adds the layer density cannot reach — +**code-vs-requirement correctness** — and it does so honestly: + +* **Rank first, judge last.** A deterministic, stdlib-only token-overlap pre-filter scores every + marked block against its requirement text and surfaces the **weak links** (low overlap). Only + those go to a model, so a large repo triggers **zero model calls per block**. The pre-filter is a + *budget heuristic*, not a correctness oracle: strong overlap → ``presumed_covered`` buys a block + out of a model call, it never proves the block correct. Lexical overlap is gameable, so + ``presumed_covered`` is not evidence of correctness — only a judged verdict is. +* **Seam, not transport.** Like the rules-judge, the model call is a pluggable ``SemanticJudgeFn`` + supplied out-of-tree; with none wired the weak links are ``UNJUDGEABLE`` (never a false verdict) + and nothing gates. This module contains no model client. +* **Advisory, never gates.** A verdict here never touches an exit code (enforced at integration). +* **Honest coverage.** Blocks with no retrievable requirement, or too little text to compare, are + reported ``UNJUDGEABLE`` — never a silent "covered". The report states what it could not judge. +* **Scoped by the frozen coverage contract.** Files a human declared ``excluded`` are skipped; + files flagged ``whole_file_claims`` (scope-only) are prioritised — that is where a green + structural number most plausibly hides wrong code. + +@cpt-algo:cpt-studio-algo-eval-semantic:p1 +""" +# @cpt-begin:cpt-studio-algo-eval-semantic:p1:inst-semantic-imports +from __future__ import annotations + +import io +import logging +import re +import tokenize as _tokenize # aliased: this module defines its own public ``tokenize`` word-splitter +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, List, Optional, Protocol, Sequence, Set, Tuple, runtime_checkable + +from .document import get_content_scoped +from .manifest import load_toml_file + +logger = logging.getLogger(__name__) + +#: The three model verdicts, plus the honest "could not compare" outcome the engine owns. +SEM_COVERED = "covered" +SEM_PARTIAL = "partial" +SEM_WRONG = "wrong" +SEM_UNJUDGEABLE = "unjudgeable" +_MODEL_VERDICTS = frozenset({SEM_COVERED, SEM_PARTIAL, SEM_WRONG}) + +#: Schema version stamped on ``SemanticReport`` / ``SemanticCalibration`` so a coverage-report +#: consumer can detect a shape change. Bump on any breaking field change. +SEMANTIC_SCHEMA_VERSION = 1 + +#: Provisional calibration constants — a comparison needs at least this many domain tokens on each +#: side, and a block scoring below the threshold is a weak link. Named here (not buried in logic) +#: so calibration on our own corpus can retune them. They are **not yet corpus-calibrated** — their +#: empirical basis is future work (see the design note); these are conservative defaults. +_MIN_TOKENS = 4 +_WEAK_LINK_THRESHOLD = 0.30 +#: Per-field cap on the requirement/code interpolated into the judge prompt, so a huge block or +#: requirement can never produce an unbounded prompt. The structured request fields stay full (for +#: the evidence guard and a host that builds its own prompt); only the prompt string is bounded. +_PROMPT_FIELD_CAP = 2000 + +#: Syntax words that are not evidence a requirement was implemented — dropped before overlap so a +#: score reflects domain vocabulary, not boilerplate shared by every block. +_STOPWORDS = frozenset({ + "the", "a", "an", "and", "or", "not", "is", "are", "be", "to", "of", "in", "on", "for", + "with", "as", "if", "else", "elif", "return", "def", "class", "self", "import", "from", + "none", "true", "false", "pass", "raise", "try", "except", "this", "that", "it", +}) +#: Split on Unicode non-word runs *and* underscore, so accented / non-Latin terms (Gebühr, Cyrillic) +#: stay whole instead of fragmenting to empty — an ASCII-only class silently deflated the pre-filter +#: on multilingual corpora and could report a real block UNJUDGEABLE. Transliteration mismatches +#: (gebuehr vs Gebühr) remain inherent to lexical overlap. +_TOKEN_SPLIT = re.compile(r"[\W_]+") +_CAMEL_SPLIT = re.compile(r"(?<=[a-z0-9])(?=[A-Z])") +# @cpt-end:cpt-studio-algo-eval-semantic:p1:inst-semantic-imports + + +# @cpt-begin:cpt-studio-algo-eval-semantic:p1:inst-semantic-datamodel +@dataclass(frozen=True) +class Pairing: + """One marked code block paired with the requirement text it cites. The engine's unit of work. + + ``requirement`` is ``None`` when no scoped requirement could be retrieved for the block's id — + an honest "unjudgeable", never treated as an empty requirement the code trivially satisfies. + """ + + block_id: str + inst: str + path: str + start_line: int + code: str + requirement: Optional[str] + + +@dataclass(frozen=True) +class Ranked: + """A pairing scored by the deterministic pre-filter.""" + + pairing: Pairing + score: Optional[float] # overlap in [0, 1]; None when unjudgeable (below the floor) + weak_link: bool # scored and below the weak-link threshold → send to the judge + reason: str # why unjudgeable / why a weak link (human-readable) + forced: bool = False # judged because it is a whole_file_claim, regardless of overlap + + +@dataclass(frozen=True) +class SemanticRequest: + """The deterministic input handed to a ``SemanticJudgeFn`` — pure, no model call.""" + + block_id: str + requirement: str + code: str + prompt: str + + +@dataclass(frozen=True) +class SemanticReply: + """A model's structured answer, parsed back by the engine.""" + + verdict: str # covered | partial | wrong + rationale: str = "" + evidence_quote: str = "" # a substring of the code the verdict rests on (grep-verified) + + +@runtime_checkable +class SemanticJudgeFn(Protocol): # pylint: disable=too-few-public-methods + """The seam the host/agent supplies: turn a ``SemanticRequest`` into a ``SemanticReply``. + + Never implemented here with a real model — that lives out-of-tree. Tests and calibration use + a deterministic stub. + + **Latency is the implementer's contract.** The engine calls this *synchronously* and does not + bound its runtime: a judge that blocks (e.g. an un-timed network call) blocks ``assess`` / + ``calibrate`` for exactly as long as it runs. An implementation that can hang MUST enforce its + own timeout and raise on expiry — a raise degrades that one finding to UNJUDGEABLE (never sinks + the run), whereas a hang has no in-engine recourse. + """ + + def __call__(self, request: SemanticRequest) -> SemanticReply: + """Return the model's verdict on ``request``.""" # pragma: no cover +# @cpt-end:cpt-studio-algo-eval-semantic:p1:inst-semantic-datamodel + + +# @cpt-begin:cpt-studio-algo-eval-semantic:p1:inst-semantic-blank +def _blank(code: str, spans: Sequence[Tuple[int, int]]) -> str: + """Replace each ``[start, end)`` character span with spaces, preserving newlines and every other + character's position — so a verbatim quote of the untouched text still matches as a substring.""" + if not spans: + return code + chars = list(code) + for start, end in spans: + for i in range(start, min(end, len(chars))): + if chars[i] != "\n": + chars[i] = " " + return "".join(chars) +# @cpt-end:cpt-studio-algo-eval-semantic:p1:inst-semantic-blank + + +# @cpt-begin:cpt-studio-algo-eval-semantic:p1:inst-semantic-strip +#: Fallback-only: strip ``#`` comment tails when a block will not tokenize (a mid-file fragment). +#: The grammar-aware ``tokenize`` pass below is the real lexer; Python's comments and string literals +#: are mutually recursive (``#`` lives in strings, ``'''`` lives in comments), so no sequence of +#: independent regexes can strip them correctly — only a single left-to-right scan can. +_LINE_COMMENT = re.compile(r"#[^\n]*") +#: Token types a string can follow to be a *statement-leading* (docstring-like) string rather than an +#: inline value — a suite start (INDENT), a statement break (NEWLINE), or a block close (DEDENT, which +#: ends an inner suite so the next token opens a fresh statement). Such prose is blanked from the +#: evidence view; an inline string (after ``=``, ``(``, …) never follows one of these. +_STATEMENT_START = frozenset({_tokenize.NEWLINE, _tokenize.INDENT, _tokenize.DEDENT}) +#: PEP 701 (Python 3.12+) tokenizes an f-string as FSTRING_START/MIDDLE/END around its interior +#: expressions, not a single STRING token. FSTRING_MIDDLE carries the literal *text* (prose); the +#: interior expressions are ordinary tokens (kept). ``None`` on < 3.12, where f-strings are STRING. +_FSTRING_MIDDLE = getattr(_tokenize, "FSTRING_MIDDLE", None) + + +def _code_views(code: str) -> Tuple[str, str, bool]: + """One ``tokenize`` pass → ``(overlap_view, evidence_view, lexed)``; comment/string spans blanked in place. + + * ``overlap_view`` blanks comments **and every string literal**, so prose (a comment, docstring, or + string echoing the requirement) can never inflate the overlap score and mask wrong code. + * ``evidence_view`` blanks comments **and docstrings** but keeps inline string literals, so a quote + of a real string the code executes (an error message, SQL, a regex) is valid evidence while a + quote lifted only from a comment or docstring is not. A docstring is a statement-leading string. + * ``lexed`` is ``False`` when the block did not tokenize (an indentation-broken or unterminated + fragment). The fallback can only comment-strip, so a string literal would survive into the + overlap view — ``overlap_score`` therefore treats ``lexed=False`` as **unjudgeable** (returns + ``None``) rather than risk a prose-inflated ``presumed_covered``. Never raises. + + Grammar-aware, so ``#`` inside a string and ``'''`` inside a comment are handled correctly, and an + f-string's literal text is blanked from overlap on Python 3.12+ (PEP 701) where it is no longer a + single STRING token. + """ + # Split on ``\n`` only, matching ``io.StringIO(...).readline`` below — NOT ``str.splitlines()``, + # which also breaks on form-feed / NEL / LS / PS and would insert phantom lines the tokenizer's + # row numbering lacks, desyncing every later span. The trailing surplus entry is never indexed. + line_start = [0] + for segment in code.split("\n"): + line_start.append(line_start[-1] + len(segment) + 1) + all_strings_and_comments: List[Tuple[int, int]] = [] # overlap view blanks these + comments_and_docstrings: List[Tuple[int, int]] = [] # evidence view blanks these + prev = _tokenize.NEWLINE # file start behaves like a statement break + try: + for tok in _tokenize.generate_tokens(io.StringIO(code).readline): + span = (line_start[tok.start[0] - 1] + tok.start[1], + line_start[tok.end[0] - 1] + tok.end[1]) + if tok.type == _tokenize.COMMENT: + all_strings_and_comments.append(span) + comments_and_docstrings.append(span) + elif tok.type == _tokenize.STRING: + all_strings_and_comments.append(span) + if prev in _STATEMENT_START: # a statement-leading string = docstring + comments_and_docstrings.append(span) + elif tok.type == _FSTRING_MIDDLE: # PEP 701 f-string literal text (Py 3.12+) + all_strings_and_comments.append(span) # blank the prose from overlap; the interior + # expressions are separate tokens, kept. An + # f-string is an inline value → kept for evidence. + if tok.type not in (_tokenize.NL, _tokenize.COMMENT): + prev = tok.type + except (_tokenize.TokenError, SyntaxError, ValueError): # IndentationError ⊂ SyntaxError + stripped = _LINE_COMMENT.sub(" ", code) # best-effort: comments only (strings survive) + return stripped, stripped, False # lexed=False → caller treats as unjudgeable + return _blank(code, all_strings_and_comments), _blank(code, comments_and_docstrings), True +# @cpt-end:cpt-studio-algo-eval-semantic:p1:inst-semantic-strip + + +# @cpt-begin:cpt-studio-algo-eval-semantic:p1:inst-semantic-tokenize +def tokenize(text: str) -> Set[str]: + """Domain tokens of ``text``: split on Unicode word boundaries *and* camelCase, casefold, drop + stopwords and 1-char fragments. Deterministic — the basis of every overlap score.""" + out: Set[str] = set() + for raw in _TOKEN_SPLIT.split(text): + for piece in _CAMEL_SPLIT.split(raw): + token = piece.casefold() + if len(token) > 1 and token not in _STOPWORDS: + out.add(token) + return out + + +def overlap_score(code: str, requirement: str) -> Optional[float]: + """Fraction of the requirement's domain tokens present in the code, in [0, 1]. + + An **advisory budget heuristic**, never a correctness proof: a strong score buys a block out of + a model call, it does not show the block is correct. Comments/docstrings/strings are stripped from + the code first so *prose* echoing the requirement cannot inflate the score. A residual gap remains + and is inherent to any lexical measure: executable identifiers named after the requirement can push + the score **above** the threshold, so the block is ``presumed_covered`` and never judged — the + judge (which only sees weak links) does not close this case; only a semantic model would. + Scored against the (smaller) requirement set — "does the code cover what the requirement asks", + not the reverse. Returns ``None`` when either side has fewer than ``_MIN_TOKENS`` domain tokens + (too little to compare honestly) **or** the block did not tokenize (the fallback can't strip + strings, so scoring it could mask wrong code) — unjudgeable, not zero. + """ + overlap_view, _evidence, lexed = _code_views(code) + if not lexed: # could not lex → cannot honestly score; never presumed_covered + return None + code_tokens = tokenize(overlap_view) + req_tokens = tokenize(requirement) + if len(code_tokens) < _MIN_TOKENS or len(req_tokens) < _MIN_TOKENS: + return None + return len(code_tokens & req_tokens) / len(req_tokens) +# @cpt-end:cpt-studio-algo-eval-semantic:p1:inst-semantic-tokenize + + +# @cpt-begin:cpt-studio-algo-eval-semantic:p1:inst-semantic-prefilter +def _rank_one(pairing: Pairing, priority: Set[str]) -> Ranked: + """Score a single pairing → ``Ranked``. Unjudgeable when no requirement or below the floor. + + A pairing in a ``priority`` (``whole_file_claims``) file is **always** a weak link regardless of + overlap: its structural coverage rests on a whole-file scope marker, so its lexical overlap is + untrustworthy (comments/leftover text inflate it) and it is exactly where wrong code hides — a + high score there must not buy a free pass. This is what "prioritise" means: always judge it. + """ + if not pairing.requirement: + return Ranked(pairing, None, False, "no retrievable requirement for this id") + score = overlap_score(pairing.code, pairing.requirement) + if score is None: + return Ranked(pairing, None, False, + "too little text to compare (below token floor) or the block did not tokenize") + if _norm_path(pairing.path) in priority: + return Ranked(pairing, score, True, + f"whole-file claim — always judged (overlap {score:.2f})", forced=True) + if score < _WEAK_LINK_THRESHOLD: + return Ranked(pairing, score, True, f"low overlap {score:.2f} < {_WEAK_LINK_THRESHOLD}") + return Ranked(pairing, score, False, f"overlap {score:.2f}") + + +def rank_pairings(pairings: Sequence[Pairing], priority_paths: Sequence[str] = ()) -> List[Ranked]: + """Rank every pairing weakest-first so the judge budget is spent where it matters. + + A ``whole_file_claims`` file is always judged (see ``_rank_one``). Sort key: weak links before + strong, then blocks in a ``priority_paths`` file ahead of the rest, then by ascending overlap. + Unjudgeable blocks sort last — they carry no signal and are reported separately as coverage gaps. + """ + priority = {_norm_path(p) for p in priority_paths} + ranked = [_rank_one(p, priority) for p in pairings] + + def key(item: Ranked) -> Tuple[int, int, float]: + if item.weak_link: + group = 0 # weak links first — where the judge budget goes + elif item.score is None: + group = 2 # unjudgeable last — no signal + else: + group = 1 # strong overlap in between + return (group, + 0 if _norm_path(item.pairing.path) in priority else 1, + item.score if item.score is not None else 1.0) + + return sorted(ranked, key=key) +# @cpt-end:cpt-studio-algo-eval-semantic:p1:inst-semantic-prefilter + + +# @cpt-begin:cpt-studio-algo-eval-semantic:p1:inst-semantic-scope +@dataclass(frozen=True) +class CoverageScope: + """Scoping read from the frozen coverage-report contract (excluded / whole-file-claim files).""" + + excluded: Set[str] = field(default_factory=set) # human-declared, skip entirely + prioritised: List[str] = field(default_factory=list) # whole_file_claims, in the producer's order + + +def _norm_path(path: str) -> str: + """Normalise a path for scope comparison: back- to forward-slashes and a stripped leading + ``./``, so a report path and a pairing path that differ only by separator style still match. + Matching stays **case-sensitive** — case-folding would wrongly merge distinct files on a + case-sensitive filesystem (the common one for this codebase).""" + norm = path.replace("\\", "/") + return norm[2:] if norm.startswith("./") else norm + + +def _paths_from(report: Dict[str, object], key: str) -> List[str]: + """Extract normalised ``path`` strings from ``report[key]`` (a list of dicts), tolerating a bad shape.""" + rows = report.get(key) + if not isinstance(rows, list): + return [] + return [_norm_path(row["path"]) for row in rows + if isinstance(row, dict) and isinstance(row.get("path"), str)] + + +def coverage_scope(report: Optional[Dict[str, object]]) -> CoverageScope: + """Read ``excluded[]`` / ``whole_file_claims[]`` from a coverage report, degrading to empty. + + The producer (the coverage-report reporting change) may not have emitted these yet; a report + without them simply yields an empty scope — skip nothing, prioritise nothing — never an error. + """ + if not isinstance(report, dict): + return CoverageScope() + return CoverageScope(excluded=set(_paths_from(report, "excluded")), + prioritised=_paths_from(report, "whole_file_claims")) +# @cpt-end:cpt-studio-algo-eval-semantic:p1:inst-semantic-scope + + +# @cpt-begin:cpt-studio-algo-eval-semantic:p1:inst-semantic-prompt +def _capped(text: str) -> str: + """Trim ``text`` to ``_PROMPT_FIELD_CAP``, marking a cut so the judge sees it was truncated.""" + return text if len(text) <= _PROMPT_FIELD_CAP else text[:_PROMPT_FIELD_CAP].rstrip() + "\n[…truncated]" + + +def build_semantic_request(ranked: Ranked) -> SemanticRequest: + """Assemble the deterministic judge prompt for one weak link — pure, no model call. The + requirement and code are capped in the prompt so it stays bounded; the returned request keeps + the full fields for the evidence guard and any host that builds its own prompt.""" + pairing = ranked.pairing + requirement = pairing.requirement or "" + prompt = ( + "You are judging whether a block of code implements the requirement it cites. This is an " + "advisory judgement; it never gates a build.\n\n" + f"REQUIREMENT (id {pairing.block_id}):\n{_capped(requirement)}\n\n" + f"CODE ({pairing.path}:{pairing.start_line}):\n{_capped(pairing.code)}\n\n" + "Answer with a verdict of 'covered', 'partial', or 'wrong', a one-line rationale, and an " + "evidence_quote copied verbatim from the CODE above that your verdict rests on.") + return SemanticRequest(pairing.block_id, requirement, pairing.code, prompt) + + +def _reply_to_verdict(reply: object) -> str: + """Map a reply to a verdict; anything unrecognised — ``None``, a bad object, a **non-string** + ``verdict`` (e.g. an int), or an unknown string — is UNJUDGEABLE. A first line of defence: it + never raises on a *missing* or non-string attribute; ``_judge_one`` additionally wraps this call + so a reply whose attribute *access* raises degrades to UNJUDGEABLE instead of sinking the run.""" + verdict = getattr(reply, "verdict", "") + if not isinstance(verdict, str): + return SEM_UNJUDGEABLE + verdict = verdict.strip().lower() + return verdict if verdict in _MODEL_VERDICTS else SEM_UNJUDGEABLE + + +# @cpt-end:cpt-studio-algo-eval-semantic:p1:inst-semantic-prompt + + +# @cpt-begin:cpt-studio-algo-eval-semantic:p1:inst-semantic-evidence +def evidence_present(code: str, quote: str) -> bool: + """Hallucination guard: the model's ``evidence_quote`` must actually occur in the *executable* + code. + + The haystack is the ``_code_views`` evidence view — comments and docstrings removed, **inline + string literals kept** — so a quote lifted only from a comment or docstring (e.g. a stale comment + describing intended-but-unimplemented behaviour) is **not** accepted, while a quote of a real + string the code executes (an error message, SQL, a regex — often the exact evidence) still is. + Whitespace-normalised so trivial reformatting does not fail a genuine quote; an empty quote is + not evidence. A deterministic check sitting *under* the non-deterministic judge. + """ + needle = " ".join(quote.split()) + haystack = " ".join(_code_views(code)[1].split()) + return bool(needle) and needle in haystack +# @cpt-end:cpt-studio-algo-eval-semantic:p1:inst-semantic-evidence + + +# @cpt-begin:cpt-studio-algo-eval-semantic:p1:inst-semantic-finding +@dataclass(frozen=True) +class SemanticFinding: + """One judged weak link — the advisory unit the report carries.""" + + block_id: str + path: str + start_line: int + verdict: str # covered | partial | wrong | unjudgeable + rationale: str + evidence_ok: bool # did evidence_present() confirm the quote? + forced: bool = False # judged because its file is a whole_file_claim, not by low overlap + + +def judge_weak_links(ranked: Sequence[Ranked], + judge_fn: Optional[SemanticJudgeFn]) -> List[SemanticFinding]: + """Judge only the weak links. With no ``judge_fn`` every weak link is UNJUDGEABLE (advisory). + + A judge that raises or returns a malformed reply degrades that one finding to UNJUDGEABLE — + it never sinks the assessment. Each verdict carries an evidence check (the hallucination + guard); a quote absent from the code sets ``evidence_ok=False`` rather than being trusted. + """ + findings: List[SemanticFinding] = [] + for item in ranked: + if not item.weak_link: + continue + findings.append(_judge_one(item, judge_fn)) + return findings + + +def _judge_one(item: Ranked, judge_fn: Optional[SemanticJudgeFn]) -> SemanticFinding: + """Judge a single weak link defensively → a ``SemanticFinding``.""" + pairing = item.pairing + if judge_fn is None: + return SemanticFinding(pairing.block_id, pairing.path, pairing.start_line, + SEM_UNJUDGEABLE, "no judge model wired (advisory)", False, item.forced) + request = build_semantic_request(item) + try: + reply = judge_fn(request) + # Marshal *inside* the try: the call is not the only thing that can raise. ``reply`` is + # duck-typed ``object``, so a lazily-parsed proxy whose ``.verdict``/``.evidence_quote`` is + # a property can raise any exception on access (getattr's default only swallows + # AttributeError). Reading here degrades that one finding to UNJUDGEABLE rather than + # aborting judge_weak_links/assess — and calibrate, which routes through here too. + verdict = _reply_to_verdict(reply) + quote = getattr(reply, "evidence_quote", "") + rationale = getattr(reply, "rationale", "") + except Exception as exc: # pylint: disable=broad-except + logger.warning("semantic: judge_fn raised on %s: %s", pairing.block_id, exc) + return SemanticFinding(pairing.block_id, pairing.path, pairing.start_line, + SEM_UNJUDGEABLE, f"judge_fn raised: {exc}", False, item.forced) + # A non-string evidence_quote/rationale (a host may return an int or a list) is coerced, not + # trusted. An unrecognised verdict names its own cause rather than echoing the model's text. + quote = quote if isinstance(quote, str) else "" + rationale = rationale if isinstance(rationale, str) else "" + if verdict == SEM_UNJUDGEABLE: + rationale = "judge returned an unrecognized verdict" + return SemanticFinding(pairing.block_id, pairing.path, pairing.start_line, verdict, + rationale, evidence_present(pairing.code, quote), item.forced) +# @cpt-end:cpt-studio-algo-eval-semantic:p1:inst-semantic-finding + + +# @cpt-begin:cpt-studio-algo-eval-semantic:p1:inst-semantic-stub +def reference_stub_judge(request: SemanticRequest) -> SemanticReply: + """A deterministic overlap ``SemanticJudgeFn`` — **not a real model.** + + Ships so tests and calibration exercise the machinery with no model wired. It re-scores the + request's own overlap and buckets it (high→covered, mid→partial, low→wrong), quoting the code + line with the most requirement tokens so the evidence guard has something real to verify. + A real judge_fn is supplied out-of-tree and replaces it. + + **Not an independent oracle.** It buckets the *same* ``overlap_score`` against the *same* + ``_WEAK_LINK_THRESHOLD`` the pre-filter already used to flag the pairing, so it is structurally + correlated with the pre-filter, not an independent check of it. Calibrating against this stub + measures the wiring, not judge quality — meaningful accuracy/consistency need a real judge. + """ + score = overlap_score(request.code, request.requirement) or 0.0 + if score >= _WEAK_LINK_THRESHOLD: + verdict = SEM_COVERED + elif score >= _WEAK_LINK_THRESHOLD / 2: + verdict = SEM_PARTIAL + else: + verdict = SEM_WRONG + return SemanticReply(verdict, f"reference stub: overlap {score:.2f}", + _best_evidence_line(request)) + + +def _best_evidence_line(request: SemanticRequest) -> str: + """The *executable* code line sharing the most tokens with the requirement — a real, verifiable + quote. Selected over the ``_code_views`` evidence view (comments/docstrings blanked, strings kept) + — the same view ``evidence_present`` matches against — so the quote it returns is one the guard + will accept, even on a block whose comment echoes the requirement.""" + req_tokens = tokenize(request.requirement) + # Split on ``\n`` only, consistent with how ``_code_views`` defines "one line" — ``str.splitlines()`` + # would fragment a line at a form-feed/NEL/etc. and could pick a truncated fragment as evidence. + lines = [line for line in _code_views(request.code)[1].split("\n") if line.strip()] + if not lines: + return "" + return max(lines, key=lambda line: len(tokenize(line) & req_tokens)).strip() +# @cpt-end:cpt-studio-algo-eval-semantic:p1:inst-semantic-stub + + +# @cpt-begin:cpt-studio-algo-eval-semantic:p1:inst-semantic-gap +@dataclass(frozen=True) +class SemanticGap: + """One block the engine could not judge — a coverage gap, located like a ``SemanticFinding``. + + Carries ``start_line`` (unlike a bare dict) so a coverage-report consumer can point at every gap + precisely, with a shape consistent with ``findings``. + """ + + block_id: str + path: str + start_line: int + reason: str # no requirement / below the token floor / judge could not verdict +# @cpt-end:cpt-studio-algo-eval-semantic:p1:inst-semantic-gap + + +# @cpt-begin:cpt-studio-algo-eval-semantic:p1:inst-semantic-report +@dataclass +class SemanticReport: + """The advisory result of assessing a set of pairings. Never carries a gate signal. + + Every pairing is accounted for: ``skipped_excluded`` (files a human excluded, dropped before + ranking) + ``assessed`` (weak links that got a real covered/partial/wrong verdict) + + ``presumed_covered`` (strong overlap, not judged) + ``len(unjudgeable)`` (coverage gaps — no + requirement, below the token floor, or the judge could not produce a verdict). So a small + ``assessed`` can never be mistaken for "only this many blocks existed". + + "Nothing to assess" and "could not assess" are distinguishable: an **empty** pairing set yields + an all-zero report with an **empty** ``unjudgeable``, whereas a resolution failure (no + requirement / below floor) yields ``unjudgeable`` **entries** — never a silent all-zero. + """ + + assessed: int # weak links that got a real covered/partial/wrong verdict + presumed_covered: int # strong overlap → not judged; a budget heuristic, NOT a + # correctness claim (lexical overlap is gameable) + unjudgeable: List[SemanticGap] # coverage gaps: no requirement / below floor / judge unjudgeable + findings: List[SemanticFinding] = field(default_factory=list) # only the real-verdict findings + skipped_excluded: int = 0 # blocks in human-excluded files, dropped before ranking + schema_version: int = SEMANTIC_SCHEMA_VERSION # report-shape version for the coverage consumer + + +def assess(pairings: Sequence[Pairing], judge_fn: Optional[SemanticJudgeFn] = None, + report: Optional[Dict[str, object]] = None) -> SemanticReport: + """Assess ``pairings`` end-to-end: scope → rank → judge weak links → honest report. + + Advisory throughout. Files in the coverage report's ``excluded`` set are dropped before + ranking (human-declared, skip safely); every block with no requirement or too little text is + listed in ``unjudgeable`` rather than silently counted covered. ``excluded`` takes precedence + over ``whole_file_claims``: a path in both is dropped here, before ranking ever sees it — the + human exclusion is the override. + """ + scope = coverage_scope(report) + in_scope = [p for p in pairings if _norm_path(p.path) not in scope.excluded] + ranked = rank_pairings(in_scope, scope.prioritised) + presumed = sum(1 for r in ranked if not r.weak_link and r.score is not None) + all_findings = judge_weak_links(ranked, judge_fn) + # A weak link the judge could not verdict (no judge wired / crash / malformed) is a coverage + # gap, not a "judged" result — it joins the unjudgeable list, not the findings. + findings = [f for f in all_findings if f.verdict in _MODEL_VERDICTS] + unjudgeable = [SemanticGap(r.pairing.block_id, r.pairing.path, r.pairing.start_line, r.reason) + for r in ranked if r.score is None] + unjudgeable += [SemanticGap(f.block_id, f.path, f.start_line, f.rationale or "judge unjudgeable") + for f in all_findings if f.verdict == SEM_UNJUDGEABLE] + return SemanticReport(assessed=len(findings), presumed_covered=presumed, + unjudgeable=unjudgeable, findings=findings, + skipped_excluded=len(pairings) - len(in_scope)) +# @cpt-end:cpt-studio-algo-eval-semantic:p1:inst-semantic-report + + +# @cpt-begin:cpt-studio-algo-eval-semantic:p1:inst-semantic-resolve +def resolve_requirement(doc_path: Path, block_id: str) -> Optional[str]: + """Fetch the requirement text scoped to ``block_id`` from a feature doc, or ``None``. + + A thin adapter over ``get_content_scoped``; a ``None`` (id absent / doc unreadable) is the + honest "unjudgeable" signal the pre-filter propagates, never a crash. The id→doc mapping that + drives this at scale is the reporting-integration follow-up; the engine only needs the lookup. + + **Security — caller contract.** ``doc_path`` is opened as given; ``block_id`` never selects a + file, so there is no traversal via the id. The caller owns the path's trust boundary: pass a + path already resolved and confirmed to be within the intended project root. The engine has no + project root of its own to check against, so it cannot enforce containment here. + """ + scoped = get_content_scoped(doc_path, id_value=block_id) + return scoped[0] if scoped else None +# @cpt-end:cpt-studio-algo-eval-semantic:p1:inst-semantic-resolve + + +# @cpt-begin:cpt-studio-algo-eval-semantic:p1:inst-semantic-gold +@dataclass +class SemanticGold: + """A human label for one pairing — the ground truth calibration compares against.""" + + verdict: str # covered | partial | wrong + rationale: str = "" + + +def load_gold(gold_path: Optional[Path]) -> Optional[SemanticGold]: + """Read a ``[gold]`` verdict label, or ``None`` when absent/malformed. Never raises. + + A missing or unreadable gold file means the pairing is not gold-backed, so its verdict is + unvalidated advisory rather than a crash. + + **Security — caller contract.** ``gold_path`` is opened as given. The caller owns the path's + trust boundary: pass a path already resolved and confirmed within the intended project root. The + engine has no project root of its own to check containment against, so it cannot enforce it here. + """ + if gold_path is None: + return None + data = load_toml_file(gold_path) # shared tolerant reader: logs + returns None on OSError/parse + if data is None: + # Case B — the reader returned None for *any* reason. Surface both sub-cases (the reader is + # silent on a missing file, and logs only the low-level detail on a read/parse failure): a + # present-but-unreadable gold file is an unexpected misconfiguration, not "no gold provided". + if gold_path.is_file(): + logger.warning("semantic: gold file present but unreadable/unparseable: %s", gold_path) + else: + logger.warning("semantic: gold file not found: %s", gold_path) + return None + section = data.get("gold") + verdict = section.get("verdict") if isinstance(section, dict) else None + # isinstance-guard before the frozenset test: a TOML array/table verdict is unhashable and would + # raise ``TypeError: unhashable type`` from ``in`` — breaking the "never raises" contract. + if not isinstance(verdict, str) or verdict not in _MODEL_VERDICTS: + logger.warning("semantic: gold needs [gold].verdict in covered|partial|wrong: %s", gold_path) + return None + return SemanticGold(verdict=verdict, rationale=str(section.get("rationale", ""))) +# @cpt-end:cpt-studio-algo-eval-semantic:p1:inst-semantic-gold + + +# @cpt-begin:cpt-studio-algo-eval-semantic:p1:inst-semantic-calibrate +@dataclass +class SemanticCalibration: + """The judge's measured quality over gold-backed pairings.""" + + accuracy: Optional[float] # fraction whose majority matches the label, over cases with a majority + consistency: Optional[float] # majority verdict's mean share of surviving runs, over cases with ≥2 + covered: List[str] # block ids that carry a gold label + runs_per_scenario: int + per_case: List[Dict[str, object]] = field(default_factory=list) + excluded: List[str] = field(default_factory=list) # unscoreable pairing / judge crash — not a mismatch + judge: str = "" # best-effort identity of the judge_fn these numbers describe + schema_version: int = SEMANTIC_SCHEMA_VERSION # calibration-shape version for the consumer + + +def _majority(verdicts: Sequence[str]) -> Tuple[str, int]: + """The most common verdict and its count. Ties resolve by sorted verdict name (canonical), so + the *display* value is independent of run order — not the first-seen order the runs produced. + The canonical pick (alphabetically-first: covered < partial < wrong) must never decide accuracy — + that would skew scoring by the gold label's rank — so ``_calibrate_case`` detects a strict tie + (no single verdict holds the top count) and excludes it from accuracy rather than trusting this.""" + counts: Dict[str, int] = {} + for verdict in verdicts: + counts[verdict] = counts.get(verdict, 0) + 1 + if not counts: + return SEM_UNJUDGEABLE, 0 + best = max(sorted(counts), key=lambda v: counts[v]) + return best, counts[best] + + +def _pairing_unscoreable(pairing: Pairing) -> bool: + """A pairing the pre-filter would mark unjudgeable — no requirement, or too little text to + compare. The judge would never see it in normal operation, so it is excluded from calibration + rather than judged on empty input and scored as a mismatch.""" + if not pairing.requirement: + return True + return overlap_score(pairing.code, pairing.requirement) is None + + +def _calibrate_case(pairing: Pairing, gold: SemanticGold, judge_fn: SemanticJudgeFn, + runs: int) -> "Tuple[bool, Optional[bool], Optional[float], Dict[str, object]]": + """Judge one gold-backed pairing ``runs`` times → ``(unscoreable, matched, consistency, row)``. + ``consistency`` is ``None`` when fewer than two runs survived (run-to-run agreement is unmeasurable + with one verdict, never a false 1.0). ``matched`` is ``None`` on a strict tie (no majority) and on a + *crash-degraded* case (``runs>=2`` reduced to one survivor — which run crashed must not flip + accuracy); an *intentional* ``runs=1`` single verdict still scores accuracy (a trivial majority). + + ``unscoreable`` is true when **no** run produced a real verdict (every run was UNJUDGEABLE — a + crash, a malformed reply, or none wired): a harness/operational outcome, not a disagreement, so + the caller excludes it. UNJUDGEABLE runs are dropped **before** majority/consistency. ``accuracy`` + is fully guarded — below two survivors ``matched`` is ``None`` (a crash that drops survivors to one + must not turn an excluded tie into a hit or miss). ``consistency`` is a share **over the survivors**, + so when a crash coincides with a *dissenting* run it is an estimate: e.g. real ``[covered, covered, + wrong]`` is 0.667, but a crash on the ``wrong`` run reads 1.0 and a crash on a ``covered`` run reads + 0.5. It is honest for what it measures (agreement among the runs that returned) and ``runs_effective`` + records the survivor count; it does not pretend a crashed run agreed. Keying off the verdicts (not a + rationale substring) means a real verdict is never falsely excluded for mentioning an error. + """ + forced = Ranked(pairing, 0.0, True, "calibration") + verdicts = [_judge_one(forced, judge_fn).verdict for _ in range(runs)] + real = [verdict for verdict in verdicts if verdict != SEM_UNJUDGEABLE] + if not real: + return True, False, None, {"block_id": pairing.block_id, "expected": gold.verdict, + "majority": SEM_UNJUDGEABLE, "matched": False, + "runs_effective": 0, "consistency": None} + majority, count = _majority(real) + # Normalise the gold side the same way ``_reply_to_verdict`` normalises the judge side, so a + # directly-built ``SemanticGold("Covered")`` / ``" covered"`` (``load_gold`` enforces lowercase, + # direct construction does not) is a formatting difference, never a false accuracy=0 mismatch. + gold_verdict = gold.verdict.strip().lower() + matched: Optional[bool] + consistency: Optional[float] + if len(real) >= 2: + # A strict tie (more than one verdict shares the top count) has no majority: scoring it against + # gold would credit the canonical (alphabetically-first) pick, deflating accuracy whenever gold + # is a later label (e.g. the safety-critical ``wrong``). ``matched=None`` leaves accuracy alone. + tied = sum(1 for verdict in set(real) if real.count(verdict) == count) > 1 + matched = None if tied else majority == gold_verdict + # Consistency = the majority verdict's share of the surviving runs (``count / len(real)``): 1.0 + # when every real run agreed, 0.5 when a 3-run case split 2:1. ``runs_effective`` records how + # many runs produced a verdict, so a case decided on fewer than ``runs`` stays visible. + consistency = round(count / len(real), 4) + elif runs == 1: + # Intentional single run: a lone clean verdict has a trivial majority, so it *does* score + # accuracy (matching the doc); consistency needs ≥2 runs to mean anything, so it stays None. + matched = majority == gold_verdict + consistency = None + else: + # Crash-degraded: ``runs`` asked for repetition but crashes left one survivor. Scoring that + # lone verdict would let *which* run crashed flip a would-be tie into a hit or miss, so a + # degraded case feeds neither denominator — gold-independent. + matched = None + consistency = None + return False, matched, consistency, {"block_id": pairing.block_id, "expected": gold.verdict, + "majority": majority, "matched": matched, + "runs_effective": len(real), "consistency": consistency} + + +def calibrate(cases: Sequence[Tuple[Pairing, SemanticGold]], judge_fn: SemanticJudgeFn, + runs: int = 3) -> SemanticCalibration: + """Run the judge ``runs`` times over each gold-backed pairing; report accuracy + consistency. + + Both are ``None`` when there is nothing to measure — never a false 0. A pairing the pre-filter + would mark unjudgeable (no requirement / below the token floor), or one whose judged majority is + UNJUDGEABLE (a crash, a malformed reply, or no judge wired), is **excluded** — a harness/ + operational outcome, not a judge mismatch — so it never deflates accuracy. ``covered`` still + lists every gold-backed pairing. + + Calibrating against ``reference_stub_judge`` is warned at runtime: the stub re-uses the + pre-filter's own overlap, so its numbers measure the machinery, not judge quality. + """ + runs = max(1, runs) + if judge_fn is reference_stub_judge: + logger.warning("semantic: calibrating against reference_stub_judge — it re-uses the pre-filter's " + "own overlap score, so accuracy/consistency measure the machinery, not judge " + "quality; wire a real judge for meaningful calibration.") + scoreable = [(pairing, gold) for pairing, gold in cases if not _pairing_unscoreable(pairing)] + excluded = [pairing.block_id for pairing, _ in cases if _pairing_unscoreable(pairing)] + outcomes = [(pairing, _calibrate_case(pairing, gold, judge_fn, runs)) + for pairing, gold in scoreable] + excluded = excluded + [pairing.block_id for pairing, out in outcomes if out[0]] # out[0]: unscoreable? + scored = [out for _, out in outcomes if not out[0]] + # Accuracy is over cases with a real majority (matched is not None — a strict tie is unmeasurable, + # not a miss); consistency only over cases with a real run-to-run measurement (≥2 survivors), so a + # one-survivor case cannot contribute a spurious 1.0. Independent denominators, each honest. + matched = [m for _, m, _, _ in scored if m is not None] + measured = [c for _, _, c, _ in scored if c is not None] + judge = getattr(judge_fn, "__qualname__", "") or type(judge_fn).__name__ + return SemanticCalibration( + accuracy=round(sum(1 for m in matched if m) / len(matched), 4) if matched else None, + consistency=round(sum(measured) / len(measured), 4) if measured else None, + covered=[pairing.block_id for pairing, _ in cases], + runs_per_scenario=runs, per_case=[row for _, _, _, row in scored], excluded=excluded, + judge=judge) +# @cpt-end:cpt-studio-algo-eval-semantic:p1:inst-semantic-calibrate diff --git a/skills/studio/scripts/studio/utils/toc.py b/skills/studio/scripts/studio/utils/toc.py index 3662ac2c..03ecfdb9 100644 --- a/skills/studio/scripts/studio/utils/toc.py +++ b/skills/studio/scripts/studio/utils/toc.py @@ -22,6 +22,8 @@ import re import argparse +import math +import unicodedata from pathlib import Path from typing import Any, Dict, List, Optional, Tuple @@ -134,13 +136,54 @@ def parse_headings( max_level: Maximum heading level to include. skip_first: If True, skip the very first heading (document title). skip_toc_heading: If True, skip headings named "Table of Contents" or "TOC". + + Thin wrapper over :func:`parse_headings_with_lines` (stripping the line + number): one fence-tracking/heading-matching implementation instead of + two that could silently diverge. """ - headings: List[Tuple[int, str]] = [] + return [ + (level, text) + for level, text, _line in parse_headings_with_lines( + lines, + min_level=min_level, + max_level=max_level, + skip_first=skip_first, + skip_toc_heading=skip_toc_heading, + ) + ] +# @cpt-end:cpt-studio-algo-traceability-validation-toc-utils:p1:inst-toc-util-parse-headings + +# @cpt-begin:cpt-studio-algo-traceability-validation-toc-utils:p1:inst-toc-util-parse-headings-lines +def parse_headings_with_lines( + lines: List[str], + *, + min_level: int = 1, + max_level: int = 6, + skip_first: bool = False, + skip_toc_heading: bool = False, +) -> List[Tuple[int, str, int]]: + """Extract ``(level, text, line_number)`` triples from markdown lines. + + Fence-aware like :func:`parse_headings` (which delegates here), and + skips a leading YAML front-matter block (see + :func:`_find_frontmatter_end`) so a ``#``-prefixed line inside + front-matter data (a comment, a value) is never mistaken for a real + heading. ``line_number`` is 1-based. + + ``skip_first``/``skip_toc_heading`` mirror :func:`parse_headings`'s own + options: ``skip_first`` drops the very first heading matched + (regardless of level, checked before the level filter, same order as + the original standalone implementation), ``skip_toc_heading`` drops + headings named "Table of Contents"/"TOC" after the level filter. + """ + headings: List[Tuple[int, str, int]] = [] fence: Optional[Tuple[str, int]] = None + frontmatter_end = _find_frontmatter_end(lines) first_skipped = False - for line in lines: - # Track fenced code blocks (``` or ~~~ with 3+ chars) + for idx, line in enumerate(lines): + if idx < frontmatter_end: + continue new_fence = _fence_update(line, fence) if new_fence != fence: fence = new_fence @@ -165,10 +208,10 @@ def parse_headings( if skip_toc_heading and text.lower() in _TOC_HEADING_NAMES: continue - headings.append((level, text)) + headings.append((level, text, idx + 1)) return headings -# @cpt-end:cpt-studio-algo-traceability-validation-toc-utils:p1:inst-toc-util-parse-headings +# @cpt-end:cpt-studio-algo-traceability-validation-toc-utils:p1:inst-toc-util-parse-headings-lines # --------------------------------------------------------------------------- # TOC building @@ -687,6 +730,229 @@ def _record_missing_toc_error( )) +# @cpt-begin:cpt-studio-algo-traceability-validation-toc-utils:p1:inst-toc-jit-readiness +# --------------------------------------------------------------------------- +# JIT-retrieval readiness — structural signals beyond TOC correctness +# --------------------------------------------------------------------------- +# These four checks are additive warnings only (never errors): they flag +# structural properties that make a document harder to navigate via +# heading-based just-in-time retrieval, without invalidating documents that +# are otherwise fine. See constructorfabric/studio#104. + +DEFAULT_MAX_SECTION_LINES = 300 +# Below this size, a missing description is not worth flagging — the whole +# point of a description is to let a caller pick the right *file* before +# reading it, among many; a trivial file doesn't need that. +MIN_LINES_FOR_DESCRIPTION_CHECK = 100 + + +def _normalize_heading_key(text: str) -> str: + """Fold a heading's text to a comparison key for duplicate detection. + + Casefolds, collapses internal whitespace runs to a single space, and + NFC-normalizes so two headings that render identically -- differing + only in case, incidental whitespace, or Unicode composition -- are + still recognized as the same title. The original text is kept for + display; only the comparison key is normalized. + """ + return unicodedata.normalize("NFC", " ".join(text.split())).casefold() + + +def _check_duplicate_heading_titles( + headings_with_lines: List[Tuple[int, str, int]], + path: Path, +) -> List[Dict[str, Any]]: + """Warn when the same heading text appears more than once. + + Duplicate titles are tolerated by anchor-suffixing elsewhere (see + ``_unique_slug``) and are NOT errors, but they make it impossible to + unambiguously address a section by its heading text alone. + """ + from . import error_codes as EC + from .constraints import error + + seen: Dict[str, int] = {} + warnings: List[Dict[str, Any]] = [] + for _level, text, line in headings_with_lines: + key = _normalize_heading_key(text) + if key in seen: + warnings.append(error( + "toc", + f"Heading `{text}` duplicates an earlier heading (first seen at line {seen[key]})", + code=EC.TOC_HEADING_DUPLICATE, + path=path, + line=line, + heading_text=text, + first_seen_line=seen[key], + )) + else: + seen[key] = line + return warnings + + +def _check_heading_depth_jumps( + headings_with_lines: List[Tuple[int, str, int]], + path: Path, +) -> List[Dict[str, Any]]: + """Warn when heading depth increases by more than one level at once. + + E.g. an H2 followed directly by an H4 skips H3 — this breaks the + "read from this heading to the next heading at the same or higher + level" boundary computation JIT retrieval relies on. + """ + from . import error_codes as EC + from .constraints import error + + warnings: List[Dict[str, Any]] = [] + prev_level: Optional[int] = None + for level, text, line in headings_with_lines: + if prev_level is not None and level > prev_level + 1: + warnings.append(error( + "toc", + f"Heading `{text}` jumps from H{prev_level} to H{level}, skipping intermediate level(s)", + code=EC.TOC_HEADING_DEPTH_JUMP, + path=path, + line=line, + heading_text=text, + from_level=prev_level, + to_level=level, + )) + prev_level = level + return warnings + + +def _check_section_lengths( + headings_with_lines: List[Tuple[int, str, int]], + total_lines: int, + path: Path, + max_section_lines: int, +) -> List[Dict[str, Any]]: + """Warn when a section's body (up to the next heading, any level) is too long. + + An oversized section with no sub-headings defeats heading-based JIT + retrieval: reading "one section" still means reading the whole thing. + + ``max_section_lines`` is validated here, independent of any CLI + argparse guard: a non-finite value (``nan``/``inf``) or a non-positive + one falls back to :data:`DEFAULT_MAX_SECTION_LINES` rather than + silently disabling the check (``nan``) or flagging virtually every + section (a negative threshold) for a direct library caller. + """ + from . import error_codes as EC + from .constraints import error + + if not math.isfinite(max_section_lines) or max_section_lines <= 0: + max_section_lines = DEFAULT_MAX_SECTION_LINES + + warnings: List[Dict[str, Any]] = [] + for i, (_level, text, line) in enumerate(headings_with_lines): + next_line = ( + headings_with_lines[i + 1][2] + if i + 1 < len(headings_with_lines) + else total_lines + 1 + ) + section_length = next_line - line + if section_length > max_section_lines: + warnings.append(error( + "toc", + f"Section `{text}` is {section_length} lines long (max recommended: {max_section_lines})", + code=EC.TOC_SECTION_TOO_LONG, + path=path, + line=line, + heading_text=text, + section_length=section_length, + max_section_lines=max_section_lines, + )) + return warnings + + +_DESCRIPTION_FIELD_RE = re.compile(r"^description\s*:\s*(.*)$") +_BLOCK_SCALAR_RE = re.compile(r"^[|>][+\-]?\d*$") + + +def _quoted_value_is_empty(value: str) -> bool: + """``value`` starts with a quote char -- True if the quoted text is empty.""" + quote = value[0] + closing = value.find(quote, 1) + inner = value[1:closing] if closing != -1 else value[1:] + return not inner.strip() + + +def _block_scalar_is_empty(body: List[str], start_index: int) -> bool: + """``value`` was a YAML block scalar marker (``|``, ``>``, ``|-``, ...) -- + its real content, if any, is on indented lines below it, not on the + marker's own line. True if the first non-blank following line isn't + indented under it (i.e. the block scalar has no content at all).""" + for line in body[start_index:]: + if not line.strip(): + continue + return not line[0].isspace() + return True + + +def _frontmatter_has_description(lines: List[str], frontmatter_end: int) -> bool: + """Check whether a YAML frontmatter block declares a non-empty ``description``. + + ``frontmatter_end`` is the index returned by :func:`_find_frontmatter_end` + (one past the closing ``---``); the body being scanned is + ``lines[1:frontmatter_end - 1]``, excluding both delimiter lines. + + A field that's present but carries no real value doesn't satisfy this: + a YAML comment (``description: # TODO``), an empty quoted string + (``description: ""``), or a block-scalar marker + (``description: |``) with nothing indented beneath it all parse as "no + description" just as much as the field being absent entirely would -- + the point of this check is to guarantee a caller gets something to + actually read, not just a matching key. + """ + body = lines[1:frontmatter_end - 1] + for i, line in enumerate(body): + match = _DESCRIPTION_FIELD_RE.match(line.strip()) + if not match: + continue + value = match.group(1).strip() + if not value or value.startswith("#"): + continue + if _BLOCK_SCALAR_RE.match(value): + if _block_scalar_is_empty(body, i + 1): + continue + elif value[0] in "\"'" and _quoted_value_is_empty(value): + continue + return True + return False + + +def _check_missing_description( + lines: List[str], + path: Path, +) -> List[Dict[str, Any]]: + """Warn when a document has no frontmatter block with a real description. + + A short description lets a caller pick the right *document* before + reading any of its headings — the same principle as heading + descriptiveness, one level up. Frontmatter that exists but carries no + ``description`` field (e.g. only a ``title``) does not satisfy this — + an empty promise is the same as no promise. Only checked above + ``MIN_LINES_FOR_DESCRIPTION_CHECK`` lines — a trivial file doesn't need + a description, and flagging every small file drowns the signal. + """ + from . import error_codes as EC + from .constraints import error + + if len(lines) < MIN_LINES_FOR_DESCRIPTION_CHECK: + return [] + frontmatter_end = _find_frontmatter_end(lines) + if frontmatter_end > 0 and _frontmatter_has_description(lines, frontmatter_end): + return [] + return [error( + "toc", + "Document has no frontmatter/description block at the top", + code=EC.TOC_MISSING_DESCRIPTION, + path=path, + line=1, + )] +# @cpt-end:cpt-studio-algo-traceability-validation-toc-utils:p1:inst-toc-jit-readiness + # @cpt-begin:cpt-studio-algo-traceability-validation-validate-toc:p1:inst-toc-compare def _validate_toc_entries( toc_entries: List[Tuple[str, str, int]], @@ -764,12 +1030,38 @@ def _append_stale_toc_warning( )) # @cpt-end:cpt-studio-algo-traceability-validation-toc-utils:p1:inst-toc-util-helpers +# @cpt-begin:cpt-studio-algo-traceability-validation-toc-utils:p1:inst-toc-jit-readiness-collect +def _collect_jit_readiness_warnings( + lines: List[str], + path: Path, + max_section_lines: int, +) -> List[Dict[str, Any]]: + """Gather all four JIT-retrieval readiness warnings for a document. + + Always parses *every* heading level, independent of whatever + ``max_heading_level`` the caller configured for TOC-completeness + checking above — these signals are about the document's real structure + (would a duplicate/depth-jump/oversized-section problem trip up + heading-based retrieval), not about which levels belong in a + human-authored TOC. Filtering by the TOC's level cap would hide real H4-H6 + issues under a shallow default (e.g. the CLI's own ``--max-level 3``). + """ + warnings: List[Dict[str, Any]] = [] + warnings.extend(_check_missing_description(lines, path)) + all_headings = parse_headings_with_lines(lines) + warnings.extend(_check_duplicate_heading_titles(all_headings, path)) + warnings.extend(_check_heading_depth_jumps(all_headings, path)) + warnings.extend(_check_section_lengths(all_headings, len(lines), path, max_section_lines)) + return warnings +# @cpt-end:cpt-studio-algo-traceability-validation-toc-utils:p1:inst-toc-jit-readiness-collect + # @cpt-begin:cpt-studio-algo-traceability-validation-toc-utils:p1:inst-toc-util-validate def validate_toc( content: str, *, artifact_path: Optional[Path] = None, max_heading_level: int = 6, + max_section_lines: int = DEFAULT_MAX_SECTION_LINES, ) -> Dict[str, List[Dict[str, Any]]]: """Validate the Table of Contents in a markdown document. @@ -784,6 +1076,11 @@ def validate_toc( 4. **Freshness** — if the TOC were regenerated, it would match the current content (catches reordering / renamed headings). + Plus four additive, warning-only JIT-retrieval readiness signals that + run regardless of TOC presence/errors above (see constructorfabric/studio#104): + duplicate heading titles, heading depth jumps, oversized sections, and a + missing top-of-file description/frontmatter block. + Returns ``{"errors": [...], "warnings": [...]}`` in the same format as ``validate_artifact_file``. """ @@ -796,8 +1093,12 @@ def validate_toc( max_heading_level, ) + # JIT-retrieval readiness signals (warning-only, run regardless of + # TOC presence below — independent of the TOC-filtered `headings`). + warnings.extend(_collect_jit_readiness_warnings(lines, path, max_section_lines)) + if not headings: - # No headings → nothing to validate + # No headings → nothing further to validate return {"errors": errors, "warnings": warnings} # @cpt-end:cpt-studio-algo-traceability-validation-toc-utils:p1:inst-toc-util-validate-init diff --git a/tests/test_doc_index.py b/tests/test_doc_index.py new file mode 100644 index 00000000..5e9d7943 --- /dev/null +++ b/tests/test_doc_index.py @@ -0,0 +1,791 @@ +"""Tests for the cached, read-once-per-file document index (doc_index.py). + +See constructorfabric/studio#104. +""" + +from __future__ import annotations + +import json +import logging +import os +from pathlib import Path + +import pytest + +from studio.commands.doc_index import cmd_doc_index +from studio.utils.doc_index import ( + annotate_section_summary, + build_doc_index, + diff_stale_sections, + get_or_build_doc_index, + infer_section_level, + load_doc_index, + save_doc_index, +) +from studio.utils.toc import parse_headings_with_lines + +_SAMPLE = ( + "# Title\n\n" + "## Section A\n\n" + "Body of A.\n\n" + "### A.1\n\n" + "Body of A.1.\n\n" + "## Section B\n\n" + "Body of B.\n" +) + + +def _write(tmp_path: Path, content: str = _SAMPLE, name: str = "doc.md") -> Path: + f = tmp_path / name + f.write_text(content, encoding="utf-8") + return f + + +@pytest.fixture +def studio_logger_propagates(): + """Force the "studio" logger to propagate for the duration of a test. + + Whichever CLI test runs first in the full suite triggers + cli.py's own _configure_studio_logging(), which sets + logging.getLogger("studio").propagate = False for the rest of the + process -- a real, ambient global-state mutation, not something this + test file controls. That silently blocks pytest's caplog (which + listens on the root logger) from ever seeing a child logger's records + for any test that runs after it. Tests asserting on log output for + "studio.utils.doc_index" opt into this fixture to stay correct + regardless of suite ordering, restoring the original value afterward. + """ + studio_logger = logging.getLogger("studio") + original = studio_logger.propagate + studio_logger.propagate = True + try: + yield + finally: + studio_logger.propagate = original + + +class TestBuildDocIndex: + def test_extracts_sections_with_line_ranges(self, tmp_path: Path): + f = _write(tmp_path) + index = build_doc_index(f) + headings = [(s["level"], s["heading"], s["line_start"], s["line_end"]) for s in index["sections"]] + assert headings == [ + (1, "Title", 1, 2), + (2, "Section A", 3, 6), + (3, "A.1", 7, 10), + (2, "Section B", 11, 14), + ] + + def test_sections_start_with_no_summary(self, tmp_path: Path): + f = _write(tmp_path) + index = build_doc_index(f) + assert all(s["summary"] is None for s in index["sections"]) + + def test_etag_present_and_stable_for_same_content(self, tmp_path: Path): + f = _write(tmp_path) + idx1 = build_doc_index(f) + idx2 = build_doc_index(f) + assert idx1["etag"] == idx2["etag"] + + def test_etag_changes_when_content_changes(self, tmp_path: Path): + f = _write(tmp_path) + idx1 = build_doc_index(f) + f.write_text(_SAMPLE + "\n## Section C\n") + idx2 = build_doc_index(f) + assert idx1["etag"] != idx2["etag"] + + def test_skips_headings_in_fenced_code(self, tmp_path: Path): + content = "# Title\n\n## Real\n\n```bash\n# not a heading\n```\n\n## Also Real\n" + f = _write(tmp_path, content) + index = build_doc_index(f) + assert [s["heading"] for s in index["sections"]] == ["Title", "Real", "Also Real"] + + def test_retrieval_sections_grouped_at_inferred_level(self, tmp_path: Path): + f = _write(tmp_path) + index = build_doc_index(f) + assert index["section_level"] == 2 + # "### A.1" (H3, off-level) stays inside "## Section A", not its own section. + assert [s["heading"] for s in index["retrieval_sections"]] == ["Section A", "Section B"] + + def test_headingless_document_has_no_retrieval_sections(self, tmp_path: Path): + f = _write(tmp_path, "Just a paragraph, no headings at all.\n") + index = build_doc_index(f) + assert index["section_level"] is None + assert index["retrieval_sections"] == [] + + def test_retrieval_section_hash_changes_only_for_the_edited_section(self, tmp_path: Path): + f = _write(tmp_path) + before = build_doc_index(f) + f.write_text(_SAMPLE.replace("Body of A.", "Body of A, edited."), encoding="utf-8") + after = build_doc_index(f) + by_heading_before = {s["heading"]: s["hash"] for s in before["retrieval_sections"]} + by_heading_after = {s["heading"]: s["hash"] for s in after["retrieval_sections"]} + assert by_heading_before["Section A"] != by_heading_after["Section A"] + assert by_heading_before["Section B"] == by_heading_after["Section B"] + + def test_trailing_whitespace_only_edit_does_not_change_the_hash(self, tmp_path: Path): + """CodeRabbit PR #109: a "trim trailing whitespace on save" editor + default changes no meaningful content and must not look like a + real edit to diff_stale_sections -- the whole point of hashing at + section granularity.""" + f = _write(tmp_path) + before = build_doc_index(f) + f.write_text(_SAMPLE.replace("Body of A.\n", "Body of A. \n"), encoding="utf-8") + after = build_doc_index(f) + by_heading_before = {s["heading"]: s["hash"] for s in before["retrieval_sections"]} + by_heading_after = {s["heading"]: s["hash"] for s in after["retrieval_sections"]} + assert by_heading_before["Section A"] == by_heading_after["Section A"] + + +class TestInferSectionLevel: + def test_uniform_level_is_chosen(self): + headings = [(2, "A", 1), (2, "B", 5), (2, "C", 9)] + assert infer_section_level(headings) == 2 + + def test_real_bug_regression_dominant_level_wins_over_a_stray_shallower_one(self): + """Reproduces the actual failure found developing this feature: a + PDF-converted document put its 8 real chapters on H5 and a single + subsection heading on H3. Picking the shallowest level present + (H3) -- or any fixed level -- turned the rest of the document into + one fake mega-section. The dominant (most-recurring) level must + win over a level that appears only once, however shallow.""" + headings = ( + [(5, f"Chapter {i}", i * 100) for i in range(1, 9)] + + [(3, "Stray Subsection", 250)] + ) + assert infer_section_level(headings) == 5 + + def test_no_headings_returns_none(self): + assert infer_section_level([]) is None + + def test_all_singleton_levels_falls_back_to_shallowest(self): + headings = [(4, "A", 1), (2, "B", 5), (6, "C", 9)] + assert infer_section_level(headings) == 2 + + def test_tie_between_recurring_levels_prefers_shallower(self): + headings = [(3, "A", 1), (3, "B", 5), (5, "C", 9), (5, "D", 13)] + assert infer_section_level(headings) == 3 + + def test_matches_real_parser_output(self, tmp_path: Path): + content = "##### Ch1\n\nbody\n\n##### Ch2\n\nbody\n\n### Odd\n\nbody\n\n##### Ch3\n\nbody\n" + f = _write(tmp_path, content) + lines = f.read_text(encoding="utf-8").split("\n") + headings = parse_headings_with_lines(lines) + assert infer_section_level(headings) == 5 + + +class TestDiffStaleSections: + def test_returns_none_when_no_cache_exists(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + assert diff_stale_sections(f) is None + + def test_returns_none_for_pre_retrieval_sections_cache_format(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + old_format = build_doc_index(f) + del old_format["retrieval_sections"] # simulate an index built before this field existed + save_doc_index(f, old_format) + assert diff_stale_sections(f) is None + + def test_no_edit_reports_everything_unchanged(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + save_doc_index(f, build_doc_index(f)) + diff = diff_stale_sections(f) + assert diff["structural_change"] is False + assert diff["changed"] == [] + assert {(e["heading"], e["line_start"]) for e in diff["unchanged"]} == { + ("Section A", 3), + ("Section B", 11), + } + + def test_editing_one_section_reports_only_that_one_changed(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + save_doc_index(f, build_doc_index(f)) + f.write_text(_SAMPLE.replace("Body of B.", "Body of B, edited."), encoding="utf-8") + diff = diff_stale_sections(f) + assert diff["structural_change"] is False + assert diff["changed"] == [{"heading": "Section B", "line_start": 11}] + assert diff["unchanged"] == [{"heading": "Section A", "line_start": 3}] + + def test_duplicate_headings_are_disambiguated_by_line_start(self, tmp_path: Path, monkeypatch): + """CodeRabbit PR #109: heading text alone can't tell two identically + named sections apart -- line_start must be returned so a caller + knows exactly which one changed.""" + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + content = "## Details\n\nFirst.\n\n## Details\n\nSecond.\n" + f = _write(tmp_path, content) + save_doc_index(f, build_doc_index(f)) + f.write_text(content.replace("Second.", "Second, edited."), encoding="utf-8") + diff = diff_stale_sections(f) + assert diff["structural_change"] is False + assert diff["unchanged"] == [{"heading": "Details", "line_start": 1}] + assert diff["changed"] == [{"heading": "Details", "line_start": 5}] + + def test_returns_none_when_file_deleted_after_caching(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + save_doc_index(f, build_doc_index(f)) + f.unlink() + assert diff_stale_sections(f) is None + + def test_adding_a_retrieval_level_heading_is_a_structural_change(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + save_doc_index(f, build_doc_index(f)) + f.write_text(_SAMPLE + "\n## Section C\n\nBody of C.\n", encoding="utf-8") + diff = diff_stale_sections(f) + assert diff["structural_change"] is True + assert diff["unchanged"] == [] + assert {e["heading"] for e in diff["changed"]} == {"Section A", "Section B", "Section C"} + + +class TestCachePersistence: + def test_load_returns_none_when_no_cache_exists(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + assert load_doc_index(f) is None + + def test_legacy_cache_with_matching_etag_but_old_schema_is_rebuilt_not_returned( + self, tmp_path: Path, monkeypatch + ): + """CodeRabbit PR #109 (second round): a cache written before + section_level/retrieval_sections existed can have a matching etag + if the file hasn't changed since -- load_doc_index() must not + return it as-is, or a caller reading those fields hits a + KeyError instead of a clean rebuild.""" + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + legacy = build_doc_index(f) + del legacy["section_level"] + del legacy["retrieval_sections"] + save_doc_index(f, legacy) + + assert load_doc_index(f) is None # not the legacy dict, and not a crash + + # The real caller path rebuilds cleanly rather than raising. + index = get_or_build_doc_index(f) + assert index["cache_hit"] is False + assert "section_level" in index + assert "retrieval_sections" in index + + def test_cmd_doc_index_does_not_crash_on_a_legacy_cache(self, tmp_path: Path, capsys, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + legacy = build_doc_index(f) + del legacy["section_level"] + del legacy["retrieval_sections"] + save_doc_index(f, legacy) + + rc = cmd_doc_index([str(f)]) + assert rc == 0 + out = json.loads(capsys.readouterr().out) + assert out["cache_hit"] is False + assert "retrieval_sections" in out + + def test_save_then_load_round_trips(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + built = build_doc_index(f) + save_doc_index(f, built) + loaded = load_doc_index(f) + assert loaded is not None + assert loaded["etag"] == built["etag"] + assert loaded["sections"] == built["sections"] + + def test_load_returns_none_when_cache_is_stale(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + save_doc_index(f, build_doc_index(f)) + f.write_text(_SAMPLE + "\n## Section C\n") # content changed after caching + assert load_doc_index(f) is None + + def test_same_size_same_line_count_edit_is_still_detected_as_stale( + self, tmp_path: Path, monkeypatch + ): + """Regression test (see PR #108 review): a same-size, same-line-count + content swap must still invalidate the cache. A byte-size + + line-count fingerprint alone cannot distinguish this from an + unchanged file -- mtime can, since a real write always advances it. + """ + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + save_doc_index(f, build_doc_index(f)) + + edited = _SAMPLE.replace("Section A", "Section Z") + assert len(edited) == len(_SAMPLE) + assert edited.count("\n") == _SAMPLE.count("\n") + f.write_text(edited, encoding="utf-8") + # Force a distinct mtime regardless of filesystem clock resolution -- + # the mechanism under test is "mtime changed", not "enough wall-clock + # time elapsed during the test run". + st = f.stat() + os.utime(f, ns=(st.st_atime_ns, st.st_mtime_ns + 1)) + + assert load_doc_index(f) is None + fresh = get_or_build_doc_index(f) + assert any(s["heading"] == "Section Z" for s in fresh["sections"]) + + def test_studio_directory_resolved_from_file_path_not_cwd( + self, tmp_path: Path, monkeypatch + ): + """CodeRabbit PR #108: the Studio directory must be resolved from the + indexed file's own location, not the process's cwd -- otherwise + indexing a file outside the caller's cwd can miss or mis-target the + cache.""" + seen_paths = [] + + def _spy(start_path): + seen_paths.append(start_path) + return tmp_path + + monkeypatch.setattr("studio.utils.files.find_studio_directory", _spy) + f = _write(tmp_path) + save_doc_index(f, build_doc_index(f)) + + assert seen_paths, "find_studio_directory was never called" + assert seen_paths[0] == f.resolve().parent + + def test_load_returns_none_on_corrupt_cache_file( + self, tmp_path: Path, monkeypatch, caplog, studio_logger_propagates + ): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + save_doc_index(f, build_doc_index(f)) + # Corrupt the cache file directly + cache_dir = tmp_path / ".cache" / "doc-index" + for cache_file in cache_dir.glob("*.json"): + cache_file.write_text("{not valid json", encoding="utf-8") + with caplog.at_level("WARNING"): + assert load_doc_index(f) is None + # CodeRabbit PR #109: real corruption is a genuine anomaly, not a + # routine cache miss -- must be visible at the CLI's default log + # level (WARNING), not buried at debug. + assert any(r.levelname == "WARNING" for r in caplog.records) + + def test_cmd_doc_index_rebuilds_cleanly_after_corrupt_cache(self, tmp_path: Path, capsys, monkeypatch): + """CodeRabbit PR #108: prove the corrupt-cache fallback at the + CLI/exit-code level, not just load_doc_index() in isolation.""" + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + save_doc_index(f, build_doc_index(f)) + cache_dir = tmp_path / ".cache" / "doc-index" + for cache_file in cache_dir.glob("*.json"): + cache_file.write_text("{not valid json", encoding="utf-8") + + rc = cmd_doc_index([str(f)]) + assert rc == 0 + out = json.loads(capsys.readouterr().out) + assert out["cache_hit"] is False + + def test_cache_missing_a_required_field_is_rebuilt_not_returned(self, tmp_path: Path, monkeypatch): + """CodeRabbit PR #108: a matching-etag cache missing "sections" + (hand-edited, or truncated mid-write) used to pass load_doc_index's + etag-only check and reach cmd_doc_index()'s len(index["sections"]) + as an unhandled KeyError.""" + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + incomplete = build_doc_index(f) + del incomplete["sections"] + save_doc_index(f, incomplete) + + assert load_doc_index(f) is None + index = get_or_build_doc_index(f) + assert index["cache_hit"] is False + assert "sections" in index + + def test_cache_from_an_older_schema_version_is_rebuilt_not_returned(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + old_schema = build_doc_index(f) + old_schema["schema_version"] = 0 + save_doc_index(f, old_schema) + + assert load_doc_index(f) is None + index = get_or_build_doc_index(f) + assert index["cache_hit"] is False + + def test_save_does_not_leave_a_temp_file_behind(self, tmp_path: Path, monkeypatch): + """CodeRabbit PR #108: save_doc_index() writes atomically (temp + file + os.replace) -- the temp file must not survive a successful + write.""" + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + save_doc_index(f, build_doc_index(f)) + cache_dir = tmp_path / ".cache" / "doc-index" + names = [p.name for p in cache_dir.iterdir()] + assert all(name.endswith(".json") for name in names) + assert load_doc_index(f) is not None + + def test_no_studio_directory_means_no_crash_and_always_none(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: None) + f = _write(tmp_path) + save_doc_index(f, build_doc_index(f)) # must no-op silently, not raise + assert load_doc_index(f) is None + + def test_studio_directory_lookup_error_means_no_crash_and_no_cache( + self, tmp_path: Path, monkeypatch, caplog, studio_logger_propagates + ): + """An OSError from find_studio_directory (e.g. an unreadable parent + directory) must degrade to 'no cache', not raise -- and it must be + logged, not silently swallowed (see PR #108 review / pylint W9001). + CodeRabbit PR #109: this is a genuine anomaly, distinct from the + ordinary (unlogged) "no Studio directory found" case, so it must + log at WARNING, visible at the CLI's default level, not DEBUG.""" + def _raise(_start_path): + raise OSError("permission denied") + + monkeypatch.setattr("studio.utils.files.find_studio_directory", _raise) + f = _write(tmp_path) + with caplog.at_level("WARNING"): + save_doc_index(f, build_doc_index(f)) # must no-op, not raise + assert load_doc_index(f) is None + assert any(r.levelname == "WARNING" for r in caplog.records) + + def test_load_returns_none_when_file_deleted_after_caching(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + save_doc_index(f, build_doc_index(f)) + f.unlink() + assert load_doc_index(f) is None + + +class TestGetOrBuildDocIndex: + def test_first_call_is_cache_miss(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + index = get_or_build_doc_index(f) + assert index["cache_hit"] is False + + def test_second_call_is_cache_hit(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + get_or_build_doc_index(f) + index = get_or_build_doc_index(f) + assert index["cache_hit"] is True + + def test_cache_hit_preserves_previously_annotated_summary(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + get_or_build_doc_index(f) + assert annotate_section_summary(f, line_start=3, summary="Covers A.") is True + index = get_or_build_doc_index(f) + assert index["cache_hit"] is True + section_a = next(s for s in index["sections"] if s["heading"] == "Section A") + assert section_a["summary"] == "Covers A." + + def test_content_change_invalidates_and_drops_stale_summaries(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + get_or_build_doc_index(f) + annotate_section_summary(f, line_start=3, summary="Covers A.") + f.write_text(_SAMPLE + "\n## Section C\n") + index = get_or_build_doc_index(f) + assert index["cache_hit"] is False + assert all(s["summary"] is None for s in index["sections"]) + + def test_force_rebuild_bypasses_valid_cache(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + get_or_build_doc_index(f) + index = get_or_build_doc_index(f, force_rebuild=True) + assert index["cache_hit"] is False + + +class TestAnnotateSectionSummary: + def test_returns_false_when_no_cache_exists_yet(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + assert annotate_section_summary(f, line_start=3, summary="x") is False + + def test_returns_false_for_unmatched_line_start(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + get_or_build_doc_index(f) + assert annotate_section_summary(f, line_start=999, summary="x") is False + + def test_returns_true_and_persists_on_match(self, tmp_path: Path, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + get_or_build_doc_index(f) + assert annotate_section_summary(f, line_start=1, summary="The title.") is True + cached = load_doc_index(f) + assert cached["sections"][0]["summary"] == "The title." + + def test_updates_matching_retrieval_section_too(self, tmp_path: Path, monkeypatch): + """CodeRabbit PR #109: annotate_section_summary() updated only + `sections`, leaving the matching `retrieval_sections` entry at + summary=None -- a caller reading retrieval_sections (the more + relevant list for a future OKF-style summarizer) couldn't see it.""" + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + get_or_build_doc_index(f) + assert annotate_section_summary(f, line_start=3, summary="Covers A.") is True + index = load_doc_index(f) + retrieval_a = next(s for s in index["retrieval_sections"] if s["heading"] == "Section A") + assert retrieval_a["summary"] == "Covers A." + + def test_off_level_heading_leaves_retrieval_sections_untouched(self, tmp_path: Path, monkeypatch): + """line_start=7 is "### A.1" -- present in `sections` but not itself + a retrieval section's start (retrieval sections are at H2 here). + Only `sections` should be updated; there's no corresponding + retrieval section to touch.""" + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + get_or_build_doc_index(f) + assert annotate_section_summary(f, line_start=7, summary="About A.1.") is True + index = load_doc_index(f) + a1 = next(s for s in index["sections"] if s["heading"] == "A.1") + assert a1["summary"] == "About A.1." + assert all(s["summary"] is None for s in index["retrieval_sections"]) + + def test_concurrent_annotations_of_different_sections_do_not_lose_either_update( + self, tmp_path: Path, monkeypatch + ): + """CodeRabbit PR #109: two concurrent read-modify-write cycles + annotating different sections of the same document must not race -- + without the lock, thread B's load could happen before thread A's + save, so thread B's own save would overwrite thread A's summary + with a stale base index. Injects a delay inside the locked section + (between load and save) to force a real overlap window if the lock + weren't actually serializing the two calls.""" + import threading + import time as time_module + + from studio.utils import doc_index as di + + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + index = get_or_build_doc_index(f) + line_a = index["sections"][1]["line_start"] # "Section A" + line_b = index["sections"][3]["line_start"] # "Section B" + + original_save = di.save_doc_index + + def slow_save(path, saved_index): + time_module.sleep(0.1) + original_save(path, saved_index) + + monkeypatch.setattr(di, "save_doc_index", slow_save) + + results: dict = {} + + def run(line_start: int, summary: str) -> None: + results[line_start] = di.annotate_section_summary(f, line_start, summary) + + t1 = threading.Thread(target=run, args=(line_a, "Summary A")) + t2 = threading.Thread(target=run, args=(line_b, "Summary B")) + t1.start() + t2.start() + t1.join(timeout=5) + t2.join(timeout=5) + + assert results == {line_a: True, line_b: True} + final = di.load_doc_index(f) + by_line = {s["line_start"]: s["summary"] for s in final["sections"]} + assert by_line[line_a] == "Summary A" + assert by_line[line_b] == "Summary B" + + +class TestReadWithStableEtag: + def test_retries_when_the_file_changes_mid_read(self, tmp_path: Path, monkeypatch): + """CodeRabbit PR #109: a write landing between reading content and + computing the etag could save headings from the *old* content + stamped with the *new* etag. Snapshotting before and after the + read, and retrying on mismatch, closes that window.""" + import studio.utils.doc_index as di + + f = _write(tmp_path) + etag_sequence = ["a", "b", "b"] # initial snapshot, then a mismatch, then a stable match + calls = {"n": 0} + + def fake_compute_etag(_path): + value = etag_sequence[calls["n"]] + calls["n"] += 1 + return value + + monkeypatch.setattr(di, "_compute_etag", fake_compute_etag) + content, etag = di._read_with_stable_etag(f) + assert content == _SAMPLE + assert etag == "b" + assert calls["n"] == 3 # one retry: initial snapshot + two read-and-check cycles + + def test_gives_up_after_max_attempts_under_sustained_contention(self, tmp_path: Path, monkeypatch): + import studio.utils.doc_index as di + + f = _write(tmp_path) + calls = {"n": 0} + + def always_different(_path): + calls["n"] += 1 + return f"etag-{calls['n']}" + + monkeypatch.setattr(di, "_compute_etag", always_different) + content, etag = di._read_with_stable_etag(f) + assert content == _SAMPLE # still returns a real read, not an error + assert calls["n"] == di._MAX_READ_ATTEMPTS + 1 + assert etag == f"etag-{calls['n']}" + + +class TestCmdDocIndex: + def test_missing_file(self, tmp_path: Path, capsys): + rc = cmd_doc_index([str(tmp_path / "nope.md")]) + assert rc == 2 + out = json.loads(capsys.readouterr().out) + assert out["status"] == "ERROR" + + def test_unreadable_file_reports_a_clean_error_not_a_raw_traceback( + self, tmp_path: Path, capsys, monkeypatch + ): + """CodeRabbit PR #108 (round 2): an OSError from get_or_build_doc_index + (e.g. a permissions failure or a race where the file vanishes after + the is_file() check) must use the same clean error contract as a + UnicodeDecodeError, not escape as a raw traceback.""" + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + + def _raise_os_error(*_a, **_k): + raise OSError("permission denied") + + monkeypatch.setattr("studio.commands.doc_index.get_or_build_doc_index", _raise_os_error) + rc = cmd_doc_index([str(f)]) + assert rc == 2 + out = json.loads(capsys.readouterr().out) + assert out["status"] == "ERROR" + + def test_basic(self, tmp_path: Path, capsys, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + rc = cmd_doc_index([str(f)]) + assert rc == 0 + out = json.loads(capsys.readouterr().out) + assert out["cache_hit"] is False + assert out["section_count"] == 4 + + def test_json_output_exposes_retrieval_sections(self, tmp_path: Path, capsys, monkeypatch): + """CodeRabbit PR #109: cmd_doc_index() built its output from `index` + but omitted retrieval_sections/section_level -- the new data this + PR adds was invisible through the CLI.""" + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + rc = cmd_doc_index([str(f)]) + assert rc == 0 + out = json.loads(capsys.readouterr().out) + assert out["section_level"] == 2 + assert out["retrieval_section_count"] == 2 + assert [s["heading"] for s in out["retrieval_sections"]] == ["Section A", "Section B"] + assert "hash" in out["retrieval_sections"][0] + + def test_non_utf8_file_reports_a_clean_error_not_a_raw_traceback(self, tmp_path: Path, capsys, monkeypatch): + """CodeRabbit PR #109: a binary/non-UTF-8 file used to crash with an + unhandled UnicodeDecodeError; must now report a clean ERROR result, + consistent with the existing "File not found" pattern.""" + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = tmp_path / "binary.md" + f.write_bytes(b"\xff\xfe\x00\x01garbage") + rc = cmd_doc_index([str(f)]) + assert rc == 2 + out = json.loads(capsys.readouterr().out) + assert out["status"] == "ERROR" + assert "utf-8" in out["message"].lower() or "UTF-8" in out["message"] + + def test_help_explains_how_section_level_is_inferred(self): + """CodeRabbit PR #109: --help gave no indication that section_level + is a heuristic, not simply "H1/H2".""" + import io + from contextlib import redirect_stdout + + buf = io.StringIO() + with redirect_stdout(buf): + with pytest.raises(SystemExit): + cmd_doc_index(["--help"]) + assert "most frequently" in buf.getvalue() + + def test_human_output_lists_retrieval_sections(self, tmp_path: Path, capsys, monkeypatch): + from studio.utils.ui import is_json_mode, set_json_mode + + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + orig = is_json_mode() + set_json_mode(False) + try: + rc = cmd_doc_index([str(f)]) + finally: + set_json_mode(orig) + assert rc == 0 + out = capsys.readouterr().out + assert "Retrieval sections (level 2, 2 section(s))" in out + assert "Section A" in out + assert "Section B" in out + + def test_second_invocation_is_cache_hit(self, tmp_path: Path, capsys, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + cmd_doc_index([str(f)]) + capsys.readouterr() + rc = cmd_doc_index([str(f)]) + assert rc == 0 + out = json.loads(capsys.readouterr().out) + assert out["cache_hit"] is True + + def test_rebuild_flag_forces_cache_miss(self, tmp_path: Path, capsys, monkeypatch): + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + cmd_doc_index([str(f)]) + capsys.readouterr() + rc = cmd_doc_index([str(f), "--rebuild"]) + assert rc == 0 + out = json.loads(capsys.readouterr().out) + assert out["cache_hit"] is False + + def test_human_output_mode(self, tmp_path: Path, capsys, monkeypatch): + from studio.utils.ui import is_json_mode, set_json_mode + + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + orig = is_json_mode() + set_json_mode(False) + try: + rc = cmd_doc_index([str(f)]) + finally: + set_json_mode(orig) + assert rc == 0 + out = capsys.readouterr().out + assert "Doc Index" in out + assert "cache miss" in out + assert "Section A" in out + + def test_human_output_mode_cache_hit(self, tmp_path: Path, capsys, monkeypatch): + from studio.utils.ui import is_json_mode, set_json_mode + + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + cmd_doc_index([str(f)]) + capsys.readouterr() + orig = is_json_mode() + set_json_mode(False) + try: + rc = cmd_doc_index([str(f)]) + finally: + set_json_mode(orig) + assert rc == 0 + assert "cache hit" in capsys.readouterr().out + + def test_human_output_mode_with_section_summary(self, tmp_path: Path, capsys, monkeypatch): + from studio.utils.ui import is_json_mode, set_json_mode + + monkeypatch.setattr("studio.utils.files.find_studio_directory", lambda *_a, **_k: tmp_path) + f = _write(tmp_path) + cmd_doc_index([str(f)]) + capsys.readouterr() + assert annotate_section_summary(f, line_start=3, summary="Covers A.") is True + orig = is_json_mode() + set_json_mode(False) + try: + rc = cmd_doc_index([str(f)]) + finally: + set_json_mode(orig) + assert rc == 0 + assert "Covers A." in capsys.readouterr().out diff --git a/tests/test_eval_semantic.py b/tests/test_eval_semantic.py new file mode 100644 index 00000000..3bb47369 --- /dev/null +++ b/tests/test_eval_semantic.py @@ -0,0 +1,990 @@ +"""Tests for the semantic-coverage engine (utils.eval_semantic). + +The headline is the adversarial pair: a marked block implementing the *wrong* behaviour must be +surfaced as a weak link and judged non-covered, while a faithful implementation must not be +touched by the judge at all. The rest pin the honesty guards: unjudgeable-not-zero, the +evidence (hallucination) guard, defensive degradation, and the advisory-never-gates discipline. +""" +from __future__ import annotations + +import random +from pathlib import Path +from typing import Callable, List, Optional + +import pytest + +from studio.utils import eval_semantic as sem +from studio.utils.eval_semantic import (Pairing, SemanticReply, SemanticReport, assess, + calibrate, coverage_scope, evidence_present, load_gold, + overlap_score, rank_pairings, reference_stub_judge, + resolve_requirement, tokenize) + +# A requirement with a clear domain vocabulary, and two code blocks: one faithful, one wrong. +_REQUIREMENT = ("Validate the user email address format and reject a malformed address before " + "saving the record.") +_CORRECT_CODE = ("def validate_email(address):\n" + " if not email_format(address):\n" + " reject_malformed(address)\n" + " return save_record(address)") +_WRONG_CODE = ("def compute_tax(amount):\n" + " rate = lookup_rate(amount)\n" + " return amount * rate") + + +def _pairing(block_id: str, code: str, requirement: Optional[str], + path: str = "mod.py", start_line: int = 10) -> Pairing: + return Pairing(block_id=block_id, inst=f"inst-{block_id}", path=path, + start_line=start_line, code=code, requirement=requirement) + + +class _SpyJudge: + """A judge_fn that records the ids it was asked to judge and returns a fixed verdict.""" + + def __init__(self, verdict: str = sem.SEM_WRONG, quote: str = "") -> None: + self.calls: List[str] = [] + self._verdict = verdict + self._quote = quote + + def __call__(self, request: sem.SemanticRequest) -> SemanticReply: + self.calls.append(request.block_id) + return SemanticReply(self._verdict, "spy", self._quote) + + +# --- tokenisation + overlap ------------------------------------------------ + +def test_tokenize_splits_camel_and_snake_and_drops_stopwords() -> None: + tokens = tokenize("validateEmail user_email and the RECORD") + assert {"validate", "email", "user", "record"} <= tokens + assert "and" not in tokens # stopwords dropped + assert "the" not in tokens + + +def test_overlap_below_token_floor_is_none_not_zero() -> None: + # Too few domain tokens on the requirement side → unjudgeable, never a false 0. + assert overlap_score(_CORRECT_CODE, "do it") is None + + +def test_overlap_is_fraction_of_requirement_tokens_present() -> None: + # Pinned to the exact value — the denominator is the REQUIREMENT token count (not code, not + # Jaccard). Asserting only >= threshold would not catch a denominator regression. + assert overlap_score(_CORRECT_CODE, _REQUIREMENT) == 0.7 + # A case where the requirement-denominator and code-denominator answers differ, pinned to the + # requirement-denominator value (2/4); a code denominator would give 2/7 ~= 0.29. + assert overlap_score("alpha beta zeta eta theta iota kappa", "alpha beta gamma delta") == 0.5 + + +# --- the adversarial pair (headline) --------------------------------------- + +def test_adversarial_wrong_is_flagged_and_correct_is_not_even_judged() -> None: + spy = _SpyJudge(verdict=sem.SEM_WRONG) + report = assess([_pairing("correct", _CORRECT_CODE, _REQUIREMENT), + _pairing("wrong", _WRONG_CODE, _REQUIREMENT)], judge_fn=spy) + judged = {f.block_id: f for f in report.findings} + # The wrong block is surfaced as a weak link and judged; the faithful one is not. + assert "wrong" in judged + assert judged["wrong"].verdict == sem.SEM_WRONG + assert "correct" not in judged + assert spy.calls == ["wrong"] # pre-filter-first: no model call on the strong block + + +def test_prefilter_first_no_model_call_when_all_blocks_are_strong() -> None: + spy = _SpyJudge() + report = assess([_pairing("a", _CORRECT_CODE, _REQUIREMENT)], judge_fn=spy) + assert spy.calls == [] + assert report.findings == [] + assert report.assessed == 0 + # accounted for as presumed-covered, not silently dropped + assert report.presumed_covered == 1 + + +def test_report_accounts_for_every_in_scope_block() -> None: + report = assess([_pairing("strong", _CORRECT_CODE, _REQUIREMENT), + _pairing("weak", _WRONG_CODE, _REQUIREMENT), + _pairing("gap", _WRONG_CODE, None)], judge_fn=_SpyJudge()) + assert report.assessed == 1 + assert report.presumed_covered == 1 + assert len(report.unjudgeable) == 1 + + +# --- honesty: unjudgeable-not-zero ----------------------------------------- + +def test_missing_requirement_is_unjudgeable_not_judged() -> None: + report = assess([_pairing("noreq", _WRONG_CODE, None)], judge_fn=_SpyJudge()) + assert report.findings == [] + assert [u.block_id for u in report.unjudgeable] == ["noreq"] + assert "no retrievable requirement" in report.unjudgeable[0].reason + + +def test_below_floor_requirement_is_unjudgeable() -> None: + report = assess([_pairing("tiny", _WRONG_CODE, "do it")], judge_fn=_SpyJudge()) + assert [u.block_id for u in report.unjudgeable] == ["tiny"] + + +def test_no_judge_wired_makes_every_weak_link_unjudgeable() -> None: + report = assess([_pairing("wrong", _WRONG_CODE, _REQUIREMENT)], judge_fn=None) + assert report.findings == [] # nothing got a real verdict + assert any(u.block_id == "wrong" for u in report.unjudgeable) # surfaced as a coverage gap + + +# --- the evidence (hallucination) guard ------------------------------------ + +def test_evidence_present_normalises_whitespace() -> None: + assert evidence_present("a b\n c", "a b c") + assert not evidence_present("real code here", "fabricated quote") + assert not evidence_present("code", "") # an empty quote is not evidence + + +def test_fabricated_quote_sets_evidence_not_ok() -> None: + spy = _SpyJudge(verdict=sem.SEM_WRONG, quote="this text is not in the code") + report = assess([_pairing("wrong", _WRONG_CODE, _REQUIREMENT)], judge_fn=spy) + assert report.findings[0].evidence_ok is False + + +def test_real_quote_sets_evidence_ok() -> None: + spy = _SpyJudge(verdict=sem.SEM_WRONG, quote="rate = lookup_rate(amount)") + report = assess([_pairing("wrong", _WRONG_CODE, _REQUIREMENT)], judge_fn=spy) + assert report.findings[0].evidence_ok is True + + +# --- defensive degradation -------------------------------------------------- + +def test_judge_that_raises_degrades_to_unjudgeable() -> None: + def boom(_request: sem.SemanticRequest) -> SemanticReply: + raise RuntimeError("model down") + + report = assess([_pairing("wrong", _WRONG_CODE, _REQUIREMENT)], judge_fn=boom) + assert report.findings == [] + assert any(u.block_id == "wrong" for u in report.unjudgeable) + + +@pytest.mark.parametrize("reply", [None, object(), SemanticReply("bogus")]) +def test_malformed_reply_is_unjudgeable(reply: object) -> None: + report = assess([_pairing("wrong", _WRONG_CODE, _REQUIREMENT)], + judge_fn=lambda _r: reply) + assert report.findings == [] + assert any(u.block_id == "wrong" for u in report.unjudgeable) + + +# --- advisory never gates --------------------------------------------------- + +def test_report_carries_no_gate_signal() -> None: + report = assess([_pairing("wrong", _WRONG_CODE, _REQUIREMENT)], + judge_fn=_SpyJudge(sem.SEM_WRONG)) + # A wrong verdict produces a finding but nothing resembling a pass/fail/exit gate. + assert isinstance(report, SemanticReport) + assert not any(hasattr(report, attr) for attr in ("gate", "exit_code", "passed")) + + +# --- the frozen coverage-report scope -------------------------------------- + +def test_excluded_files_are_skipped_before_ranking() -> None: + report = assess([_pairing("wrong", _WRONG_CODE, _REQUIREMENT, path="skip.py")], + judge_fn=_SpyJudge(), + report={"excluded": [{"path": "skip.py", "reason": "x", "declared_by": "config"}]}) + assert report.findings == [] + assert report.unjudgeable == [] + + +def test_whole_file_claims_are_prioritised_in_ranking() -> None: + weak_plain = _pairing("plain", _WRONG_CODE, _REQUIREMENT, path="plain.py") + weak_claim = _pairing("claim", _WRONG_CODE, _REQUIREMENT, path="claim.py") + ranked = rank_pairings([weak_plain, weak_claim], priority_paths=["claim.py"]) + assert ranked[0].pairing.path == "claim.py" # prioritised file bubbles to the top + + +def test_report_missing_the_fields_yields_empty_scope() -> None: + scope = coverage_scope({"some_other_key": 1}) + assert scope.excluded == set() + assert scope.prioritised == [] + assert coverage_scope(None).excluded == set() # a None report never errors + + +# --- the reference stub ----------------------------------------------------- + +def test_reference_stub_buckets_by_overlap() -> None: + covered = reference_stub_judge(sem.build_semantic_request( + sem.Ranked(_pairing("c", _CORRECT_CODE, _REQUIREMENT), 0.0, True, "x"))) + wrong = reference_stub_judge(sem.build_semantic_request( + sem.Ranked(_pairing("w", _WRONG_CODE, _REQUIREMENT), 0.0, True, "x"))) + assert covered.verdict == sem.SEM_COVERED + assert wrong.verdict == sem.SEM_WRONG + assert covered.evidence_quote # a real line, so the evidence guard can verify it + + +def test_reference_stub_on_empty_code_is_wrong_with_no_quote() -> None: + reply = reference_stub_judge(sem.SemanticRequest("e", _REQUIREMENT, "", "prompt")) + assert reply.verdict == sem.SEM_WRONG + assert reply.evidence_quote == "" + + +# --- the requirement resolver ---------------------------------------------- + +def test_resolve_requirement_reads_scoped_doc(tmp_path: Path) -> None: + doc = tmp_path / "feature.md" + doc.write_text("### cpt-studio-algo-demo\nThe demo requirement text.\n", encoding="utf-8") + assert resolve_requirement(doc, "cpt-studio-algo-demo") == "The demo requirement text." + assert resolve_requirement(doc, "cpt-studio-algo-absent") is None + + +# --- gold + calibration ----------------------------------------------------- + +def test_load_gold_valid_and_malformed(tmp_path: Path) -> None: + good = tmp_path / "gold.toml" + good.write_text('[gold]\nverdict = "wrong"\nrationale = "off"\n', encoding="utf-8") + loaded = load_gold(good) + assert loaded is not None + assert loaded.verdict == "wrong" + + bad = tmp_path / "bad.toml" + bad.write_text('[gold]\nverdict = "nonsense"\n', encoding="utf-8") + assert load_gold(bad) is None + assert load_gold(tmp_path / "missing.toml") is None + assert load_gold(None) is None + + +def test_calibrate_reports_accuracy_and_consistency() -> None: + cases = [(_pairing("w", _WRONG_CODE, _REQUIREMENT), sem.SemanticGold("wrong")), + (_pairing("c", _CORRECT_CODE, _REQUIREMENT), sem.SemanticGold("covered"))] + result = calibrate(cases, reference_stub_judge, runs=3) + assert result.accuracy == 1.0 # stub agrees with both human labels + assert result.consistency == 1.0 # deterministic stub → no run-to-run variance + assert set(result.covered) == {"w", "c"} + + +def test_calibrate_empty_is_none_not_zero() -> None: + result = calibrate([], reference_stub_judge) + assert result.accuracy is None + assert result.consistency is None + + +def test_non_string_verdict_is_unjudgeable_not_raise() -> None: + # A reply whose verdict is a truthy non-string (e.g. 123) must degrade to unjudgeable, not raise + # an AttributeError — this runs outside any try/except and calibration calls it directly. + class _IntVerdict: + verdict = 123 + + assert sem._reply_to_verdict(_IntVerdict()) == sem.SEM_UNJUDGEABLE + + +def test_calibration_excludes_a_crashing_judge() -> None: + # A judge_fn that raises is an operational failure, not a disagreement — the case is excluded, + # not scored as a mismatch, so a transient crash never deflates accuracy. + def boom(_request: sem.SemanticRequest) -> SemanticReply: + raise RuntimeError("model down") + + cal = calibrate([(_pairing("c", _WRONG_CODE, _REQUIREMENT), sem.SemanticGold("wrong"))], + boom, runs=2) + assert "c" in cal.excluded + assert cal.accuracy is None # the only case crashed → nothing scored + assert cal.per_case == [] + + +def test_calibration_excludes_an_unscoreable_pairing() -> None: + # A pairing with no requirement (unjudgeable at the pre-filter) is excluded from calibration, + # not judged on empty input and scored as a mismatch. + cal = calibrate([(_pairing("noreq", _WRONG_CODE, None), sem.SemanticGold("wrong"))], + reference_stub_judge, runs=2) + assert "noreq" in cal.excluded + assert cal.accuracy is None + + +def test_prompt_fields_are_bounded() -> None: + # A huge code block must not produce an unbounded prompt; the interpolated fields are capped, + # while the structured request field stays full for the evidence guard. + huge = "x = 1\n" * 5000 + req = sem.build_semantic_request(sem.Ranked(_pairing("big", huge, _REQUIREMENT), 0.0, True, "x")) + assert len(req.prompt) < sem._PROMPT_FIELD_CAP * 3 # bounded, not ~30k + assert "[…truncated]" in req.prompt + assert req.code == huge # the structured field is not truncated + + +# --- deep-review hardening (verified findings M1, M2, M3, M5, m2, excluded-count) --- + +def test_non_string_evidence_quote_and_rationale_do_not_sink_assessment() -> None: + # a host returning a non-string evidence_quote/rationale must not crash out of _judge_one + # (its try only wraps the model call) and sink assess()/calibrate() — degrade the fields. + class _BadReply: + verdict = "wrong" + evidence_quote = 42 # non-string + rationale = ["not", "a", "string"] # non-string + + report = assess([_pairing("wrong", _WRONG_CODE, _REQUIREMENT)], judge_fn=lambda _r: _BadReply()) + assert report.findings[0].verdict == sem.SEM_WRONG # verdict still parsed, no crash + assert report.findings[0].evidence_ok is False # non-string quote is not evidence + cal = calibrate([(_pairing("w", _WRONG_CODE, _REQUIREMENT), sem.SemanticGold("wrong"))], + lambda _r: _BadReply(), runs=2) + assert cal.accuracy == 1.0 # scored without raising + + +def test_calibration_excludes_an_unknown_verdict_reply() -> None: + # a reply mapping to UNJUDGEABLE for a non-crash reason (unknown verdict) is excluded, not + # scored as a mismatch that deflates accuracy. + cal = calibrate([(_pairing("u", _WRONG_CODE, _REQUIREMENT), sem.SemanticGold("wrong"))], + lambda _r: SemanticReply("maybe"), runs=2) + assert "u" in cal.excluded + assert cal.accuracy is None + + +def test_calibration_does_not_exclude_a_real_verdict_mentioning_an_error() -> None: + # a genuine verdict must not be excluded just because its free-text rationale mentions an + # error phrase — exclusion keys off the verdict, not a rationale substring. + reply = SemanticReply("wrong", "the branch where judge_fn raised is not covered") + cal = calibrate([(_pairing("r", _WRONG_CODE, _REQUIREMENT), sem.SemanticGold("wrong"))], + lambda _r: reply, runs=2) + assert cal.excluded == [] + assert cal.accuracy == 1.0 + + +def test_calibration_accuracy_below_one_on_disagreement() -> None: + # accuracy must be able to fall below 1 — a stub verdict that disagrees with gold is a miss. + cases = [(_pairing("hit", _WRONG_CODE, _REQUIREMENT), sem.SemanticGold("wrong")), # stub wrong == gold + (_pairing("miss", _CORRECT_CODE, _REQUIREMENT), sem.SemanticGold("wrong"))] # stub covered != gold + cal = calibrate(cases, reference_stub_judge, runs=2) + assert cal.accuracy == 0.5 + assert 0.0 < cal.accuracy < 1.0 + + +def test_calibration_consistency_below_one_for_a_flaky_judge() -> None: + # consistency must be able to fall below 1 — a judge that varies run-to-run. + calls = {"n": 0} + + def flaky(_request: sem.SemanticRequest) -> SemanticReply: + calls["n"] += 1 + return SemanticReply("covered" if calls["n"] % 2 else "wrong") + + cal = calibrate([(_pairing("f", _CORRECT_CODE, _REQUIREMENT), sem.SemanticGold("covered"))], + flaky, runs=3) + assert cal.consistency is not None + assert cal.consistency < 1.0 # covered, wrong, covered → majority 2/3 + + +def test_reference_stub_partial_bucket() -> None: + # a mid-overlap block (~0.2, in [threshold/2, threshold)) buckets to PARTIAL. + partial_code = "def validate_thing(item):\n email_field = item\n return compute_other(item)" + req = sem.build_semantic_request(sem.Ranked(_pairing("p", partial_code, _REQUIREMENT), 0.2, True, "x")) + assert reference_stub_judge(req).verdict == sem.SEM_PARTIAL + + +def test_report_counts_excluded_file_blocks() -> None: + # excluded-count: blocks in a human-excluded file are counted (skipped_excluded), not dropped. + report = assess([_pairing("keep", _WRONG_CODE, _REQUIREMENT, path="keep.py"), + _pairing("s1", _WRONG_CODE, _REQUIREMENT, path="skip.py"), + _pairing("s2", _WRONG_CODE, _REQUIREMENT, path="skip.py")], + judge_fn=_SpyJudge(sem.SEM_WRONG), + report={"excluded": [{"path": "skip.py"}]}) + assert report.skipped_excluded == 2 + assert report.assessed == 1 # only the kept block was judged + + +# --- deep-review round 2 (M1 design fix, M2 crash, M3 minority-unjudgeable, M5 accounting) --- + +def test_whole_file_claim_block_is_always_judged_despite_high_overlap() -> None: + # a whole_file_claims (prioritised) file's overlap is untrustworthy — a high-overlap block + # there must be JUDGED, not waved through as presumed_covered (the engine's headline purpose). + spy = _SpyJudge(sem.SEM_WRONG) + report = assess([_pairing("claim", _CORRECT_CODE, _REQUIREMENT, path="claim.py")], + judge_fn=spy, report={"whole_file_claims": [{"path": "claim.py"}]}) + assert spy.calls == ["claim"] # judged despite overlap 0.7 (would be strong) + assert report.presumed_covered == 0 + assert [f.verdict for f in report.findings] == [sem.SEM_WRONG] + + +def test_load_gold_non_string_verdict_returns_none_not_raise(tmp_path: Path) -> None: + # a TOML array/table verdict is unhashable; load_gold must return None, never raise. + for bad in ('verdict = ["wrong"]', 'verdict = { x = 1 }'): + p = tmp_path / "g.toml" + p.write_text(f"[gold]\n{bad}\n", encoding="utf-8") + assert load_gold(p) is None + + +def test_calibration_minority_unjudgeable_does_not_deflate_consistency() -> None: + # one transient crash among good runs must not deflate consistency or flip the exclude + # decision — UNJUDGEABLE runs are dropped before majority/consistency. + calls = {"n": 0} + + def flaky(_request: sem.SemanticRequest) -> SemanticReply: + calls["n"] += 1 + if calls["n"] == 1: + raise RuntimeError("transient") # only the first run fails + return SemanticReply("wrong") + + cal = calibrate([(_pairing("m", _WRONG_CODE, _REQUIREMENT), sem.SemanticGold("wrong"))], + flaky, runs=3) + assert cal.excluded == [] # a real verdict was produced → scored + assert cal.accuracy == 1.0 + assert cal.consistency == 1.0 # over the 2 real runs, both agreed + + +def test_assess_covered_verdict_lands_in_findings() -> None: + # a weak link the judge rules COVERED must land in findings with a real verdict, not vanish + # from every accounting bucket (it is not UNJUDGEABLE and was not presumed_covered). + report = assess([_pairing("w", _WRONG_CODE, _REQUIREMENT)], judge_fn=_SpyJudge(sem.SEM_COVERED)) + assert [f.verdict for f in report.findings] == [sem.SEM_COVERED] + assert report.assessed == 1 + + +def test_assess_accounting_identity_holds() -> None: + # every pairing is accounted for exactly once across the four buckets. + pairings = [_pairing("strong", _CORRECT_CODE, _REQUIREMENT), + _pairing("weak", _WRONG_CODE, _REQUIREMENT), + _pairing("gap", _WRONG_CODE, None), + _pairing("skip", _WRONG_CODE, _REQUIREMENT, path="x.py")] + report = assess(pairings, judge_fn=_SpyJudge(sem.SEM_PARTIAL), + report={"excluded": [{"path": "x.py"}]}) + assert (report.assessed + report.presumed_covered + + len(report.unjudgeable) + report.skipped_excluded) == len(pairings) + + +def test_majority_of_empty_is_unjudgeable() -> None: + # Defensive: _majority over no verdicts is UNJUDGEABLE with count 0, never an error. + assert sem._majority([]) == (sem.SEM_UNJUDGEABLE, 0) + + +# --- deep-review round 3 (M2 comment inflation, m2 effective-n, m3 unicode, m5/i2 gaps) --- + +def test_comment_echoing_requirement_does_not_mask_wrong_code() -> None: + # a comment/docstring that restates the requirement must NOT inflate overlap and wave the + # block through as presumed_covered — comments/strings are stripped before scoring, so wrong + # code beneath a requirement-echoing comment is still surfaced as a weak link and judged. + echoed = ('def compute_tax(amount):\n' + ' # Validate the user email address format and reject a malformed address' + ' before saving the record.\n' + ' """Validate the user email address format and reject a malformed address."""\n' + ' return amount * lookup_rate(amount)') + score = overlap_score(echoed, _REQUIREMENT) + assert score is not None # enough executable tokens to compare + assert score < sem._WEAK_LINK_THRESHOLD # not inflated by the echoing prose + spy = _SpyJudge(sem.SEM_WRONG) + report = assess([_pairing("masked", echoed, _REQUIREMENT)], judge_fn=spy) + assert spy.calls == ["masked"] # judged, not presumed_covered + assert report.presumed_covered == 0 + + +def test_tokenize_keeps_non_ascii_terms_whole() -> None: + # non-ASCII terms must stay whole, not fragment to empty — an ASCII-only split deflated the + # pre-filter and could report a real multilingual block UNJUDGEABLE. + assert "gebühr" in tokenize("Berechne die Gebühr") # not {'geb', 'hr'} + cyrillic = tokenize("проверить адрес электронной почты") + assert cyrillic # non-empty, not fragmented below the floor + assert "адрес" in cyrillic + + +def test_single_surviving_run_is_unmeasurable_for_both_accuracy_and_consistency() -> None: + # when only one run survives (others crashed), BOTH accuracy and consistency are + # unmeasurable → None. Scoring the lone survivor for accuracy would let a transient crash flip an + # excluded tie into a hit/miss, so the case feeds neither denominator (symmetric gate). + calls = {"n": 0} + + def mostly_down(_request: sem.SemanticRequest) -> SemanticReply: + calls["n"] += 1 + if calls["n"] == 1: + return SemanticReply("wrong") # exactly one real verdict + raise RuntimeError("down") + + cal = calibrate([(_pairing("s", _WRONG_CODE, _REQUIREMENT), sem.SemanticGold("wrong"))], + mostly_down, runs=3) + assert cal.excluded == [] # a real verdict was produced → in per_case + assert cal.accuracy is None # one survivor → accuracy unmeasurable + assert cal.consistency is None # one survivor → consistency unmeasurable + assert cal.per_case[0]["runs_effective"] == 1 + assert cal.per_case[0]["matched"] is None + assert cal.per_case[0]["consistency"] is None + + +def test_majority_tie_break_is_canonical_not_first_seen() -> None: + # ties resolve by sorted verdict name (covered < partial < wrong), independent of run + # order — a regression to the sibling judge's first-seen tie-break would pass every other test. + assert sem._majority(["wrong", "covered"]) == ("covered", 1) + assert sem._majority(["covered", "wrong"]) == ("covered", 1) + assert sem._majority(["wrong", "partial"]) == ("partial", 1) + + +def test_overlap_below_code_token_floor_is_none() -> None: + # the CODE side of the token floor is exercised directly — below-floor code is unjudgeable + # (None), not a judged weak link. Deleting the code-side guard would otherwise pass every test. + assert overlap_score("a b", _REQUIREMENT) is None # 0 domain tokens on the code side + assert overlap_score("", _REQUIREMENT) is None + + +# --- deep-review round 4 (F1 escape-aware strip, F2 gold-independent tie) --- + +def test_escaped_quote_string_literal_does_not_leak_requirement_text() -> None: + # A string literal that OPENS with an escaped quote and restates the requirement must be fully + # stripped from the overlap view, not leak its words into code_tokens. _code_views blanks each + # STRING token wholesale via the grammar-aware tokenizer, so an escaped internal quote is a + # non-issue (it is one token) — where the old escape-aware regex could have ended the literal early. + leaky = ('def compute_tax(amount):\n' + ' ERR = "\\"validate the user email address format and reject a malformed address\\""\n' + ' return amount * lookup_rate(amount)') + score = overlap_score(leaky, _REQUIREMENT) + assert score is not None + assert score < sem._WEAK_LINK_THRESHOLD # requirement words stripped, not leaked + spy = _SpyJudge(sem.SEM_WRONG) + report = assess([_pairing("leaky", leaky, _REQUIREMENT)], judge_fn=spy) + assert spy.calls == ["leaky"] # judged, not presumed_covered + assert report.presumed_covered == 0 + + +def test_strict_tie_is_gold_independent_and_excluded_from_accuracy() -> None: + # a strict tie (no majority) must not be scored by the gold label's alphabetical rank. It is + # excluded from accuracy (matched=None) — the SAME result whether gold is 'wrong' or 'covered' — + # while its low consistency is still reported. Previously a 1-1 tie was a forced miss on 'wrong'. + calls = {"n": 0} + + def split(_request: sem.SemanticRequest) -> SemanticReply: + calls["n"] += 1 + return SemanticReply("covered" if calls["n"] % 2 else "wrong") # covered, wrong → 1-1 tie + + cal_wrong = calibrate([(_pairing("t", _CORRECT_CODE, _REQUIREMENT), sem.SemanticGold("wrong"))], + split, runs=2) + assert cal_wrong.accuracy is None # no majority → unmeasurable, not a miss + assert cal_wrong.consistency == 0.5 # 1 of 2 agreed → reported honestly + assert cal_wrong.per_case[0]["matched"] is None + cal_covered = calibrate([(_pairing("t", _CORRECT_CODE, _REQUIREMENT), sem.SemanticGold("covered"))], + split, runs=2) + assert cal_covered.accuracy is None # identical result — gold label does not decide + + +# --- property-based invariants (seed-deterministic; generated inputs the examples never picked) --- + +# Deliberately mixes ASCII, snake/camelCase, non-ASCII (Gebühr, Cyrillic), and the noise that +# _code_views must survive: comments, plain strings, and escaped-quote string literals. +_VOCAB = ["validate", "email", "address", "reject", "malformed", "save", "record", "compute", + "tax", "amount", "rate", "lookup", "user", "format", "checkUser", "reject_all", + "Gebühr", "проверить", "адрес"] + + +def _rand_words(rng: random.Random, n: int) -> str: + return " ".join(rng.choice(_VOCAB) for _ in range(n)) + + +def _rand_code(rng: random.Random) -> str: + """Generate a code-like block: statements, comments, plain + escaped-quote string literals, + docstrings, and blanks — the shapes _code_views and tokenize must handle without raising.""" + lines = ["def f():"] + for _ in range(rng.randint(0, 7)): + kind = rng.randint(0, 4) + if kind == 0: + lines.append(f" {rng.choice(_VOCAB)}_{rng.choice(_VOCAB)} = {rng.randint(0, 9)}") + elif kind == 1: + lines.append(f" # {_rand_words(rng, rng.randint(1, 5))}") + elif kind == 2: + quote = rng.choice(['"', "'"]) + esc = rng.choice(["", "\\" + quote]) # sometimes open with an escaped quote + lines.append(f" ERR = {quote}{esc}{_rand_words(rng, rng.randint(1, 4))}{esc}{quote}") + elif kind == 3: + lines.append(f' """{_rand_words(rng, rng.randint(1, 4))}"""') + else: + lines.append("") + return "\n".join(lines) + + +def _raises(_request: sem.SemanticRequest) -> SemanticReply: + raise ValueError("model down") + + +class _HostileReply: + """A reply whose attribute access itself raises — the fail-safe boundary must absorb it.""" + + @property + def verdict(self) -> str: + raise RuntimeError("lazy parse failed") + + +def _judges(rng: random.Random) -> List[Optional[Callable[[sem.SemanticRequest], object]]]: + """The full spread of judge behaviours a host might supply — none may make assess/calibrate raise.""" + verdicts = [sem.SEM_COVERED, sem.SEM_PARTIAL, sem.SEM_WRONG, "maybe", ""] + return [None, reference_stub_judge, _SpyJudge(), _raises, + lambda _r: _HostileReply(), + lambda _r: SemanticReply(rng.choice(verdicts))] + + +def test_property_overlap_score_is_bounded_or_none() -> None: + rng = random.Random(1234) + for _ in range(400): + score = overlap_score(_rand_code(rng), _rand_words(rng, rng.randint(0, 8))) + assert score is None or (isinstance(score, float) and 0.0 <= score <= 1.0) + + +def test_property_tokenize_drops_short_and_stopwords_and_is_deterministic() -> None: + rng = random.Random(5678) + for _ in range(400): + text = _rand_code(rng) if rng.random() < 0.5 else _rand_words(rng, rng.randint(0, 10)) + tokens = tokenize(text) + assert tokens == tokenize(text) # deterministic + assert all(len(t) > 1 and t not in sem._STOPWORDS for t in tokens) + + +def test_property_assess_never_raises_and_accounting_identity_holds() -> None: + rng = random.Random(9012) + for _ in range(300): + n = rng.randint(0, 6) + pairings = [ + Pairing(block_id=f"b{i}", inst=f"inst-{i}", path=f"p{rng.randint(0, 3)}.py", + start_line=rng.randint(1, 99), code=_rand_code(rng), + requirement=(None if rng.random() < 0.2 else _rand_words(rng, rng.randint(0, 8)))) + for i in range(n)] + paths = [p.path for p in pairings] + report = {"excluded": [{"path": p} for p in paths if rng.random() < 0.25], + "whole_file_claims": [{"path": p} for p in paths if rng.random() < 0.25]} + judge = rng.choice(_judges(rng)) + result = assess(pairings, judge_fn=judge, report=report) # must not raise + assert (result.assessed + result.presumed_covered + + len(result.unjudgeable) + result.skipped_excluded) == n + assert all(f.verdict in sem._MODEL_VERDICTS for f in result.findings) + + +def test_property_calibrate_never_raises_and_metrics_are_bounded() -> None: + rng = random.Random(3456) + for _ in range(300): + cases = [ + (Pairing(block_id=f"c{i}", inst=f"inst-{i}", path="m.py", start_line=i + 1, + code=_rand_code(rng), + requirement=(None if rng.random() < 0.2 else _rand_words(rng, rng.randint(0, 8)))), + sem.SemanticGold(rng.choice([sem.SEM_COVERED, sem.SEM_PARTIAL, sem.SEM_WRONG]))) + for i in range(rng.randint(0, 4))] + judge = rng.choice([j for j in _judges(rng) if j is not None]) + cal = calibrate(cases, judge, runs=rng.randint(1, 4)) # must not raise + for metric in (cal.accuracy, cal.consistency): + assert metric is None or 0.0 <= metric <= 1.0 + assert len(cal.covered) == len(cases) + + +# --- deep-review round 5 (ainetx: evidence-strip, forced, precedence, gap start_line, path norm) --- + +def test_evidence_quote_matching_only_a_comment_is_not_evidence() -> None: + # the hallucination guard strips comments/strings like overlap_score, so a quote + # that occurs only in a comment (not executable code) sets evidence_ok=False. + code = ('def compute_tax(amount):\n' + ' # validate the user email address and reject malformed input\n' + ' result = lookup_rate(amount)\n' + ' return result * amount') + spy = _SpyJudge(verdict=sem.SEM_WRONG, quote="validate the user email address") + report = assess([_pairing("masked", code, _REQUIREMENT)], judge_fn=spy) + assert spy.calls == ["masked"] # judged (low overlap on executable tokens) + assert report.findings[0].evidence_ok is False # quote lived only in the comment + + +def test_whole_file_claim_finding_is_marked_forced() -> None: + # a judgment forced by a whole_file_claim carries forced=True; an ordinary + # weak-link judgment carries forced=False, so a report consumer can tell them apart. + spy = _SpyJudge(sem.SEM_WRONG) + report = assess([_pairing("claim", _CORRECT_CODE, _REQUIREMENT, path="claim.py"), + _pairing("weak", _WRONG_CODE, _REQUIREMENT, path="plain.py")], + judge_fn=spy, report={"whole_file_claims": [{"path": "claim.py"}]}) + by_id = {f.block_id: f for f in report.findings} + assert by_id["claim"].forced is True + assert by_id["weak"].forced is False + + +def test_excluded_takes_precedence_over_whole_file_claims() -> None: + # a path in BOTH excluded and whole_file_claims is dropped (exclusion is the + # human override, applied before ranking). + spy = _SpyJudge(sem.SEM_WRONG) + report = assess([_pairing("both", _WRONG_CODE, _REQUIREMENT, path="both.py")], + judge_fn=spy, + report={"excluded": [{"path": "both.py"}], + "whole_file_claims": [{"path": "both.py"}]}) + assert spy.calls == [] # never judged — excluded wins + assert report.skipped_excluded == 1 + + +def test_unjudgeable_gap_is_typed_and_carries_start_line() -> None: + # a coverage gap is a typed SemanticGap with start_line, located like a finding. + report = assess([_pairing("noreq", _WRONG_CODE, None, start_line=42)], judge_fn=_SpyJudge()) + gap = report.unjudgeable[0] + assert isinstance(gap, sem.SemanticGap) + assert (gap.block_id, gap.start_line) == ("noreq", 42) + + +def test_scope_paths_match_across_separator_style() -> None: + # a report path in Windows separators matches a pairing path in POSIX separators; + # scope comparison is separator-normalised. + spy = _SpyJudge(sem.SEM_WRONG) + report = assess([_pairing("x", _WRONG_CODE, _REQUIREMENT, path="pkg/mod.py")], + judge_fn=spy, report={"excluded": [{"path": "pkg\\mod.py"}]}) + assert report.skipped_excluded == 1 # matched despite backslash vs forward-slash + + +# --- deep-review round 5b (ainetx: gold-log, schema version, judge id, boundaries, formula/oracle) --- + +def test_missing_gold_file_is_logged(tmp_path: Path, caplog) -> None: + # a missing gold file (the common misconfiguration) is logged, not silently None. + import logging + with caplog.at_level(logging.WARNING): + assert load_gold(tmp_path / "nope.toml") is None + assert any("gold file not found" in r.message for r in caplog.records) + + +def test_report_and_calibration_carry_schema_version() -> None: + # the consumed report shapes carry a schema-version discriminator. + report = assess([_pairing("w", _WRONG_CODE, _REQUIREMENT)], judge_fn=_SpyJudge()) + assert report.schema_version == sem.SEMANTIC_SCHEMA_VERSION + cal = calibrate([(_pairing("w", _WRONG_CODE, _REQUIREMENT), sem.SemanticGold("wrong"))], + reference_stub_judge, runs=3) + assert cal.schema_version == sem.SEMANTIC_SCHEMA_VERSION + + +def test_calibration_records_runs_and_judge_identity() -> None: + # runs_per_scenario is asserted, and the calibration records + # which judge produced it. + cal = calibrate([(_pairing("w", _WRONG_CODE, _REQUIREMENT), sem.SemanticGold("wrong"))], + reference_stub_judge, runs=3) + assert cal.runs_per_scenario == 3 + assert cal.judge == "reference_stub_judge" + + +def test_weak_link_threshold_boundary_is_strong_not_weak() -> None: + # a block scoring EXACTLY the threshold (0.30) is strong (score < threshold is + # strict), so it is presumed-covered, not judged. + req = "alpha beta gamma delta epsilon zeta eta theta iota kappa" # 10 domain tokens + code = "alpha beta gamma lorem ipsum dolor" # shares exactly 3 → 0.30 + assert overlap_score(code, req) == 0.30 + spy = _SpyJudge(sem.SEM_WRONG) + report = assess([_pairing("edge", code, req)], judge_fn=spy) + assert spy.calls == [] # not a weak link at the exact boundary + assert report.presumed_covered == 1 + + +def test_min_tokens_floor_boundary_and_distinct_reasons() -> None: + # exactly _MIN_TOKENS (4) is sufficient; 3 is below the floor (None). And the + # "below floor" gap reason stays distinct from the "no requirement" gap reason. + assert overlap_score("alpha beta gamma delta", "alpha beta gamma delta") is not None # 4 tokens + assert overlap_score("alpha beta gamma", "alpha beta gamma") is None # 3 tokens + reason_noreq = assess([_pairing("nr", _WRONG_CODE, None)]).unjudgeable[0].reason + reason_floor = assess([_pairing("tf", "a b", "c d")]).unjudgeable[0].reason + assert reason_noreq != reason_floor + assert "requirement" in reason_noreq + assert "floor" in reason_floor + + +# --- deep-review round 5c (review pass: strip-policy split + calibration symmetry) --- + +def test_evidence_quote_of_a_real_string_literal_is_accepted() -> None: + # a quote of an inline string literal the code executes (an error message) IS evidence + # — evidence_present strips comments/docstrings but keeps string-literal contents. + code = ('def compute_tax(amount):\n' + ' if amount < 0:\n' + ' raise ValueError("amount must be positive")\n' + ' return amount * lookup_rate(amount)') + spy = _SpyJudge(verdict=sem.SEM_WRONG, quote='raise ValueError("amount must be positive")') + report = assess([_pairing("s", code, _REQUIREMENT)], judge_fn=spy) + assert spy.calls == ["s"] # low overlap → judged + assert report.findings[0].evidence_ok is True # real string quote survives the strip + + +def test_reference_stub_grounds_its_quote_on_executable_code() -> None: + # on a block whose COMMENT echoes the requirement, the stub picks an executable line + # (not the comment), so its own evidence_ok is True — it keeps its "verifiable quote" promise. + code = ('def compute_tax(amount):\n' + ' # validate the user email address and reject malformed input before saving\n' + ' return compute(amount)') + req = sem.build_semantic_request(sem.Ranked(_pairing("c", code, _REQUIREMENT), 0.1, True, "x")) + reply = reference_stub_judge(req) + assert evidence_present(code, reply.evidence_quote) is True # quote is real code, guard accepts + assert "validate" not in reply.evidence_quote # not the requirement-echoing comment + + +def test_transient_crash_does_not_flip_accuracy_via_lone_survivor() -> None: + # a 1-1 tie is excluded from accuracy (None); if a transient crash drops it to a single + # survivor, accuracy must STILL be None — not flip to a hit/miss by which run happened to crash. + tie_calls = {"n": 0} + + def split(_r: sem.SemanticRequest) -> SemanticReply: + tie_calls["n"] += 1 + return SemanticReply("wrong" if tie_calls["n"] % 2 else "partial") + + tie = calibrate([(_pairing("t", _WRONG_CODE, _REQUIREMENT), sem.SemanticGold("wrong"))], + split, runs=2) + assert tie.accuracy is None # 1-1 tie → excluded from accuracy + + crash_calls = {"n": 0} + + def split_crash(_r: sem.SemanticRequest) -> SemanticReply: + crash_calls["n"] += 1 + if crash_calls["n"] == 1: + return SemanticReply("partial") + raise RuntimeError("down") + + lone = calibrate([(_pairing("t", _WRONG_CODE, _REQUIREMENT), sem.SemanticGold("wrong"))], + split_crash, runs=2) + assert lone.accuracy is None # lone survivor → still unmeasurable, no flip + + +# --- deep-review round 6 (grammar-aware lexing: docstring-leak, #-in-string, runs=1 accuracy) --- + +def test_triple_quote_inside_comment_does_not_leak_docstring_into_overlap() -> None: + # a stray triple-quote inside a comment must NOT mis-pair with a real docstring and + # leak its requirement-echoing body into overlap. The wrong block stays a weak link (judged), not + # a false presumed_covered; a docstring-only quote is not evidence. (The old regex strip leaked.) + code = ('def compute_tax(amount):\n' + ' # prefer """ here\n' + ' """Validate the user email address format and reject a malformed address before saving."""\n' + ' return amount * lookup_rate(amount)') + assert overlap_score(code, _REQUIREMENT) < sem._WEAK_LINK_THRESHOLD # docstring did NOT leak + spy = _SpyJudge(sem.SEM_WRONG, quote="Validate the user email address format") + report = assess([_pairing("leak", code, _REQUIREMENT)], judge_fn=spy) + assert report.presumed_covered == 0 # judged, not waved through as covered + assert report.findings[0].evidence_ok is False # docstring-only quote is not evidence + + +def test_hash_inside_a_string_literal_does_not_truncate_evidence() -> None: + # a '#' inside a kept string literal must not be read as a comment and truncate the + # evidence haystack — a verbatim quote of that string is still valid evidence. + code = ('def pick_color(kind):\n' + ' if kind < 0:\n' + ' raise ValueError("bad #tag for color #FF0000")\n' + ' return compute(kind)') + spy = _SpyJudge(sem.SEM_WRONG, quote='raise ValueError("bad #tag for color #FF0000")') + report = assess([_pairing("c", code, _REQUIREMENT)], judge_fn=spy) + assert report.findings[0].evidence_ok is True # the #-bearing string survived the strip + + +def test_calibrate_runs_one_scores_accuracy() -> None: + # runs=1 is a supported input; a single clean verdict has a trivial majority and DOES + # score accuracy (consistency stays None — repeatability is unmeasurable with one run). + cal = calibrate([(_pairing("w", _WRONG_CODE, _REQUIREMENT), sem.SemanticGold("wrong"))], + reference_stub_judge, runs=1) + assert cal.accuracy == 1.0 # the one clean verdict matched gold + assert cal.consistency is None # one run → repeatability unmeasurable + + +def test_untokenizable_fragment_falls_back_without_raising() -> None: + # a mid-file fragment that won't tokenize (an unterminated string) must not + # raise — overlap falls back to a comment strip, evidence to raw code. Advisory and safe. + fragment = 'x = "unterminated\n y = compute(the_user_email_address) # note' + score = overlap_score(fragment, _REQUIREMENT) # must not raise + assert score is None or (isinstance(score, float) and 0.0 <= score <= 1.0) + assert evidence_present(fragment, "y = compute(the_user_email_address)") is True # raw fallback + + +# --- deep-review round 7 (line-offset table must match the tokenizer's \n-only line splitting) --- + +def test_line_boundary_char_does_not_desync_evidence_view() -> None: + # the offset table feeds the evidence view; a docstring after a form-feed page + # break must still be blanked (str.splitlines() breaks on \x0c but the tokenizer does not, so a + # mismatched table would leave the docstring un-blanked and accept a docstring-only quote). + code = ('def compute_tax(amount):\n' + '\x0c\n' # a form-feed page break (PEP 8 convention) + ' """validate the user email address and reject malformed input"""\n' + ' return amount * lookup_rate(amount)') + spy = _SpyJudge(sem.SEM_WRONG, quote="validate the user email address") + report = assess([_pairing("d", code, _REQUIREMENT)], judge_fn=spy) + assert spy.calls == ["d"] # judged (docstring blanked → low overlap) + assert report.findings[0].evidence_ok is False # docstring-only quote is not evidence + + +def test_many_line_boundary_chars_do_not_leak_comment_into_overlap() -> None: + # enough boundary chars before a requirement-echoing comment would, under the old + # str.splitlines() table, desync spans and leak the comment into the overlap view → false + # presumed_covered. With the \n-matched table the comment stays blanked and the block is judged. + breaks = "\x0c\n" * 40 + code = ('def compute_tax(amount):\n' + + breaks + + ' # validate the user email address format reject malformed address saving record\n' + ' return amount * lookup_rate(amount)') + assert overlap_score(code, _REQUIREMENT) < sem._WEAK_LINK_THRESHOLD # comment did not leak + spy = _SpyJudge(sem.SEM_WRONG) + report = assess([_pairing("ff", code, _REQUIREMENT)], judge_fn=spy) + assert report.presumed_covered == 0 # judged, not waved through + + +def test_dedented_statement_leading_string_is_not_evidence() -> None: + # a statement-leading string after a DEDENT is a bare string statement (prose), + # not executable logic — _STATEMENT_START includes DEDENT so it is blanked from the evidence view, + # and a quote lifted only from it is not accepted as evidence. + code = ('def compute_tax(amount):\n' + ' total = amount * 2\n' + '"""validate the user email address and reject malformed input"""\n' + 'result = total') + spy = _SpyJudge(sem.SEM_WRONG, quote="validate the user email address") + report = assess([_pairing("d", code, _REQUIREMENT)], judge_fn=spy) + assert spy.calls == ["d"] # judged (all strings blanked from overlap) + assert report.findings[0].evidence_ok is False # dedented bare string is not evidence + + +def test_fstring_prose_does_not_inflate_overlap() -> None: + # PEP 701: on Python 3.12+ an f-string tokenizes to FSTRING_START/MIDDLE/END, not one STRING + # token. The literal text must still be blanked from overlap, so a requirement-echoing f-string + # cannot mask wrong code as presumed_covered. + code = ('def compute_tax(amount):\n' + ' log(f"validate the user email address and reject malformed address before saving")\n' + ' return amount * 2') + assert overlap_score(code, _REQUIREMENT) < sem._WEAK_LINK_THRESHOLD # f-string prose blanked + spy = _SpyJudge(sem.SEM_WRONG) + report = assess([_pairing("f", code, _REQUIREMENT)], judge_fn=spy) + assert report.presumed_covered == 0 # judged, not waved through as covered + + +def test_present_but_unreadable_gold_file_is_logged(tmp_path: Path, caplog) -> None: + # A gold file that exists but fails to parse (case B) is logged as unexpected, not silently None. + import logging + bad = tmp_path / "g.toml" + bad.write_text("not = valid = toml [[[", encoding="utf-8") + with caplog.at_level(logging.WARNING, logger="studio.utils.eval_semantic"): + assert load_gold(bad) is None + assert any("present but unreadable" in r.message for r in caplog.records) + + +def test_calibrating_against_reference_stub_warns(caplog) -> None: + # The stub re-uses the pre-filter's own overlap; calibrating against it is warned at runtime so a + # caller who skipped the docstring still sees the numbers are not judge quality. + import logging + with caplog.at_level(logging.WARNING, logger="studio.utils.eval_semantic"): + calibrate([(_pairing("w", _WRONG_CODE, _REQUIREMENT), sem.SemanticGold("wrong"))], + reference_stub_judge, runs=2) + assert any("reference_stub_judge" in r.message for r in caplog.records) + + +def test_untokenizable_block_with_requirement_echoing_string_is_not_presumed_covered() -> None: + # A block that fails to tokenize (unterminated string) must not let a requirement-echoing STRING + # inflate overlap into a false presumed_covered — the fallback can't strip strings, so an + # unlexable block is unjudgeable (None), never presumed_covered. + frag = ('def compute_tax(amount):\n' + ' msg = "validate the user email address and reject a malformed address before saving\n' + ' return amount * 2') + assert overlap_score(frag, _REQUIREMENT) is None # unlexable → unjudgeable, not inflated + spy = _SpyJudge(sem.SEM_WRONG) + report = assess([_pairing("frag", frag, _REQUIREMENT)], judge_fn=spy) + assert spy.calls == [] + assert report.presumed_covered == 0 # never presumed covered + assert any(u.block_id == "frag" for u in report.unjudgeable) # surfaced as a coverage gap + + +def test_untokenizable_block_comment_quote_is_not_evidence() -> None: + # On the fallback path the evidence view is comment-stripped too, so a quote from a comment in an + # unlexable block is not accepted as evidence. + frag = 'x = "unterminated\n# validate the user email address\ny = 1' + assert evidence_present(frag, "validate the user email address") is False + + +def test_gold_verdict_is_normalized_for_accuracy() -> None: + # A directly-built SemanticGold with mixed case/whitespace must not deflate accuracy — the gold + # side is normalized like the judge side. + cal = calibrate([(_pairing("w", _WRONG_CODE, _REQUIREMENT), sem.SemanticGold(" Wrong "))], + reference_stub_judge, runs=2) + assert cal.accuracy == 1.0 # " Wrong " matches the judge's normalized "wrong" + + +def test_prompt_requirement_side_is_bounded() -> None: + # The requirement-side cap in build_semantic_request is exercised (the code-side test alone let a + # dropped requirement cap survive). A huge requirement must be truncated in the prompt. + huge_req = "email " * 5000 + req = sem.build_semantic_request(sem.Ranked(_pairing("r", _WRONG_CODE, huge_req), 0.0, True, "x")) + assert len(req.prompt) < sem._PROMPT_FIELD_CAP * 3 + assert "[…truncated]" in req.prompt + assert req.requirement == huge_req # the structured field stays full + + +def test_identifier_gaming_blind_spot_is_presumed_covered_not_judged() -> None: + # Pin the acknowledged blind spot as a visible contract: a block whose executable identifiers echo + # the requirement pushes overlap ABOVE threshold, so it is presumed_covered and never judged — a + # semantic model, not this lexical filter, would be needed to catch it. + gamed = ('def f(x):\n' + ' validate = user = email = address = format = reject = malformed = record = x\n' + ' return x') + assert overlap_score(gamed, _REQUIREMENT) >= sem._WEAK_LINK_THRESHOLD # identifiers inflate it + spy = _SpyJudge(sem.SEM_WRONG) + report = assess([_pairing("gamed", gamed, _REQUIREMENT)], judge_fn=spy) + assert spy.calls == [] # never judged — the blind spot + assert report.presumed_covered == 1 diff --git a/tests/test_toc.py b/tests/test_toc.py index 71d5504a..4e80f995 100644 --- a/tests/test_toc.py +++ b/tests/test_toc.py @@ -707,6 +707,498 @@ def test_toml_comments_in_fence_between_toc_and_heading(self): assert result["errors"] == [], f"Unexpected errors: {result['errors']}" +# --------------------------------------------------------------------------- +# JIT-retrieval readiness signals (constructorfabric/studio#104) +# --------------------------------------------------------------------------- + +class TestJitRetrievalReadiness: + def test_duplicate_heading_titles_warned(self): + content = ( + "# Title\n\n" + "## Table of Contents\n\n" + "1. [Intro](#intro)\n" + "2. [Intro](#intro-1)\n\n" + "---\n\n" + "## Intro\n\n" + "## Intro\n" + ) + result = validate_toc(content, max_heading_level=2) + assert result["errors"] == [] + codes = [w["code"] for w in result["warnings"]] + assert "toc-heading-duplicate" in codes + dup = [w for w in result["warnings"] if w["code"] == "toc-heading-duplicate"][0] + assert dup["heading_text"] == "Intro" + assert dup["first_seen_line"] == 10 + + def test_no_duplicate_warning_for_unique_headings(self): + content = ( + "# Title\n\n" + "## Table of Contents\n\n" + "1. [A](#a)\n" + "2. [B](#b)\n\n" + "---\n\n" + "## A\n\n" + "## B\n" + ) + result = validate_toc(content, max_heading_level=2) + codes = [w["code"] for w in result["warnings"]] + assert "toc-heading-duplicate" not in codes + + def test_duplicate_detection_is_case_insensitive(self): + """CodeRabbit PR #108: "Section" and "section" render identically + to a reader but compared unequal under a raw dict key.""" + content = ( + "# Title\n\n" + "## Table of Contents\n\n" + "1. [Section](#section)\n" + "2. [section](#section-1)\n\n" + "---\n\n" + "## Section\n\n" + "## section\n" + ) + result = validate_toc(content, max_heading_level=2) + codes = [w["code"] for w in result["warnings"]] + assert "toc-heading-duplicate" in codes + + def test_duplicate_detection_collapses_internal_whitespace(self): + content = ( + "# Title\n\n" + "## Table of Contents\n\n" + "1. [Setup Guide](#setup--guide)\n" + "2. [Setup Guide](#setup-guide)\n\n" + "---\n\n" + "## Setup Guide\n\n" + "## Setup Guide\n" + ) + result = validate_toc(content, max_heading_level=2) + codes = [w["code"] for w in result["warnings"]] + assert "toc-heading-duplicate" in codes + + def test_duplicate_warning_still_shows_the_original_heading_text(self): + """Normalizing the comparison key must not leak into the warning's + display text -- a reader needs to see the heading as written.""" + content = ( + "# Title\n\n" + "## Table of Contents\n\n" + "1. [SECTION](#section)\n" + "2. [section](#section-1)\n\n" + "---\n\n" + "## SECTION\n\n" + "## section\n" + ) + result = validate_toc(content, max_heading_level=2) + dup = [w for w in result["warnings"] if w["code"] == "toc-heading-duplicate"][0] + assert dup["heading_text"] == "section" + + def test_frontmatter_hash_line_is_not_parsed_as_a_heading(self): + """CodeRabbit PR #108: a `#`-prefixed line inside YAML front-matter + (a comment, or a value starting with `#`) must not be mistaken for + a real heading. Diagnostic: the front-matter line's text matches + the one real heading below it -- if front-matter weren't skipped, + it would register as a fake first occurrence and the real heading + would incorrectly warn as its "duplicate".""" + content = ( + "---\n" + "title: Foo\n" + "# Section\n" + "---\n" + "# Title\n\n" + "## Table of Contents\n\n" + "1. [Section](#section)\n\n" + "---\n\n" + "## Section\n" + ) + result = validate_toc(content, max_heading_level=2) + codes = [w["code"] for w in result["warnings"]] + assert "toc-heading-duplicate" not in codes + + def test_depth_jump_warned(self): + content = ( + "# Title\n\n" + "## Table of Contents\n\n" + "1. [A](#a)\n\n" + "---\n\n" + "## A\n\n" + "#### Skipped H3\n" + ) + result = validate_toc(content, max_heading_level=4) + codes = [w["code"] for w in result["warnings"]] + assert "toc-heading-depth-jump" in codes + jump = [w for w in result["warnings"] if w["code"] == "toc-heading-depth-jump"][0] + assert jump["from_level"] == 2 + assert jump["to_level"] == 4 + + def test_no_depth_jump_warning_for_consecutive_levels(self): + content = ( + "# Title\n\n" + "## Table of Contents\n\n" + "1. [A](#a)\n\n" + "---\n\n" + "## A\n\n" + "### A.1\n" + ) + result = validate_toc(content, max_heading_level=3) + codes = [w["code"] for w in result["warnings"]] + assert "toc-heading-depth-jump" not in codes + + def test_shallower_heading_not_a_depth_jump(self): + # Going H3 -> H1 (shallower) must never be flagged; only jumps deeper + # by more than one level are a problem. + content = ( + "# Title\n\n" + "## Table of Contents\n\n" + "1. [A](#a)\n\n" + "---\n\n" + "## A\n\n" + "### A.1\n\n" + "# Back to top level\n" + ) + result = validate_toc(content, max_heading_level=3) + codes = [w["code"] for w in result["warnings"]] + assert "toc-heading-depth-jump" not in codes + + def test_oversized_section_warned(self): + body = "\n".join(f"line {i}" for i in range(400)) + content = ( + "# Title\n\n" + "## Table of Contents\n\n" + "1. [A](#a)\n" + "2. [B](#b)\n\n" + "---\n\n" + "## A\n\n" + f"{body}\n\n" + "## B\n" + ) + result = validate_toc(content, max_heading_level=2, max_section_lines=300) + codes = [w["code"] for w in result["warnings"]] + assert "toc-section-too-long" in codes + long_section = [w for w in result["warnings"] if w["code"] == "toc-section-too-long"][0] + assert long_section["heading_text"] == "A" + assert long_section["section_length"] > 300 + + def test_nan_max_section_lines_falls_back_to_the_default_instead_of_disabling_the_check(self): + """CodeRabbit PR #108: float('nan') > anything is always False, so + an unguarded nan silently disabled the oversized-section check + entirely for a direct library caller (bypassing the CLI's + argparse(type=int) guard).""" + body = "\n".join(f"line {i}" for i in range(400)) + content = ( + "# Title\n\n## Table of Contents\n\n1. [A](#a)\n\n---\n\n## A\n\n" + body + "\n" + ) + result = validate_toc(content, max_heading_level=2, max_section_lines=float("nan")) + codes = [w["code"] for w in result["warnings"]] + assert "toc-section-too-long" in codes + + def test_negative_max_section_lines_falls_back_to_the_default_instead_of_flagging_everything(self): + content = ( + "# Title\n\n## Table of Contents\n\n1. [A](#a)\n\n---\n\n## A\n\nShort content.\n" + ) + result = validate_toc(content, max_heading_level=2, max_section_lines=-1) + codes = [w["code"] for w in result["warnings"]] + assert "toc-section-too-long" not in codes + + def test_section_within_limit_not_warned(self): + content = ( + "# Title\n\n" + "## Table of Contents\n\n" + "1. [A](#a)\n\n" + "---\n\n" + "## A\n\n" + "Short content.\n" + ) + result = validate_toc(content, max_heading_level=2, max_section_lines=300) + codes = [w["code"] for w in result["warnings"]] + assert "toc-section-too-long" not in codes + + def test_last_section_length_measured_to_end_of_file(self): + body = "\n".join(f"line {i}" for i in range(400)) + content = ( + "# Title\n\n" + "## Table of Contents\n\n" + "1. [A](#a)\n\n" + "---\n\n" + f"## A\n\n{body}\n" + ) + result = validate_toc(content, max_heading_level=2, max_section_lines=300) + codes = [w["code"] for w in result["warnings"]] + assert "toc-section-too-long" in codes + + def test_missing_description_warned_above_size_threshold(self): + filler = "\n\n".join(f"Paragraph {i} of filler text." for i in range(60)) + content = ( + "# Title\n\n" + "## Table of Contents\n\n" + "1. [A](#a)\n\n" + "---\n\n" + f"## A\n\n{filler}\n" + ) + assert len(content.split("\n")) >= 100 + result = validate_toc(content, max_heading_level=2) + codes = [w["code"] for w in result["warnings"]] + assert "toc-missing-description" in codes + + def test_missing_description_not_warned_below_size_threshold(self): + content = ( + "# Title\n\n" + "## Table of Contents\n\n" + "1. [A](#a)\n\n" + "---\n\n" + "## A\n\nShort.\n" + ) + result = validate_toc(content, max_heading_level=2) + codes = [w["code"] for w in result["warnings"]] + assert "toc-missing-description" not in codes + + def test_frontmatter_present_suppresses_missing_description(self): + filler = "\n\n".join(f"Paragraph {i} of filler text." for i in range(60)) + content = ( + "---\n" + "description: A test document.\n" + "---\n\n" + "# Title\n\n" + "## Table of Contents\n\n" + "1. [A](#a)\n\n" + "---\n\n" + f"## A\n\n{filler}\n" + ) + result = validate_toc(content, max_heading_level=2) + codes = [w["code"] for w in result["warnings"]] + assert "toc-missing-description" not in codes + + def test_frontmatter_without_description_field_still_warns(self): + """CodeRabbit PR #108: frontmatter existing is not the same as a + description existing -- a block with only unrelated fields (e.g. + title) must still warn, not be silently accepted as satisfying the + check its own name promises.""" + filler = "\n\n".join(f"Paragraph {i} of filler text." for i in range(60)) + content = ( + "---\n" + "title: A test document.\n" + "---\n\n" + "# Title\n\n" + "## Table of Contents\n\n" + "1. [A](#a)\n\n" + "---\n\n" + f"## A\n\n{filler}\n" + ) + result = validate_toc(content, max_heading_level=2) + codes = [w["code"] for w in result["warnings"]] + assert "toc-missing-description" in codes + + def test_comment_only_description_value_still_warns(self): + """CodeRabbit PR #109: `description: # TODO` matched the old regex + (`#` is non-whitespace) but is a YAML comment, not a value -- the + field is exactly as absent as if it weren't there at all.""" + filler = "\n\n".join(f"Paragraph {i} of filler text." for i in range(60)) + content = ( + "---\n" + "description: # TODO write this\n" + "---\n\n" + "# Title\n\n" + "## Table of Contents\n\n" + "1. [A](#a)\n\n" + "---\n\n" + f"## A\n\n{filler}\n" + ) + result = validate_toc(content, max_heading_level=2) + codes = [w["code"] for w in result["warnings"]] + assert "toc-missing-description" in codes + + def test_empty_quoted_description_value_still_warns(self): + """CodeRabbit PR #109: `description: ""` matched the old regex (the + opening quote is non-whitespace) but carries no actual text.""" + filler = "\n\n".join(f"Paragraph {i} of filler text." for i in range(60)) + content = ( + "---\n" + 'description: ""\n' + "---\n\n" + "# Title\n\n" + "## Table of Contents\n\n" + "1. [A](#a)\n\n" + "---\n\n" + f"## A\n\n{filler}\n" + ) + result = validate_toc(content, max_heading_level=2) + codes = [w["code"] for w in result["warnings"]] + assert "toc-missing-description" in codes + + def test_empty_single_quoted_description_value_still_warns(self): + """CodeRabbit PR #108 (round 2): single-quoted `description: ''` is + the same empty-scalar case as the double-quoted form and must warn + too -- not just the double-quoted variant.""" + filler = "\n\n".join(f"Paragraph {i} of filler text." for i in range(60)) + content = ( + "---\n" + "description: ''\n" + "---\n\n" + "# Title\n\n" + "## Table of Contents\n\n" + "1. [A](#a)\n\n" + "---\n\n" + f"## A\n\n{filler}\n" + ) + result = validate_toc(content, max_heading_level=2) + codes = [w["code"] for w in result["warnings"]] + assert "toc-missing-description" in codes + + def test_real_description_after_regex_tightening_still_suppresses_warning(self): + """Confirms the stricter check didn't overcorrect into rejecting a + genuinely populated, quoted description.""" + filler = "\n\n".join(f"Paragraph {i} of filler text." for i in range(60)) + content = ( + "---\n" + 'description: "A real, non-empty description."\n' + "---\n\n" + "# Title\n\n" + "## Table of Contents\n\n" + "1. [A](#a)\n\n" + "---\n\n" + f"## A\n\n{filler}\n" + ) + result = validate_toc(content, max_heading_level=2) + codes = [w["code"] for w in result["warnings"]] + assert "toc-missing-description" not in codes + + def test_empty_block_scalar_description_still_warns(self): + """CodeRabbit PR #109 (second round): `description: |` is a YAML + block-scalar marker -- the real content (if any) belongs on + indented lines below it, not on the marker line itself. With + nothing indented beneath it, this frontmatter has no real + description, immediately followed by the closing `---`.""" + filler = "\n\n".join(f"Paragraph {i} of filler text." for i in range(60)) + content = ( + "---\n" + "description: |\n" + "---\n\n" + "# Title\n\n" + "## Table of Contents\n\n" + "1. [A](#a)\n\n" + "---\n\n" + f"## A\n\n{filler}\n" + ) + result = validate_toc(content, max_heading_level=2) + codes = [w["code"] for w in result["warnings"]] + assert "toc-missing-description" in codes + + def test_populated_block_scalar_description_suppresses_warning(self): + """The other side of the block-scalar fix: real indented content + under `description: |` must still count as a real description.""" + filler = "\n\n".join(f"Paragraph {i} of filler text." for i in range(60)) + content = ( + "---\n" + "description: |\n" + " A real, multi-line\n" + " block-scalar description.\n" + "---\n\n" + "# Title\n\n" + "## Table of Contents\n\n" + "1. [A](#a)\n\n" + "---\n\n" + f"## A\n\n{filler}\n" + ) + result = validate_toc(content, max_heading_level=2) + codes = [w["code"] for w in result["warnings"]] + assert "toc-missing-description" not in codes + + def test_block_scalar_with_leading_blank_line_before_content_still_counts(self): + """A blank line immediately under the block-scalar marker (before + the real indented content) must be skipped, not mistaken for "no + content".""" + filler = "\n\n".join(f"Paragraph {i} of filler text." for i in range(60)) + content = ( + "---\n" + "description: |\n" + "\n" + " Real content after a leading blank line.\n" + "---\n\n" + "# Title\n\n" + "## Table of Contents\n\n" + "1. [A](#a)\n\n" + "---\n\n" + f"## A\n\n{filler}\n" + ) + result = validate_toc(content, max_heading_level=2) + codes = [w["code"] for w in result["warnings"]] + assert "toc-missing-description" not in codes + + def test_folded_block_scalar_marker_variant_is_recognized(self): + """`>` (folded) and modifiers like `|-`/`>+` are all valid YAML + block-scalar indicators, not just the bare `|`.""" + filler = "\n\n".join(f"Paragraph {i} of filler text." for i in range(60)) + content = ( + "---\n" + "description: >-\n" + "---\n\n" + "# Title\n\n" + "## Table of Contents\n\n" + "1. [A](#a)\n\n" + "---\n\n" + f"## A\n\n{filler}\n" + ) + result = validate_toc(content, max_heading_level=2) + codes = [w["code"] for w in result["warnings"]] + assert "toc-missing-description" in codes + + def test_jit_readiness_warnings_are_never_errors(self): + # All four signals are additive warnings; they must never appear + # in `errors`, regardless of how badly a document scores. (This + # fixture also trips an unrelated, pre-existing TOC-completeness + # error since "Skipped H2/H3" isn't listed in the TOC — that error + # is expected and irrelevant to what's being asserted here.) + filler = "\n\n".join(f"Paragraph {i} of filler text." for i in range(60)) + content = ( + "# Title\n\n" + "## Table of Contents\n\n" + "1. [Intro](#intro)\n" + "2. [Intro](#intro-1)\n\n" + "---\n\n" + "## Intro\n\n" + f"{filler}\n\n" + "#### Skipped H2/H3\n\n" + "## Intro\n" + ) + result = validate_toc(content, max_heading_level=4) + jit_codes = { + "toc-heading-duplicate", + "toc-heading-depth-jump", + "toc-section-too-long", + "toc-missing-description", + } + error_codes = {e["code"] for e in result["errors"]} + assert not (error_codes & jit_codes), f"JIT-readiness code leaked into errors: {error_codes}" + # This fixture triggers duplicate + depth-jump + missing-description, + # but not section-too-long (its filler is under the 300-line default). + warning_codes = {w["code"] for w in result["warnings"]} + assert {"toc-heading-duplicate", "toc-heading-depth-jump", "toc-missing-description"}.issubset( + warning_codes + ) + + def test_readiness_signals_see_headings_deeper_than_max_heading_level(self): + """CodeRabbit PR #108: readiness checks must see *every* heading + level, independent of max_heading_level (the CLI's own default is + 3). A duplicate/depth-jump/oversized-section problem below that + level must still be caught -- filtering by the TOC's level cap here + would silently hide real structural problems in H4-H6 content, as + it did against a real PDF-converted document during development.""" + body = "\n".join(f"line {i}" for i in range(400)) + content = ( + "# Title\n\n" + "## Table of Contents\n\n" + "1. [A](#a)\n\n" + "---\n\n" + "## A\n\n" + "#### Deep\n\n" + f"{body}\n\n" + "#### Deep\n" + ) + # max_heading_level=2: TOC completeness only cares about H1/H2, but + # the two duplicate/oversized H4 "Deep" headings must still surface. + result = validate_toc(content, max_heading_level=2, max_section_lines=300) + warning_codes = {w["code"] for w in result["warnings"]} + assert "toc-heading-duplicate" in warning_codes + assert "toc-section-too-long" in warning_codes + + # --------------------------------------------------------------------------- # cmd_validate_toc (integration) # --------------------------------------------------------------------------- @@ -756,6 +1248,34 @@ def test_multiple_files(self, tmp_path: Path, capsys): assert out["files_validated"] == 2 assert out["error_count"] == 1 + def test_a_read_failure_on_one_file_does_not_abort_the_batch(self, tmp_path: Path, capsys): + """CodeRabbit PR #109: an unhandled read failure on one file used to + raise out of the per-file loop, discarding results already + collected for files validated earlier in the same invocation and + never reaching the remaining files. A binary/non-UTF-8 file in the + middle of a batch must be recorded as its own ERROR result, and the + batch must still validate the file(s) after it.""" + good = tmp_path / "good.md" + good.write_text( + "# T\n\n## Table of Contents\n\n1. [A](#a)\n\n---\n\n## A\n", + encoding="utf-8", + ) + binary = tmp_path / "binary.md" + binary.write_bytes(b"\xff\xfe\x00\x01garbage") + good2 = tmp_path / "good2.md" + good2.write_text( + "# T\n\n## Table of Contents\n\n1. [B](#b)\n\n---\n\n## B\n", + encoding="utf-8", + ) + rc = cmd_validate_toc(["--max-level", "2", str(good), str(binary), str(good2)]) + assert rc == 2 + out = json.loads(capsys.readouterr().out) + assert out["files_validated"] == 3 + by_file = {r["file"]: r for r in out["results"]} + assert by_file[str(good)]["status"] == "PASS" + assert by_file[str(binary)]["status"] == "ERROR" + assert by_file[str(good2)]["status"] == "PASS" + def test_verbose_flag(self, tmp_path: Path, capsys): f = tmp_path / "doc.md" f.write_text( @@ -786,6 +1306,66 @@ def test_warn_stale_toc(self, tmp_path: Path, capsys): assert out["status"] == "WARN" assert out["warning_count"] >= 1 + def test_warn_only_file_prints_warnings_in_human_output(self, tmp_path: Path, capsys, monkeypatch): + """CodeRabbit PR #108: a WARN-only file's human-mode output used to + print just "path: WARN" with no indication of what's wrong -- the + FAIL branch already iterated warnings, but the WARN branch (the + default `else`) never did, making this PR's own headline feature + (JIT-readiness warnings) invisible outside --json.""" + from studio.utils.ui import is_json_mode, set_json_mode + + f = tmp_path / "stale.md" + f.write_text( + "# T\n\n" + "## Table of Contents\n\n" + "1. [B](#b)\n" + "2. [A](#a)\n\n" + "---\n\n" + "## A\n\n" + "## B\n", + encoding="utf-8", + ) + orig = is_json_mode() + set_json_mode(False) + try: + rc = cmd_validate_toc(["--max-level", "2", str(f)]) + finally: + set_json_mode(orig) + assert rc == 0 + out = capsys.readouterr().out + assert "warning(s)" in out + assert "⚠" in out + + def test_all_four_jit_warnings_zero_errors_exits_clean_via_cli(self, tmp_path: Path, capsys): + """CodeRabbit PR #108: prove the warn-only guarantee end to end + through cmd_validate_toc, not just validate_toc() directly -- a + document tripping JIT-readiness codes with an otherwise complete + TOC must still return rc == 0 and status WARN, zero errors. + --max-level 2 keeps the deeper H4 "Sub" heading (which trips the + depth-jump and section-too-long checks -- those see every heading + regardless of max_heading_level, per the readiness checks' own + design) out of TOC-completeness scope, so it needs no TOC entry.""" + filler = "\n\n".join(f"Paragraph {i} of filler text." for i in range(60)) + content = ( + "# Title\n\n" + "## Table of Contents\n\n" + "1. [Intro](#intro)\n" + "2. [Intro](#intro-1)\n\n" + "---\n\n" + "## Intro\n\n" + f"{filler}\n\n" + "#### Sub\n\n" + "## Intro\n" + ) + f = tmp_path / "warnonly.md" + f.write_text(content, encoding="utf-8") + rc = cmd_validate_toc([str(f), "--max-level", "2"]) + out = json.loads(capsys.readouterr().out) + assert out["status"] == "WARN" + assert out["warning_count"] >= 3 + assert out["error_count"] == 0 + assert rc == 0 + class TestCmdTocValidation: """cmd_toc post-validation and error-status paths.""" diff --git a/vulture_whitelist.py b/vulture_whitelist.py index 056beb84..1c28afcb 100644 --- a/vulture_whitelist.py +++ b/vulture_whitelist.py @@ -12,6 +12,7 @@ from studio.commands.kit import _read_conf_version from studio.commands.resolve_vars import assemble_component from studio.utils.context import LoadedKit +from studio.utils.doc_index import annotate_section_summary, diff_stale_sections from studio.utils.eval_harness import ReferencePresenceScorer, Scenario, ScorerKind, run_suite from studio.utils.eval_judge import Gold from studio.utils.manifest import ManifestLayerState @@ -39,6 +40,13 @@ _ = Gold.rules_assessed # part of the gold format; consumed by per-rule judge scoring (future) INCLUDE_ERROR = ManifestLayerState.INCLUDE_ERROR # valid enum value for future use +# doc-index summary annotation and section-level staleness diff: called by +# a future partial-rebuild caller (an LLM re-summarizing only changed +# sections), not yet reached from production paths. Exercised by tests. See +# skills/studio/scripts/studio/utils/doc_index.py. +annotate_section_summary # noqa: B018 +diff_stale_sections # noqa: B018 + # cfs map module — symbols retained for layout/configuration completeness. from studio.commands.map.layout import MAX_ROW_W # noqa: E402 from studio.commands.map.categorize import OverrideCategory # noqa: E402 @@ -72,3 +80,41 @@ record_escalation # noqa: B018 record_invocation # noqa: B018 summarize # noqa: B018 + +# eval_semantic public API — the semantic-coverage engine. Library + tests only for now; +# the `cfs` surface and coverage-report integration are the follow-up, so these are not yet +# reached from a production path. Exercised by tests. +# See skills/studio/scripts/studio/utils/eval_semantic.py. +from studio.utils.eval_semantic import ( # noqa: E402 + reference_stub_judge, + assess, + resolve_requirement, + load_gold, + calibrate, + SemanticFinding, + SemanticGap, + SemanticReport, + SemanticCalibration, +) + +reference_stub_judge # noqa: B018 +assess # noqa: B018 +resolve_requirement # noqa: B018 +load_gold # noqa: B018 +calibrate # noqa: B018 +SemanticFinding.evidence_ok # noqa: B018 +SemanticFinding.forced # noqa: B018 +SemanticGap.block_id # noqa: B018 +SemanticGap.path # noqa: B018 +SemanticGap.start_line # noqa: B018 +SemanticGap.reason # noqa: B018 +SemanticReport.presumed_covered # noqa: B018 +SemanticReport.skipped_excluded # noqa: B018 +SemanticReport.schema_version # noqa: B018 +SemanticCalibration.accuracy # noqa: B018 +SemanticCalibration.consistency # noqa: B018 +SemanticCalibration.runs_per_scenario # noqa: B018 +SemanticCalibration.per_case # noqa: B018 +SemanticCalibration.excluded # noqa: B018 +SemanticCalibration.judge # noqa: B018 +SemanticCalibration.schema_version # noqa: B018