feat(cascade): heading-nav, two-tier retrieval routing, and large-read gate

.gitignore

@@ -569,6 +569,16 @@ coverage.xml
 **/.cache/decisions.jsonl.1
 **/.cache/decisions.jsonl.lock
 
+# doc-index cache (doc_index.py) — per-project structural index for JIT
+# retrieval, read-once-per-file. Rebuilds automatically on content change.
+**/.cache/doc-index/
+
+# OKF bundle cache (okf.py) — local, regenerable concept files + index for
+# JIT retrieval's semantic fallback. Never committed: a fresh clone rebuilds
+# it (at real token cost, via an external LLM caller) rather than relying on
+# a stale copy shipped in version control.
+**/.cache/okf/
+
 # Superpowers brainstorming/planning specs (local-only)
 docs/superpowers/
 

architecture/features/core-infra.md

@@ -596,6 +596,7 @@ Enables users to install Studio globally, initialize it in any project with sens
 - [x] - `p1` - `file_action`: file-change icon printer (created/updated/unchanged/etc.) to stderr - `inst-ui-file-action`
 - [x] - `p1` - `result` JSON branch: serialize result dict as JSON to stdout in `--json` mode - `inst-ui-result-json`
 - [x] - `p1` - `result` human branch: invoke `human_fn` or generic status/message fallback to stderr - `inst-ui-result-human`
+- [x] - `p1` - `require_existing_file`: resolve a CLI file-path argument, emitting the standard "File not found" ERROR result and returning `None` when it doesn't exist -- shared by every single-file-argument command - `inst-ui-require-existing-file`
 - [x] - `p1` - Create a temporary stderr-bound logger handler with plain-message formatting for UI diagnostics - `inst-ui-stderr-handler`
 - [x] - `p1` - Emit one plain-text stderr message through the dedicated helper, allowing a logger-backed implementation internally, then close the handler - `inst-ui-stderr-emit`
 - [x] - `p1` - `relpath`: convert absolute path to cwd-relative path with fallback - `inst-ui-relpath`
@@ -718,7 +719,13 @@ Enables users to install Studio globally, initialize it in any project with sens
 4. [x] - `p1` - Redact `$HOME` to `~` recursively so no username is recorded - `inst-log-redact`
 5. [x] - `p1` - Append one schema-versioned event (run_id, decision_id, event, command, payload); never raise into the caller; rotate by size; show a one-time notice - `inst-log-record`
 6. [x] - `p1` - Typed record helpers — routing, dispatch, validation, review, escalation, and command invocation (exit code, duration, arg-shape) — that call the writer - `inst-log-api`
+   - [x] - `p1` - `record_read`: log one read-and-answer event (method, target, lines, tokens, source) in the one shared schema every JIT-retrieval method's real cost is measured in - `inst-log-read-wrapper`
 7. [x] - `p1` - Read events back oldest-first (skipping unparseable lines) and summarise counts by event and run - `inst-log-read`
+   - [x] - `p1` - `summarize_reads`: aggregate logged `"read"` events into a per-method token/line/count table - `inst-log-summarize-reads`
+
+**Supporting**:
+- [x] - `p1` - `cfs usage-report` CLI wrapper: aggregate `summarize()` and `summarize_reads()` into one payload - `inst-usage-report-cmd`
+- [x] - `p1` - Human-friendly formatter for `cfs usage-report` output - `inst-usage-report-cmd-format`
 
 ## 4. States (CDSL)
 

skills/studio/scripts/studio/cli.py

@@ -134,6 +134,34 @@ def _cmd_eval(argv: List[str]) -> int:
     from .commands.eval import cmd_eval
     return cmd_eval(argv)
 
+def _cmd_doc_index(argv: List[str]) -> int:
+    from .commands.doc_index import cmd_doc_index
+    return cmd_doc_index(argv)
+
+def _cmd_tfidf_score(argv: List[str]) -> int:
+    from .commands.tfidf import cmd_tfidf_score
+    return cmd_tfidf_score(argv)
+
+def _cmd_okf_status(argv: List[str]) -> int:
+    from .commands.okf import cmd_okf_status
+    return cmd_okf_status(argv)
+
+def _cmd_heading_nav(argv: List[str]) -> int:
+    from .commands.heading_nav import cmd_heading_nav
+    return cmd_heading_nav(argv)
+
+def _cmd_retrieve(argv: List[str]) -> int:
+    from .commands.cascade import cmd_retrieve
+    return cmd_retrieve(argv)
+
+def _cmd_read_gate(argv: List[str]) -> int:
+    from .commands.read_gate import cmd_read_gate
+    return cmd_read_gate(argv)
+
+def _cmd_usage_report(argv: List[str]) -> int:
+    from .commands.usage_report import cmd_usage_report
+    return cmd_usage_report(argv)
+
 # =============================================================================
 # ADAPTER COMMAND
 # =============================================================================
@@ -216,6 +244,13 @@ def _cmd_map(argv: List[str]) -> int:
     "resolve-vars": "Resolve template variables to absolute paths",
     "toc": "Generate/update Table of Contents",
     "chunk-input": "Chunk oversized workflow input into line-bounded Markdown files",
+    "doc-index": "Build/reuse a cached heading index for a Markdown file (read once, not per query)",
+    "tfidf-score": "Rank a Markdown file's retrieval sections against a query via TF-IDF",
+    "okf-status": "Report an OKF bundle's state for a Markdown file (missing/stale/current per section)",
+    "heading-nav": "Find a Markdown file's retrieval sections containing a query's literal text",
+    "retrieve": "Route a query through the two-tier JIT-retrieval cascade (heading-nav + TF-IDF, OKF vs. baseline)",
+    "read-gate": "Check whether a Markdown file's line count crosses the large-read confirmation threshold",
+    "usage-report": "Aggregate the local decision log's read events into a per-method token table",
     "pdsl": "Validate PDSL prompt blocks",
     "workspace-init": "Initialize multi-repo workspace",
     "workspace-add": "Add a source to workspace config",
@@ -232,7 +267,10 @@ def _cmd_map(argv: List[str]) -> int:
     ("Validation", ["validate", "validate-kits", "validate-toc", "spec-coverage", "check-language"]),
     ("Search & Navigation", ["list-ids", "list-id-kinds", "get-content", "where-defined", "where-used"]),
     ("Kit Management", ["kit"]),
-    ("Utility", ["toc", "chunk-input", "pdsl"]),
+    ("Utility", [
+        "toc", "chunk-input", "doc-index", "tfidf-score", "okf-status",
+        "heading-nav", "retrieve", "read-gate", "usage-report", "pdsl",
+    ]),
     ("Workspace", ["workspace-init", "workspace-add", "workspace-info", "workspace-sync"]),
     ("Delegation", ["delegate"]),
     ("Diagnostics", ["doctor"]),
@@ -263,6 +301,13 @@ def _cmd_map(argv: List[str]) -> int:
     "validate-toc": "_cmd_validate_toc",
     "spec-coverage": "_cmd_spec_coverage",
     "chunk-input": "_cmd_chunk_input",
+    "doc-index": "_cmd_doc_index",
+    "tfidf-score": "_cmd_tfidf_score",
+    "okf-status": "_cmd_okf_status",
+    "heading-nav": "_cmd_heading_nav",
+    "retrieve": "_cmd_retrieve",
+    "read-gate": "_cmd_read_gate",
+    "usage-report": "_cmd_usage_report",
     "workspace-init": "_cmd_workspace_init",
     "workspace-add": "_cmd_workspace_add",
     "workspace-info": "_cmd_workspace_info",
@@ -296,6 +341,13 @@ def _cmd_map(argv: List[str]) -> int:
     _cmd_validate_toc,
     _cmd_spec_coverage,
     _cmd_chunk_input,
+    _cmd_doc_index,
+    _cmd_tfidf_score,
+    _cmd_okf_status,
+    _cmd_heading_nav,
+    _cmd_retrieve,
+    _cmd_read_gate,
+    _cmd_usage_report,
     _cmd_workspace_init,
     _cmd_workspace_add,
     _cmd_workspace_info,

skills/studio/scripts/studio/commands/cascade.py

@@ -0,0 +1,69 @@
+"""Studio retrieve command — route a query against a Markdown file through
+the two-tier JIT-retrieval cascade (heading-nav + TF-IDF, falling back to an
+OKF-vs-baseline choice), and report the routing decision.
+
+Thin CLI wrapper around ``studio.utils.cascade``.
+
+@cpt-flow:cpt-studio-flow-traceability-validation-validate:p1
+"""
+
+import argparse
+from typing import List
+
+from ..utils.cascade import route_query
+from ..utils.ui import ui
+
+
+# @cpt-begin:cpt-studio-algo-traceability-validation-cascade:p1:inst-cascade-cmd
+def cmd_retrieve(argv: List[str]) -> int:
+    """Route a query against a Markdown file through the JIT-retrieval cascade."""
+    p = argparse.ArgumentParser(
+        prog="cfs retrieve",
+        description="Route a query through the two-tier JIT-retrieval cascade and report the decision.",
+    )
+    p.add_argument("file", help="Markdown file path")
+    p.add_argument("query", help="Query text")
+    p.add_argument(
+        "--margin-threshold", type=float, default=None,
+        help="Enable a numeric TF-IDF margin cutoff for a large-margin Tier 1 resolution "
+        "(default: disabled -- only an unambiguous score counts)",
+    )
+    p.add_argument(
+        "--expected-future-queries", type=int, default=None,
+        help="Expected future query volume against this document, for the OKF-vs-baseline break-even math",
+    )
+    args = p.parse_args(argv)
+
+    filepath = ui.require_existing_file(args.file)
+    if filepath is None:
+        return 2
+
+    result = route_query(
+        filepath, args.query,
+        margin_threshold=args.margin_threshold,
+        expected_future_queries=args.expected_future_queries,
+    )
+
+    output = {"file": str(filepath), **result}
+    ui.result(output, human_fn=_human_retrieve)
+    return 0
+# @cpt-end:cpt-studio-algo-traceability-validation-cascade:p1:inst-cascade-cmd
+
+
+# @cpt-begin:cpt-studio-algo-traceability-validation-cascade:p1:inst-cascade-cmd-format
+def _human_retrieve(data: dict) -> None:
+    ui.header("Retrieve")
+    ui.substep(f"query: {data['query']!r}")
+    ui.substep(f"tier: {data['tier']} ({data['reason']})")
+    for c in data["candidates"]:
+        ui.substep(f"  [{c['line_start']}-{c['line_end']}] {c['heading']}")
+    if "tier2" in data:
+        tier2 = data["tier2"]
+        ui.substep(f"tier 2 recommendation: {tier2['recommendation']} ({tier2['reason']})")
+        if tier2.get("okf_needs_rebuild"):
+            ui.substep("  OKF bundle exists but is stale/missing for this candidate -- needs a rebuild")
+    if "read_gate" in data and data["read_gate"]["needs_confirmation"]:
+        gate = data["read_gate"]
+        ui.substep(f"read gate: needs confirmation ({gate['total_lines']} lines > {gate['threshold']})")
+    ui.blank()
+# @cpt-end:cpt-studio-algo-traceability-validation-cascade:p1:inst-cascade-cmd-format

skills/studio/scripts/studio/commands/doc_index.py

@@ -0,0 +1,74 @@
+"""Studio doc-index command — build/reuse a cached structural index for a
+Markdown file, so heading-based JIT retrieval reads a file's structure once,
+not once per query.
+
+Thin CLI wrapper around ``studio.utils.doc_index``.
+
+@cpt-flow:cpt-studio-flow-traceability-validation-validate:p1
+"""
+
+import argparse
+from typing import List
+
+from ..utils.doc_index import get_or_build_doc_index
+from ..utils.ui import ui
+
+
+# @cpt-begin:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-cmd
+def cmd_doc_index(argv: List[str]) -> int:
+    """Build (or reuse the cached) structural index for a Markdown file."""
+    p = argparse.ArgumentParser(
+        prog="cfs doc-index",
+        description=(
+            "Build or reuse a cached heading/section index for a Markdown file, "
+            "so navigation reads the file's structure once, not once per query."
+        ),
+    )
+    p.add_argument("file", help="Markdown file path")
+    p.add_argument(
+        "--rebuild",
+        action="store_true",
+        help="Force a fresh build even if a valid cached index exists",
+    )
+    args = p.parse_args(argv)
+
+    filepath = ui.require_existing_file(args.file)
+    if filepath is None:
+        return 2
+
+    index = get_or_build_doc_index(filepath, force_rebuild=args.rebuild)
+
+    output = {
+        "file": str(filepath),
+        "cache_hit": index["cache_hit"],
+        "total_lines": index["total_lines"],
+        "section_count": len(index["sections"]),
+        "sections": index["sections"],
+        "section_level": index["section_level"],
+        "retrieval_section_count": len(index["retrieval_sections"]),
+        "retrieval_sections": index["retrieval_sections"],
+    }
+    ui.result(output, human_fn=_human_doc_index)
+    return 0
+# @cpt-end:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-cmd
+
+
+# @cpt-begin:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-cmd-format
+def _human_doc_index(data: dict) -> None:
+    ui.header("Doc Index")
+    hit = "cache hit — reused existing index" if data["cache_hit"] else "cache miss — built fresh index"
+    ui.substep(hit)
+    ui.substep(f"{data['section_count']} section(s), {data['total_lines']} total lines")
+    for s in data["sections"]:
+        summary = f" — {s['summary']}" if s.get("summary") else ""
+        ui.substep(f"  H{s['level']} [{s['line_start']}-{s['line_end']}] {s['heading']}{summary}")
+    ui.blank()
+
+    level = data["section_level"]
+    ui.step(f"Retrieval sections (level {level}, {data['retrieval_section_count']} section(s))" if level is not None
+             else "Retrieval sections (no headings — none inferred)")
+    for s in data["retrieval_sections"]:
+        summary = f" — {s['summary']}" if s.get("summary") else ""
+        ui.substep(f"  [{s['line_start']}-{s['line_end']}] {s['heading']} ({s['hash'][:12]}){summary}")
+    ui.blank()
+# @cpt-end:cpt-studio-algo-traceability-validation-doc-index:p1:inst-doc-index-cmd-format

skills/studio/scripts/studio/commands/heading_nav.py

@@ -0,0 +1,56 @@
+"""Studio heading-nav command — grep a Markdown file's retrieval sections
+for a query's literal text, for inspecting/benchmarking the JIT-retrieval
+mechanical gate independent of any cascade routing logic built on top of it.
+
+Thin CLI wrapper around ``studio.utils.heading_nav``.
+
+@cpt-flow:cpt-studio-flow-traceability-validation-validate:p1
+"""
+
+import argparse
+from typing import List
+
+from ..utils.heading_nav import find_sections
+from ..utils.ui import ui
+
+
+# @cpt-begin:cpt-studio-algo-traceability-validation-heading-nav:p1:inst-heading-nav-cmd
+def cmd_heading_nav(argv: List[str]) -> int:
+    """Find a Markdown file's retrieval sections containing a query literally."""
+    p = argparse.ArgumentParser(
+        prog="cfs heading-nav",
+        description="Find a Markdown file's retrieval sections containing a query's literal text.",
+    )
+    p.add_argument("file", help="Markdown file path")
+    p.add_argument("query", help="Query text to search for, literally")
+    args = p.parse_args(argv)
+
+    filepath = ui.require_existing_file(args.file)
+    if filepath is None:
+        return 2
+
+    result = find_sections(filepath, args.query)
+
+    output = {
+        "file": str(filepath),
+        "query": args.query,
+        "matches": result["matches"],
+        "first_match": result["first_match"],
+    }
+    ui.result(output, human_fn=_human_heading_nav)
+    return 0
+# @cpt-end:cpt-studio-algo-traceability-validation-heading-nav:p1:inst-heading-nav-cmd
+
+
+# @cpt-begin:cpt-studio-algo-traceability-validation-heading-nav:p1:inst-heading-nav-cmd-format
+def _human_heading_nav(data: dict) -> None:
+    ui.header("Heading-Nav Search")
+    ui.substep(f"query: {data['query']!r}")
+    if not data["matches"]:
+        ui.substep("(no matches -- this method has no semantic fallback)")
+        ui.blank()
+        return
+    for entry in data["matches"]:
+        ui.substep(f"  {entry['hit_count']}x  [{entry['line_start']}-{entry['line_end']}] {entry['heading']}")
+    ui.blank()
+# @cpt-end:cpt-studio-algo-traceability-validation-heading-nav:p1:inst-heading-nav-cmd-format

skills/studio/scripts/studio/commands/okf.py

@@ -0,0 +1,59 @@
+"""Studio okf-status command — report an OKF bundle's state for a Markdown
+file: which concept files exist, are stale, or are missing entirely,
+relative to the document's current retrieval sections.
+
+Read-only. Writing a concept file is an external caller's job (an agent
+that has actually produced a summary) via
+``studio.utils.okf.write_concept_file`` -- this command never invokes an
+LLM itself.
+
+Thin CLI wrapper around ``studio.utils.okf``.
+
+@cpt-flow:cpt-studio-flow-traceability-validation-validate:p1
+"""
+
+import argparse
+from typing import List
+
+from ..utils.okf import get_okf_status
+from ..utils.ui import ui
+
+
+# @cpt-begin:cpt-studio-algo-traceability-validation-okf:p1:inst-okf-cmd
+def cmd_okf_status(argv: List[str]) -> int:
+    """Report an OKF bundle's state for a Markdown file."""
+    p = argparse.ArgumentParser(
+        prog="cfs okf-status",
+        description="Report which OKF concept files exist, are stale, or are missing for a Markdown file.",
+    )
+    p.add_argument("file", help="Markdown file path")
+    args = p.parse_args(argv)
+
+    filepath = ui.require_existing_file(args.file)
+    if filepath is None:
+        return 2
+
+    status = get_okf_status(filepath)
+    output = {"file": str(filepath), **status}
+    ui.result(output, human_fn=_human_okf_status)
+    return 0
+# @cpt-end:cpt-studio-algo-traceability-validation-okf:p1:inst-okf-cmd
+
+
+# @cpt-begin:cpt-studio-algo-traceability-validation-okf:p1:inst-okf-cmd-format
+def _human_okf_status(data: dict) -> None:
+    ui.header("OKF Status")
+    if not data["available"]:
+        ui.substep("no Studio directory found -- OKF is unavailable for this file")
+        ui.blank()
+        return
+    ui.substep(f"bundle: {data['bundle_dir']}")
+    counts: dict = {}
+    for entry in data["entries"]:
+        counts[entry["status"]] = counts.get(entry["status"], 0) + 1
+    summary = ", ".join(f"{count} {status}" for status, count in sorted(counts.items())) or "no sections"
+    ui.substep(summary)
+    for entry in data["entries"]:
+        ui.substep(f"  [{entry['status']:>7}] [{entry['line_start']}-{entry['line_end']}] {entry['heading']}")
+    ui.blank()
+# @cpt-end:cpt-studio-algo-traceability-validation-okf:p1:inst-okf-cmd-format