diff --git a/bun.lock b/bun.lock index bb5133d..bf0006b 100644 --- a/bun.lock +++ b/bun.lock @@ -5,7 +5,7 @@ "": { "name": "semath", "dependencies": { - "wasmtex": "github:corca-ai/wasmtex#947d2c986ad4bb8b810153ab3ea17f119079e086", + "wasmtex": "github:corca-ai/wasmtex#0d34708807528f15f1898839acfe89c037b404ca", }, "devDependencies": { "@types/bun": "^1.3.14", @@ -98,6 +98,6 @@ "undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], - "wasmtex": ["wasmtex@github:corca-ai/wasmtex#947d2c9", { "peerDependencies": { "monaco-editor": "^0.55.1", "pdf-lib": "^1.17.1", "pdfjs-dist": "^5.4.624" }, "optionalPeers": ["pdf-lib"] }, "corca-ai-wasmtex-947d2c9", "sha512-rDoZsGg92JqEQIqQdtOgtoWIfyICspOrdqdaiH7gSRKK9pnwNMrt06Y7fGqWqkzNQl12VU6NWOn/3Ogdp5uqcA=="], + "wasmtex": ["wasmtex@github:corca-ai/wasmtex#0d34708", { "peerDependencies": { "monaco-editor": "^0.55.1", "pdf-lib": "^1.17.1", "pdfjs-dist": "^5.4.624" }, "optionalPeers": ["pdf-lib"] }, "corca-ai-wasmtex-0d34708", "sha512-dvkt7LqnRwrmA6h+Ciz1hO4BU92u722/KrIjvnRJWR7Terxb3Am9hQFgqal74ufjjyiravKr+ch1RuZvHVVzSQ=="], } } diff --git a/crates/semath-core/src/engine.rs b/crates/semath-core/src/engine.rs index 7d3eea2..b313dbb 100644 --- a/crates/semath-core/src/engine.rs +++ b/crates/semath-core/src/engine.rs @@ -1052,9 +1052,7 @@ impl SemathEngine { match change { ProjectChange::Upsert { document } => { let file_id = document.file_id.clone(); - let accept = self.index.documents.get(&file_id).is_none_or(|current| { - document.document_version > current.document.document_version - }); + let accept = self.accepts_upsert(&document); if accept { let previous_order = self.index.order_document(&file_id); if self.can_reuse_analysis(&document) { @@ -1346,6 +1344,21 @@ impl SemathEngine { && appended_comments_only(¤t.document.content, &next.content) } + fn accepts_upsert(&self, next: &ProjectDocument) -> bool { + let Some(current) = self.index.documents.get(&next.file_id) else { + return true; + }; + if next.document_version > current.document.document_version { + return true; + } + next.document_version == current.document.document_version + && next.content == current.document.content + && next.path == current.document.path + && next.language == current.document.language + && next.schema_version == current.document.schema_version + && analysis_fingerprint(next) != current.analysis_fingerprint + } + fn visible_definitions( &self, file_id: &str, diff --git a/crates/semath-core/src/engine_tests.rs b/crates/semath-core/src/engine_tests.rs index 3d17a4e..de63677 100644 --- a/crates/semath-core/src/engine_tests.rs +++ b/crates/semath-core/src/engine_tests.rs @@ -296,6 +296,66 @@ fn incremental_upsert_matches_the_new_document_version() { assert_eq!(view.symbol.unwrap().definitions[0].description, "the state"); } +#[test] +fn same_revision_structural_relink_reanalyzes_without_accepting_stale_text() { + let content = "Let $A$ and $B$ be events. $\\joint{A}{B}$"; + let start = content.find("\\joint").unwrap() as u32; + let source = ProjectSourceRef { + file_id: "main".into(), + path: "main.tex".into(), + range: range(start, start + "\\joint".len() as u32), + }; + let mut expanded = document("main", "main.tex", content, 1); + expanded.macros.push(ProjectMacro { + kind: ProjectMacroKind::Call, + name: "joint".into(), + source: source.clone(), + definitions: Vec::new(), + expansion: ProjectMacroExpansion { + status: ProjectMacroExpansionStatus::Expanded, + depth: 1, + editable: false, + surface: Some("A \\cap B".into()), + input_range: Some(range(start, start + "\\joint{A}{B}".len() as u32)), + notation: None, + }, + }); + let mut unresolved = document("main", "main.tex", content, 1); + unresolved.macros.push(ProjectMacro { + kind: ProjectMacroKind::Call, + name: "joint".into(), + source, + definitions: Vec::new(), + expansion: ProjectMacroExpansion { + status: ProjectMacroExpansionStatus::Unresolved, + depth: 0, + editable: false, + surface: None, + input_range: None, + notation: None, + }, + }); + + let mut project = snapshot(content); + project.documents = vec![expanded]; + let mut engine = SemathEngine::default(); + engine.reset(project).unwrap(); + let update = engine + .apply(ChangeEnvelope { + protocol_version: PROTOCOL_VERSION, + epoch: "project:1".into(), + inventory_version: 2, + analysis_generation: 2, + changes: vec![ProjectChange::Upsert { + document: Box::new(unresolved), + }], + }) + .unwrap(); + + assert_eq!(update.changed_file_ids, ["main"]); + assert_eq!(update.analyzed_file_ids, ["main"]); +} + #[test] fn append_only_comments_advance_the_version_without_semantic_reanalysis() { let original = "Let $x$ denote the input. $y=x$"; diff --git a/docs/architecture.md b/docs/architecture.md index b4d71ae..f0459fd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -67,6 +67,10 @@ Pure extractors derive document observations from immutable documents. Project m reverse-include invalidation, cancellation, and caching form the effectful shell. An edit reanalyzes only the changed document and its reverse include closure; clean and incremental rebuilds must produce the same semantic result. +When an external declaration relinks a wasmtex snapshot, Semath accepts the +same text revision only if the source identity is unchanged and the structural +fingerprint differs. This retracts generated meaning without admitting stale or +rewritten text under an old document version. English scientific prose follows the same functional-core boundary. Bounded stages segment visible spans, extract mentions and claim spans, determine each diff --git a/docs/capability-test-matrix.md b/docs/capability-test-matrix.md index a7f3fa7..48b0ae1 100644 --- a/docs/capability-test-matrix.md +++ b/docs/capability-test-matrix.md @@ -6,12 +6,12 @@ reserved for real editor, Worker, and deployment wiring. | Capability | Authoritative test | Boundary evidence | E2E responsibility | | --- | --- | --- | --- | | Notation CST, UTF-16, cursor paths, malformed input | wasmtex contract plus `bun run notation:conformance` matrix/generative tests | adapter and clean/incremental parity | editor selection wiring | -| Semantic selection and binders | Rust cursor/parser/binder tests | native/WASM parity | one semantic selection journey | +| Semantic selection and binders | Rust cursor/parser/binder properties plus the neutral eight-family cursor plan | 102 native/WASM view/navigation queries | one semantic selection journey | | Definitions, references, rename | pure scope and include-order tests | LSP mapping and both cursor edges | one navigation journey | | Canonical meaning and typed laws | Rust canonical/unifier tests plus manifest-owned corpus | protocol and native/WASM equality | one meaning-first view | | Shapes, quantities, roles, diagnostics | pure extractors and contradiction tests | Worker/LSP result mapping | reveal one source-linked conflict | | Domain packs | Rust schema-7 compiler tests, conformance, pack-derived property planning, and evaluated or probe corpus | clean package and compiled catalog | none | -| Incremental analysis | pure edit-trace planning, first-divergence comparison, shrinking, reverse-include closure, and clean-rebuild equivalence | full-path 61-document budget and 501-document scale budget | one rapid-edit wiring case | +| Incremental analysis | pure six-family lifecycle planning, first-divergence comparison, shrinking, reverse-include closure, and clean-rebuild equivalence | fixed-sample and manual full-lifecycle parity plus 61/501-document budgets | one rapid-edit wiring case | | Worker lifecycle | pure queue and generation policy tests | real engine recreation | one project-switch or crash case | | CorTeX formula meaning | pure calm-presentation and bounded view-model tests | component integration tests | one meaning and one real-conflict journey | diff --git a/docs/pack-maturity.md b/docs/pack-maturity.md index 1a3611d..8939d14 100644 --- a/docs/pack-maturity.md +++ b/docs/pack-maturity.md @@ -1,6 +1,6 @@ # Pack maturity report -This is measured repository state on 2026-08-10, not live production telemetry +This is measured repository state on 2026-08-11, not live production telemetry and not a future plan. The [quality manifest](../fixtures/corpus-manifest.json) holds approved support policy; GitHub issues hold planned work. @@ -21,14 +21,12 @@ holds approved support policy; GitHub issues hold planned work. Electromagnetism, thermodynamics/heat transfer, fluid mechanics, calculus, discrete mathematics, and optimization remain deliberately narrow slices; evaluated capability evidence is not a field-completeness claim. -- The independent frozen challenge v2 contains 48 cases across binding, - constraints, packs, presentation, resolution, and syntax. The Protocol 8 - hard-cutover engine passes 48 of 48. Three reviewed oracle corrections preserve - a complete `\\sum` token during local malformed recovery and place the - higher-order derivative cursor on `\\partial^2`, the notation being tested; - the capacitor condition pair also declares its differentiation variable as - time on both sides of the boundary. Case IDs remain frozen, and each - correction removes an unintended variable rather than relaxing an outcome. +- The independent frozen challenge v3 preserves 48 semantic boundaries across + binding, constraints, packs, presentation, resolution, and syntax, then + composes them into six realistic document shapes. The engine passes 48 of 48, + including all five decision states, source-grounded explanations, and the + neutral-versus-conflict Problems policy. Full execution remains a manual + release gate. These numbers describe the synthetic benchmark, not real-world prevalence or field completeness. diff --git a/docs/performance.md b/docs/performance.md index 45b121c..7334cb2 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -16,7 +16,8 @@ mkdir -p .artifacts && bun run budget:report The fixture set deterministically rotates through the reported ECE expression, nested modifiers and styles, dense matrices, Unicode and combining characters, -malformed recovery, and binder/rename notation. A measured leaf edit must parse +malformed recovery, binder/rename notation, sectioned multi-equation reports, +and scoped malformed neighbors. A measured leaf edit must parse and transfer only its own syntax snapshot. Append-only comments must do no semantic analysis; a separate real notation mutation must analyze only its reverse-include closure and complete within 50ms. An empty delta must also do no @@ -37,6 +38,11 @@ defaults, not live production telemetry. The dependency lock pins the wasmtex input used for a report; record the Semath and wasmtex commits when comparing reports. +The report also names every lifecycle family, exposes semantic-view p95 +separately from other cursor queries, and records deterministic failure-shrinker +input, output, and evaluation counts. Shrinking has a linear work budget; timing +is not gated on a shared runner. + The normal and scale gates cap law candidates at 20 visited rules per document. This is a structural dispatch budget, independent of installed pack count. Pure 100-pack and 500-pack fixtures additionally require a uniquely keyed form diff --git a/docs/public-api.md b/docs/public-api.md index 015e494..c9de656 100644 --- a/docs/public-api.md +++ b/docs/public-api.md @@ -17,6 +17,9 @@ hard cutover to a small meaning-first API. Hosts send a complete `ProjectSnapshot`, then ordered `ChangeEnvelope` deltas. Every request carries protocol, inventory, document, and analysis versions so stale results can be rejected. +An upsert may repeat a document version only for a wasmtex structural relink of +identical source and path. Semath requires a changed structural fingerprint; +same-version text changes remain stale and are ignored. The query surface is: diff --git a/docs/semantic-quality-scorecards.md b/docs/semantic-quality-scorecards.md index 1eeae1b..80999d1 100644 --- a/docs/semantic-quality-scorecards.md +++ b/docs/semantic-quality-scorecards.md @@ -40,13 +40,16 @@ assumption, evidence, refusal, and scope independently instead of blending them into one recognition number. The frozen recognition challenge is separate from those development and -diversity fixtures. Version 2 has 48 manually authored cases, including twelve -positive/refusal semantic boundary pairs, grouped by the layer that owns a -failure: syntax, binding, constraint, pack, resolution, or presentation. Exact -IDs and normalized document sources must not occur in development or foundation -fixtures. Association, structure, constraint, recognition, evidence, refusal, -scope, and navigation remain separate metrics. Default CI validates -the challenge schema and coverage matrix through pure tests but does not execute +diversity fixtures. Version 3 preserves the 48 independently authored v2 +semantic boundaries and places every case into one reviewed document shape: +distant prose, neighboring macros, neighboring malformed input, multiple +equations, a multi-file project, or section scope. Every case declares the +expected final decision, meaning presence, and Problems policy. The scorecard +reports decision classes, source grounding, reason integrity, and problem +visibility separately from association, structure, constraints, recognition, +evidence, refusal, scope, and navigation. Exact IDs and normalized document +sources must not occur in development or foundation fixtures. Default CI +validates the schema, pure composition, and coverage matrix but does not execute the engine over the holdout. Evaluated laws require 100% role, evidence, and refusal preservation, at least 99% precision, and at @@ -90,7 +93,10 @@ Pack validation also derives a deterministic bounded property plan from the reviewed law declarations. Its oracle is the declared transformation relation, not the production matcher. Broad generated execution and failure artifacts remain part of the manual quality workflow; only planner integrity and a small -fixed sample run in default CI. +fixed sample run in default CI. A separate cursor plan exercises 102 +native/WASM queries across eight neutral structural families and compares +semantic view, definition, references, and rename preparation at every reviewed +edge. Native/WASM parity, full-path incremental latency and memory, package integrity, and documentation are separate gates so failures remain actionable. The normal diff --git a/fixtures/challenge/recognition-v3.json b/fixtures/challenge/recognition-v3.json new file mode 100644 index 0000000..a95db27 --- /dev/null +++ b/fixtures/challenge/recognition-v3.json @@ -0,0 +1,438 @@ +{ + "schemaVersion": 3, + "baseSchemaVersion": 2, + "profiles": [ + { + "caseId": "binding-metric-long-short", + "documentShape": "distant-prose", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "binding-hedged-name-refusal", + "documentShape": "macro-neighbor", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "binding-respectively-circuit", + "documentShape": "malformed-neighbor", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "binding-role-collision-refusal", + "documentShape": "multi-equation", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "constraint-matrix-vector-compatible", + "documentShape": "project-neighbor", + "decision": { + "status": "established", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "constraint-matrix-vector-mismatch", + "documentShape": "sectioned", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "constraint-explicit-vector-shape", + "documentShape": "distant-prose", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "constraint-incompatible-role-refusal", + "documentShape": "macro-neighbor", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "pack-unfamiliar-ohm-symbols", + "documentShape": "malformed-neighbor", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "pack-cross-field-product-refusal", + "documentShape": "multi-equation", + "decision": { + "status": "conflicting", + "meaning": "absent", + "problems": "source-conflict" + } + }, + { + "caseId": "pack-event-intersection", + "documentShape": "project-neighbor", + "decision": { + "status": "established", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "pack-set-event-collision", + "documentShape": "sectioned", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "presentation-decorated-source", + "documentShape": "distant-prose", + "decision": { + "status": "ambiguous", + "meaning": "absent", + "problems": "none" + } + }, + { + "caseId": "presentation-plain-not-decorated", + "documentShape": "macro-neighbor", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "presentation-named-surface", + "documentShape": "malformed-neighbor", + "decision": { + "status": "ambiguous", + "meaning": "absent", + "problems": "none" + } + }, + { + "caseId": "presentation-opaque-command-refusal", + "documentShape": "multi-equation", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "resolution-resource-navigation", + "documentShape": "project-neighbor", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "resolution-future-definition-refusal", + "documentShape": "sectioned", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "resolution-included-definition", + "documentShape": "distant-prose", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "resolution-disconnected-refusal", + "documentShape": "macro-neighbor", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "syntax-partial-candidate", + "documentShape": "malformed-neighbor", + "decision": { + "status": "ambiguous", + "meaning": "absent", + "problems": "none" + } + }, + { + "caseId": "syntax-unknown-differential-refusal", + "documentShape": "multi-equation", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "syntax-integral-binder", + "documentShape": "project-neighbor", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "syntax-index-family", + "documentShape": "sectioned", + "decision": { + "status": "ambiguous", + "meaning": "absent", + "problems": "none" + } + }, + { + "caseId": "binding-asserted-ece-name", + "documentShape": "distant-prose", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "binding-quoted-ece-name-refusal", + "documentShape": "macro-neighbor", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "binding-shared-matrix-declaration", + "documentShape": "malformed-neighbor", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "binding-alternative-matrix-refusal", + "documentShape": "multi-equation", + "decision": { + "status": "unsupported", + "meaning": "absent", + "problems": "none" + } + }, + { + "caseId": "constraint-events-same-context", + "documentShape": "project-neighbor", + "decision": { + "status": "established", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "constraint-events-different-context-refusal", + "documentShape": "sectioned", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "constraint-passive-sign-asserted", + "documentShape": "distant-prose", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "constraint-passive-sign-negated-refusal", + "documentShape": "macro-neighbor", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "pack-electric-force-typed", + "documentShape": "malformed-neighbor", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "pack-electric-force-cross-field-refusal", + "documentShape": "multi-equation", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "pack-sensible-heat-typed", + "documentShape": "project-neighbor", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "pack-sensible-heat-untyped-refusal", + "documentShape": "sectioned", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "presentation-hat-source-preserved", + "documentShape": "distant-prose", + "decision": { + "status": "ambiguous", + "meaning": "absent", + "problems": "none" + } + }, + { + "caseId": "presentation-style-not-decoration", + "documentShape": "macro-neighbor", + "decision": { + "status": "ambiguous", + "meaning": "absent", + "problems": "none" + } + }, + { + "caseId": "presentation-declared-ece-command", + "documentShape": "malformed-neighbor", + "decision": { + "status": "ambiguous", + "meaning": "absent", + "problems": "none" + } + }, + { + "caseId": "presentation-plain-ece-not-command", + "documentShape": "multi-equation", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "resolution-prior-section-binding", + "documentShape": "project-neighbor", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "resolution-sibling-section-refusal", + "documentShape": "sectioned", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "resolution-prior-include-binding", + "documentShape": "distant-prose", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "resolution-later-include-refusal", + "documentShape": "macro-neighbor", + "decision": { + "status": "unsupported", + "meaning": "absent", + "problems": "none" + } + }, + { + "caseId": "syntax-second-partial-structure", + "documentShape": "malformed-neighbor", + "decision": { + "status": "ambiguous", + "meaning": "absent", + "problems": "none" + } + }, + { + "caseId": "syntax-opaque-second-partial-refusal", + "documentShape": "multi-equation", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + }, + { + "caseId": "syntax-complete-sum-binder", + "documentShape": "project-neighbor", + "decision": { + "status": "ambiguous", + "meaning": "absent", + "problems": "none" + } + }, + { + "caseId": "syntax-malformed-sum-refusal", + "documentShape": "sectioned", + "decision": { + "status": "partial", + "meaning": "present", + "problems": "none" + } + } + ] +} diff --git a/lib/wasm/SHA256SUMS b/lib/wasm/SHA256SUMS index bb397c1..dedac82 100644 --- a/lib/wasm/SHA256SUMS +++ b/lib/wasm/SHA256SUMS @@ -1,4 +1,4 @@ 48ebf2d7ca8844a6c43d40f6c162c5f55ec11db48f59016a7bb8fe71c3a50aee semath_wasm.js 876e88de0cb682992cbac9908e0281972bdd3d63fcc6d1faa5ad1b37832da44c semath_wasm.d.ts -d6af4925f8e5ff96735a7d81822b9a553e2109ce7905ac9bf1de67b627508623 semath_wasm_bg.wasm +4af925e54feb9a3ab25778cdc628976d1ee6deb7d58e10e9d1490b8e48f0d860 semath_wasm_bg.wasm 0e25611dc3609896c18fd79eb0eb03ddd383a7bc34df29ccfc5abfd5df4dbe72 semath_wasm_bg.wasm.d.ts diff --git a/lib/wasm/semath_wasm_bg.wasm b/lib/wasm/semath_wasm_bg.wasm index 96c898a..bf3b25b 100644 Binary files a/lib/wasm/semath_wasm_bg.wasm and b/lib/wasm/semath_wasm_bg.wasm differ diff --git a/package.json b/package.json index 6ad641d..fd4529d 100644 --- a/package.json +++ b/package.json @@ -51,15 +51,15 @@ "package:smoke": "node scripts/check-package-smoke.mjs", "pack:authoring": "bun packages/authoring/cli.mjs validate packs/*/v1.json && bun packages/authoring/cli.mjs audit-runtime packs/*/v1.json", "pack:conformance": "bun scripts/check-pack-conformance.ts", - "parity": "bun scripts/check-parity.mjs", + "parity": "bun scripts/check-parity.mjs && bun scripts/check-lifecycle.mjs", "scorecard": "SEMATH_SCORECARD_PATH=.artifacts/semantic-scorecard.json bun scripts/check-corpus.ts", - "quality": "bun run corpus:generate:check && bun run engineering:generate:check && bun run foundation:generate:check && bun run pack:authoring && bun run pack:conformance && bun run challenge && bun run corpus && bun run foundation && bun run parity && bun run budget:stable && bun run package:smoke", + "quality": "bun run corpus:generate:check && bun run engineering:generate:check && bun run foundation:generate:check && bun run pack:authoring && bun run pack:conformance && bun run challenge && bun run corpus && bun run foundation && SEMATH_LIFECYCLE_FULL=1 bun run parity && bun run budget:stable && bun run package:smoke", "test": "cargo test --workspace && bun test", "typecheck": "tsc --noEmit", "verify": "bun run check" }, "dependencies": { - "wasmtex": "github:corca-ai/wasmtex#947d2c986ad4bb8b810153ab3ea17f119079e086" + "wasmtex": "github:corca-ai/wasmtex#0d34708807528f15f1898839acfe89c037b404ca" }, "devDependencies": { "@types/bun": "^1.3.14", diff --git a/packages/evaluation/src/challenge.test.ts b/packages/evaluation/src/challenge.test.ts index 14cd8df..ffc10b9 100644 --- a/packages/evaluation/src/challenge.test.ts +++ b/packages/evaluation/src/challenge.test.ts @@ -5,6 +5,7 @@ import { CHALLENGE_METRICS, findChallengeFixtureLeaks, parseChallengeCorpus, + parseChallengeV3, scoreChallenge, type ChallengeCase, type ChallengeObservation, @@ -15,12 +16,16 @@ function cases(): ChallengeCase[] { return Array.from({ length: 48 }, (_, index) => ({ cursor: { fileId: "main", needle: "$x$" }, documents: [{ content: "$x$", fileId: "main", path: "main.tex" }], - expectation: index % 2 === 0 ? { symbol: "x" } : { excludedRelationId: "wrong" }, + expectation: + index % 2 === 0 ? { symbol: "x" } : { excludedRelationId: "wrong" }, id: `case-${index}`, metric: CHALLENGE_METRICS[index % CHALLENGE_METRICS.length]!, outcome: index % 2 === 0 ? "positive" : "refusal", owner: CHALLENGE_LAYERS[Math.floor(index / 2) % CHALLENGE_LAYERS.length]!, - variationTags: [`variation-${index}`, `boundary-pair:pair-${Math.floor(index / 2)}`], + variationTags: [ + `variation-${index}`, + `boundary-pair:pair-${Math.floor(index / 2)}`, + ], })); } @@ -28,11 +33,30 @@ function source(value: readonly ChallengeCase[]) { return { cases: value, schemaVersion: 2 }; } +function observation(item: ChallengeCase): ChallengeObservation { + return { + assumptionValues: [], + candidates: [], + caseId: item.id, + conceptIds: [], + definitions: [], + problemCount: 0, + reasonKinds: [], + relationIds: [], + shapes: [], + sourceGrounded: false, + symbols: item.expectation.symbol ? [item.expectation.symbol] : [], + }; +} + describe("independent recognition challenge", () => { test("keeps the checked-in holdout strict and coverage-complete", async () => { const fixture: unknown = JSON.parse( await readFile( - new URL("../../../fixtures/challenge/recognition-v2.json", import.meta.url), + new URL( + "../../../fixtures/challenge/recognition-v2.json", + import.meta.url, + ), "utf8", ), ); @@ -46,13 +70,56 @@ describe("independent recognition challenge", () => { onlyFiles: true, })) { const suite: unknown = JSON.parse( - await readFile(new URL(`../../../fixtures/${path}`, import.meta.url), "utf8"), + await readFile( + new URL(`../../../fixtures/${path}`, import.meta.url), + "utf8", + ), ); development.push(...parseDevelopmentCases(suite)); } expect(findChallengeFixtureLeaks(challenge.cases, development)).toEqual([]); }); + test("composes every frozen case with explicit document and decision policy", async () => { + const base: unknown = JSON.parse( + await readFile( + new URL( + "../../../fixtures/challenge/recognition-v2.json", + import.meta.url, + ), + "utf8", + ), + ); + const profile: unknown = JSON.parse( + await readFile( + new URL( + "../../../fixtures/challenge/recognition-v3.json", + import.meta.url, + ), + "utf8", + ), + ); + const challenge = parseChallengeV3(base, profile); + expect(challenge.schemaVersion).toBe(3); + expect(challenge.cases).toHaveLength(48); + expect(new Set(challenge.cases.map((item) => item.documentShape))).toEqual( + new Set([ + "distant-prose", + "macro-neighbor", + "malformed-neighbor", + "multi-equation", + "project-neighbor", + "sectioned", + ]), + ); + expect(challenge.cases.every((item) => item.decisionExpectation)).toBe( + true, + ); + expect(challenge.cases.every((item) => item.documents.length > 0)).toBe( + true, + ); + }); + test("parses a strict frozen matrix with every layer, outcome, and metric", () => { expect(parseChallengeCorpus(source(cases())).cases).toHaveLength(48); expect(() => @@ -63,13 +130,8 @@ describe("independent recognition challenge", () => { test("scores behavior by owner, metric, and outcome without blending failures", () => { const corpus = parseChallengeCorpus(source(cases())); const observations: ChallengeObservation[] = corpus.cases.map((item) => ({ - assumptionValues: [], - candidates: [], - caseId: item.id, - conceptIds: [], - definitions: [], + ...observation(item), relationIds: item.outcome === "refusal" ? [] : ["unrelated"], - shapes: [], status: "partial", symbols: item.outcome === "positive" ? ["x"] : [], })); @@ -81,6 +143,34 @@ describe("independent recognition challenge", () => { expect(scorecard.outcomes.refusal).toEqual({ passed: 24, total: 24 }); }); + test("scores decision, explanation, problem, and reason policy independently", () => { + const base = cases(); + base[0] = { + ...base[0]!, + decisionExpectation: { + meaning: "present", + problems: "none", + status: "established", + }, + }; + const corpus = { cases: base, schemaVersion: 3 } as const; + const observations = corpus.cases.map((item) => observation(item)); + observations[0] = { + ...observations[0]!, + meaningLabel: "Known relation", + problemCount: 0, + reasonKinds: ["proof"], + sourceGrounded: true, + status: "established", + symbols: ["x"], + }; + const scorecard = scoreChallenge(corpus, observations); + expect(scorecard.decisions.established).toEqual({ passed: 1, total: 1 }); + expect(scorecard.explanation).toEqual({ passed: 1, total: 1 }); + expect(scorecard.problemPolicy.none).toEqual({ passed: 1, total: 1 }); + expect(scorecard.reasonIntegrity).toEqual({ passed: 1, total: 1 }); + }); + test("rejects ambiguous cursors and incomplete coverage", () => { const values = cases(); values[0] = { @@ -88,14 +178,20 @@ describe("independent recognition challenge", () => { cursor: { fileId: "main", needle: "x" }, documents: [{ content: "$x+x$", fileId: "main", path: "main.tex" }], }; - expect(() => parseChallengeCorpus(source(values))).toThrow("must occur exactly once"); - expect(() => parseChallengeCorpus(source(cases().slice(0, 47)))).toThrow("at least 48"); + expect(() => parseChallengeCorpus(source(values))).toThrow( + "must occur exactly once", + ); + expect(() => parseChallengeCorpus(source(cases().slice(0, 47)))).toThrow( + "at least 48", + ); }); test("requires complete semantic boundary pairs and rejects development-fixture reuse", () => { const values = cases(); values[0] = { ...values[0]!, variationTags: ["boundary-pair:unpaired"] }; - expect(() => parseChallengeCorpus(source(values))).toThrow("incomplete boundary pair unpaired"); + expect(() => parseChallengeCorpus(source(values))).toThrow( + "incomplete boundary pair unpaired", + ); const challenge = cases().slice(0, 1); expect(findChallengeFixtureLeaks(challenge, challenge)).toEqual([ @@ -108,7 +204,11 @@ describe("independent recognition challenge", () => { function parseDevelopmentCases(value: unknown): DevelopmentFixtureCase[] { if (!isRecord(value) || !Array.isArray(value.cases)) return []; return value.cases.flatMap((item) => { - if (!isRecord(item) || typeof item.id !== "string" || !Array.isArray(item.documents)) { + if ( + !isRecord(item) || + typeof item.id !== "string" || + !Array.isArray(item.documents) + ) { return []; } const documents = item.documents.flatMap((document) => @@ -116,10 +216,18 @@ function parseDevelopmentCases(value: unknown): DevelopmentFixtureCase[] { typeof document.content === "string" && typeof document.fileId === "string" && typeof document.path === "string" - ? [{ content: document.content, fileId: document.fileId, path: document.path }] + ? [ + { + content: document.content, + fileId: document.fileId, + path: document.path, + }, + ] : [], ); - return documents.length === item.documents.length ? [{ documents, id: item.id }] : []; + return documents.length === item.documents.length + ? [{ documents, id: item.id }] + : []; }); } diff --git a/packages/evaluation/src/challenge.ts b/packages/evaluation/src/challenge.ts index 136bbf3..07c35ec 100644 --- a/packages/evaluation/src/challenge.ts +++ b/packages/evaluation/src/challenge.ts @@ -18,10 +18,36 @@ export const CHALLENGE_METRICS = [ "scope", "structure", ] as const; +export const CHALLENGE_DECISIONS = [ + "established", + "partial", + "ambiguous", + "conflicting", + "unsupported", +] as const; +export const CHALLENGE_DOCUMENT_SHAPES = [ + "distant-prose", + "macro-neighbor", + "malformed-neighbor", + "multi-equation", + "project-neighbor", + "sectioned", +] as const; +export const CHALLENGE_PROBLEM_POLICIES = ["none", "source-conflict"] as const; export type ChallengeLayer = (typeof CHALLENGE_LAYERS)[number]; export type ChallengeMetric = (typeof CHALLENGE_METRICS)[number]; export type ChallengeOutcome = "positive" | "refusal"; +export type ChallengeDecision = (typeof CHALLENGE_DECISIONS)[number]; +export type ChallengeDocumentShape = (typeof CHALLENGE_DOCUMENT_SHAPES)[number]; +export type ChallengeProblemPolicy = + (typeof CHALLENGE_PROBLEM_POLICIES)[number]; + +export interface ChallengeDecisionExpectation { + readonly meaning: "absent" | "present"; + readonly problems: ChallengeProblemPolicy; + readonly status: ChallengeDecision; +} export interface ChallengeExpectation { readonly assumptionValue?: string; @@ -48,6 +74,8 @@ export interface ChallengeCase { readonly needle: string; }; readonly documents: readonly CorpusDocument[]; + readonly decisionExpectation?: ChallengeDecisionExpectation; + readonly documentShape?: ChallengeDocumentShape; readonly expectation: ChallengeExpectation; readonly id: string; readonly metric: ChallengeMetric; @@ -58,7 +86,7 @@ export interface ChallengeCase { export interface ChallengeCorpus { readonly cases: readonly ChallengeCase[]; - readonly schemaVersion: 2; + readonly schemaVersion: 2 | 3; } export interface ChallengeObservation { @@ -74,9 +102,14 @@ export interface ChallengeObservation { readonly ruleId: string; readonly symbol: string; }[]; + readonly meaningLabel?: string; + readonly meaningRelationId?: string | null; + readonly problemCount: number; + readonly reasonKinds: readonly string[]; readonly relationIds: readonly string[]; readonly shapes: readonly string[]; readonly sourceNotation?: string; + readonly sourceGrounded: boolean; readonly status?: string; readonly symbols: readonly string[]; } @@ -84,11 +117,31 @@ export interface ChallengeObservation { export interface ChallengeScorecard { readonly cases: number; readonly failures: readonly string[]; - readonly layers: Readonly>; - readonly metrics: Readonly>; - readonly outcomes: Readonly>; + readonly decisions: Readonly< + Record + >; + readonly explanation: { passed: number; total: number }; + readonly layers: Readonly< + Record + >; + readonly metrics: Readonly< + Record + >; + readonly outcomes: Readonly< + Record + >; readonly passed: number; - readonly schemaVersion: 2; + readonly problemPolicy: Readonly< + Record + >; + readonly reasonIntegrity: { passed: number; total: number }; + readonly schemaVersion: 2 | 3; +} + +interface ChallengeV3Profile { + readonly caseId: string; + readonly decision: ChallengeDecisionExpectation; + readonly documentShape: ChallengeDocumentShape; } export interface DevelopmentFixtureCase { @@ -108,7 +161,8 @@ export function findChallengeFixtureLeaks( ); const leaks = new Set(); for (const item of challenge) { - if (developmentIds.has(item.id)) leaks.add(`${item.id}: duplicate fixture id`); + if (developmentIds.has(item.id)) + leaks.add(`${item.id}: duplicate fixture id`); for (const document of item.documents) { if (developmentSources.has(normalizedSource(document.content))) { leaks.add(`${item.id}: duplicate fixture source`); @@ -121,15 +175,23 @@ export function findChallengeFixtureLeaks( export function parseChallengeCorpus(value: unknown): ChallengeCorpus { const root = record(value, "challenge"); exact(root, ["schemaVersion", "cases"], "challenge"); - if (root.schemaVersion !== 2) throw new Error("challenge.schemaVersion: must be 2"); + if (root.schemaVersion !== 2) + throw new Error("challenge.schemaVersion: must be 2"); if (!Array.isArray(root.cases) || root.cases.length < 48) { throw new Error("challenge.cases: must contain at least 48 frozen cases"); } - const cases = root.cases.map((item, index) => parseCase(item, `challenge.cases[${index}]`)); - unique(cases.map((item) => item.id), "challenge.cases.id"); + const cases = root.cases.map((item, index) => + parseCase(item, `challenge.cases[${index}]`), + ); + unique( + cases.map((item) => item.id), + "challenge.cases.id", + ); for (const layer of CHALLENGE_LAYERS) { for (const outcome of ["positive", "refusal"] as const) { - if (!cases.some((item) => item.owner === layer && item.outcome === outcome)) { + if ( + !cases.some((item) => item.owner === layer && item.outcome === outcome) + ) { throw new Error(`challenge.cases: missing ${layer}/${outcome}`); } } @@ -143,14 +205,90 @@ export function parseChallengeCorpus(value: unknown): ChallengeCorpus { return { cases, schemaVersion: 2 }; } +/** + * Composes the frozen v2 semantic boundaries with a strict document-shaped v3 + * profile. The profile contains only independent test policy; it never reads + * production recognition or presentation code. + */ +export function parseChallengeV3( + baseValue: unknown, + profileValue: unknown, +): ChallengeCorpus { + const base = parseChallengeCorpus(baseValue); + const root = record(profileValue, "challenge-v3"); + exact( + root, + ["schemaVersion", "baseSchemaVersion", "profiles"], + "challenge-v3", + ); + if (root.schemaVersion !== 3) + throw new Error("challenge-v3.schemaVersion: must be 3"); + if (root.baseSchemaVersion !== 2) { + throw new Error("challenge-v3.baseSchemaVersion: must be 2"); + } + if ( + !Array.isArray(root.profiles) || + root.profiles.length !== base.cases.length + ) { + throw new Error( + `challenge-v3.profiles: must contain exactly ${base.cases.length} profiles`, + ); + } + const profiles = root.profiles.map((value, index) => + parseV3Profile(value, `challenge-v3.profiles[${index}]`), + ); + unique( + profiles.map((profile) => profile.caseId), + "challenge-v3.profiles.caseId", + ); + const profileById = new Map( + profiles.map((profile) => [profile.caseId, profile]), + ); + const unknown = profiles + .filter((profile) => !base.cases.some((item) => item.id === profile.caseId)) + .map((profile) => profile.caseId); + if (unknown.length) + throw new Error(`challenge-v3.profiles: unknown case ${unknown.sort()[0]}`); + + const cases = base.cases.map((item) => { + const profile = profileById.get(item.id); + if (!profile) + throw new Error(`challenge-v3.profiles: missing case ${item.id}`); + return shapeChallengeCase(item, profile); + }); + for (const shape of CHALLENGE_DOCUMENT_SHAPES) { + if (!cases.some((item) => item.documentShape === shape)) { + throw new Error(`challenge-v3.profiles: missing document shape ${shape}`); + } + } + for (const status of CHALLENGE_DECISIONS) { + if (!cases.some((item) => item.decisionExpectation?.status === status)) { + throw new Error(`challenge-v3.profiles: missing decision ${status}`); + } + } + for (const policy of CHALLENGE_PROBLEM_POLICIES) { + if (!cases.some((item) => item.decisionExpectation?.problems === policy)) { + throw new Error( + `challenge-v3.profiles: missing problem policy ${policy}`, + ); + } + } + return { cases, schemaVersion: 3 }; +} + export function scoreChallenge( corpus: ChallengeCorpus, observations: readonly ChallengeObservation[], ): ChallengeScorecard { const byId = new Map(observations.map((item) => [item.caseId, item])); const failures: string[] = []; - if (byId.size !== observations.length) failures.push("challenge: duplicate observations"); + if (byId.size !== observations.length) + failures.push("challenge: duplicate observations"); const passed = new Set(); + const decisionPassed = new Set(); + const explanationPassed = new Set(); + const problemPassed = new Set(); + const reasonPassed = new Set(); for (const item of corpus.cases) { const observation = byId.get(item.id); if (!observation) { @@ -158,8 +296,37 @@ export function scoreChallenge( continue; } const expected = item.expectation; + const decision = item.decisionExpectation; + const decisionMismatch = + decision && observation.status !== decision.status + ? `decision ${decision.status}` + : undefined; + const meaningPresent = observation.meaningLabel !== undefined; + const explanationMismatch = decision + ? meaningPresent !== (decision.meaning === "present") + ? `meaning ${decision.meaning}` + : expected.relationId && + observation.meaningRelationId !== expected.relationId + ? `meaning relation ${expected.relationId}` + : observation.meaningRelationId && !observation.sourceGrounded + ? "source-grounded meaning" + : undefined + : undefined; + const problemMismatch = + decision && !problemPolicyMatches(decision.problems, observation) + ? `problems ${decision.problems}` + : undefined; + const reasonMismatch = + decision && !reasonsAreValid(decision.status, observation) + ? `reason integrity for ${decision.status}` + : undefined; + if (decision && !decisionMismatch) decisionPassed.add(item.id); + if (decision && !explanationMismatch) explanationPassed.add(item.id); + if (decision && !problemMismatch) problemPassed.add(item.id); + if (decision && !reasonMismatch) reasonPassed.add(item.id); const mismatches = [ - expected.assumptionValue && !observation.assumptionValues.includes(expected.assumptionValue) + expected.assumptionValue && + !observation.assumptionValues.includes(expected.assumptionValue) ? `assumption ${expected.assumptionValue}` : undefined, expected.candidateFamily && @@ -196,7 +363,8 @@ export function scoreChallenge( ) ? `definition evidence ${expected.definitionRuleId}` : undefined, - expected.excludedConceptId && observation.conceptIds.includes(expected.excludedConceptId) + expected.excludedConceptId && + observation.conceptIds.includes(expected.excludedConceptId) ? `excluded concept ${expected.excludedConceptId}` : undefined, expected.excludedDefinitionSymbol && @@ -205,16 +373,19 @@ export function scoreChallenge( ) ? `excluded definition ${expected.excludedDefinitionSymbol}` : undefined, - expected.excludedRelationId && observation.relationIds.includes(expected.excludedRelationId) + expected.excludedRelationId && + observation.relationIds.includes(expected.excludedRelationId) ? `excluded relation ${expected.excludedRelationId}` : undefined, - expected.relationId && !observation.relationIds.includes(expected.relationId) + expected.relationId && + !observation.relationIds.includes(expected.relationId) ? `relation ${expected.relationId}` : undefined, expected.shape && !observation.shapes.includes(expected.shape) ? `shape ${expected.shape}` : undefined, - expected.sourceNotation && observation.sourceNotation !== expected.sourceNotation + expected.sourceNotation && + observation.sourceNotation !== expected.sourceNotation ? `source notation ${expected.sourceNotation}` : undefined, expected.status && observation.status !== expected.status @@ -223,8 +394,13 @@ export function scoreChallenge( expected.symbol && !observation.symbols.includes(expected.symbol) ? `symbol ${expected.symbol}` : undefined, + decisionMismatch, + explanationMismatch, + problemMismatch, + reasonMismatch, ].filter((item): item is string => Boolean(item)); - if (mismatches.length) failures.push(`${item.id}: missing ${mismatches.join(", ")}`); + if (mismatches.length) + failures.push(`${item.id}: missing ${mismatches.join(", ")}`); else passed.add(item.id); } for (const item of observations) { @@ -234,9 +410,21 @@ export function scoreChallenge( } return { cases: corpus.cases.length, + decisions: tallyExpected( + corpus.cases, + decisionPassed, + CHALLENGE_DECISIONS, + (item) => item.decisionExpectation?.status, + ), + explanation: countedExpected(corpus.cases, explanationPassed), failures: [...new Set(failures)].sort(), layers: tally(corpus.cases, passed, CHALLENGE_LAYERS, (item) => item.owner), - metrics: tally(corpus.cases, passed, CHALLENGE_METRICS, (item) => item.metric), + metrics: tally( + corpus.cases, + passed, + CHALLENGE_METRICS, + (item) => item.metric, + ), outcomes: tally( corpus.cases, passed, @@ -244,15 +432,199 @@ export function scoreChallenge( (item) => item.outcome, ), passed: passed.size, - schemaVersion: 2, + problemPolicy: tallyExpected( + corpus.cases, + problemPassed, + CHALLENGE_PROBLEM_POLICIES, + (item) => item.decisionExpectation?.problems, + ), + reasonIntegrity: countedExpected(corpus.cases, reasonPassed), + schemaVersion: corpus.schemaVersion, + }; +} + +function parseV3Profile(value: unknown, path: string): ChallengeV3Profile { + const item = record(value, path); + exact(item, ["caseId", "decision", "documentShape"], path); + const decision = record(item.decision, `${path}.decision`); + exact(decision, ["meaning", "problems", "status"], `${path}.decision`); + return { + caseId: text(item.caseId, `${path}.caseId`), + decision: { + meaning: oneOf( + decision.meaning, + ["absent", "present"] as const, + `${path}.decision.meaning`, + ), + problems: oneOf( + decision.problems, + CHALLENGE_PROBLEM_POLICIES, + `${path}.decision.problems`, + ), + status: oneOf( + decision.status, + CHALLENGE_DECISIONS, + `${path}.decision.status`, + ), + }, + documentShape: oneOf( + item.documentShape, + CHALLENGE_DOCUMENT_SHAPES, + `${path}.documentShape`, + ), + }; +} + +function shapeChallengeCase( + item: ChallengeCase, + profile: ChallengeV3Profile, +): ChallengeCase { + const target = item.documents.find( + (document) => document.fileId === item.cursor.fileId, + ); + if (!target) throw new Error(`${item.id}: missing cursor document`); + const markdown = + target.path.endsWith(".md") || target.path.endsWith(".markdown"); + const marker = item.id.replaceAll(/[^a-zA-Z0-9]/gu, "-"); + const equation = markdown + ? `\n\nA separate calibration check records $\\xi_{\\mathrm{aux}}=17$.\n` + : `\n\\[\\xi_{\\mathrm{aux}}=17\\]\n`; + const prose = + "The surrounding report compares several independent measurements. " + + "Only explicit declarations in the current scope may determine the notation below.\n\n"; + const documents = item.documents.map((document) => { + if (document.fileId !== target.fileId) return document; + const content = (() => { + switch (profile.documentShape) { + case "distant-prose": + return prose.repeat(3) + document.content + equation; + case "macro-neighbor": + return markdown + ? prose + document.content + equation + : `\\newcommand{\\auxmetric}{\\xi_{\\mathrm{aux}}}\n$\\auxmetric=17$.\n${document.content}`; + case "malformed-neighbor": + return ( + document.content + + equation + + (markdown + ? "\nAn unfinished neighbor is $\\frac{1}{" + : "\n$\\frac{1}{$") + ); + case "multi-equation": + return ( + equation + prose + document.content + equation.replace("17", "19") + ); + case "project-neighbor": + return prose + document.content; + case "sectioned": + return markdown + ? `# Background\n\n${equation}\n# Reported result\n\n${document.content}` + : `\\section{Background}\n${equation}\\section{Reported result}\n${document.content}`; + } + })(); + return { ...document, content }; + }); + if (profile.documentShape === "project-neighbor") { + documents.push({ + content: `Independent appendix for ${marker}.\n${equation}`, + fileId: `v3-neighbor-${marker}`, + path: `v3-neighbor-${marker}.${markdown ? "md" : "tex"}`, + }); + } + return { + ...item, + decisionExpectation: profile.decision, + documents, + documentShape: profile.documentShape, + variationTags: [ + ...item.variationTags, + `document-shape:${profile.documentShape}`, + ], }; } +function problemPolicyMatches( + policy: ChallengeProblemPolicy, + observation: ChallengeObservation, +): boolean { + return policy === "none" + ? observation.problemCount === 0 + : observation.problemCount > 0 && + observation.reasonKinds.includes("source-conflict"); +} + +function reasonsAreValid( + status: ChallengeDecision, + observation: ChallengeObservation, +): boolean { + const allowed: Readonly>> = { + ambiguous: new Set(["uncertainty", "engine-limit"]), + conflicting: new Set(["source-conflict"]), + established: new Set(["proof"]), + partial: new Set(["uncertainty", "engine-limit"]), + unsupported: new Set(["uncertainty", "engine-limit"]), + }; + if (observation.reasonKinds.some((kind) => !allowed[status].has(kind))) + return false; + if (status === "established") { + return ( + observation.reasonKinds.includes("proof") && observation.sourceGrounded + ); + } + if (status === "conflicting") { + return ( + observation.reasonKinds.includes("source-conflict") && + observation.sourceGrounded + ); + } + return status === "partial" || observation.reasonKinds.length > 0; +} + +function countedExpected( + cases: readonly ChallengeCase[], + passed: ReadonlySet, +): { passed: number; total: number } { + const expected = cases.filter((item) => item.decisionExpectation); + return { + passed: expected.filter((item) => passed.has(item.id)).length, + total: expected.length, + }; +} + +function tallyExpected( + items: readonly ChallengeCase[], + passed: ReadonlySet, + keys: Keys, + select: (item: ChallengeCase) => Keys[number] | undefined, +): Record { + return Object.fromEntries( + keys.map((key) => { + const selected = items.filter((item) => select(item) === key); + return [ + key, + { + passed: selected.filter((item) => passed.has(item.id)).length, + total: selected.length, + }, + ]; + }), + ) as Record; +} + function parseCase(value: unknown, path: string): ChallengeCase { const item = record(value, path); exact( item, - ["id", "documents", "cursor", "expectation", "metric", "outcome", "owner", "variationTags"], + [ + "id", + "documents", + "cursor", + "expectation", + "metric", + "outcome", + "owner", + "variationTags", + ], path, ); const id = text(item.id, `${path}.id`); @@ -261,23 +633,35 @@ function parseCase(value: unknown, path: string): ChallengeCase { } const documents = item.documents.map((value, index) => { const document = record(value, `${path}.documents[${index}]`); - exact(document, ["content", "fileId", "path"], `${path}.documents[${index}]`); + exact( + document, + ["content", "fileId", "path"], + `${path}.documents[${index}]`, + ); return { content: text(document.content, `${path}.documents[${index}].content`), fileId: text(document.fileId, `${path}.documents[${index}].fileId`), path: text(document.path, `${path}.documents[${index}].path`), }; }); - unique(documents.map((item) => item.fileId), `${path}.documents.fileId`); + unique( + documents.map((item) => item.fileId), + `${path}.documents.fileId`, + ); const cursor = record(item.cursor, `${path}.cursor`); exact(cursor, ["edge", "fileId", "needle"], `${path}.cursor`); const cursorFileId = text(cursor.fileId, `${path}.cursor.fileId`); const needle = text(cursor.needle, `${path}.cursor.needle`); - const cursorDocument = documents.find((document) => document.fileId === cursorFileId); - if (!cursorDocument) throw new Error(`${path}.cursor.fileId: unknown ${cursorFileId}`); + const cursorDocument = documents.find( + (document) => document.fileId === cursorFileId, + ); + if (!cursorDocument) + throw new Error(`${path}.cursor.fileId: unknown ${cursorFileId}`); const occurrences = cursorDocument.content.split(needle).length - 1; if (occurrences !== 1) { - throw new Error(`${path}.cursor.needle: must occur exactly once; found ${occurrences}`); + throw new Error( + `${path}.cursor.needle: must occur exactly once; found ${occurrences}`, + ); } const expectation = record(item.expectation, `${path}.expectation`); const expectationKeys = [ @@ -305,7 +689,13 @@ function parseCase(value: unknown, path: string): ChallengeCase { cursor: { ...(cursor.edge === undefined ? {} - : { edge: oneOf(cursor.edge, ["after", "before"] as const, `${path}.cursor.edge`) }), + : { + edge: oneOf( + cursor.edge, + ["after", "before"] as const, + `${path}.cursor.edge`, + ), + }), fileId: cursorFileId, needle, }, @@ -319,7 +709,11 @@ function parseCase(value: unknown, path: string): ChallengeCase { ), id, metric: oneOf(item.metric, CHALLENGE_METRICS, `${path}.metric`), - outcome: oneOf(item.outcome, ["positive", "refusal"] as const, `${path}.outcome`), + outcome: oneOf( + item.outcome, + ["positive", "refusal"] as const, + `${path}.outcome`, + ), owner: oneOf(item.owner, CHALLENGE_LAYERS, `${path}.owner`), variationTags: stringList(item.variationTags, `${path}.variationTags`), }; @@ -338,7 +732,9 @@ function validateBoundaryPairs(cases: readonly ChallengeCase[]): void { } } if (pairs.size < 12) { - throw new Error("challenge.cases: must contain at least 12 semantic boundary pairs"); + throw new Error( + "challenge.cases: must contain at least 12 semantic boundary pairs", + ); } for (const [pair, outcomes] of pairs) { if (!outcomes.has("positive") || !outcomes.has("refusal")) { @@ -365,7 +761,10 @@ function tally< const selected = items.filter((item) => select(item) === key); return [ key, - { passed: selected.filter((item) => passed.has(item.id)).length, total: selected.length }, + { + passed: selected.filter((item) => passed.has(item.id)).length, + total: selected.length, + }, ]; }), ) as Record; @@ -378,19 +777,26 @@ function record(value: unknown, path: string): Record { return value as Record; } -function exact(value: Record, keys: readonly string[], path: string): void { +function exact( + value: Record, + keys: readonly string[], + path: string, +): void { const allowed = new Set(keys); const unknown = Object.keys(value).filter((key) => !allowed.has(key)); - if (unknown.length) throw new Error(`${path}: unknown field ${unknown.sort()[0]}`); + if (unknown.length) + throw new Error(`${path}: unknown field ${unknown.sort()[0]}`); } function text(value: unknown, path: string): string { - if (typeof value !== "string" || !value.trim()) throw new Error(`${path}: must be text`); + if (typeof value !== "string" || !value.trim()) + throw new Error(`${path}: must be text`); return value; } function stringList(value: unknown, path: string): string[] { - if (!Array.isArray(value) || !value.length) throw new Error(`${path}: must be nonempty text[]`); + if (!Array.isArray(value) || !value.length) + throw new Error(`${path}: must be nonempty text[]`); const output = value.map((item, index) => text(item, `${path}[${index}]`)); unique(output, path); return output; @@ -408,5 +814,6 @@ function oneOf( } function unique(values: readonly string[], path: string): void { - if (new Set(values).size !== values.length) throw new Error(`${path}: must be unique`); + if (new Set(values).size !== values.length) + throw new Error(`${path}: must be unique`); } diff --git a/packages/evaluation/src/cursor-invariants.test.ts b/packages/evaluation/src/cursor-invariants.test.ts new file mode 100644 index 0000000..64525fd --- /dev/null +++ b/packages/evaluation/src/cursor-invariants.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, test } from "bun:test"; +import { CURSOR_INVARIANT_FAMILIES, planCursorInvariantSurfaces } from "./cursor-invariants"; + +describe("cross-stack cursor invariant planning", () => { + test("covers every reviewed structural family and every cursor edge deterministically", () => { + const surfaces = planCursorInvariantSurfaces(); + expect(surfaces).toEqual(planCursorInvariantSurfaces()); + expect(new Set(surfaces.map((surface) => surface.family))).toEqual( + new Set(CURSOR_INVARIANT_FAMILIES), + ); + expect(surfaces.every((surface) => surface.probes.length >= 2)).toBe(true); + expect( + surfaces.flatMap((surface) => surface.probes).some((probe) => probe.id.endsWith("after")), + ).toBe(true); + }); +}); diff --git a/packages/evaluation/src/cursor-invariants.ts b/packages/evaluation/src/cursor-invariants.ts new file mode 100644 index 0000000..84bb928 --- /dev/null +++ b/packages/evaluation/src/cursor-invariants.ts @@ -0,0 +1,169 @@ +export const CURSOR_INVARIANT_FAMILIES = [ + "application", + "declared-surface", + "fraction", + "macro-expansion", + "modifier", + "named-surface", + "nested-style", + "style", +] as const; + +export type CursorInvariantFamily = (typeof CURSOR_INVARIANT_FAMILIES)[number]; + +export interface CursorInvariantSurface { + readonly content: string; + readonly expectedSourceNotation: string; + readonly expectedSymbol: string; + readonly family: CursorInvariantFamily; + readonly fileId: string; + readonly id: string; + readonly path: string; + readonly probes: readonly { readonly id: string; readonly offset: number }[]; +} + +interface CursorSurfaceSeed { + readonly content: string; + readonly expectedSourceNotation: string; + readonly expectedSymbol: string; + readonly family: CursorInvariantFamily; + readonly id: string; + readonly probes: readonly { readonly delta: number; readonly id: string; readonly needle: string }[]; +} + +const SEEDS: readonly CursorSurfaceSeed[] = [ + { + content: "Let $y$ denote the prediction. Compare $\\hat y$ with the observation.", + expectedSourceNotation: "\\hat y", + expectedSymbol: "y", + family: "modifier", + id: "unbraced-hat", + probes: [ + { delta: 0, id: "modifier-start", needle: "\\hat y" }, + { delta: 5, id: "nucleus-start", needle: "\\hat y" }, + { delta: 6, id: "nucleus-after", needle: "\\hat y" }, + { delta: 6, id: "modifier-after", needle: "\\hat y" }, + ], + }, + { + content: "Let $F$ denote force. Compare $\\mathbf{F}$ with the scalar baseline.", + expectedSourceNotation: "\\mathbf{F}", + expectedSymbol: "F", + family: "style", + id: "styled-force", + probes: [ + { delta: 8, id: "body-start", needle: "\\mathbf{F}" }, + { delta: 9, id: "body-after", needle: "\\mathbf{F}" }, + { delta: 10, id: "style-after", needle: "\\mathbf{F}" }, + ], + }, + { + content: "Let $y$ denote the estimate. Compare $\\mathbf{\\hat{y}}$ with the target.", + expectedSourceNotation: "\\mathbf{\\hat{y}}", + expectedSymbol: "y", + family: "nested-style", + id: "nested-style-hat", + probes: [ + { delta: 13, id: "nucleus-start", needle: "\\mathbf{\\hat{y}}" }, + { delta: 14, id: "nucleus-after", needle: "\\mathbf{\\hat{y}}" }, + { delta: 16, id: "composite-after", needle: "\\mathbf{\\hat{y}}" }, + ], + }, + { + content: "Expected calibration error (ECE) is reported as $\\operatorname{ECE}(x)$.", + expectedSourceNotation: "\\operatorname{ECE}", + expectedSymbol: "ECE", + family: "named-surface", + id: "named-ece", + probes: [ + { delta: 14, id: "name-first", needle: "\\operatorname{ECE}" }, + { delta: 15, id: "name-middle", needle: "\\operatorname{ECE}" }, + { delta: 16, id: "name-last", needle: "\\operatorname{ECE}" }, + { delta: 18, id: "surface-after", needle: "\\operatorname{ECE}" }, + ], + }, + { + content: "Expected calibration error (ECE) is reported as $\\operatorname{ECE}(x)$.", + expectedSourceNotation: "\\operatorname{ECE}", + expectedSymbol: "ECE", + family: "application", + id: "ece-application-edge", + probes: [ + { delta: 0, id: "surface-start", needle: "\\operatorname{ECE}(x)" }, + { delta: 21, id: "application-after", needle: "\\operatorname{ECE}(x)" }, + ], + }, + { + content: "\\DeclareMathOperator{\\ECE}{ECE}\nExpected calibration error (ECE) is reported as $\\ECE(x)$.", + expectedSourceNotation: "\\ECE", + expectedSymbol: "ECE", + family: "declared-surface", + id: "declared-ece", + probes: [ + { delta: 1, id: "call-start", needle: "$\\ECE(x)" }, + { delta: 5, id: "call-after", needle: "$\\ECE(x)" }, + { delta: 8, id: "application-after", needle: "$\\ECE(x)" }, + ], + }, + { + content: "\\newcommand{\\prediction}[1]{\\hat{#1}}\nLet $y$ denote the prediction. Use $\\prediction{y}$.", + expectedSourceNotation: "\\prediction{y}", + expectedSymbol: "y", + family: "macro-expansion", + id: "prediction-macro", + probes: [ + { delta: 0, id: "call-start", needle: "\\prediction{y}" }, + { delta: 12, id: "argument-start", needle: "\\prediction{y}" }, + { delta: 14, id: "call-after", needle: "\\prediction{y}" }, + ], + }, + { + content: "Let $A$ and $B$ denote events. Use $p=\\frac{\\mathbb{P}(A \\cap B)}{\\mathbb{P}(B)}$.", + expectedSourceNotation: "A", + expectedSymbol: "A", + family: "fraction", + id: "fraction-event", + probes: [ + { delta: 0, id: "symbol-start", needle: "A \\cap B" }, + { delta: 1, id: "symbol-after", needle: "A \\cap B" }, + ], + }, +] as const; + +/** Plans exact UTF-16 probes from reviewed neutral notation surfaces. */ +export function planCursorInvariantSurfaces(): readonly CursorInvariantSurface[] { + const surfaces = SEEDS.map((seed) => ({ + content: seed.content, + expectedSourceNotation: seed.expectedSourceNotation, + expectedSymbol: seed.expectedSymbol, + family: seed.family, + fileId: `cursor-${seed.id}`, + id: seed.id, + path: `cursor-${seed.id}.tex`, + probes: seed.probes.map((probe) => { + const start = uniqueNeedleOffset(seed.content, probe.needle, `${seed.id}/${probe.id}`); + const offset = start + probe.delta; + if (offset < start || offset > start + probe.needle.length) { + throw new Error(`${seed.id}/${probe.id}: cursor is outside its reviewed surface`); + } + return { id: probe.id, offset }; + }), + })); + const families = new Set(surfaces.map((surface) => surface.family)); + for (const family of CURSOR_INVARIANT_FAMILIES) { + if (!families.has(family)) throw new Error(`cursor invariant plan is missing ${family}`); + } + const ids = surfaces.flatMap((surface) => + surface.probes.map((probe) => `${surface.id}/${probe.id}`), + ); + if (new Set(ids).size !== ids.length) throw new Error("cursor invariant probes must be unique"); + return surfaces; +} + +function uniqueNeedleOffset(content: string, needle: string, id: string): number { + const first = content.indexOf(needle); + if (first < 0 || first !== content.lastIndexOf(needle)) { + throw new Error(`${id}: probe needle must occur exactly once`); + } + return first; +} diff --git a/packages/evaluation/src/differential.test.ts b/packages/evaluation/src/differential.test.ts index a8a2864..7ec90ed 100644 --- a/packages/evaluation/src/differential.test.ts +++ b/packages/evaluation/src/differential.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test"; import { firstDifferentialFailure, planSemanticEditTrace, + planSemanticLifecycleTraces, shrinkEditTrace, } from "./differential"; @@ -14,6 +15,24 @@ describe("semantic differential planning", () => { ]); }); + test("plans every evidence lifecycle with establishment, retraction, and recovery", () => { + const traces = planSemanticLifecycleTraces(20); + expect(traces).toEqual(planSemanticLifecycleTraces(20)); + expect(traces.map((trace) => trace.family)).toEqual([ + "declaration-retraction", + "include-order", + "macro-retraction", + "malformed-recovery", + "polarity-retraction", + "typed-conflict-recovery", + ]); + for (const trace of traces) { + expect(trace.initialExpectedDecision).toBe("established"); + expect(trace.stages.at(-1)?.expectedDecision).toBe("established"); + expect(trace.stages.some((stage) => stage.expectedDecision !== "established")).toBe(true); + } + }); + test("reports the first divergent stage and exact field", () => { const shared = { decision: { status: "established" }, range: { endOffset: 4, startOffset: 3 } }; expect( diff --git a/packages/evaluation/src/differential.ts b/packages/evaluation/src/differential.ts index 11b268a..5f11320 100644 --- a/packages/evaluation/src/differential.ts +++ b/packages/evaluation/src/differential.ts @@ -23,6 +23,40 @@ export interface EditTrace { readonly steps: readonly EditTraceStep[]; } +export const SEMANTIC_LIFECYCLE_FAMILIES = [ + "declaration-retraction", + "include-order", + "macro-retraction", + "malformed-recovery", + "polarity-retraction", + "typed-conflict-recovery", +] as const; + +export type SemanticLifecycleFamily = (typeof SEMANTIC_LIFECYCLE_FAMILIES)[number]; + +export interface SemanticLifecycleDocument { + readonly content: string; + readonly fileId: string; + readonly path: string; +} + +export interface SemanticLifecycleStage { + readonly changes: readonly EditTraceStep[]; + readonly expectedDecision: "conflicting" | "established" | "not-established"; + readonly id: string; + readonly queryNeedle?: string; +} + +export interface SemanticLifecycleTrace { + readonly family: SemanticLifecycleFamily; + readonly id: string; + readonly initialDocuments: readonly SemanticLifecycleDocument[]; + readonly initialExpectedDecision: SemanticLifecycleStage["expectedDecision"]; + readonly query: { readonly fileId: string; readonly needle: string }; + readonly seed: number; + readonly stages: readonly SemanticLifecycleStage[]; +} + /** A deterministic edit history that exercises assertion, conflict, retraction and recovery. */ export function planSemanticEditTrace(seed: number): EditTrace { if (!Number.isSafeInteger(seed)) throw new Error("trace seed must be an integer"); @@ -43,6 +77,157 @@ export function planSemanticEditTrace(seed: number): EditTrace { }; } +/** + * Plans independent semantic lifecycles without consulting engine output. The + * cases establish evidence, remove or contradict it, then recover it so an + * executor can compare every incremental stage with a clean rebuild. + */ +export function planSemanticLifecycleTraces(seed: number): readonly SemanticLifecycleTrace[] { + if (!Number.isSafeInteger(seed)) throw new Error("trace seed must be an integer"); + const suffix = Math.abs(seed % 10_000); + const probabilityDefinitions = + "Let $A$ and $B$ be events in the same probability space."; + const probabilityMain = "\\input{definitions}\n$A \\cap B$."; + const localProbability = `${probabilityDefinitions}\n$A \\cap B$.`; + const traces: SemanticLifecycleTrace[] = [ + { + family: "declaration-retraction", + id: `lifecycle-${suffix}-declaration`, + initialDocuments: [ + { content: probabilityMain, fileId: "main", path: "main.tex" }, + { content: probabilityDefinitions, fileId: "definitions", path: "definitions.tex" }, + ], + initialExpectedDecision: "established", + query: { fileId: "main", needle: "A \\cap B" }, + seed, + stages: [ + { + changes: [{ fileId: "definitions", kind: "remove" }], + expectedDecision: "not-established", + id: "remove-evidence", + }, + { + changes: [{ content: probabilityDefinitions, fileId: "definitions", kind: "upsert", path: "definitions.tex" }], + expectedDecision: "established", + id: "restore-evidence", + }, + ], + }, + { + family: "include-order", + id: `lifecycle-${suffix}-include-order`, + initialDocuments: [ + { content: probabilityMain, fileId: "main", path: "main.tex" }, + { content: probabilityDefinitions, fileId: "definitions", path: "definitions.tex" }, + ], + initialExpectedDecision: "established", + query: { fileId: "main", needle: "A \\cap B" }, + seed, + stages: [ + { + changes: [{ content: "$A \\cap B$.\n\\input{definitions}", fileId: "main", kind: "upsert", path: "main.tex" }], + expectedDecision: "not-established", + id: "move-evidence-after-use", + }, + { + changes: [{ content: probabilityMain, fileId: "main", kind: "upsert", path: "main.tex" }], + expectedDecision: "established", + id: "restore-include-order", + }, + ], + }, + { + family: "macro-retraction", + id: `lifecycle-${suffix}-macro`, + initialDocuments: [ + { content: "\\newcommand{\\joint}[2]{#1 \\cap #2}", fileId: "macros", path: "macros.tex" }, + { content: `\\input{macros}\n${probabilityDefinitions}\n$\\joint{A}{B}$.`, fileId: "main", path: "main.tex" }, + ], + initialExpectedDecision: "established", + query: { fileId: "main", needle: "\\joint{A}{B}" }, + seed, + stages: [ + { + changes: [{ fileId: "macros", kind: "remove" }], + expectedDecision: "not-established", + id: "remove-macro-definition", + }, + { + changes: [{ content: "\\newcommand{\\joint}[2]{#1 \\cap #2}", fileId: "macros", kind: "upsert", path: "macros.tex" }], + expectedDecision: "established", + id: "restore-macro-definition", + }, + ], + }, + { + family: "malformed-recovery", + id: `lifecycle-${suffix}-malformed`, + initialDocuments: [{ content: localProbability, fileId: "main", path: "main.tex" }], + initialExpectedDecision: "established", + query: { fileId: "main", needle: "A \\cap B" }, + seed, + stages: [ + { + changes: [{ content: `${probabilityDefinitions}\n$A \\cap$.`, fileId: "main", kind: "upsert", path: "main.tex" }], + expectedDecision: "not-established", + id: "break-expression", + queryNeedle: "A \\cap", + }, + { + changes: [{ content: localProbability, fileId: "main", kind: "upsert", path: "main.tex" }], + expectedDecision: "established", + id: "repair-expression", + }, + ], + }, + { + family: "polarity-retraction", + id: `lifecycle-${suffix}-polarity`, + initialDocuments: [{ content: localProbability, fileId: "main", path: "main.tex" }], + initialExpectedDecision: "established", + query: { fileId: "main", needle: "A \\cap B" }, + seed, + stages: [ + { + changes: [{ content: "A and B might be events in the same probability space.\n$A \\cap B$.", fileId: "main", kind: "upsert", path: "main.tex" }], + expectedDecision: "not-established", + id: "hedge-declaration", + }, + { + changes: [{ content: localProbability, fileId: "main", kind: "upsert", path: "main.tex" }], + expectedDecision: "established", + id: "restore-assertion", + }, + ], + }, + { + family: "typed-conflict-recovery", + id: `lifecycle-${suffix}-conflict`, + initialDocuments: [{ content: localProbability, fileId: "main", path: "main.tex" }], + initialExpectedDecision: "established", + query: { fileId: "main", needle: "A \\cap B" }, + seed, + stages: [ + { + changes: [{ content: "Let $e$ be kinetic energy, $Z$ mass, and $k$ speed.\n$e=Zk$.", fileId: "main", kind: "upsert", path: "main.tex" }], + expectedDecision: "conflicting", + id: "introduce-explicit-conflict", + queryNeedle: "e=Zk", + }, + { + changes: [{ content: localProbability, fileId: "main", kind: "upsert", path: "main.tex" }], + expectedDecision: "established", + id: "remove-conflict", + }, + ], + }, + ]; + if (new Set(traces.map((trace) => trace.family)).size !== SEMANTIC_LIFECYCLE_FAMILIES.length) { + throw new Error("lifecycle trace plan is missing a required family"); + } + return traces; +} + export function firstDifferentialFailure( stages: readonly DifferentialStage[], ): DifferentialFailure | undefined { diff --git a/packages/evaluation/src/index.ts b/packages/evaluation/src/index.ts index 10a7400..1e9f7fa 100644 --- a/packages/evaluation/src/index.ts +++ b/packages/evaluation/src/index.ts @@ -1,6 +1,7 @@ export * from "./conformance"; export * from "./differential"; export * from "./challenge"; +export * from "./cursor-invariants"; export * from "./foundation"; export * from "./legacy-path-audit"; export * from "./metamorphic"; diff --git a/scripts/check-budget.ts b/scripts/check-budget.ts index 850742c..e1ef3f8 100644 --- a/scripts/check-budget.ts +++ b/scripts/check-budget.ts @@ -24,6 +24,11 @@ import { type PerformanceFixtureDocument, } from "./performance-fixtures"; import { shouldEnforceTiming } from "./performance-budget-policy"; +import { + planSemanticEditTrace, + planSemanticLifecycleTraces, + shrinkEditTrace, +} from "../packages/evaluation/src/differential"; const DOCUMENT_COUNT = positiveInteger("SEMATH_BUDGET_DOCUMENTS", 60); const STABLE_HOST_GATE = process.env.SEMATH_BUDGET_STABLE === "1"; @@ -269,6 +274,15 @@ const queryP95ByKind = Object.fromEntries( [...queryDurations].map(([kind, durations]) => [kind, percentile(durations, 0.95)]), ); const queryP95 = Math.max(...Object.values(queryP95ByKind)); +const shrinkSource = planSemanticEditTrace(0x5e_21); +let failureShrinkEvaluations = 0; +const shrunkFailure = shrinkEditTrace(shrinkSource, (candidate) => { + failureShrinkEvaluations += 1; + return candidate.steps.some((step) => step.content?.includes("matrix")); +}); +if (failureShrinkEvaluations > shrinkSource.steps.length || shrunkFailure.steps.length !== 1) { + throw new Error("budget failure shrinking exceeded deterministic linear work"); +} const peakRssGrowth = Math.max(0, Math.max(peakRss, rssAfterDispose) - rssBefore); const syntaxStats = syntax.getStats() as ReturnType & { lastInvalidatedDocuments?: number; @@ -286,10 +300,17 @@ const report = { deltaP95Ms: deltaP95, documents: DOCUMENT_COUNT + 1, engineColdMs, + failureShrink: { + evaluations: failureShrinkEvaluations, + inputSteps: shrinkSource.steps.length, + outputSteps: shrunkFailure.steps.length, + }, fixtureFamilies: [...new Set(sources.map((source) => source.family))], initialTransferBytes, peakRssGrowthBytes: peakRssGrowth, queryP95ByKind, + semanticViewP95Ms: queryP95ByKind.semanticView ?? null, + lifecycleFamilies: planSemanticLifecycleTraces(0x5e_21).map((trace) => trace.family), retainedRssGrowthBytes: retainedRssGrowth, rssGrowthByStage: { engineBytes: Math.max(0, rssAfterEngine - rssAfterSyntax), diff --git a/scripts/check-challenge.ts b/scripts/check-challenge.ts index 53bb47d..d3245e4 100644 --- a/scripts/check-challenge.ts +++ b/scripts/check-challenge.ts @@ -1,46 +1,105 @@ import { readFile } from "node:fs/promises"; import { parseChallengeCorpus, + parseChallengeV3, scoreChallenge, type ChallengeObservation, } from "../packages/evaluation/src/index"; import { runSemanticEvaluation } from "./semantic-evaluation-runner"; -const corpus = parseChallengeCorpus( +const base = JSON.parse( + await readFile( + new URL("../fixtures/challenge/recognition-v2.json", import.meta.url), + "utf8", + ), +); +const corpus = parseChallengeV3( + base, JSON.parse( await readFile( - new URL("../fixtures/challenge/recognition-v2.json", import.meta.url), + new URL("../fixtures/challenge/recognition-v3.json", import.meta.url), "utf8", ), ), ); -const results = runSemanticEvaluation(corpus.cases, "recognition-challenge-v2"); +const results = runSemanticEvaluation(corpus.cases, "recognition-challenge-v3"); const debugIds = new Set(process.env.SEMATH_CHALLENGE_DEBUG?.split(",") ?? []); const observations = corpus.cases.map((item, index): ChallengeObservation => { const result = results[index]; - const view = result?.value.kind === "semanticView" ? result.value.view : undefined; + const view = + result?.value.kind === "semanticView" ? result.value.view : undefined; + const known = + view?.decision.status === "established" || + view?.decision.status === "partial" + ? view.decision.meaning + : undefined; + const reasons = view?.decision.reasons ?? []; + const groundingReasons = reasons.filter( + (reason) => reason.kind === "proof" || reason.kind === "source-conflict", + ); + const meaningRelation = known?.relationId + ? view?.context.relations.find( + (relation) => relation.relationId === known.relationId, + ) + : undefined; const observation: ChallengeObservation = { - assumptionValues: (view?.context.assumptions ?? []).map((entry) => entry.value), + assumptionValues: (view?.context.assumptions ?? []).map( + (entry) => entry.value, + ), candidates: (view?.context.candidates ?? []).map((entry) => ({ family: entry.family, interpretation: entry.interpretation, })), caseId: item.id, - conceptIds: [...new Set((view?.context.concepts ?? []).map((entry) => entry.conceptId))], + conceptIds: [ + ...new Set( + (view?.context.concepts ?? []).map((entry) => entry.conceptId), + ), + ], definitions: (view?.symbol?.definitions ?? []).map((entry) => ({ description: entry.description, ruleId: entry.evidence.ruleId, symbol: entry.symbol, })), - relationIds: [...new Set((view?.context.relations ?? []).map((entry) => entry.relationId))], - shapes: [...new Set((view?.symbol?.shapes ?? []).map((entry) => entry.display))], + ...(known?.label ? { meaningLabel: known.label } : {}), + ...(known ? { meaningRelationId: known.relationId } : {}), + problemCount: + (view?.decision.status === "conflicting" + ? view.decision.conflicts.length + : 0) + + (view?.diagnostics ?? []).filter( + (diagnostic) => + diagnostic.severity === "error" || diagnostic.severity === "warning", + ).length, + reasonKinds: reasons.map((reason) => reason.kind), + relationIds: [ + ...new Set( + (view?.context.relations ?? []).map((entry) => entry.relationId), + ), + ], + shapes: [ + ...new Set((view?.symbol?.shapes ?? []).map((entry) => entry.display)), + ], ...(view?.symbol?.sourceNotation ? { sourceNotation: view.symbol.sourceNotation } : {}), + sourceGrounded: + (groundingReasons.length > 0 && + groundingReasons.every((reason) => + reason.evidence.some( + (evidence) => evidence.sourceRanges.length > 0, + ), + )) || + Boolean( + meaningRelation?.evidence.some( + (evidence) => evidence.sourceRanges.length > 0, + ), + ), ...(view ? { status: view.decision.status } : {}), symbols: [...new Set(view?.symbol ? [view.symbol.symbol] : [])], }; - if (debugIds.has(item.id)) console.error(JSON.stringify({ item, observation, view }, null, 2)); + if (debugIds.has(item.id)) + console.error(JSON.stringify({ item, observation, view }, null, 2)); return observation; }); const scorecard = scoreChallenge(corpus, observations); @@ -50,6 +109,12 @@ console.log( .map(([key, value]) => `${key}:${value.passed}/${value.total}`) .join(",")} ` + `metrics=${Object.entries(scorecard.metrics) + .map(([key, value]) => `${key}:${value.passed}/${value.total}`) + .join(",")} ` + + `decisions=${Object.entries(scorecard.decisions) + .map(([key, value]) => `${key}:${value.passed}/${value.total}`) + .join(",")} ` + + `problems=${Object.entries(scorecard.problemPolicy) .map(([key, value]) => `${key}:${value.passed}/${value.total}`) .join(",")}`, ); @@ -59,6 +124,11 @@ if (process.env.SEMATH_CHALLENGE_REPORT) { `${JSON.stringify(scorecard, null, 2)}\n`, ); } -if (scorecard.failures.length && process.env.SEMATH_CHALLENGE_ALLOW_FAILURES !== "1") { - throw new Error(`recognition challenge failed:\n${scorecard.failures.join("\n")}`); +if ( + scorecard.failures.length && + process.env.SEMATH_CHALLENGE_ALLOW_FAILURES !== "1" +) { + throw new Error( + `recognition challenge failed:\n${scorecard.failures.join("\n")}`, + ); } diff --git a/scripts/check-lifecycle.mjs b/scripts/check-lifecycle.mjs new file mode 100644 index 0000000..0308a4c --- /dev/null +++ b/scripts/check-lifecycle.mjs @@ -0,0 +1,200 @@ +import { readFile } from "node:fs/promises"; +import init, { SemathEngine } from "../lib/wasm/semath_wasm.js"; +import { LatexSyntaxService } from "wasmtex/syntax"; +import { adaptWasmtexDocument } from "../packages/wasmtex-adapter/src/index.ts"; +import { + firstDifferentialFailure, + planSemanticLifecycleTraces, +} from "../packages/evaluation/src/differential.ts"; +import { SEMATH_PROTOCOL_VERSION } from "../packages/protocol/src/index.ts"; + +await init({ + module_or_path: await readFile( + new URL("../lib/wasm/semath_wasm_bg.wasm", import.meta.url), + ), +}); + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); +const planned = planSemanticLifecycleTraces(0x5e_21); +const traces = process.env.SEMATH_LIFECYCLE_FULL === "1" ? planned : planned.slice(0, 2); +let comparedStages = 0; + +for (const trace of traces) { + const sources = new Map( + trace.initialDocuments.map((document) => [ + document.fileId, + { ...document, documentVersion: 1, language: "latex" }, + ]), + ); + const syntax = new LatexSyntaxService(); + syntax.reset({ documents: [...sources.values()] }); + let inventoryVersion = 1; + let analysisGeneration = 0; + const engine = new SemathEngine(); + resetEngine(engine, snapshotFrom(sources, syntax, inventoryVersion)); + assertDecision( + queryEngine(engine, trace.query, sources, inventoryVersion, analysisGeneration), + trace.initialExpectedDecision, + `${trace.id}/initial`, + ); + + for (const stage of trace.stages) { + inventoryVersion += 1; + analysisGeneration += 1; + const explicitChanges = []; + for (const change of stage.changes) { + if (change.kind === "upsert") { + const previous = sources.get(change.fileId); + const source = { + content: change.content ?? previous?.content ?? "", + documentVersion: (previous?.documentVersion ?? 0) + 1, + fileId: change.fileId, + language: "latex", + path: change.path ?? previous?.path ?? `${change.fileId}.tex`, + }; + sources.set(change.fileId, source); + syntax.upsert(source); + } else if (change.kind === "remove") { + sources.delete(change.fileId); + syntax.remove(change.fileId); + explicitChanges.push({ fileId: change.fileId, kind: "remove" }); + } else { + const previous = sources.get(change.fileId); + if (!previous || !change.path) throw new Error(`${trace.id}: invalid path change`); + const source = { ...previous, path: change.path }; + sources.set(change.fileId, source); + syntax.move(change.fileId, change.path); + explicitChanges.push({ fileId: change.fileId, kind: "path-change", path: change.path }); + } + } + const changedIds = new Set(explicitChanges.map((change) => change.fileId)); + const upserts = syntax.getInvalidatedFiles().flatMap((fileSyntax) => { + if (changedIds.has(fileSyntax.fileId)) return []; + const source = sources.get(fileSyntax.fileId); + if (!source) return []; + return [{ + document: adaptWasmtexDocument({ + content: source.content, + language: "latex", + syntax: fileSyntax, + }), + kind: "upsert", + }]; + }); + decode( + engine.applyChanges( + encode({ + analysisGeneration, + changes: [...explicitChanges, ...upserts], + epoch: "semantic-lifecycle", + inventoryVersion, + protocolVersion: SEMATH_PROTOCOL_VERSION, + }), + ), + ); + + const query = { + ...trace.query, + ...(stage.queryNeedle ? { needle: stage.queryNeedle } : {}), + }; + const incremental = queryEngine( + engine, + query, + sources, + inventoryVersion, + analysisGeneration, + ); + const clean = new SemathEngine(); + const cleanSyntax = new LatexSyntaxService(); + cleanSyntax.reset({ documents: [...sources.values()] }); + resetEngine(clean, snapshotFrom(sources, cleanSyntax, inventoryVersion)); + const rebuilt = queryEngine( + clean, + query, + sources, + inventoryVersion, + analysisGeneration, + ); + const failure = firstDifferentialFailure([ + { name: "clean", value: rebuilt.value }, + { name: "incremental", value: incremental.value }, + ]); + if (failure) { + throw new Error( + `${trace.id}/${stage.id}: ${failure.stage} diverged at ${failure.path}\n` + + `expected=${JSON.stringify(failure.expected)}\nactual=${JSON.stringify(failure.actual)}`, + ); + } + assertDecision(incremental, stage.expectedDecision, `${trace.id}/${stage.id}`); + clean.free(); + comparedStages += 1; + } + engine.free(); +} + +console.log( + `lifecycle OK: ${traces.length}/${planned.length} traces, ${comparedStages} clean/incremental stages`, +); + +function snapshotFrom(sources, syntax, inventoryVersion) { + return { + documents: [...sources.values()].map((source) => { + const fileSyntax = syntax.getFile(source.fileId); + if (!fileSyntax) throw new Error(`missing syntax for ${source.fileId}`); + return adaptWasmtexDocument({ + content: source.content, + language: "latex", + syntax: fileSyntax, + }); + }), + epoch: "semantic-lifecycle", + inventoryVersion, + mainFileId: sources.has("main") ? "main" : undefined, + projectId: "semantic-lifecycle", + protocolVersion: SEMATH_PROTOCOL_VERSION, + }; +} + +function queryEngine(engine, target, sources, inventoryVersion, analysisGeneration) { + const source = sources.get(target.fileId); + if (!source) throw new Error(`missing query document ${target.fileId}`); + const first = source.content.indexOf(target.needle); + if (first < 0 || first !== source.content.lastIndexOf(target.needle)) { + throw new Error(`query needle must occur exactly once: ${target.needle}`); + } + return decode( + engine.query( + encode({ + analysisGeneration, + documentVersion: source.documentVersion, + epoch: "semantic-lifecycle", + inventoryVersion, + protocolVersion: SEMATH_PROTOCOL_VERSION, + query: { fileId: target.fileId, kind: "semanticView", offset: first }, + }), + ), + ); +} + +function assertDecision(result, expected, label) { + const value = result?.value; + const status = value?.kind === "semanticView" ? value.view.decision.status : "missing"; + const matches = expected === "not-established" ? status !== "established" : status === expected; + if (!matches) throw new Error(`${label}: expected ${expected}, observed ${status}`); +} + +function resetEngine(engine, snapshot) { + const { documents, ...metadata } = snapshot; + engine.beginReset(encode(metadata)); + for (const document of documents) engine.ingestResetDocument(encode(document)); + return decode(engine.finishReset()); +} + +function encode(value) { + return encoder.encode(JSON.stringify(value)); +} + +function decode(value) { + return JSON.parse(decoder.decode(value)); +} diff --git a/scripts/check-parity.mjs b/scripts/check-parity.mjs index 7261097..87944b3 100644 --- a/scripts/check-parity.mjs +++ b/scripts/check-parity.mjs @@ -4,9 +4,10 @@ import init, { SemathEngine } from "../lib/wasm/semath_wasm.js"; import { LatexSyntaxService } from "wasmtex/syntax"; import { adaptWasmtexDocument } from "../packages/wasmtex-adapter/src/index.ts"; import { firstDifferentialFailure } from "../packages/evaluation/src/differential.ts"; +import { planCursorInvariantSurfaces } from "../packages/evaluation/src/cursor-invariants.ts"; import { SEMATH_PROTOCOL_VERSION } from "../packages/protocol/src/index.ts"; -const sources = [ +const baseSources = [ { content: [ "\\newcommand{\\Both}[2]{#1 \\cap #2}", @@ -47,10 +48,21 @@ const sources = [ path: "unsupported.tex", }, ]; +const cursorSurfaces = planCursorInvariantSurfaces(); +const sources = [ + ...baseSources, + ...cursorSurfaces.map((surface) => ({ + content: surface.content, + documentVersion: 1, + fileId: surface.fileId, + language: "latex", + path: surface.path, + })), +]; const snapshot = makeSnapshot(sources, 1); const probabilityOccurrence = sources[1].content.indexOf("A \\cap"); -const queries = [ +const baseQueries = [ query("probability", probabilityOccurrence), query("probability", probabilityOccurrence + 1), query("unsupported", sources[3].content.indexOf("q =") + 1), @@ -58,6 +70,17 @@ const queries = [ definitionQuery("probability", probabilityOccurrence), definitionQuery("probability", probabilityOccurrence + 1), ]; +const cursorRequests = cursorSurfaces.flatMap((surface) => + surface.probes.flatMap((probe) => + ["semanticView", "definition", "references", "prepareRename"].map((kind) => ({ + envelope: cursorQuery(surface.fileId, probe.offset, kind), + kind, + probe, + surface, + })), + ), +); +const queries = [...baseQueries, ...cursorRequests.map((request) => request.envelope)]; const fixture = { queries, snapshot }; const build = spawnSync("cargo", ["build", "--locked", "-p", "semath-native"], { @@ -87,6 +110,7 @@ assertEquivalent([ { name: "clean", value: wasmResults[0].value }, { name: "incremental", value: wasmResults[1].value }, ], "cursor-edge semantic identity"); +assertCursorInvariants(cursorRequests, wasmResults.slice(baseQueries.length)); if (reset.stats.totalDocuments !== sources.length || reset.stats.semanticNodes <= 0) { throw new Error("parity reset did not expose trustworthy analysis counters"); } @@ -188,6 +212,13 @@ function definitionQuery(fileId, offset) { return { ...query(fileId, offset), query: { fileId, kind: "definition", offset } }; } +function cursorQuery(fileId, offset, kind) { + return { + ...query(fileId, offset), + query: { fileId, kind, offset }, + }; +} + function engineQuery(target, value) { return target.query(encode(value)); } @@ -215,3 +246,50 @@ function assertEquivalent(stages, label) { ); } } + +function assertCursorInvariants(requests, results) { + const grouped = new Map(); + for (const [index, request] of requests.entries()) { + const key = `${request.surface.id}/${request.kind}`; + const values = grouped.get(key) ?? []; + values.push({ request, result: results[index] }); + grouped.set(key, values); + } + for (const [key, entries] of grouped) { + if (entries[0].request.kind === "semanticView") { + const identities = entries.map(({ request, result }) => { + const value = result?.value; + const symbol = value?.kind === "semanticView" ? value.view.symbol : undefined; + if ( + symbol?.sourceNotation !== request.surface.expectedSourceNotation || + symbol.symbol !== request.surface.expectedSymbol + ) { + throw new Error( + `${key}/${request.probe.id}: expected ${request.surface.expectedSourceNotation}/${request.surface.expectedSymbol}, ` + + `observed ${symbol?.sourceNotation ?? "none"}/${symbol?.symbol ?? "none"}`, + ); + } + return { + entityId: symbol.entityId ?? null, + occurrenceId: symbol.occurrenceId, + sourceNotation: symbol.sourceNotation, + symbol: symbol.symbol, + }; + }); + for (const identity of identities.slice(1)) { + assertEquivalent([ + { name: "clean", value: identities[0] }, + { name: "incremental", value: identity }, + ], `${key} semantic occurrence`); + } + continue; + } + const values = entries.map(({ result }) => result?.value); + for (const value of values.slice(1)) { + assertEquivalent([ + { name: "clean", value: values[0] }, + { name: "incremental", value }, + ], `${key} navigation result`); + } + } +} diff --git a/scripts/performance-fixtures.ts b/scripts/performance-fixtures.ts index c338c52..62d8fa2 100644 --- a/scripts/performance-fixtures.ts +++ b/scripts/performance-fixtures.ts @@ -7,6 +7,8 @@ export const PERFORMANCE_FIXTURE_FAMILIES = [ "unicode-and-combining", "malformed-recovery", "binder-and-rename", + "document-shaped-report", + "scoped-neighbor", ] as const; export type PerformanceFixtureFamily = (typeof PERFORMANCE_FIXTURE_FAMILIES)[number]; @@ -82,5 +84,22 @@ function fixtureBody(family: PerformanceFixtureFamily, index: number, symbol: st return `\$${symbol}=\\frac{\\hat{x_${index}}{\\left(y+z\\right.\$`; case "binder-and-rename": return `\$${symbol}=\\sum_{k=1}^{n} a_k+\\int_0^1 f(t)\\,dt\$`; + case "document-shaped-report": + return [ + "\\section{Background}", + "The surrounding experiment reports several independent measurements before the calibrated result.", + "$\\xi_{\\mathrm{aux}}=17$", + "\\section{Reported result}", + `\$${symbol}=\\operatorname{ECE}=\\sum_{m=1}^{M}\\frac{|B_m|}{n}\\left|\\operatorname{acc}(B_m)-\\operatorname{conf}(B_m)\\right|\$`, + "$\\zeta_{\\mathrm{aux}}=19$", + ].join("\n"); + case "scoped-neighbor": + return [ + "\\section{Independent notation}", + "$\\mathbf{q}_{\\mathrm{aux}}=\\frac{1}{2}$", + "\\section{Current result}", + `\$${symbol}=\\widehat{y}_{t+1}+\\operatorname{loss}(x)\$`, + "$\\frac{1}{$", + ].join("\n"); } }