diff --git a/docs/pack-maturity.md b/docs/pack-maturity.md index 95be518..17449b0 100644 --- a/docs/pack-maturity.md +++ b/docs/pack-maturity.md @@ -21,8 +21,10 @@ holds approved support policy; GitHub issues hold planned work. Electromagnetism, thermodynamics/heat transfer, fluid mechanics, calculus, discrete mathematics, and optimization remain deliberately narrow slices; evaluated capability evidence is not a field-completeness claim. -- The independent frozen challenge passes 24 of 24 cases across binding, - constraints, packs, presentation, resolution, and syntax. +- The independent frozen challenge v2 contains 48 cases across binding, + constraints, packs, presentation, resolution, and syntax. Its initial + hard-cutover baseline passes 41 of 48; the seven exposed gaps are assigned to + issues #189 and #191 rather than hidden by fixture or threshold changes. These numbers describe the synthetic benchmark, not real-world prevalence or field completeness. diff --git a/docs/public-api.md b/docs/public-api.md index c8d44cf..7324e45 100644 --- a/docs/public-api.md +++ b/docs/public-api.md @@ -1,6 +1,6 @@ # Public API -Semath is a library and language-service runtime. Protocol 4 is a deliberate +Semath is a library and language-service runtime. Protocol 6 is a deliberate hard cutover to a small meaning-first API. | Export | Responsibility | @@ -33,7 +33,7 @@ refusal explanation when appropriate. `context.assumptions` contains explicit, source-linked assumptions with their subjects; omission means none were established. Parser ASTs and legacy rewrite queries are not public. -Protocol 4 identifies every `RoleInfo` by its open, pack-qualified `conceptId`. +Protocol 6 identifies every `RoleInfo` by its open, pack-qualified `conceptId`. There is no closed role enum or unnamespaced compatibility field. Included-file role, shape, and quantity facts use the same records and retain their original evidence. diff --git a/docs/semantic-quality-scorecards.md b/docs/semantic-quality-scorecards.md index 34722f0..d51c7e9 100644 --- a/docs/semantic-quality-scorecards.md +++ b/docs/semantic-quality-scorecards.md @@ -33,10 +33,12 @@ assumption, evidence, refusal, and scope independently instead of blending them into one recognition number. The frozen recognition challenge is separate from those development and -diversity fixtures. Its cases are manually authored and grouped by the layer -that owns a failure: syntax, binding, constraint, pack, resolution, or -presentation. Association, structure, constraint, recognition, evidence, -refusal, scope, and navigation remain separate metrics. Default CI validates +diversity fixtures. Version 2 has 48 manually authored cases, including twelve +positive/refusal semantic boundary pairs, grouped by the layer that owns a +failure: syntax, binding, constraint, pack, resolution, or presentation. Exact +IDs and normalized document sources must not occur in development or foundation +fixtures. Association, structure, constraint, recognition, evidence, refusal, +scope, and navigation remain separate metrics. Default CI validates the challenge schema and coverage matrix through pure tests but does not execute the engine over the holdout. Evaluated laws require diff --git a/fixtures/challenge/recognition-v1.json b/fixtures/challenge/recognition-v1.json deleted file mode 100644 index fe97227..0000000 --- a/fixtures/challenge/recognition-v1.json +++ /dev/null @@ -1,251 +0,0 @@ -{ - "schemaVersion": 1, - "cases": [ - { - "id": "binding-metric-long-short", - "owner": "binding", - "metric": "association", - "outcome": "positive", - "documents": [{ "fileId": "main", "path": "main.md", "content": "The balanced accuracy score (BAS) summarizes the classifier. We report $\\operatorname{BAS}=0.8$." }], - "cursor": { "fileId": "main", "needle": "\\operatorname{BAS}", "edge": "after" }, - "expectation": { "symbol": "BAS", "definitionDescription": "balanced accuracy score" }, - "variationTags": ["acronym", "english-context", "named-operator"] - }, - { - "id": "binding-hedged-name-refusal", - "owner": "binding", - "metric": "refusal", - "outcome": "refusal", - "documents": [{ "fileId": "main", "path": "main.md", "content": "BAS might stand for balanced accuracy score. The table contains $\\operatorname{BAS}$." }], - "cursor": { "fileId": "main", "needle": "\\operatorname{BAS}", "edge": "before" }, - "expectation": { "excludedDefinitionSymbol": "BAS" }, - "variationTags": ["acronym", "hedged", "safe-refusal"] - }, - { - "id": "binding-respectively-circuit", - "owner": "binding", - "metric": "recognition", - "outcome": "positive", - "documents": [{ "fileId": "main", "path": "main.tex", "content": "Let $u$, $r$, and $j$ denote voltage, resistance, and electric current, respectively. Hence $u=rj$." }], - "cursor": { "fileId": "main", "needle": "u=rj", "edge": "after" }, - "expectation": { "relationId": "circuits:ohm-law" }, - "variationTags": ["respectively", "symbol-renaming", "typed"] - }, - { - "id": "binding-role-collision-refusal", - "owner": "binding", - "metric": "constraint", - "outcome": "refusal", - "documents": [{ "fileId": "main", "path": "main.tex", "content": "Let $u$ be voltage, $r$ resistance, and $j$ a differentiable function. Consider $u=rj$." }], - "cursor": { "fileId": "main", "needle": "u=rj", "edge": "before" }, - "expectation": { "excludedRelationId": "circuits:ohm-law" }, - "variationTags": ["role-conflict", "safe-refusal", "typed"] - }, - { - "id": "constraint-matrix-vector-compatible", - "owner": "constraint", - "metric": "constraint", - "outcome": "positive", - "documents": [{ "fileId": "main", "path": "main.tex", "content": "Let $H$ be an m by n matrix, $q$ an n-dimensional vector, and $z$ an m-dimensional vector. Then $z=Hq$." }], - "cursor": { "fileId": "main", "needle": "z=Hq", "edge": "after" }, - "expectation": { "relationId": "linear-algebra:matrix-vector-product" }, - "variationTags": ["explicit-shapes", "matrix-vector", "typed"] - }, - { - "id": "constraint-matrix-vector-mismatch", - "owner": "constraint", - "metric": "refusal", - "outcome": "refusal", - "documents": [{ "fileId": "main", "path": "main.tex", "content": "Let $H$ be an m by n matrix, $q$ a k-dimensional vector with $k\\ne n$, and $z$ an m-dimensional vector. Consider $z=Hq$." }], - "cursor": { "fileId": "main", "needle": "z=Hq", "edge": "before" }, - "expectation": { "excludedRelationId": "linear-algebra:matrix-vector-product" }, - "variationTags": ["dimension-mismatch", "safe-refusal", "wrong-shape"] - }, - { - "id": "constraint-explicit-vector-shape", - "owner": "constraint", - "metric": "evidence", - "outcome": "positive", - "documents": [{ "fileId": "main", "path": "main.md", "content": "Let $w$ be a p-dimensional vector. The current iterate is $w$." }], - "cursor": { "fileId": "main", "needle": "iterate is $w", "edge": "after" }, - "expectation": { "symbol": "w", "shape": "Vector[p]" }, - "variationTags": ["explicit-shapes", "role-prose", "vector-notation"] - }, - { - "id": "constraint-incompatible-role-refusal", - "owner": "constraint", - "metric": "recognition", - "outcome": "refusal", - "documents": [{ "fileId": "main", "path": "main.tex", "content": "Let $A$ and $B$ be ordinary sets rather than events. The expression is $A\\cap B$." }], - "cursor": { "fileId": "main", "needle": "A\\cap B", "edge": "after" }, - "expectation": { "excludedRelationId": "probability:event-intersection" }, - "variationTags": ["notation-collision", "role-conflict", "safe-refusal"] - }, - { - "id": "pack-unfamiliar-ohm-symbols", - "owner": "pack", - "metric": "recognition", - "outcome": "positive", - "documents": [{ "fileId": "main", "path": "main.md", "content": "In the resistor model, $e$ is voltage, $Z$ is resistance, and $k$ is electric current. Therefore $e=Zk$." }], - "cursor": { "fileId": "main", "needle": "e=Zk", "edge": "before" }, - "expectation": { "relationId": "circuits:ohm-law" }, - "variationTags": ["english-context", "symbol-renaming", "typed"] - }, - { - "id": "pack-cross-field-product-refusal", - "owner": "pack", - "metric": "refusal", - "outcome": "refusal", - "documents": [{ "fileId": "main", "path": "main.tex", "content": "Let $e$ be kinetic energy, $Z$ mass, and $k$ speed. The proposed relation is $e=Zk$." }], - "cursor": { "fileId": "main", "needle": "e=Zk", "edge": "after" }, - "expectation": { "excludedRelationId": "circuits:ohm-law" }, - "variationTags": ["cross-field", "role-mismatch", "safe-refusal"] - }, - { - "id": "pack-event-intersection", - "owner": "pack", - "metric": "association", - "outcome": "positive", - "documents": [{ "fileId": "main", "path": "main.tex", "content": "Events $C$ and $D$ belong to one probability space. Their joint event is $D\\cap C$." }], - "cursor": { "fileId": "main", "needle": "D\\cap C", "edge": "before" }, - "expectation": { "relationId": "probability:event-intersection" }, - "variationTags": ["event", "reordered-roles", "typed"] - }, - { - "id": "pack-set-event-collision", - "owner": "pack", - "metric": "evidence", - "outcome": "refusal", - "documents": [{ "fileId": "main", "path": "main.md", "content": "The symbols $C$ and $D$ name subsets of a graph vertex set, not random events. We use $C\\cap D$." }], - "cursor": { "fileId": "main", "needle": "C\\cap D", "edge": "after" }, - "expectation": { "excludedRelationId": "probability:event-intersection" }, - "variationTags": ["cross-pack", "notation-collision", "safe-refusal"] - }, - { - "id": "presentation-decorated-source", - "owner": "presentation", - "metric": "structure", - "outcome": "positive", - "documents": [{ "fileId": "main", "path": "main.tex", "content": "Compare the observation $y$ with its estimate $\\widehat{y}$." }], - "cursor": { "fileId": "main", "needle": "\\widehat{y}", "edge": "after" }, - "expectation": { "symbol": "y", "sourceNotation": "\\widehat{y}", "candidateFamily": "decoration", "candidateInterpretation": "estimate" }, - "variationTags": ["decorated", "source-preserving", "widehat"] - }, - { - "id": "presentation-plain-not-decorated", - "owner": "presentation", - "metric": "structure", - "outcome": "refusal", - "documents": [{ "fileId": "main", "path": "main.tex", "content": "The plain observation is $y$, while an estimate is written elsewhere." }], - "cursor": { "fileId": "main", "needle": "is $y", "edge": "after" }, - "expectation": { "symbol": "y", "sourceNotation": "y" }, - "variationTags": ["plain-symbol", "safe-separation", "source-preserving"] - }, - { - "id": "presentation-named-surface", - "owner": "presentation", - "metric": "navigation", - "outcome": "positive", - "documents": [{ "fileId": "main", "path": "main.tex", "content": "\\DeclareMathOperator{\\FPR}{FPR}\nThe result is $\\FPR(x)$." }], - "cursor": { "fileId": "main", "needle": "\\FPR(x)", "edge": "before" }, - "expectation": { "symbol": "FPR", "sourceNotation": "\\FPR", "definitionDescription": "FPR" }, - "variationTags": ["declaration", "named-operator", "source-preserving"] - }, - { - "id": "presentation-opaque-command-refusal", - "owner": "presentation", - "metric": "refusal", - "outcome": "refusal", - "documents": [{ "fileId": "main", "path": "main.tex", "content": "An unresolved notation is $\\mysterymetric{x}$." }], - "cursor": { "fileId": "main", "needle": "\\mysterymetric{x}", "edge": "before" }, - "expectation": { "excludedDefinitionSymbol": "\\mysterymetric" }, - "variationTags": ["opaque-macro", "safe-refusal", "unknown-command"] - }, - { - "id": "resolution-resource-navigation", - "owner": "resolution", - "metric": "navigation", - "outcome": "positive", - "documents": [{ "fileId": "main", "path": "main.tex", "content": "\\newacronym{auprc}{AUPRC}{area under the precision recall curve}\nWe report $\\operatorname{AUPRC}$." }], - "cursor": { "fileId": "main", "needle": "\\operatorname{AUPRC}", "edge": "after" }, - "expectation": { "symbol": "AUPRC", "definitionDescription": "area under the precision recall curve", "definitionRuleId": "latex-acronym-declaration" }, - "variationTags": ["acronym-resource", "evidence", "navigation"] - }, - { - "id": "resolution-future-definition-refusal", - "owner": "resolution", - "metric": "navigation", - "outcome": "refusal", - "documents": [{ "fileId": "main", "path": "main.md", "content": "First we use $\\operatorname{AUPRC}$. Later, area under the precision recall curve (AUPRC) is introduced." }], - "cursor": { "fileId": "main", "needle": "\\operatorname{AUPRC}", "edge": "before" }, - "expectation": { "excludedDefinitionSymbol": "AUPRC" }, - "variationTags": ["forward-scope", "safe-refusal", "source-order"] - }, - { - "id": "resolution-included-definition", - "owner": "resolution", - "metric": "scope", - "outcome": "positive", - "documents": [ - { "fileId": "main", "path": "main.tex", "content": "\\input{terms}\nThe estimate is $s$." }, - { "fileId": "terms", "path": "terms.tex", "content": "Let $s$ denote the latent system state." } - ], - "cursor": { "fileId": "main", "needle": "estimate is $s", "edge": "after" }, - "expectation": { "symbol": "s", "definitionDescription": "the latent system state" }, - "variationTags": ["included-declarations", "multi-file", "project-scope"] - }, - { - "id": "resolution-disconnected-refusal", - "owner": "resolution", - "metric": "scope", - "outcome": "refusal", - "documents": [ - { "fileId": "main", "path": "main.tex", "content": "The estimate is $s$." }, - { "fileId": "unused", "path": "unused.tex", "content": "Let $s$ denote the latent system state." } - ], - "cursor": { "fileId": "main", "needle": "estimate is $s", "edge": "after" }, - "expectation": { "excludedDefinitionSymbol": "s" }, - "variationTags": ["safe-refusal", "unreachable-file", "unresolved-scope"] - }, - { - "id": "syntax-partial-candidate", - "owner": "syntax", - "metric": "structure", - "outcome": "positive", - "documents": [{ "fileId": "main", "path": "main.tex", "content": "Consider the formal expression $\\partial f/\\partial x$." }], - "cursor": { "fileId": "main", "needle": "\\partial f/", "edge": "before" }, - "expectation": { "candidateFamily": "differential", "candidateInterpretation": "derivative" }, - "variationTags": ["calculus", "partial-derivative", "structural-candidate"] - }, - { - "id": "syntax-unknown-differential-refusal", - "owner": "syntax", - "metric": "refusal", - "outcome": "refusal", - "documents": [{ "fileId": "main", "path": "main.tex", "content": "The opaque form is $\\unknownpartial f/\\unknownpartial x$." }], - "cursor": { "fileId": "main", "needle": "\\unknownpartial f", "edge": "before" }, - "expectation": { "excludedDefinitionSymbol": "\\unknownpartial" }, - "variationTags": ["opaque-macro", "safe-refusal", "unknown-command"] - }, - { - "id": "syntax-integral-binder", - "owner": "syntax", - "metric": "constraint", - "outcome": "positive", - "documents": [{ "fileId": "main", "path": "main.tex", "content": "The accumulated quantity is $\\int_0^1 g(t)\\,dt$." }], - "cursor": { "fileId": "main", "needle": "\\int", "edge": "after" }, - "expectation": { "candidateFamily": "binder", "candidateInterpretation": "binder" }, - "variationTags": ["binder", "integral", "limits"] - }, - { - "id": "syntax-index-family", - "owner": "syntax", - "metric": "evidence", - "outcome": "refusal", - "documents": [{ "fileId": "main", "path": "main.tex", "content": "Without a declaration, $a_{ij}$ remains an indexed surface rather than a guessed tensor." }], - "cursor": { "fileId": "main", "needle": "a_{ij}", "edge": "after" }, - "expectation": { "candidateFamily": "script", "candidateInterpretation": "index", "excludedConceptId": "linear-algebra:tensor" }, - "variationTags": ["indexed-symbols", "safe-refusal", "structural-candidate"] - } - ] -} diff --git a/fixtures/challenge/recognition-v2.json b/fixtures/challenge/recognition-v2.json new file mode 100644 index 0000000..e5b9141 --- /dev/null +++ b/fixtures/challenge/recognition-v2.json @@ -0,0 +1,497 @@ +{ + "schemaVersion": 2, + "cases": [ + { + "id": "binding-metric-long-short", + "owner": "binding", + "metric": "association", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.md", "content": "The balanced accuracy score (BAS) summarizes the classifier. We report $\\operatorname{BAS}=0.8$." }], + "cursor": { "fileId": "main", "needle": "\\operatorname{BAS}", "edge": "after" }, + "expectation": { "symbol": "BAS", "definitionDescription": "balanced accuracy score" }, + "variationTags": ["acronym", "english-context", "named-operator"] + }, + { + "id": "binding-hedged-name-refusal", + "owner": "binding", + "metric": "refusal", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.md", "content": "BAS might stand for balanced accuracy score. The table contains $\\operatorname{BAS}$." }], + "cursor": { "fileId": "main", "needle": "\\operatorname{BAS}", "edge": "before" }, + "expectation": { "excludedDefinitionSymbol": "BAS" }, + "variationTags": ["acronym", "hedged", "safe-refusal"] + }, + { + "id": "binding-respectively-circuit", + "owner": "binding", + "metric": "recognition", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "Let $u$, $r$, and $j$ denote voltage, resistance, and electric current, respectively. Hence $u=rj$." }], + "cursor": { "fileId": "main", "needle": "u=rj", "edge": "after" }, + "expectation": { "relationId": "circuits:ohm-law" }, + "variationTags": ["respectively", "symbol-renaming", "typed"] + }, + { + "id": "binding-role-collision-refusal", + "owner": "binding", + "metric": "constraint", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "Let $u$ be voltage, $r$ resistance, and $j$ a differentiable function. Consider $u=rj$." }], + "cursor": { "fileId": "main", "needle": "u=rj", "edge": "before" }, + "expectation": { "excludedRelationId": "circuits:ohm-law" }, + "variationTags": ["role-conflict", "safe-refusal", "typed"] + }, + { + "id": "constraint-matrix-vector-compatible", + "owner": "constraint", + "metric": "constraint", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "Let $H$ be an m by n matrix, $q$ an n-dimensional vector, and $z$ an m-dimensional vector. Then $z=Hq$." }], + "cursor": { "fileId": "main", "needle": "z=Hq", "edge": "after" }, + "expectation": { "relationId": "linear-algebra:matrix-vector-product" }, + "variationTags": ["explicit-shapes", "matrix-vector", "typed"] + }, + { + "id": "constraint-matrix-vector-mismatch", + "owner": "constraint", + "metric": "refusal", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "Let $H$ be an m by n matrix, $q$ a k-dimensional vector with $k\\ne n$, and $z$ an m-dimensional vector. Consider $z=Hq$." }], + "cursor": { "fileId": "main", "needle": "z=Hq", "edge": "before" }, + "expectation": { "excludedRelationId": "linear-algebra:matrix-vector-product" }, + "variationTags": ["dimension-mismatch", "safe-refusal", "wrong-shape"] + }, + { + "id": "constraint-explicit-vector-shape", + "owner": "constraint", + "metric": "evidence", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.md", "content": "Let $w$ be a p-dimensional vector. The current iterate is $w$." }], + "cursor": { "fileId": "main", "needle": "iterate is $w", "edge": "after" }, + "expectation": { "symbol": "w", "shape": "Vector[p]" }, + "variationTags": ["explicit-shapes", "role-prose", "vector-notation"] + }, + { + "id": "constraint-incompatible-role-refusal", + "owner": "constraint", + "metric": "recognition", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "Let $A$ and $B$ be ordinary sets rather than events. The expression is $A\\cap B$." }], + "cursor": { "fileId": "main", "needle": "A\\cap B", "edge": "after" }, + "expectation": { "excludedRelationId": "probability:event-intersection" }, + "variationTags": ["notation-collision", "role-conflict", "safe-refusal"] + }, + { + "id": "pack-unfamiliar-ohm-symbols", + "owner": "pack", + "metric": "recognition", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.md", "content": "In the resistor model, $e$ is voltage, $Z$ is resistance, and $k$ is electric current. Therefore $e=Zk$." }], + "cursor": { "fileId": "main", "needle": "e=Zk", "edge": "before" }, + "expectation": { "relationId": "circuits:ohm-law" }, + "variationTags": ["english-context", "symbol-renaming", "typed"] + }, + { + "id": "pack-cross-field-product-refusal", + "owner": "pack", + "metric": "refusal", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "Let $e$ be kinetic energy, $Z$ mass, and $k$ speed. The proposed relation is $e=Zk$." }], + "cursor": { "fileId": "main", "needle": "e=Zk", "edge": "after" }, + "expectation": { "excludedRelationId": "circuits:ohm-law" }, + "variationTags": ["cross-field", "role-mismatch", "safe-refusal"] + }, + { + "id": "pack-event-intersection", + "owner": "pack", + "metric": "association", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "Events $C$ and $D$ belong to one probability space. Their joint event is $D\\cap C$." }], + "cursor": { "fileId": "main", "needle": "D\\cap C", "edge": "before" }, + "expectation": { "relationId": "probability:event-intersection" }, + "variationTags": ["event", "reordered-roles", "typed"] + }, + { + "id": "pack-set-event-collision", + "owner": "pack", + "metric": "evidence", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.md", "content": "The symbols $C$ and $D$ name subsets of a graph vertex set, not random events. We use $C\\cap D$." }], + "cursor": { "fileId": "main", "needle": "C\\cap D", "edge": "after" }, + "expectation": { "excludedRelationId": "probability:event-intersection" }, + "variationTags": ["cross-pack", "notation-collision", "safe-refusal"] + }, + { + "id": "presentation-decorated-source", + "owner": "presentation", + "metric": "structure", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "Compare the observation $y$ with its estimate $\\widehat{y}$." }], + "cursor": { "fileId": "main", "needle": "\\widehat{y}", "edge": "after" }, + "expectation": { "symbol": "y", "sourceNotation": "\\widehat{y}", "candidateFamily": "decoration", "candidateInterpretation": "estimate" }, + "variationTags": ["decorated", "source-preserving", "widehat"] + }, + { + "id": "presentation-plain-not-decorated", + "owner": "presentation", + "metric": "structure", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "The plain observation is $y$, while an estimate is written elsewhere." }], + "cursor": { "fileId": "main", "needle": "is $y", "edge": "after" }, + "expectation": { "symbol": "y", "sourceNotation": "y" }, + "variationTags": ["plain-symbol", "safe-separation", "source-preserving"] + }, + { + "id": "presentation-named-surface", + "owner": "presentation", + "metric": "navigation", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "\\DeclareMathOperator{\\FPR}{FPR}\nThe result is $\\FPR(x)$." }], + "cursor": { "fileId": "main", "needle": "\\FPR(x)", "edge": "before" }, + "expectation": { "symbol": "FPR", "sourceNotation": "\\FPR", "definitionDescription": "FPR" }, + "variationTags": ["declaration", "named-operator", "source-preserving"] + }, + { + "id": "presentation-opaque-command-refusal", + "owner": "presentation", + "metric": "refusal", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "An unresolved notation is $\\mysterymetric{x}$." }], + "cursor": { "fileId": "main", "needle": "\\mysterymetric{x}", "edge": "before" }, + "expectation": { "excludedDefinitionSymbol": "\\mysterymetric" }, + "variationTags": ["opaque-macro", "safe-refusal", "unknown-command"] + }, + { + "id": "resolution-resource-navigation", + "owner": "resolution", + "metric": "navigation", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "\\newacronym{auprc}{AUPRC}{area under the precision recall curve}\nWe report $\\operatorname{AUPRC}$." }], + "cursor": { "fileId": "main", "needle": "\\operatorname{AUPRC}", "edge": "after" }, + "expectation": { "symbol": "AUPRC", "definitionDescription": "area under the precision recall curve", "definitionRuleId": "latex-acronym-declaration" }, + "variationTags": ["acronym-resource", "evidence", "navigation"] + }, + { + "id": "resolution-future-definition-refusal", + "owner": "resolution", + "metric": "navigation", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.md", "content": "First we use $\\operatorname{AUPRC}$. Later, area under the precision recall curve (AUPRC) is introduced." }], + "cursor": { "fileId": "main", "needle": "\\operatorname{AUPRC}", "edge": "before" }, + "expectation": { "excludedDefinitionSymbol": "AUPRC" }, + "variationTags": ["forward-scope", "safe-refusal", "source-order"] + }, + { + "id": "resolution-included-definition", + "owner": "resolution", + "metric": "scope", + "outcome": "positive", + "documents": [ + { "fileId": "main", "path": "main.tex", "content": "\\input{terms}\nThe estimate is $s$." }, + { "fileId": "terms", "path": "terms.tex", "content": "Let $s$ denote the latent system state." } + ], + "cursor": { "fileId": "main", "needle": "estimate is $s", "edge": "after" }, + "expectation": { "symbol": "s", "definitionDescription": "the latent system state" }, + "variationTags": ["included-declarations", "multi-file", "project-scope"] + }, + { + "id": "resolution-disconnected-refusal", + "owner": "resolution", + "metric": "scope", + "outcome": "refusal", + "documents": [ + { "fileId": "main", "path": "main.tex", "content": "The estimate is $s$." }, + { "fileId": "unused", "path": "unused.tex", "content": "Let $s$ denote the latent system state." } + ], + "cursor": { "fileId": "main", "needle": "estimate is $s", "edge": "after" }, + "expectation": { "excludedDefinitionSymbol": "s" }, + "variationTags": ["safe-refusal", "unreachable-file", "unresolved-scope"] + }, + { + "id": "syntax-partial-candidate", + "owner": "syntax", + "metric": "structure", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "Consider the formal expression $\\partial f/\\partial x$." }], + "cursor": { "fileId": "main", "needle": "\\partial f/", "edge": "before" }, + "expectation": { "candidateFamily": "differential", "candidateInterpretation": "derivative" }, + "variationTags": ["calculus", "partial-derivative", "structural-candidate"] + }, + { + "id": "syntax-unknown-differential-refusal", + "owner": "syntax", + "metric": "refusal", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "The opaque form is $\\unknownpartial f/\\unknownpartial x$." }], + "cursor": { "fileId": "main", "needle": "\\unknownpartial f", "edge": "before" }, + "expectation": { "excludedDefinitionSymbol": "\\unknownpartial" }, + "variationTags": ["opaque-macro", "safe-refusal", "unknown-command"] + }, + { + "id": "syntax-integral-binder", + "owner": "syntax", + "metric": "constraint", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "The accumulated quantity is $\\int_0^1 g(t)\\,dt$." }], + "cursor": { "fileId": "main", "needle": "\\int", "edge": "after" }, + "expectation": { "candidateFamily": "binder", "candidateInterpretation": "binder" }, + "variationTags": ["binder", "integral", "limits"] + }, + { + "id": "syntax-index-family", + "owner": "syntax", + "metric": "evidence", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "Without a declaration, $a_{ij}$ remains an indexed surface rather than a guessed tensor." }], + "cursor": { "fileId": "main", "needle": "a_{ij}", "edge": "after" }, + "expectation": { "candidateFamily": "script", "candidateInterpretation": "index", "excludedConceptId": "linear-algebra:tensor" }, + "variationTags": ["indexed-symbols", "safe-refusal", "structural-candidate"] + }, + { + "id": "binding-asserted-ece-name", + "owner": "binding", + "metric": "association", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.md", "content": "Expected calibration error (ECE) summarizes calibration. We report $\\operatorname{ECE}$ below." }], + "cursor": { "fileId": "main", "needle": "\\operatorname{ECE}", "edge": "after" }, + "expectation": { "symbol": "ECE", "definitionDescription": "Expected calibration error" }, + "variationTags": ["acronym", "asserted", "boundary-pair:asserted-quoted-name"] + }, + { + "id": "binding-quoted-ece-name-refusal", + "owner": "binding", + "metric": "refusal", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.md", "content": "The survey says, \"ECE stands for expected calibration error.\" We report $\\operatorname{ECE}$ below." }], + "cursor": { "fileId": "main", "needle": "\\operatorname{ECE}", "edge": "before" }, + "expectation": { "excludedDefinitionSymbol": "ECE" }, + "variationTags": ["acronym", "quoted", "boundary-pair:asserted-quoted-name"] + }, + { + "id": "binding-shared-matrix-declaration", + "owner": "binding", + "metric": "association", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "Matrices $P$ and $Q$ share one coordinate basis. We inspect $Q$." }], + "cursor": { "fileId": "main", "needle": "inspect $Q", "edge": "after" }, + "expectation": { "symbol": "Q", "definitionDescription": "Matrices" }, + "variationTags": ["fronted-plural", "shared-description", "boundary-pair:asserted-alternative-declaration"] + }, + { + "id": "binding-alternative-matrix-refusal", + "owner": "binding", + "metric": "refusal", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "Either $P$ or $Q$ may denote the matrix used later. We inspect $Q$." }], + "cursor": { "fileId": "main", "needle": "inspect $Q", "edge": "before" }, + "expectation": { "excludedDefinitionSymbol": "Q" }, + "variationTags": ["alternative", "hedged", "boundary-pair:asserted-alternative-declaration"] + }, + { + "id": "constraint-events-same-context", + "owner": "constraint", + "metric": "constraint", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "Events $A$ and $B$ belong to the same probability space. Their joint event is $A\\cap B$." }], + "cursor": { "fileId": "main", "needle": "A\\cap B", "edge": "after" }, + "expectation": { "relationId": "probability:event-intersection" }, + "variationTags": ["same-context", "typed", "boundary-pair:same-different-context"] + }, + { + "id": "constraint-events-different-context-refusal", + "owner": "constraint", + "metric": "refusal", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "Event $A$ belongs to the first probability space, while event $B$ belongs to a different experiment. The formal surface is $A\\cap B$." }], + "cursor": { "fileId": "main", "needle": "A\\cap B", "edge": "before" }, + "expectation": { "excludedRelationId": "probability:event-intersection" }, + "variationTags": ["different-context", "safe-refusal", "boundary-pair:same-different-context"] + }, + { + "id": "constraint-passive-sign-asserted", + "owner": "constraint", + "metric": "evidence", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "Let $i$ be electric current, $C$ capacitance, and $v$ voltage. Under the passive sign convention, $i=C\\frac{dv}{dt}$." }], + "cursor": { "fileId": "main", "needle": "i=C", "edge": "after" }, + "expectation": { "relationId": "circuits:capacitor-current-law" }, + "variationTags": ["condition", "sign-convention", "boundary-pair:asserted-negated-condition"] + }, + { + "id": "constraint-passive-sign-negated-refusal", + "owner": "constraint", + "metric": "refusal", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "Let $i$ be electric current, $C$ capacitance, and $v$ voltage. Without adopting the passive sign convention, consider $i=C\\frac{dv}{dt}$." }], + "cursor": { "fileId": "main", "needle": "i=C", "edge": "before" }, + "expectation": { "excludedRelationId": "circuits:capacitor-current-law" }, + "variationTags": ["condition", "negated", "boundary-pair:asserted-negated-condition"] + }, + { + "id": "pack-electric-force-typed", + "owner": "pack", + "metric": "recognition", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "Let $R$ denote force, $c$ electric charge, and $G$ electric field. Therefore $R=cG$." }], + "cursor": { "fileId": "main", "needle": "R=cG", "edge": "after" }, + "expectation": { "relationId": "electromagnetism:electric-force-law" }, + "variationTags": ["electromagnetism", "symbol-renaming", "boundary-pair:typed-cross-field-force"] + }, + { + "id": "pack-electric-force-cross-field-refusal", + "owner": "pack", + "metric": "refusal", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "Let $R$ denote momentum, $c$ mass, and $G$ velocity. Therefore $R=cG$." }], + "cursor": { "fileId": "main", "needle": "R=cG", "edge": "before" }, + "expectation": { "excludedRelationId": "electromagnetism:electric-force-law" }, + "variationTags": ["cross-field", "mechanics", "boundary-pair:typed-cross-field-force"] + }, + { + "id": "pack-sensible-heat-typed", + "owner": "pack", + "metric": "recognition", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "Let $h$ be heat transfer, $m$ mass, $s$ specific heat, and $d$ temperature change. Then $h=msd$." }], + "cursor": { "fileId": "main", "needle": "h=msd", "edge": "after" }, + "expectation": { "relationId": "thermodynamics-heat-transfer:sensible-heat-relation" }, + "variationTags": ["thermal", "four-role", "boundary-pair:typed-untyped-heat"] + }, + { + "id": "pack-sensible-heat-untyped-refusal", + "owner": "pack", + "metric": "refusal", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "The four otherwise undeclared symbols appear in the formal product $h=msd$." }], + "cursor": { "fileId": "main", "needle": "h=msd", "edge": "before" }, + "expectation": { "excludedRelationId": "thermodynamics-heat-transfer:sensible-heat-relation" }, + "variationTags": ["untyped", "open-set", "boundary-pair:typed-untyped-heat"] + }, + { + "id": "presentation-hat-source-preserved", + "owner": "presentation", + "metric": "structure", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "The estimator returns $\\hat y$ for observation $y$." }], + "cursor": { "fileId": "main", "needle": "\\hat y", "edge": "after" }, + "expectation": { "symbol": "y", "sourceNotation": "\\hat y", "candidateFamily": "decoration", "candidateInterpretation": "estimate" }, + "variationTags": ["hat", "source-preserving", "boundary-pair:decorated-styled-surface"] + }, + { + "id": "presentation-style-not-decoration", + "owner": "presentation", + "metric": "structure", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "The bold display $\\mathbf{y}$ is typographic emphasis, not an estimator." }], + "cursor": { "fileId": "main", "needle": "\\mathbf{y}", "edge": "before" }, + "expectation": { "symbol": "y", "sourceNotation": "\\mathbf{y}", "excludedCandidateFamily": "decoration" }, + "variationTags": ["style", "safe-separation", "boundary-pair:decorated-styled-surface"] + }, + { + "id": "presentation-declared-ece-command", + "owner": "presentation", + "metric": "navigation", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "\\DeclareMathOperator{\\ECE}{ECE}\nWe report $\\ECE(p)$ for the model." }], + "cursor": { "fileId": "main", "needle": "\\ECE(p)", "edge": "after" }, + "expectation": { "symbol": "ECE", "sourceNotation": "\\ECE", "definitionDescription": "ECE" }, + "variationTags": ["named-operator", "declaration", "boundary-pair:named-plain-surface"] + }, + { + "id": "presentation-plain-ece-not-command", + "owner": "presentation", + "metric": "refusal", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "\\DeclareMathOperator{\\ECE}{ECE}\nThe unrelated product is $ECE$." }], + "cursor": { "fileId": "main", "needle": "$ECE", "edge": "after" }, + "expectation": { "excludedDefinitionSymbol": "ECE" }, + "variationTags": ["juxtaposition", "exact-notation-key", "boundary-pair:named-plain-surface"] + }, + { + "id": "resolution-prior-section-binding", + "owner": "resolution", + "metric": "scope", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.md", "content": "## Calibration\nExpected calibration error (ECE) measures calibration. We report $\\operatorname{ECE}$." }], + "cursor": { "fileId": "main", "needle": "\\operatorname{ECE}", "edge": "after" }, + "expectation": { "definitionDescription": "Expected calibration error", "symbol": "ECE" }, + "variationTags": ["section-scope", "prior-binding", "boundary-pair:visible-sibling-binding"] + }, + { + "id": "resolution-sibling-section-refusal", + "owner": "resolution", + "metric": "scope", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.md", "content": "## Calibration\nExpected calibration error (ECE) measures calibration.\n## Energy\nWe report $\\operatorname{ECE}$ as an unrelated label." }], + "cursor": { "fileId": "main", "needle": "\\operatorname{ECE}", "edge": "before" }, + "expectation": { "excludedDefinitionSymbol": "ECE" }, + "variationTags": ["sibling-scope", "safe-refusal", "boundary-pair:visible-sibling-binding"] + }, + { + "id": "resolution-prior-include-binding", + "owner": "resolution", + "metric": "navigation", + "outcome": "positive", + "documents": [ + { "fileId": "main", "path": "main.tex", "content": "\\input{terms}\nThe chosen variable is $z$." }, + { "fileId": "terms", "path": "terms.tex", "content": "Let $z$ denote the latent coordinate." } + ], + "cursor": { "fileId": "main", "needle": "variable is $z", "edge": "after" }, + "expectation": { "symbol": "z", "definitionDescription": "the latent coordinate" }, + "variationTags": ["include-order", "source-order", "boundary-pair:prior-later-include"] + }, + { + "id": "resolution-later-include-refusal", + "owner": "resolution", + "metric": "navigation", + "outcome": "refusal", + "documents": [ + { "fileId": "main", "path": "main.tex", "content": "The chosen variable is $z$.\n\\input{terms}" }, + { "fileId": "terms", "path": "terms.tex", "content": "Let $z$ denote the latent coordinate." } + ], + "cursor": { "fileId": "main", "needle": "variable is $z", "edge": "before" }, + "expectation": { "excludedDefinitionSymbol": "z" }, + "variationTags": ["include-order", "forward-scope", "boundary-pair:prior-later-include"] + }, + { + "id": "syntax-second-partial-structure", + "owner": "syntax", + "metric": "structure", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "The curvature term is $\\frac{\\partial^2 f}{\\partial x^2}$." }], + "cursor": { "fileId": "main", "needle": "\\partial^2 f", "edge": "after" }, + "expectation": { "candidateFamily": "differential", "candidateInterpretation": "derivative" }, + "variationTags": ["partial-derivative", "higher-order", "boundary-pair:structural-opaque-derivative"] + }, + { + "id": "syntax-opaque-second-partial-refusal", + "owner": "syntax", + "metric": "refusal", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "The unresolved surface is $\\frac{\\mysterypartial^2 f}{\\mysterypartial x^2}$." }], + "cursor": { "fileId": "main", "needle": "\\mysterypartial^2 f", "edge": "before" }, + "expectation": { "excludedCandidateFamily": "differential", "excludedDefinitionSymbol": "\\mysterypartial" }, + "variationTags": ["opaque-macro", "unknown-command", "boundary-pair:structural-opaque-derivative"] + }, + { + "id": "syntax-complete-sum-binder", + "owner": "syntax", + "metric": "constraint", + "outcome": "positive", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "The aggregate is $\\sum_{i=1}^{n} a_i$." }], + "cursor": { "fileId": "main", "needle": "\\sum", "edge": "after" }, + "expectation": { "candidateFamily": "binder", "candidateInterpretation": "binder" }, + "variationTags": ["sum", "binder", "boundary-pair:complete-malformed-binder"] + }, + { + "id": "syntax-malformed-sum-refusal", + "owner": "syntax", + "metric": "refusal", + "outcome": "refusal", + "documents": [{ "fileId": "main", "path": "main.tex", "content": "The incomplete edit is $\\sum_{i=1 a_i$." }], + "cursor": { "fileId": "main", "needle": "\\sum", "edge": "before" }, + "expectation": { "excludedCandidateFamily": "binder" }, + "variationTags": ["malformed-recovery", "safe-refusal", "boundary-pair:complete-malformed-binder"] + } + ] +} diff --git a/package.json b/package.json index c1cfc13..cbe5d86 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,8 @@ "foundation": "bun scripts/check-foundation.ts", "foundation:generate": "bun scripts/generate-foundation-corpus.ts", "foundation:generate:check": "bun scripts/generate-foundation-corpus.ts --check", + "legacy:audit": "bun scripts/audit-legacy-semantic-paths.ts", + "legacy:audit:check": "bun scripts/audit-legacy-semantic-paths.ts --check", "notation:conformance": "bun test packages/evaluation/src/notation-conformance.test.ts", "package:smoke": "node scripts/check-package-smoke.mjs", "pack:authoring": "bun packages/authoring/cli.mjs validate packs/*/v1.json && bun packages/authoring/cli.mjs audit-runtime packs/*/v1.json", diff --git a/packages/evaluation/src/challenge.test.ts b/packages/evaluation/src/challenge.test.ts index 774ae03..14cd8df 100644 --- a/packages/evaluation/src/challenge.test.ts +++ b/packages/evaluation/src/challenge.test.ts @@ -3,14 +3,16 @@ import { readFile } from "node:fs/promises"; import { CHALLENGE_LAYERS, CHALLENGE_METRICS, + findChallengeFixtureLeaks, parseChallengeCorpus, scoreChallenge, type ChallengeCase, type ChallengeObservation, + type DevelopmentFixtureCase, } from "./challenge"; function cases(): ChallengeCase[] { - return Array.from({ length: 24 }, (_, index) => ({ + return Array.from({ length: 48 }, (_, index) => ({ cursor: { fileId: "main", needle: "$x$" }, documents: [{ content: "$x$", fileId: "main", path: "main.tex" }], expectation: index % 2 === 0 ? { symbol: "x" } : { excludedRelationId: "wrong" }, @@ -18,27 +20,41 @@ function cases(): ChallengeCase[] { metric: CHALLENGE_METRICS[index % CHALLENGE_METRICS.length]!, outcome: index % 2 === 0 ? "positive" : "refusal", owner: CHALLENGE_LAYERS[Math.floor(index / 2) % CHALLENGE_LAYERS.length]!, - variationTags: [`variation-${index}`], + variationTags: [`variation-${index}`, `boundary-pair:pair-${Math.floor(index / 2)}`], })); } function source(value: readonly ChallengeCase[]) { - return { cases: value, schemaVersion: 1 }; + return { cases: value, schemaVersion: 2 }; } describe("independent recognition challenge", () => { test("keeps the checked-in holdout strict and coverage-complete", async () => { const fixture: unknown = JSON.parse( await readFile( - new URL("../../../fixtures/challenge/recognition-v1.json", import.meta.url), + new URL("../../../fixtures/challenge/recognition-v2.json", import.meta.url), "utf8", ), ); - expect(parseChallengeCorpus(fixture).cases).toHaveLength(24); + const challenge = parseChallengeCorpus(fixture); + expect(challenge.cases).toHaveLength(48); + + const development: DevelopmentFixtureCase[] = []; + const glob = new Bun.Glob("{corpus,foundation}/*.json"); + for await (const path of glob.scan({ + cwd: new URL("../../../fixtures", import.meta.url).pathname, + onlyFiles: true, + })) { + const suite: unknown = JSON.parse( + await readFile(new URL(`../../../fixtures/${path}`, import.meta.url), "utf8"), + ); + development.push(...parseDevelopmentCases(suite)); + } + expect(findChallengeFixtureLeaks(challenge.cases, development)).toEqual([]); }); test("parses a strict frozen matrix with every layer, outcome, and metric", () => { - expect(parseChallengeCorpus(source(cases())).cases).toHaveLength(24); + expect(parseChallengeCorpus(source(cases())).cases).toHaveLength(48); expect(() => parseChallengeCorpus({ ...source(cases()), compatibilityMode: true }), ).toThrow("unknown field compatibilityMode"); @@ -59,10 +75,10 @@ describe("independent recognition challenge", () => { })); observations[0] = { ...observations[0]!, symbols: [] }; const scorecard = scoreChallenge(corpus, observations); - expect(scorecard.passed).toBe(23); + expect(scorecard.passed).toBe(47); expect(scorecard.failures).toEqual(["case-0: missing symbol x"]); - expect(scorecard.layers.binding).toEqual({ passed: 3, total: 4 }); - expect(scorecard.outcomes.refusal).toEqual({ passed: 12, total: 12 }); + expect(scorecard.layers.binding).toEqual({ passed: 7, total: 8 }); + expect(scorecard.outcomes.refusal).toEqual({ passed: 24, total: 24 }); }); test("rejects ambiguous cursors and incomplete coverage", () => { @@ -73,6 +89,40 @@ describe("independent recognition challenge", () => { documents: [{ content: "$x+x$", fileId: "main", path: "main.tex" }], }; expect(() => parseChallengeCorpus(source(values))).toThrow("must occur exactly once"); - expect(() => parseChallengeCorpus(source(cases().slice(0, 23)))).toThrow("at least 24"); + expect(() => parseChallengeCorpus(source(cases().slice(0, 47)))).toThrow("at least 48"); + }); + + test("requires complete semantic boundary pairs and rejects development-fixture reuse", () => { + const values = cases(); + values[0] = { ...values[0]!, variationTags: ["boundary-pair:unpaired"] }; + expect(() => parseChallengeCorpus(source(values))).toThrow("incomplete boundary pair unpaired"); + + const challenge = cases().slice(0, 1); + expect(findChallengeFixtureLeaks(challenge, challenge)).toEqual([ + "case-0: duplicate fixture id", + "case-0: duplicate fixture source", + ]); }); }); + +function parseDevelopmentCases(value: unknown): DevelopmentFixtureCase[] { + if (!isRecord(value) || !Array.isArray(value.cases)) return []; + return value.cases.flatMap((item) => { + if (!isRecord(item) || typeof item.id !== "string" || !Array.isArray(item.documents)) { + return []; + } + const documents = item.documents.flatMap((document) => + isRecord(document) && + typeof document.content === "string" && + typeof document.fileId === "string" && + typeof document.path === "string" + ? [{ content: document.content, fileId: document.fileId, path: document.path }] + : [], + ); + return documents.length === item.documents.length ? [{ documents, id: item.id }] : []; + }); +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} diff --git a/packages/evaluation/src/challenge.ts b/packages/evaluation/src/challenge.ts index fd31871..136bbf3 100644 --- a/packages/evaluation/src/challenge.ts +++ b/packages/evaluation/src/challenge.ts @@ -31,6 +31,7 @@ export interface ChallengeExpectation { readonly definitionDescription?: string; readonly definitionRuleId?: string; readonly excludedConceptId?: string; + readonly excludedCandidateFamily?: string; readonly excludedDefinitionSymbol?: string; readonly excludedRelationId?: string; readonly relationId?: string; @@ -57,7 +58,7 @@ export interface ChallengeCase { export interface ChallengeCorpus { readonly cases: readonly ChallengeCase[]; - readonly schemaVersion: 1; + readonly schemaVersion: 2; } export interface ChallengeObservation { @@ -87,15 +88,42 @@ export interface ChallengeScorecard { readonly metrics: Readonly>; readonly outcomes: Readonly>; readonly passed: number; - readonly schemaVersion: 1; + readonly schemaVersion: 2; +} + +export interface DevelopmentFixtureCase { + readonly documents: readonly CorpusDocument[]; + readonly id: string; +} + +export function findChallengeFixtureLeaks( + challenge: readonly ChallengeCase[], + development: readonly DevelopmentFixtureCase[], +): readonly string[] { + const developmentIds = new Set(development.map((item) => item.id)); + const developmentSources = new Set( + development.flatMap((item) => + item.documents.map((document) => normalizedSource(document.content)), + ), + ); + const leaks = new Set(); + for (const item of challenge) { + if (developmentIds.has(item.id)) leaks.add(`${item.id}: duplicate fixture id`); + for (const document of item.documents) { + if (developmentSources.has(normalizedSource(document.content))) { + leaks.add(`${item.id}: duplicate fixture source`); + } + } + } + return [...leaks].sort(); } export function parseChallengeCorpus(value: unknown): ChallengeCorpus { const root = record(value, "challenge"); exact(root, ["schemaVersion", "cases"], "challenge"); - if (root.schemaVersion !== 1) throw new Error("challenge.schemaVersion: must be 1"); - if (!Array.isArray(root.cases) || root.cases.length < 24) { - throw new Error("challenge.cases: must contain at least 24 frozen cases"); + if (root.schemaVersion !== 2) throw new Error("challenge.schemaVersion: must be 2"); + if (!Array.isArray(root.cases) || root.cases.length < 48) { + throw new Error("challenge.cases: must contain at least 48 frozen cases"); } const cases = root.cases.map((item, index) => parseCase(item, `challenge.cases[${index}]`)); unique(cases.map((item) => item.id), "challenge.cases.id"); @@ -111,7 +139,8 @@ export function parseChallengeCorpus(value: unknown): ChallengeCorpus { throw new Error(`challenge.cases: missing metric ${metric}`); } } - return { cases, schemaVersion: 1 }; + validateBoundaryPairs(cases); + return { cases, schemaVersion: 2 }; } export function scoreChallenge( @@ -145,6 +174,12 @@ export function scoreChallenge( expected.conceptId && !observation.conceptIds.includes(expected.conceptId) ? `concept ${expected.conceptId}` : undefined, + expected.excludedCandidateFamily && + observation.candidates.some( + (candidate) => candidate.family === expected.excludedCandidateFamily, + ) + ? `excluded candidate ${expected.excludedCandidateFamily}` + : undefined, expected.definitionDescription && !observation.definitions.some( (definition) => @@ -209,7 +244,7 @@ export function scoreChallenge( (item) => item.outcome, ), passed: passed.size, - schemaVersion: 1, + schemaVersion: 2, }; } @@ -250,6 +285,7 @@ function parseCase(value: unknown, path: string): ChallengeCase { "candidateFamily", "candidateInterpretation", "conceptId", + "excludedCandidateFamily", "definitionDescription", "definitionRuleId", "excludedConceptId", @@ -289,6 +325,32 @@ function parseCase(value: unknown, path: string): ChallengeCase { }; } +function validateBoundaryPairs(cases: readonly ChallengeCase[]): void { + const pairs = new Map>(); + for (const item of cases) { + for (const tag of item.variationTags) { + if (!tag.startsWith("boundary-pair:")) continue; + const pair = tag.slice("boundary-pair:".length); + if (!pair) throw new Error(`${item.id}: boundary pair must have an id`); + const outcomes = pairs.get(pair) ?? new Set(); + outcomes.add(item.outcome); + pairs.set(pair, outcomes); + } + } + if (pairs.size < 12) { + throw new Error("challenge.cases: must contain at least 12 semantic boundary pairs"); + } + for (const [pair, outcomes] of pairs) { + if (!outcomes.has("positive") || !outcomes.has("refusal")) { + throw new Error(`challenge.cases: incomplete boundary pair ${pair}`); + } + } +} + +function normalizedSource(source: string): string { + return source.replace(/\s+/gu, " ").trim(); +} + function tally< const Keys extends readonly string[], Item extends { readonly id: string }, diff --git a/packages/evaluation/src/index.ts b/packages/evaluation/src/index.ts index 43aa3c8..13e6dbb 100644 --- a/packages/evaluation/src/index.ts +++ b/packages/evaluation/src/index.ts @@ -1,6 +1,7 @@ export * from "./conformance"; export * from "./challenge"; export * from "./foundation"; +export * from "./legacy-path-audit"; export * from "./metamorphic"; export * from "./model"; export * from "./notation-conformance"; diff --git a/packages/evaluation/src/legacy-path-audit.test.ts b/packages/evaluation/src/legacy-path-audit.test.ts new file mode 100644 index 0000000..d3f7a29 --- /dev/null +++ b/packages/evaluation/src/legacy-path-audit.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, test } from "bun:test"; +import { + LEGACY_SEMANTIC_PATHS, + auditLegacySemanticPaths, +} from "./legacy-path-audit"; + +describe("legacy semantic path audit", () => { + test("finds only paths whose complete legacy signature remains", () => { + const rule = LEGACY_SEMANTIC_PATHS[0]; + const complete = Object.fromEntries([ + [rule.path, rule.markers.join("\n")], + ]); + expect(auditLegacySemanticPaths(complete)).toEqual([ + { id: rule.id, ownerIssue: rule.ownerIssue, path: rule.path }, + ]); + expect(auditLegacySemanticPaths({ [rule.path]: rule.markers[0] })).toEqual([]); + }); + + test("returns findings in the reviewed ownership order", () => { + const sources: Record = {}; + for (const rule of LEGACY_SEMANTIC_PATHS) { + sources[rule.path] = `${sources[rule.path] ?? ""}\n${rule.markers.join("\n")}`; + } + expect(auditLegacySemanticPaths(sources).map((finding) => finding.id)).toEqual( + LEGACY_SEMANTIC_PATHS.map((rule) => rule.id), + ); + }); +}); diff --git a/packages/evaluation/src/legacy-path-audit.ts b/packages/evaluation/src/legacy-path-audit.ts new file mode 100644 index 0000000..124d8e1 --- /dev/null +++ b/packages/evaluation/src/legacy-path-audit.ts @@ -0,0 +1,57 @@ +export const LEGACY_SEMANTIC_PATHS = [ + { + id: "raw-law-prose-window", + ownerIssue: 189, + path: "crates/semath-core/src/law.rs", + markers: ["fn sentence_around", "document.content"], + }, + { + id: "raw-law-lexical-policy", + ownerIssue: 189, + path: "crates/semath-core/src/law.rs", + markers: ["fn context_supports_law", "contradicted"], + }, + { + id: "raw-domain-lexical-scan", + ownerIssue: 189, + path: "crates/semath-core/src/domain.rs", + markers: ["fn collect_priors", "literal_matcher"], + }, + { + id: "exhaustive-law-dispatch", + ownerIssue: 190, + path: "crates/semath-core/src/law.rs", + markers: ["for actual in canonical_expressions", "for compiled in COMPILED_LAWS.iter()"], + }, + { + id: "inline-meaning-decision", + ownerIssue: 191, + path: "crates/semath-core/src/engine.rs", + markers: ["let (status, summary, refusal) = if conflicting"], + }, + { + id: "stale-protocol-documentation", + ownerIssue: 188, + path: "docs/public-api.md", + markers: ["Protocol 4"], + }, +] as const; + +export type LegacySemanticPathId = (typeof LEGACY_SEMANTIC_PATHS)[number]["id"]; + +export interface LegacySemanticPathFinding { + readonly id: LegacySemanticPathId; + readonly ownerIssue: number; + readonly path: string; +} + +export function auditLegacySemanticPaths( + sources: Readonly>, +): readonly LegacySemanticPathFinding[] { + return LEGACY_SEMANTIC_PATHS.flatMap((rule) => { + const source = sources[rule.path]; + return source !== undefined && rule.markers.every((marker) => source.includes(marker)) + ? [{ id: rule.id, ownerIssue: rule.ownerIssue, path: rule.path }] + : []; + }); +} diff --git a/scripts/audit-legacy-semantic-paths.ts b/scripts/audit-legacy-semantic-paths.ts new file mode 100644 index 0000000..760866c --- /dev/null +++ b/scripts/audit-legacy-semantic-paths.ts @@ -0,0 +1,23 @@ +import { join } from "node:path"; +import { + LEGACY_SEMANTIC_PATHS, + auditLegacySemanticPaths, +} from "../packages/evaluation/src/legacy-path-audit"; + +const root = join(import.meta.dir, ".."); +const sources = Object.fromEntries( + await Promise.all( + [...new Set(LEGACY_SEMANTIC_PATHS.map((rule) => rule.path))].map( + async (path) => [path, await Bun.file(join(root, path)).text()] as const, + ), + ), +); +const findings = auditLegacySemanticPaths(sources); +console.log(JSON.stringify({ findings, schemaVersion: 1 }, null, 2)); +if (process.argv.includes("--check") && findings.length) { + throw new Error( + `legacy semantic paths remain:\n${findings + .map((finding) => `${finding.id} (${finding.path}, #${finding.ownerIssue})`) + .join("\n")}`, + ); +} diff --git a/scripts/check-challenge.ts b/scripts/check-challenge.ts index d13f65e..23f4bfc 100644 --- a/scripts/check-challenge.ts +++ b/scripts/check-challenge.ts @@ -9,12 +9,12 @@ import { runSemanticEvaluation } from "./semantic-evaluation-runner"; const corpus = parseChallengeCorpus( JSON.parse( await readFile( - new URL("../fixtures/challenge/recognition-v1.json", import.meta.url), + new URL("../fixtures/challenge/recognition-v2.json", import.meta.url), "utf8", ), ), ); -const results = runSemanticEvaluation(corpus.cases, "recognition-challenge-v1"); +const results = runSemanticEvaluation(corpus.cases, "recognition-challenge-v2"); const debugIds = new Set(process.env.SEMATH_CHALLENGE_DEBUG?.split(",") ?? []); const observations = corpus.cases.map((item, index): ChallengeObservation => { const result = results[index];