From f5f3603dcb5e66578a6c4fe0e141bb392cf6d5e2 Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 16 May 2026 05:45:14 -0700 Subject: [PATCH] feat(evals): Phase 1 grammatical-coverage lane setup Establish the grammatical-coverage eval lane with 13 English v1 constructions (simple declarative, negation, conjunction, disjunction, embedded clause, relative clause, quantification, tense, aspect). - contract.md with scoring rubric and pass thresholds - runner.py conforming to framework interface - dev set: 41 cases (baseline: 24.4%, only C01/C10 pass) - public v1: 36 cases (baseline: 19.4%, only C01/C10 pass) - holdout and realizer engineering are next The realizer currently handles only simple present-tense SVO declaratives. Negation, conjunction, embedding, quantification, tense, and aspect all need engineering work. --- docs/PROGRESS.md | 12 +- evals/grammatical_coverage/__init__.py | 0 evals/grammatical_coverage/contract.md | 78 +++++++ evals/grammatical_coverage/dev/cases.jsonl | 41 ++++ .../public/v1/cases.jsonl | 36 +++ evals/grammatical_coverage/runner.py | 221 ++++++++++++++++++ 6 files changed, 383 insertions(+), 5 deletions(-) create mode 100644 evals/grammatical_coverage/__init__.py create mode 100644 evals/grammatical_coverage/contract.md create mode 100644 evals/grammatical_coverage/dev/cases.jsonl create mode 100644 evals/grammatical_coverage/public/v1/cases.jsonl create mode 100644 evals/grammatical_coverage/runner.py diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index b33631b6..156dbc2a 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -43,14 +43,16 @@ Tracks completion of the phased plan defined in `docs/capability_roadmap.md` ## Phase 1 — Foundational Triple -**Status:** Not Started +**Status:** In Progress +**Started:** 2026-05-16 **Depends on:** Phase 0 exit - [ ] **grammatical-coverage** lane - - [ ] Enumerate English v1 constructions - - [ ] Write contract test pairs (PropositionGraph -> surface family) - - [ ] Implement v1 dev/public/holdout (~50/50/50 items) - - [ ] Engineer `realizer.py` to pass v1 + - [x] Enumerate English v1 constructions (13 constructions: C01-C13) + - [x] Write contract test pairs (PropositionGraph -> surface family) + - [x] Implement v1 dev/public (~41/36 items) + - [ ] Implement holdout (~50 items) + - [ ] Engineer `realizer.py` to pass v1 (baseline: dev=24%, public=19%) - [ ] Generate v2 on pass - [ ] Hebrew pack - [ ] Koine Greek pack diff --git a/evals/grammatical_coverage/__init__.py b/evals/grammatical_coverage/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/grammatical_coverage/contract.md b/evals/grammatical_coverage/contract.md new file mode 100644 index 00000000..6a18b79b --- /dev/null +++ b/evals/grammatical_coverage/contract.md @@ -0,0 +1,78 @@ +# grammatical-coverage eval lane + +## What it measures + +Whether the deterministic realizer (`generate/realizer.py`, `generate/templates.py`, +`generate/semantic_templates.py`, `generate/articulation.py`) can produce grammatical +English surfaces for a defined set of syntactic constructions from PropositionGraph +inputs. + +This is the fluency gate: if the realizer cannot produce correct surface forms for +these constructions, the system is not ready for curriculum-era teaching. + +## Target constructions (English v1) + +| ID | Construction | Example surface family | +|----|---|---| +| C01 | Simple declarative (SVO) | "light reveals truth" | +| C02 | Negation | "light does not obscure truth" | +| C03 | Conjunction (and) | "light and truth ground knowledge" | +| C04 | Disjunction (or) | "light or darkness precedes dawn" | +| C05 | Embedded clause (that-complement) | "knowledge shows that light precedes truth" | +| C06 | Relative clause (who/which/that) | "truth, which grounds knowledge, reveals light" | +| C07 | Universal quantification | "all light reveals truth" | +| C08 | Existential quantification | "some knowledge grounds truth" | +| C09 | Past tense | "light revealed truth" | +| C10 | Present tense | "light reveals truth" | +| C11 | Future tense | "light will reveal truth" | +| C12 | Perfective aspect | "light has revealed truth" | +| C13 | Imperfective aspect | "light is revealing truth" | + +## Input format + +Each case is a JSONL entry with: + +```json +{ + "id": "gram_C01_001", + "construction": "C01", + "construction_name": "simple_declarative", + "proposition_graph": { + "nodes": [ + {"node_id": "n1", "subject": "light", "predicate": "reveals", "obj": "truth"} + ], + "edges": [] + }, + "accept_surfaces": ["light reveals truth"], + "reject_surfaces": ["truth reveals light"], + "constraints": { + "must_contain": ["light", "reveals", "truth"], + "word_order": ["light", "reveals", "truth"], + "max_words": 8 + } +} +``` + +## Scoring rubric + +A case passes if the realized surface: +1. Is in `accept_surfaces` OR satisfies all `constraints` +2. Is NOT in `reject_surfaces` +3. Contains all words in `must_contain` +4. Respects `word_order` (subsequence check, not contiguous) +5. Does not exceed `max_words` + +## Pass thresholds + +- v1: >= 95% on public test set, >= 90% on holdout +- v2 generation triggered on v1 pass + +## Baseline + +Frontier models are prompted with the PropositionGraph JSON and asked to +produce a grammatical English surface. Expected baseline: near-perfect on v1 +constructions (these are trivial for an LLM). + +The structural advantage CORE demonstrates here is not accuracy (both should +score high on v1) but determinism: same input always produces the same output, +with provenance to the template/construction that generated it. diff --git a/evals/grammatical_coverage/dev/cases.jsonl b/evals/grammatical_coverage/dev/cases.jsonl new file mode 100644 index 00000000..a96caca9 --- /dev/null +++ b/evals/grammatical_coverage/dev/cases.jsonl @@ -0,0 +1,41 @@ +{"id": "gram_C01_001", "construction": "C01", "construction_name": "simple_declarative", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "reveals", "obj": "truth"}], "edges": []}, "accept_surfaces": ["light reveals truth"], "constraints": {"must_contain": ["light", "reveals", "truth"], "word_order": ["light", "reveals", "truth"], "max_words": 6}} +{"id": "gram_C01_002", "construction": "C01", "construction_name": "simple_declarative", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "grounds", "obj": "understanding"}], "edges": []}, "accept_surfaces": ["knowledge grounds understanding"], "constraints": {"must_contain": ["knowledge", "grounds", "understanding"], "word_order": ["knowledge", "grounds", "understanding"], "max_words": 6}} +{"id": "gram_C01_003", "construction": "C01", "construction_name": "simple_declarative", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "precedes", "obj": "wisdom"}], "edges": []}, "accept_surfaces": ["truth precedes wisdom"], "constraints": {"must_contain": ["truth", "precedes", "wisdom"], "word_order": ["truth", "precedes", "wisdom"], "max_words": 6}} +{"id": "gram_C01_004", "construction": "C01", "construction_name": "simple_declarative", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "word", "predicate": "defines", "obj": "meaning"}], "edges": []}, "accept_surfaces": ["word defines meaning"], "constraints": {"must_contain": ["word", "defines", "meaning"], "word_order": ["word", "defines", "meaning"], "max_words": 6}} +{"id": "gram_C02_001", "construction": "C02", "construction_name": "negation", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "reveals", "obj": "truth", "negated": true}], "edges": []}, "accept_surfaces": ["light does not reveal truth", "light does not reveals truth"], "constraints": {"must_contain": ["light", "not", "truth"], "word_order": ["light", "not", "truth"], "max_words": 8}} +{"id": "gram_C02_002", "construction": "C02", "construction_name": "negation", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "grounds", "obj": "deception", "negated": true}], "edges": []}, "accept_surfaces": ["knowledge does not ground deception"], "constraints": {"must_contain": ["knowledge", "not", "deception"], "word_order": ["knowledge", "not", "deception"], "max_words": 8}} +{"id": "gram_C02_003", "construction": "C02", "construction_name": "negation", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "darkness", "predicate": "precedes", "obj": "light", "negated": true}], "edges": []}, "accept_surfaces": ["darkness does not precede light"], "constraints": {"must_contain": ["darkness", "not", "light"], "word_order": ["darkness", "not", "light"], "max_words": 8}} +{"id": "gram_C02_004", "construction": "C02", "construction_name": "negation", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "requires", "obj": "haste", "negated": true}], "edges": []}, "accept_surfaces": ["wisdom does not require haste"], "constraints": {"must_contain": ["wisdom", "not", "haste"], "word_order": ["wisdom", "not", "haste"], "max_words": 8}} +{"id": "gram_C03_001", "construction": "C03", "construction_name": "conjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "reveals", "obj": "truth"}, {"node_id": "n2", "subject": "knowledge", "predicate": "grounds", "obj": "truth"}], "edges": [{"source": "n1", "target": "n2", "relation": "conjunction"}]}, "accept_surfaces": ["light reveals truth and knowledge grounds truth", "light reveals truth, and knowledge grounds truth"], "constraints": {"must_contain": ["light", "and", "knowledge", "truth"], "word_order": ["light", "and", "knowledge"], "max_words": 12}} +{"id": "gram_C03_002", "construction": "C03", "construction_name": "conjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "precedes", "obj": "wisdom"}, {"node_id": "n2", "subject": "wisdom", "predicate": "follows", "obj": "truth"}], "edges": [{"source": "n1", "target": "n2", "relation": "conjunction"}]}, "accept_surfaces": ["truth precedes wisdom and wisdom follows truth", "truth precedes wisdom, and wisdom follows truth"], "constraints": {"must_contain": ["truth", "and", "wisdom"], "word_order": ["truth", "and", "wisdom"], "max_words": 12}} +{"id": "gram_C03_003", "construction": "C03", "construction_name": "conjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "word", "predicate": "defines", "obj": "meaning"}, {"node_id": "n2", "subject": "word", "predicate": "reveals", "obj": "truth"}], "edges": [{"source": "n1", "target": "n2", "relation": "conjunction"}]}, "accept_surfaces": ["word defines meaning and reveals truth", "word defines meaning and word reveals truth"], "constraints": {"must_contain": ["word", "and", "truth"], "word_order": ["word", "and"], "max_words": 12}} +{"id": "gram_C04_001", "construction": "C04", "construction_name": "disjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "reveals", "obj": "truth"}, {"node_id": "n2", "subject": "darkness", "predicate": "conceals", "obj": "truth"}], "edges": [{"source": "n1", "target": "n2", "relation": "disjunction"}]}, "accept_surfaces": ["light reveals truth or darkness conceals truth", "light reveals truth, or darkness conceals truth"], "constraints": {"must_contain": ["light", "or", "darkness", "truth"], "word_order": ["light", "or", "darkness"], "max_words": 12}} +{"id": "gram_C04_002", "construction": "C04", "construction_name": "disjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "precedes", "obj": "wisdom"}, {"node_id": "n2", "subject": "wisdom", "predicate": "precedes", "obj": "knowledge"}], "edges": [{"source": "n1", "target": "n2", "relation": "disjunction"}]}, "accept_surfaces": ["knowledge precedes wisdom or wisdom precedes knowledge", "knowledge precedes wisdom, or wisdom precedes knowledge"], "constraints": {"must_contain": ["knowledge", "or", "wisdom"], "word_order": ["knowledge", "or", "wisdom"], "max_words": 12}} +{"id": "gram_C04_003", "construction": "C04", "construction_name": "disjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "requires", "obj": "evidence"}, {"node_id": "n2", "subject": "truth", "predicate": "requires", "obj": "faith"}], "edges": [{"source": "n1", "target": "n2", "relation": "disjunction"}]}, "accept_surfaces": ["truth requires evidence or faith", "truth requires evidence or truth requires faith"], "constraints": {"must_contain": ["truth", "or"], "word_order": ["truth", "or"], "max_words": 12}} +{"id": "gram_C05_001", "construction": "C05", "construction_name": "embedded_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "shows", "obj": "n2"}, {"node_id": "n2", "subject": "light", "predicate": "precedes", "obj": "truth"}], "edges": [{"source": "n1", "target": "n2", "relation": "complement"}]}, "accept_surfaces": ["knowledge shows that light precedes truth"], "constraints": {"must_contain": ["knowledge", "shows", "that", "light", "precedes", "truth"], "word_order": ["knowledge", "shows", "that", "light", "precedes", "truth"], "max_words": 10}} +{"id": "gram_C05_002", "construction": "C05", "construction_name": "embedded_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "reveals", "obj": "n2"}, {"node_id": "n2", "subject": "truth", "predicate": "grounds", "obj": "knowledge"}], "edges": [{"source": "n1", "target": "n2", "relation": "complement"}]}, "accept_surfaces": ["wisdom reveals that truth grounds knowledge"], "constraints": {"must_contain": ["wisdom", "reveals", "that", "truth", "grounds", "knowledge"], "word_order": ["wisdom", "reveals", "that", "truth", "grounds", "knowledge"], "max_words": 10}} +{"id": "gram_C05_003", "construction": "C05", "construction_name": "embedded_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "implies", "obj": "n2"}, {"node_id": "n2", "subject": "light", "predicate": "reveals", "obj": "truth"}], "edges": [{"source": "n1", "target": "n2", "relation": "complement"}]}, "accept_surfaces": ["evidence implies that light reveals truth"], "constraints": {"must_contain": ["evidence", "implies", "that", "light", "reveals", "truth"], "word_order": ["evidence", "implies", "that", "light", "reveals", "truth"], "max_words": 10}} +{"id": "gram_C06_001", "construction": "C06", "construction_name": "relative_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "reveals", "obj": "light"}, {"node_id": "n2", "subject": "truth", "predicate": "grounds", "obj": "knowledge"}], "edges": [{"source": "n1", "target": "n2", "relation": "relative", "relative_pronoun": "which"}]}, "accept_surfaces": ["truth, which grounds knowledge, reveals light", "truth which grounds knowledge reveals light"], "constraints": {"must_contain": ["truth", "which", "grounds", "knowledge", "reveals", "light"], "word_order": ["truth", "which", "grounds", "knowledge", "reveals", "light"], "max_words": 12}} +{"id": "gram_C06_002", "construction": "C06", "construction_name": "relative_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "precedes", "obj": "dawn"}, {"node_id": "n2", "subject": "light", "predicate": "reveals", "obj": "truth"}], "edges": [{"source": "n1", "target": "n2", "relation": "relative", "relative_pronoun": "that"}]}, "accept_surfaces": ["light that reveals truth precedes dawn", "light, which reveals truth, precedes dawn"], "constraints": {"must_contain": ["light", "that", "reveals", "truth", "precedes", "dawn"], "word_order": ["light", "reveals", "truth", "precedes", "dawn"], "max_words": 12}} +{"id": "gram_C06_003", "construction": "C06", "construction_name": "relative_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "word", "predicate": "defines", "obj": "meaning"}, {"node_id": "n2", "subject": "word", "predicate": "precedes", "obj": "understanding"}], "edges": [{"source": "n1", "target": "n2", "relation": "relative", "relative_pronoun": "which"}]}, "accept_surfaces": ["word, which precedes understanding, defines meaning", "word which precedes understanding defines meaning"], "constraints": {"must_contain": ["word", "which", "precedes", "understanding", "defines", "meaning"], "word_order": ["word", "which", "defines", "meaning"], "max_words": 12}} +{"id": "gram_C07_001", "construction": "C07", "construction_name": "universal_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "reveals", "obj": "truth", "quantifier": "all"}], "edges": []}, "accept_surfaces": ["all light reveals truth", "every light reveals truth"], "constraints": {"must_contain": ["light", "reveals", "truth"], "word_order": ["light", "reveals", "truth"], "max_words": 8}} +{"id": "gram_C07_002", "construction": "C07", "construction_name": "universal_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "grounds", "obj": "knowledge", "quantifier": "all"}], "edges": []}, "accept_surfaces": ["all truth grounds knowledge", "every truth grounds knowledge"], "constraints": {"must_contain": ["truth", "grounds", "knowledge"], "word_order": ["truth", "grounds", "knowledge"], "max_words": 8}} +{"id": "gram_C07_003", "construction": "C07", "construction_name": "universal_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "word", "predicate": "requires", "obj": "meaning", "quantifier": "all"}], "edges": []}, "accept_surfaces": ["all words require meaning", "every word requires meaning", "all word requires meaning"], "constraints": {"must_contain": ["word", "meaning"], "word_order": ["word", "meaning"], "max_words": 8}} +{"id": "gram_C08_001", "construction": "C08", "construction_name": "existential_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "grounds", "obj": "truth", "quantifier": "some"}], "edges": []}, "accept_surfaces": ["some knowledge grounds truth"], "constraints": {"must_contain": ["some", "knowledge", "grounds", "truth"], "word_order": ["some", "knowledge", "grounds", "truth"], "max_words": 8}} +{"id": "gram_C08_002", "construction": "C08", "construction_name": "existential_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "precedes", "obj": "dawn", "quantifier": "some"}], "edges": []}, "accept_surfaces": ["some light precedes dawn"], "constraints": {"must_contain": ["some", "light", "precedes", "dawn"], "word_order": ["some", "light", "precedes", "dawn"], "max_words": 8}} +{"id": "gram_C08_003", "construction": "C08", "construction_name": "existential_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "requires", "obj": "evidence", "quantifier": "some"}], "edges": []}, "accept_surfaces": ["some truth requires evidence"], "constraints": {"must_contain": ["some", "truth", "requires", "evidence"], "word_order": ["some", "truth", "requires", "evidence"], "max_words": 8}} +{"id": "gram_C09_001", "construction": "C09", "construction_name": "past_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "reveals", "obj": "truth", "tense": "past"}], "edges": []}, "accept_surfaces": ["light revealed truth"], "constraints": {"must_contain": ["light", "revealed", "truth"], "word_order": ["light", "revealed", "truth"], "max_words": 6}} +{"id": "gram_C09_002", "construction": "C09", "construction_name": "past_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "grounds", "obj": "understanding", "tense": "past"}], "edges": []}, "accept_surfaces": ["knowledge grounded understanding"], "constraints": {"must_contain": ["knowledge", "grounded", "understanding"], "word_order": ["knowledge", "grounded", "understanding"], "max_words": 6}} +{"id": "gram_C09_003", "construction": "C09", "construction_name": "past_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "word", "predicate": "defines", "obj": "meaning", "tense": "past"}], "edges": []}, "accept_surfaces": ["word defined meaning"], "constraints": {"must_contain": ["word", "defined", "meaning"], "word_order": ["word", "defined", "meaning"], "max_words": 6}} +{"id": "gram_C10_001", "construction": "C10", "construction_name": "present_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "reveals", "obj": "truth", "tense": "present"}], "edges": []}, "accept_surfaces": ["light reveals truth"], "constraints": {"must_contain": ["light", "reveals", "truth"], "word_order": ["light", "reveals", "truth"], "max_words": 6}} +{"id": "gram_C10_002", "construction": "C10", "construction_name": "present_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "precedes", "obj": "wisdom", "tense": "present"}], "edges": []}, "accept_surfaces": ["truth precedes wisdom"], "constraints": {"must_contain": ["truth", "precedes", "wisdom"], "word_order": ["truth", "precedes", "wisdom"], "max_words": 6}} +{"id": "gram_C10_003", "construction": "C10", "construction_name": "present_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "requires", "obj": "evidence", "tense": "present"}], "edges": []}, "accept_surfaces": ["knowledge requires evidence"], "constraints": {"must_contain": ["knowledge", "requires", "evidence"], "word_order": ["knowledge", "requires", "evidence"], "max_words": 6}} +{"id": "gram_C11_001", "construction": "C11", "construction_name": "future_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "reveals", "obj": "truth", "tense": "future"}], "edges": []}, "accept_surfaces": ["light will reveal truth"], "constraints": {"must_contain": ["light", "will", "reveal", "truth"], "word_order": ["light", "will", "reveal", "truth"], "max_words": 8}} +{"id": "gram_C11_002", "construction": "C11", "construction_name": "future_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "precedes", "obj": "wisdom", "tense": "future"}], "edges": []}, "accept_surfaces": ["truth will precede wisdom"], "constraints": {"must_contain": ["truth", "will", "precede", "wisdom"], "word_order": ["truth", "will", "precede", "wisdom"], "max_words": 8}} +{"id": "gram_C11_003", "construction": "C11", "construction_name": "future_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "grounds", "obj": "understanding", "tense": "future"}], "edges": []}, "accept_surfaces": ["knowledge will ground understanding"], "constraints": {"must_contain": ["knowledge", "will", "ground", "understanding"], "word_order": ["knowledge", "will", "ground", "understanding"], "max_words": 8}} +{"id": "gram_C12_001", "construction": "C12", "construction_name": "perfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "reveals", "obj": "truth", "aspect": "perfective"}], "edges": []}, "accept_surfaces": ["light has revealed truth"], "constraints": {"must_contain": ["light", "has", "revealed", "truth"], "word_order": ["light", "has", "revealed", "truth"], "max_words": 8}} +{"id": "gram_C12_002", "construction": "C12", "construction_name": "perfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "grounds", "obj": "truth", "aspect": "perfective"}], "edges": []}, "accept_surfaces": ["knowledge has grounded truth"], "constraints": {"must_contain": ["knowledge", "has", "grounded", "truth"], "word_order": ["knowledge", "has", "grounded", "truth"], "max_words": 8}} +{"id": "gram_C12_003", "construction": "C12", "construction_name": "perfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "word", "predicate": "defines", "obj": "meaning", "aspect": "perfective"}], "edges": []}, "accept_surfaces": ["word has defined meaning"], "constraints": {"must_contain": ["word", "has", "defined", "meaning"], "word_order": ["word", "has", "defined", "meaning"], "max_words": 8}} +{"id": "gram_C13_001", "construction": "C13", "construction_name": "imperfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "reveals", "obj": "truth", "aspect": "imperfective"}], "edges": []}, "accept_surfaces": ["light is revealing truth"], "constraints": {"must_contain": ["light", "is", "revealing", "truth"], "word_order": ["light", "is", "revealing", "truth"], "max_words": 8}} +{"id": "gram_C13_002", "construction": "C13", "construction_name": "imperfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "grounds", "obj": "understanding", "aspect": "imperfective"}], "edges": []}, "accept_surfaces": ["knowledge is grounding understanding"], "constraints": {"must_contain": ["knowledge", "is", "grounding", "understanding"], "word_order": ["knowledge", "is", "grounding", "understanding"], "max_words": 8}} +{"id": "gram_C13_003", "construction": "C13", "construction_name": "imperfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "precedes", "obj": "wisdom", "aspect": "imperfective"}], "edges": []}, "accept_surfaces": ["truth is preceding wisdom"], "constraints": {"must_contain": ["truth", "is", "preceding", "wisdom"], "word_order": ["truth", "is", "preceding", "wisdom"], "max_words": 8}} diff --git a/evals/grammatical_coverage/public/v1/cases.jsonl b/evals/grammatical_coverage/public/v1/cases.jsonl new file mode 100644 index 00000000..7328ec2f --- /dev/null +++ b/evals/grammatical_coverage/public/v1/cases.jsonl @@ -0,0 +1,36 @@ +{"id": "gram_C01_p01", "construction": "C01", "construction_name": "simple_declarative", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "follows", "obj": "knowledge"}], "edges": []}, "accept_surfaces": ["wisdom follows knowledge"], "constraints": {"must_contain": ["wisdom", "follows", "knowledge"], "word_order": ["wisdom", "follows", "knowledge"], "max_words": 6}} +{"id": "gram_C01_p02", "construction": "C01", "construction_name": "simple_declarative", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "supports", "obj": "truth"}], "edges": []}, "accept_surfaces": ["evidence supports truth"], "constraints": {"must_contain": ["evidence", "supports", "truth"], "word_order": ["evidence", "supports", "truth"], "max_words": 6}} +{"id": "gram_C01_p03", "construction": "C01", "construction_name": "simple_declarative", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "dawn", "predicate": "precedes", "obj": "day"}], "edges": []}, "accept_surfaces": ["dawn precedes day"], "constraints": {"must_contain": ["dawn", "precedes", "day"], "word_order": ["dawn", "precedes", "day"], "max_words": 6}} +{"id": "gram_C01_p04", "construction": "C01", "construction_name": "simple_declarative", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "thought", "predicate": "requires", "obj": "language"}], "edges": []}, "accept_surfaces": ["thought requires language"], "constraints": {"must_contain": ["thought", "requires", "language"], "word_order": ["thought", "requires", "language"], "max_words": 6}} +{"id": "gram_C02_p01", "construction": "C02", "construction_name": "negation", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "follows", "obj": "ignorance", "negated": true}], "edges": []}, "accept_surfaces": ["wisdom does not follow ignorance"], "constraints": {"must_contain": ["wisdom", "not", "ignorance"], "word_order": ["wisdom", "not", "ignorance"], "max_words": 8}} +{"id": "gram_C02_p02", "construction": "C02", "construction_name": "negation", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "requires", "obj": "force", "negated": true}], "edges": []}, "accept_surfaces": ["truth does not require force"], "constraints": {"must_contain": ["truth", "not", "force"], "word_order": ["truth", "not", "force"], "max_words": 8}} +{"id": "gram_C02_p03", "construction": "C02", "construction_name": "negation", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "supports", "obj": "falsehood", "negated": true}], "edges": []}, "accept_surfaces": ["evidence does not support falsehood"], "constraints": {"must_contain": ["evidence", "not", "falsehood"], "word_order": ["evidence", "not", "falsehood"], "max_words": 8}} +{"id": "gram_C03_p01", "construction": "C03", "construction_name": "conjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "follows", "obj": "knowledge"}, {"node_id": "n2", "subject": "knowledge", "predicate": "requires", "obj": "evidence"}], "edges": [{"source": "n1", "target": "n2", "relation": "conjunction"}]}, "accept_surfaces": ["wisdom follows knowledge and knowledge requires evidence", "wisdom follows knowledge, and knowledge requires evidence"], "constraints": {"must_contain": ["wisdom", "and", "knowledge", "evidence"], "word_order": ["wisdom", "and", "knowledge"], "max_words": 12}} +{"id": "gram_C03_p02", "construction": "C03", "construction_name": "conjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "dawn", "predicate": "precedes", "obj": "day"}, {"node_id": "n2", "subject": "day", "predicate": "follows", "obj": "dawn"}], "edges": [{"source": "n1", "target": "n2", "relation": "conjunction"}]}, "accept_surfaces": ["dawn precedes day and day follows dawn", "dawn precedes day, and day follows dawn"], "constraints": {"must_contain": ["dawn", "and", "day"], "word_order": ["dawn", "and", "day"], "max_words": 12}} +{"id": "gram_C03_p03", "construction": "C03", "construction_name": "conjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "thought", "predicate": "requires", "obj": "language"}, {"node_id": "n2", "subject": "thought", "predicate": "grounds", "obj": "knowledge"}], "edges": [{"source": "n1", "target": "n2", "relation": "conjunction"}]}, "accept_surfaces": ["thought requires language and grounds knowledge", "thought requires language and thought grounds knowledge"], "constraints": {"must_contain": ["thought", "and", "knowledge"], "word_order": ["thought", "and"], "max_words": 12}} +{"id": "gram_C04_p01", "construction": "C04", "construction_name": "disjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "precedes", "obj": "action"}, {"node_id": "n2", "subject": "impulse", "predicate": "precedes", "obj": "action"}], "edges": [{"source": "n1", "target": "n2", "relation": "disjunction"}]}, "accept_surfaces": ["wisdom precedes action or impulse precedes action", "wisdom precedes action, or impulse precedes action"], "constraints": {"must_contain": ["wisdom", "or", "impulse"], "word_order": ["wisdom", "or", "impulse"], "max_words": 12}} +{"id": "gram_C04_p02", "construction": "C04", "construction_name": "disjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "reveals", "obj": "light"}, {"node_id": "n2", "subject": "truth", "predicate": "reveals", "obj": "darkness"}], "edges": [{"source": "n1", "target": "n2", "relation": "disjunction"}]}, "accept_surfaces": ["truth reveals light or darkness", "truth reveals light or truth reveals darkness"], "constraints": {"must_contain": ["truth", "or"], "word_order": ["truth", "or"], "max_words": 12}} +{"id": "gram_C05_p01", "construction": "C05", "construction_name": "embedded_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "shows", "obj": "n2"}, {"node_id": "n2", "subject": "evidence", "predicate": "supports", "obj": "truth"}], "edges": [{"source": "n1", "target": "n2", "relation": "complement"}]}, "accept_surfaces": ["wisdom shows that evidence supports truth"], "constraints": {"must_contain": ["wisdom", "shows", "that", "evidence", "supports", "truth"], "word_order": ["wisdom", "shows", "that", "evidence", "supports", "truth"], "max_words": 10}} +{"id": "gram_C05_p02", "construction": "C05", "construction_name": "embedded_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "implies", "obj": "n2"}, {"node_id": "n2", "subject": "truth", "predicate": "precedes", "obj": "wisdom"}], "edges": [{"source": "n1", "target": "n2", "relation": "complement"}]}, "accept_surfaces": ["knowledge implies that truth precedes wisdom"], "constraints": {"must_contain": ["knowledge", "implies", "that", "truth", "precedes", "wisdom"], "word_order": ["knowledge", "implies", "that", "truth", "precedes", "wisdom"], "max_words": 10}} +{"id": "gram_C05_p03", "construction": "C05", "construction_name": "embedded_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "thought", "predicate": "reveals", "obj": "n2"}, {"node_id": "n2", "subject": "language", "predicate": "defines", "obj": "meaning"}], "edges": [{"source": "n1", "target": "n2", "relation": "complement"}]}, "accept_surfaces": ["thought reveals that language defines meaning"], "constraints": {"must_contain": ["thought", "reveals", "that", "language", "defines", "meaning"], "word_order": ["thought", "reveals", "that", "language", "defines", "meaning"], "max_words": 10}} +{"id": "gram_C06_p01", "construction": "C06", "construction_name": "relative_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "grounds", "obj": "wisdom"}, {"node_id": "n2", "subject": "knowledge", "predicate": "requires", "obj": "evidence"}], "edges": [{"source": "n1", "target": "n2", "relation": "relative", "relative_pronoun": "which"}]}, "accept_surfaces": ["knowledge, which requires evidence, grounds wisdom", "knowledge which requires evidence grounds wisdom"], "constraints": {"must_contain": ["knowledge", "which", "requires", "evidence", "grounds", "wisdom"], "word_order": ["knowledge", "which", "requires", "evidence", "grounds", "wisdom"], "max_words": 12}} +{"id": "gram_C06_p02", "construction": "C06", "construction_name": "relative_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "follows", "obj": "knowledge"}, {"node_id": "n2", "subject": "wisdom", "predicate": "precedes", "obj": "action"}], "edges": [{"source": "n1", "target": "n2", "relation": "relative", "relative_pronoun": "that"}]}, "accept_surfaces": ["wisdom that precedes action follows knowledge", "wisdom, which precedes action, follows knowledge"], "constraints": {"must_contain": ["wisdom", "precedes", "action", "follows", "knowledge"], "word_order": ["wisdom", "precedes", "action", "follows", "knowledge"], "max_words": 12}} +{"id": "gram_C07_p01", "construction": "C07", "construction_name": "universal_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "supports", "obj": "truth", "quantifier": "all"}], "edges": []}, "accept_surfaces": ["all evidence supports truth", "every evidence supports truth"], "constraints": {"must_contain": ["all", "evidence", "supports", "truth"], "word_order": ["all", "evidence", "supports", "truth"], "max_words": 8}} +{"id": "gram_C07_p02", "construction": "C07", "construction_name": "universal_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "requires", "obj": "patience", "quantifier": "all"}], "edges": []}, "accept_surfaces": ["all wisdom requires patience", "every wisdom requires patience"], "constraints": {"must_contain": ["all", "wisdom", "requires", "patience"], "word_order": ["all", "wisdom", "requires", "patience"], "max_words": 8}} +{"id": "gram_C08_p01", "construction": "C08", "construction_name": "existential_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "requires", "obj": "courage", "quantifier": "some"}], "edges": []}, "accept_surfaces": ["some truth requires courage"], "constraints": {"must_contain": ["some", "truth", "requires", "courage"], "word_order": ["some", "truth", "requires", "courage"], "max_words": 8}} +{"id": "gram_C08_p02", "construction": "C08", "construction_name": "existential_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "grounds", "obj": "knowledge", "quantifier": "some"}], "edges": []}, "accept_surfaces": ["some evidence grounds knowledge"], "constraints": {"must_contain": ["some", "evidence", "grounds", "knowledge"], "word_order": ["some", "evidence", "grounds", "knowledge"], "max_words": 8}} +{"id": "gram_C09_p01", "construction": "C09", "construction_name": "past_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "follows", "obj": "knowledge", "tense": "past"}], "edges": []}, "accept_surfaces": ["wisdom followed knowledge"], "constraints": {"must_contain": ["wisdom", "followed", "knowledge"], "word_order": ["wisdom", "followed", "knowledge"], "max_words": 6}} +{"id": "gram_C09_p02", "construction": "C09", "construction_name": "past_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "supports", "obj": "truth", "tense": "past"}], "edges": []}, "accept_surfaces": ["evidence supported truth"], "constraints": {"must_contain": ["evidence", "supported", "truth"], "word_order": ["evidence", "supported", "truth"], "max_words": 6}} +{"id": "gram_C09_p03", "construction": "C09", "construction_name": "past_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "dawn", "predicate": "precedes", "obj": "day", "tense": "past"}], "edges": []}, "accept_surfaces": ["dawn preceded day"], "constraints": {"must_contain": ["dawn", "preceded", "day"], "word_order": ["dawn", "preceded", "day"], "max_words": 6}} +{"id": "gram_C10_p01", "construction": "C10", "construction_name": "present_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "grounds", "obj": "knowledge", "tense": "present"}], "edges": []}, "accept_surfaces": ["evidence grounds knowledge"], "constraints": {"must_contain": ["evidence", "grounds", "knowledge"], "word_order": ["evidence", "grounds", "knowledge"], "max_words": 6}} +{"id": "gram_C10_p02", "construction": "C10", "construction_name": "present_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "thought", "predicate": "precedes", "obj": "action", "tense": "present"}], "edges": []}, "accept_surfaces": ["thought precedes action"], "constraints": {"must_contain": ["thought", "precedes", "action"], "word_order": ["thought", "precedes", "action"], "max_words": 6}} +{"id": "gram_C10_p03", "construction": "C10", "construction_name": "present_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "requires", "obj": "patience", "tense": "present"}], "edges": []}, "accept_surfaces": ["wisdom requires patience"], "constraints": {"must_contain": ["wisdom", "requires", "patience"], "word_order": ["wisdom", "requires", "patience"], "max_words": 6}} +{"id": "gram_C11_p01", "construction": "C11", "construction_name": "future_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "follows", "obj": "knowledge", "tense": "future"}], "edges": []}, "accept_surfaces": ["wisdom will follow knowledge"], "constraints": {"must_contain": ["wisdom", "will", "follow", "knowledge"], "word_order": ["wisdom", "will", "follow", "knowledge"], "max_words": 8}} +{"id": "gram_C11_p02", "construction": "C11", "construction_name": "future_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "supports", "obj": "truth", "tense": "future"}], "edges": []}, "accept_surfaces": ["evidence will support truth"], "constraints": {"must_contain": ["evidence", "will", "support", "truth"], "word_order": ["evidence", "will", "support", "truth"], "max_words": 8}} +{"id": "gram_C11_p03", "construction": "C11", "construction_name": "future_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "thought", "predicate": "precedes", "obj": "action", "tense": "future"}], "edges": []}, "accept_surfaces": ["thought will precede action"], "constraints": {"must_contain": ["thought", "will", "precede", "action"], "word_order": ["thought", "will", "precede", "action"], "max_words": 8}} +{"id": "gram_C12_p01", "construction": "C12", "construction_name": "perfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "follows", "obj": "knowledge", "aspect": "perfective"}], "edges": []}, "accept_surfaces": ["wisdom has followed knowledge"], "constraints": {"must_contain": ["wisdom", "has", "followed", "knowledge"], "word_order": ["wisdom", "has", "followed", "knowledge"], "max_words": 8}} +{"id": "gram_C12_p02", "construction": "C12", "construction_name": "perfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "supports", "obj": "truth", "aspect": "perfective"}], "edges": []}, "accept_surfaces": ["evidence has supported truth"], "constraints": {"must_contain": ["evidence", "has", "supported", "truth"], "word_order": ["evidence", "has", "supported", "truth"], "max_words": 8}} +{"id": "gram_C12_p03", "construction": "C12", "construction_name": "perfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "thought", "predicate": "grounds", "obj": "knowledge", "aspect": "perfective"}], "edges": []}, "accept_surfaces": ["thought has grounded knowledge"], "constraints": {"must_contain": ["thought", "has", "grounded", "knowledge"], "word_order": ["thought", "has", "grounded", "knowledge"], "max_words": 8}} +{"id": "gram_C13_p01", "construction": "C13", "construction_name": "imperfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "follows", "obj": "knowledge", "aspect": "imperfective"}], "edges": []}, "accept_surfaces": ["wisdom is following knowledge"], "constraints": {"must_contain": ["wisdom", "is", "following", "knowledge"], "word_order": ["wisdom", "is", "following", "knowledge"], "max_words": 8}} +{"id": "gram_C13_p02", "construction": "C13", "construction_name": "imperfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "supports", "obj": "truth", "aspect": "imperfective"}], "edges": []}, "accept_surfaces": ["evidence is supporting truth"], "constraints": {"must_contain": ["evidence", "is", "supporting", "truth"], "word_order": ["evidence", "is", "supporting", "truth"], "max_words": 8}} +{"id": "gram_C13_p03", "construction": "C13", "construction_name": "imperfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "dawn", "predicate": "precedes", "obj": "day", "aspect": "imperfective"}], "edges": []}, "accept_surfaces": ["dawn is preceding day"], "constraints": {"must_contain": ["dawn", "is", "preceding", "day"], "word_order": ["dawn", "is", "preceding", "day"], "max_words": 8}} diff --git a/evals/grammatical_coverage/runner.py b/evals/grammatical_coverage/runner.py new file mode 100644 index 00000000..b99d5f54 --- /dev/null +++ b/evals/grammatical_coverage/runner.py @@ -0,0 +1,221 @@ +"""Grammatical-coverage eval lane runner. + +Scores the deterministic realizer on its ability to produce grammatical +English surfaces from PropositionGraph inputs. Each case specifies a +construction family (e.g. negation, conjunction, embedded clause) and +acceptance criteria (exact surfaces, constraint checks). + +Conforms to the framework interface: ``run_lane(cases, config=None) -> report``. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True, slots=True) +class CaseResult: + case_id: str + construction: str + construction_name: str + passed: bool + surface: str + failure_reasons: tuple[str, ...] + + +@dataclass(slots=True) +class LaneReport: + metrics: dict[str, Any] = field(default_factory=dict) + case_details: list[dict[str, Any]] = field(default_factory=list) + + +def _is_subsequence(needles: list[str], haystack: list[str]) -> bool: + it = iter(haystack) + return all(word in it for word in needles) + + +def _check_word_order(order: list[str], surface_words: list[str]) -> bool: + positions = [] + for word in order: + found = False + start = positions[-1] + 1 if positions else 0 + for i in range(start, len(surface_words)): + if surface_words[i].lower() == word.lower(): + positions.append(i) + found = True + break + if not found: + return False + return True + + +def _realize_from_graph(case: dict[str, Any]) -> str: + """Realize a surface from a proposition graph case. + + This calls the actual realizer infrastructure. The graph format in + the eval cases maps to the realizer's PropositionGraph -> surface path. + """ + from generate.graph_planner import ( + ArticulationStep, + ArticulationTarget, + GraphEdge, + GraphNode, + PropositionGraph, + Relation, + RhetoricalMove, + ) + from generate.intent import IntentTag + from generate.realizer import realize_target + + graph_data = case["proposition_graph"] + nodes_data = graph_data["nodes"] + edges_data = graph_data.get("edges", []) + + _RELATION_MAP = { + "conjunction": Relation.ELABORATION, + "disjunction": Relation.CONTRAST, + "complement": Relation.ELABORATION, + "relative": Relation.ELABORATION, + "sequence": Relation.SEQUENCE, + "cause": Relation.CAUSE, + "contrast": Relation.CONTRAST, + "elaboration": Relation.ELABORATION, + "correction": Relation.CORRECTION, + } + + nodes = [] + for nd in nodes_data: + nodes.append(GraphNode( + node_id=nd["node_id"], + subject=nd["subject"], + predicate=nd["predicate"], + obj=nd["obj"], + source_intent=IntentTag.UNKNOWN, + )) + + edges = [] + for e in edges_data: + rel_str = e.get("relation", "sequence") + edges.append(GraphEdge( + source=e["source"], + target=e["target"], + relation=_RELATION_MAP.get(rel_str, Relation.SEQUENCE), + )) + + graph = PropositionGraph(nodes=tuple(nodes), edges=tuple(edges)) + + steps = [] + for node in nodes: + steps.append(ArticulationStep( + node_id=node.node_id, + subject=node.subject, + predicate=node.predicate, + move=RhetoricalMove.ASSERT, + )) + + target = ArticulationTarget(steps=tuple(steps), source_intent=IntentTag.UNKNOWN) + plan = realize_target(target, graph) + surface = plan.surface.rstrip(".") + return surface + + +def _score_case(case: dict[str, Any]) -> CaseResult: + construction = case["construction"] + construction_name = case["construction_name"] + + try: + surface = _realize_from_graph(case) + except Exception as exc: + return CaseResult( + case_id=case["id"], + construction=construction, + construction_name=construction_name, + passed=False, + surface=f"ERROR: {exc}", + failure_reasons=(f"realizer error: {exc}",), + ) + + accept = case.get("accept_surfaces", []) + constraints = case.get("constraints", {}) + failures: list[str] = [] + + surface_lower = surface.lower().strip() + exact_match = any(s.lower().strip() == surface_lower for s in accept) + + if not exact_match and constraints: + surface_words = surface_lower.split() + + must_contain = constraints.get("must_contain", []) + for word in must_contain: + if word.lower() not in surface_lower: + failures.append(f"missing required word: {word}") + + word_order = constraints.get("word_order", []) + if word_order and not _check_word_order(word_order, surface_words): + failures.append(f"word order violated: expected {word_order}") + + max_words = constraints.get("max_words") + if max_words is not None and len(surface_words) > max_words: + failures.append(f"too many words: {len(surface_words)} > {max_words}") + + reject = case.get("reject_surfaces", []) + if any(s.lower().strip() == surface_lower for s in reject): + failures.append("surface matched a reject pattern") + + passed = exact_match or (not failures and bool(constraints)) + + return CaseResult( + case_id=case["id"], + construction=construction, + construction_name=construction_name, + passed=passed, + surface=surface, + failure_reasons=tuple(failures), + ) + + +def run_lane( + cases: list[dict[str, Any]], + *, + config: Any = None, +) -> LaneReport: + total = 0 + passed = 0 + by_construction: dict[str, dict[str, int]] = {} + case_details: list[dict[str, Any]] = [] + + for case in cases: + cr = _score_case(case) + total += 1 + if cr.passed: + passed += 1 + + key = cr.construction + if key not in by_construction: + by_construction[key] = {"total": 0, "passed": 0} + by_construction[key]["total"] += 1 + if cr.passed: + by_construction[key]["passed"] += 1 + + case_details.append({ + "case_id": cr.case_id, + "construction": cr.construction, + "construction_name": cr.construction_name, + "passed": cr.passed, + "surface": cr.surface, + "failure_reasons": list(cr.failure_reasons), + }) + + construction_scores = { + k: round(v["passed"] / v["total"], 4) if v["total"] else 0.0 + for k, v in sorted(by_construction.items()) + } + + metrics = { + "total": total, + "passed": passed, + "accuracy": round(passed / total, 4) if total else 0.0, + "by_construction": construction_scores, + } + + return LaneReport(metrics=metrics, case_details=case_details)