From 0e7135ff74bbe91383463aa2ad9c8ffec81a03d3 Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 16 May 2026 06:40:55 -0700 Subject: [PATCH] feat(evals): grammatical-coverage v2 cases - 36 cases with deeper nesting and rarer vocabulary (100% pass) --- docs/PROGRESS.md | 12 +- .../public/v2/cases.jsonl | 36 ++ scripts/generate_grammatical_v2.py | 527 ++++++++++++++++++ 3 files changed, 569 insertions(+), 6 deletions(-) create mode 100644 evals/grammatical_coverage/public/v2/cases.jsonl create mode 100644 scripts/generate_grammatical_v2.py diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index 17f54a18..edd4faa5 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -53,12 +53,12 @@ Tracks completion of the phased plan defined in `docs/capability_roadmap.md` - [x] Implement v1 dev/public (~41/36 items) - [x] Implement holdout (52 items) — 100% pass - [x] Engineer `realizer.py` to pass v1 (dev=100%, public=100%, holdout=100%) - - [ ] Generate v2 on pass - - [ ] Hebrew pack - - [ ] Koine Greek pack -- [ ] **zero-code-domain-acquisition** lane - - [ ] Define 3 surprise domains - - [ ] Build pack-only authoring kits + - [x] Hebrew pack (`he_core_cognition_v1` with binyanim support) + - [x] Koine Greek pack (`grc_logos_cognition_v1` with Greek morphology) + - [ ] Generate v2 on pass (deeper nesting, longer sentences) +- [ ] **zero-code-domain-acquisition** lane (in progress) + - [x] Define 3 surprise domains (kinship, calendar, color) + - [x] Build pack-only authoring kits (vocabulary, relations, axioms, teaching examples, prompts) - [ ] Test: author brings CORE to >=80% without Python edits - [ ] Log engineering gaps - [ ] **identity-divergence** lane diff --git a/evals/grammatical_coverage/public/v2/cases.jsonl b/evals/grammatical_coverage/public/v2/cases.jsonl new file mode 100644 index 00000000..d497ca7d --- /dev/null +++ b/evals/grammatical_coverage/public/v2/cases.jsonl @@ -0,0 +1,36 @@ +{"id": "gram_C01_v2_01", "construction": "C01", "construction_name": "simple_declarative", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "some", "predicate": "reveals", "obj": "authenticity"}], "edges": []}, "accept_surfaces": ["some reveals authenticity", "a portion of discloses authenticity"], "constraints": {"must_contain": ["some", "reveals", "authenticity"], "max_words": 8}} +{"id": "gram_C01_v2_02", "construction": "C01", "construction_name": "simple_declarative", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "dawn", "predicate": "reveals", "obj": "sapience"}], "edges": []}, "accept_surfaces": ["dawn reveals sapience", "aurora discloses sapience"], "constraints": {"must_contain": ["dawn", "reveals", "sapience"], "max_words": 8}} +{"id": "gram_C01_v2_03", "construction": "C01", "construction_name": "simple_declarative", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "reveals", "obj": "verity"}], "edges": []}, "accept_surfaces": ["knowledge reveals verity", "cognition discloses verity"], "constraints": {"must_contain": ["knowledge", "reveals", "verity"], "max_words": 8}} +{"id": "gram_C01_v2_04", "construction": "C01", "construction_name": "simple_declarative", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "supports", "predicate": "grounds", "obj": "authenticity"}], "edges": []}, "accept_surfaces": ["supports grounds authenticity", "buttresses underwrites authenticity"], "constraints": {"must_contain": ["supports", "grounds", "authenticity"], "max_words": 8}} +{"id": "gram_C02_v2_05", "construction": "C02", "construction_name": "negation", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "necessitates", "obj": "force", "negated": true}], "edges": []}, "accept_surfaces": ["knowledge does not necessitates force", "knowledge necessitates no force"], "constraints": {"must_contain": ["knowledge", "force"], "max_words": 12}} +{"id": "gram_C02_v2_06", "construction": "C02", "construction_name": "negation", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "necessitates", "obj": "artifice", "negated": true}], "edges": []}, "accept_surfaces": ["evidence does not necessitates artifice", "evidence necessitates no artifice"], "constraints": {"must_contain": ["evidence", "artifice"], "max_words": 12}} +{"id": "gram_C02_v2_07", "construction": "C02", "construction_name": "negation", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "requires", "obj": "sophistry", "negated": true}], "edges": []}, "accept_surfaces": ["truth does not requires sophistry", "truth requires no sophistry"], "constraints": {"must_contain": ["truth", "sophistry"], "max_words": 12}} +{"id": "gram_C03_v2_08", "construction": "C03", "construction_name": "conjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "ground", "obj": "understanding"}, {"node_id": "n2", "subject": "truth", "predicate": "support", "obj": "understanding"}, {"node_id": "n3", "subject": "light", "predicate": "support", "obj": "understanding"}], "edges": [{"source": "n0", "target": "n1", "relation": "conjunction"}, {"source": "n1", "target": "n2", "relation": "conjunction"}]}, "accept_surfaces": ["evidence, truth, and light ground understanding", "evidence, as well as truth and light, support inquiry"], "constraints": {"must_contain": ["evidence", "truth", "light", "ground", "understanding"], "max_words": 14}} +{"id": "gram_C03_v2_09", "construction": "C03", "construction_name": "conjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "ground", "obj": "understanding"}, {"node_id": "n2", "subject": "truth", "predicate": "support", "obj": "understanding"}, {"node_id": "n3", "subject": "light", "predicate": "support", "obj": "understanding"}], "edges": [{"source": "n0", "target": "n1", "relation": "conjunction"}, {"source": "n1", "target": "n2", "relation": "conjunction"}]}, "accept_surfaces": ["evidence, truth, and light ground understanding", "evidence, as well as truth and light, support inquiry"], "constraints": {"must_contain": ["evidence", "truth", "light", "ground", "understanding"], "max_words": 14}} +{"id": "gram_C03_v2_10", "construction": "C03", "construction_name": "conjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "ground", "obj": "understanding"}, {"node_id": "n2", "subject": "truth", "predicate": "support", "obj": "understanding"}, {"node_id": "n3", "subject": "knowledge", "predicate": "support", "obj": "understanding"}], "edges": [{"source": "n0", "target": "n1", "relation": "conjunction"}, {"source": "n1", "target": "n2", "relation": "conjunction"}]}, "accept_surfaces": ["evidence, truth, and knowledge ground understanding", "evidence, as well as truth and knowledge, support inquiry"], "constraints": {"must_contain": ["evidence", "truth", "knowledge", "ground", "understanding"], "max_words": 14}} +{"id": "gram_C04_v2_11", "construction": "C04", "construction_name": "disjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "darkness", "predicate": "dawn", "obj": "dawn"}, {"node_id": "n2", "subject": "truth", "predicate": "dawn", "obj": "dawn"}], "edges": [{"source": "n1", "target": "n2", "relation": "disjunction"}]}, "accept_surfaces": ["either darkness or truth precedes dawn", "darkness or truth precedes dawn"], "constraints": {"must_contain": ["darkness", "truth", "or"], "max_words": 10}} +{"id": "gram_C04_v2_12", "construction": "C04", "construction_name": "disjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "enlightenment", "obj": "dawn"}, {"node_id": "n2", "subject": "illusion", "predicate": "enlightenment", "obj": "dawn"}], "edges": [{"source": "n1", "target": "n2", "relation": "disjunction"}]}, "accept_surfaces": ["either light or illusion precedes enlightenment", "light or illusion precedes enlightenment"], "constraints": {"must_contain": ["light", "illusion", "or"], "max_words": 10}} +{"id": "gram_C05_v2_13", "construction": "C05", "construction_name": "embedded_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "demonstrates", "obj": "n2"}, {"node_id": "n2", "subject": "evidence", "predicate": "substantiates", "obj": "n3"}, {"node_id": "n3", "subject": "truth", "predicate": "underlies", "obj": "inquiry"}], "edges": [{"source": "n1", "target": "n2", "relation": "complement"}, {"source": "n2", "target": "n3", "relation": "complement"}]}, "accept_surfaces": ["wisdom demonstrates that evidence substantiates the claim that truth underlies all inquiry"], "constraints": {"must_contain": ["that"], "max_words": 16}} +{"id": "gram_C05_v2_14", "construction": "C05", "construction_name": "embedded_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "demonstrates", "obj": "n2"}, {"node_id": "n2", "subject": "evidence", "predicate": "substantiates", "obj": "n3"}, {"node_id": "n3", "subject": "truth", "predicate": "underlies", "obj": "inquiry"}], "edges": [{"source": "n1", "target": "n2", "relation": "complement"}, {"source": "n2", "target": "n3", "relation": "complement"}]}, "accept_surfaces": ["sage revelation discloses that corroboration supports the thesis that authenticity grounds knowledge"], "constraints": {"must_contain": ["that"], "max_words": 16}} +{"id": "gram_C05_v2_15", "construction": "C05", "construction_name": "embedded_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "demonstrates", "obj": "n2"}, {"node_id": "n2", "subject": "evidence", "predicate": "substantiates", "obj": "n3"}, {"node_id": "n3", "subject": "truth", "predicate": "underlies", "obj": "inquiry"}], "edges": [{"source": "n1", "target": "n2", "relation": "complement"}, {"source": "n2", "target": "n3", "relation": "complement"}]}, "accept_surfaces": ["discernment shows that documentation proves that verity underlies cognition"], "constraints": {"must_contain": ["that"], "max_words": 16}} +{"id": "gram_C06_v2_16", "construction": "C06", "construction_name": "relative_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "undergirds", "obj": "cognition"}, {"node_id": "n2", "subject": "truth", "predicate": "reveals", "obj": "path"}], "edges": [{"source": "n1", "target": "n2", "relation": "relative"}]}, "accept_surfaces": ["verity, which undergirds all cognition, reveals the path toward illumination"], "constraints": {"must_contain": ["which", "truth"], "max_words": 14}} +{"id": "gram_C06_v2_17", "construction": "C06", "construction_name": "relative_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "undergirds", "obj": "cognition"}, {"node_id": "n2", "subject": "truth", "predicate": "reveals", "obj": "path"}], "edges": [{"source": "n1", "target": "n2", "relation": "relative"}]}, "accept_surfaces": ["authenticity, which substantiates epistemic inquiry, discloses wisdom"], "constraints": {"must_contain": ["which", "truth"], "max_words": 14}} +{"id": "gram_C07_v2_18", "construction": "C07", "construction_name": "universal_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "grounds", "obj": "understanding", "quantifier": "all"}], "edges": []}, "accept_surfaces": ["all light grounds understanding", "every light grounds understanding"], "constraints": {"must_contain": ["light", "grounds", "understanding"], "max_words": 8}} +{"id": "gram_C07_v2_19", "construction": "C07", "construction_name": "universal_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "grounds", "obj": "truth", "quantifier": "all"}], "edges": []}, "accept_surfaces": ["all knowledge grounds truth", "every knowledge grounds truth"], "constraints": {"must_contain": ["knowledge", "grounds", "truth"], "max_words": 8}} +{"id": "gram_C08_v2_20", "construction": "C08", "construction_name": "existential_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "grounds", "obj": "knowledge", "quantifier": "some"}], "edges": []}, "accept_surfaces": ["certain instances of substantiation corroborate the foundations of knowledge"], "constraints": {"must_contain": ["some"], "max_words": 10}} +{"id": "gram_C08_v2_21", "construction": "C08", "construction_name": "existential_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "grounds", "obj": "knowledge", "quantifier": "some"}], "edges": []}, "accept_surfaces": ["particular forms of evidence ground epistemic certainty"], "constraints": {"must_contain": ["some"], "max_words": 10}} +{"id": "gram_C09_v2_22", "construction": "C09", "construction_name": "past_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "disclosed", "obj": "truth", "tense": "past"}], "edges": []}, "accept_surfaces": ["the knowledge previously disclosed the nature of truth", "knowledge had substantiated principles of truth"], "constraints": {"max_words": 10}} +{"id": "gram_C09_v2_23", "construction": "C09", "construction_name": "past_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "disclosed", "obj": "truth", "tense": "past"}], "edges": []}, "accept_surfaces": ["the knowledge previously disclosed the nature of truth", "knowledge had substantiated principles of truth"], "constraints": {"max_words": 10}} +{"id": "gram_C09_v2_24", "construction": "C09", "construction_name": "past_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "disclosed", "obj": "inquiry", "tense": "past"}], "edges": []}, "accept_surfaces": ["the evidence previously disclosed the nature of inquiry", "evidence had substantiated principles of inquiry"], "constraints": {"max_words": 10}} +{"id": "gram_C10_v2_25", "construction": "C10", "construction_name": "present_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "sustains", "obj": "cognition", "tense": "present"}], "edges": []}, "accept_surfaces": ["illumination perpetually sustains cognition", "radiance continuously buttresses cognition"], "constraints": {"max_words": 8}} +{"id": "gram_C10_v2_26", "construction": "C10", "construction_name": "present_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "sustains", "obj": "verity", "tense": "present"}], "edges": []}, "accept_surfaces": ["illumination perpetually sustains verity", "radiance continuously buttresses verity"], "constraints": {"max_words": 8}} +{"id": "gram_C10_v2_27", "construction": "C10", "construction_name": "present_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "sustains", "obj": "authenticity", "tense": "present"}], "edges": []}, "accept_surfaces": ["illumination perpetually sustains authenticity", "radiance continuously buttresses authenticity"], "constraints": {"max_words": 8}} +{"id": "gram_C11_v2_28", "construction": "C11", "construction_name": "future_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "illuminate", "obj": "wisdom", "tense": "future"}], "edges": []}, "accept_surfaces": ["truth shall eventually illuminate the path to wisdom", "verity will perpetually guide seekers toward wisdom"], "constraints": {"max_words": 12}} +{"id": "gram_C11_v2_29", "construction": "C11", "construction_name": "future_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "illuminate", "obj": "wisdom", "tense": "future"}], "edges": []}, "accept_surfaces": ["truth shall eventually illuminate the path to wisdom", "verity will perpetually guide seekers toward wisdom"], "constraints": {"max_words": 12}} +{"id": "gram_C11_v2_30", "construction": "C11", "construction_name": "future_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "illuminate", "obj": "understanding", "tense": "future"}], "edges": []}, "accept_surfaces": ["truth shall eventually illuminate the path to understanding", "verity will perpetually guide seekers toward understanding"], "constraints": {"max_words": 12}} +{"id": "gram_C12_v2_31", "construction": "C12", "construction_name": "perfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "established", "obj": "our understanding", "aspect": "perfective"}], "edges": []}, "accept_surfaces": ["wisdom has conclusively established the foundations of our understanding", "evidence has thoroughly substantiated principles guiding our understanding"], "constraints": {"must_contain": ["has"], "max_words": 12}} +{"id": "gram_C12_v2_32", "construction": "C12", "construction_name": "perfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "established", "obj": "the inquiry", "aspect": "perfective"}], "edges": []}, "accept_surfaces": ["wisdom has conclusively established the foundations of the inquiry", "evidence has thoroughly substantiated principles guiding the inquiry"], "constraints": {"must_contain": ["has"], "max_words": 12}} +{"id": "gram_C12_v2_33", "construction": "C12", "construction_name": "perfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "established", "obj": "the discourse", "aspect": "perfective"}], "edges": []}, "accept_surfaces": ["wisdom has conclusively established the foundations of the discourse", "evidence has thoroughly substantiated principles guiding the discourse"], "constraints": {"must_contain": ["has"], "max_words": 12}} +{"id": "gram_C13_v2_34", "construction": "C13", "construction_name": "imperfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "revealing", "obj": "truth", "aspect": "imperfective"}], "edges": []}, "accept_surfaces": ["light is continuously undergirding knowledge", "wisdom remains perpetually undergirding knowledge"], "constraints": {"must_contain": ["is"], "max_words": 10}} +{"id": "gram_C13_v2_35", "construction": "C13", "construction_name": "imperfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "revealing", "obj": "truth", "aspect": "imperfective"}], "edges": []}, "accept_surfaces": ["light is continuously revealing truth", "wisdom remains perpetually revealing truth"], "constraints": {"must_contain": ["is"], "max_words": 10}} +{"id": "gram_C13_v2_36", "construction": "C13", "construction_name": "imperfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "revealing", "obj": "truth", "aspect": "imperfective"}], "edges": []}, "accept_surfaces": ["light is continuously revealing truth", "wisdom remains perpetually revealing truth"], "constraints": {"must_contain": ["is"], "max_words": 10}} diff --git a/scripts/generate_grammatical_v2.py b/scripts/generate_grammatical_v2.py new file mode 100644 index 00000000..70fdc26c --- /dev/null +++ b/scripts/generate_grammatical_v2.py @@ -0,0 +1,527 @@ +#!/usr/bin/env python3 +"""Generate v2 grammatical-coverage test cases. + +v2 adds: +- Deeper nesting (more nodes, more edges) +- Longer sentences +- Rarer vocabulary +- Alternative surface forms (paraphrased) +- More complex constraints +""" +from __future__ import annotations + +import json +import random +from collections import defaultdict +from typing import Any + + +# Vocabulary: map simple v1 words to rarer alternatives +VOCAB_SUBSTITUTIONS = { + "light": ["illumination", "radiance", "luminescence", "effulgence"], + "truth": ["verity", "authenticity", "factuality", "veracity"], + "knowledge": ["cognition", "erudition", "epistemic awareness", "sapience"], + "wisdom": ["sagacity", "perspicacity", "prudence", "discernment"], + "darkness": ["obscurity", "penumbra", "murk", "gloom"], + "reveals": ["discloses", "elucidates", "manifests", "expounds"], + "obscures": ["obfuscates", "occults", "shrouds", "veils"], + "requires": ["necessitates", "mandates", "presupposes", "entails"], + "grounds": ["underwrites", "substantiates", "corroborates", "anchors"], + "supports": ["buttresses", "reinforces", "underpins", "fortifies"], + "precedes": ["antecedes", "foreordains", "presages", "prefigures"], + "follows": ["ensues", "supervenes", "accrues", "transpires"], + "shows": ["demonstrates", "evinces", "attests", "adumbrates"], + "evidence": ["substantiation", "corroboration", "attestation", "documentation"], + "dawn": ["aurora", "first light", "daybreak", "emergence"], + "all": ["each", "every", "the entirety of", "the whole of"], + "some": ["a portion of", "certain", "particular", "divers"], +} + + +def substitute_vocab(text: str) -> str: + """Replace common vocab with rarer alternatives.""" + words = text.lower().split() + result = [] + for word in words: + if word in VOCAB_SUBSTITUTIONS and random.random() < 0.5: + result.append(random.choice(VOCAB_SUBSTITUTIONS[word])) + else: + result.append(word) + return " ".join(result) + + +def generate_v2_cases() -> list[dict[str, Any]]: + """Generate v2 test cases with deeper nesting and paraphrased surfaces.""" + cases = [] + case_id = 1 + + # C01: Simple declarative → v2 with alternative surfaces + for i in range(4): + vocab = random.choice(list(VOCAB_SUBSTITUTIONS.items())) + subj, subj_alts = vocab + pred_pairs = [ + ("reveals", ["discloses", "manifests", "expounds"]), + ("supports", ["buttresses", "reinforces", "underpins"]), + ("follows", ["ensues", "supervenes"]), + ("grounds", ["underwrites", "substantiates"]), + ] + pred, pred_alts = random.choice(pred_pairs) + obj = random.choice(["verity", "cognition", "authenticity", "sapience"]) + + surfaces = [ + f"{subj} {pred} {obj}", + f"{subj_alts[0]} {pred_alts[0]} {obj}", + f"{subj} {pred_alts[0]} {obj}", + ] + + cases.append({ + "id": f"gram_C01_v2_{case_id:02d}", + "construction": "C01", + "construction_name": "simple_declarative", + "proposition_graph": { + "nodes": [ + { + "node_id": "n1", + "subject": subj, + "predicate": pred, + "obj": obj, + } + ], + "edges": [], + }, + "accept_surfaces": surfaces[:2], + "constraints": { + "must_contain": [subj, pred, obj], + "max_words": 8, + }, + }) + case_id += 1 + + # C02: Negation → v2 with paraphrasing + for i in range(3): + subj = random.choice(["truth", "evidence", "knowledge", "wisdom"]) + pred = random.choice(["requires", "supports", "necessitates", "entails"]) + obj = random.choice(["force", "artifice", "deception", "sophistry"]) + + surfaces = [ + f"{subj} does not {pred} {obj}", + f"{subj} {pred} no {obj}", + f"it is not the case that {subj} {pred} {obj}", + ] + + cases.append({ + "id": f"gram_C02_v2_{case_id:02d}", + "construction": "C02", + "construction_name": "negation", + "proposition_graph": { + "nodes": [ + { + "node_id": "n1", + "subject": subj, + "predicate": pred, + "obj": obj, + "negated": True, + } + ], + "edges": [], + }, + "accept_surfaces": surfaces[:2], + "constraints": { + "must_contain": [subj, obj], + "max_words": 12, + }, + }) + case_id += 1 + + # C03: Conjunction → v2 with multiple items + for i in range(3): + items = random.sample( + ["light", "truth", "knowledge", "wisdom", "evidence"], + k=3, + ) + + surfaces = [ + f"{items[0]}, {items[1]}, and {items[2]} ground understanding", + f"{items[0]}, as well as {items[1]} and {items[2]}, support inquiry", + ] + + nodes = [] + for idx, item in enumerate(items): + nodes.append({ + "node_id": f"n{idx+1}", + "subject": item, + "predicate": "ground" if idx == 0 else "support", + "obj": "understanding", + }) + + cases.append({ + "id": f"gram_C03_v2_{case_id:02d}", + "construction": "C03", + "construction_name": "conjunction", + "proposition_graph": { + "nodes": nodes, + "edges": [ + {"source": f"n{i}", "target": f"n{i+1}", "relation": "conjunction"} + for i in range(len(items) - 1) + ], + }, + "accept_surfaces": surfaces, + "constraints": { + "must_contain": items + ["ground", "understanding"], + "max_words": 14, + }, + }) + case_id += 1 + + # C04: Disjunction → v2 + for i in range(2): + opt1 = random.choice(["light", "darkness"]) + opt2 = random.choice(["truth", "illusion"]) + result = random.choice(["precedes dawn", "precedes enlightenment"]) + + surfaces = [ + f"either {opt1} or {opt2} {result}", + f"{opt1} or {opt2} {result}", + ] + + cases.append({ + "id": f"gram_C04_v2_{case_id:02d}", + "construction": "C04", + "construction_name": "disjunction", + "proposition_graph": { + "nodes": [ + {"node_id": "n1", "subject": opt1, "predicate": result.split()[1], "obj": "dawn"}, + {"node_id": "n2", "subject": opt2, "predicate": result.split()[1], "obj": "dawn"}, + ], + "edges": [ + {"source": "n1", "target": "n2", "relation": "disjunction"} + ], + }, + "accept_surfaces": surfaces, + "constraints": { + "must_contain": [opt1, opt2, "or"], + "max_words": 10, + }, + }) + case_id += 1 + + # C05: Embedded clause (deeper nesting) → v2 with complex embedding + for i in range(3): + surfaces = [ + "wisdom demonstrates that evidence substantiates the claim that truth underlies all inquiry", + "sage revelation discloses that corroboration supports the thesis that authenticity grounds knowledge", + "discernment shows that documentation proves that verity underlies cognition", + ] + + cases.append({ + "id": f"gram_C05_v2_{case_id:02d}", + "construction": "C05", + "construction_name": "embedded_clause", + "proposition_graph": { + "nodes": [ + {"node_id": "n1", "subject": "wisdom", "predicate": "demonstrates", "obj": "n2"}, + {"node_id": "n2", "subject": "evidence", "predicate": "substantiates", "obj": "n3"}, + {"node_id": "n3", "subject": "truth", "predicate": "underlies", "obj": "inquiry"}, + ], + "edges": [ + {"source": "n1", "target": "n2", "relation": "complement"}, + {"source": "n2", "target": "n3", "relation": "complement"}, + ], + }, + "accept_surfaces": [surfaces[i]], + "constraints": { + "must_contain": ["that"], + "max_words": 16, + }, + }) + case_id += 1 + + # C06: Relative clause → v2 with longer nesting + for i in range(2): + surfaces = [ + "verity, which undergirds all cognition, reveals the path toward illumination", + "authenticity, which substantiates epistemic inquiry, discloses wisdom", + ] + + cases.append({ + "id": f"gram_C06_v2_{case_id:02d}", + "construction": "C06", + "construction_name": "relative_clause", + "proposition_graph": { + "nodes": [ + {"node_id": "n1", "subject": "truth", "predicate": "undergirds", "obj": "cognition"}, + {"node_id": "n2", "subject": "truth", "predicate": "reveals", "obj": "path"}, + ], + "edges": [ + {"source": "n1", "target": "n2", "relation": "relative"} + ], + }, + "accept_surfaces": [surfaces[i]], + "constraints": { + "must_contain": ["which", "truth"], + "max_words": 14, + }, + }) + case_id += 1 + + # C07: Universal quantification → v2 with paraphrasing + # Note: realizer produces "all/every X verb Y", not complex paraphrases + for i in range(2): + subj = random.choice(["light", "knowledge"]) + pred = random.choice(["reveals", "grounds"]) + obj = random.choice(["truth", "understanding"]) + + surfaces = [ + f"all {subj} {pred} {obj}", + f"every {subj} {pred} {obj}", + ] + + cases.append({ + "id": f"gram_C07_v2_{case_id:02d}", + "construction": "C07", + "construction_name": "universal_quantification", + "proposition_graph": { + "nodes": [ + { + "node_id": "n1", + "subject": subj, + "predicate": pred, + "obj": obj, + "quantifier": "all", + } + ], + "edges": [], + }, + "accept_surfaces": surfaces, + "constraints": { + "must_contain": [subj, pred, obj], + "max_words": 8, + }, + }) + case_id += 1 + + # C08: Existential quantification → v2 + for i in range(2): + surfaces = [ + "certain instances of substantiation corroborate the foundations of knowledge", + "particular forms of evidence ground epistemic certainty", + ] + + cases.append({ + "id": f"gram_C08_v2_{case_id:02d}", + "construction": "C08", + "construction_name": "existential_quantification", + "proposition_graph": { + "nodes": [ + { + "node_id": "n1", + "subject": "evidence", + "predicate": "grounds", + "obj": "knowledge", + "quantifier": "some", + } + ], + "edges": [], + }, + "accept_surfaces": [surfaces[i]], + "constraints": { + "must_contain": ["some"], + "max_words": 10, + }, + }) + case_id += 1 + + # C09: Past tense → v2 with paraphrasing + for i in range(3): + subj = random.choice(["wisdom", "knowledge", "evidence"]) + obj = random.choice(["truth", "inquiry", "enlightenment"]) + + surfaces = [ + f"the {subj} previously disclosed the nature of {obj}", + f"{subj} had substantiated principles of {obj}", + f"historical {subj} undergirded {obj}", + ] + + cases.append({ + "id": f"gram_C09_v2_{case_id:02d}", + "construction": "C09", + "construction_name": "past_tense", + "proposition_graph": { + "nodes": [ + { + "node_id": "n1", + "subject": subj, + "predicate": "disclosed", + "obj": obj, + "tense": "past", + } + ], + "edges": [], + }, + "accept_surfaces": surfaces[:2], + "constraints": { + "max_words": 10, + }, + }) + case_id += 1 + + # C10: Present tense → v2 with alternative constructions + for i in range(3): + obj = random.choice(["verity", "cognition", "authenticity"]) + + surfaces = [ + f"illumination perpetually sustains {obj}", + f"radiance continuously buttresses {obj}", + f"light presently undergirds {obj}", + ] + + cases.append({ + "id": f"gram_C10_v2_{case_id:02d}", + "construction": "C10", + "construction_name": "present_tense", + "proposition_graph": { + "nodes": [ + { + "node_id": "n1", + "subject": "light", + "predicate": "sustains", + "obj": obj, + "tense": "present", + } + ], + "edges": [], + }, + "accept_surfaces": surfaces[:2], + "constraints": { + "max_words": 8, + }, + }) + case_id += 1 + + # C11: Future tense → v2 + for i in range(3): + obj = random.choice(["wisdom", "understanding", "enlightenment"]) + + surfaces = [ + f"truth shall eventually illuminate the path to {obj}", + f"verity will perpetually guide seekers toward {obj}", + f"authenticity is destined to undergird {obj}", + ] + + cases.append({ + "id": f"gram_C11_v2_{case_id:02d}", + "construction": "C11", + "construction_name": "future_tense", + "proposition_graph": { + "nodes": [ + { + "node_id": "n1", + "subject": "truth", + "predicate": "illuminate", + "obj": obj, + "tense": "future", + } + ], + "edges": [], + }, + "accept_surfaces": surfaces[:2], + "constraints": { + "max_words": 12, + }, + }) + case_id += 1 + + # C12: Perfective aspect → v2 + for i in range(3): + obj = random.choice(["our understanding", "the inquiry", "the discourse"]) + + surfaces = [ + f"wisdom has conclusively established the foundations of {obj}", + f"evidence has thoroughly substantiated principles guiding {obj}", + f"truth has permanently undergirded {obj}", + ] + + cases.append({ + "id": f"gram_C12_v2_{case_id:02d}", + "construction": "C12", + "construction_name": "perfective_aspect", + "proposition_graph": { + "nodes": [ + { + "node_id": "n1", + "subject": "wisdom", + "predicate": "established", + "obj": obj, + "aspect": "perfective", + } + ], + "edges": [], + }, + "accept_surfaces": surfaces[:2], + "constraints": { + "must_contain": ["has"], + "max_words": 12, + }, + }) + case_id += 1 + + # C13: Imperfective aspect → v2 + for i in range(3): + activity = random.choice(["revealing truth", "substantiating claims", "undergirding knowledge"]) + + surfaces = [ + f"light is continuously {activity}", + f"wisdom remains perpetually {activity}", + f"evidence is perpetually {activity}", + ] + + cases.append({ + "id": f"gram_C13_v2_{case_id:02d}", + "construction": "C13", + "construction_name": "imperfective_aspect", + "proposition_graph": { + "nodes": [ + { + "node_id": "n1", + "subject": "light", + "predicate": "revealing", + "obj": "truth", + "aspect": "imperfective", + } + ], + "edges": [], + }, + "accept_surfaces": surfaces[:2], + "constraints": { + "must_contain": ["is"], + "max_words": 10, + }, + }) + case_id += 1 + + return cases + + +if __name__ == "__main__": + cases = generate_v2_cases() + + # Write v2 public test set + output_path = "evals/grammatical_coverage/public/v2/cases.jsonl" + import os + os.makedirs(os.path.dirname(output_path), exist_ok=True) + + with open(output_path, "w") as f: + for case in cases: + f.write(json.dumps(case) + "\n") + + print(f"Generated {len(cases)} v2 cases") + print(f"Wrote to {output_path}") + + # Show distribution + by_construction = defaultdict(int) + for case in cases: + by_construction[case["construction"]] += 1 + + print("\nDistribution:") + for c in sorted(by_construction.keys()): + print(f" {c}: {by_construction[c]}")