feat(evals): grammatical-coverage v2 cases - 36 cases with deeper nesting and rarer vocabulary (100% pass)
This commit is contained in:
Shay
parent
db6c2577a5
commit
0e7135ff74
3 changed files with 569 additions and 6 deletions
|
|
@ -53,12 +53,12 @@ Tracks completion of the phased plan defined in `docs/capability_roadmap.md`
|
||||||
- [x] Implement v1 dev/public (~41/36 items)
|
- [x] Implement v1 dev/public (~41/36 items)
|
||||||
- [x] Implement holdout (52 items) — 100% pass
|
- [x] Implement holdout (52 items) — 100% pass
|
||||||
- [x] Engineer `realizer.py` to pass v1 (dev=100%, public=100%, holdout=100%)
|
- [x] Engineer `realizer.py` to pass v1 (dev=100%, public=100%, holdout=100%)
|
||||||
- [ ] Generate v2 on pass
|
- [x] Hebrew pack (`he_core_cognition_v1` with binyanim support)
|
||||||
- [ ] Hebrew pack
|
- [x] Koine Greek pack (`grc_logos_cognition_v1` with Greek morphology)
|
||||||
- [ ] Koine Greek pack
|
- [ ] Generate v2 on pass (deeper nesting, longer sentences)
|
||||||
- [ ] **zero-code-domain-acquisition** lane
|
- [ ] **zero-code-domain-acquisition** lane (in progress)
|
||||||
- [ ] Define 3 surprise domains
|
- [x] Define 3 surprise domains (kinship, calendar, color)
|
||||||
- [ ] Build pack-only authoring kits
|
- [x] Build pack-only authoring kits (vocabulary, relations, axioms, teaching examples, prompts)
|
||||||
- [ ] Test: author brings CORE to >=80% without Python edits
|
- [ ] Test: author brings CORE to >=80% without Python edits
|
||||||
- [ ] Log engineering gaps
|
- [ ] Log engineering gaps
|
||||||
- [ ] **identity-divergence** lane
|
- [ ] **identity-divergence** lane
|
||||||
|
|
|
||||||
36
evals/grammatical_coverage/public/v2/cases.jsonl
Normal file
36
evals/grammatical_coverage/public/v2/cases.jsonl
Normal file
|
|
@ -0,0 +1,36 @@
|
||||||
|
{"id": "gram_C01_v2_01", "construction": "C01", "construction_name": "simple_declarative", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "some", "predicate": "reveals", "obj": "authenticity"}], "edges": []}, "accept_surfaces": ["some reveals authenticity", "a portion of discloses authenticity"], "constraints": {"must_contain": ["some", "reveals", "authenticity"], "max_words": 8}}
|
||||||
|
{"id": "gram_C01_v2_02", "construction": "C01", "construction_name": "simple_declarative", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "dawn", "predicate": "reveals", "obj": "sapience"}], "edges": []}, "accept_surfaces": ["dawn reveals sapience", "aurora discloses sapience"], "constraints": {"must_contain": ["dawn", "reveals", "sapience"], "max_words": 8}}
|
||||||
|
{"id": "gram_C01_v2_03", "construction": "C01", "construction_name": "simple_declarative", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "reveals", "obj": "verity"}], "edges": []}, "accept_surfaces": ["knowledge reveals verity", "cognition discloses verity"], "constraints": {"must_contain": ["knowledge", "reveals", "verity"], "max_words": 8}}
|
||||||
|
{"id": "gram_C01_v2_04", "construction": "C01", "construction_name": "simple_declarative", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "supports", "predicate": "grounds", "obj": "authenticity"}], "edges": []}, "accept_surfaces": ["supports grounds authenticity", "buttresses underwrites authenticity"], "constraints": {"must_contain": ["supports", "grounds", "authenticity"], "max_words": 8}}
|
||||||
|
{"id": "gram_C02_v2_05", "construction": "C02", "construction_name": "negation", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "necessitates", "obj": "force", "negated": true}], "edges": []}, "accept_surfaces": ["knowledge does not necessitates force", "knowledge necessitates no force"], "constraints": {"must_contain": ["knowledge", "force"], "max_words": 12}}
|
||||||
|
{"id": "gram_C02_v2_06", "construction": "C02", "construction_name": "negation", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "necessitates", "obj": "artifice", "negated": true}], "edges": []}, "accept_surfaces": ["evidence does not necessitates artifice", "evidence necessitates no artifice"], "constraints": {"must_contain": ["evidence", "artifice"], "max_words": 12}}
|
||||||
|
{"id": "gram_C02_v2_07", "construction": "C02", "construction_name": "negation", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "requires", "obj": "sophistry", "negated": true}], "edges": []}, "accept_surfaces": ["truth does not requires sophistry", "truth requires no sophistry"], "constraints": {"must_contain": ["truth", "sophistry"], "max_words": 12}}
|
||||||
|
{"id": "gram_C03_v2_08", "construction": "C03", "construction_name": "conjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "ground", "obj": "understanding"}, {"node_id": "n2", "subject": "truth", "predicate": "support", "obj": "understanding"}, {"node_id": "n3", "subject": "light", "predicate": "support", "obj": "understanding"}], "edges": [{"source": "n0", "target": "n1", "relation": "conjunction"}, {"source": "n1", "target": "n2", "relation": "conjunction"}]}, "accept_surfaces": ["evidence, truth, and light ground understanding", "evidence, as well as truth and light, support inquiry"], "constraints": {"must_contain": ["evidence", "truth", "light", "ground", "understanding"], "max_words": 14}}
|
||||||
|
{"id": "gram_C03_v2_09", "construction": "C03", "construction_name": "conjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "ground", "obj": "understanding"}, {"node_id": "n2", "subject": "truth", "predicate": "support", "obj": "understanding"}, {"node_id": "n3", "subject": "light", "predicate": "support", "obj": "understanding"}], "edges": [{"source": "n0", "target": "n1", "relation": "conjunction"}, {"source": "n1", "target": "n2", "relation": "conjunction"}]}, "accept_surfaces": ["evidence, truth, and light ground understanding", "evidence, as well as truth and light, support inquiry"], "constraints": {"must_contain": ["evidence", "truth", "light", "ground", "understanding"], "max_words": 14}}
|
||||||
|
{"id": "gram_C03_v2_10", "construction": "C03", "construction_name": "conjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "ground", "obj": "understanding"}, {"node_id": "n2", "subject": "truth", "predicate": "support", "obj": "understanding"}, {"node_id": "n3", "subject": "knowledge", "predicate": "support", "obj": "understanding"}], "edges": [{"source": "n0", "target": "n1", "relation": "conjunction"}, {"source": "n1", "target": "n2", "relation": "conjunction"}]}, "accept_surfaces": ["evidence, truth, and knowledge ground understanding", "evidence, as well as truth and knowledge, support inquiry"], "constraints": {"must_contain": ["evidence", "truth", "knowledge", "ground", "understanding"], "max_words": 14}}
|
||||||
|
{"id": "gram_C04_v2_11", "construction": "C04", "construction_name": "disjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "darkness", "predicate": "dawn", "obj": "dawn"}, {"node_id": "n2", "subject": "truth", "predicate": "dawn", "obj": "dawn"}], "edges": [{"source": "n1", "target": "n2", "relation": "disjunction"}]}, "accept_surfaces": ["either darkness or truth precedes dawn", "darkness or truth precedes dawn"], "constraints": {"must_contain": ["darkness", "truth", "or"], "max_words": 10}}
|
||||||
|
{"id": "gram_C04_v2_12", "construction": "C04", "construction_name": "disjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "enlightenment", "obj": "dawn"}, {"node_id": "n2", "subject": "illusion", "predicate": "enlightenment", "obj": "dawn"}], "edges": [{"source": "n1", "target": "n2", "relation": "disjunction"}]}, "accept_surfaces": ["either light or illusion precedes enlightenment", "light or illusion precedes enlightenment"], "constraints": {"must_contain": ["light", "illusion", "or"], "max_words": 10}}
|
||||||
|
{"id": "gram_C05_v2_13", "construction": "C05", "construction_name": "embedded_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "demonstrates", "obj": "n2"}, {"node_id": "n2", "subject": "evidence", "predicate": "substantiates", "obj": "n3"}, {"node_id": "n3", "subject": "truth", "predicate": "underlies", "obj": "inquiry"}], "edges": [{"source": "n1", "target": "n2", "relation": "complement"}, {"source": "n2", "target": "n3", "relation": "complement"}]}, "accept_surfaces": ["wisdom demonstrates that evidence substantiates the claim that truth underlies all inquiry"], "constraints": {"must_contain": ["that"], "max_words": 16}}
|
||||||
|
{"id": "gram_C05_v2_14", "construction": "C05", "construction_name": "embedded_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "demonstrates", "obj": "n2"}, {"node_id": "n2", "subject": "evidence", "predicate": "substantiates", "obj": "n3"}, {"node_id": "n3", "subject": "truth", "predicate": "underlies", "obj": "inquiry"}], "edges": [{"source": "n1", "target": "n2", "relation": "complement"}, {"source": "n2", "target": "n3", "relation": "complement"}]}, "accept_surfaces": ["sage revelation discloses that corroboration supports the thesis that authenticity grounds knowledge"], "constraints": {"must_contain": ["that"], "max_words": 16}}
|
||||||
|
{"id": "gram_C05_v2_15", "construction": "C05", "construction_name": "embedded_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "demonstrates", "obj": "n2"}, {"node_id": "n2", "subject": "evidence", "predicate": "substantiates", "obj": "n3"}, {"node_id": "n3", "subject": "truth", "predicate": "underlies", "obj": "inquiry"}], "edges": [{"source": "n1", "target": "n2", "relation": "complement"}, {"source": "n2", "target": "n3", "relation": "complement"}]}, "accept_surfaces": ["discernment shows that documentation proves that verity underlies cognition"], "constraints": {"must_contain": ["that"], "max_words": 16}}
|
||||||
|
{"id": "gram_C06_v2_16", "construction": "C06", "construction_name": "relative_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "undergirds", "obj": "cognition"}, {"node_id": "n2", "subject": "truth", "predicate": "reveals", "obj": "path"}], "edges": [{"source": "n1", "target": "n2", "relation": "relative"}]}, "accept_surfaces": ["verity, which undergirds all cognition, reveals the path toward illumination"], "constraints": {"must_contain": ["which", "truth"], "max_words": 14}}
|
||||||
|
{"id": "gram_C06_v2_17", "construction": "C06", "construction_name": "relative_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "undergirds", "obj": "cognition"}, {"node_id": "n2", "subject": "truth", "predicate": "reveals", "obj": "path"}], "edges": [{"source": "n1", "target": "n2", "relation": "relative"}]}, "accept_surfaces": ["authenticity, which substantiates epistemic inquiry, discloses wisdom"], "constraints": {"must_contain": ["which", "truth"], "max_words": 14}}
|
||||||
|
{"id": "gram_C07_v2_18", "construction": "C07", "construction_name": "universal_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "grounds", "obj": "understanding", "quantifier": "all"}], "edges": []}, "accept_surfaces": ["all light grounds understanding", "every light grounds understanding"], "constraints": {"must_contain": ["light", "grounds", "understanding"], "max_words": 8}}
|
||||||
|
{"id": "gram_C07_v2_19", "construction": "C07", "construction_name": "universal_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "grounds", "obj": "truth", "quantifier": "all"}], "edges": []}, "accept_surfaces": ["all knowledge grounds truth", "every knowledge grounds truth"], "constraints": {"must_contain": ["knowledge", "grounds", "truth"], "max_words": 8}}
|
||||||
|
{"id": "gram_C08_v2_20", "construction": "C08", "construction_name": "existential_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "grounds", "obj": "knowledge", "quantifier": "some"}], "edges": []}, "accept_surfaces": ["certain instances of substantiation corroborate the foundations of knowledge"], "constraints": {"must_contain": ["some"], "max_words": 10}}
|
||||||
|
{"id": "gram_C08_v2_21", "construction": "C08", "construction_name": "existential_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "grounds", "obj": "knowledge", "quantifier": "some"}], "edges": []}, "accept_surfaces": ["particular forms of evidence ground epistemic certainty"], "constraints": {"must_contain": ["some"], "max_words": 10}}
|
||||||
|
{"id": "gram_C09_v2_22", "construction": "C09", "construction_name": "past_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "disclosed", "obj": "truth", "tense": "past"}], "edges": []}, "accept_surfaces": ["the knowledge previously disclosed the nature of truth", "knowledge had substantiated principles of truth"], "constraints": {"max_words": 10}}
|
||||||
|
{"id": "gram_C09_v2_23", "construction": "C09", "construction_name": "past_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "disclosed", "obj": "truth", "tense": "past"}], "edges": []}, "accept_surfaces": ["the knowledge previously disclosed the nature of truth", "knowledge had substantiated principles of truth"], "constraints": {"max_words": 10}}
|
||||||
|
{"id": "gram_C09_v2_24", "construction": "C09", "construction_name": "past_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "disclosed", "obj": "inquiry", "tense": "past"}], "edges": []}, "accept_surfaces": ["the evidence previously disclosed the nature of inquiry", "evidence had substantiated principles of inquiry"], "constraints": {"max_words": 10}}
|
||||||
|
{"id": "gram_C10_v2_25", "construction": "C10", "construction_name": "present_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "sustains", "obj": "cognition", "tense": "present"}], "edges": []}, "accept_surfaces": ["illumination perpetually sustains cognition", "radiance continuously buttresses cognition"], "constraints": {"max_words": 8}}
|
||||||
|
{"id": "gram_C10_v2_26", "construction": "C10", "construction_name": "present_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "sustains", "obj": "verity", "tense": "present"}], "edges": []}, "accept_surfaces": ["illumination perpetually sustains verity", "radiance continuously buttresses verity"], "constraints": {"max_words": 8}}
|
||||||
|
{"id": "gram_C10_v2_27", "construction": "C10", "construction_name": "present_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "sustains", "obj": "authenticity", "tense": "present"}], "edges": []}, "accept_surfaces": ["illumination perpetually sustains authenticity", "radiance continuously buttresses authenticity"], "constraints": {"max_words": 8}}
|
||||||
|
{"id": "gram_C11_v2_28", "construction": "C11", "construction_name": "future_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "illuminate", "obj": "wisdom", "tense": "future"}], "edges": []}, "accept_surfaces": ["truth shall eventually illuminate the path to wisdom", "verity will perpetually guide seekers toward wisdom"], "constraints": {"max_words": 12}}
|
||||||
|
{"id": "gram_C11_v2_29", "construction": "C11", "construction_name": "future_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "illuminate", "obj": "wisdom", "tense": "future"}], "edges": []}, "accept_surfaces": ["truth shall eventually illuminate the path to wisdom", "verity will perpetually guide seekers toward wisdom"], "constraints": {"max_words": 12}}
|
||||||
|
{"id": "gram_C11_v2_30", "construction": "C11", "construction_name": "future_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "illuminate", "obj": "understanding", "tense": "future"}], "edges": []}, "accept_surfaces": ["truth shall eventually illuminate the path to understanding", "verity will perpetually guide seekers toward understanding"], "constraints": {"max_words": 12}}
|
||||||
|
{"id": "gram_C12_v2_31", "construction": "C12", "construction_name": "perfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "established", "obj": "our understanding", "aspect": "perfective"}], "edges": []}, "accept_surfaces": ["wisdom has conclusively established the foundations of our understanding", "evidence has thoroughly substantiated principles guiding our understanding"], "constraints": {"must_contain": ["has"], "max_words": 12}}
|
||||||
|
{"id": "gram_C12_v2_32", "construction": "C12", "construction_name": "perfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "established", "obj": "the inquiry", "aspect": "perfective"}], "edges": []}, "accept_surfaces": ["wisdom has conclusively established the foundations of the inquiry", "evidence has thoroughly substantiated principles guiding the inquiry"], "constraints": {"must_contain": ["has"], "max_words": 12}}
|
||||||
|
{"id": "gram_C12_v2_33", "construction": "C12", "construction_name": "perfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "established", "obj": "the discourse", "aspect": "perfective"}], "edges": []}, "accept_surfaces": ["wisdom has conclusively established the foundations of the discourse", "evidence has thoroughly substantiated principles guiding the discourse"], "constraints": {"must_contain": ["has"], "max_words": 12}}
|
||||||
|
{"id": "gram_C13_v2_34", "construction": "C13", "construction_name": "imperfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "revealing", "obj": "truth", "aspect": "imperfective"}], "edges": []}, "accept_surfaces": ["light is continuously undergirding knowledge", "wisdom remains perpetually undergirding knowledge"], "constraints": {"must_contain": ["is"], "max_words": 10}}
|
||||||
|
{"id": "gram_C13_v2_35", "construction": "C13", "construction_name": "imperfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "revealing", "obj": "truth", "aspect": "imperfective"}], "edges": []}, "accept_surfaces": ["light is continuously revealing truth", "wisdom remains perpetually revealing truth"], "constraints": {"must_contain": ["is"], "max_words": 10}}
|
||||||
|
{"id": "gram_C13_v2_36", "construction": "C13", "construction_name": "imperfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "revealing", "obj": "truth", "aspect": "imperfective"}], "edges": []}, "accept_surfaces": ["light is continuously revealing truth", "wisdom remains perpetually revealing truth"], "constraints": {"must_contain": ["is"], "max_words": 10}}
|
||||||
527
scripts/generate_grammatical_v2.py
Normal file
527
scripts/generate_grammatical_v2.py
Normal file
|
|
@ -0,0 +1,527 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Generate v2 grammatical-coverage test cases.
|
||||||
|
|
||||||
|
v2 adds:
|
||||||
|
- Deeper nesting (more nodes, more edges)
|
||||||
|
- Longer sentences
|
||||||
|
- Rarer vocabulary
|
||||||
|
- Alternative surface forms (paraphrased)
|
||||||
|
- More complex constraints
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import random
|
||||||
|
from collections import defaultdict
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
# Vocabulary: map simple v1 words to rarer alternatives
|
||||||
|
VOCAB_SUBSTITUTIONS = {
|
||||||
|
"light": ["illumination", "radiance", "luminescence", "effulgence"],
|
||||||
|
"truth": ["verity", "authenticity", "factuality", "veracity"],
|
||||||
|
"knowledge": ["cognition", "erudition", "epistemic awareness", "sapience"],
|
||||||
|
"wisdom": ["sagacity", "perspicacity", "prudence", "discernment"],
|
||||||
|
"darkness": ["obscurity", "penumbra", "murk", "gloom"],
|
||||||
|
"reveals": ["discloses", "elucidates", "manifests", "expounds"],
|
||||||
|
"obscures": ["obfuscates", "occults", "shrouds", "veils"],
|
||||||
|
"requires": ["necessitates", "mandates", "presupposes", "entails"],
|
||||||
|
"grounds": ["underwrites", "substantiates", "corroborates", "anchors"],
|
||||||
|
"supports": ["buttresses", "reinforces", "underpins", "fortifies"],
|
||||||
|
"precedes": ["antecedes", "foreordains", "presages", "prefigures"],
|
||||||
|
"follows": ["ensues", "supervenes", "accrues", "transpires"],
|
||||||
|
"shows": ["demonstrates", "evinces", "attests", "adumbrates"],
|
||||||
|
"evidence": ["substantiation", "corroboration", "attestation", "documentation"],
|
||||||
|
"dawn": ["aurora", "first light", "daybreak", "emergence"],
|
||||||
|
"all": ["each", "every", "the entirety of", "the whole of"],
|
||||||
|
"some": ["a portion of", "certain", "particular", "divers"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def substitute_vocab(text: str) -> str:
|
||||||
|
"""Replace common vocab with rarer alternatives."""
|
||||||
|
words = text.lower().split()
|
||||||
|
result = []
|
||||||
|
for word in words:
|
||||||
|
if word in VOCAB_SUBSTITUTIONS and random.random() < 0.5:
|
||||||
|
result.append(random.choice(VOCAB_SUBSTITUTIONS[word]))
|
||||||
|
else:
|
||||||
|
result.append(word)
|
||||||
|
return " ".join(result)
|
||||||
|
|
||||||
|
|
||||||
|
def generate_v2_cases() -> list[dict[str, Any]]:
|
||||||
|
"""Generate v2 test cases with deeper nesting and paraphrased surfaces."""
|
||||||
|
cases = []
|
||||||
|
case_id = 1
|
||||||
|
|
||||||
|
# C01: Simple declarative → v2 with alternative surfaces
|
||||||
|
for i in range(4):
|
||||||
|
vocab = random.choice(list(VOCAB_SUBSTITUTIONS.items()))
|
||||||
|
subj, subj_alts = vocab
|
||||||
|
pred_pairs = [
|
||||||
|
("reveals", ["discloses", "manifests", "expounds"]),
|
||||||
|
("supports", ["buttresses", "reinforces", "underpins"]),
|
||||||
|
("follows", ["ensues", "supervenes"]),
|
||||||
|
("grounds", ["underwrites", "substantiates"]),
|
||||||
|
]
|
||||||
|
pred, pred_alts = random.choice(pred_pairs)
|
||||||
|
obj = random.choice(["verity", "cognition", "authenticity", "sapience"])
|
||||||
|
|
||||||
|
surfaces = [
|
||||||
|
f"{subj} {pred} {obj}",
|
||||||
|
f"{subj_alts[0]} {pred_alts[0]} {obj}",
|
||||||
|
f"{subj} {pred_alts[0]} {obj}",
|
||||||
|
]
|
||||||
|
|
||||||
|
cases.append({
|
||||||
|
"id": f"gram_C01_v2_{case_id:02d}",
|
||||||
|
"construction": "C01",
|
||||||
|
"construction_name": "simple_declarative",
|
||||||
|
"proposition_graph": {
|
||||||
|
"nodes": [
|
||||||
|
{
|
||||||
|
"node_id": "n1",
|
||||||
|
"subject": subj,
|
||||||
|
"predicate": pred,
|
||||||
|
"obj": obj,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"edges": [],
|
||||||
|
},
|
||||||
|
"accept_surfaces": surfaces[:2],
|
||||||
|
"constraints": {
|
||||||
|
"must_contain": [subj, pred, obj],
|
||||||
|
"max_words": 8,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
case_id += 1
|
||||||
|
|
||||||
|
# C02: Negation → v2 with paraphrasing
|
||||||
|
for i in range(3):
|
||||||
|
subj = random.choice(["truth", "evidence", "knowledge", "wisdom"])
|
||||||
|
pred = random.choice(["requires", "supports", "necessitates", "entails"])
|
||||||
|
obj = random.choice(["force", "artifice", "deception", "sophistry"])
|
||||||
|
|
||||||
|
surfaces = [
|
||||||
|
f"{subj} does not {pred} {obj}",
|
||||||
|
f"{subj} {pred} no {obj}",
|
||||||
|
f"it is not the case that {subj} {pred} {obj}",
|
||||||
|
]
|
||||||
|
|
||||||
|
cases.append({
|
||||||
|
"id": f"gram_C02_v2_{case_id:02d}",
|
||||||
|
"construction": "C02",
|
||||||
|
"construction_name": "negation",
|
||||||
|
"proposition_graph": {
|
||||||
|
"nodes": [
|
||||||
|
{
|
||||||
|
"node_id": "n1",
|
||||||
|
"subject": subj,
|
||||||
|
"predicate": pred,
|
||||||
|
"obj": obj,
|
||||||
|
"negated": True,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"edges": [],
|
||||||
|
},
|
||||||
|
"accept_surfaces": surfaces[:2],
|
||||||
|
"constraints": {
|
||||||
|
"must_contain": [subj, obj],
|
||||||
|
"max_words": 12,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
case_id += 1
|
||||||
|
|
||||||
|
# C03: Conjunction → v2 with multiple items
|
||||||
|
for i in range(3):
|
||||||
|
items = random.sample(
|
||||||
|
["light", "truth", "knowledge", "wisdom", "evidence"],
|
||||||
|
k=3,
|
||||||
|
)
|
||||||
|
|
||||||
|
surfaces = [
|
||||||
|
f"{items[0]}, {items[1]}, and {items[2]} ground understanding",
|
||||||
|
f"{items[0]}, as well as {items[1]} and {items[2]}, support inquiry",
|
||||||
|
]
|
||||||
|
|
||||||
|
nodes = []
|
||||||
|
for idx, item in enumerate(items):
|
||||||
|
nodes.append({
|
||||||
|
"node_id": f"n{idx+1}",
|
||||||
|
"subject": item,
|
||||||
|
"predicate": "ground" if idx == 0 else "support",
|
||||||
|
"obj": "understanding",
|
||||||
|
})
|
||||||
|
|
||||||
|
cases.append({
|
||||||
|
"id": f"gram_C03_v2_{case_id:02d}",
|
||||||
|
"construction": "C03",
|
||||||
|
"construction_name": "conjunction",
|
||||||
|
"proposition_graph": {
|
||||||
|
"nodes": nodes,
|
||||||
|
"edges": [
|
||||||
|
{"source": f"n{i}", "target": f"n{i+1}", "relation": "conjunction"}
|
||||||
|
for i in range(len(items) - 1)
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"accept_surfaces": surfaces,
|
||||||
|
"constraints": {
|
||||||
|
"must_contain": items + ["ground", "understanding"],
|
||||||
|
"max_words": 14,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
case_id += 1
|
||||||
|
|
||||||
|
# C04: Disjunction → v2
|
||||||
|
for i in range(2):
|
||||||
|
opt1 = random.choice(["light", "darkness"])
|
||||||
|
opt2 = random.choice(["truth", "illusion"])
|
||||||
|
result = random.choice(["precedes dawn", "precedes enlightenment"])
|
||||||
|
|
||||||
|
surfaces = [
|
||||||
|
f"either {opt1} or {opt2} {result}",
|
||||||
|
f"{opt1} or {opt2} {result}",
|
||||||
|
]
|
||||||
|
|
||||||
|
cases.append({
|
||||||
|
"id": f"gram_C04_v2_{case_id:02d}",
|
||||||
|
"construction": "C04",
|
||||||
|
"construction_name": "disjunction",
|
||||||
|
"proposition_graph": {
|
||||||
|
"nodes": [
|
||||||
|
{"node_id": "n1", "subject": opt1, "predicate": result.split()[1], "obj": "dawn"},
|
||||||
|
{"node_id": "n2", "subject": opt2, "predicate": result.split()[1], "obj": "dawn"},
|
||||||
|
],
|
||||||
|
"edges": [
|
||||||
|
{"source": "n1", "target": "n2", "relation": "disjunction"}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"accept_surfaces": surfaces,
|
||||||
|
"constraints": {
|
||||||
|
"must_contain": [opt1, opt2, "or"],
|
||||||
|
"max_words": 10,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
case_id += 1
|
||||||
|
|
||||||
|
# C05: Embedded clause (deeper nesting) → v2 with complex embedding
|
||||||
|
for i in range(3):
|
||||||
|
surfaces = [
|
||||||
|
"wisdom demonstrates that evidence substantiates the claim that truth underlies all inquiry",
|
||||||
|
"sage revelation discloses that corroboration supports the thesis that authenticity grounds knowledge",
|
||||||
|
"discernment shows that documentation proves that verity underlies cognition",
|
||||||
|
]
|
||||||
|
|
||||||
|
cases.append({
|
||||||
|
"id": f"gram_C05_v2_{case_id:02d}",
|
||||||
|
"construction": "C05",
|
||||||
|
"construction_name": "embedded_clause",
|
||||||
|
"proposition_graph": {
|
||||||
|
"nodes": [
|
||||||
|
{"node_id": "n1", "subject": "wisdom", "predicate": "demonstrates", "obj": "n2"},
|
||||||
|
{"node_id": "n2", "subject": "evidence", "predicate": "substantiates", "obj": "n3"},
|
||||||
|
{"node_id": "n3", "subject": "truth", "predicate": "underlies", "obj": "inquiry"},
|
||||||
|
],
|
||||||
|
"edges": [
|
||||||
|
{"source": "n1", "target": "n2", "relation": "complement"},
|
||||||
|
{"source": "n2", "target": "n3", "relation": "complement"},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"accept_surfaces": [surfaces[i]],
|
||||||
|
"constraints": {
|
||||||
|
"must_contain": ["that"],
|
||||||
|
"max_words": 16,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
case_id += 1
|
||||||
|
|
||||||
|
# C06: Relative clause → v2 with longer nesting
|
||||||
|
for i in range(2):
|
||||||
|
surfaces = [
|
||||||
|
"verity, which undergirds all cognition, reveals the path toward illumination",
|
||||||
|
"authenticity, which substantiates epistemic inquiry, discloses wisdom",
|
||||||
|
]
|
||||||
|
|
||||||
|
cases.append({
|
||||||
|
"id": f"gram_C06_v2_{case_id:02d}",
|
||||||
|
"construction": "C06",
|
||||||
|
"construction_name": "relative_clause",
|
||||||
|
"proposition_graph": {
|
||||||
|
"nodes": [
|
||||||
|
{"node_id": "n1", "subject": "truth", "predicate": "undergirds", "obj": "cognition"},
|
||||||
|
{"node_id": "n2", "subject": "truth", "predicate": "reveals", "obj": "path"},
|
||||||
|
],
|
||||||
|
"edges": [
|
||||||
|
{"source": "n1", "target": "n2", "relation": "relative"}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"accept_surfaces": [surfaces[i]],
|
||||||
|
"constraints": {
|
||||||
|
"must_contain": ["which", "truth"],
|
||||||
|
"max_words": 14,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
case_id += 1
|
||||||
|
|
||||||
|
# C07: Universal quantification → v2 with paraphrasing
|
||||||
|
# Note: realizer produces "all/every X verb Y", not complex paraphrases
|
||||||
|
for i in range(2):
|
||||||
|
subj = random.choice(["light", "knowledge"])
|
||||||
|
pred = random.choice(["reveals", "grounds"])
|
||||||
|
obj = random.choice(["truth", "understanding"])
|
||||||
|
|
||||||
|
surfaces = [
|
||||||
|
f"all {subj} {pred} {obj}",
|
||||||
|
f"every {subj} {pred} {obj}",
|
||||||
|
]
|
||||||
|
|
||||||
|
cases.append({
|
||||||
|
"id": f"gram_C07_v2_{case_id:02d}",
|
||||||
|
"construction": "C07",
|
||||||
|
"construction_name": "universal_quantification",
|
||||||
|
"proposition_graph": {
|
||||||
|
"nodes": [
|
||||||
|
{
|
||||||
|
"node_id": "n1",
|
||||||
|
"subject": subj,
|
||||||
|
"predicate": pred,
|
||||||
|
"obj": obj,
|
||||||
|
"quantifier": "all",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"edges": [],
|
||||||
|
},
|
||||||
|
"accept_surfaces": surfaces,
|
||||||
|
"constraints": {
|
||||||
|
"must_contain": [subj, pred, obj],
|
||||||
|
"max_words": 8,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
case_id += 1
|
||||||
|
|
||||||
|
# C08: Existential quantification → v2
|
||||||
|
for i in range(2):
|
||||||
|
surfaces = [
|
||||||
|
"certain instances of substantiation corroborate the foundations of knowledge",
|
||||||
|
"particular forms of evidence ground epistemic certainty",
|
||||||
|
]
|
||||||
|
|
||||||
|
cases.append({
|
||||||
|
"id": f"gram_C08_v2_{case_id:02d}",
|
||||||
|
"construction": "C08",
|
||||||
|
"construction_name": "existential_quantification",
|
||||||
|
"proposition_graph": {
|
||||||
|
"nodes": [
|
||||||
|
{
|
||||||
|
"node_id": "n1",
|
||||||
|
"subject": "evidence",
|
||||||
|
"predicate": "grounds",
|
||||||
|
"obj": "knowledge",
|
||||||
|
"quantifier": "some",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"edges": [],
|
||||||
|
},
|
||||||
|
"accept_surfaces": [surfaces[i]],
|
||||||
|
"constraints": {
|
||||||
|
"must_contain": ["some"],
|
||||||
|
"max_words": 10,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
case_id += 1
|
||||||
|
|
||||||
|
# C09: Past tense → v2 with paraphrasing
|
||||||
|
for i in range(3):
|
||||||
|
subj = random.choice(["wisdom", "knowledge", "evidence"])
|
||||||
|
obj = random.choice(["truth", "inquiry", "enlightenment"])
|
||||||
|
|
||||||
|
surfaces = [
|
||||||
|
f"the {subj} previously disclosed the nature of {obj}",
|
||||||
|
f"{subj} had substantiated principles of {obj}",
|
||||||
|
f"historical {subj} undergirded {obj}",
|
||||||
|
]
|
||||||
|
|
||||||
|
cases.append({
|
||||||
|
"id": f"gram_C09_v2_{case_id:02d}",
|
||||||
|
"construction": "C09",
|
||||||
|
"construction_name": "past_tense",
|
||||||
|
"proposition_graph": {
|
||||||
|
"nodes": [
|
||||||
|
{
|
||||||
|
"node_id": "n1",
|
||||||
|
"subject": subj,
|
||||||
|
"predicate": "disclosed",
|
||||||
|
"obj": obj,
|
||||||
|
"tense": "past",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"edges": [],
|
||||||
|
},
|
||||||
|
"accept_surfaces": surfaces[:2],
|
||||||
|
"constraints": {
|
||||||
|
"max_words": 10,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
case_id += 1
|
||||||
|
|
||||||
|
# C10: Present tense → v2 with alternative constructions
|
||||||
|
for i in range(3):
|
||||||
|
obj = random.choice(["verity", "cognition", "authenticity"])
|
||||||
|
|
||||||
|
surfaces = [
|
||||||
|
f"illumination perpetually sustains {obj}",
|
||||||
|
f"radiance continuously buttresses {obj}",
|
||||||
|
f"light presently undergirds {obj}",
|
||||||
|
]
|
||||||
|
|
||||||
|
cases.append({
|
||||||
|
"id": f"gram_C10_v2_{case_id:02d}",
|
||||||
|
"construction": "C10",
|
||||||
|
"construction_name": "present_tense",
|
||||||
|
"proposition_graph": {
|
||||||
|
"nodes": [
|
||||||
|
{
|
||||||
|
"node_id": "n1",
|
||||||
|
"subject": "light",
|
||||||
|
"predicate": "sustains",
|
||||||
|
"obj": obj,
|
||||||
|
"tense": "present",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"edges": [],
|
||||||
|
},
|
||||||
|
"accept_surfaces": surfaces[:2],
|
||||||
|
"constraints": {
|
||||||
|
"max_words": 8,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
case_id += 1
|
||||||
|
|
||||||
|
# C11: Future tense → v2
|
||||||
|
for i in range(3):
|
||||||
|
obj = random.choice(["wisdom", "understanding", "enlightenment"])
|
||||||
|
|
||||||
|
surfaces = [
|
||||||
|
f"truth shall eventually illuminate the path to {obj}",
|
||||||
|
f"verity will perpetually guide seekers toward {obj}",
|
||||||
|
f"authenticity is destined to undergird {obj}",
|
||||||
|
]
|
||||||
|
|
||||||
|
cases.append({
|
||||||
|
"id": f"gram_C11_v2_{case_id:02d}",
|
||||||
|
"construction": "C11",
|
||||||
|
"construction_name": "future_tense",
|
||||||
|
"proposition_graph": {
|
||||||
|
"nodes": [
|
||||||
|
{
|
||||||
|
"node_id": "n1",
|
||||||
|
"subject": "truth",
|
||||||
|
"predicate": "illuminate",
|
||||||
|
"obj": obj,
|
||||||
|
"tense": "future",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"edges": [],
|
||||||
|
},
|
||||||
|
"accept_surfaces": surfaces[:2],
|
||||||
|
"constraints": {
|
||||||
|
"max_words": 12,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
case_id += 1
|
||||||
|
|
||||||
|
# C12: Perfective aspect → v2
|
||||||
|
for i in range(3):
|
||||||
|
obj = random.choice(["our understanding", "the inquiry", "the discourse"])
|
||||||
|
|
||||||
|
surfaces = [
|
||||||
|
f"wisdom has conclusively established the foundations of {obj}",
|
||||||
|
f"evidence has thoroughly substantiated principles guiding {obj}",
|
||||||
|
f"truth has permanently undergirded {obj}",
|
||||||
|
]
|
||||||
|
|
||||||
|
cases.append({
|
||||||
|
"id": f"gram_C12_v2_{case_id:02d}",
|
||||||
|
"construction": "C12",
|
||||||
|
"construction_name": "perfective_aspect",
|
||||||
|
"proposition_graph": {
|
||||||
|
"nodes": [
|
||||||
|
{
|
||||||
|
"node_id": "n1",
|
||||||
|
"subject": "wisdom",
|
||||||
|
"predicate": "established",
|
||||||
|
"obj": obj,
|
||||||
|
"aspect": "perfective",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"edges": [],
|
||||||
|
},
|
||||||
|
"accept_surfaces": surfaces[:2],
|
||||||
|
"constraints": {
|
||||||
|
"must_contain": ["has"],
|
||||||
|
"max_words": 12,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
case_id += 1
|
||||||
|
|
||||||
|
# C13: Imperfective aspect → v2
|
||||||
|
for i in range(3):
|
||||||
|
activity = random.choice(["revealing truth", "substantiating claims", "undergirding knowledge"])
|
||||||
|
|
||||||
|
surfaces = [
|
||||||
|
f"light is continuously {activity}",
|
||||||
|
f"wisdom remains perpetually {activity}",
|
||||||
|
f"evidence is perpetually {activity}",
|
||||||
|
]
|
||||||
|
|
||||||
|
cases.append({
|
||||||
|
"id": f"gram_C13_v2_{case_id:02d}",
|
||||||
|
"construction": "C13",
|
||||||
|
"construction_name": "imperfective_aspect",
|
||||||
|
"proposition_graph": {
|
||||||
|
"nodes": [
|
||||||
|
{
|
||||||
|
"node_id": "n1",
|
||||||
|
"subject": "light",
|
||||||
|
"predicate": "revealing",
|
||||||
|
"obj": "truth",
|
||||||
|
"aspect": "imperfective",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"edges": [],
|
||||||
|
},
|
||||||
|
"accept_surfaces": surfaces[:2],
|
||||||
|
"constraints": {
|
||||||
|
"must_contain": ["is"],
|
||||||
|
"max_words": 10,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
case_id += 1
|
||||||
|
|
||||||
|
return cases
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
cases = generate_v2_cases()
|
||||||
|
|
||||||
|
# Write v2 public test set
|
||||||
|
output_path = "evals/grammatical_coverage/public/v2/cases.jsonl"
|
||||||
|
import os
|
||||||
|
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||||
|
|
||||||
|
with open(output_path, "w") as f:
|
||||||
|
for case in cases:
|
||||||
|
f.write(json.dumps(case) + "\n")
|
||||||
|
|
||||||
|
print(f"Generated {len(cases)} v2 cases")
|
||||||
|
print(f"Wrote to {output_path}")
|
||||||
|
|
||||||
|
# Show distribution
|
||||||
|
by_construction = defaultdict(int)
|
||||||
|
for case in cases:
|
||||||
|
by_construction[case["construction"]] += 1
|
||||||
|
|
||||||
|
print("\nDistribution:")
|
||||||
|
for c in sorted(by_construction.keys()):
|
||||||
|
print(f" {c}: {by_construction[c]}")
|
||||||
Loading…
Reference in a new issue