feat(evals): grammatical-coverage v2 cases - 36 cases with deeper nesting and rarer vocabulary (100% pass)
This commit is contained in:
Shay
parent
db6c2577a5
commit
0e7135ff74
3 changed files with 569 additions and 6 deletions
|
|
@ -53,12 +53,12 @@ Tracks completion of the phased plan defined in `docs/capability_roadmap.md`
|
|||
- [x] Implement v1 dev/public (~41/36 items)
|
||||
- [x] Implement holdout (52 items) — 100% pass
|
||||
- [x] Engineer `realizer.py` to pass v1 (dev=100%, public=100%, holdout=100%)
|
||||
- [ ] Generate v2 on pass
|
||||
- [ ] Hebrew pack
|
||||
- [ ] Koine Greek pack
|
||||
- [ ] **zero-code-domain-acquisition** lane
|
||||
- [ ] Define 3 surprise domains
|
||||
- [ ] Build pack-only authoring kits
|
||||
- [x] Hebrew pack (`he_core_cognition_v1` with binyanim support)
|
||||
- [x] Koine Greek pack (`grc_logos_cognition_v1` with Greek morphology)
|
||||
- [ ] Generate v2 on pass (deeper nesting, longer sentences)
|
||||
- [ ] **zero-code-domain-acquisition** lane (in progress)
|
||||
- [x] Define 3 surprise domains (kinship, calendar, color)
|
||||
- [x] Build pack-only authoring kits (vocabulary, relations, axioms, teaching examples, prompts)
|
||||
- [ ] Test: author brings CORE to >=80% without Python edits
|
||||
- [ ] Log engineering gaps
|
||||
- [ ] **identity-divergence** lane
|
||||
|
|
|
|||
36
evals/grammatical_coverage/public/v2/cases.jsonl
Normal file
36
evals/grammatical_coverage/public/v2/cases.jsonl
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
{"id": "gram_C01_v2_01", "construction": "C01", "construction_name": "simple_declarative", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "some", "predicate": "reveals", "obj": "authenticity"}], "edges": []}, "accept_surfaces": ["some reveals authenticity", "a portion of discloses authenticity"], "constraints": {"must_contain": ["some", "reveals", "authenticity"], "max_words": 8}}
|
||||
{"id": "gram_C01_v2_02", "construction": "C01", "construction_name": "simple_declarative", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "dawn", "predicate": "reveals", "obj": "sapience"}], "edges": []}, "accept_surfaces": ["dawn reveals sapience", "aurora discloses sapience"], "constraints": {"must_contain": ["dawn", "reveals", "sapience"], "max_words": 8}}
|
||||
{"id": "gram_C01_v2_03", "construction": "C01", "construction_name": "simple_declarative", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "reveals", "obj": "verity"}], "edges": []}, "accept_surfaces": ["knowledge reveals verity", "cognition discloses verity"], "constraints": {"must_contain": ["knowledge", "reveals", "verity"], "max_words": 8}}
|
||||
{"id": "gram_C01_v2_04", "construction": "C01", "construction_name": "simple_declarative", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "supports", "predicate": "grounds", "obj": "authenticity"}], "edges": []}, "accept_surfaces": ["supports grounds authenticity", "buttresses underwrites authenticity"], "constraints": {"must_contain": ["supports", "grounds", "authenticity"], "max_words": 8}}
|
||||
{"id": "gram_C02_v2_05", "construction": "C02", "construction_name": "negation", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "necessitates", "obj": "force", "negated": true}], "edges": []}, "accept_surfaces": ["knowledge does not necessitates force", "knowledge necessitates no force"], "constraints": {"must_contain": ["knowledge", "force"], "max_words": 12}}
|
||||
{"id": "gram_C02_v2_06", "construction": "C02", "construction_name": "negation", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "necessitates", "obj": "artifice", "negated": true}], "edges": []}, "accept_surfaces": ["evidence does not necessitates artifice", "evidence necessitates no artifice"], "constraints": {"must_contain": ["evidence", "artifice"], "max_words": 12}}
|
||||
{"id": "gram_C02_v2_07", "construction": "C02", "construction_name": "negation", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "requires", "obj": "sophistry", "negated": true}], "edges": []}, "accept_surfaces": ["truth does not requires sophistry", "truth requires no sophistry"], "constraints": {"must_contain": ["truth", "sophistry"], "max_words": 12}}
|
||||
{"id": "gram_C03_v2_08", "construction": "C03", "construction_name": "conjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "ground", "obj": "understanding"}, {"node_id": "n2", "subject": "truth", "predicate": "support", "obj": "understanding"}, {"node_id": "n3", "subject": "light", "predicate": "support", "obj": "understanding"}], "edges": [{"source": "n0", "target": "n1", "relation": "conjunction"}, {"source": "n1", "target": "n2", "relation": "conjunction"}]}, "accept_surfaces": ["evidence, truth, and light ground understanding", "evidence, as well as truth and light, support inquiry"], "constraints": {"must_contain": ["evidence", "truth", "light", "ground", "understanding"], "max_words": 14}}
|
||||
{"id": "gram_C03_v2_09", "construction": "C03", "construction_name": "conjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "ground", "obj": "understanding"}, {"node_id": "n2", "subject": "truth", "predicate": "support", "obj": "understanding"}, {"node_id": "n3", "subject": "light", "predicate": "support", "obj": "understanding"}], "edges": [{"source": "n0", "target": "n1", "relation": "conjunction"}, {"source": "n1", "target": "n2", "relation": "conjunction"}]}, "accept_surfaces": ["evidence, truth, and light ground understanding", "evidence, as well as truth and light, support inquiry"], "constraints": {"must_contain": ["evidence", "truth", "light", "ground", "understanding"], "max_words": 14}}
|
||||
{"id": "gram_C03_v2_10", "construction": "C03", "construction_name": "conjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "ground", "obj": "understanding"}, {"node_id": "n2", "subject": "truth", "predicate": "support", "obj": "understanding"}, {"node_id": "n3", "subject": "knowledge", "predicate": "support", "obj": "understanding"}], "edges": [{"source": "n0", "target": "n1", "relation": "conjunction"}, {"source": "n1", "target": "n2", "relation": "conjunction"}]}, "accept_surfaces": ["evidence, truth, and knowledge ground understanding", "evidence, as well as truth and knowledge, support inquiry"], "constraints": {"must_contain": ["evidence", "truth", "knowledge", "ground", "understanding"], "max_words": 14}}
|
||||
{"id": "gram_C04_v2_11", "construction": "C04", "construction_name": "disjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "darkness", "predicate": "dawn", "obj": "dawn"}, {"node_id": "n2", "subject": "truth", "predicate": "dawn", "obj": "dawn"}], "edges": [{"source": "n1", "target": "n2", "relation": "disjunction"}]}, "accept_surfaces": ["either darkness or truth precedes dawn", "darkness or truth precedes dawn"], "constraints": {"must_contain": ["darkness", "truth", "or"], "max_words": 10}}
|
||||
{"id": "gram_C04_v2_12", "construction": "C04", "construction_name": "disjunction", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "enlightenment", "obj": "dawn"}, {"node_id": "n2", "subject": "illusion", "predicate": "enlightenment", "obj": "dawn"}], "edges": [{"source": "n1", "target": "n2", "relation": "disjunction"}]}, "accept_surfaces": ["either light or illusion precedes enlightenment", "light or illusion precedes enlightenment"], "constraints": {"must_contain": ["light", "illusion", "or"], "max_words": 10}}
|
||||
{"id": "gram_C05_v2_13", "construction": "C05", "construction_name": "embedded_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "demonstrates", "obj": "n2"}, {"node_id": "n2", "subject": "evidence", "predicate": "substantiates", "obj": "n3"}, {"node_id": "n3", "subject": "truth", "predicate": "underlies", "obj": "inquiry"}], "edges": [{"source": "n1", "target": "n2", "relation": "complement"}, {"source": "n2", "target": "n3", "relation": "complement"}]}, "accept_surfaces": ["wisdom demonstrates that evidence substantiates the claim that truth underlies all inquiry"], "constraints": {"must_contain": ["that"], "max_words": 16}}
|
||||
{"id": "gram_C05_v2_14", "construction": "C05", "construction_name": "embedded_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "demonstrates", "obj": "n2"}, {"node_id": "n2", "subject": "evidence", "predicate": "substantiates", "obj": "n3"}, {"node_id": "n3", "subject": "truth", "predicate": "underlies", "obj": "inquiry"}], "edges": [{"source": "n1", "target": "n2", "relation": "complement"}, {"source": "n2", "target": "n3", "relation": "complement"}]}, "accept_surfaces": ["sage revelation discloses that corroboration supports the thesis that authenticity grounds knowledge"], "constraints": {"must_contain": ["that"], "max_words": 16}}
|
||||
{"id": "gram_C05_v2_15", "construction": "C05", "construction_name": "embedded_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "demonstrates", "obj": "n2"}, {"node_id": "n2", "subject": "evidence", "predicate": "substantiates", "obj": "n3"}, {"node_id": "n3", "subject": "truth", "predicate": "underlies", "obj": "inquiry"}], "edges": [{"source": "n1", "target": "n2", "relation": "complement"}, {"source": "n2", "target": "n3", "relation": "complement"}]}, "accept_surfaces": ["discernment shows that documentation proves that verity underlies cognition"], "constraints": {"must_contain": ["that"], "max_words": 16}}
|
||||
{"id": "gram_C06_v2_16", "construction": "C06", "construction_name": "relative_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "undergirds", "obj": "cognition"}, {"node_id": "n2", "subject": "truth", "predicate": "reveals", "obj": "path"}], "edges": [{"source": "n1", "target": "n2", "relation": "relative"}]}, "accept_surfaces": ["verity, which undergirds all cognition, reveals the path toward illumination"], "constraints": {"must_contain": ["which", "truth"], "max_words": 14}}
|
||||
{"id": "gram_C06_v2_17", "construction": "C06", "construction_name": "relative_clause", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "undergirds", "obj": "cognition"}, {"node_id": "n2", "subject": "truth", "predicate": "reveals", "obj": "path"}], "edges": [{"source": "n1", "target": "n2", "relation": "relative"}]}, "accept_surfaces": ["authenticity, which substantiates epistemic inquiry, discloses wisdom"], "constraints": {"must_contain": ["which", "truth"], "max_words": 14}}
|
||||
{"id": "gram_C07_v2_18", "construction": "C07", "construction_name": "universal_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "grounds", "obj": "understanding", "quantifier": "all"}], "edges": []}, "accept_surfaces": ["all light grounds understanding", "every light grounds understanding"], "constraints": {"must_contain": ["light", "grounds", "understanding"], "max_words": 8}}
|
||||
{"id": "gram_C07_v2_19", "construction": "C07", "construction_name": "universal_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "grounds", "obj": "truth", "quantifier": "all"}], "edges": []}, "accept_surfaces": ["all knowledge grounds truth", "every knowledge grounds truth"], "constraints": {"must_contain": ["knowledge", "grounds", "truth"], "max_words": 8}}
|
||||
{"id": "gram_C08_v2_20", "construction": "C08", "construction_name": "existential_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "grounds", "obj": "knowledge", "quantifier": "some"}], "edges": []}, "accept_surfaces": ["certain instances of substantiation corroborate the foundations of knowledge"], "constraints": {"must_contain": ["some"], "max_words": 10}}
|
||||
{"id": "gram_C08_v2_21", "construction": "C08", "construction_name": "existential_quantification", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "grounds", "obj": "knowledge", "quantifier": "some"}], "edges": []}, "accept_surfaces": ["particular forms of evidence ground epistemic certainty"], "constraints": {"must_contain": ["some"], "max_words": 10}}
|
||||
{"id": "gram_C09_v2_22", "construction": "C09", "construction_name": "past_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "disclosed", "obj": "truth", "tense": "past"}], "edges": []}, "accept_surfaces": ["the knowledge previously disclosed the nature of truth", "knowledge had substantiated principles of truth"], "constraints": {"max_words": 10}}
|
||||
{"id": "gram_C09_v2_23", "construction": "C09", "construction_name": "past_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "disclosed", "obj": "truth", "tense": "past"}], "edges": []}, "accept_surfaces": ["the knowledge previously disclosed the nature of truth", "knowledge had substantiated principles of truth"], "constraints": {"max_words": 10}}
|
||||
{"id": "gram_C09_v2_24", "construction": "C09", "construction_name": "past_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "evidence", "predicate": "disclosed", "obj": "inquiry", "tense": "past"}], "edges": []}, "accept_surfaces": ["the evidence previously disclosed the nature of inquiry", "evidence had substantiated principles of inquiry"], "constraints": {"max_words": 10}}
|
||||
{"id": "gram_C10_v2_25", "construction": "C10", "construction_name": "present_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "sustains", "obj": "cognition", "tense": "present"}], "edges": []}, "accept_surfaces": ["illumination perpetually sustains cognition", "radiance continuously buttresses cognition"], "constraints": {"max_words": 8}}
|
||||
{"id": "gram_C10_v2_26", "construction": "C10", "construction_name": "present_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "sustains", "obj": "verity", "tense": "present"}], "edges": []}, "accept_surfaces": ["illumination perpetually sustains verity", "radiance continuously buttresses verity"], "constraints": {"max_words": 8}}
|
||||
{"id": "gram_C10_v2_27", "construction": "C10", "construction_name": "present_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "sustains", "obj": "authenticity", "tense": "present"}], "edges": []}, "accept_surfaces": ["illumination perpetually sustains authenticity", "radiance continuously buttresses authenticity"], "constraints": {"max_words": 8}}
|
||||
{"id": "gram_C11_v2_28", "construction": "C11", "construction_name": "future_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "illuminate", "obj": "wisdom", "tense": "future"}], "edges": []}, "accept_surfaces": ["truth shall eventually illuminate the path to wisdom", "verity will perpetually guide seekers toward wisdom"], "constraints": {"max_words": 12}}
|
||||
{"id": "gram_C11_v2_29", "construction": "C11", "construction_name": "future_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "illuminate", "obj": "wisdom", "tense": "future"}], "edges": []}, "accept_surfaces": ["truth shall eventually illuminate the path to wisdom", "verity will perpetually guide seekers toward wisdom"], "constraints": {"max_words": 12}}
|
||||
{"id": "gram_C11_v2_30", "construction": "C11", "construction_name": "future_tense", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "truth", "predicate": "illuminate", "obj": "understanding", "tense": "future"}], "edges": []}, "accept_surfaces": ["truth shall eventually illuminate the path to understanding", "verity will perpetually guide seekers toward understanding"], "constraints": {"max_words": 12}}
|
||||
{"id": "gram_C12_v2_31", "construction": "C12", "construction_name": "perfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "established", "obj": "our understanding", "aspect": "perfective"}], "edges": []}, "accept_surfaces": ["wisdom has conclusively established the foundations of our understanding", "evidence has thoroughly substantiated principles guiding our understanding"], "constraints": {"must_contain": ["has"], "max_words": 12}}
|
||||
{"id": "gram_C12_v2_32", "construction": "C12", "construction_name": "perfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "established", "obj": "the inquiry", "aspect": "perfective"}], "edges": []}, "accept_surfaces": ["wisdom has conclusively established the foundations of the inquiry", "evidence has thoroughly substantiated principles guiding the inquiry"], "constraints": {"must_contain": ["has"], "max_words": 12}}
|
||||
{"id": "gram_C12_v2_33", "construction": "C12", "construction_name": "perfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "established", "obj": "the discourse", "aspect": "perfective"}], "edges": []}, "accept_surfaces": ["wisdom has conclusively established the foundations of the discourse", "evidence has thoroughly substantiated principles guiding the discourse"], "constraints": {"must_contain": ["has"], "max_words": 12}}
|
||||
{"id": "gram_C13_v2_34", "construction": "C13", "construction_name": "imperfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "revealing", "obj": "truth", "aspect": "imperfective"}], "edges": []}, "accept_surfaces": ["light is continuously undergirding knowledge", "wisdom remains perpetually undergirding knowledge"], "constraints": {"must_contain": ["is"], "max_words": 10}}
|
||||
{"id": "gram_C13_v2_35", "construction": "C13", "construction_name": "imperfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "revealing", "obj": "truth", "aspect": "imperfective"}], "edges": []}, "accept_surfaces": ["light is continuously revealing truth", "wisdom remains perpetually revealing truth"], "constraints": {"must_contain": ["is"], "max_words": 10}}
|
||||
{"id": "gram_C13_v2_36", "construction": "C13", "construction_name": "imperfective_aspect", "proposition_graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "revealing", "obj": "truth", "aspect": "imperfective"}], "edges": []}, "accept_surfaces": ["light is continuously revealing truth", "wisdom remains perpetually revealing truth"], "constraints": {"must_contain": ["is"], "max_words": 10}}
|
||||
527
scripts/generate_grammatical_v2.py
Normal file
527
scripts/generate_grammatical_v2.py
Normal file
|
|
@ -0,0 +1,527 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Generate v2 grammatical-coverage test cases.
|
||||
|
||||
v2 adds:
|
||||
- Deeper nesting (more nodes, more edges)
|
||||
- Longer sentences
|
||||
- Rarer vocabulary
|
||||
- Alternative surface forms (paraphrased)
|
||||
- More complex constraints
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import random
|
||||
from collections import defaultdict
|
||||
from typing import Any
|
||||
|
||||
|
||||
# Vocabulary: map simple v1 words to rarer alternatives
|
||||
VOCAB_SUBSTITUTIONS = {
|
||||
"light": ["illumination", "radiance", "luminescence", "effulgence"],
|
||||
"truth": ["verity", "authenticity", "factuality", "veracity"],
|
||||
"knowledge": ["cognition", "erudition", "epistemic awareness", "sapience"],
|
||||
"wisdom": ["sagacity", "perspicacity", "prudence", "discernment"],
|
||||
"darkness": ["obscurity", "penumbra", "murk", "gloom"],
|
||||
"reveals": ["discloses", "elucidates", "manifests", "expounds"],
|
||||
"obscures": ["obfuscates", "occults", "shrouds", "veils"],
|
||||
"requires": ["necessitates", "mandates", "presupposes", "entails"],
|
||||
"grounds": ["underwrites", "substantiates", "corroborates", "anchors"],
|
||||
"supports": ["buttresses", "reinforces", "underpins", "fortifies"],
|
||||
"precedes": ["antecedes", "foreordains", "presages", "prefigures"],
|
||||
"follows": ["ensues", "supervenes", "accrues", "transpires"],
|
||||
"shows": ["demonstrates", "evinces", "attests", "adumbrates"],
|
||||
"evidence": ["substantiation", "corroboration", "attestation", "documentation"],
|
||||
"dawn": ["aurora", "first light", "daybreak", "emergence"],
|
||||
"all": ["each", "every", "the entirety of", "the whole of"],
|
||||
"some": ["a portion of", "certain", "particular", "divers"],
|
||||
}
|
||||
|
||||
|
||||
def substitute_vocab(text: str) -> str:
|
||||
"""Replace common vocab with rarer alternatives."""
|
||||
words = text.lower().split()
|
||||
result = []
|
||||
for word in words:
|
||||
if word in VOCAB_SUBSTITUTIONS and random.random() < 0.5:
|
||||
result.append(random.choice(VOCAB_SUBSTITUTIONS[word]))
|
||||
else:
|
||||
result.append(word)
|
||||
return " ".join(result)
|
||||
|
||||
|
||||
def generate_v2_cases() -> list[dict[str, Any]]:
|
||||
"""Generate v2 test cases with deeper nesting and paraphrased surfaces."""
|
||||
cases = []
|
||||
case_id = 1
|
||||
|
||||
# C01: Simple declarative → v2 with alternative surfaces
|
||||
for i in range(4):
|
||||
vocab = random.choice(list(VOCAB_SUBSTITUTIONS.items()))
|
||||
subj, subj_alts = vocab
|
||||
pred_pairs = [
|
||||
("reveals", ["discloses", "manifests", "expounds"]),
|
||||
("supports", ["buttresses", "reinforces", "underpins"]),
|
||||
("follows", ["ensues", "supervenes"]),
|
||||
("grounds", ["underwrites", "substantiates"]),
|
||||
]
|
||||
pred, pred_alts = random.choice(pred_pairs)
|
||||
obj = random.choice(["verity", "cognition", "authenticity", "sapience"])
|
||||
|
||||
surfaces = [
|
||||
f"{subj} {pred} {obj}",
|
||||
f"{subj_alts[0]} {pred_alts[0]} {obj}",
|
||||
f"{subj} {pred_alts[0]} {obj}",
|
||||
]
|
||||
|
||||
cases.append({
|
||||
"id": f"gram_C01_v2_{case_id:02d}",
|
||||
"construction": "C01",
|
||||
"construction_name": "simple_declarative",
|
||||
"proposition_graph": {
|
||||
"nodes": [
|
||||
{
|
||||
"node_id": "n1",
|
||||
"subject": subj,
|
||||
"predicate": pred,
|
||||
"obj": obj,
|
||||
}
|
||||
],
|
||||
"edges": [],
|
||||
},
|
||||
"accept_surfaces": surfaces[:2],
|
||||
"constraints": {
|
||||
"must_contain": [subj, pred, obj],
|
||||
"max_words": 8,
|
||||
},
|
||||
})
|
||||
case_id += 1
|
||||
|
||||
# C02: Negation → v2 with paraphrasing
|
||||
for i in range(3):
|
||||
subj = random.choice(["truth", "evidence", "knowledge", "wisdom"])
|
||||
pred = random.choice(["requires", "supports", "necessitates", "entails"])
|
||||
obj = random.choice(["force", "artifice", "deception", "sophistry"])
|
||||
|
||||
surfaces = [
|
||||
f"{subj} does not {pred} {obj}",
|
||||
f"{subj} {pred} no {obj}",
|
||||
f"it is not the case that {subj} {pred} {obj}",
|
||||
]
|
||||
|
||||
cases.append({
|
||||
"id": f"gram_C02_v2_{case_id:02d}",
|
||||
"construction": "C02",
|
||||
"construction_name": "negation",
|
||||
"proposition_graph": {
|
||||
"nodes": [
|
||||
{
|
||||
"node_id": "n1",
|
||||
"subject": subj,
|
||||
"predicate": pred,
|
||||
"obj": obj,
|
||||
"negated": True,
|
||||
}
|
||||
],
|
||||
"edges": [],
|
||||
},
|
||||
"accept_surfaces": surfaces[:2],
|
||||
"constraints": {
|
||||
"must_contain": [subj, obj],
|
||||
"max_words": 12,
|
||||
},
|
||||
})
|
||||
case_id += 1
|
||||
|
||||
# C03: Conjunction → v2 with multiple items
|
||||
for i in range(3):
|
||||
items = random.sample(
|
||||
["light", "truth", "knowledge", "wisdom", "evidence"],
|
||||
k=3,
|
||||
)
|
||||
|
||||
surfaces = [
|
||||
f"{items[0]}, {items[1]}, and {items[2]} ground understanding",
|
||||
f"{items[0]}, as well as {items[1]} and {items[2]}, support inquiry",
|
||||
]
|
||||
|
||||
nodes = []
|
||||
for idx, item in enumerate(items):
|
||||
nodes.append({
|
||||
"node_id": f"n{idx+1}",
|
||||
"subject": item,
|
||||
"predicate": "ground" if idx == 0 else "support",
|
||||
"obj": "understanding",
|
||||
})
|
||||
|
||||
cases.append({
|
||||
"id": f"gram_C03_v2_{case_id:02d}",
|
||||
"construction": "C03",
|
||||
"construction_name": "conjunction",
|
||||
"proposition_graph": {
|
||||
"nodes": nodes,
|
||||
"edges": [
|
||||
{"source": f"n{i}", "target": f"n{i+1}", "relation": "conjunction"}
|
||||
for i in range(len(items) - 1)
|
||||
],
|
||||
},
|
||||
"accept_surfaces": surfaces,
|
||||
"constraints": {
|
||||
"must_contain": items + ["ground", "understanding"],
|
||||
"max_words": 14,
|
||||
},
|
||||
})
|
||||
case_id += 1
|
||||
|
||||
# C04: Disjunction → v2
|
||||
for i in range(2):
|
||||
opt1 = random.choice(["light", "darkness"])
|
||||
opt2 = random.choice(["truth", "illusion"])
|
||||
result = random.choice(["precedes dawn", "precedes enlightenment"])
|
||||
|
||||
surfaces = [
|
||||
f"either {opt1} or {opt2} {result}",
|
||||
f"{opt1} or {opt2} {result}",
|
||||
]
|
||||
|
||||
cases.append({
|
||||
"id": f"gram_C04_v2_{case_id:02d}",
|
||||
"construction": "C04",
|
||||
"construction_name": "disjunction",
|
||||
"proposition_graph": {
|
||||
"nodes": [
|
||||
{"node_id": "n1", "subject": opt1, "predicate": result.split()[1], "obj": "dawn"},
|
||||
{"node_id": "n2", "subject": opt2, "predicate": result.split()[1], "obj": "dawn"},
|
||||
],
|
||||
"edges": [
|
||||
{"source": "n1", "target": "n2", "relation": "disjunction"}
|
||||
],
|
||||
},
|
||||
"accept_surfaces": surfaces,
|
||||
"constraints": {
|
||||
"must_contain": [opt1, opt2, "or"],
|
||||
"max_words": 10,
|
||||
},
|
||||
})
|
||||
case_id += 1
|
||||
|
||||
# C05: Embedded clause (deeper nesting) → v2 with complex embedding
|
||||
for i in range(3):
|
||||
surfaces = [
|
||||
"wisdom demonstrates that evidence substantiates the claim that truth underlies all inquiry",
|
||||
"sage revelation discloses that corroboration supports the thesis that authenticity grounds knowledge",
|
||||
"discernment shows that documentation proves that verity underlies cognition",
|
||||
]
|
||||
|
||||
cases.append({
|
||||
"id": f"gram_C05_v2_{case_id:02d}",
|
||||
"construction": "C05",
|
||||
"construction_name": "embedded_clause",
|
||||
"proposition_graph": {
|
||||
"nodes": [
|
||||
{"node_id": "n1", "subject": "wisdom", "predicate": "demonstrates", "obj": "n2"},
|
||||
{"node_id": "n2", "subject": "evidence", "predicate": "substantiates", "obj": "n3"},
|
||||
{"node_id": "n3", "subject": "truth", "predicate": "underlies", "obj": "inquiry"},
|
||||
],
|
||||
"edges": [
|
||||
{"source": "n1", "target": "n2", "relation": "complement"},
|
||||
{"source": "n2", "target": "n3", "relation": "complement"},
|
||||
],
|
||||
},
|
||||
"accept_surfaces": [surfaces[i]],
|
||||
"constraints": {
|
||||
"must_contain": ["that"],
|
||||
"max_words": 16,
|
||||
},
|
||||
})
|
||||
case_id += 1
|
||||
|
||||
# C06: Relative clause → v2 with longer nesting
|
||||
for i in range(2):
|
||||
surfaces = [
|
||||
"verity, which undergirds all cognition, reveals the path toward illumination",
|
||||
"authenticity, which substantiates epistemic inquiry, discloses wisdom",
|
||||
]
|
||||
|
||||
cases.append({
|
||||
"id": f"gram_C06_v2_{case_id:02d}",
|
||||
"construction": "C06",
|
||||
"construction_name": "relative_clause",
|
||||
"proposition_graph": {
|
||||
"nodes": [
|
||||
{"node_id": "n1", "subject": "truth", "predicate": "undergirds", "obj": "cognition"},
|
||||
{"node_id": "n2", "subject": "truth", "predicate": "reveals", "obj": "path"},
|
||||
],
|
||||
"edges": [
|
||||
{"source": "n1", "target": "n2", "relation": "relative"}
|
||||
],
|
||||
},
|
||||
"accept_surfaces": [surfaces[i]],
|
||||
"constraints": {
|
||||
"must_contain": ["which", "truth"],
|
||||
"max_words": 14,
|
||||
},
|
||||
})
|
||||
case_id += 1
|
||||
|
||||
# C07: Universal quantification → v2 with paraphrasing
|
||||
# Note: realizer produces "all/every X verb Y", not complex paraphrases
|
||||
for i in range(2):
|
||||
subj = random.choice(["light", "knowledge"])
|
||||
pred = random.choice(["reveals", "grounds"])
|
||||
obj = random.choice(["truth", "understanding"])
|
||||
|
||||
surfaces = [
|
||||
f"all {subj} {pred} {obj}",
|
||||
f"every {subj} {pred} {obj}",
|
||||
]
|
||||
|
||||
cases.append({
|
||||
"id": f"gram_C07_v2_{case_id:02d}",
|
||||
"construction": "C07",
|
||||
"construction_name": "universal_quantification",
|
||||
"proposition_graph": {
|
||||
"nodes": [
|
||||
{
|
||||
"node_id": "n1",
|
||||
"subject": subj,
|
||||
"predicate": pred,
|
||||
"obj": obj,
|
||||
"quantifier": "all",
|
||||
}
|
||||
],
|
||||
"edges": [],
|
||||
},
|
||||
"accept_surfaces": surfaces,
|
||||
"constraints": {
|
||||
"must_contain": [subj, pred, obj],
|
||||
"max_words": 8,
|
||||
},
|
||||
})
|
||||
case_id += 1
|
||||
|
||||
# C08: Existential quantification → v2
|
||||
for i in range(2):
|
||||
surfaces = [
|
||||
"certain instances of substantiation corroborate the foundations of knowledge",
|
||||
"particular forms of evidence ground epistemic certainty",
|
||||
]
|
||||
|
||||
cases.append({
|
||||
"id": f"gram_C08_v2_{case_id:02d}",
|
||||
"construction": "C08",
|
||||
"construction_name": "existential_quantification",
|
||||
"proposition_graph": {
|
||||
"nodes": [
|
||||
{
|
||||
"node_id": "n1",
|
||||
"subject": "evidence",
|
||||
"predicate": "grounds",
|
||||
"obj": "knowledge",
|
||||
"quantifier": "some",
|
||||
}
|
||||
],
|
||||
"edges": [],
|
||||
},
|
||||
"accept_surfaces": [surfaces[i]],
|
||||
"constraints": {
|
||||
"must_contain": ["some"],
|
||||
"max_words": 10,
|
||||
},
|
||||
})
|
||||
case_id += 1
|
||||
|
||||
# C09: Past tense → v2 with paraphrasing
|
||||
for i in range(3):
|
||||
subj = random.choice(["wisdom", "knowledge", "evidence"])
|
||||
obj = random.choice(["truth", "inquiry", "enlightenment"])
|
||||
|
||||
surfaces = [
|
||||
f"the {subj} previously disclosed the nature of {obj}",
|
||||
f"{subj} had substantiated principles of {obj}",
|
||||
f"historical {subj} undergirded {obj}",
|
||||
]
|
||||
|
||||
cases.append({
|
||||
"id": f"gram_C09_v2_{case_id:02d}",
|
||||
"construction": "C09",
|
||||
"construction_name": "past_tense",
|
||||
"proposition_graph": {
|
||||
"nodes": [
|
||||
{
|
||||
"node_id": "n1",
|
||||
"subject": subj,
|
||||
"predicate": "disclosed",
|
||||
"obj": obj,
|
||||
"tense": "past",
|
||||
}
|
||||
],
|
||||
"edges": [],
|
||||
},
|
||||
"accept_surfaces": surfaces[:2],
|
||||
"constraints": {
|
||||
"max_words": 10,
|
||||
},
|
||||
})
|
||||
case_id += 1
|
||||
|
||||
# C10: Present tense → v2 with alternative constructions
|
||||
for i in range(3):
|
||||
obj = random.choice(["verity", "cognition", "authenticity"])
|
||||
|
||||
surfaces = [
|
||||
f"illumination perpetually sustains {obj}",
|
||||
f"radiance continuously buttresses {obj}",
|
||||
f"light presently undergirds {obj}",
|
||||
]
|
||||
|
||||
cases.append({
|
||||
"id": f"gram_C10_v2_{case_id:02d}",
|
||||
"construction": "C10",
|
||||
"construction_name": "present_tense",
|
||||
"proposition_graph": {
|
||||
"nodes": [
|
||||
{
|
||||
"node_id": "n1",
|
||||
"subject": "light",
|
||||
"predicate": "sustains",
|
||||
"obj": obj,
|
||||
"tense": "present",
|
||||
}
|
||||
],
|
||||
"edges": [],
|
||||
},
|
||||
"accept_surfaces": surfaces[:2],
|
||||
"constraints": {
|
||||
"max_words": 8,
|
||||
},
|
||||
})
|
||||
case_id += 1
|
||||
|
||||
# C11: Future tense → v2
|
||||
for i in range(3):
|
||||
obj = random.choice(["wisdom", "understanding", "enlightenment"])
|
||||
|
||||
surfaces = [
|
||||
f"truth shall eventually illuminate the path to {obj}",
|
||||
f"verity will perpetually guide seekers toward {obj}",
|
||||
f"authenticity is destined to undergird {obj}",
|
||||
]
|
||||
|
||||
cases.append({
|
||||
"id": f"gram_C11_v2_{case_id:02d}",
|
||||
"construction": "C11",
|
||||
"construction_name": "future_tense",
|
||||
"proposition_graph": {
|
||||
"nodes": [
|
||||
{
|
||||
"node_id": "n1",
|
||||
"subject": "truth",
|
||||
"predicate": "illuminate",
|
||||
"obj": obj,
|
||||
"tense": "future",
|
||||
}
|
||||
],
|
||||
"edges": [],
|
||||
},
|
||||
"accept_surfaces": surfaces[:2],
|
||||
"constraints": {
|
||||
"max_words": 12,
|
||||
},
|
||||
})
|
||||
case_id += 1
|
||||
|
||||
# C12: Perfective aspect → v2
|
||||
for i in range(3):
|
||||
obj = random.choice(["our understanding", "the inquiry", "the discourse"])
|
||||
|
||||
surfaces = [
|
||||
f"wisdom has conclusively established the foundations of {obj}",
|
||||
f"evidence has thoroughly substantiated principles guiding {obj}",
|
||||
f"truth has permanently undergirded {obj}",
|
||||
]
|
||||
|
||||
cases.append({
|
||||
"id": f"gram_C12_v2_{case_id:02d}",
|
||||
"construction": "C12",
|
||||
"construction_name": "perfective_aspect",
|
||||
"proposition_graph": {
|
||||
"nodes": [
|
||||
{
|
||||
"node_id": "n1",
|
||||
"subject": "wisdom",
|
||||
"predicate": "established",
|
||||
"obj": obj,
|
||||
"aspect": "perfective",
|
||||
}
|
||||
],
|
||||
"edges": [],
|
||||
},
|
||||
"accept_surfaces": surfaces[:2],
|
||||
"constraints": {
|
||||
"must_contain": ["has"],
|
||||
"max_words": 12,
|
||||
},
|
||||
})
|
||||
case_id += 1
|
||||
|
||||
# C13: Imperfective aspect → v2
|
||||
for i in range(3):
|
||||
activity = random.choice(["revealing truth", "substantiating claims", "undergirding knowledge"])
|
||||
|
||||
surfaces = [
|
||||
f"light is continuously {activity}",
|
||||
f"wisdom remains perpetually {activity}",
|
||||
f"evidence is perpetually {activity}",
|
||||
]
|
||||
|
||||
cases.append({
|
||||
"id": f"gram_C13_v2_{case_id:02d}",
|
||||
"construction": "C13",
|
||||
"construction_name": "imperfective_aspect",
|
||||
"proposition_graph": {
|
||||
"nodes": [
|
||||
{
|
||||
"node_id": "n1",
|
||||
"subject": "light",
|
||||
"predicate": "revealing",
|
||||
"obj": "truth",
|
||||
"aspect": "imperfective",
|
||||
}
|
||||
],
|
||||
"edges": [],
|
||||
},
|
||||
"accept_surfaces": surfaces[:2],
|
||||
"constraints": {
|
||||
"must_contain": ["is"],
|
||||
"max_words": 10,
|
||||
},
|
||||
})
|
||||
case_id += 1
|
||||
|
||||
return cases
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cases = generate_v2_cases()
|
||||
|
||||
# Write v2 public test set
|
||||
output_path = "evals/grammatical_coverage/public/v2/cases.jsonl"
|
||||
import os
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
|
||||
with open(output_path, "w") as f:
|
||||
for case in cases:
|
||||
f.write(json.dumps(case) + "\n")
|
||||
|
||||
print(f"Generated {len(cases)} v2 cases")
|
||||
print(f"Wrote to {output_path}")
|
||||
|
||||
# Show distribution
|
||||
by_construction = defaultdict(int)
|
||||
for case in cases:
|
||||
by_construction[case["construction"]] += 1
|
||||
|
||||
print("\nDistribution:")
|
||||
for c in sorted(by_construction.keys()):
|
||||
print(f" {c}: {by_construction[c]}")
|
||||
Loading…
Reference in a new issue