test(comprehend): generative wrong=0 hardening — reader is faithful or refuses
Proves the wrong=0 invariant GENERALIZES beyond the 8-case gold lanes. Over 1000 randomly generated single-word problems across all three domains, the reader either refuses or reproduces the EXACT verdict the independent oracle gives on the ground-truth structure the prose encodes — it never changes the answer. Non-circular: the generated structure S is ground truth; we render prose P(S), then compare oracle(project(comprehend(P(S)))) to oracle(S) computed directly. The oracle is independent of the reader, so agreement = lossless carry and refusal = honest decline; both are wrong=0. Verified to BITE: flipping a comparator direction makes the total_ordering case produce a reversed sort the test catches. Single-word vocab on purpose — multi-word NPs are the known gold-canonicalization wall the reader refuses; the generator stays in the readable regime where faithfulness is the whole claim. Anti-overfit: random vocab proves the templates key on function words + order, not on memorized gold content.
This commit is contained in:
Shay
parent
e831ed2615
commit
d1908ec4c8
1 changed files with 188 additions and 0 deletions
188
tests/test_comprehension_wrong_zero_property.py
Normal file
188
tests/test_comprehension_wrong_zero_property.py
Normal file
|
|
@ -0,0 +1,188 @@
|
|||
"""Generative wrong=0 hardening for the comprehension organ (anti-overfit).
|
||||
|
||||
The end-to-end lane tests prove wrong=0 on the 8-case staged gold lanes. This file
|
||||
proves the same invariant GENERALIZES: over hundreds of randomly generated
|
||||
single-word problems, the reader is FAITHFUL — it either refuses, or it reproduces
|
||||
the exact verdict the independent oracle gives on the ground-truth structure the
|
||||
prose encodes. It NEVER changes the answer.
|
||||
|
||||
Why this is non-circular: the generated structure ``S`` is the ground truth. We
|
||||
render prose ``P(S)`` with a trivial single-word renderer, then run the
|
||||
comprehension path ``oracle(project(comprehend(P(S))))`` and compare it to
|
||||
``oracle(S)`` computed directly. The oracle is independent of the reader; agreement
|
||||
means the reader carried the meaning losslessly, refusal means it declined — both
|
||||
are wrong=0. A reader that silently mis-read would commit a DIFFERENT verdict and
|
||||
the test would bite.
|
||||
|
||||
Single-word vocab on purpose: multi-word NPs are a known gold-canonicalization wall
|
||||
(the reader refuses them), so the generator stays in the readable single-word
|
||||
regime where faithfulness is the whole claim.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
|
||||
from evals.set_membership.oracle import OracleError as SetError
|
||||
from evals.set_membership.oracle import oracle_answer as set_oracle
|
||||
from evals.syllogism.oracle import OracleError as SylError
|
||||
from evals.syllogism.oracle import oracle_answer as syl_oracle
|
||||
from evals.total_ordering.oracle import OracleError as OrdError
|
||||
from evals.total_ordering.oracle import oracle_answer as ord_oracle
|
||||
from generate.meaning_graph.projectors import (
|
||||
to_set_membership,
|
||||
to_syllogism,
|
||||
to_total_ordering,
|
||||
)
|
||||
from generate.meaning_graph.reader import Refusal, comprehend
|
||||
|
||||
_TERMS = [f"t{i}" for i in range(8)] # single-word identifiers -> trivial plurals
|
||||
|
||||
|
||||
def _committed(comp_or_refusal, projector, oracle, error):
|
||||
"""Run the comprehension path; return the committed oracle answer or None
|
||||
(None == refused / unprojectable / oracle-refused — all wrong=0-safe)."""
|
||||
if isinstance(comp_or_refusal, Refusal):
|
||||
return None
|
||||
projected = projector(comp_or_refusal)
|
||||
if projected is None:
|
||||
return None
|
||||
try:
|
||||
return oracle(*projected)
|
||||
except error:
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Syllogism — the richest grammar (A/E/I/O premises + therefore-conclusion)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
_FORM_PREMISE = {
|
||||
"A": lambda s, p: f"All {s}s are {p}s.",
|
||||
"E": lambda s, p: f"No {s}s are {p}s.",
|
||||
"I": lambda s, p: f"Some {s}s are {p}s.",
|
||||
"O": lambda s, p: f"Some {s}s are not {p}s.",
|
||||
}
|
||||
_FORM_CONCLUSION = {
|
||||
"A": lambda s, p: f"Therefore all {s}s are {p}s.",
|
||||
"E": lambda s, p: f"Therefore no {s}s are {p}s.",
|
||||
"I": lambda s, p: f"Therefore some {s}s are {p}s.",
|
||||
"O": lambda s, p: f"Therefore some {s}s are not {p}s.",
|
||||
}
|
||||
|
||||
|
||||
def test_syllogism_reader_is_faithful_or_refuses() -> None:
|
||||
rng = random.Random(20260605)
|
||||
checked = committed = 0
|
||||
for _ in range(400):
|
||||
a, b, c = rng.sample(_TERMS, 3)
|
||||
# Two premises + a conclusion over the three terms; random forms.
|
||||
(s1, p1), (s2, p2), (sc, pc) = (
|
||||
rng.sample([a, b, c], 2),
|
||||
rng.sample([a, b, c], 2),
|
||||
rng.sample([a, b, c], 2),
|
||||
)
|
||||
f1, f2, fc = (rng.choice("AEIO"), rng.choice("AEIO"), rng.choice("AEIO"))
|
||||
prose = " ".join(
|
||||
[_FORM_PREMISE[f1](s1, p1), _FORM_PREMISE[f2](s2, p2), _FORM_CONCLUSION[fc](sc, pc)]
|
||||
)
|
||||
structure = {
|
||||
"terms": sorted({a, b, c}),
|
||||
"domain_size": 3,
|
||||
"premises": [
|
||||
{"form": f1, "subject": s1, "predicate": p1},
|
||||
{"form": f2, "subject": s2, "predicate": p2},
|
||||
],
|
||||
}
|
||||
query = {"kind": "validity", "conclusion": {"form": fc, "subject": sc, "predicate": pc}}
|
||||
try:
|
||||
expected = syl_oracle(structure, query)
|
||||
except SylError:
|
||||
expected = None # inconsistent premises -> ground truth refuses
|
||||
got = _committed(comprehend(prose), to_syllogism, syl_oracle, SylError)
|
||||
checked += 1
|
||||
if got is not None:
|
||||
committed += 1
|
||||
assert got == expected, (prose, got, expected)
|
||||
assert committed > 50 # the generator actually exercises the committed path
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Total ordering — random strict chains, sort + compare queries
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_total_ordering_reader_is_faithful_or_refuses() -> None:
|
||||
rng = random.Random(7)
|
||||
committed = 0
|
||||
for _ in range(300):
|
||||
n = rng.randint(2, 5)
|
||||
chain = rng.sample(_TERMS, n) # chain[0] < chain[1] < ... (strict)
|
||||
rels = [{"less": chain[i], "greater": chain[i + 1]} for i in range(n - 1)]
|
||||
# Render the chain as comparative clauses joined into one sentence.
|
||||
clauses = [f"{lo} is below {hi}" for lo, hi in zip(chain, chain[1:])]
|
||||
facts = ", and ".join(clauses) + "."
|
||||
if rng.random() < 0.5:
|
||||
order = rng.choice(["ascending", "descending"])
|
||||
prose = f"{facts} Sort {order}."
|
||||
query = {"kind": "sort", "order": order}
|
||||
else:
|
||||
x, y = rng.sample(chain, 2)
|
||||
prose = f"{facts} Compare {x} with {y}."
|
||||
query = {"kind": "compare", "left": x, "right": y}
|
||||
structure = {"items": sorted(set(chain)), "relations": rels}
|
||||
try:
|
||||
expected = ord_oracle(structure, query)
|
||||
except OrdError:
|
||||
expected = None
|
||||
got = _committed(comprehend(prose), to_total_ordering, ord_oracle, OrdError)
|
||||
if got is not None:
|
||||
committed += 1
|
||||
assert got == expected, (prose, got, expected)
|
||||
assert committed > 50
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Set membership — random members + subset chains, member + subset queries
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_set_membership_reader_is_faithful_or_refuses() -> None:
|
||||
rng = random.Random(99)
|
||||
committed = 0
|
||||
for _ in range(300):
|
||||
classes = rng.sample(_TERMS, rng.randint(2, 4))
|
||||
individuals = [f"e{i}" for i in range(rng.randint(1, 3))]
|
||||
member_facts = [(ind, rng.choice(classes)) for ind in individuals]
|
||||
# a short subset chain over a prefix of the classes
|
||||
subset_facts = [
|
||||
(classes[i], classes[i + 1]) for i in range(len(classes) - 1) if rng.random() < 0.7
|
||||
]
|
||||
member_lines = [f"{ind} is a {cls}." for ind, cls in member_facts]
|
||||
subset_lines = [f"All {a}s are {b}s." for a, b in subset_facts]
|
||||
sets = [
|
||||
{"id": c, "members": sorted({i for i, cl in member_facts if cl == c})} for c in classes
|
||||
]
|
||||
structure = {
|
||||
"elements": sorted(individuals),
|
||||
"sets": sets,
|
||||
"subsets": [{"subset": a, "superset": b} for a, b in subset_facts],
|
||||
}
|
||||
if rng.random() < 0.5 and individuals:
|
||||
ind = rng.choice(individuals)
|
||||
target = rng.choice(classes)
|
||||
prose = " ".join(member_lines + subset_lines + [f"Is {ind} a {target}?"])
|
||||
query = {"kind": "member", "element": ind, "set": target}
|
||||
else:
|
||||
a, b = rng.sample(classes, 2)
|
||||
prose = " ".join(member_lines + subset_lines + [f"Are all {a}s {b}s?"])
|
||||
query = {"kind": "subset", "subset": a, "superset": b}
|
||||
try:
|
||||
expected = set_oracle(structure, query)
|
||||
except SetError:
|
||||
expected = None
|
||||
got = _committed(comprehend(prose), to_set_membership, set_oracle, SetError)
|
||||
if got is not None:
|
||||
committed += 1
|
||||
assert got == expected, (prose, got, expected)
|
||||
assert committed > 50
|
||||
Loading…
Reference in a new issue