From d1908ec4c801967487df18e14e9cc306115cab3d Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 5 Jun 2026 21:22:13 -0700 Subject: [PATCH] =?UTF-8?q?test(comprehend):=20generative=20wrong=3D0=20ha?= =?UTF-8?q?rdening=20=E2=80=94=20reader=20is=20faithful=20or=20refuses?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Proves the wrong=0 invariant GENERALIZES beyond the 8-case gold lanes. Over 1000 randomly generated single-word problems across all three domains, the reader either refuses or reproduces the EXACT verdict the independent oracle gives on the ground-truth structure the prose encodes — it never changes the answer. Non-circular: the generated structure S is ground truth; we render prose P(S), then compare oracle(project(comprehend(P(S)))) to oracle(S) computed directly. The oracle is independent of the reader, so agreement = lossless carry and refusal = honest decline; both are wrong=0. Verified to BITE: flipping a comparator direction makes the total_ordering case produce a reversed sort the test catches. Single-word vocab on purpose — multi-word NPs are the known gold-canonicalization wall the reader refuses; the generator stays in the readable regime where faithfulness is the whole claim. Anti-overfit: random vocab proves the templates key on function words + order, not on memorized gold content. --- .../test_comprehension_wrong_zero_property.py | 188 ++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 tests/test_comprehension_wrong_zero_property.py diff --git a/tests/test_comprehension_wrong_zero_property.py b/tests/test_comprehension_wrong_zero_property.py new file mode 100644 index 00000000..a64e2eb7 --- /dev/null +++ b/tests/test_comprehension_wrong_zero_property.py @@ -0,0 +1,188 @@ +"""Generative wrong=0 hardening for the comprehension organ (anti-overfit). + +The end-to-end lane tests prove wrong=0 on the 8-case staged gold lanes. This file +proves the same invariant GENERALIZES: over hundreds of randomly generated +single-word problems, the reader is FAITHFUL — it either refuses, or it reproduces +the exact verdict the independent oracle gives on the ground-truth structure the +prose encodes. It NEVER changes the answer. + +Why this is non-circular: the generated structure ``S`` is the ground truth. We +render prose ``P(S)`` with a trivial single-word renderer, then run the +comprehension path ``oracle(project(comprehend(P(S))))`` and compare it to +``oracle(S)`` computed directly. The oracle is independent of the reader; agreement +means the reader carried the meaning losslessly, refusal means it declined — both +are wrong=0. A reader that silently mis-read would commit a DIFFERENT verdict and +the test would bite. + +Single-word vocab on purpose: multi-word NPs are a known gold-canonicalization wall +(the reader refuses them), so the generator stays in the readable single-word +regime where faithfulness is the whole claim. +""" + +from __future__ import annotations + +import random + +from evals.set_membership.oracle import OracleError as SetError +from evals.set_membership.oracle import oracle_answer as set_oracle +from evals.syllogism.oracle import OracleError as SylError +from evals.syllogism.oracle import oracle_answer as syl_oracle +from evals.total_ordering.oracle import OracleError as OrdError +from evals.total_ordering.oracle import oracle_answer as ord_oracle +from generate.meaning_graph.projectors import ( + to_set_membership, + to_syllogism, + to_total_ordering, +) +from generate.meaning_graph.reader import Refusal, comprehend + +_TERMS = [f"t{i}" for i in range(8)] # single-word identifiers -> trivial plurals + + +def _committed(comp_or_refusal, projector, oracle, error): + """Run the comprehension path; return the committed oracle answer or None + (None == refused / unprojectable / oracle-refused — all wrong=0-safe).""" + if isinstance(comp_or_refusal, Refusal): + return None + projected = projector(comp_or_refusal) + if projected is None: + return None + try: + return oracle(*projected) + except error: + return None + + +# --------------------------------------------------------------------------- # +# Syllogism — the richest grammar (A/E/I/O premises + therefore-conclusion) +# --------------------------------------------------------------------------- # + +_FORM_PREMISE = { + "A": lambda s, p: f"All {s}s are {p}s.", + "E": lambda s, p: f"No {s}s are {p}s.", + "I": lambda s, p: f"Some {s}s are {p}s.", + "O": lambda s, p: f"Some {s}s are not {p}s.", +} +_FORM_CONCLUSION = { + "A": lambda s, p: f"Therefore all {s}s are {p}s.", + "E": lambda s, p: f"Therefore no {s}s are {p}s.", + "I": lambda s, p: f"Therefore some {s}s are {p}s.", + "O": lambda s, p: f"Therefore some {s}s are not {p}s.", +} + + +def test_syllogism_reader_is_faithful_or_refuses() -> None: + rng = random.Random(20260605) + checked = committed = 0 + for _ in range(400): + a, b, c = rng.sample(_TERMS, 3) + # Two premises + a conclusion over the three terms; random forms. + (s1, p1), (s2, p2), (sc, pc) = ( + rng.sample([a, b, c], 2), + rng.sample([a, b, c], 2), + rng.sample([a, b, c], 2), + ) + f1, f2, fc = (rng.choice("AEIO"), rng.choice("AEIO"), rng.choice("AEIO")) + prose = " ".join( + [_FORM_PREMISE[f1](s1, p1), _FORM_PREMISE[f2](s2, p2), _FORM_CONCLUSION[fc](sc, pc)] + ) + structure = { + "terms": sorted({a, b, c}), + "domain_size": 3, + "premises": [ + {"form": f1, "subject": s1, "predicate": p1}, + {"form": f2, "subject": s2, "predicate": p2}, + ], + } + query = {"kind": "validity", "conclusion": {"form": fc, "subject": sc, "predicate": pc}} + try: + expected = syl_oracle(structure, query) + except SylError: + expected = None # inconsistent premises -> ground truth refuses + got = _committed(comprehend(prose), to_syllogism, syl_oracle, SylError) + checked += 1 + if got is not None: + committed += 1 + assert got == expected, (prose, got, expected) + assert committed > 50 # the generator actually exercises the committed path + + +# --------------------------------------------------------------------------- # +# Total ordering — random strict chains, sort + compare queries +# --------------------------------------------------------------------------- # + + +def test_total_ordering_reader_is_faithful_or_refuses() -> None: + rng = random.Random(7) + committed = 0 + for _ in range(300): + n = rng.randint(2, 5) + chain = rng.sample(_TERMS, n) # chain[0] < chain[1] < ... (strict) + rels = [{"less": chain[i], "greater": chain[i + 1]} for i in range(n - 1)] + # Render the chain as comparative clauses joined into one sentence. + clauses = [f"{lo} is below {hi}" for lo, hi in zip(chain, chain[1:])] + facts = ", and ".join(clauses) + "." + if rng.random() < 0.5: + order = rng.choice(["ascending", "descending"]) + prose = f"{facts} Sort {order}." + query = {"kind": "sort", "order": order} + else: + x, y = rng.sample(chain, 2) + prose = f"{facts} Compare {x} with {y}." + query = {"kind": "compare", "left": x, "right": y} + structure = {"items": sorted(set(chain)), "relations": rels} + try: + expected = ord_oracle(structure, query) + except OrdError: + expected = None + got = _committed(comprehend(prose), to_total_ordering, ord_oracle, OrdError) + if got is not None: + committed += 1 + assert got == expected, (prose, got, expected) + assert committed > 50 + + +# --------------------------------------------------------------------------- # +# Set membership — random members + subset chains, member + subset queries +# --------------------------------------------------------------------------- # + + +def test_set_membership_reader_is_faithful_or_refuses() -> None: + rng = random.Random(99) + committed = 0 + for _ in range(300): + classes = rng.sample(_TERMS, rng.randint(2, 4)) + individuals = [f"e{i}" for i in range(rng.randint(1, 3))] + member_facts = [(ind, rng.choice(classes)) for ind in individuals] + # a short subset chain over a prefix of the classes + subset_facts = [ + (classes[i], classes[i + 1]) for i in range(len(classes) - 1) if rng.random() < 0.7 + ] + member_lines = [f"{ind} is a {cls}." for ind, cls in member_facts] + subset_lines = [f"All {a}s are {b}s." for a, b in subset_facts] + sets = [ + {"id": c, "members": sorted({i for i, cl in member_facts if cl == c})} for c in classes + ] + structure = { + "elements": sorted(individuals), + "sets": sets, + "subsets": [{"subset": a, "superset": b} for a, b in subset_facts], + } + if rng.random() < 0.5 and individuals: + ind = rng.choice(individuals) + target = rng.choice(classes) + prose = " ".join(member_lines + subset_lines + [f"Is {ind} a {target}?"]) + query = {"kind": "member", "element": ind, "set": target} + else: + a, b = rng.sample(classes, 2) + prose = " ".join(member_lines + subset_lines + [f"Are all {a}s {b}s?"]) + query = {"kind": "subset", "subset": a, "superset": b} + try: + expected = set_oracle(structure, query) + except SetError: + expected = None + got = _committed(comprehend(prose), to_set_membership, set_oracle, SetError) + if got is not None: + committed += 1 + assert got == expected, (prose, got, expected) + assert committed > 50