From 59974865efdeca810c1a7b3e18b34e3266cabbbf Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 6 Jun 2026 16:40:15 -0700 Subject: [PATCH] feat(comprehension): question target in the graph (PR-1) + setup-oracle lane (grade the reading) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two coupled, additive, off-serving changes toward the typed math-comprehension organ. No serving path touched; the relational_metric answer lane stays 15/15 wrong=0. PR-1 — QuantQuery → BoundUnknown. comprehend_quantitative now emits the question target as a BoundUnknown INSIDE the binding-graph (symbol_id, state_index="terminal", question_form "count"|"total", expected_unit), so the graph is a real question-bearing mathematical object and its canonical serialization carries the target. The external QuantQuery is RETAINED, consistent-by-construction, so the two consumers (to_relational_metric, realize/quantitative) are byte-identical; a follow-up rewires them onto graph.unknowns and drops the duplicate field. Setup-oracle lane (evals/setup_oracle) — grade the READING, not the answer. The relational_metric lane scores answers, which can bless a semantically-wrong derivation that coincidentally lands on the right number (the exact hazard the held-out measurements + the 2/87 resolve_pooled probe exposed). The setup-oracle compares the reader's comprehended STRUCTURE — a span-free signature of facts + typed equations + the BoundUnknown target — against the INDEPENDENT gold structure (the relational_metric cases' own relations/query, authored separately from the binding-graph reader). A structural mismatch is setup_wrong, the wrong=0-critical count, even when the answer would be right. v1 grades structure (units deferred — covered by admissibility). The reader reads all 15 cases with the gold structure (setup_wrong=0); a meaningful-fail test proves the oracle catches a right-answer/wrong-structure reading (it is not decoration). `python -m evals.setup_oracle` exits nonzero iff setup_wrong > 0. This is the measurement rig BEFORE investing in frame families: setup_wrong=0 is the gate; serving must not move while setup_wrong > 0. It is the first milestone of the math-comprehension organ, not a path to "solve GSM8K". Verified: setup-oracle 15/15 setup_correct wrong=0; quantitative + setup-oracle unit tests (17); realize-binding-graph + binding-graph + architectural invariants (183). --- evals/setup_oracle/__init__.py | 28 +++++++ evals/setup_oracle/__main__.py | 23 ++++++ evals/setup_oracle/runner.py | 81 ++++++++++++++++++++ evals/setup_oracle/signature.py | 76 ++++++++++++++++++ generate/quantitative_comprehension.py | 19 ++++- tests/test_quantitative_comprehension.py | 23 ++++++ tests/test_setup_oracle.py | 98 ++++++++++++++++++++++++ 7 files changed, 347 insertions(+), 1 deletion(-) create mode 100644 evals/setup_oracle/__init__.py create mode 100644 evals/setup_oracle/__main__.py create mode 100644 evals/setup_oracle/runner.py create mode 100644 evals/setup_oracle/signature.py create mode 100644 tests/test_setup_oracle.py diff --git a/evals/setup_oracle/__init__.py b/evals/setup_oracle/__init__.py new file mode 100644 index 00000000..078d4d22 --- /dev/null +++ b/evals/setup_oracle/__init__.py @@ -0,0 +1,28 @@ +"""Setup-oracle lane — grade the READING (the semantic setup), not the answer. + +The relational_metric lane scores answers: `comprehend → project → oracle_answer → +compare gold integer`. That hides a hazard the held-out measurements exposed — a WRONG +reading can produce a coincidentally-correct number and still pass. The setup-oracle +closes that gap: it compares the reader's comprehended STRUCTURE (the relations it read ++ the BoundUnknown question target it emitted) against the INDEPENDENT gold structure +(the relational_metric cases' own `relations`/`query`, authored separately from the +binding-graph reader). A wrong setup is a first-class failure even when its answer is right. + +`setup_wrong` is the load-bearing, wrong=0-critical count: a reading that misrepresents +the problem. v1 grades structure (facts + equations + question target/state/form); unit +modelling stays covered by the admissibility tests (a documented signature extension). +""" + +from evals.setup_oracle.runner import run +from evals.setup_oracle.signature import ( + gold_unknown_signature, + reader_unknown_signature, + relation_signature, +) + +__all__ = [ + "gold_unknown_signature", + "reader_unknown_signature", + "relation_signature", + "run", +] diff --git a/evals/setup_oracle/__main__.py b/evals/setup_oracle/__main__.py new file mode 100644 index 00000000..f1874400 --- /dev/null +++ b/evals/setup_oracle/__main__.py @@ -0,0 +1,23 @@ +"""CLI: print the setup-oracle report. + + python -m evals.setup_oracle + +Exit 0 iff ``setup_wrong == 0`` — the gate the milestone rests on (a wrong reading must +never pass, and serving must not move while setup_wrong > 0). +""" + +from __future__ import annotations + +import json + +from evals.setup_oracle.runner import run + + +def main() -> int: + report = run() + print(json.dumps(report, indent=2, default=str)) + return 0 if report["setup_wrong"] == 0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/setup_oracle/runner.py b/evals/setup_oracle/runner.py new file mode 100644 index 00000000..016c2525 --- /dev/null +++ b/evals/setup_oracle/runner.py @@ -0,0 +1,81 @@ +"""Setup-oracle runner — grade the reader's comprehended structure vs gold structure. + +For each relational_metric case: comprehend the prose into a binding-graph, project it +to relations + read its question target, and compare the (relations, target) SIGNATURE +to the case's INDEPENDENT gold (`relations` + `query`). A structural mismatch is +``setup_wrong`` — the wrong=0-critical count — even if the answer would be right. + +This is a STRICTER gate than the relational_metric (answer) lane: it requires the +reader to have read the problem the way the gold says it reads, not merely to land on +the gold number. It is the gate every future frame family must pass before serving. +""" + +from __future__ import annotations + +from typing import Any + +from evals.relational_metric.runner import _load_cases +from evals.setup_oracle.signature import ( + gold_unknown_signature, + reader_unknown_signature, + relation_signature, +) +from generate.meaning_graph.reader import Refusal +from generate.quantitative_comprehension import comprehend_quantitative, to_relational_metric + + +def run() -> dict[str, Any]: + """Score the reader's setup against the independent gold setup, structure-only.""" + cases = _load_cases() + setup_correct = setup_wrong = setup_refused = 0 + wrongs: list[dict[str, Any]] = [] + + for case in cases: + comp = comprehend_quantitative(case["text"]) + if isinstance(comp, Refusal): + setup_refused += 1 + continue + projected = to_relational_metric(comp) + if projected is None: + setup_refused += 1 + continue + reader_relations, _reader_query = projected + + reader_sig = relation_signature(reader_relations) + gold_sig = relation_signature(case["relations"]) + reader_unk = reader_unknown_signature(comp.binding_graph) + gold_unk = gold_unknown_signature(case["relations"], case["query"]) + + if reader_sig == gold_sig and reader_unk == gold_unk: + setup_correct += 1 + else: + setup_wrong += 1 + wrongs.append( + { + "id": case.get("id"), + "relations_match": reader_sig == gold_sig, + "target_match": reader_unk == gold_unk, + "reader_relations": reader_sig, + "gold_relations": gold_sig, + "reader_target": reader_unk, + "gold_target": gold_unk, + } + ) + + return { + "lane": "setup_oracle", + "grades": "structure-only (facts + equations + question target); units deferred", + "total": len(cases), + "setup_correct": setup_correct, + "setup_wrong": setup_wrong, + "setup_refused": setup_refused, + "wrongs": wrongs, + "counts": { + "setup_correct": setup_correct, + "setup_wrong": setup_wrong, + "setup_refused": setup_refused, + }, + } + + +__all__ = ["run"] diff --git a/evals/setup_oracle/signature.py b/evals/setup_oracle/signature.py new file mode 100644 index 00000000..83b95103 --- /dev/null +++ b/evals/setup_oracle/signature.py @@ -0,0 +1,76 @@ +"""Span-free structural signatures for the setup-oracle. + +A *signature* is a deterministic, order-independent, span-free projection of the +mathematical SETUP — what the reader claims the problem says, stripped of input +offsets and surface tokens. Two readings are setup-equivalent iff their signatures +are equal. Used to compare the reader's comprehended structure against the +independent gold structure (the relational_metric cases' own `relations`/`query`). + +v1 grades: facts (entity, value), equations (the typed relation shape), and the +question target (symbol, state-index, question-form). Unit modelling is intentionally +NOT in the signature yet — it is covered by the admissibility tests, and a future +extension adds an expected-unit axis once the gold carries it. +""" + +from __future__ import annotations + +from typing import Any + +from generate.binding_graph.model import SemanticSymbolicBindingGraph + + +def relation_signature(relations: list[dict[str, Any]]) -> tuple[tuple, ...]: + """Canonicalize a list of relations (the ``to_relational_metric`` / gold shape) + into a sorted, order-independent tuple of typed relation tuples.""" + out: list[tuple] = [] + for r in relations: + kind = r["kind"] + if kind == "fact": + out.append(("fact", r["entity"], int(r["value"]))) + elif kind in ("more_than", "fewer_than"): + out.append((kind, r["entity"], r["ref"], int(r["delta"]))) + elif kind == "sum_of": + out.append(("sum_of", r["entity"], tuple(sorted(r["parts"])))) + else: # an unknown relation kind is itself a structural difference, not a crash + out.append(("unhandled_kind", kind, r.get("entity", ""))) + return tuple(sorted(out, key=repr)) + + +def gold_unknown_signature( + relations: list[dict[str, Any]], query: dict[str, Any] +) -> tuple[str, str, str]: + """The expected question-target signature, derived from the INDEPENDENT gold. + + A query whose target is an aggregate (the gold contains a ``sum_of`` producing it) + is a ``total`` form; otherwise a ``count``. All current cases ask the terminal state. + """ + form = "total" if any(r["kind"] == "sum_of" for r in relations) else "count" + return (query["entity"], "terminal", form) + + +def _state_token(state_index: Any) -> str: + if isinstance(state_index, str): + return state_index + # An Operation state-index (ADR-0135) — name it by its operation index. + return f"op{getattr(state_index, 'operation_index', '?')}" + + +def reader_unknown_signature(graph: SemanticSymbolicBindingGraph) -> tuple[str, str, str]: + """The reader's question-target signature from ``graph.unknowns`` (PR-1). + + A graph that does not carry exactly one unknown is itself a structural defect — it + is reported as a distinguished ``MALFORMED`` signature so it can never silently + compare equal to a well-formed gold target. + """ + unknowns = graph.unknowns + if len(unknowns) != 1: + return ("MALFORMED", str(len(unknowns)), "") + u = unknowns[0] + return (u.symbol_id, _state_token(u.state_index), u.question_form) + + +__all__ = [ + "gold_unknown_signature", + "reader_unknown_signature", + "relation_signature", +] diff --git a/generate/quantitative_comprehension.py b/generate/quantitative_comprehension.py index 4ad14a63..d5ef5a21 100644 --- a/generate/quantitative_comprehension.py +++ b/generate/quantitative_comprehension.py @@ -34,6 +34,7 @@ from generate.binding_graph.admissibility import AdmissibilityError, check_admis from generate.binding_graph.model import ( BoundEquation, BoundFact, + BoundUnknown, SemanticSymbolicBindingGraph, SourceSpanLink, SymbolBinding, @@ -271,9 +272,25 @@ def comprehend_quantitative(text: str, source_id: str = "input") -> QuantCompreh ) ) + # The question target lives INSIDE the graph (ADR-0135): a BoundUnknown bound to + # the asked symbol at the terminal state. The form is "total" for an aggregate + # query ("how many do X and Y have"), else "count". ``query`` is retained as a + # consistent-by-construction convenience for the existing relational_metric + # projection + realize path; a follow-up collapses it onto graph.unknowns. + unknown = BoundUnknown( + symbol_id=ask_entity, + question_span=_span(ask_entity), + state_index="terminal", + question_form="total" if sum_eq is not None else "count", + expected_unit=ask_unit, + ) + try: graph = SemanticSymbolicBindingGraph( - symbols=tuple(symbols), facts=bound_facts, equations=tuple(equations) + symbols=tuple(symbols), + facts=bound_facts, + equations=tuple(equations), + unknowns=(unknown,), ) except Exception as exc: # noqa: BLE001 — surface construction refusal return Refusal("invalid_binding_graph", repr(exc)) diff --git a/tests/test_quantitative_comprehension.py b/tests/test_quantitative_comprehension.py index be9e7958..b5e0ee43 100644 --- a/tests/test_quantitative_comprehension.py +++ b/tests/test_quantitative_comprehension.py @@ -35,6 +35,29 @@ def test_fact_and_more_than_build_binding_graph() -> None: assert comp.query.entity == "mia" +def test_question_target_is_a_bound_unknown_in_the_graph() -> None: + # PR-1: the question target lives INSIDE the graph (a BoundUnknown at the terminal + # state), not only as the external QuantQuery. + comp = _comp("Liam has 6 stickers. Mia has 4 more stickers than Liam. How many stickers does Mia have?") + unknowns = comp.binding_graph.unknowns + assert len(unknowns) == 1 + u = unknowns[0] + assert u.symbol_id == "mia" + assert u.state_index == "terminal" + assert u.question_form == "count" + assert u.expected_unit == "item" + # The graph's canonical serialization now carries the target. + assert "state=terminal" in comp.binding_graph.to_canonical_string() + # Retained convenience stays consistent with the in-graph unknown. + assert comp.query.entity == u.symbol_id + + +def test_sum_query_target_is_total_form_unknown() -> None: + comp = _comp("Dan has 7 coins. Eva has 9 more coins than Dan. How many coins do Dan and Eva have?") + (u,) = comp.binding_graph.unknowns + assert u.symbol_id == "total" and u.question_form == "total" and u.state_index == "terminal" + + def test_count_nouns_resolve_to_item_dimension() -> None: # Unknown sortal nouns become the count dimension (item); admissibility admits. comp = _comp("Kim has 2 marbles. Leo has 3 more marbles than Kim. How many marbles does Leo have?") diff --git a/tests/test_setup_oracle.py b/tests/test_setup_oracle.py new file mode 100644 index 00000000..05ba3540 --- /dev/null +++ b/tests/test_setup_oracle.py @@ -0,0 +1,98 @@ +"""Setup-oracle lane — grade the reading (structure), not the answer. + +Two obligations: +1. The current reader reads all 15 relational_metric cases with the gold STRUCTURE + (``setup_wrong == 0``) — the gate the milestone rests on. +2. The oracle MEANINGFULLY FAILS — a reading that lands on the right number via the + WRONG structure is ``setup_wrong``. Without this, structure-grading would be + decoration; with it, "did we read it right?" is falsifiable. +""" + +from __future__ import annotations + +from evals.setup_oracle import ( + gold_unknown_signature, + reader_unknown_signature, + relation_signature, + run, +) +from generate.binding_graph.model import ( + BoundFact, + SemanticSymbolicBindingGraph, + SourceSpanLink, + SymbolBinding, +) + + +def _span() -> SourceSpanLink: + return SourceSpanLink(source_id="t", start=0, end=1, text="x") + + +# --------------------------------------------------------------------------- # +# Obligation 1 — the reader reads the gold structure on every case +# --------------------------------------------------------------------------- # + + +def test_all_cases_setup_correct_wrong_zero() -> None: + report = run() + assert report["total"] == 15 + assert report["setup_correct"] == 15 + assert report["setup_wrong"] == 0 # the load-bearing count + assert report["setup_refused"] == 0 + + +# --------------------------------------------------------------------------- # +# Obligation 2 — the oracle is not decoration (it catches wrong readings) +# --------------------------------------------------------------------------- # + + +def test_right_answer_wrong_structure_is_caught() -> None: + # Gold: mia = liam + 4 over liam = 6 (answer 10, read as a relation). + gold = [ + {"kind": "fact", "entity": "liam", "value": 6}, + {"kind": "more_than", "entity": "mia", "ref": "liam", "delta": 4}, + ] + # A reading that lands on the SAME answer (mia = 10) but flattens the relation + # into a bare fact — the right number, the wrong reading. + wrong_structure = [{"kind": "fact", "entity": "mia", "value": 10}] + assert relation_signature(gold) != relation_signature(wrong_structure) + + +def test_signature_catches_wrong_operation() -> None: + more = [{"kind": "more_than", "entity": "y", "ref": "x", "delta": 6}] + fewer = [{"kind": "fewer_than", "entity": "y", "ref": "x", "delta": 6}] + assert relation_signature(more) != relation_signature(fewer) + + +def test_signature_is_order_independent() -> None: + a = [ + {"kind": "fact", "entity": "x", "value": 1}, + {"kind": "more_than", "entity": "y", "ref": "x", "delta": 2}, + ] + assert relation_signature(a) == relation_signature(list(reversed(a))) + + +def test_wrong_question_target_is_caught() -> None: + rels = [ + {"kind": "fact", "entity": "dan", "value": 7}, + {"kind": "more_than", "entity": "eva", "ref": "dan", "delta": 9}, + {"kind": "sum_of", "entity": "total", "parts": ["dan", "eva"]}, + ] + # Gold asks the total; a reader that targeted "eva" instead is a different reading. + assert gold_unknown_signature(rels, {"entity": "total"}) == ("total", "terminal", "total") + assert gold_unknown_signature(rels, {"entity": "total"}) != ("eva", "terminal", "count") + + +def test_malformed_graph_target_never_matches_gold() -> None: + # A graph carrying no question target (pre-PR-1 shape) must report MALFORMED and + # never silently compare equal to a well-formed gold target. + graph = SemanticSymbolicBindingGraph( + symbols=(SymbolBinding(symbol_id="x", name="x", semantic_role="count", + source_span=_span(), introduced_by="t", entity="x", unit="item"),), + facts=(BoundFact(symbol_id="x", value="1", source_span=_span(), unit="item"),), + equations=(), + unknowns=(), + ) + sig = reader_unknown_signature(graph) + assert sig[0] == "MALFORMED" + assert sig != ("x", "terminal", "count")