59974865ef
Two coupled, additive, off-serving changes toward the typed math-comprehension organ. No serving path touched; the relational_metric answer lane stays 15/15 wrong=0. PR-1 — QuantQuery → BoundUnknown. comprehend_quantitative now emits the question target as a BoundUnknown INSIDE the binding-graph (symbol_id, state_index="terminal", question_form "count"|"total", expected_unit), so the graph is a real question-bearing mathematical object and its canonical serialization carries the target. The external QuantQuery is RETAINED, consistent-by-construction, so the two consumers (to_relational_metric, realize/quantitative) are byte-identical; a follow-up rewires them onto graph.unknowns and drops the duplicate field. Setup-oracle lane (evals/setup_oracle) — grade the READING, not the answer. The relational_metric lane scores answers, which can bless a semantically-wrong derivation that coincidentally lands on the right number (the exact hazard the held-out measurements + the 2/87 resolve_pooled probe exposed). The setup-oracle compares the reader's comprehended STRUCTURE — a span-free signature of facts + typed equations + the BoundUnknown target — against the INDEPENDENT gold structure (the relational_metric cases' own relations/query, authored separately from the binding-graph reader). A structural mismatch is setup_wrong, the wrong=0-critical count, even when the answer would be right. v1 grades structure (units deferred — covered by admissibility). The reader reads all 15 cases with the gold structure (setup_wrong=0); a meaningful-fail test proves the oracle catches a right-answer/wrong-structure reading (it is not decoration). `python -m evals.setup_oracle` exits nonzero iff setup_wrong > 0. This is the measurement rig BEFORE investing in frame families: setup_wrong=0 is the gate; serving must not move while setup_wrong > 0. It is the first milestone of the math-comprehension organ, not a path to "solve GSM8K". Verified: setup-oracle 15/15 setup_correct wrong=0; quantitative + setup-oracle unit tests (17); realize-binding-graph + binding-graph + architectural invariants (183).
98 lines
3.9 KiB
Python
98 lines
3.9 KiB
Python
"""Setup-oracle lane — grade the reading (structure), not the answer.
|
|
|
|
Two obligations:
|
|
1. The current reader reads all 15 relational_metric cases with the gold STRUCTURE
|
|
(``setup_wrong == 0``) — the gate the milestone rests on.
|
|
2. The oracle MEANINGFULLY FAILS — a reading that lands on the right number via the
|
|
WRONG structure is ``setup_wrong``. Without this, structure-grading would be
|
|
decoration; with it, "did we read it right?" is falsifiable.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from evals.setup_oracle import (
|
|
gold_unknown_signature,
|
|
reader_unknown_signature,
|
|
relation_signature,
|
|
run,
|
|
)
|
|
from generate.binding_graph.model import (
|
|
BoundFact,
|
|
SemanticSymbolicBindingGraph,
|
|
SourceSpanLink,
|
|
SymbolBinding,
|
|
)
|
|
|
|
|
|
def _span() -> SourceSpanLink:
|
|
return SourceSpanLink(source_id="t", start=0, end=1, text="x")
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Obligation 1 — the reader reads the gold structure on every case
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_all_cases_setup_correct_wrong_zero() -> None:
|
|
report = run()
|
|
assert report["total"] == 15
|
|
assert report["setup_correct"] == 15
|
|
assert report["setup_wrong"] == 0 # the load-bearing count
|
|
assert report["setup_refused"] == 0
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Obligation 2 — the oracle is not decoration (it catches wrong readings)
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_right_answer_wrong_structure_is_caught() -> None:
|
|
# Gold: mia = liam + 4 over liam = 6 (answer 10, read as a relation).
|
|
gold = [
|
|
{"kind": "fact", "entity": "liam", "value": 6},
|
|
{"kind": "more_than", "entity": "mia", "ref": "liam", "delta": 4},
|
|
]
|
|
# A reading that lands on the SAME answer (mia = 10) but flattens the relation
|
|
# into a bare fact — the right number, the wrong reading.
|
|
wrong_structure = [{"kind": "fact", "entity": "mia", "value": 10}]
|
|
assert relation_signature(gold) != relation_signature(wrong_structure)
|
|
|
|
|
|
def test_signature_catches_wrong_operation() -> None:
|
|
more = [{"kind": "more_than", "entity": "y", "ref": "x", "delta": 6}]
|
|
fewer = [{"kind": "fewer_than", "entity": "y", "ref": "x", "delta": 6}]
|
|
assert relation_signature(more) != relation_signature(fewer)
|
|
|
|
|
|
def test_signature_is_order_independent() -> None:
|
|
a = [
|
|
{"kind": "fact", "entity": "x", "value": 1},
|
|
{"kind": "more_than", "entity": "y", "ref": "x", "delta": 2},
|
|
]
|
|
assert relation_signature(a) == relation_signature(list(reversed(a)))
|
|
|
|
|
|
def test_wrong_question_target_is_caught() -> None:
|
|
rels = [
|
|
{"kind": "fact", "entity": "dan", "value": 7},
|
|
{"kind": "more_than", "entity": "eva", "ref": "dan", "delta": 9},
|
|
{"kind": "sum_of", "entity": "total", "parts": ["dan", "eva"]},
|
|
]
|
|
# Gold asks the total; a reader that targeted "eva" instead is a different reading.
|
|
assert gold_unknown_signature(rels, {"entity": "total"}) == ("total", "terminal", "total")
|
|
assert gold_unknown_signature(rels, {"entity": "total"}) != ("eva", "terminal", "count")
|
|
|
|
|
|
def test_malformed_graph_target_never_matches_gold() -> None:
|
|
# A graph carrying no question target (pre-PR-1 shape) must report MALFORMED and
|
|
# never silently compare equal to a well-formed gold target.
|
|
graph = SemanticSymbolicBindingGraph(
|
|
symbols=(SymbolBinding(symbol_id="x", name="x", semantic_role="count",
|
|
source_span=_span(), introduced_by="t", entity="x", unit="item"),),
|
|
facts=(BoundFact(symbol_id="x", value="1", source_span=_span(), unit="item"),),
|
|
equations=(),
|
|
unknowns=(),
|
|
)
|
|
sig = reader_unknown_signature(graph)
|
|
assert sig[0] == "MALFORMED"
|
|
assert sig != ("x", "terminal", "count")
|