5b193280a5
Pure-gold. Defines what "correct" means for the R1 (comparative / derived-symbol / multi-step) shapes the reader cannot read yet, and verifies the current reader REFUSES them rather than misreading them. NO frame extraction, no parser change, no serving. - evals/setup_oracle/r1_gold.jsonl — 10 INDEPENDENT, self-contained fixtures (id, text, relations, expected_units, query, notes) covering: twice/half-as-many (multiplicative), more/fewer-than -> total, a multi-step derived chain (jon=3*ivy; kim=jon+2), a derived subtotal reused downstream (total=lee+mae; per_box=total/3), an inverse target, and the hard negatives (ambiguous referent, missing base quantity, distractor quantity). The gold is the authority — each fixture is reviewable without joining files. - run_r1() scores the CURRENT reader against this gold. Investigative result (the deliverable): **10/10 setup_refused, setup_wrong=0, setup_correct=0.** The reader refuses every R1 shape with a TYPED reason (non_digit_quantity / no_quantity_template / unreadable_quantity_clause / admissibility_refused) — it NEVER misreads an R1 case as a simpler form. So the wrong=0 posture holds all the way down, and PR-5c (the R1 frame) starts from a clean foundation: every R1 case is currently a safe refusal, and the frame's job is refusal -> correct, with the oracle ensuring no setup_wrong is ever introduced. `python -m evals.setup_oracle r1` prints the auditable R1 report (exit 0 iff setup_wrong==0). The 15-case setup gate + relational_metric answer lane are untouched.
163 lines
6.9 KiB
Python
163 lines
6.9 KiB
Python
"""Setup-oracle lane — grade the reading (structure), not the answer.
|
|
|
|
Two obligations:
|
|
1. The current reader reads all 15 relational_metric cases with the gold STRUCTURE
|
|
(``setup_wrong == 0``) — the gate the milestone rests on.
|
|
2. The oracle MEANINGFULLY FAILS — a reading that lands on the right number via the
|
|
WRONG structure is ``setup_wrong``. Without this, structure-grading would be
|
|
decoration; with it, "did we read it right?" is falsifiable.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from evals.setup_oracle import (
|
|
gold_unknown_signature,
|
|
reader_symbol_units,
|
|
reader_unknown_signature,
|
|
relation_signature,
|
|
run,
|
|
symbol_unit_signature,
|
|
)
|
|
from generate.binding_graph.model import (
|
|
BoundFact,
|
|
BoundUnknown,
|
|
SemanticSymbolicBindingGraph,
|
|
SourceSpanLink,
|
|
SymbolBinding,
|
|
)
|
|
|
|
|
|
def _span() -> SourceSpanLink:
|
|
return SourceSpanLink(source_id="t", start=0, end=1, text="x")
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Obligation 1 — the reader reads the gold structure on every case
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_all_cases_setup_correct_wrong_zero() -> None:
|
|
report = run()
|
|
assert report["total"] == 15
|
|
assert report["setup_correct"] == 15
|
|
assert report["setup_wrong"] == 0 # the load-bearing count
|
|
assert report["setup_refused"] == 0
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Obligation 2 — the oracle is not decoration (it catches wrong readings)
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_right_answer_wrong_structure_is_caught() -> None:
|
|
# Gold: mia = liam + 4 over liam = 6 (answer 10, read as a relation).
|
|
gold = [
|
|
{"kind": "fact", "entity": "liam", "value": 6},
|
|
{"kind": "more_than", "entity": "mia", "ref": "liam", "delta": 4},
|
|
]
|
|
# A reading that lands on the SAME answer (mia = 10) but flattens the relation
|
|
# into a bare fact — the right number, the wrong reading.
|
|
wrong_structure = [{"kind": "fact", "entity": "mia", "value": 10}]
|
|
assert relation_signature(gold) != relation_signature(wrong_structure)
|
|
|
|
|
|
def test_signature_catches_wrong_operation() -> None:
|
|
more = [{"kind": "more_than", "entity": "y", "ref": "x", "delta": 6}]
|
|
fewer = [{"kind": "fewer_than", "entity": "y", "ref": "x", "delta": 6}]
|
|
assert relation_signature(more) != relation_signature(fewer)
|
|
|
|
|
|
def test_signature_is_order_independent() -> None:
|
|
a = [
|
|
{"kind": "fact", "entity": "x", "value": 1},
|
|
{"kind": "more_than", "entity": "y", "ref": "x", "delta": 2},
|
|
]
|
|
assert relation_signature(a) == relation_signature(list(reversed(a)))
|
|
|
|
|
|
def test_wrong_question_target_is_caught() -> None:
|
|
rels = [
|
|
{"kind": "fact", "entity": "dan", "value": 7},
|
|
{"kind": "more_than", "entity": "eva", "ref": "dan", "delta": 9},
|
|
{"kind": "sum_of", "entity": "total", "parts": ["dan", "eva"]},
|
|
]
|
|
# Gold asks the total; a reader that targeted "eva" instead is a different reading.
|
|
units = {"total": "item"}
|
|
assert gold_unknown_signature(rels, {"entity": "total"}, units) == ("total", "terminal", "total", "item")
|
|
assert gold_unknown_signature(rels, {"entity": "total"}, units) != ("eva", "terminal", "count", "item")
|
|
|
|
|
|
def test_malformed_graph_target_never_matches_gold() -> None:
|
|
# A graph carrying no question target (pre-PR-1 shape) must report MALFORMED and
|
|
# never silently compare equal to a well-formed gold target.
|
|
graph = SemanticSymbolicBindingGraph(
|
|
symbols=(SymbolBinding(symbol_id="x", name="x", semantic_role="count",
|
|
source_span=_span(), introduced_by="t", entity="x", unit="item"),),
|
|
facts=(BoundFact(symbol_id="x", value="1", source_span=_span(), unit="item"),),
|
|
equations=(),
|
|
unknowns=(),
|
|
)
|
|
sig = reader_unknown_signature(graph)
|
|
assert sig[0] == "MALFORMED"
|
|
assert sig != ("x", "terminal", "count", "item")
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# PR-5a — the ruler is now UNIT-AWARE (structure can match while units diverge)
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_unit_mismatch_is_caught_even_when_structure_matches() -> None:
|
|
# Same structure (a single fact about x), but the reader modelled a different unit.
|
|
# The setup-oracle must FAIL — a unit-wrong reading is not a correct setup.
|
|
gold_units = symbol_unit_signature({"x": "item"})
|
|
reader_units_wrong = symbol_unit_signature({"x": "meter"})
|
|
assert gold_units != reader_units_wrong
|
|
assert symbol_unit_signature({"x": "item"}) == symbol_unit_signature({"x": "item"})
|
|
|
|
|
|
def test_target_unit_mismatch_is_caught() -> None:
|
|
# Structure + symbol + state + form all agree, but the target's expected unit differs.
|
|
rels = [{"kind": "fact", "entity": "x", "value": 1}]
|
|
assert gold_unknown_signature(rels, {"entity": "x"}, {"x": "item"}) != gold_unknown_signature(
|
|
rels, {"entity": "x"}, {"x": "dollars"}
|
|
)
|
|
|
|
|
|
def test_reader_units_read_from_the_binding_graph() -> None:
|
|
# The reader's unit signature comes from the GRAPH's symbols, not the answer projection.
|
|
graph = SemanticSymbolicBindingGraph(
|
|
symbols=(
|
|
SymbolBinding(symbol_id="iris", name="iris", semantic_role="count",
|
|
source_span=_span(), introduced_by="t", entity="iris", unit="dollars"),
|
|
SymbolBinding(symbol_id="jack", name="jack", semantic_role="count",
|
|
source_span=_span(), introduced_by="t", entity="jack", unit="dollars"),
|
|
),
|
|
facts=(BoundFact(symbol_id="iris", value="100", source_span=_span(), unit="dollars"),),
|
|
equations=(),
|
|
unknowns=(BoundUnknown(symbol_id="jack", question_span=_span(), state_index="terminal",
|
|
question_form="count", expected_unit="dollars"),),
|
|
)
|
|
assert reader_symbol_units(graph) == (("iris", "dollars"), ("jack", "dollars"))
|
|
assert reader_unknown_signature(graph) == ("jack", "terminal", "count", "dollars")
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# PR-5b — independent R1 gold: the reader must REFUSE, never MISREAD
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_r1_reader_refuses_never_misreads() -> None:
|
|
from evals.setup_oracle import run_r1
|
|
|
|
r = run_r1()
|
|
assert r["total"] == 10
|
|
# THE success criterion: the reader misreads NO unsupported R1 case. A wrong setup
|
|
# here would be a pre-existing hazard to fix before the R1 frame (PR-5c).
|
|
assert r["setup_wrong"] == 0
|
|
# The reader cannot read these shapes yet -> every one is a safe (typed) refusal.
|
|
assert r["setup_refused"] == 10
|
|
assert r["setup_correct"] == 0
|
|
for d in r["details"]:
|
|
assert d["outcome"] == "refused"
|
|
assert d.get("reason") # a typed refusal reason, never a silent drop
|