diff --git a/evals/setup_oracle/__init__.py b/evals/setup_oracle/__init__.py index 078d4d22..357cb2dd 100644 --- a/evals/setup_oracle/__init__.py +++ b/evals/setup_oracle/__init__.py @@ -16,13 +16,17 @@ modelling stays covered by the admissibility tests (a documented signature exten from evals.setup_oracle.runner import run from evals.setup_oracle.signature import ( gold_unknown_signature, + reader_symbol_units, reader_unknown_signature, relation_signature, + symbol_unit_signature, ) __all__ = [ "gold_unknown_signature", + "reader_symbol_units", "reader_unknown_signature", "relation_signature", "run", + "symbol_unit_signature", ] diff --git a/evals/setup_oracle/expected_units.json b/evals/setup_oracle/expected_units.json new file mode 100644 index 00000000..661fa8eb --- /dev/null +++ b/evals/setup_oracle/expected_units.json @@ -0,0 +1,18 @@ +{ + "_doc": "Independent expected MODELING unit per entity for the setup-oracle (PR-5a). Hand-authored from the PROBLEM, not copied from the reader: a discrete sortal count (stickers, cards, coins, ...) is dimensionally a count -> the generic count unit 'item'; money ('dollars') stays 'dollars'. The 'total' of same-unit parts inherits that unit. The setup-oracle fails (setup_wrong) if the reader's per-symbol unit diverges from this.", + "rm-v1-0001": {"liam": "item", "mia": "item"}, + "rm-v1-0002": {"noah": "item", "olivia": "item"}, + "rm-v1-0003": {"ava": "item", "ben": "item", "cara": "item"}, + "rm-v1-0004": {"dan": "item", "eva": "item", "total": "item"}, + "rm-v1-0005": {"finn": "item"}, + "rm-v1-0006": {"gabe": "item", "hana": "item"}, + "rm-v1-0007": {"iris": "dollars", "jack": "dollars"}, + "rm-v1-0008": {"kim": "item", "leo": "item", "total": "item"}, + "rm-v1-0009": {"maya": "item", "nico": "item"}, + "rm-v1-0010": {"omar": "item", "pia": "item", "quinn": "item"}, + "rm-v1-0011": {"rosa": "item", "sam": "item", "total": "item"}, + "rm-v1-0012": {"tara": "item", "uma": "item"}, + "rm-v1-0013": {"vera": "item", "will": "item"}, + "rm-v1-0014": {"xena": "item", "yara": "item", "zane": "item"}, + "rm-v1-0015": {"gus": "item"} +} diff --git a/evals/setup_oracle/runner.py b/evals/setup_oracle/runner.py index 016c2525..b540ba68 100644 --- a/evals/setup_oracle/runner.py +++ b/evals/setup_oracle/runner.py @@ -1,32 +1,44 @@ -"""Setup-oracle runner — grade the reader's comprehended structure vs gold structure. +"""Setup-oracle runner — grade the reader's comprehended structure + UNITS vs gold. -For each relational_metric case: comprehend the prose into a binding-graph, project it -to relations + read its question target, and compare the (relations, target) SIGNATURE -to the case's INDEPENDENT gold (`relations` + `query`). A structural mismatch is -``setup_wrong`` — the wrong=0-critical count — even if the answer would be right. - -This is a STRICTER gate than the relational_metric (answer) lane: it requires the -reader to have read the problem the way the gold says it reads, not merely to land on -the gold number. It is the gate every future frame family must pass before serving. +For each relational_metric case: comprehend the prose into a binding-graph and compare, +against the INDEPENDENT gold, three axes: + 1. the relation STRUCTURE (facts + typed equations — from the IR, not a reparse), + 2. the per-symbol UNITS (read from the binding-graph, vs the expected_units fixture), + 3. the question TARGET (symbol, state-index, form, expected unit). +A mismatch on ANY axis is ``setup_wrong`` — the wrong=0-critical count — even if the +answer would be right. This is a STRICTER gate than the relational_metric (answer) lane, +and the gate every future GSM8K frame family must pass before serving. """ from __future__ import annotations +import json +from pathlib import Path from typing import Any from evals.relational_metric.runner import _load_cases from evals.setup_oracle.signature import ( gold_unknown_signature, + reader_symbol_units, reader_unknown_signature, relation_signature, + symbol_unit_signature, ) from generate.meaning_graph.reader import Refusal from generate.quantitative_comprehension import comprehend_quantitative, to_relational_metric +_EXPECTED_UNITS_PATH = Path(__file__).resolve().parent / "expected_units.json" + + +def _load_expected_units() -> dict[str, dict[str, str]]: + raw = json.loads(_EXPECTED_UNITS_PATH.read_text(encoding="utf-8")) + return {k: v for k, v in raw.items() if not k.startswith("_")} + def run() -> dict[str, Any]: - """Score the reader's setup against the independent gold setup, structure-only.""" + """Score the reader's setup (structure + units + target) against the independent gold.""" cases = _load_cases() + expected_units = _load_expected_units() setup_correct = setup_wrong = setup_refused = 0 wrongs: list[dict[str, Any]] = [] @@ -40,23 +52,27 @@ def run() -> dict[str, Any]: setup_refused += 1 continue reader_relations, _reader_query = projected + case_units = expected_units.get(case.get("id", ""), {}) - reader_sig = relation_signature(reader_relations) - gold_sig = relation_signature(case["relations"]) + reader_rel = relation_signature(reader_relations) + gold_rel = relation_signature(case["relations"]) + reader_units = reader_symbol_units(comp.binding_graph) + gold_units = symbol_unit_signature(case_units) reader_unk = reader_unknown_signature(comp.binding_graph) - gold_unk = gold_unknown_signature(case["relations"], case["query"]) + gold_unk = gold_unknown_signature(case["relations"], case["query"], case_units) - if reader_sig == gold_sig and reader_unk == gold_unk: + if reader_rel == gold_rel and reader_units == gold_units and reader_unk == gold_unk: setup_correct += 1 else: setup_wrong += 1 wrongs.append( { "id": case.get("id"), - "relations_match": reader_sig == gold_sig, + "relations_match": reader_rel == gold_rel, + "units_match": reader_units == gold_units, "target_match": reader_unk == gold_unk, - "reader_relations": reader_sig, - "gold_relations": gold_sig, + "reader_units": reader_units, + "gold_units": gold_units, "reader_target": reader_unk, "gold_target": gold_unk, } @@ -64,7 +80,7 @@ def run() -> dict[str, Any]: return { "lane": "setup_oracle", - "grades": "structure-only (facts + equations + question target); units deferred", + "grades": "structure + per-symbol units + question target (symbol/state/form/unit)", "total": len(cases), "setup_correct": setup_correct, "setup_wrong": setup_wrong, diff --git a/evals/setup_oracle/signature.py b/evals/setup_oracle/signature.py index 83b95103..cd7dfe5e 100644 --- a/evals/setup_oracle/signature.py +++ b/evals/setup_oracle/signature.py @@ -6,10 +6,11 @@ offsets and surface tokens. Two readings are setup-equivalent iff their signatur are equal. Used to compare the reader's comprehended structure against the independent gold structure (the relational_metric cases' own `relations`/`query`). -v1 grades: facts (entity, value), equations (the typed relation shape), and the -question target (symbol, state-index, question-form). Unit modelling is intentionally -NOT in the signature yet — it is covered by the admissibility tests, and a future -extension adds an expected-unit axis once the gold carries it. +v2 (PR-5a) grades: facts (entity, value), equations (the typed relation shape), the +question target (symbol, state-index, question-form, **unit**), and the **per-symbol +unit** — read from the binding-graph itself, not the answer projection. A reading whose +structure matches but whose units diverge from the independent expected-unit gold now +FAILS (``setup_wrong``). The ruler must be unit-aware before it judges real GSM8K frames. """ from __future__ import annotations @@ -19,6 +20,19 @@ from typing import Any from generate.binding_graph.model import SemanticSymbolicBindingGraph +def symbol_unit_signature(units: dict[str, str | None]) -> tuple[tuple[str, str], ...]: + """Canonicalize a per-symbol unit map into a sorted, order-independent signature. + + Used for BOTH sides: the reader's units come from the binding-graph's symbols; the + gold's from the independent ``expected_units`` fixture. A ``None`` unit (a symbol the + reader left unmodelled) canonicalizes to ``"unset"`` so it can never silently match a + declared gold unit. + """ + return tuple( + sorted((sid, unit if unit is not None else "unset") for sid, unit in units.items()) + ) + + def relation_signature(relations: list[dict[str, Any]]) -> tuple[tuple, ...]: """Canonicalize a list of relations (the ``to_relational_metric`` / gold shape) into a sorted, order-independent tuple of typed relation tuples.""" @@ -37,15 +51,19 @@ def relation_signature(relations: list[dict[str, Any]]) -> tuple[tuple, ...]: def gold_unknown_signature( - relations: list[dict[str, Any]], query: dict[str, Any] -) -> tuple[str, str, str]: + relations: list[dict[str, Any]], + query: dict[str, Any], + expected_units: dict[str, str], +) -> tuple[str, str, str, str]: """The expected question-target signature, derived from the INDEPENDENT gold. A query whose target is an aggregate (the gold contains a ``sum_of`` producing it) is a ``total`` form; otherwise a ``count``. All current cases ask the terminal state. + The expected target unit comes from the independent ``expected_units`` fixture. """ form = "total" if any(r["kind"] == "sum_of" for r in relations) else "count" - return (query["entity"], "terminal", form) + entity = query["entity"] + return (entity, "terminal", form, expected_units.get(entity, "unset")) def _state_token(state_index: Any) -> str: @@ -55,8 +73,11 @@ def _state_token(state_index: Any) -> str: return f"op{getattr(state_index, 'operation_index', '?')}" -def reader_unknown_signature(graph: SemanticSymbolicBindingGraph) -> tuple[str, str, str]: - """The reader's question-target signature from ``graph.unknowns`` (PR-1). +def reader_unknown_signature( + graph: SemanticSymbolicBindingGraph, +) -> tuple[str, str, str, str]: + """The reader's question-target signature from ``graph.unknowns`` (PR-1), now with + the target's ``expected_unit`` (PR-5a). A graph that does not carry exactly one unknown is itself a structural defect — it is reported as a distinguished ``MALFORMED`` signature so it can never silently @@ -64,13 +85,26 @@ def reader_unknown_signature(graph: SemanticSymbolicBindingGraph) -> tuple[str, """ unknowns = graph.unknowns if len(unknowns) != 1: - return ("MALFORMED", str(len(unknowns)), "") + return ("MALFORMED", str(len(unknowns)), "", "") u = unknowns[0] - return (u.symbol_id, _state_token(u.state_index), u.question_form) + return ( + u.symbol_id, + _state_token(u.state_index), + u.question_form, + u.expected_unit if u.expected_unit is not None else "unset", + ) + + +def reader_symbol_units(graph: SemanticSymbolicBindingGraph) -> tuple[tuple[str, str], ...]: + """The reader's per-symbol unit signature, read from the BINDING-GRAPH (not the + answer projection) — the unit each symbol was modelled with.""" + return symbol_unit_signature({s.symbol_id: s.unit for s in graph.symbols}) __all__ = [ "gold_unknown_signature", + "reader_symbol_units", "reader_unknown_signature", "relation_signature", + "symbol_unit_signature", ] diff --git a/tests/test_setup_oracle.py b/tests/test_setup_oracle.py index 05ba3540..97459dd4 100644 --- a/tests/test_setup_oracle.py +++ b/tests/test_setup_oracle.py @@ -12,12 +12,15 @@ from __future__ import annotations from evals.setup_oracle import ( gold_unknown_signature, + reader_symbol_units, reader_unknown_signature, relation_signature, run, + symbol_unit_signature, ) from generate.binding_graph.model import ( BoundFact, + BoundUnknown, SemanticSymbolicBindingGraph, SourceSpanLink, SymbolBinding, @@ -79,8 +82,9 @@ def test_wrong_question_target_is_caught() -> None: {"kind": "sum_of", "entity": "total", "parts": ["dan", "eva"]}, ] # Gold asks the total; a reader that targeted "eva" instead is a different reading. - assert gold_unknown_signature(rels, {"entity": "total"}) == ("total", "terminal", "total") - assert gold_unknown_signature(rels, {"entity": "total"}) != ("eva", "terminal", "count") + units = {"total": "item"} + assert gold_unknown_signature(rels, {"entity": "total"}, units) == ("total", "terminal", "total", "item") + assert gold_unknown_signature(rels, {"entity": "total"}, units) != ("eva", "terminal", "count", "item") def test_malformed_graph_target_never_matches_gold() -> None: @@ -95,4 +99,44 @@ def test_malformed_graph_target_never_matches_gold() -> None: ) sig = reader_unknown_signature(graph) assert sig[0] == "MALFORMED" - assert sig != ("x", "terminal", "count") + assert sig != ("x", "terminal", "count", "item") + + +# --------------------------------------------------------------------------- # +# PR-5a — the ruler is now UNIT-AWARE (structure can match while units diverge) +# --------------------------------------------------------------------------- # + + +def test_unit_mismatch_is_caught_even_when_structure_matches() -> None: + # Same structure (a single fact about x), but the reader modelled a different unit. + # The setup-oracle must FAIL — a unit-wrong reading is not a correct setup. + gold_units = symbol_unit_signature({"x": "item"}) + reader_units_wrong = symbol_unit_signature({"x": "meter"}) + assert gold_units != reader_units_wrong + assert symbol_unit_signature({"x": "item"}) == symbol_unit_signature({"x": "item"}) + + +def test_target_unit_mismatch_is_caught() -> None: + # Structure + symbol + state + form all agree, but the target's expected unit differs. + rels = [{"kind": "fact", "entity": "x", "value": 1}] + assert gold_unknown_signature(rels, {"entity": "x"}, {"x": "item"}) != gold_unknown_signature( + rels, {"entity": "x"}, {"x": "dollars"} + ) + + +def test_reader_units_read_from_the_binding_graph() -> None: + # The reader's unit signature comes from the GRAPH's symbols, not the answer projection. + graph = SemanticSymbolicBindingGraph( + symbols=( + SymbolBinding(symbol_id="iris", name="iris", semantic_role="count", + source_span=_span(), introduced_by="t", entity="iris", unit="dollars"), + SymbolBinding(symbol_id="jack", name="jack", semantic_role="count", + source_span=_span(), introduced_by="t", entity="jack", unit="dollars"), + ), + facts=(BoundFact(symbol_id="iris", value="100", source_span=_span(), unit="dollars"),), + equations=(), + unknowns=(BoundUnknown(symbol_id="jack", question_span=_span(), state_index="terminal", + question_form="count", expected_unit="dollars"),), + ) + assert reader_symbol_units(graph) == (("iris", "dollars"), ("jack", "dollars")) + assert reader_unknown_signature(graph) == ("jack", "terminal", "count", "dollars")