Merge pull request #611 from AssetOverflow/feat/setup-oracle-units

feat(setup-oracle): make the ruler UNIT-AWARE (setup-oracle v2, PR-5a)

evals/setup_oracle/__init__.py

@@ -16,13 +16,17 @@ modelling stays covered by the admissibility tests (a documented signature exten
 from evals.setup_oracle.runner import run
 from evals.setup_oracle.signature import (
     gold_unknown_signature,
+    reader_symbol_units,
     reader_unknown_signature,
     relation_signature,
+    symbol_unit_signature,
 )
 
 __all__ = [
     "gold_unknown_signature",
+    "reader_symbol_units",
     "reader_unknown_signature",
     "relation_signature",
     "run",
+    "symbol_unit_signature",
 ]

evals/setup_oracle/expected_units.json

@@ -0,0 +1,18 @@
+{
+  "_doc": "Independent expected MODELING unit per entity for the setup-oracle (PR-5a). Hand-authored from the PROBLEM, not copied from the reader: a discrete sortal count (stickers, cards, coins, ...) is dimensionally a count -> the generic count unit 'item'; money ('dollars') stays 'dollars'. The 'total' of same-unit parts inherits that unit. The setup-oracle fails (setup_wrong) if the reader's per-symbol unit diverges from this.",
+  "rm-v1-0001": {"liam": "item", "mia": "item"},
+  "rm-v1-0002": {"noah": "item", "olivia": "item"},
+  "rm-v1-0003": {"ava": "item", "ben": "item", "cara": "item"},
+  "rm-v1-0004": {"dan": "item", "eva": "item", "total": "item"},
+  "rm-v1-0005": {"finn": "item"},
+  "rm-v1-0006": {"gabe": "item", "hana": "item"},
+  "rm-v1-0007": {"iris": "dollars", "jack": "dollars"},
+  "rm-v1-0008": {"kim": "item", "leo": "item", "total": "item"},
+  "rm-v1-0009": {"maya": "item", "nico": "item"},
+  "rm-v1-0010": {"omar": "item", "pia": "item", "quinn": "item"},
+  "rm-v1-0011": {"rosa": "item", "sam": "item", "total": "item"},
+  "rm-v1-0012": {"tara": "item", "uma": "item"},
+  "rm-v1-0013": {"vera": "item", "will": "item"},
+  "rm-v1-0014": {"xena": "item", "yara": "item", "zane": "item"},
+  "rm-v1-0015": {"gus": "item"}
+}

evals/setup_oracle/runner.py

@@ -1,32 +1,44 @@
-"""Setup-oracle runner — grade the reader's comprehended structure vs gold structure.
+"""Setup-oracle runner — grade the reader's comprehended structure + UNITS vs gold.
 
-For each relational_metric case: comprehend the prose into a binding-graph, project it
+For each relational_metric case: comprehend the prose into a binding-graph and compare,
-to relations + read its question target, and compare the (relations, target) SIGNATURE
+against the INDEPENDENT gold, three axes:
-to the case's INDEPENDENT gold (`relations` + `query`). A structural mismatch is
+  1. the relation STRUCTURE (facts + typed equations — from the IR, not a reparse),
-``setup_wrong`` — the wrong=0-critical count — even if the answer would be right.
+  2. the per-symbol UNITS (read from the binding-graph, vs the expected_units fixture),
-
+  3. the question TARGET (symbol, state-index, form, expected unit).
-This is a STRICTER gate than the relational_metric (answer) lane: it requires the
+A mismatch on ANY axis is ``setup_wrong`` — the wrong=0-critical count — even if the
-reader to have read the problem the way the gold says it reads, not merely to land on
+answer would be right. This is a STRICTER gate than the relational_metric (answer) lane,
-the gold number. It is the gate every future frame family must pass before serving.
+and the gate every future GSM8K frame family must pass before serving.
 """
 
 from __future__ import annotations
 
+import json
+from pathlib import Path
 from typing import Any
 
 from evals.relational_metric.runner import _load_cases
 from evals.setup_oracle.signature import (
     gold_unknown_signature,
+    reader_symbol_units,
     reader_unknown_signature,
     relation_signature,
+    symbol_unit_signature,
 )
 from generate.meaning_graph.reader import Refusal
 from generate.quantitative_comprehension import comprehend_quantitative, to_relational_metric
 
+_EXPECTED_UNITS_PATH = Path(__file__).resolve().parent / "expected_units.json"
+
+
+def _load_expected_units() -> dict[str, dict[str, str]]:
+    raw = json.loads(_EXPECTED_UNITS_PATH.read_text(encoding="utf-8"))
+    return {k: v for k, v in raw.items() if not k.startswith("_")}
+
 
 def run() -> dict[str, Any]:
-    """Score the reader's setup against the independent gold setup, structure-only."""
+    """Score the reader's setup (structure + units + target) against the independent gold."""
     cases = _load_cases()
+    expected_units = _load_expected_units()
     setup_correct = setup_wrong = setup_refused = 0
     wrongs: list[dict[str, Any]] = []
 
@@ -40,23 +52,27 @@ def run() -> dict[str, Any]:
             setup_refused += 1
             continue
         reader_relations, _reader_query = projected
+        case_units = expected_units.get(case.get("id", ""), {})
 
-        reader_sig = relation_signature(reader_relations)
+        reader_rel = relation_signature(reader_relations)
-        gold_sig = relation_signature(case["relations"])
+        gold_rel = relation_signature(case["relations"])
+        reader_units = reader_symbol_units(comp.binding_graph)
+        gold_units = symbol_unit_signature(case_units)
         reader_unk = reader_unknown_signature(comp.binding_graph)
-        gold_unk = gold_unknown_signature(case["relations"], case["query"])
+        gold_unk = gold_unknown_signature(case["relations"], case["query"], case_units)
 
-        if reader_sig == gold_sig and reader_unk == gold_unk:
+        if reader_rel == gold_rel and reader_units == gold_units and reader_unk == gold_unk:
             setup_correct += 1
         else:
             setup_wrong += 1
             wrongs.append(
                 {
                     "id": case.get("id"),
-                    "relations_match": reader_sig == gold_sig,
+                    "relations_match": reader_rel == gold_rel,
+                    "units_match": reader_units == gold_units,
                     "target_match": reader_unk == gold_unk,
-                    "reader_relations": reader_sig,
+                    "reader_units": reader_units,
-                    "gold_relations": gold_sig,
+                    "gold_units": gold_units,
                     "reader_target": reader_unk,
                     "gold_target": gold_unk,
                 }
@@ -64,7 +80,7 @@ def run() -> dict[str, Any]:
 
     return {
         "lane": "setup_oracle",
-        "grades": "structure-only (facts + equations + question target); units deferred",
+        "grades": "structure + per-symbol units + question target (symbol/state/form/unit)",
         "total": len(cases),
         "setup_correct": setup_correct,
         "setup_wrong": setup_wrong,

evals/setup_oracle/signature.py

@@ -6,10 +6,11 @@ offsets and surface tokens. Two readings are setup-equivalent iff their signatur
 are equal. Used to compare the reader's comprehended structure against the
 independent gold structure (the relational_metric cases' own `relations`/`query`).
 
-v1 grades: facts (entity, value), equations (the typed relation shape), and the
+v2 (PR-5a) grades: facts (entity, value), equations (the typed relation shape), the
-question target (symbol, state-index, question-form). Unit modelling is intentionally
+question target (symbol, state-index, question-form, **unit**), and the **per-symbol
-NOT in the signature yet — it is covered by the admissibility tests, and a future
+unit** — read from the binding-graph itself, not the answer projection. A reading whose
-extension adds an expected-unit axis once the gold carries it.
+structure matches but whose units diverge from the independent expected-unit gold now
+FAILS (``setup_wrong``). The ruler must be unit-aware before it judges real GSM8K frames.
 """
 
 from __future__ import annotations
@@ -19,6 +20,19 @@ from typing import Any
 from generate.binding_graph.model import SemanticSymbolicBindingGraph
 
 
+def symbol_unit_signature(units: dict[str, str | None]) -> tuple[tuple[str, str], ...]:
+    """Canonicalize a per-symbol unit map into a sorted, order-independent signature.
+
+    Used for BOTH sides: the reader's units come from the binding-graph's symbols; the
+    gold's from the independent ``expected_units`` fixture. A ``None`` unit (a symbol the
+    reader left unmodelled) canonicalizes to ``"unset"`` so it can never silently match a
+    declared gold unit.
+    """
+    return tuple(
+        sorted((sid, unit if unit is not None else "unset") for sid, unit in units.items())
+    )
+
+
 def relation_signature(relations: list[dict[str, Any]]) -> tuple[tuple, ...]:
     """Canonicalize a list of relations (the ``to_relational_metric`` / gold shape)
     into a sorted, order-independent tuple of typed relation tuples."""
@@ -37,15 +51,19 @@ def relation_signature(relations: list[dict[str, Any]]) -> tuple[tuple, ...]:
 
 
 def gold_unknown_signature(
-    relations: list[dict[str, Any]], query: dict[str, Any]
+    relations: list[dict[str, Any]],
-) -> tuple[str, str, str]:
+    query: dict[str, Any],
+    expected_units: dict[str, str],
+) -> tuple[str, str, str, str]:
     """The expected question-target signature, derived from the INDEPENDENT gold.
 
     A query whose target is an aggregate (the gold contains a ``sum_of`` producing it)
     is a ``total`` form; otherwise a ``count``. All current cases ask the terminal state.
+    The expected target unit comes from the independent ``expected_units`` fixture.
     """
     form = "total" if any(r["kind"] == "sum_of" for r in relations) else "count"
-    return (query["entity"], "terminal", form)
+    entity = query["entity"]
+    return (entity, "terminal", form, expected_units.get(entity, "unset"))
 
 
 def _state_token(state_index: Any) -> str:
@@ -55,8 +73,11 @@ def _state_token(state_index: Any) -> str:
     return f"op{getattr(state_index, 'operation_index', '?')}"
 
 
-def reader_unknown_signature(graph: SemanticSymbolicBindingGraph) -> tuple[str, str, str]:
+def reader_unknown_signature(
-    """The reader's question-target signature from ``graph.unknowns`` (PR-1).
+    graph: SemanticSymbolicBindingGraph,
+) -> tuple[str, str, str, str]:
+    """The reader's question-target signature from ``graph.unknowns`` (PR-1), now with
+    the target's ``expected_unit`` (PR-5a).
 
     A graph that does not carry exactly one unknown is itself a structural defect — it
     is reported as a distinguished ``MALFORMED`` signature so it can never silently
@@ -64,13 +85,26 @@ def reader_unknown_signature(graph: SemanticSymbolicBindingGraph) -> tuple[str,
     """
     unknowns = graph.unknowns
     if len(unknowns) != 1:
-        return ("MALFORMED", str(len(unknowns)), "")
+        return ("MALFORMED", str(len(unknowns)), "", "")
     u = unknowns[0]
-    return (u.symbol_id, _state_token(u.state_index), u.question_form)
+    return (
+        u.symbol_id,
+        _state_token(u.state_index),
+        u.question_form,
+        u.expected_unit if u.expected_unit is not None else "unset",
+    )
+
+
+def reader_symbol_units(graph: SemanticSymbolicBindingGraph) -> tuple[tuple[str, str], ...]:
+    """The reader's per-symbol unit signature, read from the BINDING-GRAPH (not the
+    answer projection) — the unit each symbol was modelled with."""
+    return symbol_unit_signature({s.symbol_id: s.unit for s in graph.symbols})
 
 
 __all__ = [
     "gold_unknown_signature",
+    "reader_symbol_units",
     "reader_unknown_signature",
     "relation_signature",
+    "symbol_unit_signature",
 ]

tests/test_setup_oracle.py

@@ -12,12 +12,15 @@ from __future__ import annotations
 
 from evals.setup_oracle import (
     gold_unknown_signature,
+    reader_symbol_units,
     reader_unknown_signature,
     relation_signature,
     run,
+    symbol_unit_signature,
 )
 from generate.binding_graph.model import (
     BoundFact,
+    BoundUnknown,
     SemanticSymbolicBindingGraph,
     SourceSpanLink,
     SymbolBinding,
@@ -79,8 +82,9 @@ def test_wrong_question_target_is_caught() -> None:
         {"kind": "sum_of", "entity": "total", "parts": ["dan", "eva"]},
     ]
     # Gold asks the total; a reader that targeted "eva" instead is a different reading.
-    assert gold_unknown_signature(rels, {"entity": "total"}) == ("total", "terminal", "total")
+    units = {"total": "item"}
-    assert gold_unknown_signature(rels, {"entity": "total"}) != ("eva", "terminal", "count")
+    assert gold_unknown_signature(rels, {"entity": "total"}, units) == ("total", "terminal", "total", "item")
+    assert gold_unknown_signature(rels, {"entity": "total"}, units) != ("eva", "terminal", "count", "item")
 
 
 def test_malformed_graph_target_never_matches_gold() -> None:
@@ -95,4 +99,44 @@ def test_malformed_graph_target_never_matches_gold() -> None:
     )
     sig = reader_unknown_signature(graph)
     assert sig[0] == "MALFORMED"
-    assert sig != ("x", "terminal", "count")
+    assert sig != ("x", "terminal", "count", "item")
+
+
+# --------------------------------------------------------------------------- #
+# PR-5a — the ruler is now UNIT-AWARE (structure can match while units diverge)
+# --------------------------------------------------------------------------- #
+
+
+def test_unit_mismatch_is_caught_even_when_structure_matches() -> None:
+    # Same structure (a single fact about x), but the reader modelled a different unit.
+    # The setup-oracle must FAIL — a unit-wrong reading is not a correct setup.
+    gold_units = symbol_unit_signature({"x": "item"})
+    reader_units_wrong = symbol_unit_signature({"x": "meter"})
+    assert gold_units != reader_units_wrong
+    assert symbol_unit_signature({"x": "item"}) == symbol_unit_signature({"x": "item"})
+
+
+def test_target_unit_mismatch_is_caught() -> None:
+    # Structure + symbol + state + form all agree, but the target's expected unit differs.
+    rels = [{"kind": "fact", "entity": "x", "value": 1}]
+    assert gold_unknown_signature(rels, {"entity": "x"}, {"x": "item"}) != gold_unknown_signature(
+        rels, {"entity": "x"}, {"x": "dollars"}
+    )
+
+
+def test_reader_units_read_from_the_binding_graph() -> None:
+    # The reader's unit signature comes from the GRAPH's symbols, not the answer projection.
+    graph = SemanticSymbolicBindingGraph(
+        symbols=(
+            SymbolBinding(symbol_id="iris", name="iris", semantic_role="count",
+                          source_span=_span(), introduced_by="t", entity="iris", unit="dollars"),
+            SymbolBinding(symbol_id="jack", name="jack", semantic_role="count",
+                          source_span=_span(), introduced_by="t", entity="jack", unit="dollars"),
+        ),
+        facts=(BoundFact(symbol_id="iris", value="100", source_span=_span(), unit="dollars"),),
+        equations=(),
+        unknowns=(BoundUnknown(symbol_id="jack", question_span=_span(), state_index="terminal",
+                               question_form="count", expected_unit="dollars"),),
+    )
+    assert reader_symbol_units(graph) == (("iris", "dollars"), ("jack", "dollars"))
+    assert reader_unknown_signature(graph) == ("jack", "terminal", "count", "dollars")