core-labs/core
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Merge pull request #611 from AssetOverflow/feat/setup-oracle-units

Browse source 
feat(setup-oracle): make the ruler UNIT-AWARE (setup-oracle v2, PR-5a)
This commit is contained in:
Shay 2026-06-06 17:13:20 -07:00 committed by GitHub
commit 6c8f2b8332
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 148 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
evals/setup_oracle/__init__.py
View file

@ -16,13 +16,17 @@ modelling stays covered by the admissibility tests (a documented signature exten
from evals.setup_oracle.runner import run
from evals.setup_oracle.signature import (
gold_unknown_signature,
reader_symbol_units,
reader_unknown_signature,
relation_signature,
symbol_unit_signature,
)
__all__ = [
"gold_unknown_signature",
"reader_symbol_units",
"reader_unknown_signature",
"relation_signature",
"run",
"symbol_unit_signature",
]

18
evals/setup_oracle/expected_units.json Normal file
View file

@ -0,0 +1,18 @@
{
"_doc": "Independent expected MODELING unit per entity for the setup-oracle (PR-5a). Hand-authored from the PROBLEM, not copied from the reader: a discrete sortal count (stickers, cards, coins, ...) is dimensionally a count -> the generic count unit 'item'; money ('dollars') stays 'dollars'. The 'total' of same-unit parts inherits that unit. The setup-oracle fails (setup_wrong) if the reader's per-symbol unit diverges from this.",
"rm-v1-0001": {"liam": "item", "mia": "item"},
"rm-v1-0002": {"noah": "item", "olivia": "item"},
"rm-v1-0003": {"ava": "item", "ben": "item", "cara": "item"},
"rm-v1-0004": {"dan": "item", "eva": "item", "total": "item"},
"rm-v1-0005": {"finn": "item"},
"rm-v1-0006": {"gabe": "item", "hana": "item"},
"rm-v1-0007": {"iris": "dollars", "jack": "dollars"},
"rm-v1-0008": {"kim": "item", "leo": "item", "total": "item"},
"rm-v1-0009": {"maya": "item", "nico": "item"},
"rm-v1-0010": {"omar": "item", "pia": "item", "quinn": "item"},
"rm-v1-0011": {"rosa": "item", "sam": "item", "total": "item"},
"rm-v1-0012": {"tara": "item", "uma": "item"},
"rm-v1-0013": {"vera": "item", "will": "item"},
"rm-v1-0014": {"xena": "item", "yara": "item", "zane": "item"},
"rm-v1-0015": {"gus": "item"}
}

52
evals/setup_oracle/runner.py
View file

@ -1,32 +1,44 @@
"""Setup-oracle runner — grade the reader's comprehended structure vs gold structure.
"""Setup-oracle runner — grade the reader's comprehended structure + UNITS vs gold.
For each relational_metric case: comprehend the prose into a binding-graph, project it
to relations + read its question target, and compare the (relations, target) SIGNATURE
to the case's INDEPENDENT gold (`relations` + `query`). A structural mismatch is
``setup_wrong`` — the wrong=0-critical count — even if the answer would be right.
This is a STRICTER gate than the relational_metric (answer) lane: it requires the
reader to have read the problem the way the gold says it reads, not merely to land on
the gold number. It is the gate every future frame family must pass before serving.
For each relational_metric case: comprehend the prose into a binding-graph and compare,
against the INDEPENDENT gold, three axes:
1. the relation STRUCTURE (facts + typed equations — from the IR, not a reparse),
2. the per-symbol UNITS (read from the binding-graph, vs the expected_units fixture),
3. the question TARGET (symbol, state-index, form, expected unit).
A mismatch on ANY axis is ``setup_wrong`` — the wrong=0-critical count — even if the
answer would be right. This is a STRICTER gate than the relational_metric (answer) lane,
and the gate every future GSM8K frame family must pass before serving.
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
from evals.relational_metric.runner import _load_cases
from evals.setup_oracle.signature import (
gold_unknown_signature,
reader_symbol_units,
reader_unknown_signature,
relation_signature,
symbol_unit_signature,
)
from generate.meaning_graph.reader import Refusal
from generate.quantitative_comprehension import comprehend_quantitative, to_relational_metric
_EXPECTED_UNITS_PATH = Path(__file__).resolve().parent / "expected_units.json"
def _load_expected_units() -> dict[str, dict[str, str]]:
raw = json.loads(_EXPECTED_UNITS_PATH.read_text(encoding="utf-8"))
return {k: v for k, v in raw.items() if not k.startswith("_")}
def run() -> dict[str, Any]:
"""Score the reader's setup against the independent gold setup, structure-only."""
"""Score the reader's setup (structure + units + target) against the independent gold."""
cases = _load_cases()
expected_units = _load_expected_units()
setup_correct = setup_wrong = setup_refused = 0
wrongs: list[dict[str, Any]] = []
@ -40,23 +52,27 @@ def run() -> dict[str, Any]:
setup_refused += 1
continue
reader_relations, _reader_query = projected
case_units = expected_units.get(case.get("id", ""), {})
reader_sig = relation_signature(reader_relations)
gold_sig = relation_signature(case["relations"])
reader_rel = relation_signature(reader_relations)
gold_rel = relation_signature(case["relations"])
reader_units = reader_symbol_units(comp.binding_graph)
gold_units = symbol_unit_signature(case_units)
reader_unk = reader_unknown_signature(comp.binding_graph)
gold_unk = gold_unknown_signature(case["relations"], case["query"])
gold_unk = gold_unknown_signature(case["relations"], case["query"], case_units)
if reader_sig == gold_sig and reader_unk == gold_unk:
if reader_rel == gold_rel and reader_units == gold_units and reader_unk == gold_unk:
setup_correct += 1
else:
setup_wrong += 1
wrongs.append(
{
"id": case.get("id"),
"relations_match": reader_sig == gold_sig,
"relations_match": reader_rel == gold_rel,
"units_match": reader_units == gold_units,
"target_match": reader_unk == gold_unk,
"reader_relations": reader_sig,
"gold_relations": gold_sig,
"reader_units": reader_units,
"gold_units": gold_units,
"reader_target": reader_unk,
"gold_target": gold_unk,
}
@ -64,7 +80,7 @@ def run() -> dict[str, Any]:
return {
"lane": "setup_oracle",
"grades": "structure-only (facts + equations + question target); units deferred",
"grades": "structure + per-symbol units + question target (symbol/state/form/unit)",
"total": len(cases),
"setup_correct": setup_correct,
"setup_wrong": setup_wrong,

56
evals/setup_oracle/signature.py
View file

@ -6,10 +6,11 @@ offsets and surface tokens. Two readings are setup-equivalent iff their signatur
are equal. Used to compare the reader's comprehended structure against the
independent gold structure (the relational_metric cases' own `relations`/`query`).
v1 grades: facts (entity, value), equations (the typed relation shape), and the
question target (symbol, state-index, question-form). Unit modelling is intentionally
NOT in the signature yet — it is covered by the admissibility tests, and a future
extension adds an expected-unit axis once the gold carries it.
v2 (PR-5a) grades: facts (entity, value), equations (the typed relation shape), the
question target (symbol, state-index, question-form, **unit**), and the **per-symbol
unit** — read from the binding-graph itself, not the answer projection. A reading whose
structure matches but whose units diverge from the independent expected-unit gold now
FAILS (``setup_wrong``). The ruler must be unit-aware before it judges real GSM8K frames.
"""
from __future__ import annotations
@ -19,6 +20,19 @@ from typing import Any
from generate.binding_graph.model import SemanticSymbolicBindingGraph
def symbol_unit_signature(units: dict[str, str | None]) -> tuple[tuple[str, str], ...]:
"""Canonicalize a per-symbol unit map into a sorted, order-independent signature.
Used for BOTH sides: the reader's units come from the binding-graph's symbols; the
gold's from the independent ``expected_units`` fixture. A ``None`` unit (a symbol the
reader left unmodelled) canonicalizes to ``"unset"`` so it can never silently match a
declared gold unit.
"""
return tuple(
sorted((sid, unit if unit is not None else "unset") for sid, unit in units.items())
)
def relation_signature(relations: list[dict[str, Any]]) -> tuple[tuple, ...]:
"""Canonicalize a list of relations (the ``to_relational_metric`` / gold shape)
into a sorted, order-independent tuple of typed relation tuples."""
@ -37,15 +51,19 @@ def relation_signature(relations: list[dict[str, Any]]) -> tuple[tuple, ...]:
def gold_unknown_signature(
relations: list[dict[str, Any]], query: dict[str, Any]
) -> tuple[str, str, str]:
relations: list[dict[str, Any]],
query: dict[str, Any],
expected_units: dict[str, str],
) -> tuple[str, str, str, str]:
"""The expected question-target signature, derived from the INDEPENDENT gold.
A query whose target is an aggregate (the gold contains a ``sum_of`` producing it)
is a ``total`` form; otherwise a ``count``. All current cases ask the terminal state.
The expected target unit comes from the independent ``expected_units`` fixture.
"""
form = "total" if any(r["kind"] == "sum_of" for r in relations) else "count"
return (query["entity"], "terminal", form)
entity = query["entity"]
return (entity, "terminal", form, expected_units.get(entity, "unset"))
def _state_token(state_index: Any) -> str:
@ -55,8 +73,11 @@ def _state_token(state_index: Any) -> str:
return f"op{getattr(state_index, 'operation_index', '?')}"
def reader_unknown_signature(graph: SemanticSymbolicBindingGraph) -> tuple[str, str, str]:
"""The reader's question-target signature from ``graph.unknowns`` (PR-1).
def reader_unknown_signature(
graph: SemanticSymbolicBindingGraph,
) -> tuple[str, str, str, str]:
"""The reader's question-target signature from ``graph.unknowns`` (PR-1), now with
the target's ``expected_unit`` (PR-5a).
A graph that does not carry exactly one unknown is itself a structural defect — it
is reported as a distinguished ``MALFORMED`` signature so it can never silently
@ -64,13 +85,26 @@ def reader_unknown_signature(graph: SemanticSymbolicBindingGraph) -> tuple[str,
"""
unknowns = graph.unknowns
if len(unknowns) != 1:
return ("MALFORMED", str(len(unknowns)), "")
return ("MALFORMED", str(len(unknowns)), "", "")
u = unknowns[0]
return (u.symbol_id, _state_token(u.state_index), u.question_form)
return (
u.symbol_id,
_state_token(u.state_index),
u.question_form,
u.expected_unit if u.expected_unit is not None else "unset",
)
def reader_symbol_units(graph: SemanticSymbolicBindingGraph) -> tuple[tuple[str, str], ...]:
"""The reader's per-symbol unit signature, read from the BINDING-GRAPH (not the
answer projection) — the unit each symbol was modelled with."""
return symbol_unit_signature({s.symbol_id: s.unit for s in graph.symbols})
__all__ = [
"gold_unknown_signature",
"reader_symbol_units",
"reader_unknown_signature",
"relation_signature",
"symbol_unit_signature",
]

50
tests/test_setup_oracle.py
View file

@ -12,12 +12,15 @@ from __future__ import annotations
from evals.setup_oracle import (
gold_unknown_signature,
reader_symbol_units,
reader_unknown_signature,
relation_signature,
run,
symbol_unit_signature,
)
from generate.binding_graph.model import (
BoundFact,
BoundUnknown,
SemanticSymbolicBindingGraph,
SourceSpanLink,
SymbolBinding,
@ -79,8 +82,9 @@ def test_wrong_question_target_is_caught() -> None:
{"kind": "sum_of", "entity": "total", "parts": ["dan", "eva"]},
]
# Gold asks the total; a reader that targeted "eva" instead is a different reading.
assert gold_unknown_signature(rels, {"entity": "total"}) == ("total", "terminal", "total")
assert gold_unknown_signature(rels, {"entity": "total"}) != ("eva", "terminal", "count")
units = {"total": "item"}
assert gold_unknown_signature(rels, {"entity": "total"}, units) == ("total", "terminal", "total", "item")
assert gold_unknown_signature(rels, {"entity": "total"}, units) != ("eva", "terminal", "count", "item")
def test_malformed_graph_target_never_matches_gold() -> None:
@ -95,4 +99,44 @@ def test_malformed_graph_target_never_matches_gold() -> None:
)
sig = reader_unknown_signature(graph)
assert sig[0] == "MALFORMED"
assert sig != ("x", "terminal", "count")
assert sig != ("x", "terminal", "count", "item")
# --------------------------------------------------------------------------- #
# PR-5a — the ruler is now UNIT-AWARE (structure can match while units diverge)
# --------------------------------------------------------------------------- #
def test_unit_mismatch_is_caught_even_when_structure_matches() -> None:
# Same structure (a single fact about x), but the reader modelled a different unit.
# The setup-oracle must FAIL — a unit-wrong reading is not a correct setup.
gold_units = symbol_unit_signature({"x": "item"})
reader_units_wrong = symbol_unit_signature({"x": "meter"})
assert gold_units != reader_units_wrong
assert symbol_unit_signature({"x": "item"}) == symbol_unit_signature({"x": "item"})
def test_target_unit_mismatch_is_caught() -> None:
# Structure + symbol + state + form all agree, but the target's expected unit differs.
rels = [{"kind": "fact", "entity": "x", "value": 1}]
assert gold_unknown_signature(rels, {"entity": "x"}, {"x": "item"}) != gold_unknown_signature(
rels, {"entity": "x"}, {"x": "dollars"}
)
def test_reader_units_read_from_the_binding_graph() -> None:
# The reader's unit signature comes from the GRAPH's symbols, not the answer projection.
graph = SemanticSymbolicBindingGraph(
symbols=(
SymbolBinding(symbol_id="iris", name="iris", semantic_role="count",
source_span=_span(), introduced_by="t", entity="iris", unit="dollars"),
SymbolBinding(symbol_id="jack", name="jack", semantic_role="count",
source_span=_span(), introduced_by="t", entity="jack", unit="dollars"),
),
facts=(BoundFact(symbol_id="iris", value="100", source_span=_span(), unit="dollars"),),
equations=(),
unknowns=(BoundUnknown(symbol_id="jack", question_span=_span(), state_index="terminal",
question_form="count", expected_unit="dollars"),),
)
assert reader_symbol_units(graph) == (("iris", "dollars"), ("jack", "dollars"))
assert reader_unknown_signature(graph) == ("jack", "terminal", "count", "dollars")