Merge pull request #608 from AssetOverflow/feat/quant-bound-unknown-and-setup-oracle

feat(comprehension): question target in the graph (PR-1) + setup-oracle lane

evals/setup_oracle/__init__.py

@@ -0,0 +1,28 @@
+"""Setup-oracle lane — grade the READING (the semantic setup), not the answer.
+
+The relational_metric lane scores answers: `comprehend → project → oracle_answer →
+compare gold integer`. That hides a hazard the held-out measurements exposed — a WRONG
+reading can produce a coincidentally-correct number and still pass. The setup-oracle
+closes that gap: it compares the reader's comprehended STRUCTURE (the relations it read
++ the BoundUnknown question target it emitted) against the INDEPENDENT gold structure
+(the relational_metric cases' own `relations`/`query`, authored separately from the
+binding-graph reader). A wrong setup is a first-class failure even when its answer is right.
+
+`setup_wrong` is the load-bearing, wrong=0-critical count: a reading that misrepresents
+the problem. v1 grades structure (facts + equations + question target/state/form); unit
+modelling stays covered by the admissibility tests (a documented signature extension).
+"""
+
+from evals.setup_oracle.runner import run
+from evals.setup_oracle.signature import (
+    gold_unknown_signature,
+    reader_unknown_signature,
+    relation_signature,
+)
+
+__all__ = [
+    "gold_unknown_signature",
+    "reader_unknown_signature",
+    "relation_signature",
+    "run",
+]

evals/setup_oracle/__main__.py

@@ -0,0 +1,23 @@
+"""CLI: print the setup-oracle report.
+
+    python -m evals.setup_oracle
+
+Exit 0 iff ``setup_wrong == 0`` — the gate the milestone rests on (a wrong reading must
+never pass, and serving must not move while setup_wrong > 0).
+"""
+
+from __future__ import annotations
+
+import json
+
+from evals.setup_oracle.runner import run
+
+
+def main() -> int:
+    report = run()
+    print(json.dumps(report, indent=2, default=str))
+    return 0 if report["setup_wrong"] == 0 else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

evals/setup_oracle/runner.py

@@ -0,0 +1,81 @@
+"""Setup-oracle runner — grade the reader's comprehended structure vs gold structure.
+
+For each relational_metric case: comprehend the prose into a binding-graph, project it
+to relations + read its question target, and compare the (relations, target) SIGNATURE
+to the case's INDEPENDENT gold (`relations` + `query`). A structural mismatch is
+``setup_wrong`` — the wrong=0-critical count — even if the answer would be right.
+
+This is a STRICTER gate than the relational_metric (answer) lane: it requires the
+reader to have read the problem the way the gold says it reads, not merely to land on
+the gold number. It is the gate every future frame family must pass before serving.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from evals.relational_metric.runner import _load_cases
+from evals.setup_oracle.signature import (
+    gold_unknown_signature,
+    reader_unknown_signature,
+    relation_signature,
+)
+from generate.meaning_graph.reader import Refusal
+from generate.quantitative_comprehension import comprehend_quantitative, to_relational_metric
+
+
+def run() -> dict[str, Any]:
+    """Score the reader's setup against the independent gold setup, structure-only."""
+    cases = _load_cases()
+    setup_correct = setup_wrong = setup_refused = 0
+    wrongs: list[dict[str, Any]] = []
+
+    for case in cases:
+        comp = comprehend_quantitative(case["text"])
+        if isinstance(comp, Refusal):
+            setup_refused += 1
+            continue
+        projected = to_relational_metric(comp)
+        if projected is None:
+            setup_refused += 1
+            continue
+        reader_relations, _reader_query = projected
+
+        reader_sig = relation_signature(reader_relations)
+        gold_sig = relation_signature(case["relations"])
+        reader_unk = reader_unknown_signature(comp.binding_graph)
+        gold_unk = gold_unknown_signature(case["relations"], case["query"])
+
+        if reader_sig == gold_sig and reader_unk == gold_unk:
+            setup_correct += 1
+        else:
+            setup_wrong += 1
+            wrongs.append(
+                {
+                    "id": case.get("id"),
+                    "relations_match": reader_sig == gold_sig,
+                    "target_match": reader_unk == gold_unk,
+                    "reader_relations": reader_sig,
+                    "gold_relations": gold_sig,
+                    "reader_target": reader_unk,
+                    "gold_target": gold_unk,
+                }
+            )
+
+    return {
+        "lane": "setup_oracle",
+        "grades": "structure-only (facts + equations + question target); units deferred",
+        "total": len(cases),
+        "setup_correct": setup_correct,
+        "setup_wrong": setup_wrong,
+        "setup_refused": setup_refused,
+        "wrongs": wrongs,
+        "counts": {
+            "setup_correct": setup_correct,
+            "setup_wrong": setup_wrong,
+            "setup_refused": setup_refused,
+        },
+    }
+
+
+__all__ = ["run"]

evals/setup_oracle/signature.py

@@ -0,0 +1,76 @@
+"""Span-free structural signatures for the setup-oracle.
+
+A *signature* is a deterministic, order-independent, span-free projection of the
+mathematical SETUP — what the reader claims the problem says, stripped of input
+offsets and surface tokens. Two readings are setup-equivalent iff their signatures
+are equal. Used to compare the reader's comprehended structure against the
+independent gold structure (the relational_metric cases' own `relations`/`query`).
+
+v1 grades: facts (entity, value), equations (the typed relation shape), and the
+question target (symbol, state-index, question-form). Unit modelling is intentionally
+NOT in the signature yet — it is covered by the admissibility tests, and a future
+extension adds an expected-unit axis once the gold carries it.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from generate.binding_graph.model import SemanticSymbolicBindingGraph
+
+
+def relation_signature(relations: list[dict[str, Any]]) -> tuple[tuple, ...]:
+    """Canonicalize a list of relations (the ``to_relational_metric`` / gold shape)
+    into a sorted, order-independent tuple of typed relation tuples."""
+    out: list[tuple] = []
+    for r in relations:
+        kind = r["kind"]
+        if kind == "fact":
+            out.append(("fact", r["entity"], int(r["value"])))
+        elif kind in ("more_than", "fewer_than"):
+            out.append((kind, r["entity"], r["ref"], int(r["delta"])))
+        elif kind == "sum_of":
+            out.append(("sum_of", r["entity"], tuple(sorted(r["parts"]))))
+        else:  # an unknown relation kind is itself a structural difference, not a crash
+            out.append(("unhandled_kind", kind, r.get("entity", "")))
+    return tuple(sorted(out, key=repr))
+
+
+def gold_unknown_signature(
+    relations: list[dict[str, Any]], query: dict[str, Any]
+) -> tuple[str, str, str]:
+    """The expected question-target signature, derived from the INDEPENDENT gold.
+
+    A query whose target is an aggregate (the gold contains a ``sum_of`` producing it)
+    is a ``total`` form; otherwise a ``count``. All current cases ask the terminal state.
+    """
+    form = "total" if any(r["kind"] == "sum_of" for r in relations) else "count"
+    return (query["entity"], "terminal", form)
+
+
+def _state_token(state_index: Any) -> str:
+    if isinstance(state_index, str):
+        return state_index
+    # An Operation state-index (ADR-0135) — name it by its operation index.
+    return f"op{getattr(state_index, 'operation_index', '?')}"
+
+
+def reader_unknown_signature(graph: SemanticSymbolicBindingGraph) -> tuple[str, str, str]:
+    """The reader's question-target signature from ``graph.unknowns`` (PR-1).
+
+    A graph that does not carry exactly one unknown is itself a structural defect — it
+    is reported as a distinguished ``MALFORMED`` signature so it can never silently
+    compare equal to a well-formed gold target.
+    """
+    unknowns = graph.unknowns
+    if len(unknowns) != 1:
+        return ("MALFORMED", str(len(unknowns)), "")
+    u = unknowns[0]
+    return (u.symbol_id, _state_token(u.state_index), u.question_form)
+
+
+__all__ = [
+    "gold_unknown_signature",
+    "reader_unknown_signature",
+    "relation_signature",
+]

generate/quantitative_comprehension.py

@@ -34,6 +34,7 @@ from generate.binding_graph.admissibility import AdmissibilityError, check_admis
 from generate.binding_graph.model import (
     BoundEquation,
     BoundFact,
+    BoundUnknown,
     SemanticSymbolicBindingGraph,
     SourceSpanLink,
     SymbolBinding,
@@ -271,9 +272,25 @@ def comprehend_quantitative(text: str, source_id: str = "input") -> QuantCompreh
             )
         )
 
+    # The question target lives INSIDE the graph (ADR-0135): a BoundUnknown bound to
+    # the asked symbol at the terminal state. The form is "total" for an aggregate
+    # query ("how many do X and Y have"), else "count". ``query`` is retained as a
+    # consistent-by-construction convenience for the existing relational_metric
+    # projection + realize path; a follow-up collapses it onto graph.unknowns.
+    unknown = BoundUnknown(
+        symbol_id=ask_entity,
+        question_span=_span(ask_entity),
+        state_index="terminal",
+        question_form="total" if sum_eq is not None else "count",
+        expected_unit=ask_unit,
+    )
+
     try:
         graph = SemanticSymbolicBindingGraph(
-            symbols=tuple(symbols), facts=bound_facts, equations=tuple(equations)
+            symbols=tuple(symbols),
+            facts=bound_facts,
+            equations=tuple(equations),
+            unknowns=(unknown,),
         )
     except Exception as exc:  # noqa: BLE001 — surface construction refusal
         return Refusal("invalid_binding_graph", repr(exc))

tests/test_quantitative_comprehension.py

@@ -35,6 +35,29 @@ def test_fact_and_more_than_build_binding_graph() -> None:
     assert comp.query.entity == "mia"
 
 
+def test_question_target_is_a_bound_unknown_in_the_graph() -> None:
+    # PR-1: the question target lives INSIDE the graph (a BoundUnknown at the terminal
+    # state), not only as the external QuantQuery.
+    comp = _comp("Liam has 6 stickers. Mia has 4 more stickers than Liam. How many stickers does Mia have?")
+    unknowns = comp.binding_graph.unknowns
+    assert len(unknowns) == 1
+    u = unknowns[0]
+    assert u.symbol_id == "mia"
+    assert u.state_index == "terminal"
+    assert u.question_form == "count"
+    assert u.expected_unit == "item"
+    # The graph's canonical serialization now carries the target.
+    assert "state=terminal" in comp.binding_graph.to_canonical_string()
+    # Retained convenience stays consistent with the in-graph unknown.
+    assert comp.query.entity == u.symbol_id
+
+
+def test_sum_query_target_is_total_form_unknown() -> None:
+    comp = _comp("Dan has 7 coins. Eva has 9 more coins than Dan. How many coins do Dan and Eva have?")
+    (u,) = comp.binding_graph.unknowns
+    assert u.symbol_id == "total" and u.question_form == "total" and u.state_index == "terminal"
+
+
 def test_count_nouns_resolve_to_item_dimension() -> None:
     # Unknown sortal nouns become the count dimension (item); admissibility admits.
     comp = _comp("Kim has 2 marbles. Leo has 3 more marbles than Kim. How many marbles does Leo have?")

tests/test_setup_oracle.py

@@ -0,0 +1,98 @@
+"""Setup-oracle lane — grade the reading (structure), not the answer.
+
+Two obligations:
+1. The current reader reads all 15 relational_metric cases with the gold STRUCTURE
+   (``setup_wrong == 0``) — the gate the milestone rests on.
+2. The oracle MEANINGFULLY FAILS — a reading that lands on the right number via the
+   WRONG structure is ``setup_wrong``. Without this, structure-grading would be
+   decoration; with it, "did we read it right?" is falsifiable.
+"""
+
+from __future__ import annotations
+
+from evals.setup_oracle import (
+    gold_unknown_signature,
+    reader_unknown_signature,
+    relation_signature,
+    run,
+)
+from generate.binding_graph.model import (
+    BoundFact,
+    SemanticSymbolicBindingGraph,
+    SourceSpanLink,
+    SymbolBinding,
+)
+
+
+def _span() -> SourceSpanLink:
+    return SourceSpanLink(source_id="t", start=0, end=1, text="x")
+
+
+# --------------------------------------------------------------------------- #
+# Obligation 1 — the reader reads the gold structure on every case
+# --------------------------------------------------------------------------- #
+
+
+def test_all_cases_setup_correct_wrong_zero() -> None:
+    report = run()
+    assert report["total"] == 15
+    assert report["setup_correct"] == 15
+    assert report["setup_wrong"] == 0  # the load-bearing count
+    assert report["setup_refused"] == 0
+
+
+# --------------------------------------------------------------------------- #
+# Obligation 2 — the oracle is not decoration (it catches wrong readings)
+# --------------------------------------------------------------------------- #
+
+
+def test_right_answer_wrong_structure_is_caught() -> None:
+    # Gold: mia = liam + 4 over liam = 6  (answer 10, read as a relation).
+    gold = [
+        {"kind": "fact", "entity": "liam", "value": 6},
+        {"kind": "more_than", "entity": "mia", "ref": "liam", "delta": 4},
+    ]
+    # A reading that lands on the SAME answer (mia = 10) but flattens the relation
+    # into a bare fact — the right number, the wrong reading.
+    wrong_structure = [{"kind": "fact", "entity": "mia", "value": 10}]
+    assert relation_signature(gold) != relation_signature(wrong_structure)
+
+
+def test_signature_catches_wrong_operation() -> None:
+    more = [{"kind": "more_than", "entity": "y", "ref": "x", "delta": 6}]
+    fewer = [{"kind": "fewer_than", "entity": "y", "ref": "x", "delta": 6}]
+    assert relation_signature(more) != relation_signature(fewer)
+
+
+def test_signature_is_order_independent() -> None:
+    a = [
+        {"kind": "fact", "entity": "x", "value": 1},
+        {"kind": "more_than", "entity": "y", "ref": "x", "delta": 2},
+    ]
+    assert relation_signature(a) == relation_signature(list(reversed(a)))
+
+
+def test_wrong_question_target_is_caught() -> None:
+    rels = [
+        {"kind": "fact", "entity": "dan", "value": 7},
+        {"kind": "more_than", "entity": "eva", "ref": "dan", "delta": 9},
+        {"kind": "sum_of", "entity": "total", "parts": ["dan", "eva"]},
+    ]
+    # Gold asks the total; a reader that targeted "eva" instead is a different reading.
+    assert gold_unknown_signature(rels, {"entity": "total"}) == ("total", "terminal", "total")
+    assert gold_unknown_signature(rels, {"entity": "total"}) != ("eva", "terminal", "count")
+
+
+def test_malformed_graph_target_never_matches_gold() -> None:
+    # A graph carrying no question target (pre-PR-1 shape) must report MALFORMED and
+    # never silently compare equal to a well-formed gold target.
+    graph = SemanticSymbolicBindingGraph(
+        symbols=(SymbolBinding(symbol_id="x", name="x", semantic_role="count",
+                               source_span=_span(), introduced_by="t", entity="x", unit="item"),),
+        facts=(BoundFact(symbol_id="x", value="1", source_span=_span(), unit="item"),),
+        equations=(),
+        unknowns=(),
+    )
+    sig = reader_unknown_signature(graph)
+    assert sig[0] == "MALFORMED"
+    assert sig != ("x", "terminal", "count")