feat(comprehension): add R1 answer-oracle lane

evals/setup_oracle/runner.py

@@ -16,6 +16,7 @@ import json
 from pathlib import Path
 from typing import Any
 
+from evals.relational_metric.oracle import OracleError, oracle_answer
 from evals.relational_metric.runner import _load_cases
 from evals.setup_oracle.signature import (
     gold_unknown_signature,
@@ -103,6 +104,42 @@ def run() -> dict[str, Any]:
     }
 
 
+def _score_setup_fixture(fx: dict[str, Any]) -> tuple[str, str | None, dict[str, Any] | None]:
+    """Score one independent setup fixture by structure, not answer.
+
+    Returns ``(outcome, reason, detail)`` where outcome is ``correct`` / ``refused`` /
+    ``WRONG``. Kept small so PR-6b can reuse the same setup gate before evaluating
+    answers; answer scoring must never run on a structurally wrong reading.
+    """
+    comp = comprehend_quantitative(fx["text"])
+    if isinstance(comp, Refusal):
+        return "refused", comp.reason, None
+    projected = to_relational_metric(comp)
+    if projected is None:
+        return "refused", "unprojectable", None
+    reader_relations, _ = projected
+    units = fx["expected_units"]
+
+    reader_rel = relation_signature(reader_relations)
+    gold_rel = relation_signature(fx["relations"])
+    reader_units = reader_symbol_units(comp.binding_graph)
+    gold_units = symbol_unit_signature(units)
+    reader_unk = reader_unknown_signature(comp.binding_graph)
+    gold_unk = gold_unknown_signature(fx["relations"], fx["query"], units)
+
+    if reader_rel == gold_rel and reader_units == gold_units and reader_unk == gold_unk:
+        return "correct", None, None
+    return "WRONG", None, {
+        "relations_match": reader_rel == gold_rel,
+        "units_match": reader_units == gold_units,
+        "target_match": reader_unk == gold_unk,
+        "reader_relations": reader_rel,
+        "gold_relations": gold_rel,
+        "reader_target": reader_unk,
+        "gold_target": gold_unk,
+    }
+
+
 def run_r1() -> dict[str, Any]:
     """Score the CURRENT reader against the independent, self-contained R1 gold.
 
@@ -117,44 +154,16 @@ def run_r1() -> dict[str, Any]:
     details: list[dict[str, Any]] = []
 
     for fx in fixtures:
-        comp = comprehend_quantitative(fx["text"])
+        outcome, reason, detail = _score_setup_fixture(fx)
-        if isinstance(comp, Refusal):
+        if outcome == "correct":
-            setup_refused += 1
-            details.append({"id": fx["id"], "outcome": "refused", "reason": comp.reason})
-            continue
-        projected = to_relational_metric(comp)
-        if projected is None:
-            setup_refused += 1
-            details.append({"id": fx["id"], "outcome": "refused", "reason": "unprojectable"})
-            continue
-        reader_relations, _ = projected
-        units = fx["expected_units"]
-
-        reader_rel = relation_signature(reader_relations)
-        gold_rel = relation_signature(fx["relations"])
-        reader_units = reader_symbol_units(comp.binding_graph)
-        gold_units = symbol_unit_signature(units)
-        reader_unk = reader_unknown_signature(comp.binding_graph)
-        gold_unk = gold_unknown_signature(fx["relations"], fx["query"], units)
-
-        if reader_rel == gold_rel and reader_units == gold_units and reader_unk == gold_unk:
             setup_correct += 1
             details.append({"id": fx["id"], "outcome": "correct"})
+        elif outcome == "refused":
+            setup_refused += 1
+            details.append({"id": fx["id"], "outcome": "refused", "reason": reason})
         else:
             setup_wrong += 1
-            details.append(
+            details.append({"id": fx["id"], "outcome": "WRONG", **(detail or {})})
-                {
-                    "id": fx["id"],
-                    "outcome": "WRONG",
-                    "relations_match": reader_rel == gold_rel,
-                    "units_match": reader_units == gold_units,
-                    "target_match": reader_unk == gold_unk,
-                    "reader_relations": reader_rel,
-                    "gold_relations": gold_rel,
-                    "reader_target": reader_unk,
-                    "gold_target": gold_unk,
-                }
-            )
 
     return {
         "lane": "setup_oracle_r1",
@@ -171,4 +180,60 @@ def run_r1() -> dict[str, Any]:
     }
 
 
-__all__ = ["run", "run_r1"]
+def run_r1_answers() -> dict[str, Any]:
+    """Off-serving answer lane for setup-correct R1 fixtures (PR-6b).
+
+    This does NOT expand the reader or serving path. It only asks the independent
+    relational oracle to evaluate fixtures whose setup already matches the independent R1
+    gold. Unsupported fixtures remain refused; setup-wrong fixtures are not answer-scored.
+    """
+    fixtures = _load_r1_gold()
+    correct = wrong = refused = setup_wrong = gold_error = 0
+    details: list[dict[str, Any]] = []
+
+    for fx in fixtures:
+        outcome, reason, detail = _score_setup_fixture(fx)
+        if outcome == "WRONG":
+            setup_wrong += 1
+            details.append({"id": fx["id"], "outcome": "setup_WRONG", **(detail or {})})
+            continue
+        if outcome == "refused":
+            refused += 1
+            details.append({"id": fx["id"], "outcome": "refused", "reason": reason})
+            continue
+
+        try:
+            got = oracle_answer(fx["relations"], fx["query"])
+        except OracleError as exc:
+            gold_error += 1
+            details.append({"id": fx["id"], "outcome": "gold_error", "reason": str(exc)})
+            continue
+
+        expected = fx.get("gold")
+        if got == expected:
+            correct += 1
+            details.append({"id": fx["id"], "outcome": "correct", "answer": got})
+        else:
+            wrong += 1
+            details.append({"id": fx["id"], "outcome": "WRONG", "answer": got, "gold": expected})
+
+    return {
+        "lane": "setup_oracle_r1_answers",
+        "total": len(fixtures),
+        "correct": correct,
+        "wrong": wrong,
+        "refused": refused,
+        "setup_wrong": setup_wrong,
+        "gold_error": gold_error,
+        "details": details,
+        "counts": {
+            "correct": correct,
+            "wrong": wrong,
+            "refused": refused,
+            "setup_wrong": setup_wrong,
+            "gold_error": gold_error,
+        },
+    }
+
+
+__all__ = ["run", "run_r1", "run_r1_answers"]