diff --git a/evals/setup_oracle/runner.py b/evals/setup_oracle/runner.py index c63edc4b..53d83eeb 100644 --- a/evals/setup_oracle/runner.py +++ b/evals/setup_oracle/runner.py @@ -16,6 +16,7 @@ import json from pathlib import Path from typing import Any +from evals.relational_metric.oracle import OracleError, oracle_answer from evals.relational_metric.runner import _load_cases from evals.setup_oracle.signature import ( gold_unknown_signature, @@ -103,6 +104,42 @@ def run() -> dict[str, Any]: } +def _score_setup_fixture(fx: dict[str, Any]) -> tuple[str, str | None, dict[str, Any] | None]: + """Score one independent setup fixture by structure, not answer. + + Returns ``(outcome, reason, detail)`` where outcome is ``correct`` / ``refused`` / + ``WRONG``. Kept small so PR-6b can reuse the same setup gate before evaluating + answers; answer scoring must never run on a structurally wrong reading. + """ + comp = comprehend_quantitative(fx["text"]) + if isinstance(comp, Refusal): + return "refused", comp.reason, None + projected = to_relational_metric(comp) + if projected is None: + return "refused", "unprojectable", None + reader_relations, _ = projected + units = fx["expected_units"] + + reader_rel = relation_signature(reader_relations) + gold_rel = relation_signature(fx["relations"]) + reader_units = reader_symbol_units(comp.binding_graph) + gold_units = symbol_unit_signature(units) + reader_unk = reader_unknown_signature(comp.binding_graph) + gold_unk = gold_unknown_signature(fx["relations"], fx["query"], units) + + if reader_rel == gold_rel and reader_units == gold_units and reader_unk == gold_unk: + return "correct", None, None + return "WRONG", None, { + "relations_match": reader_rel == gold_rel, + "units_match": reader_units == gold_units, + "target_match": reader_unk == gold_unk, + "reader_relations": reader_rel, + "gold_relations": gold_rel, + "reader_target": reader_unk, + "gold_target": gold_unk, + } + + def run_r1() -> dict[str, Any]: """Score the CURRENT reader against the independent, self-contained R1 gold. @@ -117,44 +154,16 @@ def run_r1() -> dict[str, Any]: details: list[dict[str, Any]] = [] for fx in fixtures: - comp = comprehend_quantitative(fx["text"]) - if isinstance(comp, Refusal): - setup_refused += 1 - details.append({"id": fx["id"], "outcome": "refused", "reason": comp.reason}) - continue - projected = to_relational_metric(comp) - if projected is None: - setup_refused += 1 - details.append({"id": fx["id"], "outcome": "refused", "reason": "unprojectable"}) - continue - reader_relations, _ = projected - units = fx["expected_units"] - - reader_rel = relation_signature(reader_relations) - gold_rel = relation_signature(fx["relations"]) - reader_units = reader_symbol_units(comp.binding_graph) - gold_units = symbol_unit_signature(units) - reader_unk = reader_unknown_signature(comp.binding_graph) - gold_unk = gold_unknown_signature(fx["relations"], fx["query"], units) - - if reader_rel == gold_rel and reader_units == gold_units and reader_unk == gold_unk: + outcome, reason, detail = _score_setup_fixture(fx) + if outcome == "correct": setup_correct += 1 details.append({"id": fx["id"], "outcome": "correct"}) + elif outcome == "refused": + setup_refused += 1 + details.append({"id": fx["id"], "outcome": "refused", "reason": reason}) else: setup_wrong += 1 - details.append( - { - "id": fx["id"], - "outcome": "WRONG", - "relations_match": reader_rel == gold_rel, - "units_match": reader_units == gold_units, - "target_match": reader_unk == gold_unk, - "reader_relations": reader_rel, - "gold_relations": gold_rel, - "reader_target": reader_unk, - "gold_target": gold_unk, - } - ) + details.append({"id": fx["id"], "outcome": "WRONG", **(detail or {})}) return { "lane": "setup_oracle_r1", @@ -171,4 +180,60 @@ def run_r1() -> dict[str, Any]: } -__all__ = ["run", "run_r1"] +def run_r1_answers() -> dict[str, Any]: + """Off-serving answer lane for setup-correct R1 fixtures (PR-6b). + + This does NOT expand the reader or serving path. It only asks the independent + relational oracle to evaluate fixtures whose setup already matches the independent R1 + gold. Unsupported fixtures remain refused; setup-wrong fixtures are not answer-scored. + """ + fixtures = _load_r1_gold() + correct = wrong = refused = setup_wrong = gold_error = 0 + details: list[dict[str, Any]] = [] + + for fx in fixtures: + outcome, reason, detail = _score_setup_fixture(fx) + if outcome == "WRONG": + setup_wrong += 1 + details.append({"id": fx["id"], "outcome": "setup_WRONG", **(detail or {})}) + continue + if outcome == "refused": + refused += 1 + details.append({"id": fx["id"], "outcome": "refused", "reason": reason}) + continue + + try: + got = oracle_answer(fx["relations"], fx["query"]) + except OracleError as exc: + gold_error += 1 + details.append({"id": fx["id"], "outcome": "gold_error", "reason": str(exc)}) + continue + + expected = fx.get("gold") + if got == expected: + correct += 1 + details.append({"id": fx["id"], "outcome": "correct", "answer": got}) + else: + wrong += 1 + details.append({"id": fx["id"], "outcome": "WRONG", "answer": got, "gold": expected}) + + return { + "lane": "setup_oracle_r1_answers", + "total": len(fixtures), + "correct": correct, + "wrong": wrong, + "refused": refused, + "setup_wrong": setup_wrong, + "gold_error": gold_error, + "details": details, + "counts": { + "correct": correct, + "wrong": wrong, + "refused": refused, + "setup_wrong": setup_wrong, + "gold_error": gold_error, + }, + } + + +__all__ = ["run", "run_r1", "run_r1_answers"]