feat(comprehension): add R1 answer-oracle lane
This commit is contained in:
Shay
parent
ce8897e84f
commit
c364d434da
1 changed files with 100 additions and 35 deletions
|
|
@ -16,6 +16,7 @@ import json
|
|||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from evals.relational_metric.oracle import OracleError, oracle_answer
|
||||
from evals.relational_metric.runner import _load_cases
|
||||
from evals.setup_oracle.signature import (
|
||||
gold_unknown_signature,
|
||||
|
|
@ -103,6 +104,42 @@ def run() -> dict[str, Any]:
|
|||
}
|
||||
|
||||
|
||||
def _score_setup_fixture(fx: dict[str, Any]) -> tuple[str, str | None, dict[str, Any] | None]:
|
||||
"""Score one independent setup fixture by structure, not answer.
|
||||
|
||||
Returns ``(outcome, reason, detail)`` where outcome is ``correct`` / ``refused`` /
|
||||
``WRONG``. Kept small so PR-6b can reuse the same setup gate before evaluating
|
||||
answers; answer scoring must never run on a structurally wrong reading.
|
||||
"""
|
||||
comp = comprehend_quantitative(fx["text"])
|
||||
if isinstance(comp, Refusal):
|
||||
return "refused", comp.reason, None
|
||||
projected = to_relational_metric(comp)
|
||||
if projected is None:
|
||||
return "refused", "unprojectable", None
|
||||
reader_relations, _ = projected
|
||||
units = fx["expected_units"]
|
||||
|
||||
reader_rel = relation_signature(reader_relations)
|
||||
gold_rel = relation_signature(fx["relations"])
|
||||
reader_units = reader_symbol_units(comp.binding_graph)
|
||||
gold_units = symbol_unit_signature(units)
|
||||
reader_unk = reader_unknown_signature(comp.binding_graph)
|
||||
gold_unk = gold_unknown_signature(fx["relations"], fx["query"], units)
|
||||
|
||||
if reader_rel == gold_rel and reader_units == gold_units and reader_unk == gold_unk:
|
||||
return "correct", None, None
|
||||
return "WRONG", None, {
|
||||
"relations_match": reader_rel == gold_rel,
|
||||
"units_match": reader_units == gold_units,
|
||||
"target_match": reader_unk == gold_unk,
|
||||
"reader_relations": reader_rel,
|
||||
"gold_relations": gold_rel,
|
||||
"reader_target": reader_unk,
|
||||
"gold_target": gold_unk,
|
||||
}
|
||||
|
||||
|
||||
def run_r1() -> dict[str, Any]:
|
||||
"""Score the CURRENT reader against the independent, self-contained R1 gold.
|
||||
|
||||
|
|
@ -117,44 +154,16 @@ def run_r1() -> dict[str, Any]:
|
|||
details: list[dict[str, Any]] = []
|
||||
|
||||
for fx in fixtures:
|
||||
comp = comprehend_quantitative(fx["text"])
|
||||
if isinstance(comp, Refusal):
|
||||
setup_refused += 1
|
||||
details.append({"id": fx["id"], "outcome": "refused", "reason": comp.reason})
|
||||
continue
|
||||
projected = to_relational_metric(comp)
|
||||
if projected is None:
|
||||
setup_refused += 1
|
||||
details.append({"id": fx["id"], "outcome": "refused", "reason": "unprojectable"})
|
||||
continue
|
||||
reader_relations, _ = projected
|
||||
units = fx["expected_units"]
|
||||
|
||||
reader_rel = relation_signature(reader_relations)
|
||||
gold_rel = relation_signature(fx["relations"])
|
||||
reader_units = reader_symbol_units(comp.binding_graph)
|
||||
gold_units = symbol_unit_signature(units)
|
||||
reader_unk = reader_unknown_signature(comp.binding_graph)
|
||||
gold_unk = gold_unknown_signature(fx["relations"], fx["query"], units)
|
||||
|
||||
if reader_rel == gold_rel and reader_units == gold_units and reader_unk == gold_unk:
|
||||
outcome, reason, detail = _score_setup_fixture(fx)
|
||||
if outcome == "correct":
|
||||
setup_correct += 1
|
||||
details.append({"id": fx["id"], "outcome": "correct"})
|
||||
elif outcome == "refused":
|
||||
setup_refused += 1
|
||||
details.append({"id": fx["id"], "outcome": "refused", "reason": reason})
|
||||
else:
|
||||
setup_wrong += 1
|
||||
details.append(
|
||||
{
|
||||
"id": fx["id"],
|
||||
"outcome": "WRONG",
|
||||
"relations_match": reader_rel == gold_rel,
|
||||
"units_match": reader_units == gold_units,
|
||||
"target_match": reader_unk == gold_unk,
|
||||
"reader_relations": reader_rel,
|
||||
"gold_relations": gold_rel,
|
||||
"reader_target": reader_unk,
|
||||
"gold_target": gold_unk,
|
||||
}
|
||||
)
|
||||
details.append({"id": fx["id"], "outcome": "WRONG", **(detail or {})})
|
||||
|
||||
return {
|
||||
"lane": "setup_oracle_r1",
|
||||
|
|
@ -171,4 +180,60 @@ def run_r1() -> dict[str, Any]:
|
|||
}
|
||||
|
||||
|
||||
__all__ = ["run", "run_r1"]
|
||||
def run_r1_answers() -> dict[str, Any]:
|
||||
"""Off-serving answer lane for setup-correct R1 fixtures (PR-6b).
|
||||
|
||||
This does NOT expand the reader or serving path. It only asks the independent
|
||||
relational oracle to evaluate fixtures whose setup already matches the independent R1
|
||||
gold. Unsupported fixtures remain refused; setup-wrong fixtures are not answer-scored.
|
||||
"""
|
||||
fixtures = _load_r1_gold()
|
||||
correct = wrong = refused = setup_wrong = gold_error = 0
|
||||
details: list[dict[str, Any]] = []
|
||||
|
||||
for fx in fixtures:
|
||||
outcome, reason, detail = _score_setup_fixture(fx)
|
||||
if outcome == "WRONG":
|
||||
setup_wrong += 1
|
||||
details.append({"id": fx["id"], "outcome": "setup_WRONG", **(detail or {})})
|
||||
continue
|
||||
if outcome == "refused":
|
||||
refused += 1
|
||||
details.append({"id": fx["id"], "outcome": "refused", "reason": reason})
|
||||
continue
|
||||
|
||||
try:
|
||||
got = oracle_answer(fx["relations"], fx["query"])
|
||||
except OracleError as exc:
|
||||
gold_error += 1
|
||||
details.append({"id": fx["id"], "outcome": "gold_error", "reason": str(exc)})
|
||||
continue
|
||||
|
||||
expected = fx.get("gold")
|
||||
if got == expected:
|
||||
correct += 1
|
||||
details.append({"id": fx["id"], "outcome": "correct", "answer": got})
|
||||
else:
|
||||
wrong += 1
|
||||
details.append({"id": fx["id"], "outcome": "WRONG", "answer": got, "gold": expected})
|
||||
|
||||
return {
|
||||
"lane": "setup_oracle_r1_answers",
|
||||
"total": len(fixtures),
|
||||
"correct": correct,
|
||||
"wrong": wrong,
|
||||
"refused": refused,
|
||||
"setup_wrong": setup_wrong,
|
||||
"gold_error": gold_error,
|
||||
"details": details,
|
||||
"counts": {
|
||||
"correct": correct,
|
||||
"wrong": wrong,
|
||||
"refused": refused,
|
||||
"setup_wrong": setup_wrong,
|
||||
"gold_error": gold_error,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
__all__ = ["run", "run_r1", "run_r1_answers"]
|
||||
|
|
|
|||
Loading…
Reference in a new issue