feat(rate): single-rate reader + run_reader lane + answer-choice reuse (R3d)

generate/rate_comprehension/reader.py: read_rate_problem comprehends explicit single-rate prose into a RateProblem via structural recognizers — rate value (<N> <plural> per <singular>), duration (for/in <N> <unit>), standalone quantity (outside those spans), query (how many <unit> by unit-match, or speed in <X> per <Y> -> rate). The compound-unit-consistency check is the wrong=0 gate: a duration whose unit isn't the rate denominator (miles/hour for 30 minutes) REFUSES rate_unit_mismatch, never converts.

Refuses combined_rates (>=2 rate clauses), temporal_state (clock markers), missing_rate/time/quantity (underdetermined). run_reader lane + 'reader' CLI. R3 reader: setup_correct 8 / setup_wrong 0 / refused_correct 4 -> answers 6 correct / 0 wrong / 6 refused (the 6/0/6 target). Reuses R2 verify_answer_choice end-to-end. Off-serving. 6 reader tests + constructed unit-mismatch guard.
This commit is contained in:
Shay 2026-06-07 23:07:51 -07:00
parent 6560300101
commit 6b1fdbff45
5 changed files with 264 additions and 6 deletions

View file

@ -1,19 +1,28 @@
"""CLI: validate the R3 rate gold.
"""CLI: validate the R3 rate gold, or grade the R3 reader against it.
python -m evals.rate_oracle # validate rate_gold.jsonl; exit 0 iff invalid == 0
python -m evals.rate_oracle reader # grade the reader; exit 0 iff setup_wrong == 0
# and reason_mismatch == 0
"""
from __future__ import annotations
import json
import sys
from evals.rate_oracle.runner import run
from evals.rate_oracle.runner import run, run_reader
def main() -> int:
lane = sys.argv[1] if len(sys.argv) > 1 else ""
if lane == "reader":
report = run_reader()
ok = report["setup_wrong"] == 0 and report["reason_mismatch"] == 0
else:
report = run()
ok = report["invalid"] == 0
print(json.dumps(report, indent=2, default=str))
return 0 if report["invalid"] == 0 else 1
return 0 if ok else 1
if __name__ == "__main__":

View file

@ -129,4 +129,62 @@ def run() -> dict[str, Any]:
}
__all__ = ["EXPECTATIONS", "READER_REASONS", "SOLVER_REASONS", "gold_to_problem", "run", "validate_fixture"]
def run_reader() -> dict[str, Any]:
"""Grade the R3 rate reader against the gold (R3d).
Well-formed fixtures (``solved`` / ``solver_refuses``) must read to a setup whose signature
equals the gold's (``setup_correct``); a refusal is a miss; a mismatch is ``setup_wrong``.
``reader_refuses`` fixtures must refuse with the gold's ``reader_reason`` (``refused_correct``);
a refusal with the wrong reason is ``reason_mismatch``; producing a setup is ``setup_wrong``.
Exit-0 criterion: ``setup_wrong == 0 and reason_mismatch == 0``.
"""
from generate.meaning_graph.reader import Refusal
from generate.rate_comprehension.reader import read_rate_problem
fixtures = _load_rate_gold()
setup_correct = setup_wrong = setup_refused = refused_correct = reason_mismatch = 0
details: list[dict[str, Any]] = []
for fx in fixtures:
out = read_rate_problem(fx["text"])
fid = fx.get("id")
if fx["expect"] in ("solved", "solver_refuses"):
if isinstance(out, Refusal):
setup_refused += 1
details.append({"id": fid, "outcome": "setup_refused", "reason": out.reason})
elif rate_setup_signature(out) == rate_setup_signature(gold_to_problem(fx)):
setup_correct += 1
details.append({"id": fid, "outcome": "setup_correct"})
else:
setup_wrong += 1
details.append({"id": fid, "outcome": "setup_WRONG"})
else: # reader_refuses
if isinstance(out, Refusal) and out.reason == fx["reader_reason"]:
refused_correct += 1
details.append({"id": fid, "outcome": "refused_correct", "reason": out.reason})
elif isinstance(out, Refusal):
reason_mismatch += 1
details.append({"id": fid, "outcome": "reason_mismatch", "got": out.reason, "want": fx["reader_reason"]})
else:
setup_wrong += 1
details.append({"id": fid, "outcome": "setup_WRONG_over_read"})
return {
"lane": "rate_oracle_reader",
"total": len(fixtures),
"setup_correct": setup_correct,
"setup_wrong": setup_wrong,
"setup_refused": setup_refused,
"refused_correct": refused_correct,
"reason_mismatch": reason_mismatch,
"details": details,
}
__all__ = [
"EXPECTATIONS",
"READER_REASONS",
"SOLVER_REASONS",
"gold_to_problem",
"run",
"run_reader",
"validate_fixture",
]

View file

@ -13,6 +13,7 @@ conversion, no multi-equation systems. Those are later R3 slices.
from __future__ import annotations
from generate.rate_comprehension.model import RateProblem, RateRole
from generate.rate_comprehension.reader import read_rate_problem
from generate.rate_comprehension.solver import answer_unit, solve_rate
from generate.rate_comprehension.units import (
BaseUnit,
@ -32,6 +33,7 @@ __all__ = [
"answer_unit",
"rate_from_quantity_over_time",
"rate_times_time",
"read_rate_problem",
"solve_rate",
"time_from_quantity_over_rate",
]

View file

@ -0,0 +1,124 @@
"""Single-rate prose reader (R3d): explicit rate prose -> RateProblem.
Reads ONLY explicit single-rate problems and refuses everything else combined/multi-rate,
clock-time/temporal, unit-mismatched durations, and underdetermined (missing-piece) prose. The
compound-unit consistency check (R3a) is the wrong=0 gate: a duration whose unit is not the rate's
denominator (``60 miles per hour for 30 minutes``) REFUSES rather than silently converting.
Recognized clauses (structural, not fixed strings):
- rate value : ``<N> <plural> per <singular>`` -> rate value + RateUnit(num, denom)
- duration/time : ``(for|in) <N> <unit>`` -> time value + unit
- quantity : a standalone ``<N> <unit>`` (outside the rate/duration spans)
- query : ``how many <unit>`` (role by unit match) | ``speed in <X> per <Y>`` (-> rate)
Refusals: ``combined_rates`` (2 rate clauses), ``temporal_state`` (clock markers),
``rate_unit_mismatch`` (duration/quantity unit rate unit), ``missing_rate`` / ``missing_time`` /
``missing_quantity`` (underdetermined). Off-serving; deterministic.
"""
from __future__ import annotations
import re
from generate.meaning_graph.reader import Refusal
from generate.rate_comprehension.model import RateProblem
from generate.rate_comprehension.units import RateUnit
_RATE_VALUE = re.compile(r"\b(\d+)\s+([a-z]+)\s+per\s+([a-z]+)\b")
_RATE_QUERY = re.compile(r"\bspeed in ([a-z]+) per ([a-z]+)\b")
_DURATION = re.compile(r"\b(?:for|in)\s+(\d+)\s+([a-z]+)\b")
_HOW_MANY = re.compile(r"\bhow many ([a-z]+)\b")
_DIGIT_NOUN = re.compile(r"\b(\d+)\s+([a-z]+)\b")
_TEMPORAL = re.compile(r"\b\d+\s*(?:am|pm)\b|\barrived\b|\bleft at\b|\bo'?clock\b")
def _singular(noun: str) -> str:
if noun.endswith("es") and noun[:-2].endswith(("x", "s", "z", "ch", "sh")):
return noun[:-2]
if noun.endswith("s") and len(noun) > 1:
return noun[:-1]
return noun
def read_rate_problem(text: str) -> RateProblem | Refusal:
"""Comprehend explicit single-rate prose into a typed RateProblem, or refuse."""
if not text or not text.strip():
return Refusal("empty")
t = text.lower()
if _TEMPORAL.search(t):
return Refusal("temporal_state", "elapsed clock time is not an explicit duration")
rate_clauses = _RATE_VALUE.findall(t)
if len(rate_clauses) >= 2:
return Refusal("combined_rates", "more than one rate is multi-rate (R3.2)")
# rate value clause (≤1)
rate_value: int | None = None
num_unit = denom_unit = None
rate_match = _RATE_VALUE.search(t)
if rate_match:
rate_value = int(rate_match.group(1))
num_unit, denom_unit = _singular(rate_match.group(2)), _singular(rate_match.group(3))
# duration / time clause
dur = _DURATION.search(t)
time_value = int(dur.group(1)) if dur else None
time_unit = _singular(dur.group(2)) if dur else None
# standalone quantity: a digit-noun outside the rate-clause and duration spans
spans = [m.span() for m in (rate_match, dur) if m is not None]
quantity_value = quantity_unit = None
for m in _DIGIT_NOUN.finditer(t):
if any(s <= m.start() < e for s, e in spans):
continue
quantity_value, quantity_unit = int(m.group(1)), _singular(m.group(2))
break
# query role + rate unit
rate_query = _RATE_QUERY.search(t)
how_many = _HOW_MANY.search(t)
if rate_query:
query = "rate"
rate_unit = RateUnit(_singular(rate_query.group(1)), _singular(rate_query.group(2)))
elif how_many:
asked = _singular(how_many.group(1))
if rate_value is None:
return Refusal("missing_rate", "no rate given and rate is not the question")
rate_unit = RateUnit(num_unit, denom_unit)
if asked == num_unit:
query = "quantity"
elif asked == denom_unit:
query = "time"
else:
return Refusal("query_target_unrecognized", asked)
else:
return Refusal("no_query")
# compound-unit consistency (the wrong=0 gate)
if time_value is not None and time_unit != rate_unit.denominator:
return Refusal("rate_unit_mismatch", f"duration {time_unit!r} ≠ rate denominator {rate_unit.denominator!r}")
if quantity_value is not None and quantity_unit != rate_unit.numerator:
return Refusal("rate_unit_mismatch", f"quantity {quantity_unit!r} ≠ rate numerator {rate_unit.numerator!r}")
# assemble by query (refusing an underdetermined setup)
if query == "quantity":
if rate_value is None:
return Refusal("missing_rate")
if time_value is None:
return Refusal("missing_time")
return RateProblem(rate_unit, rate_value, time_value, None, "quantity")
if query == "rate":
if quantity_value is None:
return Refusal("missing_quantity")
if time_value is None:
return Refusal("missing_time")
return RateProblem(rate_unit, None, time_value, quantity_value, "rate")
# query == "time"
if rate_value is None:
return Refusal("missing_rate")
if quantity_value is None:
return Refusal("missing_quantity")
return RateProblem(rate_unit, rate_value, None, quantity_value, "time")
__all__ = ["read_rate_problem"]

65
tests/test_rate_reader.py Normal file
View file

@ -0,0 +1,65 @@
"""Tests for the R3 single-rate reader (R3d).
Pins wrong=0: every well-formed fixture reads to exactly the gold setup; every reader_refuses
fixture refuses with its gold reason; and end to end every solved fixture reads solves
ties to its labeled answer (reusing R2's answer-choice verifier). Plus the unit-mismatch gate.
"""
from __future__ import annotations
from evals.rate_oracle.runner import _load_rate_gold, gold_to_problem, run_reader
from evals.rate_oracle.signature import rate_setup_signature
from generate.answer_choices.verify import ChoiceVerdict, verify_answer_choice
from generate.meaning_graph.reader import Refusal
from generate.rate_comprehension.reader import read_rate_problem
from generate.rate_comprehension.solver import solve_rate
def _by(expect: str) -> list[dict]:
return [f for f in _load_rate_gold() if f["expect"] == expect]
def test_reader_lane_is_wrong_zero_and_complete() -> None:
r = run_reader()
assert r["setup_wrong"] == 0 and r["reason_mismatch"] == 0
assert r["setup_refused"] == 0
assert r["setup_correct"] == 8 # 6 solved + 2 solver_refuses (both have valid setups)
assert r["refused_correct"] == 4
def test_reads_every_well_formed_fixture_to_gold_signature() -> None:
for fx in _by("solved") + _by("solver_refuses"):
out = read_rate_problem(fx["text"])
assert not isinstance(out, Refusal), f"{fx['id']}: {getattr(out, 'reason', '')}"
assert rate_setup_signature(out) == rate_setup_signature(gold_to_problem(fx)), fx["id"]
def test_refuses_every_reader_refuse_fixture_with_reason() -> None:
for fx in _by("reader_refuses"):
out = read_rate_problem(fx["text"])
assert isinstance(out, Refusal) and out.reason == fx["reader_reason"], fx["id"]
def test_read_solve_verify_end_to_end_for_solved() -> None:
for fx in _by("solved"):
problem = read_rate_problem(fx["text"])
assert not isinstance(problem, Refusal), fx["id"]
value = solve_rate(problem)
assert value == fx["gold"], fx["id"]
verdict = verify_answer_choice(value, fx["options"], fx["answer"])
assert isinstance(verdict, ChoiceVerdict) and verdict.status == "consistent"
assert verdict.computed_label == fx["answer"]
def test_read_then_solver_refuses_for_solver_refuse_fixtures() -> None:
for fx in _by("solver_refuses"):
problem = read_rate_problem(fx["text"])
assert not isinstance(problem, Refusal), fx["id"]
out = solve_rate(problem)
assert isinstance(out, Refusal) and out.reason == fx["solver_reason"], fx["id"]
def test_constructed_unit_mismatch_refuses() -> None:
# rate per hour, duration in days — does not compose -> refuse (never converts).
out = read_rate_problem("A car travels 70 miles per hour for 2 days. How many miles does it travel?")
assert isinstance(out, Refusal) and out.reason == "rate_unit_mismatch"