From 6b1fdbff45ed10d024beebf811008c7c17f56299 Mon Sep 17 00:00:00 2001 From: Shay Date: Sun, 7 Jun 2026 23:07:51 -0700 Subject: [PATCH] feat(rate): single-rate reader + run_reader lane + answer-choice reuse (R3d) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit generate/rate_comprehension/reader.py: read_rate_problem comprehends explicit single-rate prose into a RateProblem via structural recognizers — rate value ( per ), duration (for/in ), standalone quantity (outside those spans), query (how many by unit-match, or speed in per -> rate). The compound-unit-consistency check is the wrong=0 gate: a duration whose unit isn't the rate denominator (miles/hour for 30 minutes) REFUSES rate_unit_mismatch, never converts. Refuses combined_rates (>=2 rate clauses), temporal_state (clock markers), missing_rate/time/quantity (underdetermined). run_reader lane + 'reader' CLI. R3 reader: setup_correct 8 / setup_wrong 0 / refused_correct 4 -> answers 6 correct / 0 wrong / 6 refused (the 6/0/6 target). Reuses R2 verify_answer_choice end-to-end. Off-serving. 6 reader tests + constructed unit-mismatch guard. --- evals/rate_oracle/__main__.py | 19 +++- evals/rate_oracle/runner.py | 60 +++++++++++- generate/rate_comprehension/__init__.py | 2 + generate/rate_comprehension/reader.py | 124 ++++++++++++++++++++++++ tests/test_rate_reader.py | 65 +++++++++++++ 5 files changed, 264 insertions(+), 6 deletions(-) create mode 100644 generate/rate_comprehension/reader.py create mode 100644 tests/test_rate_reader.py diff --git a/evals/rate_oracle/__main__.py b/evals/rate_oracle/__main__.py index 697534d9..a59b7d13 100644 --- a/evals/rate_oracle/__main__.py +++ b/evals/rate_oracle/__main__.py @@ -1,19 +1,28 @@ -"""CLI: validate the R3 rate gold. +"""CLI: validate the R3 rate gold, or grade the R3 reader against it. - python -m evals.rate_oracle # validate rate_gold.jsonl; exit 0 iff invalid == 0 + python -m evals.rate_oracle # validate rate_gold.jsonl; exit 0 iff invalid == 0 + python -m evals.rate_oracle reader # grade the reader; exit 0 iff setup_wrong == 0 + # and reason_mismatch == 0 """ from __future__ import annotations import json +import sys -from evals.rate_oracle.runner import run +from evals.rate_oracle.runner import run, run_reader def main() -> int: - report = run() + lane = sys.argv[1] if len(sys.argv) > 1 else "" + if lane == "reader": + report = run_reader() + ok = report["setup_wrong"] == 0 and report["reason_mismatch"] == 0 + else: + report = run() + ok = report["invalid"] == 0 print(json.dumps(report, indent=2, default=str)) - return 0 if report["invalid"] == 0 else 1 + return 0 if ok else 1 if __name__ == "__main__": diff --git a/evals/rate_oracle/runner.py b/evals/rate_oracle/runner.py index b8054178..1499bffe 100644 --- a/evals/rate_oracle/runner.py +++ b/evals/rate_oracle/runner.py @@ -129,4 +129,62 @@ def run() -> dict[str, Any]: } -__all__ = ["EXPECTATIONS", "READER_REASONS", "SOLVER_REASONS", "gold_to_problem", "run", "validate_fixture"] +def run_reader() -> dict[str, Any]: + """Grade the R3 rate reader against the gold (R3d). + + Well-formed fixtures (``solved`` / ``solver_refuses``) must read to a setup whose signature + equals the gold's (``setup_correct``); a refusal is a miss; a mismatch is ``setup_wrong``. + ``reader_refuses`` fixtures must refuse with the gold's ``reader_reason`` (``refused_correct``); + a refusal with the wrong reason is ``reason_mismatch``; producing a setup is ``setup_wrong``. + Exit-0 criterion: ``setup_wrong == 0 and reason_mismatch == 0``. + """ + from generate.meaning_graph.reader import Refusal + from generate.rate_comprehension.reader import read_rate_problem + + fixtures = _load_rate_gold() + setup_correct = setup_wrong = setup_refused = refused_correct = reason_mismatch = 0 + details: list[dict[str, Any]] = [] + for fx in fixtures: + out = read_rate_problem(fx["text"]) + fid = fx.get("id") + if fx["expect"] in ("solved", "solver_refuses"): + if isinstance(out, Refusal): + setup_refused += 1 + details.append({"id": fid, "outcome": "setup_refused", "reason": out.reason}) + elif rate_setup_signature(out) == rate_setup_signature(gold_to_problem(fx)): + setup_correct += 1 + details.append({"id": fid, "outcome": "setup_correct"}) + else: + setup_wrong += 1 + details.append({"id": fid, "outcome": "setup_WRONG"}) + else: # reader_refuses + if isinstance(out, Refusal) and out.reason == fx["reader_reason"]: + refused_correct += 1 + details.append({"id": fid, "outcome": "refused_correct", "reason": out.reason}) + elif isinstance(out, Refusal): + reason_mismatch += 1 + details.append({"id": fid, "outcome": "reason_mismatch", "got": out.reason, "want": fx["reader_reason"]}) + else: + setup_wrong += 1 + details.append({"id": fid, "outcome": "setup_WRONG_over_read"}) + return { + "lane": "rate_oracle_reader", + "total": len(fixtures), + "setup_correct": setup_correct, + "setup_wrong": setup_wrong, + "setup_refused": setup_refused, + "refused_correct": refused_correct, + "reason_mismatch": reason_mismatch, + "details": details, + } + + +__all__ = [ + "EXPECTATIONS", + "READER_REASONS", + "SOLVER_REASONS", + "gold_to_problem", + "run", + "run_reader", + "validate_fixture", +] diff --git a/generate/rate_comprehension/__init__.py b/generate/rate_comprehension/__init__.py index 112aa471..c00e8d0f 100644 --- a/generate/rate_comprehension/__init__.py +++ b/generate/rate_comprehension/__init__.py @@ -13,6 +13,7 @@ conversion, no multi-equation systems. Those are later R3 slices. from __future__ import annotations from generate.rate_comprehension.model import RateProblem, RateRole +from generate.rate_comprehension.reader import read_rate_problem from generate.rate_comprehension.solver import answer_unit, solve_rate from generate.rate_comprehension.units import ( BaseUnit, @@ -32,6 +33,7 @@ __all__ = [ "answer_unit", "rate_from_quantity_over_time", "rate_times_time", + "read_rate_problem", "solve_rate", "time_from_quantity_over_rate", ] diff --git a/generate/rate_comprehension/reader.py b/generate/rate_comprehension/reader.py new file mode 100644 index 00000000..8cc056b7 --- /dev/null +++ b/generate/rate_comprehension/reader.py @@ -0,0 +1,124 @@ +"""Single-rate prose reader (R3d): explicit rate prose -> RateProblem. + +Reads ONLY explicit single-rate problems and refuses everything else — combined/multi-rate, +clock-time/temporal, unit-mismatched durations, and underdetermined (missing-piece) prose. The +compound-unit consistency check (R3a) is the wrong=0 gate: a duration whose unit is not the rate's +denominator (``60 miles per hour for 30 minutes``) REFUSES rather than silently converting. + +Recognized clauses (structural, not fixed strings): + - rate value : `` per `` -> rate value + RateUnit(num, denom) + - duration/time : ``(for|in) `` -> time value + unit + - quantity : a standalone `` `` (outside the rate/duration spans) + - query : ``how many `` (role by unit match) | ``speed in per `` (-> rate) + +Refusals: ``combined_rates`` (≥2 rate clauses), ``temporal_state`` (clock markers), +``rate_unit_mismatch`` (duration/quantity unit ≠ rate unit), ``missing_rate`` / ``missing_time`` / +``missing_quantity`` (underdetermined). Off-serving; deterministic. +""" + +from __future__ import annotations + +import re + +from generate.meaning_graph.reader import Refusal +from generate.rate_comprehension.model import RateProblem +from generate.rate_comprehension.units import RateUnit + +_RATE_VALUE = re.compile(r"\b(\d+)\s+([a-z]+)\s+per\s+([a-z]+)\b") +_RATE_QUERY = re.compile(r"\bspeed in ([a-z]+) per ([a-z]+)\b") +_DURATION = re.compile(r"\b(?:for|in)\s+(\d+)\s+([a-z]+)\b") +_HOW_MANY = re.compile(r"\bhow many ([a-z]+)\b") +_DIGIT_NOUN = re.compile(r"\b(\d+)\s+([a-z]+)\b") +_TEMPORAL = re.compile(r"\b\d+\s*(?:am|pm)\b|\barrived\b|\bleft at\b|\bo'?clock\b") + + +def _singular(noun: str) -> str: + if noun.endswith("es") and noun[:-2].endswith(("x", "s", "z", "ch", "sh")): + return noun[:-2] + if noun.endswith("s") and len(noun) > 1: + return noun[:-1] + return noun + + +def read_rate_problem(text: str) -> RateProblem | Refusal: + """Comprehend explicit single-rate prose into a typed RateProblem, or refuse.""" + if not text or not text.strip(): + return Refusal("empty") + t = text.lower() + + if _TEMPORAL.search(t): + return Refusal("temporal_state", "elapsed clock time is not an explicit duration") + rate_clauses = _RATE_VALUE.findall(t) + if len(rate_clauses) >= 2: + return Refusal("combined_rates", "more than one rate is multi-rate (R3.2)") + + # rate value clause (≤1) + rate_value: int | None = None + num_unit = denom_unit = None + rate_match = _RATE_VALUE.search(t) + if rate_match: + rate_value = int(rate_match.group(1)) + num_unit, denom_unit = _singular(rate_match.group(2)), _singular(rate_match.group(3)) + + # duration / time clause + dur = _DURATION.search(t) + time_value = int(dur.group(1)) if dur else None + time_unit = _singular(dur.group(2)) if dur else None + + # standalone quantity: a digit-noun outside the rate-clause and duration spans + spans = [m.span() for m in (rate_match, dur) if m is not None] + quantity_value = quantity_unit = None + for m in _DIGIT_NOUN.finditer(t): + if any(s <= m.start() < e for s, e in spans): + continue + quantity_value, quantity_unit = int(m.group(1)), _singular(m.group(2)) + break + + # query role + rate unit + rate_query = _RATE_QUERY.search(t) + how_many = _HOW_MANY.search(t) + if rate_query: + query = "rate" + rate_unit = RateUnit(_singular(rate_query.group(1)), _singular(rate_query.group(2))) + elif how_many: + asked = _singular(how_many.group(1)) + if rate_value is None: + return Refusal("missing_rate", "no rate given and rate is not the question") + rate_unit = RateUnit(num_unit, denom_unit) + if asked == num_unit: + query = "quantity" + elif asked == denom_unit: + query = "time" + else: + return Refusal("query_target_unrecognized", asked) + else: + return Refusal("no_query") + + # compound-unit consistency (the wrong=0 gate) + if time_value is not None and time_unit != rate_unit.denominator: + return Refusal("rate_unit_mismatch", f"duration {time_unit!r} ≠ rate denominator {rate_unit.denominator!r}") + if quantity_value is not None and quantity_unit != rate_unit.numerator: + return Refusal("rate_unit_mismatch", f"quantity {quantity_unit!r} ≠ rate numerator {rate_unit.numerator!r}") + + # assemble by query (refusing an underdetermined setup) + if query == "quantity": + if rate_value is None: + return Refusal("missing_rate") + if time_value is None: + return Refusal("missing_time") + return RateProblem(rate_unit, rate_value, time_value, None, "quantity") + if query == "rate": + if quantity_value is None: + return Refusal("missing_quantity") + if time_value is None: + return Refusal("missing_time") + return RateProblem(rate_unit, None, time_value, quantity_value, "rate") + # query == "time" + if rate_value is None: + return Refusal("missing_rate") + if quantity_value is None: + return Refusal("missing_quantity") + return RateProblem(rate_unit, rate_value, None, quantity_value, "time") + + +__all__ = ["read_rate_problem"] diff --git a/tests/test_rate_reader.py b/tests/test_rate_reader.py new file mode 100644 index 00000000..e2327780 --- /dev/null +++ b/tests/test_rate_reader.py @@ -0,0 +1,65 @@ +"""Tests for the R3 single-rate reader (R3d). + +Pins wrong=0: every well-formed fixture reads to exactly the gold setup; every reader_refuses +fixture refuses with its gold reason; and — end to end — every solved fixture reads → solves → +ties to its labeled answer (reusing R2's answer-choice verifier). Plus the unit-mismatch gate. +""" + +from __future__ import annotations + +from evals.rate_oracle.runner import _load_rate_gold, gold_to_problem, run_reader +from evals.rate_oracle.signature import rate_setup_signature +from generate.answer_choices.verify import ChoiceVerdict, verify_answer_choice +from generate.meaning_graph.reader import Refusal +from generate.rate_comprehension.reader import read_rate_problem +from generate.rate_comprehension.solver import solve_rate + + +def _by(expect: str) -> list[dict]: + return [f for f in _load_rate_gold() if f["expect"] == expect] + + +def test_reader_lane_is_wrong_zero_and_complete() -> None: + r = run_reader() + assert r["setup_wrong"] == 0 and r["reason_mismatch"] == 0 + assert r["setup_refused"] == 0 + assert r["setup_correct"] == 8 # 6 solved + 2 solver_refuses (both have valid setups) + assert r["refused_correct"] == 4 + + +def test_reads_every_well_formed_fixture_to_gold_signature() -> None: + for fx in _by("solved") + _by("solver_refuses"): + out = read_rate_problem(fx["text"]) + assert not isinstance(out, Refusal), f"{fx['id']}: {getattr(out, 'reason', '')}" + assert rate_setup_signature(out) == rate_setup_signature(gold_to_problem(fx)), fx["id"] + + +def test_refuses_every_reader_refuse_fixture_with_reason() -> None: + for fx in _by("reader_refuses"): + out = read_rate_problem(fx["text"]) + assert isinstance(out, Refusal) and out.reason == fx["reader_reason"], fx["id"] + + +def test_read_solve_verify_end_to_end_for_solved() -> None: + for fx in _by("solved"): + problem = read_rate_problem(fx["text"]) + assert not isinstance(problem, Refusal), fx["id"] + value = solve_rate(problem) + assert value == fx["gold"], fx["id"] + verdict = verify_answer_choice(value, fx["options"], fx["answer"]) + assert isinstance(verdict, ChoiceVerdict) and verdict.status == "consistent" + assert verdict.computed_label == fx["answer"] + + +def test_read_then_solver_refuses_for_solver_refuse_fixtures() -> None: + for fx in _by("solver_refuses"): + problem = read_rate_problem(fx["text"]) + assert not isinstance(problem, Refusal), fx["id"] + out = solve_rate(problem) + assert isinstance(out, Refusal) and out.reason == fx["solver_reason"], fx["id"] + + +def test_constructed_unit_mismatch_refuses() -> None: + # rate per hour, duration in days — does not compose -> refuse (never converts). + out = read_rate_problem("A car travels 70 miles per hour for 2 days. How many miles does it travel?") + assert isinstance(out, Refusal) and out.reason == "rate_unit_mismatch"