feat(rate): R3 RateProblem model + rate gold + setup oracle (R3b)

generate/rate_comprehension/model.py: RateProblem (quantity = rate × time, exactly one unknown == query; rate_unit fixes all three units by construction; structural guard refuses zero/two unknowns). evals/rate_oracle/: span-free signature, a 12-fixture gold (6 solved / 2 solver_refuses / 2... reader_refuses), closed expect+reason taxonomy, gold-validation runner + CLI. No reader yet — proves the ruler (12/12 valid).

Gold shape gives the 6/0/6 target: 6 single-rate solved (forward + both inverses, non-temporal 'per box' included), 2 non-integer-inverse solver refusals, 4 reader refusals (unit mismatch minutes-vs-hour, missing time, combined rates, clock-time temporal). Off-serving. 9 oracle + 10 unit tests; per-branch meaningful-fail.

evals/rate_oracle/__init__.py

@@ -0,0 +1,7 @@
+"""R3 rate setup oracle (off-serving) — the independent ruler for the single-rate organ.
+
+Grades a comprehended ``RateProblem`` against independent gold by a span-free canonical signature
+(``signature``), backed by a reviewable gold corpus (``rate_gold.jsonl``) and a validation runner
+(``runner``). The R3 twin of ``evals.constraint_oracle``. Imports no ``generate.derivation`` /
+``core.reliability_gate`` — disjoint from the GSM8K serving path.
+"""

evals/rate_oracle/__main__.py

@@ -0,0 +1,20 @@
+"""CLI: validate the R3 rate gold.
+
+    python -m evals.rate_oracle   # validate rate_gold.jsonl; exit 0 iff invalid == 0
+"""
+
+from __future__ import annotations
+
+import json
+
+from evals.rate_oracle.runner import run
+
+
+def main() -> int:
+    report = run()
+    print(json.dumps(report, indent=2, default=str))
+    return 0 if report["invalid"] == 0 else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

evals/rate_oracle/rate_gold.jsonl

@@ -0,0 +1,12 @@
+{"id": "r3-01-distance", "expect": "solved", "text": "A car travels 60 miles per hour for 3 hours. How many miles does it travel?", "rate_unit": {"numerator": "mile", "denominator": "hour"}, "rate": 60, "time": 3, "quantity": null, "query": "quantity", "gold": 180, "options": {"A": 120, "B": 150, "C": 180, "D": 200}, "answer": "C", "notes": "Forward: quantity = rate × time = 60 mile/hour × 3 hour = 180 mile. Unit composition exact."}
+{"id": "r3-02-earnings", "expect": "solved", "text": "A worker earns 15 dollars per hour for 8 hours. How many dollars does she earn?", "rate_unit": {"numerator": "dollar", "denominator": "hour"}, "rate": 15, "time": 8, "quantity": null, "query": "quantity", "gold": 120, "options": {"A": 120, "B": 23, "C": 105, "D": 135}, "answer": "A", "notes": "Forward: 15 dollar/hour × 8 hour = 120 dollar."}
+{"id": "r3-03-widgets", "expect": "solved", "text": "A machine makes 12 widgets per minute. It runs for 5 minutes. How many widgets does it make?", "rate_unit": {"numerator": "widget", "denominator": "minute"}, "rate": 12, "time": 5, "quantity": null, "query": "quantity", "gold": 60, "options": {"A": 17, "B": 60, "C": 7, "D": 72}, "answer": "B", "notes": "Forward: 12 widget/minute × 5 minute = 60 widget."}
+{"id": "r3-04-items", "expect": "solved", "text": "A packer fills 8 items per box for 6 boxes. How many items does the packer pack?", "rate_unit": {"numerator": "item", "denominator": "box"}, "rate": 8, "time": 6, "quantity": null, "query": "quantity", "gold": 48, "options": {"A": 14, "B": 48, "C": 42, "D": 54}, "answer": "B", "notes": "Forward with a non-temporal denominator (per box): 8 item/box × 6 box = 48 item. The 'time' slot is the rate's denominator, not necessarily a clock unit."}
+{"id": "r3-05-speed", "expect": "solved", "text": "A car travels 180 miles in 3 hours. What is its speed in miles per hour?", "rate_unit": {"numerator": "mile", "denominator": "hour"}, "rate": null, "time": 3, "quantity": 180, "query": "rate", "gold": 60, "options": {"A": 540, "B": 60, "C": 90, "D": 30}, "answer": "B", "notes": "Inverse (find rate): rate = quantity ÷ time = 180 mile ÷ 3 hour = 60 mile/hour. Exact division."}
+{"id": "r3-06-runtime", "expect": "solved", "text": "A machine makes 60 widgets at 12 widgets per minute. How many minutes did it run?", "rate_unit": {"numerator": "widget", "denominator": "minute"}, "rate": 12, "time": null, "quantity": 60, "query": "time", "gold": 5, "options": {"A": 5, "B": 720, "C": 48, "D": 6}, "answer": "A", "notes": "Inverse (find time): time = quantity ÷ rate = 60 widget ÷ 12 widget/minute = 5 minute. Exact division."}
+{"id": "r3-07-non-integer-rate", "expect": "solver_refuses", "solver_reason": "non_integer_solution", "text": "A car travels 100 miles in 3 hours. What is its speed in miles per hour?", "rate_unit": {"numerator": "mile", "denominator": "hour"}, "rate": null, "time": 3, "quantity": 100, "query": "rate", "gold": null, "notes": "Inverse with a non-exact division: 100 ÷ 3 is not an integer. The solver REFUSES (non_integer_solution), never rounds to 33."}
+{"id": "r3-08-non-integer-time", "expect": "solver_refuses", "solver_reason": "non_integer_solution", "text": "A machine makes 50 widgets at 12 widgets per minute. How many minutes did it run?", "rate_unit": {"numerator": "widget", "denominator": "minute"}, "rate": 12, "time": null, "quantity": 50, "query": "time", "gold": null, "notes": "Inverse with a non-exact division: 50 ÷ 12 is not an integer. The solver REFUSES (non_integer_solution)."}
+{"id": "r3-09-unit-mismatch", "expect": "reader_refuses", "reader_reason": "rate_unit_mismatch", "text": "A car travels 60 miles per hour for 30 minutes. How many miles does it travel?", "gold": null, "notes": "The rate is per HOUR but the duration is in MINUTES: mile/hour × minute does not compose. v1 does NOT silently convert minutes to hours (that would introduce rational quantities + unit conversion). The reader must REFUSE (rate_unit_mismatch)."}
+{"id": "r3-10-missing-time", "expect": "reader_refuses", "reader_reason": "missing_time", "text": "A car travels 60 miles per hour. How many miles does it travel?", "gold": null, "notes": "Only the rate is given — no duration. With one known the equation is underdetermined; the reader must REFUSE (missing_time), never invent a time."}
+{"id": "r3-11-combined-rates", "expect": "reader_refuses", "reader_reason": "combined_rates", "text": "A car travels 60 miles per hour for 2 hours and then 40 miles per hour for 3 hours. How many miles does it travel?", "gold": null, "notes": "Two rates over two segments — combined/multi-rate is R3.2, not single-rate v1. The reader must REFUSE (combined_rates) rather than read one segment."}
+{"id": "r3-12-temporal-state", "expect": "reader_refuses", "reader_reason": "temporal_state", "text": "A car left at 2 pm and arrived at 5 pm traveling 60 miles per hour. How many miles did it travel?", "gold": null, "notes": "Elapsed clock-time (2 pm to 5 pm) is a temporal-state computation, not an explicit duration. v1 does NOT compute elapsed intervals; the reader must REFUSE (temporal_state)."}

evals/rate_oracle/runner.py

@@ -0,0 +1,132 @@
+"""R3 rate setup-oracle runner — the ruler before reader capability (R3b).
+
+Validates that the independent rate gold is internally coherent and deserializes into the typed
+``RateProblem`` IR: every fixture matches its closed ``expect`` taxonomy, well-formed setups
+construct (exactly one unknown == the query), and — for ``solved`` fixtures — the multiple-choice
+key agrees with the gold value. No reader yet (lands R3d); this proves the ruler. The R3 twin of
+``evals.constraint_oracle.runner``.
+
+Exit 0 iff ``invalid == 0``.
+
+``expect`` taxonomy (closed):
+  - ``solved``         — full single-rate setup; ``gold`` is the int answer; ``options[answer] == gold``.
+  - ``solver_refuses`` — full setup, but a non-exact inverse; ``solver_reason`` says why; no gold.
+  - ``reader_refuses`` — prose the reader must refuse (missing piece / unit mismatch / combined /
+                         temporal); ``reader_reason`` says why; no setup, no gold.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from evals.rate_oracle.signature import rate_setup_signature
+from generate.rate_comprehension.model import RateProblem
+from generate.rate_comprehension.units import RateUnit, UnitError
+
+_RATE_GOLD_PATH = Path(__file__).resolve().parent / "rate_gold.jsonl"
+
+EXPECTATIONS = frozenset({"solved", "solver_refuses", "reader_refuses"})
+SOLVER_REASONS = frozenset({"non_integer_solution"})
+#: Closed reader-refusal set the gold uses; extended (with a fixture) as the reader grows.
+READER_REASONS = frozenset(
+    {
+        "rate_unit_mismatch",
+        "missing_rate",
+        "missing_time",
+        "missing_quantity",
+        "two_unknowns",
+        "combined_rates",
+        "temporal_state",
+    }
+)
+
+
+def _load_rate_gold() -> list[dict[str, Any]]:
+    return [
+        json.loads(line)
+        for line in _RATE_GOLD_PATH.read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+
+
+def gold_to_problem(fx: dict[str, Any]) -> RateProblem:
+    """Deserialize a fixture's setup fields into the typed RateProblem IR."""
+    ru = fx["rate_unit"]
+    return RateProblem(
+        rate_unit=RateUnit(ru["numerator"], ru["denominator"]),
+        rate=fx.get("rate"),
+        time=fx.get("time"),
+        quantity=fx.get("quantity"),
+        query=fx["query"],
+    )
+
+
+def validate_fixture(fx: dict[str, Any]) -> tuple[str, str | None]:
+    """Validate one gold fixture's coherence. Returns ``(outcome, reason)``."""
+    expect = fx.get("expect")
+    if expect not in EXPECTATIONS:
+        return "invalid", f"unknown_expect:{expect!r}"
+
+    if expect == "reader_refuses":
+        if fx.get("reader_reason") not in READER_REASONS:
+            return "invalid", f"unknown_reader_reason:{fx.get('reader_reason')!r}"
+        if fx.get("gold") is not None:
+            return "invalid", "reader_refuses_has_gold"
+        return "valid", None
+
+    # solved | solver_refuses require a well-formed single-rate setup.
+    try:
+        problem = gold_to_problem(fx)
+    except (KeyError, TypeError, ValueError, UnitError) as exc:
+        return "invalid", f"malformed_setup:{exc}"
+    if rate_setup_signature(problem) != rate_setup_signature(problem):
+        return "invalid", "nondeterministic_signature"  # pragma: no cover - determinism guard
+
+    if expect == "solver_refuses":
+        if fx.get("solver_reason") not in SOLVER_REASONS:
+            return "invalid", f"unknown_solver_reason:{fx.get('solver_reason')!r}"
+        if fx.get("gold") is not None:
+            return "invalid", "solver_refuses_has_gold"
+        return "valid", None
+
+    # solved: integer gold + coherent multiple-choice key.
+    gold = fx.get("gold")
+    if not isinstance(gold, int) or isinstance(gold, bool):
+        return "invalid", "solved_needs_int_gold"
+    options, answer = fx.get("options"), fx.get("answer")
+    if not isinstance(options, dict) or answer not in options:
+        return "invalid", "missing_or_unlabeled_answer"
+    if options[answer] != gold:
+        return "invalid", "answer_key_incoherent"
+    return "valid", None
+
+
+def run() -> dict[str, Any]:
+    """Validate every R3 rate gold fixture. Exit-0 criterion: ``invalid == 0``."""
+    fixtures = _load_rate_gold()
+    valid = invalid = 0
+    by_expect: dict[str, int] = {}
+    details: list[dict[str, Any]] = []
+    for fx in fixtures:
+        outcome, reason = validate_fixture(fx)
+        expect = fx.get("expect", "?")
+        by_expect[expect] = by_expect.get(expect, 0) + 1
+        if outcome == "valid":
+            valid += 1
+            details.append({"id": fx.get("id"), "outcome": "valid", "expect": expect})
+        else:
+            invalid += 1
+            details.append({"id": fx.get("id"), "outcome": "invalid", "reason": reason})
+    return {
+        "lane": "rate_oracle_gold_validation",
+        "total": len(fixtures),
+        "valid": valid,
+        "invalid": invalid,
+        "by_expect": by_expect,
+        "details": details,
+    }
+
+
+__all__ = ["EXPECTATIONS", "READER_REASONS", "SOLVER_REASONS", "gold_to_problem", "run", "validate_fixture"]

evals/rate_oracle/signature.py

@@ -0,0 +1,36 @@
+"""Span-free canonical signature for the R3 rate setup oracle (R3b).
+
+A deterministic, order-independent projection of a single-rate setup — the rate unit, the known
+values keyed by role, and the query — used to compare a reader's comprehended ``RateProblem``
+against the independent gold. The R3 twin of ``evals.setup_oracle.signature`` /
+``evals.constraint_oracle.signature``. Pure, deterministic.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from generate.rate_comprehension.model import RateProblem
+
+
+def rate_setup_signature(problem: RateProblem) -> dict[str, Any]:
+    """Canonical ``(rate_unit, knowns, query)`` signature of a single-rate setup."""
+    knowns = tuple(
+        sorted(
+            (role, value)
+            for role, value in (
+                ("rate", problem.rate),
+                ("time", problem.time),
+                ("quantity", problem.quantity),
+            )
+            if value is not None
+        )
+    )
+    return {
+        "rate_unit": (problem.rate_unit.numerator, problem.rate_unit.denominator),
+        "knowns": knowns,
+        "query": problem.query,
+    }
+
+
+__all__ = ["rate_setup_signature"]

generate/rate_comprehension/__init__.py

@@ -12,6 +12,7 @@ conversion, no multi-equation systems. Those are later R3 slices.
 
 from __future__ import annotations
 
+from generate.rate_comprehension.model import RateProblem, RateRole
 from generate.rate_comprehension.units import (
     BaseUnit,
     RateUnit,
@@ -23,6 +24,8 @@ from generate.rate_comprehension.units import (
 
 __all__ = [
     "BaseUnit",
+    "RateProblem",
+    "RateRole",
     "RateUnit",
     "UnitError",
     "rate_from_quantity_over_time",

generate/rate_comprehension/model.py

@@ -0,0 +1,53 @@
+"""Typed model for an R3 single-rate problem (R3b).
+
+A single-rate problem is the fixed equation ``quantity = rate × time`` with **exactly one**
+unknown (the query). ``rate_unit`` (e.g. ``mile/hour``) is the single source of unit truth: the
+quantity is measured in ``rate_unit.numerator`` and the time in ``rate_unit.denominator``, so the
+three units are consistent by construction. The two non-query slots carry integer values; the
+query slot is ``None``.
+
+Pure data with a structural guard: exactly the query slot is unknown (illegal states — zero or two
+unknowns — cannot be represented). Off-serving; deterministic.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+from generate.rate_comprehension.units import RateUnit
+
+RateRole = Literal["rate", "time", "quantity"]
+
+
+@dataclass(frozen=True, slots=True)
+class RateProblem:
+    """``quantity = rate × time`` with one unknown. ``rate_unit`` fixes all three units."""
+
+    rate_unit: RateUnit
+    rate: int | None
+    time: int | None
+    quantity: int | None
+    query: RateRole
+
+    def __post_init__(self) -> None:
+        slots: dict[str, int | None] = {"rate": self.rate, "time": self.time, "quantity": self.quantity}
+        unknown = [role for role, value in slots.items() if value is None]
+        if unknown != [self.query]:
+            raise ValueError(
+                f"exactly the query slot must be the unknown; query={self.query!r}, unknown={unknown}"
+            )
+        for role, value in slots.items():
+            if value is not None and (not isinstance(value, int) or isinstance(value, bool)):
+                raise ValueError(f"{role} value must be int; got {value!r}")
+
+    @property
+    def quantity_unit(self) -> str:
+        return self.rate_unit.numerator
+
+    @property
+    def time_unit(self) -> str:
+        return self.rate_unit.denominator
+
+
+__all__ = ["RateProblem", "RateRole"]

tests/test_rate_oracle.py

@@ -0,0 +1,94 @@
+"""Tests for the R3 RateProblem model + rate setup oracle (R3b).
+
+Pins the model's exactly-one-unknown guard and that the gold ruler is coherent + non-vacuous
+(each validator branch fires under its violation).
+"""
+
+from __future__ import annotations
+
+import copy
+
+import pytest
+
+from evals.rate_oracle.runner import (
+    READER_REASONS,
+    SOLVER_REASONS,
+    _load_rate_gold,
+    gold_to_problem,
+    run,
+    validate_fixture,
+)
+from evals.rate_oracle.signature import rate_setup_signature
+from generate.rate_comprehension.model import RateProblem
+from generate.rate_comprehension.units import RateUnit
+
+
+def _solved() -> dict:
+    return copy.deepcopy(next(f for f in _load_rate_gold() if f["expect"] == "solved"))
+
+
+def test_run_validates_all_gold() -> None:
+    r = run()
+    assert r["invalid"] == 0 and r["valid"] == r["total"] == 12
+    assert r["by_expect"] == {"solved": 6, "solver_refuses": 2, "reader_refuses": 4}
+
+
+def test_gold_to_problem_roundtrips() -> None:
+    p = gold_to_problem(next(f for f in _load_rate_gold() if f["id"] == "r3-01-distance"))
+    assert p.rate == 60 and p.time == 3 and p.quantity is None and p.query == "quantity"
+    assert (p.quantity_unit, p.time_unit) == ("mile", "hour")
+
+
+def test_model_requires_exactly_one_unknown() -> None:
+    with pytest.raises(ValueError):  # two unknowns
+        RateProblem(RateUnit("mile", "hour"), None, None, 180, "rate")
+    with pytest.raises(ValueError):  # zero unknowns
+        RateProblem(RateUnit("mile", "hour"), 60, 3, 180, "quantity")
+    with pytest.raises(ValueError):  # the named query is not the unknown
+        RateProblem(RateUnit("mile", "hour"), 60, None, 180, "rate")
+
+
+def test_signature_is_deterministic_and_keyed_by_role() -> None:
+    p = gold_to_problem(next(f for f in _load_rate_gold() if f["id"] == "r3-05-speed"))
+    sig = rate_setup_signature(p)
+    assert sig == rate_setup_signature(p)
+    assert sig["query"] == "rate" and sig["rate_unit"] == ("mile", "hour")
+    assert dict(sig["knowns"]) == {"time": 3, "quantity": 180}
+
+
+def test_reasons_in_gold_are_closed() -> None:
+    for fx in _load_rate_gold():
+        if fx["expect"] == "solver_refuses":
+            assert fx["solver_reason"] in SOLVER_REASONS
+        elif fx["expect"] == "reader_refuses":
+            assert fx["reader_reason"] in READER_REASONS
+
+
+# --- meaningful-fail: each invalid branch fires under exactly its violation ------------ #
+
+
+def test_validator_rejects_incoherent_answer_key() -> None:
+    fx = _solved()
+    fx["answer"] = next(k for k in fx["options"] if fx["options"][k] != fx["gold"])
+    assert validate_fixture(fx) == ("invalid", "answer_key_incoherent")
+
+
+def test_validator_rejects_reader_refuse_carrying_gold() -> None:
+    fx = copy.deepcopy(next(f for f in _load_rate_gold() if f["expect"] == "reader_refuses"))
+    fx["gold"] = 5
+    assert validate_fixture(fx) == ("invalid", "reader_refuses_has_gold")
+
+
+def test_validator_rejects_unknown_reasons() -> None:
+    rr = copy.deepcopy(next(f for f in _load_rate_gold() if f["expect"] == "reader_refuses"))
+    rr["reader_reason"] = "made_up"
+    assert validate_fixture(rr)[0] == "invalid"
+    sr = copy.deepcopy(next(f for f in _load_rate_gold() if f["expect"] == "solver_refuses"))
+    sr["solver_reason"] = "made_up"
+    assert validate_fixture(sr)[0] == "invalid"
+
+
+def test_validator_rejects_two_unknown_setup() -> None:
+    fx = _solved()
+    fx["time"] = None  # now both time and quantity unknown -> model __post_init__ raises
+    assert validate_fixture(fx)[0] == "invalid"