diff --git a/evals/rate_oracle/__init__.py b/evals/rate_oracle/__init__.py new file mode 100644 index 00000000..9eb78bd4 --- /dev/null +++ b/evals/rate_oracle/__init__.py @@ -0,0 +1,7 @@ +"""R3 rate setup oracle (off-serving) — the independent ruler for the single-rate organ. + +Grades a comprehended ``RateProblem`` against independent gold by a span-free canonical signature +(``signature``), backed by a reviewable gold corpus (``rate_gold.jsonl``) and a validation runner +(``runner``). The R3 twin of ``evals.constraint_oracle``. Imports no ``generate.derivation`` / +``core.reliability_gate`` — disjoint from the GSM8K serving path. +""" diff --git a/evals/rate_oracle/__main__.py b/evals/rate_oracle/__main__.py new file mode 100644 index 00000000..697534d9 --- /dev/null +++ b/evals/rate_oracle/__main__.py @@ -0,0 +1,20 @@ +"""CLI: validate the R3 rate gold. + + python -m evals.rate_oracle # validate rate_gold.jsonl; exit 0 iff invalid == 0 +""" + +from __future__ import annotations + +import json + +from evals.rate_oracle.runner import run + + +def main() -> int: + report = run() + print(json.dumps(report, indent=2, default=str)) + return 0 if report["invalid"] == 0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/rate_oracle/rate_gold.jsonl b/evals/rate_oracle/rate_gold.jsonl new file mode 100644 index 00000000..3a9709b9 --- /dev/null +++ b/evals/rate_oracle/rate_gold.jsonl @@ -0,0 +1,12 @@ +{"id": "r3-01-distance", "expect": "solved", "text": "A car travels 60 miles per hour for 3 hours. How many miles does it travel?", "rate_unit": {"numerator": "mile", "denominator": "hour"}, "rate": 60, "time": 3, "quantity": null, "query": "quantity", "gold": 180, "options": {"A": 120, "B": 150, "C": 180, "D": 200}, "answer": "C", "notes": "Forward: quantity = rate × time = 60 mile/hour × 3 hour = 180 mile. Unit composition exact."} +{"id": "r3-02-earnings", "expect": "solved", "text": "A worker earns 15 dollars per hour for 8 hours. How many dollars does she earn?", "rate_unit": {"numerator": "dollar", "denominator": "hour"}, "rate": 15, "time": 8, "quantity": null, "query": "quantity", "gold": 120, "options": {"A": 120, "B": 23, "C": 105, "D": 135}, "answer": "A", "notes": "Forward: 15 dollar/hour × 8 hour = 120 dollar."} +{"id": "r3-03-widgets", "expect": "solved", "text": "A machine makes 12 widgets per minute. It runs for 5 minutes. How many widgets does it make?", "rate_unit": {"numerator": "widget", "denominator": "minute"}, "rate": 12, "time": 5, "quantity": null, "query": "quantity", "gold": 60, "options": {"A": 17, "B": 60, "C": 7, "D": 72}, "answer": "B", "notes": "Forward: 12 widget/minute × 5 minute = 60 widget."} +{"id": "r3-04-items", "expect": "solved", "text": "A packer fills 8 items per box for 6 boxes. How many items does the packer pack?", "rate_unit": {"numerator": "item", "denominator": "box"}, "rate": 8, "time": 6, "quantity": null, "query": "quantity", "gold": 48, "options": {"A": 14, "B": 48, "C": 42, "D": 54}, "answer": "B", "notes": "Forward with a non-temporal denominator (per box): 8 item/box × 6 box = 48 item. The 'time' slot is the rate's denominator, not necessarily a clock unit."} +{"id": "r3-05-speed", "expect": "solved", "text": "A car travels 180 miles in 3 hours. What is its speed in miles per hour?", "rate_unit": {"numerator": "mile", "denominator": "hour"}, "rate": null, "time": 3, "quantity": 180, "query": "rate", "gold": 60, "options": {"A": 540, "B": 60, "C": 90, "D": 30}, "answer": "B", "notes": "Inverse (find rate): rate = quantity ÷ time = 180 mile ÷ 3 hour = 60 mile/hour. Exact division."} +{"id": "r3-06-runtime", "expect": "solved", "text": "A machine makes 60 widgets at 12 widgets per minute. How many minutes did it run?", "rate_unit": {"numerator": "widget", "denominator": "minute"}, "rate": 12, "time": null, "quantity": 60, "query": "time", "gold": 5, "options": {"A": 5, "B": 720, "C": 48, "D": 6}, "answer": "A", "notes": "Inverse (find time): time = quantity ÷ rate = 60 widget ÷ 12 widget/minute = 5 minute. Exact division."} +{"id": "r3-07-non-integer-rate", "expect": "solver_refuses", "solver_reason": "non_integer_solution", "text": "A car travels 100 miles in 3 hours. What is its speed in miles per hour?", "rate_unit": {"numerator": "mile", "denominator": "hour"}, "rate": null, "time": 3, "quantity": 100, "query": "rate", "gold": null, "notes": "Inverse with a non-exact division: 100 ÷ 3 is not an integer. The solver REFUSES (non_integer_solution), never rounds to 33."} +{"id": "r3-08-non-integer-time", "expect": "solver_refuses", "solver_reason": "non_integer_solution", "text": "A machine makes 50 widgets at 12 widgets per minute. How many minutes did it run?", "rate_unit": {"numerator": "widget", "denominator": "minute"}, "rate": 12, "time": null, "quantity": 50, "query": "time", "gold": null, "notes": "Inverse with a non-exact division: 50 ÷ 12 is not an integer. The solver REFUSES (non_integer_solution)."} +{"id": "r3-09-unit-mismatch", "expect": "reader_refuses", "reader_reason": "rate_unit_mismatch", "text": "A car travels 60 miles per hour for 30 minutes. How many miles does it travel?", "gold": null, "notes": "The rate is per HOUR but the duration is in MINUTES: mile/hour × minute does not compose. v1 does NOT silently convert minutes to hours (that would introduce rational quantities + unit conversion). The reader must REFUSE (rate_unit_mismatch)."} +{"id": "r3-10-missing-time", "expect": "reader_refuses", "reader_reason": "missing_time", "text": "A car travels 60 miles per hour. How many miles does it travel?", "gold": null, "notes": "Only the rate is given — no duration. With one known the equation is underdetermined; the reader must REFUSE (missing_time), never invent a time."} +{"id": "r3-11-combined-rates", "expect": "reader_refuses", "reader_reason": "combined_rates", "text": "A car travels 60 miles per hour for 2 hours and then 40 miles per hour for 3 hours. How many miles does it travel?", "gold": null, "notes": "Two rates over two segments — combined/multi-rate is R3.2, not single-rate v1. The reader must REFUSE (combined_rates) rather than read one segment."} +{"id": "r3-12-temporal-state", "expect": "reader_refuses", "reader_reason": "temporal_state", "text": "A car left at 2 pm and arrived at 5 pm traveling 60 miles per hour. How many miles did it travel?", "gold": null, "notes": "Elapsed clock-time (2 pm to 5 pm) is a temporal-state computation, not an explicit duration. v1 does NOT compute elapsed intervals; the reader must REFUSE (temporal_state)."} diff --git a/evals/rate_oracle/runner.py b/evals/rate_oracle/runner.py new file mode 100644 index 00000000..b8054178 --- /dev/null +++ b/evals/rate_oracle/runner.py @@ -0,0 +1,132 @@ +"""R3 rate setup-oracle runner — the ruler before reader capability (R3b). + +Validates that the independent rate gold is internally coherent and deserializes into the typed +``RateProblem`` IR: every fixture matches its closed ``expect`` taxonomy, well-formed setups +construct (exactly one unknown == the query), and — for ``solved`` fixtures — the multiple-choice +key agrees with the gold value. No reader yet (lands R3d); this proves the ruler. The R3 twin of +``evals.constraint_oracle.runner``. + +Exit 0 iff ``invalid == 0``. + +``expect`` taxonomy (closed): + - ``solved`` — full single-rate setup; ``gold`` is the int answer; ``options[answer] == gold``. + - ``solver_refuses`` — full setup, but a non-exact inverse; ``solver_reason`` says why; no gold. + - ``reader_refuses`` — prose the reader must refuse (missing piece / unit mismatch / combined / + temporal); ``reader_reason`` says why; no setup, no gold. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from evals.rate_oracle.signature import rate_setup_signature +from generate.rate_comprehension.model import RateProblem +from generate.rate_comprehension.units import RateUnit, UnitError + +_RATE_GOLD_PATH = Path(__file__).resolve().parent / "rate_gold.jsonl" + +EXPECTATIONS = frozenset({"solved", "solver_refuses", "reader_refuses"}) +SOLVER_REASONS = frozenset({"non_integer_solution"}) +#: Closed reader-refusal set the gold uses; extended (with a fixture) as the reader grows. +READER_REASONS = frozenset( + { + "rate_unit_mismatch", + "missing_rate", + "missing_time", + "missing_quantity", + "two_unknowns", + "combined_rates", + "temporal_state", + } +) + + +def _load_rate_gold() -> list[dict[str, Any]]: + return [ + json.loads(line) + for line in _RATE_GOLD_PATH.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def gold_to_problem(fx: dict[str, Any]) -> RateProblem: + """Deserialize a fixture's setup fields into the typed RateProblem IR.""" + ru = fx["rate_unit"] + return RateProblem( + rate_unit=RateUnit(ru["numerator"], ru["denominator"]), + rate=fx.get("rate"), + time=fx.get("time"), + quantity=fx.get("quantity"), + query=fx["query"], + ) + + +def validate_fixture(fx: dict[str, Any]) -> tuple[str, str | None]: + """Validate one gold fixture's coherence. Returns ``(outcome, reason)``.""" + expect = fx.get("expect") + if expect not in EXPECTATIONS: + return "invalid", f"unknown_expect:{expect!r}" + + if expect == "reader_refuses": + if fx.get("reader_reason") not in READER_REASONS: + return "invalid", f"unknown_reader_reason:{fx.get('reader_reason')!r}" + if fx.get("gold") is not None: + return "invalid", "reader_refuses_has_gold" + return "valid", None + + # solved | solver_refuses require a well-formed single-rate setup. + try: + problem = gold_to_problem(fx) + except (KeyError, TypeError, ValueError, UnitError) as exc: + return "invalid", f"malformed_setup:{exc}" + if rate_setup_signature(problem) != rate_setup_signature(problem): + return "invalid", "nondeterministic_signature" # pragma: no cover - determinism guard + + if expect == "solver_refuses": + if fx.get("solver_reason") not in SOLVER_REASONS: + return "invalid", f"unknown_solver_reason:{fx.get('solver_reason')!r}" + if fx.get("gold") is not None: + return "invalid", "solver_refuses_has_gold" + return "valid", None + + # solved: integer gold + coherent multiple-choice key. + gold = fx.get("gold") + if not isinstance(gold, int) or isinstance(gold, bool): + return "invalid", "solved_needs_int_gold" + options, answer = fx.get("options"), fx.get("answer") + if not isinstance(options, dict) or answer not in options: + return "invalid", "missing_or_unlabeled_answer" + if options[answer] != gold: + return "invalid", "answer_key_incoherent" + return "valid", None + + +def run() -> dict[str, Any]: + """Validate every R3 rate gold fixture. Exit-0 criterion: ``invalid == 0``.""" + fixtures = _load_rate_gold() + valid = invalid = 0 + by_expect: dict[str, int] = {} + details: list[dict[str, Any]] = [] + for fx in fixtures: + outcome, reason = validate_fixture(fx) + expect = fx.get("expect", "?") + by_expect[expect] = by_expect.get(expect, 0) + 1 + if outcome == "valid": + valid += 1 + details.append({"id": fx.get("id"), "outcome": "valid", "expect": expect}) + else: + invalid += 1 + details.append({"id": fx.get("id"), "outcome": "invalid", "reason": reason}) + return { + "lane": "rate_oracle_gold_validation", + "total": len(fixtures), + "valid": valid, + "invalid": invalid, + "by_expect": by_expect, + "details": details, + } + + +__all__ = ["EXPECTATIONS", "READER_REASONS", "SOLVER_REASONS", "gold_to_problem", "run", "validate_fixture"] diff --git a/evals/rate_oracle/signature.py b/evals/rate_oracle/signature.py new file mode 100644 index 00000000..8a9f0fa1 --- /dev/null +++ b/evals/rate_oracle/signature.py @@ -0,0 +1,36 @@ +"""Span-free canonical signature for the R3 rate setup oracle (R3b). + +A deterministic, order-independent projection of a single-rate setup — the rate unit, the known +values keyed by role, and the query — used to compare a reader's comprehended ``RateProblem`` +against the independent gold. The R3 twin of ``evals.setup_oracle.signature`` / +``evals.constraint_oracle.signature``. Pure, deterministic. +""" + +from __future__ import annotations + +from typing import Any + +from generate.rate_comprehension.model import RateProblem + + +def rate_setup_signature(problem: RateProblem) -> dict[str, Any]: + """Canonical ``(rate_unit, knowns, query)`` signature of a single-rate setup.""" + knowns = tuple( + sorted( + (role, value) + for role, value in ( + ("rate", problem.rate), + ("time", problem.time), + ("quantity", problem.quantity), + ) + if value is not None + ) + ) + return { + "rate_unit": (problem.rate_unit.numerator, problem.rate_unit.denominator), + "knowns": knowns, + "query": problem.query, + } + + +__all__ = ["rate_setup_signature"] diff --git a/generate/rate_comprehension/__init__.py b/generate/rate_comprehension/__init__.py index cd7d6021..2988fec6 100644 --- a/generate/rate_comprehension/__init__.py +++ b/generate/rate_comprehension/__init__.py @@ -12,6 +12,7 @@ conversion, no multi-equation systems. Those are later R3 slices. from __future__ import annotations +from generate.rate_comprehension.model import RateProblem, RateRole from generate.rate_comprehension.units import ( BaseUnit, RateUnit, @@ -23,6 +24,8 @@ from generate.rate_comprehension.units import ( __all__ = [ "BaseUnit", + "RateProblem", + "RateRole", "RateUnit", "UnitError", "rate_from_quantity_over_time", diff --git a/generate/rate_comprehension/model.py b/generate/rate_comprehension/model.py new file mode 100644 index 00000000..d2ddc3fd --- /dev/null +++ b/generate/rate_comprehension/model.py @@ -0,0 +1,53 @@ +"""Typed model for an R3 single-rate problem (R3b). + +A single-rate problem is the fixed equation ``quantity = rate × time`` with **exactly one** +unknown (the query). ``rate_unit`` (e.g. ``mile/hour``) is the single source of unit truth: the +quantity is measured in ``rate_unit.numerator`` and the time in ``rate_unit.denominator``, so the +three units are consistent by construction. The two non-query slots carry integer values; the +query slot is ``None``. + +Pure data with a structural guard: exactly the query slot is unknown (illegal states — zero or two +unknowns — cannot be represented). Off-serving; deterministic. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +from generate.rate_comprehension.units import RateUnit + +RateRole = Literal["rate", "time", "quantity"] + + +@dataclass(frozen=True, slots=True) +class RateProblem: + """``quantity = rate × time`` with one unknown. ``rate_unit`` fixes all three units.""" + + rate_unit: RateUnit + rate: int | None + time: int | None + quantity: int | None + query: RateRole + + def __post_init__(self) -> None: + slots: dict[str, int | None] = {"rate": self.rate, "time": self.time, "quantity": self.quantity} + unknown = [role for role, value in slots.items() if value is None] + if unknown != [self.query]: + raise ValueError( + f"exactly the query slot must be the unknown; query={self.query!r}, unknown={unknown}" + ) + for role, value in slots.items(): + if value is not None and (not isinstance(value, int) or isinstance(value, bool)): + raise ValueError(f"{role} value must be int; got {value!r}") + + @property + def quantity_unit(self) -> str: + return self.rate_unit.numerator + + @property + def time_unit(self) -> str: + return self.rate_unit.denominator + + +__all__ = ["RateProblem", "RateRole"] diff --git a/tests/test_rate_oracle.py b/tests/test_rate_oracle.py new file mode 100644 index 00000000..98644069 --- /dev/null +++ b/tests/test_rate_oracle.py @@ -0,0 +1,94 @@ +"""Tests for the R3 RateProblem model + rate setup oracle (R3b). + +Pins the model's exactly-one-unknown guard and that the gold ruler is coherent + non-vacuous +(each validator branch fires under its violation). +""" + +from __future__ import annotations + +import copy + +import pytest + +from evals.rate_oracle.runner import ( + READER_REASONS, + SOLVER_REASONS, + _load_rate_gold, + gold_to_problem, + run, + validate_fixture, +) +from evals.rate_oracle.signature import rate_setup_signature +from generate.rate_comprehension.model import RateProblem +from generate.rate_comprehension.units import RateUnit + + +def _solved() -> dict: + return copy.deepcopy(next(f for f in _load_rate_gold() if f["expect"] == "solved")) + + +def test_run_validates_all_gold() -> None: + r = run() + assert r["invalid"] == 0 and r["valid"] == r["total"] == 12 + assert r["by_expect"] == {"solved": 6, "solver_refuses": 2, "reader_refuses": 4} + + +def test_gold_to_problem_roundtrips() -> None: + p = gold_to_problem(next(f for f in _load_rate_gold() if f["id"] == "r3-01-distance")) + assert p.rate == 60 and p.time == 3 and p.quantity is None and p.query == "quantity" + assert (p.quantity_unit, p.time_unit) == ("mile", "hour") + + +def test_model_requires_exactly_one_unknown() -> None: + with pytest.raises(ValueError): # two unknowns + RateProblem(RateUnit("mile", "hour"), None, None, 180, "rate") + with pytest.raises(ValueError): # zero unknowns + RateProblem(RateUnit("mile", "hour"), 60, 3, 180, "quantity") + with pytest.raises(ValueError): # the named query is not the unknown + RateProblem(RateUnit("mile", "hour"), 60, None, 180, "rate") + + +def test_signature_is_deterministic_and_keyed_by_role() -> None: + p = gold_to_problem(next(f for f in _load_rate_gold() if f["id"] == "r3-05-speed")) + sig = rate_setup_signature(p) + assert sig == rate_setup_signature(p) + assert sig["query"] == "rate" and sig["rate_unit"] == ("mile", "hour") + assert dict(sig["knowns"]) == {"time": 3, "quantity": 180} + + +def test_reasons_in_gold_are_closed() -> None: + for fx in _load_rate_gold(): + if fx["expect"] == "solver_refuses": + assert fx["solver_reason"] in SOLVER_REASONS + elif fx["expect"] == "reader_refuses": + assert fx["reader_reason"] in READER_REASONS + + +# --- meaningful-fail: each invalid branch fires under exactly its violation ------------ # + + +def test_validator_rejects_incoherent_answer_key() -> None: + fx = _solved() + fx["answer"] = next(k for k in fx["options"] if fx["options"][k] != fx["gold"]) + assert validate_fixture(fx) == ("invalid", "answer_key_incoherent") + + +def test_validator_rejects_reader_refuse_carrying_gold() -> None: + fx = copy.deepcopy(next(f for f in _load_rate_gold() if f["expect"] == "reader_refuses")) + fx["gold"] = 5 + assert validate_fixture(fx) == ("invalid", "reader_refuses_has_gold") + + +def test_validator_rejects_unknown_reasons() -> None: + rr = copy.deepcopy(next(f for f in _load_rate_gold() if f["expect"] == "reader_refuses")) + rr["reader_reason"] = "made_up" + assert validate_fixture(rr)[0] == "invalid" + sr = copy.deepcopy(next(f for f in _load_rate_gold() if f["expect"] == "solver_refuses")) + sr["solver_reason"] = "made_up" + assert validate_fixture(sr)[0] == "invalid" + + +def test_validator_rejects_two_unknown_setup() -> None: + fx = _solved() + fx["time"] = None # now both time and quantity unknown -> model __post_init__ raises + assert validate_fixture(fx)[0] == "invalid"