e77f93c485
Closes the sibling vacuity CMB-a surfaced: R3's evals/rate_oracle solver_refuses branch accepted any fixture labelled non_integer_solution without checking the arithmetic, and the solved branch never recomputed the gold. A curator could certify an arithmetically-impossible refusal or a wrong gold (with a self-consistent answer key) as valid — the 'obligation that can't meaningfully fail = decoration' case CLAUDE.md forbids. Adds _canonical_outcome (independent reimplementation of the single-rate algebra WITH R3.2 conversion — NOT a call to generate.rate_comprehension.solver, which would be circular). The validator now cross-checks every solved gold (== computed) and every solver_refuses reason (genuine) against it, with dedicated meaningful-fail tests anchored on hand-written literals. Also removes the dead determinism self-comparison. R3 gold stays 13/13 valid (7/2/4); reader 9/0/4 unchanged; 26 oracle/reader tests pass. Off-serving; serving + other organs untouched.
236 lines
10 KiB
Python
236 lines
10 KiB
Python
"""R3 rate setup-oracle runner — the ruler before reader capability (R3b).
|
||
|
||
Validates that the independent rate gold is internally coherent and deserializes into the typed
|
||
``RateProblem`` IR: every fixture matches its closed ``expect`` taxonomy, well-formed setups
|
||
construct (exactly one unknown == the query), and the label is **cross-checked against the canonical
|
||
arithmetic** (``_canonical_outcome``): a ``solved`` gold must equal the computed answer and agree
|
||
with its multiple-choice key, and a ``solver_refuses`` reason must be the genuine reason the setup
|
||
refuses. This makes the ruler *meaningfully fail* on a mislabelled fixture (CLAUDE.md
|
||
proof-obligation rule) rather than asserting it. ``_canonical_outcome`` validates gold coherence
|
||
only; it is not a runtime solver (that is ``generate.rate_comprehension.solver``). The R3 twin of
|
||
``evals.constraint_oracle.runner``.
|
||
|
||
Exit 0 iff ``invalid == 0``.
|
||
|
||
``expect`` taxonomy (closed):
|
||
- ``solved`` — full single-rate setup; ``gold`` is the int answer; ``options[answer] == gold``.
|
||
- ``solver_refuses`` — full setup, but a non-exact inverse; ``solver_reason`` says why; no gold.
|
||
- ``reader_refuses`` — prose the reader must refuse (missing piece / unit mismatch / combined /
|
||
temporal); ``reader_reason`` says why; no setup, no gold.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
from fractions import Fraction
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from evals.rate_oracle.signature import rate_setup_signature
|
||
from generate.rate_comprehension.conversion import ConversionError, convert_time
|
||
from generate.rate_comprehension.model import RateProblem
|
||
from generate.rate_comprehension.units import RateUnit, UnitError
|
||
|
||
_RATE_GOLD_PATH = Path(__file__).resolve().parent / "rate_gold.jsonl"
|
||
|
||
EXPECTATIONS = frozenset({"solved", "solver_refuses", "reader_refuses"})
|
||
SOLVER_REASONS = frozenset({"non_integer_solution"})
|
||
#: Closed reader-refusal set the gold uses; extended (with a fixture) as the reader grows.
|
||
READER_REASONS = frozenset(
|
||
{
|
||
"rate_unit_mismatch",
|
||
"missing_rate",
|
||
"missing_time",
|
||
"missing_quantity",
|
||
"two_unknowns",
|
||
"combined_rates",
|
||
"temporal_state",
|
||
}
|
||
)
|
||
|
||
|
||
def _load_rate_gold() -> list[dict[str, Any]]:
|
||
return [
|
||
json.loads(line)
|
||
for line in _RATE_GOLD_PATH.read_text(encoding="utf-8").splitlines()
|
||
if line.strip()
|
||
]
|
||
|
||
|
||
def gold_to_problem(fx: dict[str, Any]) -> RateProblem:
|
||
"""Deserialize a fixture's setup fields into the typed RateProblem IR."""
|
||
ru = fx["rate_unit"]
|
||
return RateProblem(
|
||
rate_unit=RateUnit(ru["numerator"], ru["denominator"]),
|
||
rate=fx.get("rate"),
|
||
time=fx.get("time"),
|
||
quantity=fx.get("quantity"),
|
||
query=fx["query"],
|
||
time_unit=fx.get("time_unit"),
|
||
)
|
||
|
||
|
||
def _canonical_outcome(problem: RateProblem) -> tuple[str, int | None, str | None]:
|
||
"""The canonical answer/refusal for a well-formed single-rate setup, used ONLY to validate gold
|
||
coherence — **not** a runtime solver (that is ``generate.rate_comprehension.solver``). An
|
||
independent reimplementation of the rate algebra (``rate × convert(time)`` and its inverses) so
|
||
the oracle's ``solved`` / ``solver_refuses`` labels can *meaningfully fail* (CLAUDE.md
|
||
proof-obligation rule) rather than being asserted. Returns ``("solved", value, None)`` or
|
||
``("refused", None, reason)``.
|
||
"""
|
||
ru = problem.rate_unit
|
||
try:
|
||
if problem.query == "quantity":
|
||
assert problem.rate is not None and problem.time is not None
|
||
value = Fraction(problem.rate) * convert_time(problem.time, problem.time_unit, ru.denominator)
|
||
elif problem.query == "rate":
|
||
assert problem.quantity is not None and problem.time is not None
|
||
value = Fraction(problem.quantity) / convert_time(problem.time, problem.time_unit, ru.denominator)
|
||
else: # time — answered in the rate's denominator unit
|
||
assert problem.quantity is not None and problem.rate is not None
|
||
value = Fraction(problem.quantity) / Fraction(problem.rate)
|
||
except ConversionError:
|
||
# A non-convertible duration in a solved/solver_refuses setup is incoherent gold — the
|
||
# convertibility refusal (``rate_unit_mismatch``) is the reader's boundary, not the solver's.
|
||
return "refused", None, "rate_unit_mismatch"
|
||
if value.denominator != 1:
|
||
return "refused", None, "non_integer_solution"
|
||
return "solved", int(value), None
|
||
|
||
|
||
def validate_fixture(fx: dict[str, Any]) -> tuple[str, str | None]:
|
||
"""Validate one gold fixture's coherence. Returns ``(outcome, reason)``."""
|
||
expect = fx.get("expect")
|
||
if expect not in EXPECTATIONS:
|
||
return "invalid", f"unknown_expect:{expect!r}"
|
||
|
||
if expect == "reader_refuses":
|
||
if fx.get("reader_reason") not in READER_REASONS:
|
||
return "invalid", f"unknown_reader_reason:{fx.get('reader_reason')!r}"
|
||
if fx.get("gold") is not None:
|
||
return "invalid", "reader_refuses_has_gold"
|
||
return "valid", None
|
||
|
||
# solved | solver_refuses require a well-formed single-rate setup. (Signature determinism is
|
||
# proven by the model being a pure function over a frozen dataclass — see the oracle tests — not
|
||
# by a runtime self-comparison.)
|
||
try:
|
||
problem = gold_to_problem(fx)
|
||
except (KeyError, TypeError, ValueError, UnitError) as exc:
|
||
return "invalid", f"malformed_setup:{exc}"
|
||
kind, value, reason = _canonical_outcome(problem)
|
||
|
||
if expect == "solver_refuses":
|
||
if fx.get("solver_reason") not in SOLVER_REASONS:
|
||
return "invalid", f"unknown_solver_reason:{fx.get('solver_reason')!r}"
|
||
if fx.get("gold") is not None:
|
||
return "invalid", "solver_refuses_has_gold"
|
||
# The label must be the GENUINE reason this setup refuses — not asserted.
|
||
if kind != "refused":
|
||
return "invalid", "solver_refuses_is_actually_solvable"
|
||
if reason != fx["solver_reason"]:
|
||
return "invalid", f"solver_reason_mismatch:expected_{reason}"
|
||
return "valid", None
|
||
|
||
# solved: integer gold + coherent multiple-choice key + gold == the canonical computed answer.
|
||
gold = fx.get("gold")
|
||
if not isinstance(gold, int) or isinstance(gold, bool):
|
||
return "invalid", "solved_needs_int_gold"
|
||
options, answer = fx.get("options"), fx.get("answer")
|
||
if not isinstance(options, dict) or answer not in options:
|
||
return "invalid", "missing_or_unlabeled_answer"
|
||
if options[answer] != gold:
|
||
return "invalid", "answer_key_incoherent"
|
||
if kind != "solved":
|
||
return "invalid", "solved_is_not_canonically_solvable"
|
||
if value != gold:
|
||
return "invalid", "gold_does_not_match_computed_answer"
|
||
return "valid", None
|
||
|
||
|
||
def run() -> dict[str, Any]:
|
||
"""Validate every R3 rate gold fixture. Exit-0 criterion: ``invalid == 0``."""
|
||
fixtures = _load_rate_gold()
|
||
valid = invalid = 0
|
||
by_expect: dict[str, int] = {}
|
||
details: list[dict[str, Any]] = []
|
||
for fx in fixtures:
|
||
outcome, reason = validate_fixture(fx)
|
||
expect = fx.get("expect", "?")
|
||
by_expect[expect] = by_expect.get(expect, 0) + 1
|
||
if outcome == "valid":
|
||
valid += 1
|
||
details.append({"id": fx.get("id"), "outcome": "valid", "expect": expect})
|
||
else:
|
||
invalid += 1
|
||
details.append({"id": fx.get("id"), "outcome": "invalid", "reason": reason})
|
||
return {
|
||
"lane": "rate_oracle_gold_validation",
|
||
"total": len(fixtures),
|
||
"valid": valid,
|
||
"invalid": invalid,
|
||
"by_expect": by_expect,
|
||
"details": details,
|
||
}
|
||
|
||
|
||
def run_reader() -> dict[str, Any]:
|
||
"""Grade the R3 rate reader against the gold (R3d).
|
||
|
||
Well-formed fixtures (``solved`` / ``solver_refuses``) must read to a setup whose signature
|
||
equals the gold's (``setup_correct``); a refusal is a miss; a mismatch is ``setup_wrong``.
|
||
``reader_refuses`` fixtures must refuse with the gold's ``reader_reason`` (``refused_correct``);
|
||
a refusal with the wrong reason is ``reason_mismatch``; producing a setup is ``setup_wrong``.
|
||
Exit-0 criterion: ``setup_wrong == 0 and reason_mismatch == 0``.
|
||
"""
|
||
from generate.meaning_graph.reader import Refusal
|
||
from generate.rate_comprehension.reader import read_rate_problem
|
||
|
||
fixtures = _load_rate_gold()
|
||
setup_correct = setup_wrong = setup_refused = refused_correct = reason_mismatch = 0
|
||
details: list[dict[str, Any]] = []
|
||
for fx in fixtures:
|
||
out = read_rate_problem(fx["text"])
|
||
fid = fx.get("id")
|
||
if fx["expect"] in ("solved", "solver_refuses"):
|
||
if isinstance(out, Refusal):
|
||
setup_refused += 1
|
||
details.append({"id": fid, "outcome": "setup_refused", "reason": out.reason})
|
||
elif rate_setup_signature(out) == rate_setup_signature(gold_to_problem(fx)):
|
||
setup_correct += 1
|
||
details.append({"id": fid, "outcome": "setup_correct"})
|
||
else:
|
||
setup_wrong += 1
|
||
details.append({"id": fid, "outcome": "setup_WRONG"})
|
||
else: # reader_refuses
|
||
if isinstance(out, Refusal) and out.reason == fx["reader_reason"]:
|
||
refused_correct += 1
|
||
details.append({"id": fid, "outcome": "refused_correct", "reason": out.reason})
|
||
elif isinstance(out, Refusal):
|
||
reason_mismatch += 1
|
||
details.append({"id": fid, "outcome": "reason_mismatch", "got": out.reason, "want": fx["reader_reason"]})
|
||
else:
|
||
setup_wrong += 1
|
||
details.append({"id": fid, "outcome": "setup_WRONG_over_read"})
|
||
return {
|
||
"lane": "rate_oracle_reader",
|
||
"total": len(fixtures),
|
||
"setup_correct": setup_correct,
|
||
"setup_wrong": setup_wrong,
|
||
"setup_refused": setup_refused,
|
||
"refused_correct": refused_correct,
|
||
"reason_mismatch": reason_mismatch,
|
||
"details": details,
|
||
}
|
||
|
||
|
||
__all__ = [
|
||
"EXPECTATIONS",
|
||
"READER_REASONS",
|
||
"SOLVER_REASONS",
|
||
"_canonical_outcome",
|
||
"gold_to_problem",
|
||
"run",
|
||
"run_reader",
|
||
"validate_fixture",
|
||
]
|