test(rate-oracle): port _canonical_outcome non-vacuous validation to R3 (R3-vac)

Closes the sibling vacuity CMB-a surfaced: R3's evals/rate_oracle solver_refuses branch
accepted any fixture labelled non_integer_solution without checking the arithmetic, and the
solved branch never recomputed the gold. A curator could certify an arithmetically-impossible
refusal or a wrong gold (with a self-consistent answer key) as valid — the 'obligation that
can't meaningfully fail = decoration' case CLAUDE.md forbids.

Adds _canonical_outcome (independent reimplementation of the single-rate algebra WITH R3.2
conversion — NOT a call to generate.rate_comprehension.solver, which would be circular). The
validator now cross-checks every solved gold (== computed) and every solver_refuses reason
(genuine) against it, with dedicated meaningful-fail tests anchored on hand-written literals.
Also removes the dead determinism self-comparison.

R3 gold stays 13/13 valid (7/2/4); reader 9/0/4 unchanged; 26 oracle/reader tests pass.
Off-serving; serving + other organs untouched.

evals/rate_oracle/runner.py

@@ -2,8 +2,12 @@
 
 Validates that the independent rate gold is internally coherent and deserializes into the typed
 ``RateProblem`` IR: every fixture matches its closed ``expect`` taxonomy, well-formed setups
-construct (exactly one unknown == the query), and — for ``solved`` fixtures — the multiple-choice
-key agrees with the gold value. No reader yet (lands R3d); this proves the ruler. The R3 twin of
+construct (exactly one unknown == the query), and the label is **cross-checked against the canonical
+arithmetic** (``_canonical_outcome``): a ``solved`` gold must equal the computed answer and agree
+with its multiple-choice key, and a ``solver_refuses`` reason must be the genuine reason the setup
+refuses. This makes the ruler *meaningfully fail* on a mislabelled fixture (CLAUDE.md
+proof-obligation rule) rather than asserting it. ``_canonical_outcome`` validates gold coherence
+only; it is not a runtime solver (that is ``generate.rate_comprehension.solver``). The R3 twin of
 ``evals.constraint_oracle.runner``.
 
 Exit 0 iff ``invalid == 0``.
@@ -18,10 +22,12 @@ Exit 0 iff ``invalid == 0``.
 from __future__ import annotations
 
 import json
+from fractions import Fraction
 from pathlib import Path
 from typing import Any
 
 from evals.rate_oracle.signature import rate_setup_signature
+from generate.rate_comprehension.conversion import ConversionError, convert_time
 from generate.rate_comprehension.model import RateProblem
 from generate.rate_comprehension.units import RateUnit, UnitError
 
@@ -64,6 +70,34 @@ def gold_to_problem(fx: dict[str, Any]) -> RateProblem:
     )
 
 
+def _canonical_outcome(problem: RateProblem) -> tuple[str, int | None, str | None]:
+    """The canonical answer/refusal for a well-formed single-rate setup, used ONLY to validate gold
+    coherence — **not** a runtime solver (that is ``generate.rate_comprehension.solver``). An
+    independent reimplementation of the rate algebra (``rate × convert(time)`` and its inverses) so
+    the oracle's ``solved`` / ``solver_refuses`` labels can *meaningfully fail* (CLAUDE.md
+    proof-obligation rule) rather than being asserted. Returns ``("solved", value, None)`` or
+    ``("refused", None, reason)``.
+    """
+    ru = problem.rate_unit
+    try:
+        if problem.query == "quantity":
+            assert problem.rate is not None and problem.time is not None
+            value = Fraction(problem.rate) * convert_time(problem.time, problem.time_unit, ru.denominator)
+        elif problem.query == "rate":
+            assert problem.quantity is not None and problem.time is not None
+            value = Fraction(problem.quantity) / convert_time(problem.time, problem.time_unit, ru.denominator)
+        else:  # time — answered in the rate's denominator unit
+            assert problem.quantity is not None and problem.rate is not None
+            value = Fraction(problem.quantity) / Fraction(problem.rate)
+    except ConversionError:
+        # A non-convertible duration in a solved/solver_refuses setup is incoherent gold — the
+        # convertibility refusal (``rate_unit_mismatch``) is the reader's boundary, not the solver's.
+        return "refused", None, "rate_unit_mismatch"
+    if value.denominator != 1:
+        return "refused", None, "non_integer_solution"
+    return "solved", int(value), None
+
+
 def validate_fixture(fx: dict[str, Any]) -> tuple[str, str | None]:
     """Validate one gold fixture's coherence. Returns ``(outcome, reason)``."""
     expect = fx.get("expect")
@@ -77,22 +111,28 @@ def validate_fixture(fx: dict[str, Any]) -> tuple[str, str | None]:
             return "invalid", "reader_refuses_has_gold"
         return "valid", None
 
-    # solved | solver_refuses require a well-formed single-rate setup.
+    # solved | solver_refuses require a well-formed single-rate setup. (Signature determinism is
+    # proven by the model being a pure function over a frozen dataclass — see the oracle tests — not
+    # by a runtime self-comparison.)
     try:
         problem = gold_to_problem(fx)
     except (KeyError, TypeError, ValueError, UnitError) as exc:
         return "invalid", f"malformed_setup:{exc}"
-    if rate_setup_signature(problem) != rate_setup_signature(problem):
-        return "invalid", "nondeterministic_signature"  # pragma: no cover - determinism guard
+    kind, value, reason = _canonical_outcome(problem)
 
     if expect == "solver_refuses":
         if fx.get("solver_reason") not in SOLVER_REASONS:
             return "invalid", f"unknown_solver_reason:{fx.get('solver_reason')!r}"
         if fx.get("gold") is not None:
             return "invalid", "solver_refuses_has_gold"
+        # The label must be the GENUINE reason this setup refuses — not asserted.
+        if kind != "refused":
+            return "invalid", "solver_refuses_is_actually_solvable"
+        if reason != fx["solver_reason"]:
+            return "invalid", f"solver_reason_mismatch:expected_{reason}"
         return "valid", None
 
-    # solved: integer gold + coherent multiple-choice key.
+    # solved: integer gold + coherent multiple-choice key + gold == the canonical computed answer.
     gold = fx.get("gold")
     if not isinstance(gold, int) or isinstance(gold, bool):
         return "invalid", "solved_needs_int_gold"
@@ -101,6 +141,10 @@ def validate_fixture(fx: dict[str, Any]) -> tuple[str, str | None]:
         return "invalid", "missing_or_unlabeled_answer"
     if options[answer] != gold:
         return "invalid", "answer_key_incoherent"
+    if kind != "solved":
+        return "invalid", "solved_is_not_canonically_solvable"
+    if value != gold:
+        return "invalid", "gold_does_not_match_computed_answer"
     return "valid", None
 
 
@@ -184,6 +228,7 @@ __all__ = [
     "EXPECTATIONS",
     "READER_REASONS",
     "SOLVER_REASONS",
+    "_canonical_outcome",
     "gold_to_problem",
     "run",
     "run_reader",

tests/test_rate_oracle.py

@@ -92,3 +92,50 @@ def test_validator_rejects_two_unknown_setup() -> None:
     fx = _solved()
     fx["time"] = None  # now both time and quantity unknown -> model __post_init__ raises
     assert validate_fixture(fx)[0] == "invalid"
+
+
+# --- R3-vac: the canonical-outcome cross-check makes solver_refuses / solved non-vacuous --------- #
+
+
+def test_validator_rejects_solver_refuse_that_is_actually_solvable() -> None:
+    # r3-01 solves exactly to 180; mislabelling it solver_refuses must be rejected, not blessed.
+    fx = _solved()
+    fx["expect"] = "solver_refuses"
+    fx["solver_reason"] = "non_integer_solution"
+    for k in ("gold", "options", "answer"):
+        fx.pop(k, None)
+    assert validate_fixture(fx) == ("invalid", "solver_refuses_is_actually_solvable")
+
+
+def test_validator_rejects_solver_reason_mismatch() -> None:
+    # A non-convertible duration genuinely refuses, but for rate_unit_mismatch (reader's boundary),
+    # not the labelled non_integer_solution.
+    fx = _solved()
+    fx["expect"] = "solver_refuses"
+    fx["solver_reason"] = "non_integer_solution"
+    fx["time_unit"] = "gallon"  # does not convert to the rate denominator (hour)
+    for k in ("gold", "options", "answer"):
+        fx.pop(k, None)
+    assert validate_fixture(fx) == ("invalid", "solver_reason_mismatch:expected_rate_unit_mismatch")
+
+
+def test_validator_rejects_wrong_gold_even_with_coherent_answer_key() -> None:
+    fx = _solved()  # r3-01: 60 mile/hour x 3 hour = 180
+    fx["gold"] = 181
+    fx["options"] = {**fx["options"], fx["answer"]: 181}  # keep options[answer] == gold
+    assert validate_fixture(fx) == ("invalid", "gold_does_not_match_computed_answer")
+
+
+def test_canonical_outcome_matrix() -> None:
+    from evals.rate_oracle.runner import _canonical_outcome
+
+    mph = RateUnit("mile", "hour")
+    wpm = RateUnit("widget", "minute")
+    # forward, no conversion: 60 x 3 = 180
+    assert _canonical_outcome(RateProblem(mph, 60, 3, None, "quantity", time_unit="hour")) == ("solved", 180, None)
+    # R3.2 conversion: 60 mile/hour for 30 minutes -> 30
+    assert _canonical_outcome(RateProblem(mph, 60, 30, None, "quantity", time_unit="minute")) == ("solved", 30, None)
+    # inverse rate, non-integer (100 / 3) -> refuse
+    assert _canonical_outcome(RateProblem(mph, None, 3, 100, "rate", time_unit="hour")) == ("refused", None, "non_integer_solution")
+    # inverse time, exact: 60 / 12 = 5
+    assert _canonical_outcome(RateProblem(wpm, 12, None, 60, "time", time_unit="minute")) == ("solved", 5, None)