diff --git a/evals/combined_rate_oracle/__main__.py b/evals/combined_rate_oracle/__main__.py index 2508f2a6..47c8dd68 100644 --- a/evals/combined_rate_oracle/__main__.py +++ b/evals/combined_rate_oracle/__main__.py @@ -1,6 +1,7 @@ -"""CLI: validate the combined-rate gold ruler (CMB-a). +"""CLI: validate the combined-rate gold ruler, or grade the CMB solver against it. - python -m evals.combined_rate_oracle # validate combined_rate_gold.jsonl; exit 0 iff invalid == 0 + python -m evals.combined_rate_oracle # validate combined_rate_gold.jsonl; exit 0 iff invalid == 0 + python -m evals.combined_rate_oracle solver # grade the solver (CMB-b); exit 0 iff no wrong The reader grading lane (``reader`` arg) lands with the reader (CMB-c). """ @@ -8,14 +9,21 @@ The reader grading lane (``reader`` arg) lands with the reader (CMB-c). from __future__ import annotations import json +import sys -from evals.combined_rate_oracle.runner import run +from evals.combined_rate_oracle.runner import run, run_solver def main() -> int: - report = run() + lane = sys.argv[1] if len(sys.argv) > 1 else "" + if lane == "solver": + report = run_solver() + ok = report["solved_wrong"] == 0 and report["refuse_wrong"] == 0 + else: + report = run() + ok = report["invalid"] == 0 print(json.dumps(report, indent=2, default=str)) - return 0 if report["invalid"] == 0 else 1 + return 0 if ok else 1 if __name__ == "__main__": diff --git a/evals/combined_rate_oracle/combined_rate_gold.jsonl b/evals/combined_rate_oracle/combined_rate_gold.jsonl index b4088d2a..405363c0 100644 --- a/evals/combined_rate_oracle/combined_rate_gold.jsonl +++ b/evals/combined_rate_oracle/combined_rate_gold.jsonl @@ -8,6 +8,7 @@ {"id": "cmb-08-paint-non-integer-time", "expect": "solver_refuses", "solver_reason": "non_integer_solution", "text": "Anna paints 3 rooms per hour and Ben paints 2 rooms per hour. Working together, how many hours do they take to paint 12 rooms?", "rate_unit": {"numerator": "room", "denominator": "hour"}, "rate_a": 3, "rate_b": 2, "combine_mode": "sum", "time": null, "time_unit": "hour", "quantity": 12, "query": "time", "gold": null, "notes": "Well-formed setup. effective_rate = 5; 12 / 5 is not an integer -> solver REFUSES (non_integer_solution), never rounds."} {"id": "cmb-07b-net-negative", "expect": "solver_refuses", "solver_reason": "non_positive_net_rate", "text": "A pump fills a tank at 2 liters per minute while a drain removes 5 liters per minute. How many liters are in the tank after 3 minutes?", "rate_unit": {"numerator": "liter", "denominator": "minute"}, "rate_a": 2, "rate_b": 5, "combine_mode": "difference", "time": 3, "time_unit": "minute", "quantity": null, "query": "quantity", "gold": null, "notes": "effective_rate = 2-5 = -3 (drain exceeds fill) -> NEGATIVE net. The accumulate query is not supported by a non-positive net rate -> solver REFUSES (non_positive_net_rate); it never returns a negative quantity (-9). Exercises the eff<0 branch (cmb-07 covers the eff=0 boundary point)."} {"id": "cmb-07c-net-zero-time-query", "expect": "solver_refuses", "solver_reason": "non_positive_net_rate", "text": "A pump fills a tank at 4 liters per minute while a drain removes 4 liters per minute. How many minutes does it take to put 12 liters in the tank?", "rate_unit": {"numerator": "liter", "denominator": "minute"}, "rate_a": 4, "rate_b": 4, "combine_mode": "difference", "time": null, "time_unit": "minute", "quantity": 12, "query": "time", "gold": null, "notes": "effective_rate = 4-4 = 0; a time query would compute 12 / 0. The non_positive_net_rate guard refuses BEFORE dividing -> no division by zero. Exercises the eff=0 + query=time cell."} +{"id": "cmb-07d-net-negative-time-query", "expect": "solver_refuses", "solver_reason": "non_positive_net_rate", "text": "A pump fills a tank at 2 liters per minute while a drain removes 5 liters per minute. How many minutes does it take to put 9 liters in the tank?", "rate_unit": {"numerator": "liter", "denominator": "minute"}, "rate_a": 2, "rate_b": 5, "combine_mode": "difference", "time": null, "time_unit": "minute", "quantity": 9, "query": "time", "gold": null, "notes": "effective_rate = 2-5 = -3 (NEGATIVE net) on a time query — never fills, so 'how long to reach 9' is unanswerable. REFUSE (non_positive_net_rate). Closes the eff<0 + query=time cell (cmb-07b is eff<0/quantity, cmb-07c is eff=0/time). All inputs positive; only the net is negative."} {"id": "cmb-09-unit-mismatch", "expect": "reader_refuses", "reader_reason": "rate_unit_mismatch", "text": "Anna paints 3 rooms per hour and a pipe fills 2 liters per minute. Working together, how much do they complete in 4 hours?", "gold": null, "notes": "Two rates but incompatible units (room/hour vs liter/minute) — they do not share one compound unit, so no single CombinedRateProblem is representable. REFUSE (rate_unit_mismatch)."} {"id": "cmb-10-combine-ambiguous", "expect": "reader_refuses", "reader_reason": "combine_mode_ambiguous", "text": "One machine runs at 3 widgets per minute. Another machine runs at 5 widgets per minute. How many widgets are produced in 4 minutes?", "gold": null, "notes": "Two rates, same unit, but NO explicit cooperation ('together'/'combined'/'both') or opposing-flow ('fills ... drains') cue. Direction must not be inferred from vibes. REFUSE (combine_mode_ambiguous)."} {"id": "cmb-11-missing-second-rate", "expect": "reader_refuses", "reader_reason": "missing_second_rate", "text": "Anna and Ben paint a house together. Anna paints 3 rooms per hour. How many rooms do they paint in 4 hours?", "gold": null, "notes": "A cooperation cue IS present ('together') and only ONE rate is given -> REFUSE (missing_second_rate). The CMB domain-entry rule is 2-dimensional (rate-count x cue), NOT cue-gated: (two rates + clear sum/difference cue -> solved); (two rates + no cue -> combine_mode_ambiguous, cmb-10); (one rate + combination cue -> missing_second_rate, here); (one rate or foreign + no cue -> not_combined_rate_shaped, cmb-15). So two rates alone make it CMB's domain even with no cue, while a single rate needs a combination cue to be CMB's substantive domain rather than R3's."} diff --git a/evals/combined_rate_oracle/runner.py b/evals/combined_rate_oracle/runner.py index ef5585d9..ce44f1fe 100644 --- a/evals/combined_rate_oracle/runner.py +++ b/evals/combined_rate_oracle/runner.py @@ -187,6 +187,56 @@ def run() -> dict[str, Any]: } +def run_solver() -> dict[str, Any]: + """Grade the CMB solver against the gold setups (CMB-b). + + For every fixture with a setup (``solved`` / ``solver_refuses``), solve the gold setup and + compare to the gold label: a ``solved`` fixture must produce the gold int; a ``solver_refuses`` + fixture must produce a ``Refusal`` carrying the gold ``solver_reason``. ``reader_refuses`` + fixtures have no setup and are skipped (the reader's lane, CMB-c). This grades the runtime + solver against the *committed* gold values (not against ``_canonical_outcome``). Note both the + solver and ``_canonical_outcome`` delegate net-rate arithmetic to ``model.effective_rate``, so + the hand-computed literal tests — not path-independence — are the anchor against a shared + ``effective_rate`` bug. Exit-0 criterion: ``solved_wrong == 0 and refuse_wrong == 0``. + """ + from generate.combined_rate_comprehension.solver import solve_combined_rate + from generate.meaning_graph.reader import Refusal + + fixtures = _load_combined_rate_gold() + solved_correct = solved_wrong = refuse_correct = refuse_wrong = skipped = 0 + details: list[dict[str, Any]] = [] + for fx in fixtures: + fid = fx.get("id") + if fx["expect"] == "reader_refuses": + skipped += 1 + continue + out = solve_combined_rate(gold_to_problem(fx)) + if fx["expect"] == "solved": + if not isinstance(out, Refusal) and out == fx["gold"]: + solved_correct += 1 + details.append({"id": fid, "outcome": "solved_correct"}) + else: + solved_wrong += 1 + details.append({"id": fid, "outcome": "solved_WRONG", "got": str(out), "want": fx["gold"]}) + else: # solver_refuses + if isinstance(out, Refusal) and out.reason == fx["solver_reason"]: + refuse_correct += 1 + details.append({"id": fid, "outcome": "refuse_correct", "reason": out.reason}) + else: + refuse_wrong += 1 + details.append({"id": fid, "outcome": "refuse_WRONG", "got": str(out), "want": fx["solver_reason"]}) + return { + "lane": "combined_rate_oracle_solver", + "total": len(fixtures), + "solved_correct": solved_correct, + "solved_wrong": solved_wrong, + "refuse_correct": refuse_correct, + "refuse_wrong": refuse_wrong, + "skipped_reader_refuses": skipped, + "details": details, + } + + __all__ = [ "COMBINE_MODES", "EXPECTATIONS", @@ -197,5 +247,6 @@ __all__ = [ "_load_combined_rate_gold", "gold_to_problem", "run", + "run_solver", "validate_fixture", ] diff --git a/generate/combined_rate_comprehension/model.py b/generate/combined_rate_comprehension/model.py index 461d3e0f..3e5428c7 100644 --- a/generate/combined_rate_comprehension/model.py +++ b/generate/combined_rate_comprehension/model.py @@ -23,9 +23,12 @@ Pure data with a structural guard: the two rates must be present, and exactly th licenses are known/unknown (illegal states — a missing rate, the wrong slot unknown, an over-specified ``effective_rate`` query — cannot be represented). -``effective_rate`` is derived (a property), and for ``difference`` mode it MAY be ``<= 0`` -(``rate_a <= rate_b``). The model does NOT refuse that — a non-positive net rate is the *solver's* -boundary (``non_positive_net_rate``, CMB-b), not a malformed setup. Off-serving; deterministic. +The two rates and the known time/quantity are **positive ints** — a non-positive rate or a +non-positive duration/quantity is nonsensical and cannot be represented (so the solver can never +receive a path that yields a negative answer). The *net* rate, by contrast, MAY be ``<= 0``: +``effective_rate`` is derived (a property), and for ``difference`` mode with ``rate_a <= rate_b`` +it is ``<= 0``. The model does NOT refuse that — a non-positive net rate is the *solver's* boundary +(``non_positive_net_rate``, CMB-b), not a malformed setup. Off-serving; deterministic. No unit conversion in v1 (``time_unit`` defaults to the rate denominator and the v1 gold never crosses units). """ @@ -68,8 +71,8 @@ class CombinedRateProblem: if self.time_unit is None: object.__setattr__(self, "time_unit", self.rate_unit.denominator) for role, value in (("rate_a", self.rate_a), ("rate_b", self.rate_b)): - if not isinstance(value, int) or isinstance(value, bool): - raise ValueError(f"{role} must be a known int (two explicit rates); got {value!r}") + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + raise ValueError(f"{role} must be a positive int (two explicit rates); got {value!r}") if self.combine_mode not in ("sum", "difference"): raise ValueError(f"combine_mode must be 'sum' or 'difference'; got {self.combine_mode!r}") if self.query not in _QUERY_SLOTS: @@ -86,8 +89,8 @@ class CombinedRateProblem: raise ValueError(f"query={self.query!r}: slot {role!r} must be the unknown (None)") for role in known_slots: value = slots[role] - if not isinstance(value, int) or isinstance(value, bool): - raise ValueError(f"{role} value must be int; got {value!r}") + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + raise ValueError(f"{role} value must be a positive int; got {value!r}") @property def effective_rate(self) -> int: diff --git a/generate/combined_rate_comprehension/solver.py b/generate/combined_rate_comprehension/solver.py new file mode 100644 index 00000000..9a5e0b82 --- /dev/null +++ b/generate/combined_rate_comprehension/solver.py @@ -0,0 +1,54 @@ +"""Exact integer combined-rate solver (CMB-b). + +Solves the queried slot of a :class:`CombinedRateProblem` over the **effective rate** — + +```text +effective_rate = rate_a + rate_b (combine_mode == "sum") +effective_rate = rate_a - rate_b (combine_mode == "difference") +query effective_rate: return effective_rate (well-defined even if <= 0) +query quantity: effective_rate × time (exact int) +query time: quantity ÷ effective_rate (exact int or REFUSE) +``` + +Two refusals, the closed CMB solver taxonomy: + +- ``non_positive_net_rate`` — a ``quantity`` or ``time`` query whose net rate is ``<= 0`` cannot + accumulate or finish (and guards the ``eff == 0`` time query from dividing by zero). The + ``effective_rate`` query is exempt: the net rate is a well-defined answer even when ``<= 0``. +- ``non_integer_solution`` — a ``time`` query that does not divide exactly; never rounds. + +Pure integer arithmetic — **no float, no Fraction** (CMB v1 crosses no units, so no rational +conversion is needed; the reader will refuse cross-unit problems, CMB-c). Off-serving (imports no +``generate.derivation`` / ``core.reliability_gate``); deterministic. This is the runtime solver; the +oracle's ``_canonical_outcome`` is a separate gold-coherence path. Note both this solver and +``_canonical_outcome`` delegate the net-rate arithmetic to ``model.effective_rate``, so the +solver lane grades the solver against the *committed static gold*, and the hand-computed literal +tests (``test_literal_grid_values``, plus the inline ``(rate_a ± rate_b) × time`` in +``test_quantity_query_is_always_integral``) — **not** path-independence — are the anchor against a +shared ``effective_rate`` bug. +""" + +from __future__ import annotations + +from generate.combined_rate_comprehension.model import CombinedRateProblem +from generate.meaning_graph.reader import Refusal + + +def solve_combined_rate(problem: CombinedRateProblem) -> int | Refusal: + """Solve the queried slot exactly over the effective rate, or refuse with the closed taxonomy.""" + eff = problem.effective_rate + if problem.query == "effective_rate": + return eff # the net rate is the answer, well-defined even when <= 0 + if eff <= 0: + return Refusal("non_positive_net_rate", f"effective_rate={eff}") + if problem.query == "quantity": + assert problem.time is not None # guaranteed by the model's per-query slot guard + return eff * problem.time + # query == "time": exact integer division in the rate's denominator unit, or refuse. + assert problem.quantity is not None + if problem.quantity % eff != 0: + return Refusal("non_integer_solution", f"{problem.quantity}/{eff}") + return problem.quantity // eff + + +__all__ = ["solve_combined_rate"] diff --git a/tests/test_combined_rate_oracle.py b/tests/test_combined_rate_oracle.py index 839157e6..c2535548 100644 --- a/tests/test_combined_rate_oracle.py +++ b/tests/test_combined_rate_oracle.py @@ -43,8 +43,8 @@ def _solved() -> dict: def test_run_validates_all_gold() -> None: r = run() - assert r["invalid"] == 0 and r["valid"] == r["total"] == 17 - assert r["by_expect"] == {"solved": 6, "solver_refuses": 4, "reader_refuses": 7} + assert r["invalid"] == 0 and r["valid"] == r["total"] == 18 + assert r["by_expect"] == {"solved": 6, "solver_refuses": 5, "reader_refuses": 7} def test_solved_grid_covers_every_mode_query_cell() -> None: @@ -114,6 +114,22 @@ def test_non_positive_net_rate_is_a_well_formed_setup_not_a_model_error() -> Non assert p.effective_rate == 0 +def test_model_rejects_non_positive_rates_and_known_slots() -> None: + # Negative/zero INPUTS are nonsensical and must be unrepresentable (a non-positive NET rate is + # the solver's call, but a non-positive rate magnitude or duration/quantity is a malformed setup) + # — otherwise the solver could emit a negative answer (wrong=0 breach). + with pytest.raises(ValueError): # negative rate magnitude + CombinedRateProblem(-3, 2, _ROOM_HOUR, "sum", 4, None, "quantity") + with pytest.raises(ValueError): # zero rate magnitude + CombinedRateProblem(3, 0, _ROOM_HOUR, "sum", 4, None, "quantity") + with pytest.raises(ValueError): # negative known duration + CombinedRateProblem(3, 2, _ROOM_HOUR, "sum", -3, None, "quantity") + with pytest.raises(ValueError): # negative known quantity + CombinedRateProblem(3, 2, _ROOM_HOUR, "sum", None, -5, "time") + with pytest.raises(ValueError): # zero known duration + CombinedRateProblem(3, 2, _ROOM_HOUR, "sum", 0, None, "quantity") + + # --- signature ------------------------------------------------------------------------- # diff --git a/tests/test_combined_rate_solver.py b/tests/test_combined_rate_solver.py new file mode 100644 index 00000000..da104899 --- /dev/null +++ b/tests/test_combined_rate_solver.py @@ -0,0 +1,112 @@ +"""Tests for the exact combined-rate solver (CMB-b). + +Pins wrong=0 for the solver lane: every solved gold setup solves to its gold int; every +solver_refuses setup refuses with its gold reason; the effective_rate query returns the net rate +even when non-positive; non-positive net and non-integer time refuse (never round, never go +negative); and the answer is always an exact int, never a float. Literal-anchored values keep the +solver honest independently of the gold and the oracle's _canonical_outcome. +""" + +from __future__ import annotations + +from evals.combined_rate_oracle.runner import _load_combined_rate_gold, gold_to_problem, run_solver +from generate.combined_rate_comprehension.model import CombinedRateProblem +from generate.combined_rate_comprehension.solver import solve_combined_rate +from generate.combined_rate_comprehension.units import RateUnit +from generate.meaning_graph.reader import Refusal + +_ROOM_HOUR = RateUnit("room", "hour") +_LITER_MIN = RateUnit("liter", "minute") + + +def _by(expect: str) -> list[dict]: + return [f for f in _load_combined_rate_gold() if f["expect"] == expect] + + +def test_solver_lane_is_wrong_zero_and_complete() -> None: + r = run_solver() + assert r["solved_wrong"] == 0 and r["refuse_wrong"] == 0 + assert r["solved_correct"] == 6 + assert r["refuse_correct"] == 5 + assert r["skipped_reader_refuses"] == 7 + + +def test_solves_every_solved_fixture_to_gold() -> None: + for fx in _by("solved"): + out = solve_combined_rate(gold_to_problem(fx)) + assert out == fx["gold"], f"{fx['id']}: got {out!r}, want {fx['gold']}" + + +def test_refuses_every_solver_refuse_fixture_with_reason() -> None: + for fx in _by("solver_refuses"): + out = solve_combined_rate(gold_to_problem(fx)) + assert isinstance(out, Refusal) and out.reason == fx["solver_reason"], fx["id"] + + +def test_literal_grid_values() -> None: + # Hand-computed expected answers — the independent anchor (not gold, not _canonical_outcome). + assert solve_combined_rate(CombinedRateProblem(3, 2, _ROOM_HOUR, "sum", 4, None, "quantity")) == 20 + assert solve_combined_rate(CombinedRateProblem(5, 2, _LITER_MIN, "difference", 6, None, "quantity")) == 18 + assert solve_combined_rate(CombinedRateProblem(3, 2, _ROOM_HOUR, "sum", None, 20, "time")) == 4 + assert solve_combined_rate(CombinedRateProblem(5, 2, _LITER_MIN, "difference", None, 18, "time")) == 6 + assert solve_combined_rate(CombinedRateProblem(9, 4, _LITER_MIN, "difference", None, None, "effective_rate")) == 5 + assert solve_combined_rate(CombinedRateProblem(6, 4, _LITER_MIN, "sum", None, None, "effective_rate")) == 10 + + +def test_effective_rate_query_returns_net_even_when_nonpositive() -> None: + # The net rate is a well-defined answer even at/below zero — the effective_rate query never refuses. + assert solve_combined_rate(CombinedRateProblem(4, 4, _LITER_MIN, "difference", None, None, "effective_rate")) == 0 + assert solve_combined_rate(CombinedRateProblem(2, 5, _LITER_MIN, "difference", None, None, "effective_rate")) == -3 + + +def test_non_positive_net_rate_refuses_quantity_and_time() -> None: + # Full (eff<=0) x (quantity, time) grid: eff==0/quantity, eff<0/quantity, eff==0/time (would /0), + # eff<0/time. All inputs positive; only the net rate is non-positive. + for p in ( + CombinedRateProblem(4, 4, _LITER_MIN, "difference", 5, None, "quantity"), + CombinedRateProblem(2, 5, _LITER_MIN, "difference", 3, None, "quantity"), + CombinedRateProblem(4, 4, _LITER_MIN, "difference", None, 12, "time"), + CombinedRateProblem(2, 5, _LITER_MIN, "difference", None, 9, "time"), + ): + out = solve_combined_rate(p) + assert isinstance(out, Refusal) and out.reason == "non_positive_net_rate" + + +def test_non_integer_time_refuses() -> None: + # 12 / (3+2) = 2.4 -> refuse, never round to 2 or 3. + out = solve_combined_rate(CombinedRateProblem(3, 2, _ROOM_HOUR, "sum", None, 12, "time")) + assert isinstance(out, Refusal) and out.reason == "non_integer_solution" + + +def test_quantity_query_is_always_integral() -> None: + # eff * time is integral for any integer rates/time -> never a non_integer refusal on quantity. + # Expected values are computed INLINE (not via model.effective_rate) — the real anchor against a + # shared effective_rate bug — for BOTH sum and difference modes. + for ra, rb, t in ((3, 2, 7), (5, 2, 9), (1, 1, 100)): + out = solve_combined_rate(CombinedRateProblem(ra, rb, _ROOM_HOUR, "sum", t, None, "quantity")) + assert isinstance(out, int) and out == (ra + rb) * t + for ra, rb, t in ((5, 2, 6), (9, 4, 3), (7, 1, 8)): # rate_a > rate_b so eff > 0 + out = solve_combined_rate(CombinedRateProblem(ra, rb, _LITER_MIN, "difference", t, None, "quantity")) + assert isinstance(out, int) and out == (ra - rb) * t + + +def test_solver_answer_is_int_never_float_or_bool() -> None: + out = solve_combined_rate(CombinedRateProblem(5, 2, _LITER_MIN, "difference", None, 18, "time")) + assert type(out) is int # not float, not bool + + +def test_solver_module_is_off_serving() -> None: + import ast + from pathlib import Path + + import generate.combined_rate_comprehension.solver as solver_mod + + forbidden = ("generate.derivation", "core.reliability_gate") + for node in ast.walk(ast.parse(Path(str(solver_mod.__file__)).read_text(encoding="utf-8"))): + names = ( + [a.name for a in node.names] if isinstance(node, ast.Import) + else [node.module or ""] if isinstance(node, ast.ImportFrom) + else [] + ) + for name in names: + assert not any(name.startswith(t) for t in forbidden), f"solver imports {name}"