From 36bf2a64562843358e0953e0677594ea653aea9a Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 8 Jun 2026 06:58:14 -0700 Subject: [PATCH] =?UTF-8?q?feat(combined-rate):=20CMB-a=20=E2=80=94=20comb?= =?UTF-8?q?ined-rate=20setup=20ruler=20(model=20+=20gold=20+=20oracle)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First rung of the combined-rate ladder, off-serving (imports no generate.derivation / core.reliability_gate). Claims exactly: the combined-rate setup ruler is defined and gold-valid. No reader/solver/wiring yet — no capability claim. - generate/combined_rate_comprehension/: local RateUnit (decoupled from R3), CombinedRateProblem (two-explicit-rates + per-query-slot guard; effective_rate = rate_a +/- rate_b; non-positive net left to the solver, not the model). - evals/combined_rate_oracle/: span-free signature (sum commutative / difference ordered) + a NON-VACUOUS validator. 17 gold fixtures: 6 solved (full combine_mode x query grid), 4 solver_refuses, 7 reader_refuses (the complete refusal taxonomy + the 2x2 domain-entry grid). Adversarial 5-lens verification returned fix_first; the validator now cross-checks every solved gold and solver_refuses reason against the canonical arithmetic (_canonical_outcome) so a mislabelled / arithmetically-impossible fixture is rejected (meaningful-fail per CLAUDE.md), with dedicated tests. Added eff<0 and eff=0/time coverage fixtures; removed a dead determinism guard. gold 17/17 valid; 25 oracle tests; R1/R2/R3 + router-hygiene + serving all unchanged. Doc: docs/analysis/cmb-a-combined-rate-ruler-2026-06-08.md --- .../cmb-a-combined-rate-ruler-2026-06-08.md | 120 ++++++++ evals/combined_rate_oracle/__init__.py | 8 + evals/combined_rate_oracle/__main__.py | 22 ++ .../combined_rate_gold.jsonl | 17 ++ evals/combined_rate_oracle/runner.py | 201 +++++++++++++ evals/combined_rate_oracle/signature.py | 45 +++ .../combined_rate_comprehension/__init__.py | 23 ++ generate/combined_rate_comprehension/model.py | 105 +++++++ generate/combined_rate_comprehension/units.py | 41 +++ tests/test_combined_rate_oracle.py | 280 ++++++++++++++++++ 10 files changed, 862 insertions(+) create mode 100644 docs/analysis/cmb-a-combined-rate-ruler-2026-06-08.md create mode 100644 evals/combined_rate_oracle/__init__.py create mode 100644 evals/combined_rate_oracle/__main__.py create mode 100644 evals/combined_rate_oracle/combined_rate_gold.jsonl create mode 100644 evals/combined_rate_oracle/runner.py create mode 100644 evals/combined_rate_oracle/signature.py create mode 100644 generate/combined_rate_comprehension/__init__.py create mode 100644 generate/combined_rate_comprehension/model.py create mode 100644 generate/combined_rate_comprehension/units.py create mode 100644 tests/test_combined_rate_oracle.py diff --git a/docs/analysis/cmb-a-combined-rate-ruler-2026-06-08.md b/docs/analysis/cmb-a-combined-rate-ruler-2026-06-08.md new file mode 100644 index 00000000..e3f51274 --- /dev/null +++ b/docs/analysis/cmb-a-combined-rate-ruler-2026-06-08.md @@ -0,0 +1,120 @@ +# CMB-a — combined-rate setup ruler (model + gold + oracle) + +**As of:** CMB-a, on `main` + `feat/cmb-a-combined-rate-oracle`. + +CMB-a is the first rung of the combined-rate ladder. It claims **exactly one thing**: *the +combined-rate setup ruler is defined and gold-valid.* No reader (CMB-c), no solver (CMB-b), no +router/contemplation wiring (CMB-d) — and therefore **no capability claim yet**. + +Combined rate is a new semantic object, not single-rate with a bigger parser: two explicit rates +over one shared unit, combined by an explicit mode, then single-rate algebra over the result — + +```text +effective_rate = rate_a + rate_b (sum — cooperation: "working together") +effective_rate = rate_a - rate_b (difference — opposing flow: "fills ... drains") +quantity = effective_rate × time | time = quantity ÷ effective_rate | effective_rate +``` + +so it gets its own organ (`generate/combined_rate_comprehension/`) and its own ruler +(`evals/combined_rate_oracle/`), deliberately **not** folded into `rate_comprehension`. If the rate +organs converge later, a shared rate algebra can be extracted then (the local `RateUnit` copy is +the seam). + +## Lane state +- combined-rate gold **17 / 17 valid** — **6 solved** (the full `combine_mode × query` grid), + **4 solver_refuses**, **7 reader_refuses**. +- R1 **7/0/3** · R2 **10/0/0** · R3 gold **13/13** / reader **9/0/4** — all unchanged. +- router-organ-hygiene invariant **green** · off-serving (imports no `generate.derivation` / + `core.reliability_gate`, AST-checked) · GSM8K serving **unchanged**. + +```bash +.venv/bin/python -m evals.combined_rate_oracle # 17/17 valid +.venv/bin/python -m pytest tests/test_combined_rate_oracle.py -q # 25 passed +``` + +## Model — `CombinedRateProblem` + +Two rates are **always known** (that is what makes it combined-rate); `rate_unit` is the single +unit slot for both (so two different units is a *reader* refusal, not representable). The query +selects the unknown: + +| query | known | unknown | +|---|---|---| +| `quantity` | `time` | `quantity` | +| `time` | `quantity` | `time` | +| `effective_rate` | — | both `time` and `quantity` | + +`effective_rate` is a derived property and **may be `<= 0`** for `difference` (`rate_a <= rate_b`); +the model does **not** refuse that — a non-positive net rate is the *solver's* boundary +(`non_positive_net_rate`, CMB-b), not a malformed setup. + +## The 2×2 domain-entry grid (load-bearing for CMB-c/d hygiene) + +Domain entry is **two-dimensional** (rate-count × combination cue), not cue-gated: + +| | combination cue | no cue | +|---|---|---| +| **two rates** | solved (sum/difference) | `combine_mode_ambiguous` | +| **one rate** | `missing_second_rate` | `not_combined_rate_shaped` | + +Two rates alone make it CMB's domain even with no cue (→ `combine_mode_ambiguous`, a substantive +refusal). A single rate needs a combination cue to be CMB's *substantive* domain +(`missing_second_rate`); without one it is single-rate R3 territory and CMB must step aside with +`not_combined_rate_shaped` (→ the `input_shape` family, the router-organ-hygiene invariant). This +is the CMB analogue of R3.1's `missing_rate`-only-when-duration-present fix. + +## Closed taxonomies + +- `combine_mode ∈ {sum, difference}`; `query ∈ {quantity, time, effective_rate}`. +- **solver_reasons**: `non_positive_net_rate`, `non_integer_solution`. +- **reader_reasons**: `rate_unit_mismatch`, `combine_mode_ambiguous`, `missing_second_rate`, + `three_or_more_rates`, `reciprocal_work_rate_deferred`, `clock_interval_deferred`, + `not_combined_rate_shaped`. + +## The validator is non-vacuous (adversarial-verification finding) + +A 5-lens adversarial pass (independent recompute, hygiene-boundary refutation, model/signature +holes, proof-obligation rigor, forward wrong=0 hazards) + an adjudicator returned **`fix_first`**. +The real hazard: the first-draft `validate_fixture` accepted *any* fixture labelled +`solver_refuses` regardless of the arithmetic — a positive net rate (`5−2=3`) labelled +`non_positive_net_rate`, or a `quantity` query (always integral) labelled `non_integer_solution`, +would have been certified as valid gold and fed a wrong target to CMB-b. That is precisely the +"schema obligation that can't meaningfully fail = decoration, not proof" case CLAUDE.md forbids +(and the **solved** branch had the identical latent hole: a wrong `gold` with a self-consistent +answer key). + +**Fix:** a single canonical-outcome reference (`_canonical_outcome`) inside the oracle now +validates *both* the solved gold and the solver_refuses reason against the real arithmetic — a +mislabelled or arithmetically-impossible fixture is rejected (`solver_refuses_is_actually_solvable` +/ `solver_reason_mismatch:…` / `gold_does_not_match_computed_answer`). Each new branch has a +dedicated meaningful-fail test. + +> **Scope note (surfaced deliberately):** the slice was scoped "oracle = canonical structure, not +> solving." Computing the canonical answer *purely to validate gold coherence* is slightly more +> than structure-only — but the meaningful-fail doctrine is a hard CLAUDE.md invariant, and a ruler +> that cannot check its own gold is decoration. `_canonical_outcome` is **not** a runtime solver +> (that is CMB-b) and does not touch reader-independence (which lives in text→setup, CMB-c). + +Two coverage fixtures were also added for the `eff<0` (`cmb-07b`) and `eff=0/time` (`cmb-07c`, +which would otherwise divide by zero) cells, the dead determinism self-comparison was removed, and +three uncovered validator branches got dedicated tests. + +> **Sibling follow-up (noted, not fixed here):** R3's `evals/rate_oracle` has the *same* latent +> vacuity — its `solver_refuses` branch does not cross-check `non_integer_solution` against the +> arithmetic. A later PR should port `_canonical_outcome`-style validation to the R3 oracle. + +## CMB ladder + +```text +CMB-a (this slice) model + gold + setup oracle "the ruler is defined and gold-valid" +CMB-b exact solver (effective_rate algebra, int-or-refuse) +CMB-c reader (prose -> CombinedRateProblem | Refusal), graded by this oracle +CMB-d router/contemplation wiring + failure-family update (hygiene proven) +CMB-e capability ledger entry +``` + +## Deferred (named) + +`reciprocal` work-rate (`1/(1/a + 1/b)`), 3+ rates, mixed-unit combined rates (cross with R3.2 time +conversion), clock intervals, relative speed with opposite directions — each has a `reader_reason` +in the ruler so it refuses cleanly until built. diff --git a/evals/combined_rate_oracle/__init__.py b/evals/combined_rate_oracle/__init__.py new file mode 100644 index 00000000..9436e3ae --- /dev/null +++ b/evals/combined_rate_oracle/__init__.py @@ -0,0 +1,8 @@ +"""Combined-rate setup oracle (off-serving) — the independent ruler for the combined-rate organ. + +Grades a comprehended ``CombinedRateProblem`` against independent gold by a span-free canonical +signature (``signature``), backed by a reviewable gold corpus (``combined_rate_gold.jsonl``) and a +validation runner (``runner``). The CMB twin of ``evals.rate_oracle`` / ``evals.constraint_oracle``. +Imports no ``generate.derivation`` / ``core.reliability_gate`` — disjoint from the GSM8K serving +path. CMB-a ships the ruler only; the reader grading lane lands with the reader (CMB-c). +""" diff --git a/evals/combined_rate_oracle/__main__.py b/evals/combined_rate_oracle/__main__.py new file mode 100644 index 00000000..2508f2a6 --- /dev/null +++ b/evals/combined_rate_oracle/__main__.py @@ -0,0 +1,22 @@ +"""CLI: validate the combined-rate gold ruler (CMB-a). + + python -m evals.combined_rate_oracle # validate combined_rate_gold.jsonl; exit 0 iff invalid == 0 + +The reader grading lane (``reader`` arg) lands with the reader (CMB-c). +""" + +from __future__ import annotations + +import json + +from evals.combined_rate_oracle.runner import run + + +def main() -> int: + report = run() + print(json.dumps(report, indent=2, default=str)) + return 0 if report["invalid"] == 0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/combined_rate_oracle/combined_rate_gold.jsonl b/evals/combined_rate_oracle/combined_rate_gold.jsonl new file mode 100644 index 00000000..b4088d2a --- /dev/null +++ b/evals/combined_rate_oracle/combined_rate_gold.jsonl @@ -0,0 +1,17 @@ +{"id": "cmb-01-paint-sum-quantity", "expect": "solved", "text": "Anna paints 3 rooms per hour and Ben paints 2 rooms per hour. Working together, how many rooms do they paint in 4 hours?", "rate_unit": {"numerator": "room", "denominator": "hour"}, "rate_a": 3, "rate_b": 2, "combine_mode": "sum", "time": 4, "time_unit": "hour", "quantity": null, "query": "quantity", "gold": 20, "options": {"A": 20, "B": 12, "C": 8, "D": 5}, "answer": "A", "notes": "Cooperation cue 'working together' -> sum. effective_rate = 3+2 = 5 room/hour; 5 x 4 = 20 room. Distractors: 12=only Anna, 8=only Ben, 5=forgot to multiply by time."} +{"id": "cmb-02-tank-difference-quantity", "expect": "solved", "text": "A pipe fills a tank at 5 liters per minute while a drain removes 2 liters per minute. How many liters are in the tank after 6 minutes?", "rate_unit": {"numerator": "liter", "denominator": "minute"}, "rate_a": 5, "rate_b": 2, "combine_mode": "difference", "time": 6, "time_unit": "minute", "quantity": null, "query": "quantity", "gold": 18, "options": {"A": 18, "B": 42, "C": 30, "D": 12}, "answer": "A", "notes": "Opposing-flow cue 'fills ... removes' -> difference. effective_rate = 5-2 = 3 liter/minute; 3 x 6 = 18 liter. Distractors: 42=sum-mode (5+2)x6, 30=only fill, 12=only drain."} +{"id": "cmb-03-paint-sum-time", "expect": "solved", "text": "Anna paints 3 rooms per hour and Ben paints 2 rooms per hour. Working together, how many hours do they take to paint 20 rooms?", "rate_unit": {"numerator": "room", "denominator": "hour"}, "rate_a": 3, "rate_b": 2, "combine_mode": "sum", "time": null, "time_unit": "hour", "quantity": 20, "query": "time", "gold": 4, "options": {"A": 4, "B": 10, "C": 100, "D": 5}, "answer": "A", "notes": "sum, query time: 20 / (3+2) = 4 hour. Distractors: 10=only Ben (20/2), 100=20x5, 5=effective rate."} +{"id": "cmb-04-tank-difference-time", "expect": "solved", "text": "A pipe fills a tank at 5 liters per minute while a drain removes 2 liters per minute. How many minutes does it take to put 18 liters in the tank?", "rate_unit": {"numerator": "liter", "denominator": "minute"}, "rate_a": 5, "rate_b": 2, "combine_mode": "difference", "time": null, "time_unit": "minute", "quantity": 18, "query": "time", "gold": 6, "options": {"A": 6, "B": 9, "C": 54, "D": 3}, "answer": "A", "notes": "difference, query time: 18 / (5-2) = 6 minute. Distractors: 9=only drain inverse, 54=18x3, 3=net rate."} +{"id": "cmb-05-tank-difference-rate", "expect": "solved", "text": "A pipe fills a tank at 9 liters per minute while a drain removes 4 liters per minute. What is the net rate in liters per minute at which the tank fills?", "rate_unit": {"numerator": "liter", "denominator": "minute"}, "rate_a": 9, "rate_b": 4, "combine_mode": "difference", "time": null, "time_unit": "minute", "quantity": null, "query": "effective_rate", "gold": 5, "options": {"A": 5, "B": 13, "C": 36, "D": 4}, "answer": "A", "notes": "difference, query effective_rate: 9-4 = 5 liter/minute. Distractors: 13=sum-mode, 36=9x4, 4=only drain."} +{"id": "cmb-06-hoses-sum-rate", "expect": "solved", "text": "Two hoses fill a pool together, one at 6 gallons per minute and one at 4 gallons per minute. What is their combined fill rate in gallons per minute?", "rate_unit": {"numerator": "gallon", "denominator": "minute"}, "rate_a": 6, "rate_b": 4, "combine_mode": "sum", "time": null, "time_unit": "minute", "quantity": null, "query": "effective_rate", "gold": 10, "options": {"A": 10, "B": 2, "C": 24, "D": 6}, "answer": "A", "notes": "sum, query effective_rate: 6+4 = 10 gallon/minute. Distractors: 2=difference-mode, 24=6x4, 6=one hose."} +{"id": "cmb-07-tank-non-positive-net", "expect": "solver_refuses", "solver_reason": "non_positive_net_rate", "text": "A pump fills a tank at 4 liters per minute while a drain removes 4 liters per minute. How many liters are in the tank after 5 minutes?", "rate_unit": {"numerator": "liter", "denominator": "minute"}, "rate_a": 4, "rate_b": 4, "combine_mode": "difference", "time": 5, "time_unit": "minute", "quantity": null, "query": "quantity", "gold": null, "notes": "Well-formed setup (reader can read it: two rates, opposing-flow cue, duration, quantity query). effective_rate = 4-4 = 0. The accumulate query is not supported by a non-positive net rate -> solver REFUSES (non_positive_net_rate); it does NOT silently return 0."} +{"id": "cmb-08-paint-non-integer-time", "expect": "solver_refuses", "solver_reason": "non_integer_solution", "text": "Anna paints 3 rooms per hour and Ben paints 2 rooms per hour. Working together, how many hours do they take to paint 12 rooms?", "rate_unit": {"numerator": "room", "denominator": "hour"}, "rate_a": 3, "rate_b": 2, "combine_mode": "sum", "time": null, "time_unit": "hour", "quantity": 12, "query": "time", "gold": null, "notes": "Well-formed setup. effective_rate = 5; 12 / 5 is not an integer -> solver REFUSES (non_integer_solution), never rounds."} +{"id": "cmb-07b-net-negative", "expect": "solver_refuses", "solver_reason": "non_positive_net_rate", "text": "A pump fills a tank at 2 liters per minute while a drain removes 5 liters per minute. How many liters are in the tank after 3 minutes?", "rate_unit": {"numerator": "liter", "denominator": "minute"}, "rate_a": 2, "rate_b": 5, "combine_mode": "difference", "time": 3, "time_unit": "minute", "quantity": null, "query": "quantity", "gold": null, "notes": "effective_rate = 2-5 = -3 (drain exceeds fill) -> NEGATIVE net. The accumulate query is not supported by a non-positive net rate -> solver REFUSES (non_positive_net_rate); it never returns a negative quantity (-9). Exercises the eff<0 branch (cmb-07 covers the eff=0 boundary point)."} +{"id": "cmb-07c-net-zero-time-query", "expect": "solver_refuses", "solver_reason": "non_positive_net_rate", "text": "A pump fills a tank at 4 liters per minute while a drain removes 4 liters per minute. How many minutes does it take to put 12 liters in the tank?", "rate_unit": {"numerator": "liter", "denominator": "minute"}, "rate_a": 4, "rate_b": 4, "combine_mode": "difference", "time": null, "time_unit": "minute", "quantity": 12, "query": "time", "gold": null, "notes": "effective_rate = 4-4 = 0; a time query would compute 12 / 0. The non_positive_net_rate guard refuses BEFORE dividing -> no division by zero. Exercises the eff=0 + query=time cell."} +{"id": "cmb-09-unit-mismatch", "expect": "reader_refuses", "reader_reason": "rate_unit_mismatch", "text": "Anna paints 3 rooms per hour and a pipe fills 2 liters per minute. Working together, how much do they complete in 4 hours?", "gold": null, "notes": "Two rates but incompatible units (room/hour vs liter/minute) — they do not share one compound unit, so no single CombinedRateProblem is representable. REFUSE (rate_unit_mismatch)."} +{"id": "cmb-10-combine-ambiguous", "expect": "reader_refuses", "reader_reason": "combine_mode_ambiguous", "text": "One machine runs at 3 widgets per minute. Another machine runs at 5 widgets per minute. How many widgets are produced in 4 minutes?", "gold": null, "notes": "Two rates, same unit, but NO explicit cooperation ('together'/'combined'/'both') or opposing-flow ('fills ... drains') cue. Direction must not be inferred from vibes. REFUSE (combine_mode_ambiguous)."} +{"id": "cmb-11-missing-second-rate", "expect": "reader_refuses", "reader_reason": "missing_second_rate", "text": "Anna and Ben paint a house together. Anna paints 3 rooms per hour. How many rooms do they paint in 4 hours?", "gold": null, "notes": "A cooperation cue IS present ('together') and only ONE rate is given -> REFUSE (missing_second_rate). The CMB domain-entry rule is 2-dimensional (rate-count x cue), NOT cue-gated: (two rates + clear sum/difference cue -> solved); (two rates + no cue -> combine_mode_ambiguous, cmb-10); (one rate + combination cue -> missing_second_rate, here); (one rate or foreign + no cue -> not_combined_rate_shaped, cmb-15). So two rates alone make it CMB's domain even with no cue, while a single rate needs a combination cue to be CMB's substantive domain rather than R3's."} +{"id": "cmb-12-three-rates", "expect": "reader_refuses", "reader_reason": "three_or_more_rates", "text": "Anna paints 3 rooms per hour, Ben paints 2 rooms per hour, and Carla paints 4 rooms per hour. Working together, how many rooms do they paint in 2 hours?", "gold": null, "notes": "Three explicit rates. CMB v1 combines exactly two. REFUSE (three_or_more_rates) — deferred to a later slice."} +{"id": "cmb-13-reciprocal-work-rate", "expect": "reader_refuses", "reader_reason": "reciprocal_work_rate_deferred", "text": "Anna can paint a house in 3 hours, and Ben can paint the same house in 6 hours. Working together, how many hours do they take?", "gold": null, "notes": "Durations to complete a whole job, NOT explicit per-unit rates — the classic reciprocal work-rate form 1/(1/3 + 1/6). A different representation (reciprocal rates, rational arithmetic). REFUSE (reciprocal_work_rate_deferred) — deferred."} +{"id": "cmb-14-clock-interval", "expect": "reader_refuses", "reader_reason": "clock_interval_deferred", "text": "A pipe fills a tank at 5 liters per minute and another fills it at 3 liters per minute, running together from 2 pm to 5 pm. How many liters are in the tank?", "gold": null, "notes": "Elapsed clock time (2 pm to 5 pm), not an explicit duration. REFUSE (clock_interval_deferred) — deferred (the CMB twin of R3's temporal_state)."} +{"id": "cmb-15-not-combined-shaped", "expect": "reader_refuses", "reader_reason": "not_combined_rate_shaped", "text": "A car travels 60 miles per hour for 3 hours. How many miles does it travel?", "gold": null, "notes": "A single explicit rate with no second rate and no combination cue — single-rate R3 territory, not CMB's. CMB must step aside, NOT claim a substantive boundary. REFUSE (not_combined_rate_shaped) -> maps to the input_shape family in CMB-d (router-organ-hygiene invariant)."} diff --git a/evals/combined_rate_oracle/runner.py b/evals/combined_rate_oracle/runner.py new file mode 100644 index 00000000..ef5585d9 --- /dev/null +++ b/evals/combined_rate_oracle/runner.py @@ -0,0 +1,201 @@ +"""Combined-rate setup-oracle runner — the ruler before reader capability (CMB-a). + +Validates that the independent combined-rate gold is internally coherent and deserializes into the +typed ``CombinedRateProblem`` IR: every fixture matches its closed ``expect`` taxonomy, well-formed +setups construct (two explicit rates, exactly the query's slot unknown), and — for ``solved`` +fixtures — the multiple-choice key agrees with the gold value. No reader yet (lands CMB-c); this +proves the ruler. The CMB twin of ``evals.rate_oracle.runner``. + +Exit 0 iff ``invalid == 0``. + +The validator is **non-vacuous**: it cross-checks every ``solved`` gold and every +``solver_refuses`` reason against the canonical arithmetic (``_canonical_outcome``) so a fixture +whose label contradicts its own setup (a positive net rate labelled refused, a wrong gold value, +a non-integer reason on a quantity query) is rejected — the ruler can *meaningfully fail*, not just +assert (CLAUDE.md proof-obligation rule). ``_canonical_outcome`` validates gold coherence only; it +is not a runtime solver (that is CMB-b). + +``expect`` taxonomy (closed): + - ``solved`` — full combined-rate setup; ``gold`` is the int answer; ``options[answer] == gold`` + **and** ``gold`` equals the canonical computed answer. + - ``solver_refuses`` — full, well-formed setup whose canonical outcome is a refusal for exactly + the stated ``solver_reason`` (non-positive net rate / non-exact division); no gold. + - ``reader_refuses`` — prose the reader must refuse (mismatched units / ambiguous combine mode / + missing second rate / >=3 rates / reciprocal or clock forms / not + combined-rate-shaped); ``reader_reason`` says why; no setup, no gold. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from generate.combined_rate_comprehension.model import CombinedRateProblem +from generate.combined_rate_comprehension.units import RateUnit, UnitError + +_GOLD_PATH = Path(__file__).resolve().parent / "combined_rate_gold.jsonl" + +EXPECTATIONS = frozenset({"solved", "solver_refuses", "reader_refuses"}) +COMBINE_MODES = frozenset({"sum", "difference"}) +QUERIES = frozenset({"quantity", "time", "effective_rate"}) +SOLVER_REASONS = frozenset({"non_positive_net_rate", "non_integer_solution"}) +#: Closed reader-refusal set the gold uses; extended (with a fixture) as the reader grows. +#: ``not_combined_rate_shaped`` is the hygiene "not my domain" reason (maps to the ``input_shape`` +#: family in CMB-d); the rest are substantive combined-rate boundaries. +READER_REASONS = frozenset( + { + "rate_unit_mismatch", + "combine_mode_ambiguous", + "missing_second_rate", + "three_or_more_rates", + "reciprocal_work_rate_deferred", + "clock_interval_deferred", + "not_combined_rate_shaped", + } +) + + +def _load_combined_rate_gold() -> list[dict[str, Any]]: + return [ + json.loads(line) + for line in _GOLD_PATH.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def gold_to_problem(fx: dict[str, Any]) -> CombinedRateProblem: + """Deserialize a fixture's setup fields into the typed CombinedRateProblem IR.""" + ru = fx["rate_unit"] + return CombinedRateProblem( + rate_a=fx.get("rate_a"), + rate_b=fx.get("rate_b"), + rate_unit=RateUnit(ru["numerator"], ru["denominator"]), + combine_mode=fx["combine_mode"], + time=fx.get("time"), + quantity=fx.get("quantity"), + query=fx["query"], + time_unit=fx.get("time_unit"), + ) + + +def _canonical_outcome(problem: CombinedRateProblem) -> tuple[str, int | None, str | None]: + """The canonical answer/refusal for a well-formed setup, used ONLY to validate gold coherence. + + This is **not** a runtime solver (that is CMB-b) and not a runtime decode path — it is the + ruler checking its own gold, so the oracle's ``solved`` / ``solver_refuses`` labels can + *meaningfully fail* (CLAUDE.md proof-obligation rule) rather than being asserted. Returns + ``("solved", value, None)`` or ``("refused", None, reason)`` with ``reason`` in + :data:`SOLVER_REASONS`. + + - ``effective_rate`` query: the net rate is well-defined even when ``<= 0`` (a net-draining + tank) — answered, never refused. + - ``quantity`` / ``time`` queries: a non-positive net rate cannot accumulate or finish -> + ``non_positive_net_rate`` (this also guards the ``eff == 0`` time query from dividing by 0). + - ``time`` query: an exact integer or ``non_integer_solution`` (never rounds). ``quantity`` + queries are ``eff * time`` and are always integral, so they never refuse for non-integrality. + """ + eff = problem.effective_rate + if problem.query == "effective_rate": + return "solved", eff, None + if eff <= 0: + return "refused", None, "non_positive_net_rate" + if problem.query == "quantity": + assert problem.time is not None # guaranteed by the model's per-query slot guard + return "solved", eff * problem.time, None + # query == "time": quantity is the known (model guard); exact integer or refuse. + assert problem.quantity is not None + if problem.quantity % eff != 0: + return "refused", None, "non_integer_solution" + return "solved", problem.quantity // eff, None + + +def validate_fixture(fx: dict[str, Any]) -> tuple[str, str | None]: + """Validate one gold fixture's coherence. Returns ``(outcome, reason)``.""" + expect = fx.get("expect") + if expect not in EXPECTATIONS: + return "invalid", f"unknown_expect:{expect!r}" + + if expect == "reader_refuses": + if fx.get("reader_reason") not in READER_REASONS: + return "invalid", f"unknown_reader_reason:{fx.get('reader_reason')!r}" + if fx.get("gold") is not None: + return "invalid", "reader_refuses_has_gold" + return "valid", None + + # solved | solver_refuses require a well-formed combined-rate setup. (Signature determinism is + # proven by the model being a pure function over a frozen dataclass — see the oracle tests — + # not by a runtime self-comparison.) + try: + problem = gold_to_problem(fx) + except (KeyError, TypeError, ValueError, UnitError) as exc: + return "invalid", f"malformed_setup:{exc}" + kind, value, reason = _canonical_outcome(problem) + + if expect == "solver_refuses": + if fx.get("solver_reason") not in SOLVER_REASONS: + return "invalid", f"unknown_solver_reason:{fx.get('solver_reason')!r}" + if fx.get("gold") is not None: + return "invalid", "solver_refuses_has_gold" + # The label must be the GENUINE reason this setup refuses — not asserted. A positive, + # exactly-solvable setup mislabeled solver_refuses, or the wrong refusal reason, is invalid. + if kind != "refused": + return "invalid", "solver_refuses_is_actually_solvable" + if reason != fx["solver_reason"]: + return "invalid", f"solver_reason_mismatch:expected_{reason}" + return "valid", None + + # solved: integer gold + coherent multiple-choice key + gold == the canonical computed answer. + gold = fx.get("gold") + if not isinstance(gold, int) or isinstance(gold, bool): + return "invalid", "solved_needs_int_gold" + options, answer = fx.get("options"), fx.get("answer") + if not isinstance(options, dict) or answer not in options: + return "invalid", "missing_or_unlabeled_answer" + if options[answer] != gold: + return "invalid", "answer_key_incoherent" + if kind != "solved": + return "invalid", "solved_is_not_canonically_solvable" + if value != gold: + return "invalid", "gold_does_not_match_computed_answer" + return "valid", None + + +def run() -> dict[str, Any]: + """Validate every combined-rate gold fixture. Exit-0 criterion: ``invalid == 0``.""" + fixtures = _load_combined_rate_gold() + valid = invalid = 0 + by_expect: dict[str, int] = {} + details: list[dict[str, Any]] = [] + for fx in fixtures: + outcome, reason = validate_fixture(fx) + expect = fx.get("expect", "?") + by_expect[expect] = by_expect.get(expect, 0) + 1 + if outcome == "valid": + valid += 1 + details.append({"id": fx.get("id"), "outcome": "valid", "expect": expect}) + else: + invalid += 1 + details.append({"id": fx.get("id"), "outcome": "invalid", "reason": reason}) + return { + "lane": "combined_rate_oracle_gold_validation", + "total": len(fixtures), + "valid": valid, + "invalid": invalid, + "by_expect": by_expect, + "details": details, + } + + +__all__ = [ + "COMBINE_MODES", + "EXPECTATIONS", + "QUERIES", + "READER_REASONS", + "SOLVER_REASONS", + "_canonical_outcome", + "_load_combined_rate_gold", + "gold_to_problem", + "run", + "validate_fixture", +] diff --git a/evals/combined_rate_oracle/signature.py b/evals/combined_rate_oracle/signature.py new file mode 100644 index 00000000..932c4d69 --- /dev/null +++ b/evals/combined_rate_oracle/signature.py @@ -0,0 +1,45 @@ +"""Span-free canonical signature for the combined-rate setup oracle (CMB-a). + +A deterministic, order-independent projection of a combined-rate setup — the combine mode, the +rate unit, the two rates, the known values keyed by role, and the query — used to compare a +reader's comprehended ``CombinedRateProblem`` against the independent gold. The CMB twin of +``evals.rate_oracle.signature``. + +Canonical form respects the algebra: ``sum`` is commutative, so its two rates are **sorted** ("3 +and 2 together" and "2 and 3 together" are the same thought); ``difference`` is **not** +commutative (which rate is the drain matters), so its rates keep ``(rate_a, rate_b)`` order. Pure, +deterministic. +""" + +from __future__ import annotations + +from typing import Any + +from generate.combined_rate_comprehension.model import CombinedRateProblem + + +def combined_rate_setup_signature(problem: CombinedRateProblem) -> dict[str, Any]: + """Canonical ``(combine_mode, rate_unit, rates, knowns, query)`` signature.""" + rates: tuple[int, int] = (problem.rate_a, problem.rate_b) + if problem.combine_mode == "sum": + rates = tuple(sorted(rates)) # type: ignore[assignment] # commutative -> canonicalize order + knowns = tuple( + sorted( + (role, value) + for role, value in (("time", problem.time), ("quantity", problem.quantity)) + if value is not None + ) + ) + return { + "combine_mode": problem.combine_mode, + "rate_unit": (problem.rate_unit.numerator, problem.rate_unit.denominator), + "rates": rates, + "knowns": knowns, + "query": problem.query, + # The duration's unit is part of the setup (forward-compat with conversion); in v1 it is + # always the rate denominator, but the signature carries it for parity with R3. + "time_unit": problem.time_unit, + } + + +__all__ = ["combined_rate_setup_signature"] diff --git a/generate/combined_rate_comprehension/__init__.py b/generate/combined_rate_comprehension/__init__.py new file mode 100644 index 00000000..24f883af --- /dev/null +++ b/generate/combined_rate_comprehension/__init__.py @@ -0,0 +1,23 @@ +"""Combined-rate comprehension organ (off-serving) — two explicit rates over one shared unit. + +The semantic object R3 (single-rate) does NOT have: an **effective rate** formed by combining two +explicit rates that share a compound unit, under an explicit ``combine_mode``: + +```text +effective_rate = rate_a + rate_b (sum — cooperation: "working together") +effective_rate = rate_a - rate_b (difference — opposing flow: "fills ... drains") +``` + +then the single-rate algebra over that effective rate (``quantity = effective_rate × time``). + +A separate organ, deliberately NOT folded into ``rate_comprehension``: combined rate is not +single-rate with a bigger parser — it introduces a new model and ruler. If the rate organs later +converge, a shared rate algebra can be extracted then. Off-serving: imports no +``generate.derivation`` / ``core.reliability_gate``, so it cannot move the sealed GSM8K serving +metric. Deterministic; no clock, no randomness. + +CMB-a (this slice) defines the model + the independent gold ruler + the setup oracle only. It +claims exactly one thing: *the combined-rate setup ruler is defined and gold-valid.* No reader +(CMB-c), no solver (CMB-b), no router/contemplation wiring (CMB-d) — and therefore no capability +claim yet. +""" diff --git a/generate/combined_rate_comprehension/model.py b/generate/combined_rate_comprehension/model.py new file mode 100644 index 00000000..461d3e0f --- /dev/null +++ b/generate/combined_rate_comprehension/model.py @@ -0,0 +1,105 @@ +"""Typed model for a combined-rate problem (CMB-a). + +A combined-rate problem is two explicit rates over one shared unit, combined by an explicit mode, +then single-rate algebra over the result: + +```text +effective_rate = rate_a + rate_b (combine_mode == "sum") +effective_rate = rate_a - rate_b (combine_mode == "difference") +quantity = effective_rate × time (query == "quantity") +time = quantity ÷ effective_rate (query == "time") +effective_rate (query == "effective_rate") +``` + +The two rates are **always known** (that is what makes it a combined-rate problem — two explicit +rates); ``rate_unit`` is the single source of unit truth for both. The query selects which derived +slot is asked: + +- ``quantity`` — ``time`` is the known, ``quantity`` is the unknown (``None``). +- ``time`` — ``quantity`` is the known, ``time`` is the unknown (``None``). +- ``effective_rate`` — neither ``time`` nor ``quantity`` is needed; both are ``None``. + +Pure data with a structural guard: the two rates must be present, and exactly the slots the query +licenses are known/unknown (illegal states — a missing rate, the wrong slot unknown, an +over-specified ``effective_rate`` query — cannot be represented). + +``effective_rate`` is derived (a property), and for ``difference`` mode it MAY be ``<= 0`` +(``rate_a <= rate_b``). The model does NOT refuse that — a non-positive net rate is the *solver's* +boundary (``non_positive_net_rate``, CMB-b), not a malformed setup. Off-serving; deterministic. +No unit conversion in v1 (``time_unit`` defaults to the rate denominator and the v1 gold never +crosses units). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +from generate.combined_rate_comprehension.units import RateUnit + +CombineMode = Literal["sum", "difference"] +CombinedRateQuery = Literal["quantity", "time", "effective_rate"] + +#: The slots each query licenses as (known, unknown). ``effective_rate`` needs neither time nor +#: quantity; the other two trade exactly one known for one unknown. +_QUERY_SLOTS: dict[str, tuple[frozenset[str], frozenset[str]]] = { + "quantity": (frozenset({"time"}), frozenset({"quantity"})), + "time": (frozenset({"quantity"}), frozenset({"time"})), + "effective_rate": (frozenset(), frozenset({"time", "quantity"})), +} + + +@dataclass(frozen=True, slots=True) +class CombinedRateProblem: + """Two explicit rates over one unit, combined by ``combine_mode``, with one queried slot.""" + + rate_a: int + rate_b: int + rate_unit: RateUnit + combine_mode: CombineMode + time: int | None + quantity: int | None + query: CombinedRateQuery + #: The duration's time unit (forward-compat with conversion). Defaults to the rate denominator; + #: v1 never crosses units, so it always equals ``rate_unit.denominator`` here. + time_unit: str | None = None + + def __post_init__(self) -> None: + if self.time_unit is None: + object.__setattr__(self, "time_unit", self.rate_unit.denominator) + for role, value in (("rate_a", self.rate_a), ("rate_b", self.rate_b)): + if not isinstance(value, int) or isinstance(value, bool): + raise ValueError(f"{role} must be a known int (two explicit rates); got {value!r}") + if self.combine_mode not in ("sum", "difference"): + raise ValueError(f"combine_mode must be 'sum' or 'difference'; got {self.combine_mode!r}") + if self.query not in _QUERY_SLOTS: + raise ValueError(f"query must be one of {sorted(_QUERY_SLOTS)}; got {self.query!r}") + known_slots, unknown_slots = _QUERY_SLOTS[self.query] + slots: dict[str, int | None] = {"time": self.time, "quantity": self.quantity} + known = {role for role, value in slots.items() if value is not None} + if known != known_slots: + raise ValueError( + f"query={self.query!r} licenses knowns {sorted(known_slots)}; got {sorted(known)}" + ) + for role in unknown_slots: + if slots[role] is not None: + raise ValueError(f"query={self.query!r}: slot {role!r} must be the unknown (None)") + for role in known_slots: + value = slots[role] + if not isinstance(value, int) or isinstance(value, bool): + raise ValueError(f"{role} value must be int; got {value!r}") + + @property + def effective_rate(self) -> int: + """The combined rate (pure derivation). MAY be ``<= 0`` for ``difference``; the solver, + not the model, owns the ``non_positive_net_rate`` refusal.""" + if self.combine_mode == "sum": + return self.rate_a + self.rate_b + return self.rate_a - self.rate_b + + @property + def quantity_unit(self) -> str: + return self.rate_unit.numerator + + +__all__ = ["CombineMode", "CombinedRateProblem", "CombinedRateQuery"] diff --git a/generate/combined_rate_comprehension/units.py b/generate/combined_rate_comprehension/units.py new file mode 100644 index 00000000..430aff2b --- /dev/null +++ b/generate/combined_rate_comprehension/units.py @@ -0,0 +1,41 @@ +"""Compound-unit value type for the combined-rate organ (CMB-a). + +A combined-rate problem carries two rates over ONE shared compound unit (``rooms/hour``, +``liters/minute``): both contributors are measured the same way, so the model holds a single +``RateUnit``. Two rates with *different* units do not compose — that is a reader refusal +(``rate_unit_mismatch``), representable only because there is one unit slot, not two. + +Deliberately a **local** copy of the single-rate organ's ``RateUnit`` rather than an import from +``generate.rate_comprehension`` — the two rate organs are kept disjoint (CMB-a does not depend on +R3) until a shared rate algebra is extracted. The duplication is intentional and tiny; if/when the +rate organs converge (a later slice), promote this to a shared ``rate_algebra`` module and have +both import it. No unit conversion in CMB v1 (mirrors R3's v1 boundary). Deterministic. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +class UnitError(ValueError): + """A malformed or non-composing unit — refuse, never fabricate a unit.""" + + +@dataclass(frozen=True, slots=True) +class RateUnit: + """A compound rate unit ``numerator / denominator`` — ``room/hour``, ``liter/minute``.""" + + numerator: str + denominator: str + + def __post_init__(self) -> None: + if not self.numerator or not self.denominator: + raise UnitError(f"RateUnit needs non-empty units; got {self!r}") + if self.numerator == self.denominator: + raise UnitError(f"a rate's numerator and denominator must differ; got {self}") + + def __str__(self) -> str: + return f"{self.numerator}/{self.denominator}" + + +__all__ = ["RateUnit", "UnitError"] diff --git a/tests/test_combined_rate_oracle.py b/tests/test_combined_rate_oracle.py new file mode 100644 index 00000000..839157e6 --- /dev/null +++ b/tests/test_combined_rate_oracle.py @@ -0,0 +1,280 @@ +"""Tests for the combined-rate CombinedRateProblem model + setup oracle (CMB-a). + +Pins the model's two-explicit-rates + per-query-slot guards, the sum/difference effective-rate +derivation (and that a non-positive net rate is a SOLVER concern, not a malformed setup), the +canonical signature (sum commutative, difference ordered), and that the gold ruler is coherent + +non-vacuous (each validator branch fires under its violation). No reader yet (CMB-c). +""" + +from __future__ import annotations + +import copy +from pathlib import Path + +import pytest + +from evals.combined_rate_oracle.runner import ( + COMBINE_MODES, + QUERIES, + READER_REASONS, + SOLVER_REASONS, + _load_combined_rate_gold, + gold_to_problem, + run, + validate_fixture, +) +from evals.combined_rate_oracle.signature import combined_rate_setup_signature +from generate.combined_rate_comprehension.model import CombinedRateProblem +from generate.combined_rate_comprehension.units import RateUnit + +_ROOM_HOUR = RateUnit("room", "hour") + + +def _of(expect: str) -> dict: + return copy.deepcopy(next(f for f in _load_combined_rate_gold() if f["expect"] == expect)) + + +def _solved() -> dict: + return copy.deepcopy(next(f for f in _load_combined_rate_gold() if f["expect"] == "solved")) + + +# --- gold validity --------------------------------------------------------------------- # + + +def test_run_validates_all_gold() -> None: + r = run() + assert r["invalid"] == 0 and r["valid"] == r["total"] == 17 + assert r["by_expect"] == {"solved": 6, "solver_refuses": 4, "reader_refuses": 7} + + +def test_solved_grid_covers_every_mode_query_cell() -> None: + # The six solved fixtures are exactly the full combine_mode x query grid (2 x 3). + solved = [f for f in _load_combined_rate_gold() if f["expect"] == "solved"] + cells = {(f["combine_mode"], f["query"]) for f in solved} + assert cells == {(m, q) for m in COMBINE_MODES for q in QUERIES} + + +def test_gold_to_problem_roundtrips() -> None: + p = gold_to_problem(next(f for f in _load_combined_rate_gold() if f["id"] == "cmb-01-paint-sum-quantity")) + assert (p.rate_a, p.rate_b, p.combine_mode) == (3, 2, "sum") + assert p.time == 4 and p.quantity is None and p.query == "quantity" + assert (p.quantity_unit, p.time_unit) == ("room", "hour") + assert p.effective_rate == 5 + + +def test_reasons_and_modes_in_gold_are_closed() -> None: + for fx in _load_combined_rate_gold(): + if fx["expect"] == "solver_refuses": + assert fx["solver_reason"] in SOLVER_REASONS + elif fx["expect"] == "reader_refuses": + assert fx["reader_reason"] in READER_REASONS + else: # solved + assert fx["combine_mode"] in COMBINE_MODES and fx["query"] in QUERIES + + +# --- model guards ---------------------------------------------------------------------- # + + +def test_model_requires_two_explicit_rates() -> None: + with pytest.raises(ValueError): + CombinedRateProblem(None, 2, _ROOM_HOUR, "sum", 4, None, "quantity") # type: ignore[arg-type] + with pytest.raises(ValueError): + CombinedRateProblem(3, None, _ROOM_HOUR, "sum", 4, None, "quantity") # type: ignore[arg-type] + + +def test_model_query_slot_guard() -> None: + # query=quantity: quantity must be the unknown, time the known. + with pytest.raises(ValueError): # zero unknowns (quantity also given) + CombinedRateProblem(3, 2, _ROOM_HOUR, "sum", 4, 20, "quantity") + with pytest.raises(ValueError): # two unknowns (time also missing) + CombinedRateProblem(3, 2, _ROOM_HOUR, "sum", None, None, "quantity") + # query=time: time must be the unknown, quantity the known. + with pytest.raises(ValueError): + CombinedRateProblem(3, 2, _ROOM_HOUR, "sum", 4, None, "time") + # query=effective_rate: neither time nor quantity may be present (over-specified). + with pytest.raises(ValueError): + CombinedRateProblem(3, 2, _ROOM_HOUR, "sum", 4, None, "effective_rate") + + +def test_model_rejects_unknown_combine_mode_and_query() -> None: + with pytest.raises(ValueError): + CombinedRateProblem(3, 2, _ROOM_HOUR, "average", 4, None, "quantity") # type: ignore[arg-type] + with pytest.raises(ValueError): + CombinedRateProblem(3, 2, _ROOM_HOUR, "sum", 4, None, "rate") # type: ignore[arg-type] + + +def test_effective_rate_derivation() -> None: + assert CombinedRateProblem(3, 2, _ROOM_HOUR, "sum", 4, None, "quantity").effective_rate == 5 + assert CombinedRateProblem(9, 4, _ROOM_HOUR, "difference", 4, None, "quantity").effective_rate == 5 + + +def test_non_positive_net_rate_is_a_well_formed_setup_not_a_model_error() -> None: + # 4 - 4 = 0: the model MUST construct it (the solver, not the model, refuses non_positive_net_rate). + p = CombinedRateProblem(4, 4, _ROOM_HOUR, "difference", 5, None, "quantity") + assert p.effective_rate == 0 + + +# --- signature ------------------------------------------------------------------------- # + + +def test_signature_is_deterministic_and_keyed() -> None: + p = gold_to_problem(next(f for f in _load_combined_rate_gold() if f["id"] == "cmb-05-tank-difference-rate")) + sig = combined_rate_setup_signature(p) + assert sig == combined_rate_setup_signature(p) + assert sig["query"] == "effective_rate" and sig["combine_mode"] == "difference" + assert sig["rates"] == (9, 4) and sig["rate_unit"] == ("liter", "minute") + + +def test_signature_sum_is_commutative_but_difference_is_ordered() -> None: + sum_ab = combined_rate_setup_signature(CombinedRateProblem(3, 2, _ROOM_HOUR, "sum", 4, None, "quantity")) + sum_ba = combined_rate_setup_signature(CombinedRateProblem(2, 3, _ROOM_HOUR, "sum", 4, None, "quantity")) + assert sum_ab == sum_ba and sum_ab["rates"] == (2, 3) # sorted + dif_ab = combined_rate_setup_signature(CombinedRateProblem(5, 2, _ROOM_HOUR, "difference", 6, None, "quantity")) + dif_ba = combined_rate_setup_signature(CombinedRateProblem(2, 5, _ROOM_HOUR, "difference", 6, None, "quantity")) + assert dif_ab != dif_ba and dif_ab["rates"] == (5, 2) # order preserved (which is the drain matters) + + +# --- meaningful-fail: each invalid branch fires under exactly its violation ------------- # + + +def test_validator_rejects_incoherent_answer_key() -> None: + fx = _solved() + fx["answer"] = next(k for k in fx["options"] if fx["options"][k] != fx["gold"]) + assert validate_fixture(fx) == ("invalid", "answer_key_incoherent") + + +def test_validator_rejects_reader_refuse_carrying_gold() -> None: + fx = _of("reader_refuses") + fx["gold"] = 5 + assert validate_fixture(fx) == ("invalid", "reader_refuses_has_gold") + + +def test_validator_rejects_solver_refuse_carrying_gold() -> None: + fx = _of("solver_refuses") + fx["gold"] = 5 + assert validate_fixture(fx) == ("invalid", "solver_refuses_has_gold") + + +def test_validator_rejects_unknown_reasons() -> None: + # Pin the FULL tuple (CLAUDE.md proof-obligation): a substitution returning a different invalid + # reason must not pass silently. + rr = _of("reader_refuses") + rr["reader_reason"] = "made_up" + assert validate_fixture(rr) == ("invalid", "unknown_reader_reason:'made_up'") + sr = _of("solver_refuses") + sr["solver_reason"] = "made_up" + assert validate_fixture(sr) == ("invalid", "unknown_solver_reason:'made_up'") + + +def test_validator_rejects_unknown_expect() -> None: + assert validate_fixture({"expect": "bogus"}) == ("invalid", "unknown_expect:'bogus'") + + +def test_validator_rejects_missing_or_unlabeled_answer() -> None: + fx = _solved() + fx["answer"] = "Z" # label not in options + assert validate_fixture(fx) == ("invalid", "missing_or_unlabeled_answer") + + +def test_validator_rejects_non_int_gold() -> None: + fx = _solved() + fx["gold"] = "twenty" + assert validate_fixture(fx) == ("invalid", "solved_needs_int_gold") + + +# --- the canonical-outcome cross-check (H1/H2/G5): label/gold must match the arithmetic --------- # + + +def _solver_refuse_fixture(**over: object) -> dict: + """A well-formed solver_refuses-shaped fixture (default: cmb-07b-like, eff=-3 -> refuses).""" + base = { + "id": "t", "expect": "solver_refuses", "solver_reason": "non_positive_net_rate", + "rate_unit": {"numerator": "liter", "denominator": "minute"}, + "rate_a": 2, "rate_b": 5, "combine_mode": "difference", + "time": 3, "time_unit": "minute", "quantity": None, "query": "quantity", "gold": None, + } + base.update(over) + return base + + +def test_validator_rejects_solver_refuse_that_is_actually_solvable() -> None: + # H1: positive net rate (5-2=3) mislabeled non_positive_net_rate. + h1 = _solver_refuse_fixture(rate_a=5, rate_b=2) + assert validate_fixture(h1) == ("invalid", "solver_refuses_is_actually_solvable") + # H2: a quantity query (eff*time is always integral) mislabeled non_integer_solution. + h2 = _solver_refuse_fixture( + combine_mode="sum", rate_a=3, rate_b=2, solver_reason="non_integer_solution" + ) + assert validate_fixture(h2) == ("invalid", "solver_refuses_is_actually_solvable") + + +def test_validator_rejects_solver_reason_mismatch() -> None: + # Genuinely refuses, but for the WRONG reason: eff=-3 on a time query is non_positive, not non_integer. + fx = _solver_refuse_fixture(query="time", time=None, quantity=12, solver_reason="non_integer_solution") + assert validate_fixture(fx) == ("invalid", "solver_reason_mismatch:expected_non_positive_net_rate") + + +def test_validator_rejects_wrong_gold_even_with_coherent_answer_key() -> None: + # G5: gold is internally answer-key-coherent but does not equal the canonical computed answer. + fx = _solved() # cmb-01: (3+2)*4 = 20 + fx["gold"] = 21 + fx["options"] = {**fx["options"], fx["answer"]: 21} # keep options[answer] == gold + assert validate_fixture(fx) == ("invalid", "gold_does_not_match_computed_answer") + + +def test_canonical_outcome_matrix() -> None: + from evals.combined_rate_oracle.runner import _canonical_outcome + + def cro(ra, rb, mode, time, qty, query): + return _canonical_outcome(CombinedRateProblem(ra, rb, _ROOM_HOUR, mode, time, qty, query)) + + assert cro(3, 2, "sum", 4, None, "quantity") == ("solved", 20, None) + assert cro(5, 2, "difference", 6, None, "quantity") == ("solved", 18, None) + assert cro(3, 2, "sum", None, 20, "time") == ("solved", 4, None) + assert cro(5, 2, "difference", None, 18, "time") == ("solved", 6, None) + assert cro(9, 4, "difference", None, None, "effective_rate") == ("solved", 5, None) + assert cro(6, 4, "sum", None, None, "effective_rate") == ("solved", 10, None) + # refusals + assert cro(4, 4, "difference", 5, None, "quantity") == ("refused", None, "non_positive_net_rate") + assert cro(4, 4, "difference", None, 12, "time") == ("refused", None, "non_positive_net_rate") + assert cro(3, 2, "sum", None, 12, "time") == ("refused", None, "non_integer_solution") + # an effective_rate query is well-defined even when the net is non-positive (never refuses) + assert cro(4, 4, "difference", None, None, "effective_rate") == ("solved", 0, None) + + +def test_validator_rejects_malformed_setup_missing_rate() -> None: + fx = _solved() + fx["rate_b"] = None # only one explicit rate -> model __post_init__ raises + assert validate_fixture(fx)[0] == "invalid" + + +def test_validator_rejects_unknown_combine_mode_in_gold() -> None: + fx = _solved() + fx["combine_mode"] = "average" # not a real combine mode -> model raises -> malformed_setup + outcome, reason = validate_fixture(fx) + assert outcome == "invalid" and reason is not None and reason.startswith("malformed_setup") + + +# --- off-serving guarantee ------------------------------------------------------------- # + + +def test_combined_rate_organ_is_off_serving() -> None: + # Check actual IMPORTS (via AST), not raw text — the docstrings legitimately *name* the + # serving modules to assert disjointness, so a substring scan would false-positive. + import ast + + import evals.combined_rate_oracle as oracle_pkg + import generate.combined_rate_comprehension as organ_pkg + + forbidden = ("generate.derivation", "core.reliability_gate") + for pkg in (organ_pkg, oracle_pkg): + for py in Path(str(pkg.__file__)).resolve().parent.glob("*.py"): + for node in ast.walk(ast.parse(py.read_text(encoding="utf-8"))): + if isinstance(node, ast.Import): + names = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom): + names = [node.module or ""] + else: + continue + for name in names: + assert not any(name.startswith(t) for t in forbidden), f"{py.name}: imports {name}"