diff --git a/evals/combined_rate_oracle/__main__.py b/evals/combined_rate_oracle/__main__.py index 47c8dd68..2670eac8 100644 --- a/evals/combined_rate_oracle/__main__.py +++ b/evals/combined_rate_oracle/__main__.py @@ -1,9 +1,9 @@ -"""CLI: validate the combined-rate gold ruler, or grade the CMB solver against it. +"""CLI: validate the combined-rate gold ruler, or grade the CMB solver / reader against it. python -m evals.combined_rate_oracle # validate combined_rate_gold.jsonl; exit 0 iff invalid == 0 python -m evals.combined_rate_oracle solver # grade the solver (CMB-b); exit 0 iff no wrong - -The reader grading lane (``reader`` arg) lands with the reader (CMB-c). + python -m evals.combined_rate_oracle reader # grade the reader (CMB-c); exit 0 iff setup_wrong == 0 + # and reason_mismatch == 0 """ from __future__ import annotations @@ -11,7 +11,7 @@ from __future__ import annotations import json import sys -from evals.combined_rate_oracle.runner import run, run_solver +from evals.combined_rate_oracle.runner import run, run_reader, run_solver def main() -> int: @@ -19,6 +19,9 @@ def main() -> int: if lane == "solver": report = run_solver() ok = report["solved_wrong"] == 0 and report["refuse_wrong"] == 0 + elif lane == "reader": + report = run_reader() + ok = report["setup_wrong"] == 0 and report["reason_mismatch"] == 0 else: report = run() ok = report["invalid"] == 0 diff --git a/evals/combined_rate_oracle/runner.py b/evals/combined_rate_oracle/runner.py index ce44f1fe..179d43a0 100644 --- a/evals/combined_rate_oracle/runner.py +++ b/evals/combined_rate_oracle/runner.py @@ -237,6 +237,58 @@ def run_solver() -> dict[str, Any]: } +def run_reader() -> dict[str, Any]: + """Grade the CMB reader against the gold (CMB-c). + + Well-formed fixtures (``solved`` / ``solver_refuses``) must read to a setup whose signature + equals the gold's (``setup_correct``); a refusal is a miss (``setup_refused``); a mismatch is + ``setup_wrong``. ``reader_refuses`` fixtures must refuse with the gold ``reader_reason`` + (``refused_correct``); a refusal with the wrong reason is ``reason_mismatch``; producing a setup + is ``setup_wrong`` (over-read). Exit-0 criterion: ``setup_wrong == 0 and reason_mismatch == 0``. + """ + from generate.combined_rate_comprehension.reader import read_combined_rate_problem + from generate.meaning_graph.reader import Refusal + + from evals.combined_rate_oracle.signature import combined_rate_setup_signature + + fixtures = _load_combined_rate_gold() + setup_correct = setup_wrong = setup_refused = refused_correct = reason_mismatch = 0 + details: list[dict[str, Any]] = [] + for fx in fixtures: + out = read_combined_rate_problem(fx["text"]) + fid = fx.get("id") + if fx["expect"] in ("solved", "solver_refuses"): + if isinstance(out, Refusal): + setup_refused += 1 + details.append({"id": fid, "outcome": "setup_refused", "reason": out.reason}) + elif combined_rate_setup_signature(out) == combined_rate_setup_signature(gold_to_problem(fx)): + setup_correct += 1 + details.append({"id": fid, "outcome": "setup_correct"}) + else: + setup_wrong += 1 + details.append({"id": fid, "outcome": "setup_WRONG"}) + else: # reader_refuses + if isinstance(out, Refusal) and out.reason == fx["reader_reason"]: + refused_correct += 1 + details.append({"id": fid, "outcome": "refused_correct", "reason": out.reason}) + elif isinstance(out, Refusal): + reason_mismatch += 1 + details.append({"id": fid, "outcome": "reason_mismatch", "got": out.reason, "want": fx["reader_reason"]}) + else: + setup_wrong += 1 + details.append({"id": fid, "outcome": "setup_WRONG_over_read"}) + return { + "lane": "combined_rate_oracle_reader", + "total": len(fixtures), + "setup_correct": setup_correct, + "setup_wrong": setup_wrong, + "setup_refused": setup_refused, + "refused_correct": refused_correct, + "reason_mismatch": reason_mismatch, + "details": details, + } + + __all__ = [ "COMBINE_MODES", "EXPECTATIONS", @@ -247,6 +299,7 @@ __all__ = [ "_load_combined_rate_gold", "gold_to_problem", "run", + "run_reader", "run_solver", "validate_fixture", ] diff --git a/generate/combined_rate_comprehension/reader.py b/generate/combined_rate_comprehension/reader.py new file mode 100644 index 00000000..aa7f0cb2 --- /dev/null +++ b/generate/combined_rate_comprehension/reader.py @@ -0,0 +1,214 @@ +"""Combined-rate prose reader (CMB-c): explicit two-rate prose -> CombinedRateProblem. + +Reads ONLY explicit combined-rate problems — two rates over one shared unit, combined by an +explicit cooperative (``sum``) or opposing-flow (``difference``) cue — and refuses everything else +with the closed CMB reader taxonomy. It does NOT solve: a non-positive net rate or a non-integer +answer is the *solver's* boundary (CMB-b), reached only after this reader produces a setup. + +The wrong=0 spine is the CMB-a **2×2 domain-entry grid** (rate-count × combination cue): + +```text + | combination cue | no cue + two rates | parse (sum/difference) | combine_mode_ambiguous (same unit only) + one rate | missing_second_rate | not_combined_rate_shaped <- step aside (R3 territory) +``` + +plus "combined-shaped but deferred / malformed" refusals that fire ONLY once a genuine two-rate +combination cue is present, so CMB never claims a substantive boundary on R1/R2/R3 text: +``three_or_more_rates``, ``rate_unit_mismatch`` (the two rates differ in unit), +``clock_interval_deferred``, ``reciprocal_work_rate_deferred``. + +**Robustness (each guards a wrong=0 / hygiene failure mode an adversarial pass found):** + - cues are **whole-word** regexes, never substrings (``exempt`` is not a drain; ``bothered`` is + not ``both``); + - ``difference`` requires an explicit **fill AND drain pair**, and assigns ``rate_a`` = fill + (minuend) / ``rate_b`` = drain (subtrahend) **by role**, not by text order (a drain listed + first must still subtract); + - the duration and the time-query target are read from the **query clause** (after the last rate + clause), so a preamble number ("50 liters already in the tank …") is never mistaken for them; + - the rate regex rejects **decimals** (``3.5 pages per hour`` is not an integer rate -> step aside); + - substantive refusals are **gated behind a combination cue** — foreign two-/three-rate prose with + no cue steps aside as ``not_combined_rate_shaped`` (router-organ hygiene), never a substantive + boundary; and **sequential segments** (each rate carrying its own adjacent duration) step aside. + +Off-serving (imports no ``generate.derivation`` / ``core.reliability_gate``); deterministic. +""" + +from __future__ import annotations + +import re + +from generate.combined_rate_comprehension.model import CombinedRateProblem +from generate.combined_rate_comprehension.units import RateUnit +from generate.meaning_graph.reader import Refusal + +# An integer rate `` per ``; the lookbehind rejects a digit that is part of a +# decimal ("3.5 pages per hour" -> no match -> the reader steps aside rather than read "5"). +_RATE_VALUE = re.compile(r"(? str: + if noun.endswith("es") and noun[:-2].endswith(("x", "s", "z", "ch", "sh")): + return noun[:-2] + if noun.endswith("s") and len(noun) > 1: + return noun[:-1] + return noun + + +def _clause_role(t: str, start: int) -> str: + """``drain`` / ``fill`` / ``other`` for the rate clause at *start*, by the verb directly before + its number (within ``_LEADIN`` chars). Drain wins ties — a drain verb is the load-bearing sign.""" + leadin = t[max(0, start - _LEADIN) : start] + if _DRAIN_RE.search(leadin): + return "drain" + if _FILL_RE.search(leadin): + return "fill" + return "other" + + +def _is_sequential(t: str, rate_spans: list[tuple[int, int]]) -> bool: + """True iff BOTH rate clauses carry their own following duration (sequential segments — "60 mph + for 2 hours and then 40 mph for 3 hours" — which is R3.x, not a simultaneous combination). The + window tolerates a short connector ("also for", "then for").""" + return all(_DURATION.search(t[e : e + 28]) is not None for _s, e in rate_spans) + + +def _digit_noun_after(t: str, start: int, noun: str) -> int | None: + """The first `` `` value at/after *start* — anchored to AFTER the ``how many`` question, + so neither a premise number nor a transitional distractor inside the query clause is mistaken for + the queried quantity.""" + for m in _DIGIT_NOUN.finditer(t, start): + if _singular(m.group(2)) == noun: + return int(m.group(1)) + return None + + +def read_combined_rate_problem(text: str) -> CombinedRateProblem | Refusal: + """Comprehend explicit combined-rate prose into a typed CombinedRateProblem, or refuse.""" + if not text or not text.strip(): + return Refusal("empty") + t = text.lower() + + rate_clauses = _RATE_VALUE.findall(t) # [(value, plural, denom_singular), ...] + rate_spans = [m.span() for m in _RATE_VALUE.finditer(t)] + n = len(rate_clauses) + coop = _COOP_RE.search(t) is not None + n_durations = len(_DURATION.findall(t)) + + # --- the grid's rate-count axis ----------------------------------------------------------- # + if n == 0: + # No explicit "per" rate. A cooperative cue + ≥2 completion durations + a "can"-style + # capability is the reciprocal work-rate form (1/(1/a+1/b)); else not CMB's domain. + if coop and n_durations >= 2 and _RECIPROCAL_RE.search(t): + return Refusal("reciprocal_work_rate_deferred", "durations-to-complete, not explicit rates") + return Refusal("not_combined_rate_shaped", "no explicit rate clause") + if n >= 3: + # Substantive ONLY with a combination cue; otherwise foreign prose that merely has ≥3 rate + # patterns -> step aside (hygiene). + if coop or any(_clause_role(t, s) == "drain" for s, _e in rate_spans): + return Refusal("three_or_more_rates", f"{n} explicit rates; CMB v1 combines exactly two") + return Refusal("not_combined_rate_shaped", "three or more rates but no combination cue") + if n == 1: + # One rate is substantive (missing a second) ONLY with a cooperative cue AND two conjoined + # agents in the premise; a lone rate (or an incidental "combined"/"both") is R3 territory. + if coop and any(m.start() < rate_spans[0][0] for m in _AGENT_CONJ.finditer(t)): + return Refusal("missing_second_rate", "a cooperation cue and two agents but only one explicit rate") + return Refusal("not_combined_rate_shaped", "a single rate with no two-agent combination cue") + + # --- n == 2 -------------------------------------------------------------------------------- # + u0 = (_singular(rate_clauses[0][1]), _singular(rate_clauses[0][2])) + u1 = (_singular(rate_clauses[1][1]), _singular(rate_clauses[1][2])) + if _is_sequential(t, rate_spans): + return Refusal("not_combined_rate_shaped", "two rates with their own durations are sequential segments") + + # The combine mode comes from the two clauses' OWN roles: a clean fill/drain opposition is the + # only difference; a cooperative cue is the only sum. Anything else has no usable cue. + roles = [_clause_role(t, s) for s, _e in rate_spans] + if sorted(roles) == ["drain", "fill"]: + mode = "difference" + elif coop: + mode = "sum" + else: + mode = None + if mode is None: + # combine_mode_ambiguous is substantive (CMB's domain), so it fires ONLY when the question + # actually asks to COMBINE the two same-unit rates — a total quantity/time, or the net/combined + # rate. Two same-unit rates merely COMPARED ("which is faster?"), or with a different unit, or + # with no combined query, are not a CMB problem -> step aside (router hygiene). + if u0 == u1: + hm = _HOW_MANY.search(t) + asked0 = _singular(hm.group(1)) if hm else None + if asked0 in u0 or _EFF_RATE_QUERY.search(t, rate_spans[-1][1]) is not None: + return Refusal("combine_mode_ambiguous", "two same-unit rates and a combined query but no mode cue") + return Refusal("not_combined_rate_shaped", "two rates that are not a clean combined-rate problem") + # A combination cue IS present — now substantive combined-rate refusals are legitimate. + if u0 != u1: + return Refusal("rate_unit_mismatch", f"two rates differ in unit: {u0} vs {u1}") + if _CLOCK.search(t): + return Refusal("clock_interval_deferred", "an elapsed clock interval is not an explicit duration") + + unit = RateUnit(*u0) + if mode == "difference": + # rate_a = fill (minuend), rate_b = drain (subtrahend), BY ROLE not by text order. + drain_idx = roles.index("drain") + rate_a, rate_b = ( + (int(rate_clauses[1][0]), int(rate_clauses[0][0])) + if drain_idx == 0 + else (int(rate_clauses[0][0]), int(rate_clauses[1][0])) + ) + else: + rate_a, rate_b = int(rate_clauses[0][0]), int(rate_clauses[1][0]) + + # --- query slot + assembly: read the queried slot from AFTER the "how many" question, so a + # premise number or a transitional time marker is never mistaken for it. --------------------- # + if _EFF_RATE_QUERY.search(t, rate_spans[-1][1]): + return CombinedRateProblem(rate_a, rate_b, unit, mode, None, None, "effective_rate") + + how_many = _HOW_MANY.search(t) + if how_many is None: + return Refusal("not_combined_rate_shaped", "no recognizable query slot") # defensive (no gold) + asked = _singular(how_many.group(1)) + q_start = how_many.end() + + if asked == unit.numerator: # quantity query — needs a duration in the rate's denominator unit + dur = _DURATION.search(t, q_start) + if dur is None: + return Refusal("not_combined_rate_shaped", "quantity query without a duration") # defensive + time_value, time_unit = int(dur.group(1)), _singular(dur.group(2)) + if time_unit != unit.denominator: + # CMB v1 crosses no units (R3.2 conversion is single-rate only) — refuse rather than + # silently treat a non-denominator duration as the rate's unit. + return Refusal("rate_unit_mismatch", f"duration {time_unit!r} != rate denominator {unit.denominator!r}") + return CombinedRateProblem(rate_a, rate_b, unit, mode, time_value, None, "quantity", time_unit=time_unit) + + if asked == unit.denominator: # time query — needs the target quantity (in the numerator unit) + qty = _digit_noun_after(t, q_start, unit.numerator) + if qty is None: + return Refusal("not_combined_rate_shaped", "time query without a target quantity") # defensive + return CombinedRateProblem(rate_a, rate_b, unit, mode, None, qty, "time") + + return Refusal("not_combined_rate_shaped", f"unrecognized query target {asked!r}") # defensive + + +__all__ = ["read_combined_rate_problem"] diff --git a/tests/test_combined_rate_reader.py b/tests/test_combined_rate_reader.py new file mode 100644 index 00000000..d19accc1 --- /dev/null +++ b/tests/test_combined_rate_reader.py @@ -0,0 +1,339 @@ +"""Tests for the combined-rate prose reader (CMB-c). + +Pins wrong=0 for the reader: every well-formed gold fixture reads to exactly the gold setup (and +solves end-to-end to its labeled answer via CMB-b); every reader_refuses fixture refuses with its +gold reason; the CMB-a 2x2 domain-entry grid (with fresh vocabulary, proving the reader is +structural not string-matched); the solver-boundary fixtures are PARSED then refused by the solver, +not the reader; and router-organ hygiene — CMB steps aside on every foreign R1/R2/R3 problem. +""" + +from __future__ import annotations + +from pathlib import Path + +from evals.combined_rate_oracle.runner import _load_combined_rate_gold, gold_to_problem, run_reader +from evals.combined_rate_oracle.signature import combined_rate_setup_signature +from evals.constraint_oracle.runner import _load_r2_gold +from evals.rate_oracle.runner import _load_rate_gold +from evals.setup_oracle.runner import _load_r1_gold +from generate.answer_choices.verify import ChoiceVerdict, verify_answer_choice +from generate.combined_rate_comprehension.model import CombinedRateProblem +from generate.combined_rate_comprehension.reader import read_combined_rate_problem +from generate.combined_rate_comprehension.solver import solve_combined_rate +from generate.meaning_graph.reader import Refusal + +_PAGE_HOUR = ("page", "hour") + + +def _by(expect: str) -> list[dict]: + return [f for f in _load_combined_rate_gold() if f["expect"] == expect] + + +# --- the reader lane + gold round-trip ---------------------------------------------------- # + + +def test_reader_lane_is_wrong_zero_and_complete() -> None: + r = run_reader() + assert r["setup_wrong"] == 0 and r["reason_mismatch"] == 0 + assert r["setup_refused"] == 0 + assert r["setup_correct"] == 11 # 6 solved + 5 solver_refuses (all parse) + assert r["refused_correct"] == 7 + + +def test_reads_every_well_formed_fixture_to_gold_signature() -> None: + for fx in _by("solved") + _by("solver_refuses"): + out = read_combined_rate_problem(fx["text"]) + assert not isinstance(out, Refusal), f"{fx['id']}: refused {getattr(out, 'reason', '')}" + assert combined_rate_setup_signature(out) == combined_rate_setup_signature(gold_to_problem(fx)), fx["id"] + + +def test_refuses_every_reader_refuse_fixture_with_reason() -> None: + for fx in _by("reader_refuses"): + out = read_combined_rate_problem(fx["text"]) + assert isinstance(out, Refusal) and out.reason == fx["reader_reason"], fx["id"] + + +def test_read_solve_verify_end_to_end_for_solved() -> None: + for fx in _by("solved"): + problem = read_combined_rate_problem(fx["text"]) + assert not isinstance(problem, Refusal), fx["id"] + value = solve_combined_rate(problem) + assert value == fx["gold"], fx["id"] + verdict = verify_answer_choice(value, fx["options"], fx["answer"]) + assert isinstance(verdict, ChoiceVerdict) and verdict.status == "consistent" + assert verdict.computed_label == fx["answer"], fx["id"] + + +def test_solver_boundary_fixtures_are_parsed_by_reader_then_refused_by_solver() -> None: + # The division of labor: the reader OWNS the setup (non-positive net / non-integer are valid + # combined-rate setups), the solver OWNS solvability. The reader must NOT refuse these. + for fx in _by("solver_refuses"): + problem = read_combined_rate_problem(fx["text"]) + assert not isinstance(problem, Refusal), f"{fx['id']}: reader wrongly refused a solver-boundary case" + out = solve_combined_rate(problem) + assert isinstance(out, Refusal) and out.reason == fx["solver_reason"], fx["id"] + + +# --- the 2x2 domain-entry grid, with FRESH vocabulary (structural, not gold-string-matched) ---- # + + +def test_grid_two_rates_plus_cooperative_cue_parses_sum() -> None: + out = read_combined_rate_problem( + "Carla types 4 pages per hour and Dave types 3 pages per hour. Working together, how many pages in 5 hours?" + ) + assert isinstance(out, CombinedRateProblem) + assert out.combine_mode == "sum" and out.query == "quantity" + assert solve_combined_rate(out) == 35 # (4 + 3) * 5 + + +def test_grid_two_rates_plus_opposing_cue_parses_difference() -> None: + out = read_combined_rate_problem( + "A tap adds 7 liters per minute while a leak removes 2 liters per minute. How many liters after 4 minutes?" + ) + assert isinstance(out, CombinedRateProblem) + assert out.combine_mode == "difference" and out.query == "quantity" + assert solve_combined_rate(out) == 20 # (7 - 2) * 4 + + +def test_grid_two_rates_no_cue_is_combine_mode_ambiguous() -> None: + out = read_combined_rate_problem( + "Machine X makes 4 bolts per minute. Machine Y makes 6 bolts per minute. How many bolts in 3 minutes?" + ) + assert isinstance(out, Refusal) and out.reason == "combine_mode_ambiguous" + + +def test_grid_one_rate_plus_cue_is_missing_second_rate() -> None: + out = read_combined_rate_problem( + "Sam and Tess mow a lawn together. Sam mows 2 lawns per hour. How many lawns in 3 hours?" + ) + assert isinstance(out, Refusal) and out.reason == "missing_second_rate" + + +def test_grid_one_rate_no_cue_steps_aside() -> None: + out = read_combined_rate_problem("A printer prints 5 pages per minute for 4 minutes. How many pages does it print?") + assert isinstance(out, Refusal) and out.reason == "not_combined_rate_shaped" + + +# --- each remaining reader refusal reason, meaningful-fail (fresh text) -------------------- # + + +def test_rate_unit_mismatch() -> None: + out = read_combined_rate_problem( + "A clerk files 3 forms per hour and a pump moves 2 gallons per minute. Working together, how much in 4 hours?" + ) + assert isinstance(out, Refusal) and out.reason == "rate_unit_mismatch" + + +def test_three_or_more_rates() -> None: + out = read_combined_rate_problem( + "A types 3 pages per hour, B types 2 pages per hour, and C types 5 pages per hour. Together, how many pages in 2 hours?" + ) + assert isinstance(out, Refusal) and out.reason == "three_or_more_rates" + + +def test_reciprocal_work_rate_deferred() -> None: + out = read_combined_rate_problem( + "Maria can paint a fence in 4 hours, and Nina can paint the same fence in 6 hours. Working together, how many hours?" + ) + assert isinstance(out, Refusal) and out.reason == "reciprocal_work_rate_deferred" + + +def test_clock_interval_deferred() -> None: + out = read_combined_rate_problem( + "One hose fills at 3 liters per minute and another fills at 2 liters per minute, together from 1 pm to 4 pm. How many liters?" + ) + assert isinstance(out, Refusal) and out.reason == "clock_interval_deferred" + + +def test_sequential_segments_step_aside_not_ambiguous() -> None: + # Two rate clauses, each with its OWN duration -> sequential (R3.x), NOT a combination. Must + # step aside, never claim the substantive combine_mode_ambiguous (hygiene). + out = read_combined_rate_problem( + "A car goes 60 miles per hour for 2 hours and then 40 miles per hour for 3 hours. How many miles total?" + ) + assert isinstance(out, Refusal) and out.reason == "not_combined_rate_shaped" + + +# --- router-organ hygiene: CMB steps aside on every foreign R1/R2/R3 problem --------------- # + + +def test_steps_aside_on_all_foreign_r1_r2_r3_gold() -> None: + # The ONLY refusal reason CMB may use on foreign text is the input_shape-family + # not_combined_rate_shaped — never a substantive boundary (router-organ-hygiene invariant). + for load in (_load_r1_gold, _load_r2_gold, _load_rate_gold): + for fx in load(): + out = read_combined_rate_problem(fx["text"]) + assert isinstance(out, Refusal), f"{fx['id']}: CMB produced a setup on foreign text" + assert out.reason == "not_combined_rate_shaped", f"{fx['id']}: substantive reason {out.reason!r} on foreign text" + + +def test_does_not_steal_ordinary_single_rate_problems() -> None: + # Plain R3 single-rate prose (incl. the clock/temporal r3 case) is R3's; CMB must step aside. + for text in ( + "A car travels 60 miles per hour for 3 hours. How many miles does it travel?", + "A worker earns 15 dollars per hour for 8 hours. How many dollars does she earn?", + "A machine makes 12 widgets per minute. It runs for 5 minutes. How many widgets does it make?", + ): + out = read_combined_rate_problem(text) + assert isinstance(out, Refusal) and out.reason == "not_combined_rate_shaped", text + + +# --- regression tests for the adversarial-found defects (wrong=0 / hygiene) --------------- # + + +def test_difference_mode_is_role_based_not_positional() -> None: + # Drain listed FIRST must still subtract: 3 (fill) - 8 (drain) = -5 -> solver refuses, NOT a + # positive answer of (8-3)*5. Both orders must agree on the role assignment. + drain_first = read_combined_rate_problem( + "A drain removes 8 liters per minute while a pipe fills a tank at 3 liters per minute. How many liters after 5 minutes?" + ) + assert isinstance(drain_first, CombinedRateProblem) + assert drain_first.combine_mode == "difference" and drain_first.effective_rate == -5 + assert isinstance(solve_combined_rate(drain_first), Refusal) + fill_first = read_combined_rate_problem( + "A pump fills a tank at 9 liters per minute while a drain removes 3 liters per minute. How many liters after 5 minutes?" + ) + assert isinstance(fill_first, CombinedRateProblem) and solve_combined_rate(fill_first) == 30 + + +def test_lone_drain_word_is_cooperative_not_opposing() -> None: + # "drain the backlog" is a verb, not opposing flow; with no fill counterpart it is a cooperative + # (sum) problem, not difference. + out = read_combined_rate_problem( + "Both workers drain the backlog at 5 tasks per hour and 3 tasks per hour. How many tasks in 4 hours?" + ) + assert isinstance(out, CombinedRateProblem) and out.combine_mode == "sum" + assert solve_combined_rate(out) == 32 + + +def test_mid_sentence_combined_rate_does_not_steal_quantity_query() -> None: + out = read_combined_rate_problem( + "Machine A fills 6 tanks per hour and Machine B fills 4 tanks per hour. At their combined rate, how many tanks in 5 hours?" + ) + assert isinstance(out, CombinedRateProblem) and out.query == "quantity" + assert solve_combined_rate(out) == 50 + + +def test_preamble_number_is_not_the_query_target() -> None: + # The "in 3 hours / 15 rooms" preamble must not be read as the query duration/quantity. + out = read_combined_rate_problem( + "In 3 hours, Anna and Ben painted 15 rooms. Anna paints 3 rooms per hour and Ben paints 2 rooms per hour. " + "Working together, how many rooms do they paint in 4 hours?" + ) + assert isinstance(out, CombinedRateProblem) and out.time == 4 + assert solve_combined_rate(out) == 20 + + +def test_decimal_rates_step_aside() -> None: + out = read_combined_rate_problem( + "Carla types 3.5 pages per hour and Dave types 2.5 pages per hour. Working together, how many pages in 4 hours?" + ) + assert isinstance(out, Refusal) and out.reason == "not_combined_rate_shaped" + + +def test_incidental_combination_words_step_aside() -> None: + # 'combined'/'both'/'together' used as non-combination words on single-rate text must NOT + # over-claim missing_second_rate (hygiene) — step aside. + for text in ( + "Anna earns 3 dollars per hour. If they work together, how many dollars does Anna earn in 4 hours?", + "A machine produces 5 parts per hour. The combined output target is 25 parts. How many hours does it take?", + "A study entitled Combined Effects found workers process 4 tasks per hour. How many tasks in 8 hours?", + ): + out = read_combined_rate_problem(text) + assert isinstance(out, Refusal) and out.reason == "not_combined_rate_shaped", text + + +def test_foreign_multi_rate_text_steps_aside_not_substantive() -> None: + # Substantive refusals (rate_unit_mismatch / three_or_more_rates) must NOT fire on foreign prose + # that merely contains 2+ rate patterns with no combination cue (router-organ hygiene). + for text in ( + "The server handles 100 requests per second and costs 2 dollars per hour. How many requests in 10 seconds?", + "A types 3 pages per hour, B types 2 pages per hour, and C types 5 pages per hour. How many pages in 2 hours?", + ): + out = read_combined_rate_problem(text) + assert isinstance(out, Refusal) and out.reason == "not_combined_rate_shaped", text + + +def test_duration_unit_mismatch_refuses() -> None: + # A combined problem whose duration unit differs from the rate denominator: CMB v1 does not + # convert (that is R3.2, single-rate only) -> rate_unit_mismatch. Pins the line-144 exit. + out = read_combined_rate_problem( + "Pipe A fills 5 rooms per hour and Pipe B fills 3 rooms per hour. Working together, how many rooms in 30 minutes?" + ) + assert isinstance(out, Refusal) and out.reason == "rate_unit_mismatch" + + +# --- second-pass regression: incidental cues, distractor numbers, loose sequential ------- # + + +def test_incidental_drain_noun_does_not_force_difference() -> None: + # "the drain stays closed" / "drain from the tank" are not drain RATES — two fills cooperating + # must stay sum, never flip to difference. + a = read_combined_rate_problem( + "The pump fills 8 liters per minute and the hose fills 4 liters per minute together. The drain stays closed. How many liters in 3 minutes?" + ) + assert isinstance(a, CombinedRateProblem) and a.combine_mode == "sum" and solve_combined_rate(a) == 36 + b = read_combined_rate_problem( + "Pump A fills 10 liters per minute and pump B fills 6 liters per minute together. How many liters drain from the tank in 2 minutes?" + ) + assert isinstance(b, CombinedRateProblem) and b.combine_mode == "sum" and solve_combined_rate(b) == 32 + + +def test_unresolvable_drain_role_steps_aside_not_wrong_answer() -> None: + # A drain verb that does not cleanly govern one rate clause -> no clean opposition -> refuse + # (safe), never emit a positive answer with an inverted role. + out = read_combined_rate_problem( + "Pipe A pumps 10 gallons per minute, draining the reservoir, and Pipe B fills 3 gallons per minute. How many gallons in 2 minutes?" + ) + assert isinstance(out, Refusal) and out.reason == "combine_mode_ambiguous" + + +def test_time_query_quantity_ignores_preamble_distractor() -> None: + out = read_combined_rate_problem( + "Pump A adds 6 liters per minute and pump B adds 4 liters per minute together. " + "There are already 50 liters in the tank. How many minutes to fill 100 liters?" + ) + assert isinstance(out, CombinedRateProblem) and out.quantity == 100 and solve_combined_rate(out) == 10 + + +def test_quantity_query_duration_ignores_transitional_distractor() -> None: + out = read_combined_rate_problem( + "Pipe A fills 10 liters per minute and pipe B fills 6 liters per minute together. " + "After 2 minutes and 3 seconds, how many liters are added in 4 minutes?" + ) + assert isinstance(out, CombinedRateProblem) and out.time == 4 and solve_combined_rate(out) == 64 + + +def test_sequential_segments_with_loose_connector_step_aside() -> None: + out = read_combined_rate_problem( + "Machine A produces 10 units per hour, also for 3 hours, and machine B produces 5 units per hour, " + "also for 2 hours. Both machines work together. How many units total?" + ) + assert isinstance(out, Refusal) and out.reason == "not_combined_rate_shaped" + + +def test_compared_rates_with_no_combined_query_step_aside() -> None: + # Two same-unit rates that are COMPARED, not combined ("which is faster?" / "what is the + # difference?"), are not a CMB problem -> step aside, never the substantive combine_mode_ambiguous. + for text in ( + "A car drives 60 miles per hour. A train drives 80 miles per hour. Which is faster?", + "A printer does 5 pages per minute. A scanner does 3 pages per minute. What is the difference?", + ): + out = read_combined_rate_problem(text) + assert isinstance(out, Refusal) and out.reason == "not_combined_rate_shaped", text + + +def test_reader_module_is_off_serving() -> None: + import ast + + import generate.combined_rate_comprehension.reader as reader_mod + + forbidden = ("generate.derivation", "core.reliability_gate") + for node in ast.walk(ast.parse(Path(str(reader_mod.__file__)).read_text(encoding="utf-8"))): + names = ( + [a.name for a in node.names] if isinstance(node, ast.Import) + else [node.module or ""] if isinstance(node, ast.ImportFrom) + else [] + ) + for name in names: + assert not any(name.startswith(t) for t in forbidden), f"reader imports {name}"