feat(combined-rate): CMB-c — combined-rate prose reader (prose -> CombinedRateProblem | Refusal)
Third rung of the CMB ladder. Reads explicit combined-rate prose into a canonical CombinedRateProblem or a closed reader refusal; graded by the CMB-a ruler + CMB-b solver via a new run_reader lane. Off-serving; no router/contemplation/serving/ledger. Claim: 'English prose reads into a canonical CMB setup or a closed reader refusal.' Enforces the CMB-a 2x2 domain-entry grid + the deferral taxonomy (rate_unit_mismatch, combine_mode_ambiguous, missing_second_rate, three_or_more_rates, reciprocal_work_rate_deferred, clock_interval_deferred, not_combined_rate_shaped). Non-positive-net / non-integer stay the SOLVER's refusals (the reader parses them). Router-organ hygiene: foreign R1/R2/R3 text steps aside as not_combined_rate_shaped (the input_shape family), never a substantive boundary (0 breaches on all 36 R1/R2/R3 gold). THREE adversarial verification rounds found 11 real wrong=0/hygiene hazards a curated-gold lane cannot surface; all fixed + regression-tested. Root-cause fixes (not point-patches): - cues are whole-word; difference = a CLEAN fill-vs-drain OPPOSITION classified per rate clause by the verb in its own lead-in; rate_a=fill/rate_b=drain BY ROLE (a drain listed first still subtracts); incidental drain nouns / drain-verbs-between-clauses no longer flip the mode; - query duration + time-query quantity are read only from AFTER the 'how many' question, so neither a premise number nor a transitional time marker is mistaken for them; - the rate regex rejects decimals; substantive refusals are gated behind a real combination cue; combine_mode_ambiguous requires a genuine COMBINED query (a compared pair steps aside); - sequential segments (each rate carrying its own duration, tolerant of 'also for') step aside. reader lane 11/0/0 + 7 refused-correct; gold 18/18; 19+17+3 adversarial triggers across 3 rounds; 126 tests (CMB + router/hygiene + R1/R2/R3 readers); serving + siblings unchanged. Next (ladder): CMB-d router/contemplation wiring -> CMB-e ledger.
This commit is contained in:
Shay
parent
93fb55b813
commit
47aaa09019
4 changed files with 613 additions and 4 deletions
|
|
@ -1,9 +1,9 @@
|
|||
"""CLI: validate the combined-rate gold ruler, or grade the CMB solver against it.
|
||||
"""CLI: validate the combined-rate gold ruler, or grade the CMB solver / reader against it.
|
||||
|
||||
python -m evals.combined_rate_oracle # validate combined_rate_gold.jsonl; exit 0 iff invalid == 0
|
||||
python -m evals.combined_rate_oracle solver # grade the solver (CMB-b); exit 0 iff no wrong
|
||||
|
||||
The reader grading lane (``reader`` arg) lands with the reader (CMB-c).
|
||||
python -m evals.combined_rate_oracle reader # grade the reader (CMB-c); exit 0 iff setup_wrong == 0
|
||||
# and reason_mismatch == 0
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -11,7 +11,7 @@ from __future__ import annotations
|
|||
import json
|
||||
import sys
|
||||
|
||||
from evals.combined_rate_oracle.runner import run, run_solver
|
||||
from evals.combined_rate_oracle.runner import run, run_reader, run_solver
|
||||
|
||||
|
||||
def main() -> int:
|
||||
|
|
@ -19,6 +19,9 @@ def main() -> int:
|
|||
if lane == "solver":
|
||||
report = run_solver()
|
||||
ok = report["solved_wrong"] == 0 and report["refuse_wrong"] == 0
|
||||
elif lane == "reader":
|
||||
report = run_reader()
|
||||
ok = report["setup_wrong"] == 0 and report["reason_mismatch"] == 0
|
||||
else:
|
||||
report = run()
|
||||
ok = report["invalid"] == 0
|
||||
|
|
|
|||
|
|
@ -237,6 +237,58 @@ def run_solver() -> dict[str, Any]:
|
|||
}
|
||||
|
||||
|
||||
def run_reader() -> dict[str, Any]:
|
||||
"""Grade the CMB reader against the gold (CMB-c).
|
||||
|
||||
Well-formed fixtures (``solved`` / ``solver_refuses``) must read to a setup whose signature
|
||||
equals the gold's (``setup_correct``); a refusal is a miss (``setup_refused``); a mismatch is
|
||||
``setup_wrong``. ``reader_refuses`` fixtures must refuse with the gold ``reader_reason``
|
||||
(``refused_correct``); a refusal with the wrong reason is ``reason_mismatch``; producing a setup
|
||||
is ``setup_wrong`` (over-read). Exit-0 criterion: ``setup_wrong == 0 and reason_mismatch == 0``.
|
||||
"""
|
||||
from generate.combined_rate_comprehension.reader import read_combined_rate_problem
|
||||
from generate.meaning_graph.reader import Refusal
|
||||
|
||||
from evals.combined_rate_oracle.signature import combined_rate_setup_signature
|
||||
|
||||
fixtures = _load_combined_rate_gold()
|
||||
setup_correct = setup_wrong = setup_refused = refused_correct = reason_mismatch = 0
|
||||
details: list[dict[str, Any]] = []
|
||||
for fx in fixtures:
|
||||
out = read_combined_rate_problem(fx["text"])
|
||||
fid = fx.get("id")
|
||||
if fx["expect"] in ("solved", "solver_refuses"):
|
||||
if isinstance(out, Refusal):
|
||||
setup_refused += 1
|
||||
details.append({"id": fid, "outcome": "setup_refused", "reason": out.reason})
|
||||
elif combined_rate_setup_signature(out) == combined_rate_setup_signature(gold_to_problem(fx)):
|
||||
setup_correct += 1
|
||||
details.append({"id": fid, "outcome": "setup_correct"})
|
||||
else:
|
||||
setup_wrong += 1
|
||||
details.append({"id": fid, "outcome": "setup_WRONG"})
|
||||
else: # reader_refuses
|
||||
if isinstance(out, Refusal) and out.reason == fx["reader_reason"]:
|
||||
refused_correct += 1
|
||||
details.append({"id": fid, "outcome": "refused_correct", "reason": out.reason})
|
||||
elif isinstance(out, Refusal):
|
||||
reason_mismatch += 1
|
||||
details.append({"id": fid, "outcome": "reason_mismatch", "got": out.reason, "want": fx["reader_reason"]})
|
||||
else:
|
||||
setup_wrong += 1
|
||||
details.append({"id": fid, "outcome": "setup_WRONG_over_read"})
|
||||
return {
|
||||
"lane": "combined_rate_oracle_reader",
|
||||
"total": len(fixtures),
|
||||
"setup_correct": setup_correct,
|
||||
"setup_wrong": setup_wrong,
|
||||
"setup_refused": setup_refused,
|
||||
"refused_correct": refused_correct,
|
||||
"reason_mismatch": reason_mismatch,
|
||||
"details": details,
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"COMBINE_MODES",
|
||||
"EXPECTATIONS",
|
||||
|
|
@ -247,6 +299,7 @@ __all__ = [
|
|||
"_load_combined_rate_gold",
|
||||
"gold_to_problem",
|
||||
"run",
|
||||
"run_reader",
|
||||
"run_solver",
|
||||
"validate_fixture",
|
||||
]
|
||||
|
|
|
|||
214
generate/combined_rate_comprehension/reader.py
Normal file
214
generate/combined_rate_comprehension/reader.py
Normal file
|
|
@ -0,0 +1,214 @@
|
|||
"""Combined-rate prose reader (CMB-c): explicit two-rate prose -> CombinedRateProblem.
|
||||
|
||||
Reads ONLY explicit combined-rate problems — two rates over one shared unit, combined by an
|
||||
explicit cooperative (``sum``) or opposing-flow (``difference``) cue — and refuses everything else
|
||||
with the closed CMB reader taxonomy. It does NOT solve: a non-positive net rate or a non-integer
|
||||
answer is the *solver's* boundary (CMB-b), reached only after this reader produces a setup.
|
||||
|
||||
The wrong=0 spine is the CMB-a **2×2 domain-entry grid** (rate-count × combination cue):
|
||||
|
||||
```text
|
||||
| combination cue | no cue
|
||||
two rates | parse (sum/difference) | combine_mode_ambiguous (same unit only)
|
||||
one rate | missing_second_rate | not_combined_rate_shaped <- step aside (R3 territory)
|
||||
```
|
||||
|
||||
plus "combined-shaped but deferred / malformed" refusals that fire ONLY once a genuine two-rate
|
||||
combination cue is present, so CMB never claims a substantive boundary on R1/R2/R3 text:
|
||||
``three_or_more_rates``, ``rate_unit_mismatch`` (the two rates differ in unit),
|
||||
``clock_interval_deferred``, ``reciprocal_work_rate_deferred``.
|
||||
|
||||
**Robustness (each guards a wrong=0 / hygiene failure mode an adversarial pass found):**
|
||||
- cues are **whole-word** regexes, never substrings (``exempt`` is not a drain; ``bothered`` is
|
||||
not ``both``);
|
||||
- ``difference`` requires an explicit **fill AND drain pair**, and assigns ``rate_a`` = fill
|
||||
(minuend) / ``rate_b`` = drain (subtrahend) **by role**, not by text order (a drain listed
|
||||
first must still subtract);
|
||||
- the duration and the time-query target are read from the **query clause** (after the last rate
|
||||
clause), so a preamble number ("50 liters already in the tank …") is never mistaken for them;
|
||||
- the rate regex rejects **decimals** (``3.5 pages per hour`` is not an integer rate -> step aside);
|
||||
- substantive refusals are **gated behind a combination cue** — foreign two-/three-rate prose with
|
||||
no cue steps aside as ``not_combined_rate_shaped`` (router-organ hygiene), never a substantive
|
||||
boundary; and **sequential segments** (each rate carrying its own adjacent duration) step aside.
|
||||
|
||||
Off-serving (imports no ``generate.derivation`` / ``core.reliability_gate``); deterministic.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from generate.combined_rate_comprehension.model import CombinedRateProblem
|
||||
from generate.combined_rate_comprehension.units import RateUnit
|
||||
from generate.meaning_graph.reader import Refusal
|
||||
|
||||
# An integer rate ``<N> <plural> per <singular>``; the lookbehind rejects a digit that is part of a
|
||||
# decimal ("3.5 pages per hour" -> no match -> the reader steps aside rather than read "5").
|
||||
_RATE_VALUE = re.compile(r"(?<![\d.])(\d+)\s+([a-z]+)\s+per\s+([a-z]+)\b")
|
||||
_DURATION = re.compile(r"\b(?:for|in|after)\s+(\d+)\s+([a-z]+)\b")
|
||||
_DIGIT_NOUN = re.compile(r"(?<![\d.])(\d+)\s+([a-z]+)\b")
|
||||
_HOW_MANY = re.compile(r"\bhow many ([a-z]+)\b")
|
||||
# An effective-rate question — constrained to a direct "what … net/combined … rate" so a mid-sentence
|
||||
# "at their combined rate, how many …" does NOT steal the slot from a quantity question.
|
||||
_EFF_RATE_QUERY = re.compile(r"\bwhat\b[^.?!]*\b(?:net|combined)\b[^.?!]*\brate\b")
|
||||
_CLOCK = re.compile(r"\b\d+\s*(?:am|pm)\b|\bo'?clock\b")
|
||||
# Two conjoined agents in the premise ("Anna and Ben …") — the strong signal for missing_second_rate.
|
||||
_AGENT_CONJ = re.compile(r"\b[a-z]+\s+and\s+[a-z]+\b")
|
||||
|
||||
#: Whole-word fill / drain verbs. Detected PER RATE CLAUSE (in the clause's own lead-in), never
|
||||
#: globally — an incidental "the drain stays closed" or a "draining" that governs a different clause
|
||||
#: must not flip the mode. ``difference`` is a *clean opposition*: one fill clause + one drain clause.
|
||||
_FILL_RE = re.compile(r"\b(?:fills?|filling|filled|adds?|adding|added|pours?|pouring|injects?|pumps?\s+in)\b")
|
||||
_DRAIN_RE = re.compile(r"\b(?:drains?|draining|drained|removes?|removing|removed|leaks?|leaking|leaked|empties|emptied|emptying|siphons?|siphoning)\b")
|
||||
_COOP_RE = re.compile(r"\b(?:together|combined|both)\b")
|
||||
_RECIPROCAL_RE = re.compile(r"\bcan\b") # "X can do the job in N time" — the reciprocal work-rate form
|
||||
|
||||
#: How far before a rate's number to look for its governing fill/drain verb ("a pipe fills a tank at
|
||||
#: 5", "while a drain removes 2"). Kept tight so a verb governing the OTHER clause is not captured.
|
||||
_LEADIN = 25
|
||||
|
||||
|
||||
def _singular(noun: str) -> str:
|
||||
if noun.endswith("es") and noun[:-2].endswith(("x", "s", "z", "ch", "sh")):
|
||||
return noun[:-2]
|
||||
if noun.endswith("s") and len(noun) > 1:
|
||||
return noun[:-1]
|
||||
return noun
|
||||
|
||||
|
||||
def _clause_role(t: str, start: int) -> str:
|
||||
"""``drain`` / ``fill`` / ``other`` for the rate clause at *start*, by the verb directly before
|
||||
its number (within ``_LEADIN`` chars). Drain wins ties — a drain verb is the load-bearing sign."""
|
||||
leadin = t[max(0, start - _LEADIN) : start]
|
||||
if _DRAIN_RE.search(leadin):
|
||||
return "drain"
|
||||
if _FILL_RE.search(leadin):
|
||||
return "fill"
|
||||
return "other"
|
||||
|
||||
|
||||
def _is_sequential(t: str, rate_spans: list[tuple[int, int]]) -> bool:
|
||||
"""True iff BOTH rate clauses carry their own following duration (sequential segments — "60 mph
|
||||
for 2 hours and then 40 mph for 3 hours" — which is R3.x, not a simultaneous combination). The
|
||||
window tolerates a short connector ("also for", "then for")."""
|
||||
return all(_DURATION.search(t[e : e + 28]) is not None for _s, e in rate_spans)
|
||||
|
||||
|
||||
def _digit_noun_after(t: str, start: int, noun: str) -> int | None:
|
||||
"""The first ``<N> <noun>`` value at/after *start* — anchored to AFTER the ``how many`` question,
|
||||
so neither a premise number nor a transitional distractor inside the query clause is mistaken for
|
||||
the queried quantity."""
|
||||
for m in _DIGIT_NOUN.finditer(t, start):
|
||||
if _singular(m.group(2)) == noun:
|
||||
return int(m.group(1))
|
||||
return None
|
||||
|
||||
|
||||
def read_combined_rate_problem(text: str) -> CombinedRateProblem | Refusal:
|
||||
"""Comprehend explicit combined-rate prose into a typed CombinedRateProblem, or refuse."""
|
||||
if not text or not text.strip():
|
||||
return Refusal("empty")
|
||||
t = text.lower()
|
||||
|
||||
rate_clauses = _RATE_VALUE.findall(t) # [(value, plural, denom_singular), ...]
|
||||
rate_spans = [m.span() for m in _RATE_VALUE.finditer(t)]
|
||||
n = len(rate_clauses)
|
||||
coop = _COOP_RE.search(t) is not None
|
||||
n_durations = len(_DURATION.findall(t))
|
||||
|
||||
# --- the grid's rate-count axis ----------------------------------------------------------- #
|
||||
if n == 0:
|
||||
# No explicit "per" rate. A cooperative cue + ≥2 completion durations + a "can"-style
|
||||
# capability is the reciprocal work-rate form (1/(1/a+1/b)); else not CMB's domain.
|
||||
if coop and n_durations >= 2 and _RECIPROCAL_RE.search(t):
|
||||
return Refusal("reciprocal_work_rate_deferred", "durations-to-complete, not explicit rates")
|
||||
return Refusal("not_combined_rate_shaped", "no explicit rate clause")
|
||||
if n >= 3:
|
||||
# Substantive ONLY with a combination cue; otherwise foreign prose that merely has ≥3 rate
|
||||
# patterns -> step aside (hygiene).
|
||||
if coop or any(_clause_role(t, s) == "drain" for s, _e in rate_spans):
|
||||
return Refusal("three_or_more_rates", f"{n} explicit rates; CMB v1 combines exactly two")
|
||||
return Refusal("not_combined_rate_shaped", "three or more rates but no combination cue")
|
||||
if n == 1:
|
||||
# One rate is substantive (missing a second) ONLY with a cooperative cue AND two conjoined
|
||||
# agents in the premise; a lone rate (or an incidental "combined"/"both") is R3 territory.
|
||||
if coop and any(m.start() < rate_spans[0][0] for m in _AGENT_CONJ.finditer(t)):
|
||||
return Refusal("missing_second_rate", "a cooperation cue and two agents but only one explicit rate")
|
||||
return Refusal("not_combined_rate_shaped", "a single rate with no two-agent combination cue")
|
||||
|
||||
# --- n == 2 -------------------------------------------------------------------------------- #
|
||||
u0 = (_singular(rate_clauses[0][1]), _singular(rate_clauses[0][2]))
|
||||
u1 = (_singular(rate_clauses[1][1]), _singular(rate_clauses[1][2]))
|
||||
if _is_sequential(t, rate_spans):
|
||||
return Refusal("not_combined_rate_shaped", "two rates with their own durations are sequential segments")
|
||||
|
||||
# The combine mode comes from the two clauses' OWN roles: a clean fill/drain opposition is the
|
||||
# only difference; a cooperative cue is the only sum. Anything else has no usable cue.
|
||||
roles = [_clause_role(t, s) for s, _e in rate_spans]
|
||||
if sorted(roles) == ["drain", "fill"]:
|
||||
mode = "difference"
|
||||
elif coop:
|
||||
mode = "sum"
|
||||
else:
|
||||
mode = None
|
||||
if mode is None:
|
||||
# combine_mode_ambiguous is substantive (CMB's domain), so it fires ONLY when the question
|
||||
# actually asks to COMBINE the two same-unit rates — a total quantity/time, or the net/combined
|
||||
# rate. Two same-unit rates merely COMPARED ("which is faster?"), or with a different unit, or
|
||||
# with no combined query, are not a CMB problem -> step aside (router hygiene).
|
||||
if u0 == u1:
|
||||
hm = _HOW_MANY.search(t)
|
||||
asked0 = _singular(hm.group(1)) if hm else None
|
||||
if asked0 in u0 or _EFF_RATE_QUERY.search(t, rate_spans[-1][1]) is not None:
|
||||
return Refusal("combine_mode_ambiguous", "two same-unit rates and a combined query but no mode cue")
|
||||
return Refusal("not_combined_rate_shaped", "two rates that are not a clean combined-rate problem")
|
||||
# A combination cue IS present — now substantive combined-rate refusals are legitimate.
|
||||
if u0 != u1:
|
||||
return Refusal("rate_unit_mismatch", f"two rates differ in unit: {u0} vs {u1}")
|
||||
if _CLOCK.search(t):
|
||||
return Refusal("clock_interval_deferred", "an elapsed clock interval is not an explicit duration")
|
||||
|
||||
unit = RateUnit(*u0)
|
||||
if mode == "difference":
|
||||
# rate_a = fill (minuend), rate_b = drain (subtrahend), BY ROLE not by text order.
|
||||
drain_idx = roles.index("drain")
|
||||
rate_a, rate_b = (
|
||||
(int(rate_clauses[1][0]), int(rate_clauses[0][0]))
|
||||
if drain_idx == 0
|
||||
else (int(rate_clauses[0][0]), int(rate_clauses[1][0]))
|
||||
)
|
||||
else:
|
||||
rate_a, rate_b = int(rate_clauses[0][0]), int(rate_clauses[1][0])
|
||||
|
||||
# --- query slot + assembly: read the queried slot from AFTER the "how many" question, so a
|
||||
# premise number or a transitional time marker is never mistaken for it. --------------------- #
|
||||
if _EFF_RATE_QUERY.search(t, rate_spans[-1][1]):
|
||||
return CombinedRateProblem(rate_a, rate_b, unit, mode, None, None, "effective_rate")
|
||||
|
||||
how_many = _HOW_MANY.search(t)
|
||||
if how_many is None:
|
||||
return Refusal("not_combined_rate_shaped", "no recognizable query slot") # defensive (no gold)
|
||||
asked = _singular(how_many.group(1))
|
||||
q_start = how_many.end()
|
||||
|
||||
if asked == unit.numerator: # quantity query — needs a duration in the rate's denominator unit
|
||||
dur = _DURATION.search(t, q_start)
|
||||
if dur is None:
|
||||
return Refusal("not_combined_rate_shaped", "quantity query without a duration") # defensive
|
||||
time_value, time_unit = int(dur.group(1)), _singular(dur.group(2))
|
||||
if time_unit != unit.denominator:
|
||||
# CMB v1 crosses no units (R3.2 conversion is single-rate only) — refuse rather than
|
||||
# silently treat a non-denominator duration as the rate's unit.
|
||||
return Refusal("rate_unit_mismatch", f"duration {time_unit!r} != rate denominator {unit.denominator!r}")
|
||||
return CombinedRateProblem(rate_a, rate_b, unit, mode, time_value, None, "quantity", time_unit=time_unit)
|
||||
|
||||
if asked == unit.denominator: # time query — needs the target quantity (in the numerator unit)
|
||||
qty = _digit_noun_after(t, q_start, unit.numerator)
|
||||
if qty is None:
|
||||
return Refusal("not_combined_rate_shaped", "time query without a target quantity") # defensive
|
||||
return CombinedRateProblem(rate_a, rate_b, unit, mode, None, qty, "time")
|
||||
|
||||
return Refusal("not_combined_rate_shaped", f"unrecognized query target {asked!r}") # defensive
|
||||
|
||||
|
||||
__all__ = ["read_combined_rate_problem"]
|
||||
339
tests/test_combined_rate_reader.py
Normal file
339
tests/test_combined_rate_reader.py
Normal file
|
|
@ -0,0 +1,339 @@
|
|||
"""Tests for the combined-rate prose reader (CMB-c).
|
||||
|
||||
Pins wrong=0 for the reader: every well-formed gold fixture reads to exactly the gold setup (and
|
||||
solves end-to-end to its labeled answer via CMB-b); every reader_refuses fixture refuses with its
|
||||
gold reason; the CMB-a 2x2 domain-entry grid (with fresh vocabulary, proving the reader is
|
||||
structural not string-matched); the solver-boundary fixtures are PARSED then refused by the solver,
|
||||
not the reader; and router-organ hygiene — CMB steps aside on every foreign R1/R2/R3 problem.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from evals.combined_rate_oracle.runner import _load_combined_rate_gold, gold_to_problem, run_reader
|
||||
from evals.combined_rate_oracle.signature import combined_rate_setup_signature
|
||||
from evals.constraint_oracle.runner import _load_r2_gold
|
||||
from evals.rate_oracle.runner import _load_rate_gold
|
||||
from evals.setup_oracle.runner import _load_r1_gold
|
||||
from generate.answer_choices.verify import ChoiceVerdict, verify_answer_choice
|
||||
from generate.combined_rate_comprehension.model import CombinedRateProblem
|
||||
from generate.combined_rate_comprehension.reader import read_combined_rate_problem
|
||||
from generate.combined_rate_comprehension.solver import solve_combined_rate
|
||||
from generate.meaning_graph.reader import Refusal
|
||||
|
||||
_PAGE_HOUR = ("page", "hour")
|
||||
|
||||
|
||||
def _by(expect: str) -> list[dict]:
|
||||
return [f for f in _load_combined_rate_gold() if f["expect"] == expect]
|
||||
|
||||
|
||||
# --- the reader lane + gold round-trip ---------------------------------------------------- #
|
||||
|
||||
|
||||
def test_reader_lane_is_wrong_zero_and_complete() -> None:
|
||||
r = run_reader()
|
||||
assert r["setup_wrong"] == 0 and r["reason_mismatch"] == 0
|
||||
assert r["setup_refused"] == 0
|
||||
assert r["setup_correct"] == 11 # 6 solved + 5 solver_refuses (all parse)
|
||||
assert r["refused_correct"] == 7
|
||||
|
||||
|
||||
def test_reads_every_well_formed_fixture_to_gold_signature() -> None:
|
||||
for fx in _by("solved") + _by("solver_refuses"):
|
||||
out = read_combined_rate_problem(fx["text"])
|
||||
assert not isinstance(out, Refusal), f"{fx['id']}: refused {getattr(out, 'reason', '')}"
|
||||
assert combined_rate_setup_signature(out) == combined_rate_setup_signature(gold_to_problem(fx)), fx["id"]
|
||||
|
||||
|
||||
def test_refuses_every_reader_refuse_fixture_with_reason() -> None:
|
||||
for fx in _by("reader_refuses"):
|
||||
out = read_combined_rate_problem(fx["text"])
|
||||
assert isinstance(out, Refusal) and out.reason == fx["reader_reason"], fx["id"]
|
||||
|
||||
|
||||
def test_read_solve_verify_end_to_end_for_solved() -> None:
|
||||
for fx in _by("solved"):
|
||||
problem = read_combined_rate_problem(fx["text"])
|
||||
assert not isinstance(problem, Refusal), fx["id"]
|
||||
value = solve_combined_rate(problem)
|
||||
assert value == fx["gold"], fx["id"]
|
||||
verdict = verify_answer_choice(value, fx["options"], fx["answer"])
|
||||
assert isinstance(verdict, ChoiceVerdict) and verdict.status == "consistent"
|
||||
assert verdict.computed_label == fx["answer"], fx["id"]
|
||||
|
||||
|
||||
def test_solver_boundary_fixtures_are_parsed_by_reader_then_refused_by_solver() -> None:
|
||||
# The division of labor: the reader OWNS the setup (non-positive net / non-integer are valid
|
||||
# combined-rate setups), the solver OWNS solvability. The reader must NOT refuse these.
|
||||
for fx in _by("solver_refuses"):
|
||||
problem = read_combined_rate_problem(fx["text"])
|
||||
assert not isinstance(problem, Refusal), f"{fx['id']}: reader wrongly refused a solver-boundary case"
|
||||
out = solve_combined_rate(problem)
|
||||
assert isinstance(out, Refusal) and out.reason == fx["solver_reason"], fx["id"]
|
||||
|
||||
|
||||
# --- the 2x2 domain-entry grid, with FRESH vocabulary (structural, not gold-string-matched) ---- #
|
||||
|
||||
|
||||
def test_grid_two_rates_plus_cooperative_cue_parses_sum() -> None:
|
||||
out = read_combined_rate_problem(
|
||||
"Carla types 4 pages per hour and Dave types 3 pages per hour. Working together, how many pages in 5 hours?"
|
||||
)
|
||||
assert isinstance(out, CombinedRateProblem)
|
||||
assert out.combine_mode == "sum" and out.query == "quantity"
|
||||
assert solve_combined_rate(out) == 35 # (4 + 3) * 5
|
||||
|
||||
|
||||
def test_grid_two_rates_plus_opposing_cue_parses_difference() -> None:
|
||||
out = read_combined_rate_problem(
|
||||
"A tap adds 7 liters per minute while a leak removes 2 liters per minute. How many liters after 4 minutes?"
|
||||
)
|
||||
assert isinstance(out, CombinedRateProblem)
|
||||
assert out.combine_mode == "difference" and out.query == "quantity"
|
||||
assert solve_combined_rate(out) == 20 # (7 - 2) * 4
|
||||
|
||||
|
||||
def test_grid_two_rates_no_cue_is_combine_mode_ambiguous() -> None:
|
||||
out = read_combined_rate_problem(
|
||||
"Machine X makes 4 bolts per minute. Machine Y makes 6 bolts per minute. How many bolts in 3 minutes?"
|
||||
)
|
||||
assert isinstance(out, Refusal) and out.reason == "combine_mode_ambiguous"
|
||||
|
||||
|
||||
def test_grid_one_rate_plus_cue_is_missing_second_rate() -> None:
|
||||
out = read_combined_rate_problem(
|
||||
"Sam and Tess mow a lawn together. Sam mows 2 lawns per hour. How many lawns in 3 hours?"
|
||||
)
|
||||
assert isinstance(out, Refusal) and out.reason == "missing_second_rate"
|
||||
|
||||
|
||||
def test_grid_one_rate_no_cue_steps_aside() -> None:
|
||||
out = read_combined_rate_problem("A printer prints 5 pages per minute for 4 minutes. How many pages does it print?")
|
||||
assert isinstance(out, Refusal) and out.reason == "not_combined_rate_shaped"
|
||||
|
||||
|
||||
# --- each remaining reader refusal reason, meaningful-fail (fresh text) -------------------- #
|
||||
|
||||
|
||||
def test_rate_unit_mismatch() -> None:
|
||||
out = read_combined_rate_problem(
|
||||
"A clerk files 3 forms per hour and a pump moves 2 gallons per minute. Working together, how much in 4 hours?"
|
||||
)
|
||||
assert isinstance(out, Refusal) and out.reason == "rate_unit_mismatch"
|
||||
|
||||
|
||||
def test_three_or_more_rates() -> None:
|
||||
out = read_combined_rate_problem(
|
||||
"A types 3 pages per hour, B types 2 pages per hour, and C types 5 pages per hour. Together, how many pages in 2 hours?"
|
||||
)
|
||||
assert isinstance(out, Refusal) and out.reason == "three_or_more_rates"
|
||||
|
||||
|
||||
def test_reciprocal_work_rate_deferred() -> None:
|
||||
out = read_combined_rate_problem(
|
||||
"Maria can paint a fence in 4 hours, and Nina can paint the same fence in 6 hours. Working together, how many hours?"
|
||||
)
|
||||
assert isinstance(out, Refusal) and out.reason == "reciprocal_work_rate_deferred"
|
||||
|
||||
|
||||
def test_clock_interval_deferred() -> None:
|
||||
out = read_combined_rate_problem(
|
||||
"One hose fills at 3 liters per minute and another fills at 2 liters per minute, together from 1 pm to 4 pm. How many liters?"
|
||||
)
|
||||
assert isinstance(out, Refusal) and out.reason == "clock_interval_deferred"
|
||||
|
||||
|
||||
def test_sequential_segments_step_aside_not_ambiguous() -> None:
|
||||
# Two rate clauses, each with its OWN duration -> sequential (R3.x), NOT a combination. Must
|
||||
# step aside, never claim the substantive combine_mode_ambiguous (hygiene).
|
||||
out = read_combined_rate_problem(
|
||||
"A car goes 60 miles per hour for 2 hours and then 40 miles per hour for 3 hours. How many miles total?"
|
||||
)
|
||||
assert isinstance(out, Refusal) and out.reason == "not_combined_rate_shaped"
|
||||
|
||||
|
||||
# --- router-organ hygiene: CMB steps aside on every foreign R1/R2/R3 problem --------------- #
|
||||
|
||||
|
||||
def test_steps_aside_on_all_foreign_r1_r2_r3_gold() -> None:
|
||||
# The ONLY refusal reason CMB may use on foreign text is the input_shape-family
|
||||
# not_combined_rate_shaped — never a substantive boundary (router-organ-hygiene invariant).
|
||||
for load in (_load_r1_gold, _load_r2_gold, _load_rate_gold):
|
||||
for fx in load():
|
||||
out = read_combined_rate_problem(fx["text"])
|
||||
assert isinstance(out, Refusal), f"{fx['id']}: CMB produced a setup on foreign text"
|
||||
assert out.reason == "not_combined_rate_shaped", f"{fx['id']}: substantive reason {out.reason!r} on foreign text"
|
||||
|
||||
|
||||
def test_does_not_steal_ordinary_single_rate_problems() -> None:
|
||||
# Plain R3 single-rate prose (incl. the clock/temporal r3 case) is R3's; CMB must step aside.
|
||||
for text in (
|
||||
"A car travels 60 miles per hour for 3 hours. How many miles does it travel?",
|
||||
"A worker earns 15 dollars per hour for 8 hours. How many dollars does she earn?",
|
||||
"A machine makes 12 widgets per minute. It runs for 5 minutes. How many widgets does it make?",
|
||||
):
|
||||
out = read_combined_rate_problem(text)
|
||||
assert isinstance(out, Refusal) and out.reason == "not_combined_rate_shaped", text
|
||||
|
||||
|
||||
# --- regression tests for the adversarial-found defects (wrong=0 / hygiene) --------------- #
|
||||
|
||||
|
||||
def test_difference_mode_is_role_based_not_positional() -> None:
|
||||
# Drain listed FIRST must still subtract: 3 (fill) - 8 (drain) = -5 -> solver refuses, NOT a
|
||||
# positive answer of (8-3)*5. Both orders must agree on the role assignment.
|
||||
drain_first = read_combined_rate_problem(
|
||||
"A drain removes 8 liters per minute while a pipe fills a tank at 3 liters per minute. How many liters after 5 minutes?"
|
||||
)
|
||||
assert isinstance(drain_first, CombinedRateProblem)
|
||||
assert drain_first.combine_mode == "difference" and drain_first.effective_rate == -5
|
||||
assert isinstance(solve_combined_rate(drain_first), Refusal)
|
||||
fill_first = read_combined_rate_problem(
|
||||
"A pump fills a tank at 9 liters per minute while a drain removes 3 liters per minute. How many liters after 5 minutes?"
|
||||
)
|
||||
assert isinstance(fill_first, CombinedRateProblem) and solve_combined_rate(fill_first) == 30
|
||||
|
||||
|
||||
def test_lone_drain_word_is_cooperative_not_opposing() -> None:
|
||||
# "drain the backlog" is a verb, not opposing flow; with no fill counterpart it is a cooperative
|
||||
# (sum) problem, not difference.
|
||||
out = read_combined_rate_problem(
|
||||
"Both workers drain the backlog at 5 tasks per hour and 3 tasks per hour. How many tasks in 4 hours?"
|
||||
)
|
||||
assert isinstance(out, CombinedRateProblem) and out.combine_mode == "sum"
|
||||
assert solve_combined_rate(out) == 32
|
||||
|
||||
|
||||
def test_mid_sentence_combined_rate_does_not_steal_quantity_query() -> None:
|
||||
out = read_combined_rate_problem(
|
||||
"Machine A fills 6 tanks per hour and Machine B fills 4 tanks per hour. At their combined rate, how many tanks in 5 hours?"
|
||||
)
|
||||
assert isinstance(out, CombinedRateProblem) and out.query == "quantity"
|
||||
assert solve_combined_rate(out) == 50
|
||||
|
||||
|
||||
def test_preamble_number_is_not_the_query_target() -> None:
|
||||
# The "in 3 hours / 15 rooms" preamble must not be read as the query duration/quantity.
|
||||
out = read_combined_rate_problem(
|
||||
"In 3 hours, Anna and Ben painted 15 rooms. Anna paints 3 rooms per hour and Ben paints 2 rooms per hour. "
|
||||
"Working together, how many rooms do they paint in 4 hours?"
|
||||
)
|
||||
assert isinstance(out, CombinedRateProblem) and out.time == 4
|
||||
assert solve_combined_rate(out) == 20
|
||||
|
||||
|
||||
def test_decimal_rates_step_aside() -> None:
|
||||
out = read_combined_rate_problem(
|
||||
"Carla types 3.5 pages per hour and Dave types 2.5 pages per hour. Working together, how many pages in 4 hours?"
|
||||
)
|
||||
assert isinstance(out, Refusal) and out.reason == "not_combined_rate_shaped"
|
||||
|
||||
|
||||
def test_incidental_combination_words_step_aside() -> None:
|
||||
# 'combined'/'both'/'together' used as non-combination words on single-rate text must NOT
|
||||
# over-claim missing_second_rate (hygiene) — step aside.
|
||||
for text in (
|
||||
"Anna earns 3 dollars per hour. If they work together, how many dollars does Anna earn in 4 hours?",
|
||||
"A machine produces 5 parts per hour. The combined output target is 25 parts. How many hours does it take?",
|
||||
"A study entitled Combined Effects found workers process 4 tasks per hour. How many tasks in 8 hours?",
|
||||
):
|
||||
out = read_combined_rate_problem(text)
|
||||
assert isinstance(out, Refusal) and out.reason == "not_combined_rate_shaped", text
|
||||
|
||||
|
||||
def test_foreign_multi_rate_text_steps_aside_not_substantive() -> None:
|
||||
# Substantive refusals (rate_unit_mismatch / three_or_more_rates) must NOT fire on foreign prose
|
||||
# that merely contains 2+ rate patterns with no combination cue (router-organ hygiene).
|
||||
for text in (
|
||||
"The server handles 100 requests per second and costs 2 dollars per hour. How many requests in 10 seconds?",
|
||||
"A types 3 pages per hour, B types 2 pages per hour, and C types 5 pages per hour. How many pages in 2 hours?",
|
||||
):
|
||||
out = read_combined_rate_problem(text)
|
||||
assert isinstance(out, Refusal) and out.reason == "not_combined_rate_shaped", text
|
||||
|
||||
|
||||
def test_duration_unit_mismatch_refuses() -> None:
|
||||
# A combined problem whose duration unit differs from the rate denominator: CMB v1 does not
|
||||
# convert (that is R3.2, single-rate only) -> rate_unit_mismatch. Pins the line-144 exit.
|
||||
out = read_combined_rate_problem(
|
||||
"Pipe A fills 5 rooms per hour and Pipe B fills 3 rooms per hour. Working together, how many rooms in 30 minutes?"
|
||||
)
|
||||
assert isinstance(out, Refusal) and out.reason == "rate_unit_mismatch"
|
||||
|
||||
|
||||
# --- second-pass regression: incidental cues, distractor numbers, loose sequential ------- #
|
||||
|
||||
|
||||
def test_incidental_drain_noun_does_not_force_difference() -> None:
|
||||
# "the drain stays closed" / "drain from the tank" are not drain RATES — two fills cooperating
|
||||
# must stay sum, never flip to difference.
|
||||
a = read_combined_rate_problem(
|
||||
"The pump fills 8 liters per minute and the hose fills 4 liters per minute together. The drain stays closed. How many liters in 3 minutes?"
|
||||
)
|
||||
assert isinstance(a, CombinedRateProblem) and a.combine_mode == "sum" and solve_combined_rate(a) == 36
|
||||
b = read_combined_rate_problem(
|
||||
"Pump A fills 10 liters per minute and pump B fills 6 liters per minute together. How many liters drain from the tank in 2 minutes?"
|
||||
)
|
||||
assert isinstance(b, CombinedRateProblem) and b.combine_mode == "sum" and solve_combined_rate(b) == 32
|
||||
|
||||
|
||||
def test_unresolvable_drain_role_steps_aside_not_wrong_answer() -> None:
|
||||
# A drain verb that does not cleanly govern one rate clause -> no clean opposition -> refuse
|
||||
# (safe), never emit a positive answer with an inverted role.
|
||||
out = read_combined_rate_problem(
|
||||
"Pipe A pumps 10 gallons per minute, draining the reservoir, and Pipe B fills 3 gallons per minute. How many gallons in 2 minutes?"
|
||||
)
|
||||
assert isinstance(out, Refusal) and out.reason == "combine_mode_ambiguous"
|
||||
|
||||
|
||||
def test_time_query_quantity_ignores_preamble_distractor() -> None:
|
||||
out = read_combined_rate_problem(
|
||||
"Pump A adds 6 liters per minute and pump B adds 4 liters per minute together. "
|
||||
"There are already 50 liters in the tank. How many minutes to fill 100 liters?"
|
||||
)
|
||||
assert isinstance(out, CombinedRateProblem) and out.quantity == 100 and solve_combined_rate(out) == 10
|
||||
|
||||
|
||||
def test_quantity_query_duration_ignores_transitional_distractor() -> None:
|
||||
out = read_combined_rate_problem(
|
||||
"Pipe A fills 10 liters per minute and pipe B fills 6 liters per minute together. "
|
||||
"After 2 minutes and 3 seconds, how many liters are added in 4 minutes?"
|
||||
)
|
||||
assert isinstance(out, CombinedRateProblem) and out.time == 4 and solve_combined_rate(out) == 64
|
||||
|
||||
|
||||
def test_sequential_segments_with_loose_connector_step_aside() -> None:
|
||||
out = read_combined_rate_problem(
|
||||
"Machine A produces 10 units per hour, also for 3 hours, and machine B produces 5 units per hour, "
|
||||
"also for 2 hours. Both machines work together. How many units total?"
|
||||
)
|
||||
assert isinstance(out, Refusal) and out.reason == "not_combined_rate_shaped"
|
||||
|
||||
|
||||
def test_compared_rates_with_no_combined_query_step_aside() -> None:
|
||||
# Two same-unit rates that are COMPARED, not combined ("which is faster?" / "what is the
|
||||
# difference?"), are not a CMB problem -> step aside, never the substantive combine_mode_ambiguous.
|
||||
for text in (
|
||||
"A car drives 60 miles per hour. A train drives 80 miles per hour. Which is faster?",
|
||||
"A printer does 5 pages per minute. A scanner does 3 pages per minute. What is the difference?",
|
||||
):
|
||||
out = read_combined_rate_problem(text)
|
||||
assert isinstance(out, Refusal) and out.reason == "not_combined_rate_shaped", text
|
||||
|
||||
|
||||
def test_reader_module_is_off_serving() -> None:
|
||||
import ast
|
||||
|
||||
import generate.combined_rate_comprehension.reader as reader_mod
|
||||
|
||||
forbidden = ("generate.derivation", "core.reliability_gate")
|
||||
for node in ast.walk(ast.parse(Path(str(reader_mod.__file__)).read_text(encoding="utf-8"))):
|
||||
names = (
|
||||
[a.name for a in node.names] if isinstance(node, ast.Import)
|
||||
else [node.module or ""] if isinstance(node, ast.ImportFrom)
|
||||
else []
|
||||
)
|
||||
for name in names:
|
||||
assert not any(name.startswith(t) for t in forbidden), f"reader imports {name}"
|
||||
Loading…
Reference in a new issue