4ddaef4d00
generate/constraint_comprehension/reader.py: read_constraint_problem comprehends two-category constraint prose into a typed ConstraintProblem via five recognizers — C5 category-pair (from coefficient clauses; >2 -> too_many_categories), C6 coefficient (Each <cat> holds/has/costs/is-worth <N> <unit>; shared measured unit), C7 total-count (collective-unit sentence -> x+y=N; absent -> missing_total_count), C8 weighted-total (measured-unit sentence -> ax+by=T; absent -> missing_weighted_total), C9 query-target (How many <cat> -> the asked unknown). Refuses, never mis-assembles. Reconciles the sketch's 'no equal coefficients' note: equal coefficients are a SOLVER refusal (indistinguishable_weights), not a reader one — the reader reads r2-010 setup_correct and the solver refuses it. run_reader lane + 'reader' CLI subcommand. R2 reader: setup_correct 10 / setup_wrong 0 / refused_correct 3; R1 7/0/3 and 15-case 15/0/0 unchanged. Off-serving. 8 reader tests incl. full read->solve->verify chain + meaningful-fail guards.
99 lines
4.8 KiB
Python
99 lines
4.8 KiB
Python
"""Tests for the R2 two-category reader (C5–C9).
|
||
|
||
Pins the wrong=0 reader contract: every well-formed fixture reads to EXACTLY the gold setup
|
||
signature, every ``reader_refuses`` fixture refuses with its gold reason, and — proving each
|
||
reader slice end to end — every solved fixture reads -> solves -> ties to its labeled answer.
|
||
The defensive guards (coefficient unit mismatch, off-category query) are exercised by
|
||
construction; the too-many-categories guard is shown meaningful-fail against a 2-category twin.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from evals.constraint_oracle.runner import _load_r2_gold, gold_to_problem, run_reader
|
||
from evals.constraint_oracle.signature import constraint_setup_signature
|
||
from generate.answer_choices.verify import ChoiceVerdict, verify_answer_choice
|
||
from generate.constraint_comprehension.reader import read_constraint_problem
|
||
from generate.constraint_comprehension.solver import answer_constraint_problem
|
||
from generate.meaning_graph.reader import Refusal
|
||
|
||
|
||
def _by_expect(expect: str) -> list[dict]:
|
||
return [f for f in _load_r2_gold() if f["expect"] == expect]
|
||
|
||
|
||
def test_reader_lane_is_wrong_zero_and_complete() -> None:
|
||
r = run_reader()
|
||
assert r["setup_wrong"] == 0
|
||
assert r["reason_mismatch"] == 0
|
||
assert r["setup_refused"] == 0 # every well-formed fixture reads at C9
|
||
assert r["setup_correct"] == 10 # 7 solved + 3 solver_refuses (all have valid setups)
|
||
assert r["refused_correct"] == 3 # the three reader_refuses fixtures
|
||
|
||
|
||
def test_reader_reads_every_well_formed_fixture_to_gold_signature() -> None:
|
||
for fx in _by_expect("solved") + _by_expect("solver_refuses"):
|
||
out = read_constraint_problem(fx["text"])
|
||
assert not isinstance(out, Refusal), f"{fx['id']} refused: {getattr(out, 'reason', '')}"
|
||
assert constraint_setup_signature(out) == constraint_setup_signature(gold_to_problem(fx)), fx["id"]
|
||
|
||
|
||
def test_reader_refuses_every_reader_refuse_fixture_with_its_reason() -> None:
|
||
for fx in _by_expect("reader_refuses"):
|
||
out = read_constraint_problem(fx["text"])
|
||
assert isinstance(out, Refusal), fx["id"]
|
||
assert out.reason == fx["reader_reason"], f"{fx['id']}: {out.reason} != {fx['reader_reason']}"
|
||
|
||
|
||
def test_read_solve_verify_end_to_end_for_solved() -> None:
|
||
# The full chain a reader slice must prove: read -> solve -> tie to the labeled option.
|
||
for fx in _by_expect("solved"):
|
||
problem = read_constraint_problem(fx["text"])
|
||
assert not isinstance(problem, Refusal), fx["id"]
|
||
value = answer_constraint_problem(problem)
|
||
assert value == fx["gold"], fx["id"]
|
||
verdict = verify_answer_choice(value, fx["options"], fx["answer"], noun=problem.query.unit)
|
||
assert isinstance(verdict, ChoiceVerdict) and verdict.status == "consistent"
|
||
assert verdict.computed_label == fx["answer"]
|
||
|
||
|
||
def test_read_then_solver_refuses_for_solver_refuse_fixtures() -> None:
|
||
# The reader reads the setup; the SOLVER owns solvability — read correct, solve refuses.
|
||
for fx in _by_expect("solver_refuses"):
|
||
problem = read_constraint_problem(fx["text"])
|
||
assert not isinstance(problem, Refusal), fx["id"]
|
||
out = answer_constraint_problem(problem)
|
||
assert isinstance(out, Refusal) and out.reason == fx["solver_reason"], fx["id"]
|
||
|
||
|
||
# --- defensive guards (no gold fixture; exercised by construction) --------------------- #
|
||
|
||
|
||
def test_coefficient_unit_mismatch_refuses() -> None:
|
||
out = read_constraint_problem(
|
||
"A shop has 5 things, all cars and trucks. Each car has 4 wheels and each truck costs "
|
||
"3 dollars. The things total 20 wheels. How many cars are there?"
|
||
)
|
||
assert isinstance(out, Refusal) and out.reason == "coefficient_unit_mismatch"
|
||
|
||
|
||
def test_off_category_query_refuses() -> None:
|
||
out = read_constraint_problem(
|
||
"A school rents 6 buses for a trip. Each large bus holds 50 students and each small bus "
|
||
"holds 30 students. The buses carry 260 students in total. How many vans are there?"
|
||
)
|
||
assert isinstance(out, Refusal) and out.reason == "query_target_not_a_category"
|
||
|
||
|
||
def test_too_many_categories_is_meaningful_fail_against_two_category_twin() -> None:
|
||
# The 3-category text refuses; the SAME template with exactly two categories reads.
|
||
three = read_constraint_problem(
|
||
"A lot has 10 vehicles. Each car has 4 wheels, each motorcycle has 2 wheels, and each "
|
||
"truck has 6 wheels. Together the vehicles have 34 wheels. How many cars are there?"
|
||
)
|
||
assert isinstance(three, Refusal) and three.reason == "too_many_categories"
|
||
two = read_constraint_problem(
|
||
"A lot has 10 vehicles. Each car has 4 wheels and each motorcycle has 2 wheels. "
|
||
"Together the vehicles have 32 wheels. How many cars are there?"
|
||
)
|
||
assert not isinstance(two, Refusal)
|
||
assert {u.symbol for u in two.unknowns} == {"car", "motorcycle"}
|