From f9ef9e56a46062fc88150807cae733d7861c2486 Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 6 Jun 2026 21:07:19 -0700 Subject: [PATCH] =?UTF-8?q?feat(comprehension):=20aggregate-then-divide=20?= =?UTF-8?q?partition=20frame=20=E2=80=94=20"split=20equally=20into=20N=20b?= =?UTF-8?q?oxes"=20(PR-6d)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-6d adds the partition frame: combine all parts into a total, then split that total equally into N containers. r1-06-subtotal-reused moves refused → correct — the FIRST case where the divisor applies to a DERIVED symbol (the total), not a directly given fact. That is real progress toward GSM8K setup comprehension, where intermediate quantities are the norm. Scope (kept narrow on purpose): No new relation kind. No new arithmetic operation. No rational support. No rounding/flooring. No serving path touched. The frame reuses the already-ratified pieces — SumOf(parts) + Div(Symbol(total), Literal(N)) → divide_by — so this PR is reader-only (no IR / admissibility / oracle / signature change). Frame grammar: "They combine their and split them equally into N ." + "How many are in each ?" -> total = sum(all facts); per_ = total / N; ask per_. wrong=0 boundaries: - Exact-divisibility still gates the ANSWER, now over a derived total: 5+6=11, 11/3 is non-exact -> the setup reads correctly but the answer REFUSES (never floors). Setup comprehension and answer exactness are cleanly separated. - Partition/query coherence: a partition is read ONLY together with its "in each " query (and vice versa); container mismatch (box vs jar) refuses. Prevents over-reading a story detail into an unused derived value. Meaningful-fail verified: disabling the guard makes a dangling partition wrongly comprehend. Gates: R1 setup: 4 correct / 0 wrong / 6 refused R1 answers: 4 correct / 0 wrong / 6 refused / setup_wrong 0 / gold_error 0 15-case setup: 15 / 0 / 0 97 PR-6d tests + 99 relational/invariant tests green. Reader is off-serving (no generate.derivation / core.reliability_gate import). --- evals/setup_oracle/r1_gold.jsonl | 2 +- generate/quantitative_comprehension.py | 100 +++++++++++++++++++++-- tests/test_quantitative_comprehension.py | 72 ++++++++++++++++ tests/test_setup_oracle.py | 14 ++-- 4 files changed, 174 insertions(+), 14 deletions(-) diff --git a/evals/setup_oracle/r1_gold.jsonl b/evals/setup_oracle/r1_gold.jsonl index 6cb9d356..42e276db 100644 --- a/evals/setup_oracle/r1_gold.jsonl +++ b/evals/setup_oracle/r1_gold.jsonl @@ -3,7 +3,7 @@ {"id": "r1-03-more-total", "text": "Finn has 10 books. Evan has 5 more books than Finn. How many books do Evan and Finn have altogether?", "relations": [{"kind": "fact", "entity": "finn", "value": 10}, {"kind": "more_than", "entity": "evan", "ref": "finn", "delta": 5}, {"kind": "sum_of", "entity": "total", "parts": ["evan", "finn"]}], "expected_units": {"finn": "item", "evan": "item", "total": "item"}, "query": {"entity": "total", "unit": "item"}, "notes": "Additive + aggregate. 'altogether' (not the bare 'have') may fall outside the strict query template -> likely REFUSE; if read, structure must match exactly."} {"id": "r1-04-fewer-total", "text": "Gail has 20 cards. Hank has 6 fewer cards than Gail. How many cards do Gail and Hank have in total?", "relations": [{"kind": "fact", "entity": "gail", "value": 20}, {"kind": "fewer_than", "entity": "hank", "ref": "gail", "delta": 6}, {"kind": "sum_of", "entity": "total", "parts": ["gail", "hank"]}], "expected_units": {"gail": "item", "hank": "item", "total": "item"}, "query": {"entity": "total", "unit": "item"}, "notes": "Additive(fewer) + aggregate. 'in total' qualifier likely outside the query template -> REFUSE expected."} {"id": "r1-05-chain", "text": "Ivy has 4 pens. Jon has 3 times as many pens as Ivy. Kim has 2 more pens than Jon. How many pens does Kim have?", "relations": [{"kind": "fact", "entity": "ivy", "value": 4}, {"kind": "times_as_many", "entity": "jon", "ref": "ivy", "factor": 3}, {"kind": "more_than", "entity": "kim", "ref": "jon", "delta": 2}], "expected_units": {"ivy": "item", "jon": "item", "kim": "item"}, "query": {"entity": "kim", "unit": "item"}, "gold": 14, "notes": "Multi-step derived chain: jon=3*ivy (intermediate), kim=jon+2. The multiplicative middle step has no template -> must REFUSE the whole reading, never read a partial/wrong chain."} -{"id": "r1-06-subtotal-reused", "text": "Lee has 5 hats. Mae has 7 hats. They combine their hats and split them equally into 3 boxes. How many hats are in each box?", "relations": [{"kind": "fact", "entity": "lee", "value": 5}, {"kind": "fact", "entity": "mae", "value": 7}, {"kind": "sum_of", "entity": "total", "parts": ["lee", "mae"]}, {"kind": "divide_by", "entity": "per_box", "ref": "total", "divisor": 3}], "expected_units": {"lee": "item", "mae": "item", "total": "item", "per_box": "item"}, "query": {"entity": "per_box", "unit": "item"}, "notes": "Derived subtotal reused downstream: total = lee + mae, per_box = total / 3. No partition template -> must REFUSE."} +{"id": "r1-06-subtotal-reused", "text": "Lee has 5 hats. Mae has 7 hats. They combine their hats and split them equally into 3 boxes. How many hats are in each box?", "relations": [{"kind": "fact", "entity": "lee", "value": 5}, {"kind": "fact", "entity": "mae", "value": 7}, {"kind": "sum_of", "entity": "total", "parts": ["lee", "mae"]}, {"kind": "divide_by", "entity": "per_box", "ref": "total", "divisor": 3}], "expected_units": {"lee": "item", "mae": "item", "total": "item", "per_box": "item"}, "query": {"entity": "per_box", "unit": "item"}, "gold": 4, "notes": "Aggregate-then-divide partition (PR-6d): semantic source is equal PARTITION ('split equally into 3 boxes'); the mathematical setup is total = lee + mae (sum_of) then per_box = total / 3 (divide_by), reusing SumOf + Div with NO new relation kind. First derived-subtotal reuse: the divide's ref is itself a derived symbol. Exact-divisibility still gates the answer (total % 3 == 0; 12 / 3 = 4)."} {"id": "r1-07-inverse", "text": "Nia has 9 more beads than Omar. Nia has 15 beads. How many beads does Omar have?", "relations": [{"kind": "fact", "entity": "nia", "value": 15}, {"kind": "more_than", "entity": "nia", "ref": "omar", "delta": 9}, {"kind": "ask_base", "entity": "omar"}], "expected_units": {"nia": "item", "omar": "item"}, "query": {"entity": "omar", "unit": "item"}, "notes": "Inverse target: omar = nia - 9 = 6, asked via the base of a more_than. The reader may read the STRUCTURE (nia=15, nia=omar+9, ask omar) — that setup is correct; only the solver can't invert. So setup_correct OR refuse, but NEVER setup_wrong."} {"id": "r1-08-ambiguous-referent", "text": "Pat has 5 marbles. He has 3 more than her. How many marbles does she have?", "relations": [], "expected_units": {}, "query": {"entity": "she", "unit": "item"}, "notes": "Ambiguous pronoun referents (he/her/she) with no grounded base for 'her'/'she' -> must REFUSE (no honest reading), never bind a guessed referent."} {"id": "r1-09-missing-base", "text": "Quinn has twice as many toys as Rosa. How many toys does Quinn have?", "relations": [{"kind": "times_as_many", "entity": "quinn", "ref": "rosa", "factor": 2}], "expected_units": {"quinn": "item", "rosa": "item"}, "query": {"entity": "quinn", "unit": "item"}, "notes": "Missing base quantity: Rosa's count is never given, so Quinn is underdetermined -> must REFUSE."} diff --git a/generate/quantitative_comprehension.py b/generate/quantitative_comprehension.py index f96f29e1..e1d8e8e3 100644 --- a/generate/quantitative_comprehension.py +++ b/generate/quantitative_comprehension.py @@ -166,6 +166,34 @@ class _Div: unit: str +@dataclass(frozen=True, slots=True) +class _Partition: + """Aggregate-then-divide: combine all facts into a ``total`` then split that total + equally into ``divisor`` parts (R1, "They combine their X and split them equally + into N boxes"). The semantic source is equal PARTITION; the mathematical setup is + ``total = sum(facts)`` + ``per_ = total / divisor`` — reusing ``SumOf`` + + ``Div(Symbol, Literal)``, NO new relation kind (the divisor is exact integer + division, the same wrong=0 boundary as PR-6c).""" + + unit: str # the unit combined and split (hats -> item) + divisor: int # number of equal parts (3 boxes) + container: str # SINGULAR container noun (box) — must match the perquery's + + +def _singular(noun: str) -> str: + """Conservative singularization for container nouns (``boxes`` -> ``box``, + ``bags`` -> ``bag``); already-singular nouns (``box``) pass through unchanged. + + Used ONLY to canonicalize the partition container so the "split into N boxes" + sentence and the "in each box" query name the same ``per_`` symbol. + """ + if noun.endswith("es") and noun[:-2].endswith(("x", "s", "z", "ch", "sh")): + return noun[:-2] + if noun.endswith("s") and len(noun) > 1: + return noun[:-1] + return noun + + #: Word factors for "twice/double/triple ... as many" (a multiply by a dimensionless int). _FACTOR_WORDS: dict[str, int] = {"twice": 2, "double": 2, "triple": 3, "quadruple": 4} @@ -218,17 +246,40 @@ def _parse_sentence(body: str, detail: str): if not toks: return None - if len(toks) >= 5 and toks[0] == "how" and toks[1] == "many" and toks[-1] == "have": + if len(toks) >= 5 and toks[0] == "how" and toks[1] == "many": unit = _resolve_unit(_ident(toks[2], detail)) - rest = toks[3:-1] # between "" and "have" - if rest and rest[0] == "does" and len(rest) == 2: - return ("query", _ident(rest[1], detail), unit) - if rest and rest[0] == "do": - parts = [_ident(t, detail) for t in rest[1:] if t != "and"] - if len(parts) >= 2: - return ("sumquery", tuple(parts), unit) + # "How many are in each ?" -> the partition per-container target. + if len(toks) == 7 and toks[3] == "are" and toks[4] == "in" and toks[5] == "each": + return ("perquery", _singular(_ident(toks[6], detail)), unit) + if toks[-1] == "have": + rest = toks[3:-1] # between "" and "have" + if rest and rest[0] == "does" and len(rest) == 2: + return ("query", _ident(rest[1], detail), unit) + if rest and rest[0] == "do": + parts = [_ident(t, detail) for t in rest[1:] if t != "and"] + if len(parts) >= 2: + return ("sumquery", tuple(parts), unit) raise _QReject("unreadable_quantity_query", detail) + # Partition: "They combine their and split them equally into ." + if ( + len(toks) == 11 + and toks[0] == "they" + and toks[1] == "combine" + and toks[2] == "their" + and toks[4] == "and" + and toks[5] == "split" + and toks[6] == "them" + and toks[7] == "equally" + and toks[8] == "into" + and toks[9].isdigit() + ): + return _Partition( + unit=_resolve_unit(_ident(toks[3], detail)), + divisor=_int(toks[9], detail), + container=_singular(_ident(toks[10], detail)), + ) + if len(toks) >= 4 and toks[1] == "has": entity = _ident(toks[0], detail) # Multiplicative comparative is checked BEFORE the digit gate (its factor may be @@ -265,6 +316,7 @@ def comprehend_quantitative(text: str, source_id: str = "input") -> QuantCompreh eqs: list[_Eq] = [] muls: list[_Mul] = [] divs: list[_Div] = [] + partitions: list[_Partition] = [] queries: list[tuple] = [] try: @@ -280,6 +332,8 @@ def comprehend_quantitative(text: str, source_id: str = "input") -> QuantCompreh muls.append(spec) elif isinstance(spec, _Div): divs.append(spec) + elif isinstance(spec, _Partition): + partitions.append(spec) else: queries.append(spec) except _QReject as rej: @@ -287,6 +341,9 @@ def comprehend_quantitative(text: str, source_id: str = "input") -> QuantCompreh if len(queries) != 1 or not facts: return Refusal("no_single_quantity_query") + if len(partitions) > 1: + return Refusal("multiple_partitions") + partition = partitions[0] if partitions else None unit_of: dict[str, str] = {} role_of: dict[str, str] = {} @@ -300,9 +357,28 @@ def comprehend_quantitative(text: str, source_id: str = "input") -> QuantCompreh unit_of[d.entity], role_of[d.entity] = d.unit, "count" query = queries[0] + # A partition is read ONLY together with its "in each " query, and vice + # versa — a partition without that target, or that target without a partition, refuses. + if (partition is not None) != (query[0] == "perquery"): + return Refusal("partition_query_mismatch") + sum_eq: tuple[str, tuple[str, ...]] | None = None + partition_eq: tuple[str, str, int] | None = None # (per_box, total, divisor) if query[0] == "query": ask_entity, ask_unit = query[1], query[2] + elif query[0] == "perquery": + # Aggregate-then-divide: total = sum(all facts); per_ = total / divisor. + container, ask_unit = query[1], query[2] + assert partition is not None # guaranteed by the mismatch guard above + if partition.container != container: + return Refusal("partition_container_mismatch") + ask_entity = "per_" + container + unit_of.setdefault("total", partition.unit) + role_of["total"] = "total" + unit_of.setdefault(ask_entity, partition.unit) + role_of[ask_entity] = "count" + sum_eq = ("total", tuple(f.entity for f in facts)) + partition_eq = (ask_entity, "total", partition.divisor) else: # sumquery -> synthesize a total symbol + sum equation parts, ask_unit = query[1], query[2] ask_entity = "total" @@ -322,6 +398,9 @@ def comprehend_quantitative(text: str, source_id: str = "input") -> QuantCompreh if sum_eq is not None: referenced.add(sum_eq[0]) referenced.update(sum_eq[1]) + if partition_eq is not None: + referenced.add(partition_eq[0]) + referenced.add(partition_eq[1]) referenced.add(ask_entity) symbols = [ @@ -358,6 +437,11 @@ def comprehend_quantitative(text: str, source_id: str = "input") -> QuantCompreh if sum_eq is not None: lhs, parts = sum_eq expr_specs.append((lhs, SumOf(tuple(Symbol(p) for p in parts)))) + # The partition divide is appended AFTER the sum so ``total`` is forward-resolved + # before ``per_ = total / divisor`` (the oracle substitutes in this order). + if partition_eq is not None: + lhs, ref, divisor = partition_eq + expr_specs.append((lhs, Div(Symbol(ref), Literal(divisor)))) # equations: shell -> REAL admissibility -> rebuild (NEVER stamp "admitted"). equations: list[BoundEquation] = [] diff --git a/tests/test_quantitative_comprehension.py b/tests/test_quantitative_comprehension.py index 2d87275f..69c4241e 100644 --- a/tests/test_quantitative_comprehension.py +++ b/tests/test_quantitative_comprehension.py @@ -8,6 +8,8 @@ reviewer's "do not stamp admissibility" guard, made executable. from __future__ import annotations +import pytest + from generate.binding_graph.model import ( BoundFact, BoundUnknown, @@ -218,3 +220,73 @@ def test_half_as_many_missing_base_refuses() -> None: # "half as many ... as Rod" with no value for Rod -> ungrounded base -> REFUSE. comp = comprehend_quantitative("Sue has half as many pears as Rod. How many pears does Sue have?") assert isinstance(comp, Refusal) + + +# --------------------------------------------------------------------------- # +# PR-6d — aggregate-then-divide partition (SumOf + Div, no new relation kind) +# --------------------------------------------------------------------------- # + +_PARTITION_TEXT = ( + "Lee has 5 hats. Mae has 7 hats. They combine their hats and split them " + "equally into 3 boxes. How many hats are in each box?" +) + + +def test_partition_builds_sum_then_divide() -> None: + # PR-6d: one sentence synthesizes TWO derived symbols — total = lee + mae (sum_of) + # and per_box = total / 3 (divide_by, the FIRST divide whose ref is itself derived). + comp = _comp(_PARTITION_TEXT) + by_lhs = {e.lhs_symbol_id: e for e in comp.binding_graph.equations} + total = by_lhs["total"] + assert total.operation_kind == "add" + assert total.rhs_canonical == "lee + mae" + assert total.dependencies == frozenset({"lee", "mae"}) + per_box = by_lhs["per_box"] + assert per_box.operation_kind == "divide" + assert per_box.rhs_canonical == "total / 3" + assert per_box.dependencies == frozenset({"total"}) # ref is a DERIVED symbol + assert total.admissibility_status == per_box.admissibility_status == "admitted" + assert single_unknown(comp.binding_graph).symbol_id == "per_box" + # Only the modelled entities — the partition introduces no proof-machinery symbol. + assert {s.symbol_id for s in comp.binding_graph.symbols} == {"lee", "mae", "total", "per_box"} + + +def test_partition_without_its_query_refuses() -> None: + # A partition sentence whose question is a plain "does X have" (not "in each box") + # is incoherent -> REFUSE, never read a dangling partition. + comp = comprehend_quantitative( + "Lee has 5 hats. Mae has 7 hats. They combine their hats and split them " + "equally into 3 boxes. How many hats does Lee have?" + ) + assert isinstance(comp, Refusal) + + +def test_per_each_query_without_partition_refuses() -> None: + # "in each box" with no partition sentence -> no per-box symbol exists -> REFUSE. + comp = comprehend_quantitative("Lee has 5 hats. How many hats are in each box?") + assert isinstance(comp, Refusal) + + +def test_partition_container_mismatch_refuses() -> None: + # Split into boxes but asked "in each jar" -> container mismatch -> REFUSE. + comp = comprehend_quantitative( + "Lee has 5 hats. Mae has 7 hats. They combine their hats and split them " + "equally into 3 boxes. How many hats are in each jar?" + ) + assert isinstance(comp, Refusal) + + +def test_partition_setup_correct_but_non_exact_answer_refuses() -> None: + # The reading is correct (total = 5 + 6, per_box = total / 3), but 11 % 3 != 0, so + # the answer oracle REFUSES — exact-divisibility still gates the partition's answer. + from evals.relational_metric.oracle import OracleError, oracle_answer + + comp = _comp( + "Lee has 5 hats. Mae has 6 hats. They combine their hats and split them " + "equally into 3 boxes. How many hats are in each box?" + ) + projected = to_relational_metric(comp) + assert projected is not None # the SETUP is readable + relations, query = projected + with pytest.raises(OracleError): # but 11 / 3 is non-exact -> the answer refuses + oracle_answer(relations, query) diff --git a/tests/test_setup_oracle.py b/tests/test_setup_oracle.py index c92ccba8..3fa394ae 100644 --- a/tests/test_setup_oracle.py +++ b/tests/test_setup_oracle.py @@ -165,8 +165,10 @@ def test_r1_comparative_supported_rest_refused_wrong_zero() -> None: assert by_id["r1-05-chain"] == "correct" # Divisive frame (PR-6c): "half as many" (r1-02). assert by_id["r1-02-half"] == "correct" - assert r["setup_correct"] == 3 - assert r["setup_refused"] == 7 + # Partition frame (PR-6d): aggregate-then-divide "split equally into 3 boxes" (r1-06). + assert by_id["r1-06-subtotal-reused"] == "correct" + assert r["setup_correct"] == 4 + assert r["setup_refused"] == 6 # No detail is ever WRONG, and every non-correct one is a typed refusal. for d in r["details"]: assert d["outcome"] in ("correct", "refused") @@ -211,14 +213,16 @@ def test_r1_answer_lane_scores_only_setup_correct_fixtures() -> None: assert r["setup_wrong"] == 0 assert r["wrong"] == 0 assert r["gold_error"] == 0 - assert r["correct"] == 3 - assert r["refused"] == 7 + assert r["correct"] == 4 + assert r["refused"] == 6 by_id = {d["id"]: d for d in r["details"]} assert by_id["r1-01-twice"] == {"id": "r1-01-twice", "outcome": "correct", "answer": 12} assert by_id["r1-02-half"] == {"id": "r1-02-half", "outcome": "correct", "answer": 4} assert by_id["r1-05-chain"] == {"id": "r1-05-chain", "outcome": "correct", "answer": 14} + # PR-6d: the partition's derived per-box answer (total 12 / 3 boxes = 4). + assert by_id["r1-06-subtotal-reused"] == {"id": "r1-06-subtotal-reused", "outcome": "correct", "answer": 4} for fixture_id, detail in by_id.items(): - if fixture_id not in {"r1-01-twice", "r1-02-half", "r1-05-chain"}: + if fixture_id not in {"r1-01-twice", "r1-02-half", "r1-05-chain", "r1-06-subtotal-reused"}: assert detail["outcome"] == "refused" assert detail.get("reason")