diff --git a/evals/setup_oracle/r1_gold.jsonl b/evals/setup_oracle/r1_gold.jsonl index 42e276db..76007365 100644 --- a/evals/setup_oracle/r1_gold.jsonl +++ b/evals/setup_oracle/r1_gold.jsonl @@ -1,7 +1,7 @@ {"id": "r1-01-twice", "text": "Anna has 6 apples. Bella has twice as many apples as Anna. How many apples does Bella have?", "relations": [{"kind": "fact", "entity": "anna", "value": 6}, {"kind": "times_as_many", "entity": "bella", "ref": "anna", "factor": 2}], "expected_units": {"anna": "item", "bella": "item"}, "query": {"entity": "bella", "unit": "item"}, "gold": 12, "notes": "Multiplicative: bella = 2 * anna. The reader has no multiplicative template AND 'twice' is non-digit -> must REFUSE, never misread as a fact/additive."} {"id": "r1-02-half", "text": "Carl has 8 coins. Dora has half as many coins as Carl. How many coins does Dora have?", "relations": [{"kind": "fact", "entity": "carl", "value": 8}, {"kind": "divide_by", "entity": "dora", "ref": "carl", "divisor": 2}], "expected_units": {"carl": "item", "dora": "item"}, "query": {"entity": "dora", "unit": "item"}, "gold": 4, "notes": "Divisive: dora = carl / 2 = 4 (exact integer division). 'half as many' fits the [Y has WORD as many UNIT as X] template; the WORD maps to a divisor, not a factor (PR-6c). Exact-divisibility is the wrong=0 boundary: an odd base would refuse, never round."} -{"id": "r1-03-more-total", "text": "Finn has 10 books. Evan has 5 more books than Finn. How many books do Evan and Finn have altogether?", "relations": [{"kind": "fact", "entity": "finn", "value": 10}, {"kind": "more_than", "entity": "evan", "ref": "finn", "delta": 5}, {"kind": "sum_of", "entity": "total", "parts": ["evan", "finn"]}], "expected_units": {"finn": "item", "evan": "item", "total": "item"}, "query": {"entity": "total", "unit": "item"}, "notes": "Additive + aggregate. 'altogether' (not the bare 'have') may fall outside the strict query template -> likely REFUSE; if read, structure must match exactly."} -{"id": "r1-04-fewer-total", "text": "Gail has 20 cards. Hank has 6 fewer cards than Gail. How many cards do Gail and Hank have in total?", "relations": [{"kind": "fact", "entity": "gail", "value": 20}, {"kind": "fewer_than", "entity": "hank", "ref": "gail", "delta": 6}, {"kind": "sum_of", "entity": "total", "parts": ["gail", "hank"]}], "expected_units": {"gail": "item", "hank": "item", "total": "item"}, "query": {"entity": "total", "unit": "item"}, "notes": "Additive(fewer) + aggregate. 'in total' qualifier likely outside the query template -> REFUSE expected."} +{"id": "r1-03-more-total", "text": "Finn has 10 books. Evan has 5 more books than Finn. How many books do Evan and Finn have altogether?", "relations": [{"kind": "fact", "entity": "finn", "value": 10}, {"kind": "more_than", "entity": "evan", "ref": "finn", "delta": 5}, {"kind": "sum_of", "entity": "total", "parts": ["evan", "finn"]}], "expected_units": {"finn": "item", "evan": "item", "total": "item"}, "query": {"entity": "total", "unit": "item"}, "gold": 25, "notes": "Additive + aggregate (aggregate-query slice): the query closes with a trailing qualifier 'altogether' after 'have'; the recognizer strips it and reads the multi-part sum, honored ONLY for the multi-part form. evan = finn + 5 = 15; total = evan + finn = 25. Parts are grounded (finn fact, evan via more_than) and unit-compatible; an ungrounded or unit-incompatible part refuses downstream at admissibility (unit_unbound / unit_mismatch)."} +{"id": "r1-04-fewer-total", "text": "Gail has 20 cards. Hank has 6 fewer cards than Gail. How many cards do Gail and Hank have in total?", "relations": [{"kind": "fact", "entity": "gail", "value": 20}, {"kind": "fewer_than", "entity": "hank", "ref": "gail", "delta": 6}, {"kind": "sum_of", "entity": "total", "parts": ["gail", "hank"]}], "expected_units": {"gail": "item", "hank": "item", "total": "item"}, "query": {"entity": "total", "unit": "item"}, "gold": 34, "notes": "Additive(fewer) + aggregate (aggregate-query slice): the query closes with a two-token qualifier 'in total' after 'have'; the recognizer strips it and reads the multi-part sum, honored ONLY for the multi-part form. hank = gail - 6 = 14; total = gail + hank = 34."} {"id": "r1-05-chain", "text": "Ivy has 4 pens. Jon has 3 times as many pens as Ivy. Kim has 2 more pens than Jon. How many pens does Kim have?", "relations": [{"kind": "fact", "entity": "ivy", "value": 4}, {"kind": "times_as_many", "entity": "jon", "ref": "ivy", "factor": 3}, {"kind": "more_than", "entity": "kim", "ref": "jon", "delta": 2}], "expected_units": {"ivy": "item", "jon": "item", "kim": "item"}, "query": {"entity": "kim", "unit": "item"}, "gold": 14, "notes": "Multi-step derived chain: jon=3*ivy (intermediate), kim=jon+2. The multiplicative middle step has no template -> must REFUSE the whole reading, never read a partial/wrong chain."} {"id": "r1-06-subtotal-reused", "text": "Lee has 5 hats. Mae has 7 hats. They combine their hats and split them equally into 3 boxes. How many hats are in each box?", "relations": [{"kind": "fact", "entity": "lee", "value": 5}, {"kind": "fact", "entity": "mae", "value": 7}, {"kind": "sum_of", "entity": "total", "parts": ["lee", "mae"]}, {"kind": "divide_by", "entity": "per_box", "ref": "total", "divisor": 3}], "expected_units": {"lee": "item", "mae": "item", "total": "item", "per_box": "item"}, "query": {"entity": "per_box", "unit": "item"}, "gold": 4, "notes": "Aggregate-then-divide partition (PR-6d): semantic source is equal PARTITION ('split equally into 3 boxes'); the mathematical setup is total = lee + mae (sum_of) then per_box = total / 3 (divide_by), reusing SumOf + Div with NO new relation kind. First derived-subtotal reuse: the divide's ref is itself a derived symbol. Exact-divisibility still gates the answer (total % 3 == 0; 12 / 3 = 4)."} {"id": "r1-07-inverse", "text": "Nia has 9 more beads than Omar. Nia has 15 beads. How many beads does Omar have?", "relations": [{"kind": "fact", "entity": "nia", "value": 15}, {"kind": "more_than", "entity": "nia", "ref": "omar", "delta": 9}, {"kind": "ask_base", "entity": "omar"}], "expected_units": {"nia": "item", "omar": "item"}, "query": {"entity": "omar", "unit": "item"}, "notes": "Inverse target: omar = nia - 9 = 6, asked via the base of a more_than. The reader may read the STRUCTURE (nia=15, nia=omar+9, ask omar) — that setup is correct; only the solver can't invert. So setup_correct OR refuse, but NEVER setup_wrong."} diff --git a/generate/quantitative_comprehension.py b/generate/quantitative_comprehension.py index e1d8e8e3..db802762 100644 --- a/generate/quantitative_comprehension.py +++ b/generate/quantitative_comprehension.py @@ -18,7 +18,7 @@ Templates (function-word + order; digits only — a non-digit quantity REFUSES): - `` has more than `` -> BoundEquation(Y = X + N) op=add - `` has fewer than `` -> BoundEquation(Y = X - N) op=subtract - query ``How many does have`` -> ask Y - - query ``How many do and have`` -> total = X + Y; ask total + - query ``How many do and have [altogether|in total]`` -> total = X + Y; ask total Refusal-first: an unparseable clause, a non-digit quantity, a non-identifier name, a missing/duplicated query, or an admissibility refusal all return a typed @@ -251,9 +251,22 @@ def _parse_sentence(body: str, detail: str): # "How many are in each ?" -> the partition per-container target. if len(toks) == 7 and toks[3] == "are" and toks[4] == "in" and toks[5] == "each": return ("perquery", _singular(_ident(toks[6], detail)), unit) - if toks[-1] == "have": - rest = toks[3:-1] # between "" and "have" - if rest and rest[0] == "does" and len(rest) == 2: + # An aggregate query may close with a qualifier AFTER "have": + # "... have altogether?" or "... have in total?". Strip it so the + # "have"-terminal templates apply; the qualifier is honored ONLY for the + # multi-part aggregate (sumquery) form, never the single-entity query + # ("does X have altogether?" is nonsensical -> refuses). It adds no new + # relation kind: the parts still flow through sum_of, and an ungrounded or + # unit-incompatible part is refused downstream by admissibility + # (unit_unbound / unit_mismatch), so the recognizer cannot over-read. + core, aggregate = toks, False + if toks[-1] == "altogether": + core, aggregate = toks[:-1], True + elif toks[-1] == "total" and toks[-2] == "in": + core, aggregate = toks[:-2], True + if core[-1] == "have": + rest = core[3:-1] # between "" and "have" + if not aggregate and rest and rest[0] == "does" and len(rest) == 2: return ("query", _ident(rest[1], detail), unit) if rest and rest[0] == "do": parts = [_ident(t, detail) for t in rest[1:] if t != "and"] diff --git a/tests/test_quantitative_comprehension.py b/tests/test_quantitative_comprehension.py index 69c4241e..d0dc6169 100644 --- a/tests/test_quantitative_comprehension.py +++ b/tests/test_quantitative_comprehension.py @@ -290,3 +290,74 @@ def test_partition_setup_correct_but_non_exact_answer_refuses() -> None: relations, query = projected with pytest.raises(OracleError): # but 11 / 3 is non-exact -> the answer refuses oracle_answer(relations, query) + + +# --------------------------------------------------------------------------- # +# Additive aggregate query variants: "... have altogether?" / "... have in total?" +# A trailing qualifier after "have" is stripped and honored ONLY for the multi-part +# aggregate (sumquery) form. No new arithmetic, no new relation kind: the parts flow +# through sum_of, and admissibility still gates grounding + unit-compatibility. +# --------------------------------------------------------------------------- # + + +def test_aggregate_query_altogether_reads_and_sums() -> None: + from evals.relational_metric.oracle import oracle_answer + + comp = _comp( + "Finn has 10 books. Evan has 5 more books than Finn. " + "How many books do Evan and Finn have altogether?" + ) + assert single_unknown(comp.binding_graph).symbol_id == "total" + total_eq = next(e for e in comp.binding_graph.equations if e.lhs_symbol_id == "total") + assert total_eq.operation_kind == "add" + assert set(total_eq.dependencies) == {"evan", "finn"} + relations, query = to_relational_metric(comp) + assert oracle_answer(relations, query) == 25 # evan=15, finn=10 + + +def test_aggregate_query_in_total_reads_and_sums() -> None: + from evals.relational_metric.oracle import oracle_answer + + comp = _comp( + "Gail has 20 cards. Hank has 6 fewer cards than Gail. " + "How many cards do Gail and Hank have in total?" + ) + assert single_unknown(comp.binding_graph).symbol_id == "total" + relations, query = to_relational_metric(comp) + assert oracle_answer(relations, query) == 34 # gail=20, hank=14 + + +def test_aggregate_qualifier_on_single_entity_refuses() -> None: + # The qualifier is honored ONLY for the multi-part form. A single-entity query + # carrying "altogether" is nonsensical and must REFUSE. This is load-bearing: the + # ``not aggregate`` guard is what blocks the "does X have" template from firing on + # an aggregate-qualified question and silently reading a single grounded fact. + comp = comprehend_quantitative("Anna has 6 apples. How many apples does Anna have altogether?") + assert isinstance(comp, Refusal) + assert comp.reason == "unreadable_quantity_query" + + +def test_aggregate_query_ungrounded_part_refuses() -> None: + # Widening the recognizer cannot admit an UNGROUNDED part: "zoe" has no fact or + # derivation, so its unit is unbound and the sum's admissibility REFUSES rather than + # fabricating a partial total. (wrong=0 boundary — the recognizer over-reads the + # surface, admissibility refuses to ground it.) + comp = comprehend_quantitative( + "Finn has 10 books. Evan has 5 more books than Finn. " + "How many books do Evan and Zoe have altogether?" + ) + assert isinstance(comp, Refusal) + assert comp.reason == "admissibility_refused" + assert "unit_unbound" in comp.detail + + +def test_aggregate_query_unit_incompatible_part_refuses() -> None: + # ... and cannot admit UNIT-INCOMPATIBLE parts: dollars (currency) + books (item) + # is a mixed-dimension sum, refused by the REAL additive unit check. + comp = comprehend_quantitative( + "Anna has 5 dollars. Bella has 3 books. " + "How many books do Anna and Bella have altogether?" + ) + assert isinstance(comp, Refusal) + assert comp.reason == "admissibility_refused" + assert "unit_mismatch" in comp.detail diff --git a/tests/test_setup_oracle.py b/tests/test_setup_oracle.py index 3fa394ae..2f04ed08 100644 --- a/tests/test_setup_oracle.py +++ b/tests/test_setup_oracle.py @@ -167,8 +167,13 @@ def test_r1_comparative_supported_rest_refused_wrong_zero() -> None: assert by_id["r1-02-half"] == "correct" # Partition frame (PR-6d): aggregate-then-divide "split equally into 3 boxes" (r1-06). assert by_id["r1-06-subtotal-reused"] == "correct" - assert r["setup_correct"] == 4 - assert r["setup_refused"] == 6 + # Aggregate-query frame (aggregate-query slice): additive total asked via a trailing + # qualifier — "altogether" (r1-03) and "in total" (r1-04). Phrasing-only widening of + # the existing sum_of; no new arithmetic or relation kind. + assert by_id["r1-03-more-total"] == "correct" + assert by_id["r1-04-fewer-total"] == "correct" + assert r["setup_correct"] == 6 + assert r["setup_refused"] == 4 # No detail is ever WRONG, and every non-correct one is a typed refusal. for d in r["details"]: assert d["outcome"] in ("correct", "refused") @@ -213,16 +218,23 @@ def test_r1_answer_lane_scores_only_setup_correct_fixtures() -> None: assert r["setup_wrong"] == 0 assert r["wrong"] == 0 assert r["gold_error"] == 0 - assert r["correct"] == 4 - assert r["refused"] == 6 + assert r["correct"] == 6 + assert r["refused"] == 4 by_id = {d["id"]: d for d in r["details"]} assert by_id["r1-01-twice"] == {"id": "r1-01-twice", "outcome": "correct", "answer": 12} assert by_id["r1-02-half"] == {"id": "r1-02-half", "outcome": "correct", "answer": 4} assert by_id["r1-05-chain"] == {"id": "r1-05-chain", "outcome": "correct", "answer": 14} # PR-6d: the partition's derived per-box answer (total 12 / 3 boxes = 4). assert by_id["r1-06-subtotal-reused"] == {"id": "r1-06-subtotal-reused", "outcome": "correct", "answer": 4} + # Aggregate-query slice: additive totals via "altogether" / "in total". + assert by_id["r1-03-more-total"] == {"id": "r1-03-more-total", "outcome": "correct", "answer": 25} + assert by_id["r1-04-fewer-total"] == {"id": "r1-04-fewer-total", "outcome": "correct", "answer": 34} + _supported = { + "r1-01-twice", "r1-02-half", "r1-05-chain", "r1-06-subtotal-reused", + "r1-03-more-total", "r1-04-fewer-total", + } for fixture_id, detail in by_id.items(): - if fixture_id not in {"r1-01-twice", "r1-02-half", "r1-05-chain", "r1-06-subtotal-reused"}: + if fixture_id not in _supported: assert detail["outcome"] == "refused" assert detail.get("reason")