From 7f67cea40066ab7d4d8e172f3a1a67b3deddb5a2 Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 23 May 2026 19:42:55 -0700 Subject: [PATCH] =?UTF-8?q?feat(ADR-0131.G.5):=20aggregate=20answer=20comp?= =?UTF-8?q?osition=20=E2=80=94=20combined/together=20cues=20wired,=20axis?= =?UTF-8?q?=20lane=2020/20,=20wrong=3D=3D0=20(#197)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the vocabulary gap: `combined` and `together` added to `_Q_TOTAL_RE` and `_Q_ENTITY_RE` tail alternations. Both map to `entity=None` semantics; the solver's existing sum path is unchanged. Ships: - Parser one-line regex extension (`generate/math_candidate_parser.py`) - 20-case curated axis lane (`G5_aggregate/v1/`) — 5 shapes × 4 cues - Runner + byte-equal report (20/20 pass, wrong=0) - 25 tests covering cue vocab, 2/3-entity sums, degenerate aggregate, refusals, byte-equality, B3 regression guard, GSM8K safety rail - ADR-0131.G.5 No admission movement on GSM8K probe (statement-parse bottleneck unchanged). --- ...R-0131.G.5-aggregate-answer-composition.md | 89 +++++++ .../G5_aggregate/__init__.py | 0 .../G5_aggregate/v1/__init__.py | 0 .../G5_aggregate/v1/cases.jsonl | 20 ++ .../G5_aggregate/v1/report.json | 218 ++++++++++++++++++ .../G5_aggregate/v1/runner.py | 129 +++++++++++ generate/math_candidate_parser.py | 12 +- tests/test_adr_0131_G5_aggregate.py | 162 +++++++++++++ 8 files changed, 626 insertions(+), 4 deletions(-) create mode 100644 docs/decisions/ADR-0131.G.5-aggregate-answer-composition.md create mode 100644 evals/math_capability_axes/G5_aggregate/__init__.py create mode 100644 evals/math_capability_axes/G5_aggregate/v1/__init__.py create mode 100644 evals/math_capability_axes/G5_aggregate/v1/cases.jsonl create mode 100644 evals/math_capability_axes/G5_aggregate/v1/report.json create mode 100644 evals/math_capability_axes/G5_aggregate/v1/runner.py create mode 100644 tests/test_adr_0131_G5_aggregate.py diff --git a/docs/decisions/ADR-0131.G.5-aggregate-answer-composition.md b/docs/decisions/ADR-0131.G.5-aggregate-answer-composition.md new file mode 100644 index 00000000..a847bef9 --- /dev/null +++ b/docs/decisions/ADR-0131.G.5-aggregate-answer-composition.md @@ -0,0 +1,89 @@ +# ADR-0131.G.5 — Aggregate Answer Composition + +**Status:** Accepted +**Parent:** [ADR-0131.G — GSM8K Coverage Probe](ADR-0131.G-gsm8k-coverage-probe.md) +**Date:** 2026-05-23 + +## Context + +The aggregate-answer path — questions like "How many apples do they have +altogether?" — was functionally complete before this ADR. The parser +(`_Q_TOTAL_RE` in `generate/math_candidate_parser.py`) already emitted +`Unknown(entity=None, unit=)` for aggregate cues, and the solver +(`generate/math_solver.py`) already summed all terminal state entries +matching the questioned unit when `entity is None`. + +What was missing: + +1. **Vocabulary gap (now closed):** `"combined"` and `"together"` were + absent from `_Q_TOTAL_RE`'s tail alternation, causing questions using + those cues to be refused even when the solver would have produced the + correct sum. +2. **No pinned lane:** no curated axis cases proved the 2-entity, + 3-entity, and degenerate aggregate paths end-to-end through + `parse_and_solve`. + +## Decision + +### Closed aggregate-cue vocabulary + +Exactly four cues are admitted: + +| Cue | Example tail | +|-----|-------------| +| `in total` | "How many apples do they have in total?" | +| `altogether` | "How many apples do they have altogether?" | +| `combined` | "How many apples do they have combined?" | +| `together` | "How many apples do they have together?" | + +All four map to `entity=None` semantics — the solver sums all state +entries whose unit matches the questioned unit, across all entities. + +### Solver path (pre-existing) + +The `entity is None` branch in `_resolve_unknown` was not changed. It +sums `v for (_, unit), v in state.items() if unit == unknown.unit`. +This ADR extends the cue vocabulary and pins the lane, not the solver. + +### Axis lane + +20 curated cases at `evals/math_capability_axes/G5_aggregate/v1/cases.jsonl`: + +| Shape | Count | Purpose | +|-------|-------|---------| +| 2-entity sum, no operations | 4 | one case per cue | +| 3-entity sum, no operations | 4 | one case per cue | +| 2-entity sum with add/subtract op | 4 | mixed cues | +| Single-entity degenerate | 4 | regression guard | +| Refusal: outside closed cue | 4 | wrong==0 probe | + +Refusal cases use question forms outside the closed `_Q_TOTAL_RE` +pattern (e.g., "How many apples does everyone have?", "What is the +total number of coins?") to verify the parser correctly refuses +paraphrases not in the closed cue set. + +### Gate + +`wrong == 0` on every axis case. GSM8K `admitted_wrong == 0` preserved +(no admission movement expected — all 50 sample cases still refuse at +statement parsing; question-layer work cannot lift that). + +## Deferred + +- **Implicit aggregation without a cue word:** "How many apples do Sam + and Tom have?" requires coreference resolution (named-entity → + pronoun-equivalent grouping). Out of scope for the closed-cue model. +- **Rate-based aggregation:** "How many dollars did they earn in total?" + where the unit derives from a rate operation. Requires rate-verb + support in the statement parser. +- **GSM8K admission lift:** all 50 sample cases fail at statement + parsing (rate verbs, compound sentences, implicit entities). + Question-layer cue extensions cannot move that number. + +## Evidence + +- Axis runner: `evals/math_capability_axes/G5_aggregate/v1/runner.py` +- Report: `evals/math_capability_axes/G5_aggregate/v1/report.json` +- Tests: `tests/test_adr_0131_G5_aggregate.py` +- B3 lane unchanged. +- GSM8K `admitted_wrong == 0` preserved. diff --git a/evals/math_capability_axes/G5_aggregate/__init__.py b/evals/math_capability_axes/G5_aggregate/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/math_capability_axes/G5_aggregate/v1/__init__.py b/evals/math_capability_axes/G5_aggregate/v1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/math_capability_axes/G5_aggregate/v1/cases.jsonl b/evals/math_capability_axes/G5_aggregate/v1/cases.jsonl new file mode 100644 index 00000000..f329b6ae --- /dev/null +++ b/evals/math_capability_axes/G5_aggregate/v1/cases.jsonl @@ -0,0 +1,20 @@ +{"case_id": "G5-2ent-001", "category": "2entity_no_op", "cue": "altogether", "problem": "Sam has 5 apples. Tom has 3 apples. How many apples do they have altogether?", "expected_answer": 8.0} +{"case_id": "G5-2ent-002", "category": "2entity_no_op", "cue": "in total", "problem": "Alice has 7 books. Bob has 4 books. How many books do they have in total?", "expected_answer": 11.0} +{"case_id": "G5-2ent-003", "category": "2entity_no_op", "cue": "combined", "problem": "Maya has 6 coins. Leo has 9 coins. How many coins do they have combined?", "expected_answer": 15.0} +{"case_id": "G5-2ent-004", "category": "2entity_no_op", "cue": "together", "problem": "Jade has 12 stickers. Finn has 8 stickers. How many stickers do they have together?", "expected_answer": 20.0} +{"case_id": "G5-3ent-001", "category": "3entity_no_op", "cue": "altogether", "problem": "Sam has 5 apples. Tom has 3 apples. Amy has 2 apples. How many apples do they have altogether?", "expected_answer": 10.0} +{"case_id": "G5-3ent-002", "category": "3entity_no_op", "cue": "in total", "problem": "Alice has 4 books. Bob has 6 books. Carol has 2 books. How many books do they have in total?", "expected_answer": 12.0} +{"case_id": "G5-3ent-003", "category": "3entity_no_op", "cue": "combined", "problem": "Maya has 10 coins. Leo has 5 coins. Nina has 3 coins. How many coins do they have combined?", "expected_answer": 18.0} +{"case_id": "G5-3ent-004", "category": "3entity_no_op", "cue": "together", "problem": "Jade has 7 stickers. Finn has 4 stickers. Rex has 9 stickers. How many stickers do they have together?", "expected_answer": 20.0} +{"case_id": "G5-op-001", "category": "2entity_with_op", "cue": "altogether", "problem": "Sam has 5 apples. Sam buys 3 apples. Tom has 4 apples. How many apples do they have altogether?", "expected_answer": 12.0} +{"case_id": "G5-op-002", "category": "2entity_with_op", "cue": "combined", "problem": "Alice has 10 books. Alice loses 2 books. Bob has 6 books. How many books do they have combined?", "expected_answer": 14.0} +{"case_id": "G5-op-003", "category": "2entity_with_op", "cue": "in total", "problem": "Maya has 8 coins. Leo has 5 coins. Leo finds 3 coins. How many coins do they have in total?", "expected_answer": 16.0} +{"case_id": "G5-op-004", "category": "2entity_with_op", "cue": "together", "problem": "Jade has 12 stickers. Jade gives away 4 stickers. Finn has 8 stickers. How many stickers do they have together?", "expected_answer": 16.0} +{"case_id": "G5-degen-001", "category": "single_entity_total_cue", "cue": "in total", "problem": "Sam has 5 apples. How many apples do they have in total?", "expected_answer": 5.0} +{"case_id": "G5-degen-002", "category": "single_entity_total_cue", "cue": "altogether", "problem": "Alice has 7 books. Alice buys 3 books. How many books do they have altogether?", "expected_answer": 10.0} +{"case_id": "G5-degen-003", "category": "single_entity_total_cue", "cue": "combined", "problem": "Maya has 9 coins. Maya loses 2 coins. How many coins do they have combined?", "expected_answer": 7.0} +{"case_id": "G5-degen-004", "category": "single_entity_total_cue", "cue": "together", "problem": "Finn has 6 stickers. How many stickers do they have together?", "expected_answer": 6.0} +{"case_id": "G5-refuse-001", "category": "refusal_outside_closed_cue", "cue": "none", "problem": "Sam has 5 apples. Tom has 3 apples. How many apples does everyone have?", "expected_answer": null} +{"case_id": "G5-refuse-002", "category": "refusal_outside_closed_cue", "cue": "none", "problem": "Alice has 4 coins. Bob has 6 coins. What is the total number of coins?", "expected_answer": null} +{"case_id": "G5-refuse-003", "category": "refusal_outside_closed_cue", "cue": "none", "problem": "Maya has 10 books. Leo has 5 books. How many books do Sam and Leo have?", "expected_answer": null} +{"case_id": "G5-refuse-004", "category": "refusal_outside_closed_cue", "cue": "none", "problem": "Jade has 8 stickers. Finn has 4 stickers. How many stickers are there?", "expected_answer": null} diff --git a/evals/math_capability_axes/G5_aggregate/v1/report.json b/evals/math_capability_axes/G5_aggregate/v1/report.json new file mode 100644 index 00000000..34432ed4 --- /dev/null +++ b/evals/math_capability_axes/G5_aggregate/v1/report.json @@ -0,0 +1,218 @@ +{ + "adr": "0131.G.5", + "axis": "aggregate", + "cases_path": "evals/math_capability_axes/G5_aggregate/v1/cases.jsonl", + "metrics": { + "cases_total": 20, + "pass_rate": 1.0, + "passed": 20, + "wrong": 0, + "wrong_count_is_zero": true, + "wrong_rate": 0.0 + }, + "per_case": [ + { + "answer": 8.0, + "case_id": "G5-2ent-001", + "category": "2entity_no_op", + "cue": "altogether", + "expected_answer": 8.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 11.0, + "case_id": "G5-2ent-002", + "category": "2entity_no_op", + "cue": "in total", + "expected_answer": 11.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 15.0, + "case_id": "G5-2ent-003", + "category": "2entity_no_op", + "cue": "combined", + "expected_answer": 15.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 20.0, + "case_id": "G5-2ent-004", + "category": "2entity_no_op", + "cue": "together", + "expected_answer": 20.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 10.0, + "case_id": "G5-3ent-001", + "category": "3entity_no_op", + "cue": "altogether", + "expected_answer": 10.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 12.0, + "case_id": "G5-3ent-002", + "category": "3entity_no_op", + "cue": "in total", + "expected_answer": 12.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 18.0, + "case_id": "G5-3ent-003", + "category": "3entity_no_op", + "cue": "combined", + "expected_answer": 18.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 20.0, + "case_id": "G5-3ent-004", + "category": "3entity_no_op", + "cue": "together", + "expected_answer": 20.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 12.0, + "case_id": "G5-op-001", + "category": "2entity_with_op", + "cue": "altogether", + "expected_answer": 12.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 14.0, + "case_id": "G5-op-002", + "category": "2entity_with_op", + "cue": "combined", + "expected_answer": 14.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 16.0, + "case_id": "G5-op-003", + "category": "2entity_with_op", + "cue": "in total", + "expected_answer": 16.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 16.0, + "case_id": "G5-op-004", + "category": "2entity_with_op", + "cue": "together", + "expected_answer": 16.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 5.0, + "case_id": "G5-degen-001", + "category": "single_entity_total_cue", + "cue": "in total", + "expected_answer": 5.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 10.0, + "case_id": "G5-degen-002", + "category": "single_entity_total_cue", + "cue": "altogether", + "expected_answer": 10.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 7.0, + "case_id": "G5-degen-003", + "category": "single_entity_total_cue", + "cue": "combined", + "expected_answer": 7.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 6.0, + "case_id": "G5-degen-004", + "category": "single_entity_total_cue", + "cue": "together", + "expected_answer": 6.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": null, + "case_id": "G5-refuse-001", + "category": "refusal_outside_closed_cue", + "cue": "none", + "expected_answer": null, + "outcome": "pass", + "reason": "" + }, + { + "answer": null, + "case_id": "G5-refuse-002", + "category": "refusal_outside_closed_cue", + "cue": "none", + "expected_answer": null, + "outcome": "pass", + "reason": "" + }, + { + "answer": null, + "case_id": "G5-refuse-003", + "category": "refusal_outside_closed_cue", + "cue": "none", + "expected_answer": null, + "outcome": "pass", + "reason": "" + }, + { + "answer": null, + "case_id": "G5-refuse-004", + "category": "refusal_outside_closed_cue", + "cue": "none", + "expected_answer": null, + "outcome": "pass", + "reason": "" + } + ], + "per_category": { + "2entity_no_op": { + "pass": 4, + "wrong": 0 + }, + "2entity_with_op": { + "pass": 4, + "wrong": 0 + }, + "3entity_no_op": { + "pass": 4, + "wrong": 0 + }, + "refusal_outside_closed_cue": { + "pass": 4, + "wrong": 0 + }, + "single_entity_total_cue": { + "pass": 4, + "wrong": 0 + } + }, + "schema_version": 1 +} diff --git a/evals/math_capability_axes/G5_aggregate/v1/runner.py b/evals/math_capability_axes/G5_aggregate/v1/runner.py new file mode 100644 index 00000000..b8d4b720 --- /dev/null +++ b/evals/math_capability_axes/G5_aggregate/v1/runner.py @@ -0,0 +1,129 @@ +"""ADR-0131.G.5 — Capability axis runner for aggregate answer composition. + +Exercises the ``entity=None`` sum path in :mod:`generate.math_solver` via +:func:`generate.math_candidate_graph.parse_and_solve` against curated +coverage cases that are independent of GSM8K. + +Per-case classification: + +| Case category | pass criterion | +|-----------------------------|-------------------------------------------| +| 2entity_no_op | answer == expected_answer (exact float) | +| 3entity_no_op | answer == expected_answer | +| 2entity_with_op | answer == expected_answer | +| single_entity_total_cue | answer == expected_answer | +| refusal_outside_closed_cue | answer is None (question not admitted) | + +``wrong`` is non-zero only if a positive case returns the wrong numeric +answer or a refusal case emits a numeric answer. ``wrong == 0`` is the +load-bearing gate (ADR-0114a Obligation #4). + +Determinism: case order in ``cases.jsonl`` is the report order; same +input file → byte-equal report. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from generate.math_candidate_graph import parse_and_solve + +_HERE = Path(__file__).resolve().parent +_CASES_PATH = _HERE / "cases.jsonl" +_REPORT_PATH = _HERE / "report.json" + + +def _load_cases() -> list[dict[str, Any]]: + return [ + json.loads(line) + for line in _CASES_PATH.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def _score_case(case: dict[str, Any]) -> dict[str, Any]: + r = parse_and_solve(case["problem"]) + exp = case["expected_answer"] + category = case["category"] + + if exp is not None: + if r.answer == exp: + outcome, reason = "pass", "" + elif r.answer is None: + outcome = "wrong" + reason = f"expected {exp} but got refusal: {r.refusal_reason}" + else: + outcome = "wrong" + reason = f"expected {exp} but got {r.answer}" + else: + if r.answer is None: + outcome, reason = "pass", "" + else: + outcome = "wrong" + reason = f"expected refusal but got answer {r.answer}" + + return { + "case_id": case["case_id"], + "category": category, + "cue": case.get("cue", ""), + "outcome": outcome, + "reason": reason, + "answer": r.answer, + "expected_answer": exp, + } + + +def build_report() -> dict[str, Any]: + cases = _load_cases() + per_case = [_score_case(c) for c in cases] + total = len(per_case) + passed = sum(1 for d in per_case if d["outcome"] == "pass") + wrong = sum(1 for d in per_case if d["outcome"] == "wrong") + by_category: dict[str, dict[str, int]] = {} + for d in per_case: + slot = by_category.setdefault(d["category"], {"pass": 0, "wrong": 0}) + slot[d["outcome"]] = slot.get(d["outcome"], 0) + 1 + return { + "schema_version": 1, + "adr": "0131.G.5", + "axis": "aggregate", + "cases_path": "evals/math_capability_axes/G5_aggregate/v1/cases.jsonl", + "metrics": { + "cases_total": total, + "passed": passed, + "wrong": wrong, + "pass_rate": (passed / total) if total else 0.0, + "wrong_rate": (wrong / total) if total else 0.0, + "wrong_count_is_zero": wrong == 0, + }, + "per_category": { + k: dict(sorted(v.items())) for k, v in sorted(by_category.items()) + }, + "per_case": per_case, + } + + +def write_report(report: dict[str, Any]) -> None: + _REPORT_PATH.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def main() -> int: + report = build_report() + write_report(report) + m = report["metrics"] + print( + f"ADR-0131.G.5 aggregate: passed {m['passed']}/{m['cases_total']} " + f"({m['pass_rate']:.1%}); wrong={m['wrong']} (gate: must be 0)" + ) + for cat, counts in report["per_category"].items(): + print(f" {cat:30s} {counts}") + return 0 if m["wrong_count_is_zero"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/generate/math_candidate_parser.py b/generate/math_candidate_parser.py index aaec29f6..a8213dd4 100644 --- a/generate/math_candidate_parser.py +++ b/generate/math_candidate_parser.py @@ -685,11 +685,15 @@ class CandidateUnknown: Two question shapes in P3 scope: - - ``How many does have [left|now|in total|altogether]?`` + - ``How many does have [left|now|in total|altogether|combined|together]?`` → ``Unknown(entity=, unit=)`` - - ``How many do they have [left|now|in total|altogether]?`` + - ``How many do they have [left|now|in total|altogether|combined|together]?`` → ``Unknown(entity=None, unit=)`` (total-across) + Closed aggregate-cue vocabulary: ``in total``, ``altogether``, + ``combined``, ``together``. All four map to ``entity=None`` on the + total-across form. + The round-trip filter for questions checks the unit token and (when present) the entity token both appear in the source span. """ @@ -703,13 +707,13 @@ class CandidateUnknown: _Q_ENTITY_RE: Final[re.Pattern[str]] = re.compile( r"^How\s+many\s+(?P\w+)\s+(?:does|do)\s+" rf"(?P{_ENTITY})" - r"\s+have(?:\s+(?:left|now|in\s+total|altogether)){0,2}\s*\??$", + r"\s+have(?:\s+(?:left|now|in\s+total|altogether|combined|together)){0,2}\s*\??$", flags=re.IGNORECASE, ) _Q_TOTAL_RE: Final[re.Pattern[str]] = re.compile( r"^How\s+many\s+(?P\w+)\s+do\s+they\s+have" - r"(?:\s+(?:in\s+total|altogether|left|now)){0,2}\s*\??$", + r"(?:\s+(?:in\s+total|altogether|combined|together|left|now)){0,2}\s*\??$", flags=re.IGNORECASE, ) diff --git a/tests/test_adr_0131_G5_aggregate.py b/tests/test_adr_0131_G5_aggregate.py new file mode 100644 index 00000000..0aada72e --- /dev/null +++ b/tests/test_adr_0131_G5_aggregate.py @@ -0,0 +1,162 @@ +"""ADR-0131.G.5 — Aggregate answer composition axis lane tests. + +Pins the closed aggregate-cue vocabulary (``in total``, ``altogether``, +``combined``, ``together``) and the end-to-end ``parse_and_solve`` path +for 2-entity, 3-entity, single-entity, and refusal shapes. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from evals.math_capability_axes.G5_aggregate.v1.runner import build_report +from generate.math_candidate_graph import parse_and_solve +from generate.math_candidate_parser import extract_question_candidates + +_REPO = Path(__file__).resolve().parent.parent +_GSM8K_LEGACY_REPORT = ( + _REPO / "evals/gsm8k_math/train_sample/v1/train_sample_coverage_report.json" +) +_GSM8K_CG_REPORT = _REPO / "evals/gsm8k_math/train_sample/v1/report.json" + + +# ── Cue vocabulary tests ───────────────────────────────────────────── + + +class TestCueVocabulary: + """Verify that combined and together parse to entity=None.""" + + @pytest.mark.parametrize("cue", ["combined", "together", "altogether", "in total"]) + def test_cue_parses_to_entity_none(self, cue: str) -> None: + q = f"How many apples do they have {cue}?" + cands = extract_question_candidates(q) + assert len(cands) >= 1, f"no candidate for cue {cue!r}" + assert cands[0].unknown.entity is None + assert cands[0].unknown.unit == "apples" + + def test_closed_cue_docstring_lists_all_four(self) -> None: + import generate.math_candidate_parser as mod + + src = Path(mod.__file__).read_text(encoding="utf-8") + for cue in ("in total", "altogether", "combined", "together"): + assert cue in src, f"cue {cue!r} missing from parser source" + + +# ── End-to-end parse_and_solve tests ───────────────────────────────── + + +class TestTwoEntityNoOp: + @pytest.mark.parametrize( + "problem, expected", + [ + ("Sam has 5 apples. Tom has 3 apples. How many apples do they have altogether?", 8.0), + ("Alice has 7 books. Bob has 4 books. How many books do they have in total?", 11.0), + ("Maya has 6 coins. Leo has 9 coins. How many coins do they have combined?", 15.0), + ("Jade has 12 stickers. Finn has 8 stickers. How many stickers do they have together?", 20.0), + ], + ) + def test_two_entity_sum(self, problem: str, expected: float) -> None: + r = parse_and_solve(problem) + assert r.answer == expected + assert r.refusal_reason is None + + +class TestThreeEntityNoOp: + @pytest.mark.parametrize( + "problem, expected", + [ + ("Sam has 5 apples. Tom has 3 apples. Amy has 2 apples. How many apples do they have altogether?", 10.0), + ("Alice has 4 books. Bob has 6 books. Carol has 2 books. How many books do they have in total?", 12.0), + ("Maya has 10 coins. Leo has 5 coins. Nina has 3 coins. How many coins do they have combined?", 18.0), + ("Jade has 7 stickers. Finn has 4 stickers. Rex has 9 stickers. How many stickers do they have together?", 20.0), + ], + ) + def test_three_entity_sum(self, problem: str, expected: float) -> None: + r = parse_and_solve(problem) + assert r.answer == expected + assert r.refusal_reason is None + + +class TestSingleEntityDegenerate: + def test_single_entity_identity(self) -> None: + r = parse_and_solve("Sam has 5 apples. How many apples do they have in total?") + assert r.answer == 5.0 + + def test_single_entity_with_op(self) -> None: + r = parse_and_solve("Alice has 7 books. Alice buys 3 books. How many books do they have altogether?") + assert r.answer == 10.0 + + +class TestMismatchedUnitRefusal: + @pytest.mark.parametrize( + "problem", + [ + "Sam has 5 apples. Tom has 3 apples. How many apples does everyone have?", + "Alice has 4 coins. Bob has 6 coins. What is the total number of coins?", + "Maya has 10 books. Leo has 5 books. How many books do Sam and Leo have?", + "Jade has 8 stickers. Finn has 4 stickers. How many stickers are there?", + ], + ) + def test_outside_closed_cue_refuses(self, problem: str) -> None: + r = parse_and_solve(problem) + assert r.answer is None, f"expected refusal but got {r.answer}" + + +# ── Axis lane gate ─────────────────────────────────────────────────── + + +class TestAxisLaneGate: + def test_wrong_is_zero(self) -> None: + report = build_report() + assert report["metrics"]["wrong"] == 0 + assert report["metrics"]["wrong_count_is_zero"] is True + + def test_report_byte_equal_across_runs(self) -> None: + r1 = build_report() + r2 = build_report() + s1 = json.dumps(r1, indent=2, sort_keys=True) + s2 = json.dumps(r2, indent=2, sort_keys=True) + assert s1 == s2 + + def test_all_categories_present(self) -> None: + report = build_report() + expected_cats = { + "2entity_no_op", + "3entity_no_op", + "2entity_with_op", + "single_entity_total_cue", + "refusal_outside_closed_cue", + } + assert set(report["per_category"].keys()) == expected_cats + + +# ── B3 regression guard ────────────────────────────────────────────── + + +def test_b3_lane_still_passes() -> None: + """B3 bounded-grammar lane must remain green after G5 changes.""" + from evals.math_bounded_grammar.v1.runner import build_report as b3_build, load_cases + + cases_path = _REPO / "evals" / "math_bounded_grammar" / "v1" / "cases.jsonl" + report = b3_build(load_cases(cases_path)) + assert report["metrics"]["wrong"] == 0, ( + f"B3 lane regression: wrong={report['metrics']['wrong']}" + ) + + +# ── GSM8K safety rail ──────────────────────────────────────────────── + + +def test_gsm8k_legacy_probe_safety_rail_intact() -> None: + """ADR-0131.G invariant: legacy probe still shows admitted_wrong == 0.""" + data = json.loads(_GSM8K_LEGACY_REPORT.read_text(encoding="utf-8")) + assert data["metrics"]["admitted_wrong"] == 0 + + +def test_gsm8k_candidate_graph_probe_wrong_zero() -> None: + """ADR-0131.G invariant: candidate-graph probe shows wrong == 0.""" + data = json.loads(_GSM8K_CG_REPORT.read_text(encoding="utf-8")) + assert data["counts"]["wrong"] == 0