From e7a1ffb72eaca96ec1ad3b22f558ea86e6a99d7c Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 23 May 2026 21:20:52 -0700 Subject: [PATCH] =?UTF-8?q?feat(ADR-0136.S.2):=20conditional-op=20question?= =?UTF-8?q?=20=E2=80=94=20gsm8k-0042=20admits,=20wrong=3D=3D0=20(#203)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds CandidateConditionalOpQuestion + extractor for the closed shape: "If , how many does []?" In parse_and_solve, when the question yields exactly one such candidate and exactly one matching InitialPossession exists by (entity, unit) across all statement sentences, computes initial_value ± operand (verb polarity) and emits when answer >= 0; refuses otherwise. Structurally identical to S.1 capacity/earnings short-circuits. GSM8K probe: 2/50 → 3/50 (+0042, answer=30.0), wrong stays 0. - generate/math_candidate_parser.py: _COND_SUBTRACT_VERBS / _COND_ADD_VERBS closed sets; _COND_OP_Q_RE; extract_conditional_op_question_candidates - generate/math_candidate_graph.py: short-circuit after earnings path - tests/test_adr_0136_S2_conditional_op.py: 25 tests (extractor unit tests, end-to-end short-circuit, B3 + S.1 regression guards, post-S.2 honest admission count) - docs/decisions/ADR-0136.S.2-conditional-op-question.md --- .../ADR-0136.S.2-conditional-op-question.md | 106 ++++++++++ generate/math_candidate_graph.py | 31 +++ generate/math_candidate_parser.py | 111 +++++++++- tests/test_adr_0136_S2_conditional_op.py | 194 ++++++++++++++++++ 4 files changed, 440 insertions(+), 2 deletions(-) create mode 100644 docs/decisions/ADR-0136.S.2-conditional-op-question.md create mode 100644 tests/test_adr_0136_S2_conditional_op.py diff --git a/docs/decisions/ADR-0136.S.2-conditional-op-question.md b/docs/decisions/ADR-0136.S.2-conditional-op-question.md new file mode 100644 index 00000000..d33f4861 --- /dev/null +++ b/docs/decisions/ADR-0136.S.2-conditional-op-question.md @@ -0,0 +1,106 @@ +# ADR-0136.S.2 — Conditional-Op Question (Statement-Layer Corridor) + +**Status:** Active +**Date:** 2026-05-23 +**Parent:** [ADR-0136](./ADR-0136-statement-layer-corridor.md) + +--- + +## Context + +After S.0 (context-sentence classifier) and S.1 (rate/event statement parsing) +landed, the GSM8K train-sample probe sat at **2/50** admitted (`gsm8k-0014`, +`gsm8k-0018`), `wrong == 0`. + +A taxonomy pass over the remaining 48 refused cases identified `gsm8k-0042` +as the next single-barrier unlock: + +``` +Ella has 4 bags with 20 apples in each bag and six bags with 25 apples in +each bag. If Ella sells 200 apples, how many apples does Ella has left? +``` + +The initial-state sentence already parses via `_CONJ_EMBEDDED_RE` (ADR-0131.G.4 +embedded quantifier) to `InitialPossession(entity="Ella", value=230, +unit="apples")`. The **only** barrier is the question form, which neither +`_Q_ENTITY_RE` nor `_Q_TOTAL_RE` cover: the `If , +how many does [left|...]?` shape combines a +conditional-action operand with the entity-recall question. + +--- + +## Decision + +Add a **conditional-op question** extractor and a corresponding short-circuit +in `parse_and_solve` that: + +1. Matches the closed shape `If , how many + does []?` +2. Classifies `` against two closed sets: + - `_COND_SUBTRACT_VERBS` (sell/sells/sold, give/gives/gave, eat/eats/ate, + use, lose, spend, donate, remove, take, send, pay, drop, throw) + - `_COND_ADD_VERBS` (buy/buys/bought, get/gets/got, receive, find, add, + collect, pick, earn, gain) +3. Refuses on any of: unknown verb, unit mismatch (`` vs `` + after canonicalization), entity mismatch (`` vs `` + case-insensitively), `N <= 0`. +4. In `parse_and_solve`: if the question yields exactly one + `CandidateConditionalOpQuestion`, collect all `extract_initial_candidates` + from every statement sentence and look for **exactly one** matching IC + by `(entity, unit)`. If found, compute `initial_value ± operand` by verb + polarity; emit only when `answer >= 0`. Refuses otherwise. + +The short-circuit is structurally identical to the S.1 capacity/earnings +paths: it bypasses graph construction, returns `selected_graph=None`, and +preserves `wrong == 0` by refusing rather than guessing on any ambiguity. + +--- + +## Invariants + +- **`admitted_wrong == 0`** preserved by: + - Single-match (entity, unit) requirement before emission + - Non-negative answer gate + - Closed verb sets — no wildcards +- **Context-filler safety rail** unchanged (S.0) +- **No solver/graph/verifier changes** — extractor lives in + `math_candidate_parser.py`; short-circuit lives in `math_candidate_graph.py` + +--- + +## Honest GSM8K delta + +| Stage | Admitted | Wrong | +|---|---|---| +| Pre-S.0 | 0/50 | 0 | +| Post-S.1 | 1/50 (`0014`) | 0 | +| Post-S.0 classifier | 2/50 (`+0018`) | 0 | +| **Post-S.2** | **3/50** (`+0042`) | **0** | + +`gsm8k-0042` admits with `answer == 30.0` (expected: 30). + +--- + +## Consequences + +- The S.x corridor now spans `parser` (S.1 capacity/earnings, S.2 question + shape) and `pre-pass classifier` (S.0). Future phases (S.3 compound + statements, S.4 coreference) extend this pattern. +- The canonical GSM8K runner (`evals/gsm8k_math/runner.py`) still asserts + `selected_graph is not None` on admission and therefore cannot score the + short-circuit admissions. The `report.json` artifact remains stale + (0/50/0); the honest count is asserted via direct `parse_and_solve` test + (`test_gsm8k_post_s2_admission_honest`). Aligning the canonical runner + with the short-circuit paths is deferred (out of S.2 scope). + +--- + +## Deferred + +- Conditional-op question forms with **more than one** operand (e.g. + "If she gives away 3 and buys 5, …") — needs two-op composition. +- Question forms without `does ` aux (e.g. "how many apples + remain?") — needs a sibling regex. +- Conditional questions where the conditional and the question reference + **different** units that are unit-related (e.g. dollars↔cents) — needs + unit-relation taxonomy. diff --git a/generate/math_candidate_graph.py b/generate/math_candidate_graph.py index 4f025c86..f0c7166f 100644 --- a/generate/math_candidate_graph.py +++ b/generate/math_candidate_graph.py @@ -44,6 +44,7 @@ from generate.math_candidate_parser import ( classify_sentence, extract_capacity_candidates, extract_capacity_question_candidates, + extract_conditional_op_question_candidates, extract_earnings_candidates, extract_earnings_question_candidates, extract_initial_candidates, @@ -393,6 +394,36 @@ def parse_and_solve(text: str) -> CandidateGraphResult: branches_enumerated=0, branches_admissible=0, ) + # ADR-0136.S.2 — Conditional-op question short-circuit. + # Shape: "If , how many does + # [left|...]?" — given exactly one matching initial-state + # candidate for (entity, unit) across all statement sentences, the + # answer is initial_value ± operand by verb polarity. Refuses on any + # ambiguity (multiple matching ICs, no IC, negative answer); preserves + # wrong == 0. + cond_qs = extract_conditional_op_question_candidates(question_sentences[0]) + if len(cond_qs) == 1: + cq = cond_qs[0] + all_ic: list[CandidateInitial] = [] + for s in statement_sentences: + all_ic.extend(extract_initial_candidates(s)) + matching = [ + ic for ic in all_ic + if ic.initial.entity.lower() == cq.entity.lower() + and ic.initial.quantity.unit == cq.unit + ] + if len(matching) == 1: + val = matching[0].initial.quantity.value + answer = val - cq.operand if cq.op == "subtract" else val + cq.operand + if answer >= 0: + return CandidateGraphResult( + answer=answer, + selected_graph=None, + refusal_reason=None, + branches_enumerated=1, + branches_admissible=1, + ) + # Per-sentence choice spaces (after round-trip filter + tiebreaker). per_sentence_choices: list[list[SentenceChoice]] = [] for s in statement_sentences: diff --git a/generate/math_candidate_parser.py b/generate/math_candidate_parser.py index c0680b94..9f059807 100644 --- a/generate/math_candidate_parser.py +++ b/generate/math_candidate_parser.py @@ -707,13 +707,13 @@ class CandidateUnknown: _Q_ENTITY_RE: Final[re.Pattern[str]] = re.compile( r"^How\s+many\s+(?P\w+)\s+(?:does|do)\s+" rf"(?P{_ENTITY})" - r"\s+have(?:\s+(?:left|now|in\s+total|altogether|combined|together)){0,2}\s*\??$", + r"\s+have(?:\s+(?:left|now|in\s+total|altogether|combined|together|in\s+all)){0,2}\s*\??$", flags=re.IGNORECASE, ) _Q_TOTAL_RE: Final[re.Pattern[str]] = re.compile( r"^How\s+many\s+(?P\w+)\s+do\s+they\s+have" - r"(?:\s+(?:in\s+total|altogether|combined|together|left|now)){0,2}\s*\??$", + r"(?:\s+(?:in\s+total|altogether|combined|together|in\s+all|left|now)){0,2}\s*\??$", flags=re.IGNORECASE, ) @@ -1897,3 +1897,110 @@ def extract_earnings_question_candidates( source_span=sentence, ) ] + + +# --------------------------------------------------------------------------- +# ADR-0136.S.2 — Conditional-op question +# --------------------------------------------------------------------------- +# +# Target shape (gsm8k-0042): +# "If , how many does left?" +# +# Routes through the parse_and_solve short-circuit: given a single matching +# initial-state candidate for (entity, unit), the answer is +# initial_value ± operand depending on verb polarity. No graph built; +# refuses on any ambiguity (unit mismatch, entity mismatch, multiple +# matching ICs, negative answer). + +_COND_SUBTRACT_VERBS: Final[frozenset[str]] = frozenset({ + "sell", "sells", "sold", + "give", "gives", "gave", + "eat", "eats", "ate", + "use", "uses", "used", + "lose", "loses", "lost", + "spend", "spends", "spent", + "donate", "donates", "donated", + "remove", "removes", "removed", + "take", "takes", "took", + "send", "sends", "sent", + "pay", "pays", "paid", + "drop", "drops", "dropped", + "throw", "throws", "threw", +}) + +_COND_ADD_VERBS: Final[frozenset[str]] = frozenset({ + "buy", "buys", "bought", + "get", "gets", "got", + "receive", "receives", "received", + "find", "finds", "found", + "add", "adds", "added", + "collect", "collects", "collected", + "pick", "picks", "picked", + "earn", "earns", "earned", + "gain", "gains", "gained", +}) + +_COND_VERB_PATTERN: Final[str] = ( + r"(?:" + "|".join( + re.escape(v) + for v in sorted(_COND_SUBTRACT_VERBS | _COND_ADD_VERBS, key=len, reverse=True) + ) + r")" +) + + +@dataclass(frozen=True, slots=True) +class CandidateConditionalOpQuestion: + entity: str + op: Literal["add", "subtract"] + operand: float + unit: str + source_span: str + + +# "If , how many does [ ]?" +_COND_OP_Q_RE: Final[re.Pattern[str]] = re.compile( + rf"^If\s+(?P{_ENTITY})\s+" + rf"(?P{_COND_VERB_PATTERN})\s+" + r"(?P\d+(?:\.\d+)?)\s+(?P\w+),\s+" + r"how\s+many\s+(?P\w+)\s+does\s+" + rf"(?P{_ENTITY})\s+(?:has|have|had)" + r"(?:\s+(?:left|now|remaining|away|in\s+total|altogether))?" + r"\s*\??\s*$", + flags=re.IGNORECASE, +) + + +def extract_conditional_op_question_candidates( + sentence: str, +) -> list[CandidateConditionalOpQuestion]: + s = sentence.strip() + m = _COND_OP_Q_RE.match(s) + if m is None: + return [] + verb = m.group("verb").lower() + if verb in _COND_SUBTRACT_VERBS: + op: Literal["add", "subtract"] = "subtract" + elif verb in _COND_ADD_VERBS: + op = "add" + else: + return [] + unit = _canonicalize_unit(m.group("unit")) + unit2 = _canonicalize_unit(m.group("unit2")) + if unit != unit2: + return [] + entity = _normalize_entity(m.group("entity")) + entity2 = _normalize_entity(m.group("entity2")) + if entity.lower() != entity2.lower(): + return [] + n = float(m.group("n")) + if n <= 0: + return [] + return [ + CandidateConditionalOpQuestion( + entity=entity, + op=op, + operand=n, + unit=unit, + source_span=sentence, + ) + ] diff --git a/tests/test_adr_0136_S2_conditional_op.py b/tests/test_adr_0136_S2_conditional_op.py new file mode 100644 index 00000000..4cacd2d4 --- /dev/null +++ b/tests/test_adr_0136_S2_conditional_op.py @@ -0,0 +1,194 @@ +"""ADR-0136.S.2 — Conditional-op question tests. + +Pins the new `extract_conditional_op_question_candidates` extractor and +the short-circuit path in `parse_and_solve` for the shape: + + "If , how many does + [left|now|remaining|...]?" +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from generate.math_candidate_graph import parse_and_solve +from generate.math_candidate_parser import ( + _COND_ADD_VERBS, + _COND_SUBTRACT_VERBS, + extract_conditional_op_question_candidates, +) + + +_REPO = Path(__file__).resolve().parent.parent + + +# ── Regex extractor tests ──────────────────────────────────────────── + + +class TestConditionalOpExtractor: + def test_subtract_canonical(self) -> None: + cands = extract_conditional_op_question_candidates( + "If Ella sells 200 apples, how many apples does Ella has left?" + ) + assert len(cands) == 1 + c = cands[0] + assert c.entity == "Ella" + assert c.op == "subtract" + assert c.operand == 200.0 + assert c.unit == "apples" + + def test_add_canonical(self) -> None: + cands = extract_conditional_op_question_candidates( + "If Bob buys 5 apples, how many apples does Bob have now?" + ) + assert len(cands) == 1 + c = cands[0] + assert c.op == "add" + assert c.operand == 5.0 + + @pytest.mark.parametrize( + "verb", ["sells", "gives", "eats", "uses", "loses", "spends", "donates"] + ) + def test_subtract_verbs(self, verb: str) -> None: + cands = extract_conditional_op_question_candidates( + f"If Alice {verb} 3 apples, how many apples does Alice have left?" + ) + assert len(cands) == 1 + assert cands[0].op == "subtract" + + @pytest.mark.parametrize( + "verb", ["buys", "gets", "receives", "finds", "collects", "earns"] + ) + def test_add_verbs(self, verb: str) -> None: + cands = extract_conditional_op_question_candidates( + f"If Alice {verb} 3 apples, how many apples does Alice have now?" + ) + assert len(cands) == 1 + assert cands[0].op == "add" + + def test_unit_mismatch_refuses(self) -> None: + cands = extract_conditional_op_question_candidates( + "If Ella sells 200 apples, how many oranges does Ella have left?" + ) + assert cands == [] + + def test_entity_mismatch_refuses(self) -> None: + cands = extract_conditional_op_question_candidates( + "If Ella sells 200 apples, how many apples does Bob have left?" + ) + assert cands == [] + + def test_unknown_verb_refuses(self) -> None: + cands = extract_conditional_op_question_candidates( + "If Ella juggles 200 apples, how many apples does Ella have left?" + ) + assert cands == [] + + def test_zero_operand_refuses(self) -> None: + cands = extract_conditional_op_question_candidates( + "If Ella sells 0 apples, how many apples does Ella have left?" + ) + assert cands == [] + + def test_verb_sets_disjoint(self) -> None: + assert _COND_SUBTRACT_VERBS.isdisjoint(_COND_ADD_VERBS) + + +# ── End-to-end short-circuit tests ─────────────────────────────────── + + +class TestConditionalOpEndToEnd: + def test_gsm8k_0042_admits_30(self) -> None: + """The proof case for S.2.""" + r = parse_and_solve( + "Ella has 4 bags with 20 apples in each bag and " + "six bags with 25 apples in each bag. " + "If Ella sells 200 apples, how many apples does Ella has left?" + ) + assert r.answer == 30.0, f"got {r.answer} ({r.refusal_reason})" + + def test_simple_subtract(self) -> None: + r = parse_and_solve( + "Bob has 100 apples. " + "If Bob eats 30 apples, how many apples does Bob have left?" + ) + assert r.answer == 70.0 + + def test_simple_add(self) -> None: + r = parse_and_solve( + "Alice has 12 apples. " + "If Alice buys 8 apples, how many apples does Alice have now?" + ) + assert r.answer == 20.0 + + def test_negative_result_refuses(self) -> None: + """Selling more than you have must refuse (never produce negative).""" + r = parse_and_solve( + "Bob has 10 apples. " + "If Bob sells 50 apples, how many apples does Bob have left?" + ) + assert r.answer is None + + def test_no_matching_initial_state_refuses(self) -> None: + """Question about apples but no initial-state apples → refuse.""" + r = parse_and_solve( + "Bob has 10 oranges. " + "If Bob sells 5 apples, how many apples does Bob have left?" + ) + assert r.answer is None + + def test_no_matching_entity_refuses(self) -> None: + r = parse_and_solve( + "Bob has 100 apples. " + "If Alice sells 30 apples, how many apples does Alice have left?" + ) + assert r.answer is None + + +# ── B3 + S.1 regression guards ─────────────────────────────────────── + + +def test_b3_lane_still_passes() -> None: + from evals.math_bounded_grammar.v1.runner import build_report, load_cases + + cases_path = _REPO / "evals" / "math_bounded_grammar" / "v1" / "cases.jsonl" + report = build_report(load_cases(cases_path)) + assert report["metrics"]["wrong"] == 0 + + +def test_s1_axis_lane_still_passes() -> None: + from evals.math_capability_axes.S1_rate_events.v1.runner import build_report + + report = build_report() + assert report["metrics"]["wrong"] == 0 + + +# ── GSM8K safety rail ──────────────────────────────────────────────── + + +def test_gsm8k_post_s2_admission_honest() -> None: + """Post-S.2: 3 admissions expected (0014, 0018, 0042); wrong stays 0.""" + cases = [ + json.loads(line) + for line in ( + _REPO / "evals/gsm8k_math/train_sample/v1/cases.jsonl" + ).read_text(encoding="utf-8").splitlines() + if line.strip() + ] + admitted: list[str] = [] + wrong: list[tuple[str, float, float]] = [] + for c in cases: + r = parse_and_solve(c["question"]) + if r.answer is not None: + if r.answer == c["answer_numeric"]: + admitted.append(c["case_id"]) + else: + wrong.append((c["case_id"], r.answer, c["answer_numeric"])) + assert wrong == [], f"wrong admissions: {wrong}" + assert "gsm8k-train-sample-v1-0014" in admitted # S.1 capacity + assert "gsm8k-train-sample-v1-0018" in admitted # S.1 inverted + assert "gsm8k-train-sample-v1-0042" in admitted # S.2 cond-op + assert len(admitted) >= 3