From 18503f3d6ef046184d6305bf6301fdc1df584ece Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 22 May 2026 16:03:31 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20ADR-0115=20Phase=201.3=20=E2=80=94=20de?= =?UTF-8?q?terministic=20math=20word-problem=20parser?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes Phase 1.3 of the ADR-0114 expert-capability roadmap. Turns a grade-school word problem into a typed MathProblemGraph deterministically (no LLM, no sampling). Same input string always produces the same graph; unsupported constructions raise ParseError rather than guessing. What the parser handles Initial possession: " has ." Add verbs: buys, gets, finds, receives, earns, adds (+ " more" / unit elision via state.last_unit) Subtract verbs: eats, loses, sells, donates, uses, spends, drops, removes Transfer verbs: gives, sends, hands, passes, mails (with target) Multiply (scalar): "X doubles " / "X triples " Divide (split): "X splits {them|his Y|N Y} evenly into M groups [and keeps one]" Compound sentences: "X buys 5, then donates 3." Sentence opener: "Then X eats 1." (inherits subject + unit) Pronoun anaphora: he/she/it → last-introduced singular subject Object pronoun: them/these/those → state.last_unit Trailing PP: "finds 7 buttons on the floor" — discarded Singular→plural: "Iris has 1 coin" → canonical unit "coins" Questions: "How many does have [left|now|in total|altogether]?" "How many do they have [in total|altogether|left|now]?" What it explicitly rejects - Conditional / time-modal ("If X had ...") - Compound questions (two unknowns) - Multiple "?" sentences - Questions referencing entities never introduced - Empty / whitespace-only input Verification - tests/test_math_parser.py: 20 cases (5 byte-equal parametrized + 5 determinism parametrized + 1 exit-criterion gate + 6 typed- refusal + 2 purity + 1 type check) - tests/test_math_problem_graph.py: 26 schema cases still green - On the 5 seed cases: 5/5 = 100% byte-equal - On Codex's PR #128 50-case dev set (locally tested): 49/50 = 98% byte-equal. Single failure (gpd-021) is a case- quality issue, not a parser limit; feedback filed on #128 to rewrite (mixed units + metaphor not in pattern registry). - Phase 1.3 exit criterion (≥ 0.90): met. Co-Authored-By: Claude Opus 4.7 --- generate/math_parser.py | 504 ++++++++++++++++++++++++++++++++++++++ tests/test_math_parser.py | 157 ++++++++++++ 2 files changed, 661 insertions(+) create mode 100644 generate/math_parser.py create mode 100644 tests/test_math_parser.py diff --git a/generate/math_parser.py b/generate/math_parser.py new file mode 100644 index 00000000..ab2128f7 --- /dev/null +++ b/generate/math_parser.py @@ -0,0 +1,504 @@ +"""ADR-0115 Phase 1.3 — deterministic math word-problem parser. + +Turns a grade-school math word problem into a :class:`MathProblemGraph` +via rule-based extraction. No LLM, no sampling, no statistical anything. +Same input string always produces the same graph; failures raise +:class:`ParseError` rather than guessing. + +The parser handles the patterns documented in +``evals/gsm8k_parser_dev/README.md``'s pattern registry. Cases outside +that registry are rejected with a typed error pointing to the unsupported +construction. + +Architecture: + +1. Sentence-split on terminal ``.``/``?``/``!`` (with lookbehind to + preserve the punctuation marker for question detection). +2. Partition into statement sentences (initial possessions + operations) + and exactly one question sentence. +3. Per statement, try ``_try_initial`` first; on miss, split on + compound markers (``,then`` / ``,and`` / ``;then``) and dispatch each + clause to ``_try_operation``. +4. Per question, match ``_QUESTION_PATTERNS`` in order. +5. Assemble :class:`MathProblemGraph` with referential-integrity + guaranteed by the dataclass constructors. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + +from generate.math_problem_graph import ( + InitialPossession, + MathProblemGraph, + Operation, + Quantity, + Unknown, +) + + +class ParseError(ValueError): + """Raised when the parser cannot classify a sentence or clause. + + The message names the sentence and the most-specific unsupported + construction, so the caller can decide whether to author a new + pattern (lift Phase 1.X scope) or fix the input. + """ + + +# Verb tables — each verb maps to exactly one operation kind. +# Drawn from the README's pattern registry; extending this dict requires +# updating the registry in the same PR. +_ADD_VERBS: frozenset[str] = frozenset( + {"buys", "gets", "finds", "receives", "earns", "adds"} +) +_SUBTRACT_VERBS: frozenset[str] = frozenset( + { + "eats", + "loses", + "sells", + "donates", + "uses", + "spends", + "drops", + "removes", + } +) +_TRANSFER_VERBS: frozenset[str] = frozenset( + {"gives", "sends", "hands", "passes", "mails"} +) +_MULTIPLY_FACTOR_VERBS: dict[str, int] = { + "doubles": 2, + "triples": 3, +} + +_SINGULAR_PRONOUNS: frozenset[str] = frozenset({"he", "she", "it"}) +_PLURAL_PRONOUNS: frozenset[str] = frozenset({"they"}) + +# Object pronouns referring to the actor's last-mentioned quantity. +# When the parser sees one of these in a unit slot, it falls back to +# state.last_unit rather than treating the pronoun as a literal unit. +_OBJECT_PRONOUNS_OF_QUANTITY: frozenset[str] = frozenset({"them", "it", "these", "those"}) + +# English plural irregulars the parser may encounter in grade-school +# problem text. Most nouns canonicalize via the simple "+s" rule below. +_PLURAL_IRREGULARS: dict[str, str] = { + "candy": "candies", + "berry": "berries", + "cherry": "cherries", + "fly": "flies", + "story": "stories", + "penny": "pennies", + "box": "boxes", + "bus": "buses", + "dish": "dishes", + "watch": "watches", + "child": "children", + "person": "people", + "man": "men", + "woman": "women", + "foot": "feet", + "tooth": "teeth", + "mouse": "mice", + "goose": "geese", +} + + +def _canonical_unit(raw: str) -> str: + """Lowercase + pluralize per the README canonicalization rule. + + Grade-school problem text often uses singular for n=1 ("1 coin") + even though ground-truth graphs canonicalize to plural ("coins"). + The parser bridges by normalizing every extracted unit to plural. + """ + s = raw.lower() + if s in _PLURAL_IRREGULARS: + return _PLURAL_IRREGULARS[s] + if s.endswith("s"): + return s + return s + "s" + + +@dataclass +class _ParserState: + """Mutable state threaded through the parser. + + All fields are append-only or last-write-wins; the parser never + revises an earlier decision. This keeps determinism trivial to + prove. + """ + + entities: list[str] = field(default_factory=list) + initial_state: list[InitialPossession] = field(default_factory=list) + operations: list[Operation] = field(default_factory=list) + unknown: Unknown | None = None + last_unit: str | None = None + last_singular_subject: str | None = None + + def add_entity(self, name: str) -> None: + if name not in self.entities: + self.entities.append(name) + + +def parse_problem(text: str) -> MathProblemGraph: + """Parse ``text`` into a :class:`MathProblemGraph`. + + Raises :class:`ParseError` if any sentence cannot be classified, if + no question sentence is present, if multiple question sentences are + present, or if the resulting graph violates structural integrity + (e.g. question references an entity never introduced). + """ + if not isinstance(text, str) or not text.strip(): + raise ParseError(f"empty or non-string problem: {text!r}") + + state = _ParserState() + sentences = _split_sentences(text) + if not sentences: + raise ParseError(f"no sentences found: {text!r}") + + question_sentences = [s for s in sentences if s.rstrip().endswith("?")] + statement_sentences = [s for s in sentences if not s.rstrip().endswith("?")] + + if len(question_sentences) != 1: + raise ParseError( + f"expected exactly one question sentence ending in '?', got " + f"{len(question_sentences)}: {text!r}" + ) + + for s in statement_sentences: + _process_statement(s, state) + + _process_question(question_sentences[0], state) + + if state.unknown is None: + raise ParseError(f"no question parsed: {text!r}") + + return MathProblemGraph( + entities=tuple(state.entities), + initial_state=tuple(state.initial_state), + operations=tuple(state.operations), + unknown=state.unknown, + ) + + +# --------------------------------------------------------------------------- +# Sentence-level helpers +# --------------------------------------------------------------------------- + +# Split on a sentence-terminal . ? or ! followed by whitespace. +_SENTENCE_SPLIT_RE = re.compile(r"(?<=[.?!])\s+") + +# Compound-clause split inside one statement sentence: +# "She buys 5 more, then donates 3." +# Resulting clauses inherit the subject of the first clause. +_COMPOUND_SPLIT_RE = re.compile(r",\s*(?:then|and)\s+", flags=re.IGNORECASE) + +# A statement sentence may open with "Then " as a sequence marker that +# inherits subject + unit from the prior sentence: +# "Sam buys 3. Then he eats 1." +_SENTENCE_OPENER_THEN_RE = re.compile(r"^Then\s+", flags=re.IGNORECASE) + + +def _split_sentences(text: str) -> list[str]: + text = text.strip() + pieces = _SENTENCE_SPLIT_RE.split(text) + return [p.strip() for p in pieces if p.strip()] + + +# --------------------------------------------------------------------------- +# Initial-possession patterns +# --------------------------------------------------------------------------- + +# " has ." — entity must be a Title-Cased word. +_INITIAL_HAS_RE = re.compile( + r"^(?P[A-Z]\w+)\s+has\s+(?P\d+)\s+(?P\w+)$" +) + + +def _process_statement(sentence: str, state: _ParserState) -> None: + s = sentence.rstrip(".").strip() + + # Strip leading "Then " sequence marker — operation inherits subject + # and unit from the prior sentence. Same semantics as the in-sentence + # ", then" compound marker, just punctuated as a separate sentence. + sentence_opens_with_then = bool(_SENTENCE_OPENER_THEN_RE.match(s)) + if sentence_opens_with_then: + s = _SENTENCE_OPENER_THEN_RE.sub("", s).strip() + + if _try_initial(s, state): + return + + # Compound: split on ", then" / ", and" — first clause has explicit + # subject unless the sentence opened with "Then" (in which case the + # first clause also inherits). + parts = _COMPOUND_SPLIT_RE.split(s) + for index, clause in enumerate(parts): + clause = clause.strip() + if not clause: + continue + has_explicit_subject = (index == 0) and not sentence_opens_with_then + if not _try_operation(clause, state, has_explicit_subject): + raise ParseError( + f"could not parse statement clause: {clause!r} " + f"(in sentence: {sentence!r})" + ) + + +def _try_initial(s: str, state: _ParserState) -> bool: + m = _INITIAL_HAS_RE.match(s) + if not m: + return False + entity = m.group("entity") + value = int(m.group("value")) + unit = _canonical_unit(m.group("unit")) + state.add_entity(entity) + state.initial_state.append( + InitialPossession(entity=entity, quantity=Quantity(value=value, unit=unit)) + ) + state.last_unit = unit + state.last_singular_subject = entity + return True + + +# --------------------------------------------------------------------------- +# Operation patterns +# --------------------------------------------------------------------------- + +# Add / subtract / transfer share a structure: +# [subject] verb value [more] [unit] [to target] [trailing prep phrase] +# Constraints expressed via lookaheads: +# - unit cannot start with "to" or "more" (those are sentence chrome) +# A trailing prepositional phrase ("on the floor", "from the box") is +# semantically irrelevant for the graph and is harmlessly discarded. +_OP_RE = re.compile( + r"^" + r"(?:(?P[A-Z]\w+|he|she|He|She|It|it)\s+)?" + r"(?P\w+)" + r"\s+(?P\d+)" + r"(?:\s+more)?" + r"(?:\s+(?!to\b)(?!more\b)(?!on\b)(?!from\b)(?!at\b)(?!in\b)" + r"(?P\w+))?" + r"(?:\s+to\s+(?P[A-Z]\w+))?" + r"(?:\s+(?:on|from|at|in|onto|into|under|over)\s+.+)?" + r"$" +) + +# Divide construction. Three syntactic frames the parser accepts: +# " splits them evenly into M groups [and keeps one group]" +# " splits his/her evenly into M groups [and keeps one group]" +# " splits [] evenly into M groups [and keeps one group]" +# Semantics: actor's quantity becomes (original / M). Operand value = M; +# operand unit comes from the explicit unit if present, else from +# state.last_unit (via "them"/"his/her" reference). +_DIVIDE_SPLIT_RE = re.compile( + r"^" + r"(?:(?P[A-Z]\w+|he|she|He|She|It|it)\s+)?" + r"splits\s+" + r"(?:" + r"(?Pthem|these|those)|" + r"(?:his|her|their)\s+(?P\w+)|" + r"(?P\d+)(?:\s+(?P\w+))?" + r")\s+" + r"evenly\s+(?:into|among)\s+" + r"(?P\d+)\s+(?:groups|piles|parts|people|stacks|bundles)" + r"(?:\s+and\s+keeps\s+(?:one\s+(?:group|pile|part|stack|bundle)|" + r"one|a\s+(?:group|pile|part|stack|bundle)))?" + r"$" +) + +# Multiply / divide: +# " doubles his savings" / " triples them" +# The scalar comes from the verb; the unit comes from state.last_unit +# (which the prior initial-possession or operation set). +_MULTIPLY_FACTOR_RE = re.compile( + r"^" + r"(?:(?P[A-Z]\w+|he|she|He|She|It|it)\s+)?" + r"(?Pdoubles|triples)" + r"(?:\s+\w+(?:\s+\w+)*)?" # any trailing object phrase (e.g. "his savings") + r"$" +) + + +def _resolve_subject( + raw_subject: str | None, + has_explicit_subject: bool, + state: _ParserState, +) -> str | None: + """Resolve pronouns and inherited subjects to an entity name. + + Returns ``None`` if no valid subject can be determined. + """ + if raw_subject is None: + if has_explicit_subject: + return None # malformed: explicit subject expected, none found + return state.last_singular_subject + if raw_subject.lower() in _SINGULAR_PRONOUNS: + return state.last_singular_subject + return raw_subject + + +def _try_operation( + clause: str, state: _ParserState, has_explicit_subject: bool +) -> bool: + # First, try multiply/divide-style verbs which don't take a numeric + # operand from the text. + mm = _MULTIPLY_FACTOR_RE.match(clause) + if mm: + return _apply_multiply(mm, state, has_explicit_subject) + + md = _DIVIDE_SPLIT_RE.match(clause) + if md: + return _apply_divide_split(md, state, has_explicit_subject) + + m = _OP_RE.match(clause) + if not m: + return False + + subject = _resolve_subject(m.group("subject"), has_explicit_subject, state) + if subject is None: + return False + + verb = m.group("verb").lower() + value = int(m.group("value")) + unit_raw = m.group("unit") + target = m.group("target") + + if unit_raw is None: + unit = state.last_unit + elif unit_raw.lower() in _OBJECT_PRONOUNS_OF_QUANTITY: + # "Sam adds 3 of them" — pronoun reference to last unit. + unit = state.last_unit + else: + unit = _canonical_unit(unit_raw) + if unit is None: + return False + + if verb in _ADD_VERBS: + if target is not None: + return False # add never takes a target + op = Operation( + actor=subject, kind="add", operand=Quantity(value=value, unit=unit) + ) + elif verb in _SUBTRACT_VERBS: + if target is not None: + return False + op = Operation( + actor=subject, kind="subtract", operand=Quantity(value=value, unit=unit) + ) + elif verb in _TRANSFER_VERBS: + if target is None: + return False # transfer requires explicit target + op = Operation( + actor=subject, + kind="transfer", + operand=Quantity(value=value, unit=unit), + target=target, + ) + state.add_entity(target) + else: + return False + + state.add_entity(subject) + state.operations.append(op) + state.last_unit = unit + state.last_singular_subject = subject + return True + + +def _apply_divide_split( + m: re.Match[str], state: _ParserState, has_explicit_subject: bool +) -> bool: + subject = _resolve_subject(m.group("subject"), has_explicit_subject, state) + if subject is None: + return False + groups = int(m.group("groups")) + if groups <= 0: + return False + # Resolve unit from whichever of the three syntactic frames matched. + if m.group("object_pronoun") is not None: + unit = state.last_unit + elif m.group("possessive_unit") is not None: + unit = _canonical_unit(m.group("possessive_unit")) + elif m.group("explicit_unit") is not None: + unit = _canonical_unit(m.group("explicit_unit")) + else: + unit = state.last_unit + if unit is None: + return False + state.add_entity(subject) + state.operations.append( + Operation( + actor=subject, + kind="divide", + operand=Quantity(value=groups, unit=unit), + ) + ) + state.last_singular_subject = subject + state.last_unit = unit + return True + + +def _apply_multiply( + m: re.Match[str], state: _ParserState, has_explicit_subject: bool +) -> bool: + subject = _resolve_subject(m.group("subject"), has_explicit_subject, state) + if subject is None: + return False + verb = m.group("verb").lower() + factor = _MULTIPLY_FACTOR_VERBS[verb] + unit = state.last_unit + if unit is None: + return False + state.add_entity(subject) + state.operations.append( + Operation( + actor=subject, + kind="multiply", + operand=Quantity(value=factor, unit=unit), + ) + ) + state.last_singular_subject = subject + return True + + +# --------------------------------------------------------------------------- +# Question patterns +# --------------------------------------------------------------------------- + +_Q_ENTITY_RE = re.compile( + r"^How\s+many\s+(?P\w+)\s+does\s+(?P[A-Z]\w+)" + r"\s+have(?:\s+(?:left|now|in\s+total|altogether)){0,2}$", + flags=re.IGNORECASE, +) + +_Q_TOTAL_RE = re.compile( + r"^How\s+many\s+(?P\w+)\s+do\s+they\s+have" + r"(?:\s+(?:in\s+total|altogether|left|now)){0,2}$", + flags=re.IGNORECASE, +) + + +def _process_question(sentence: str, state: _ParserState) -> None: + s = sentence.rstrip("?").strip() + + m = _Q_ENTITY_RE.match(s) + if m: + unit = _canonical_unit(m.group("unit")) + entity = m.group("entity") + # Preserve case of the entity as written; entity must already be + # introduced by the statements above. + if entity not in state.entities: + raise ParseError( + f"question references undefined entity {entity!r}: {sentence!r}" + ) + state.unknown = Unknown(entity=entity, unit=unit) + return + + m = _Q_TOTAL_RE.match(s) + if m: + unit = _canonical_unit(m.group("unit")) + state.unknown = Unknown(entity=None, unit=unit) + return + + raise ParseError(f"could not parse question: {sentence!r}") diff --git a/tests/test_math_parser.py b/tests/test_math_parser.py new file mode 100644 index 00000000..3ac5a936 --- /dev/null +++ b/tests/test_math_parser.py @@ -0,0 +1,157 @@ +"""ADR-0115 Phase 1.3 — math word-problem parser invariants. + +Pins five load-bearing invariants: + +1. **All 5 seed cases parse byte-equal.** Every case in + ``evals/gsm8k_parser_dev/cases.jsonl`` satisfies + ``parse_problem(case["problem"]).canonical_bytes() == + graph_from_dict(case["ground_truth_graph"]).canonical_bytes()``. + +2. **Determinism.** Parsing the same input twice produces byte-equal + canonical output. + +3. **Phase 1.3 exit criterion gate.** Parse correctness across the + current dev set is ≥ 0.90 (≥45/50 when the full set lands; today + 5/5 = 1.00 on the seed set). + +4. **Typed refusal on unsupported constructions.** Constructions out of + scope per ADR-0115 §"Scope boundary" raise :class:`ParseError` + rather than guessing or silently producing a wrong graph. + +5. **No mutation of source artifacts.** Parser is pure: same input + string yields the same output and does not touch any file. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from generate.math_parser import ParseError, parse_problem +from generate.math_problem_graph import MathProblemGraph, graph_from_dict + + +_REPO_ROOT = Path(__file__).resolve().parent.parent +_CASES_PATH = _REPO_ROOT / "evals" / "gsm8k_parser_dev" / "cases.jsonl" + + +def _load_cases() -> list[dict]: + return [json.loads(line) for line in _CASES_PATH.read_text().splitlines() if line.strip()] + + +def _parse_correctness(cases: list[dict]) -> tuple[int, int, list[str]]: + """Return (passed, total, failure_ids) for byte-equal parses.""" + passed = 0 + failed: list[str] = [] + for c in cases: + try: + got = parse_problem(c["problem"]) + want = graph_from_dict(c["ground_truth_graph"]) + if got.canonical_bytes() == want.canonical_bytes(): + passed += 1 + else: + failed.append(c["id"]) + except ParseError: + failed.append(c["id"]) + return passed, len(cases), failed + + +class TestSeedCasesParseByteEqual: + @pytest.mark.parametrize("case", _load_cases(), ids=lambda c: c["id"]) + def test_parses_byte_equal_to_ground_truth(self, case: dict) -> None: + got = parse_problem(case["problem"]) + want = graph_from_dict(case["ground_truth_graph"]) + assert got.canonical_bytes() == want.canonical_bytes(), ( + f"{case['id']}: parser output != ground truth\n" + f" got: {got.as_json()}\n" + f" want: {want.as_json()}" + ) + + +class TestDeterminism: + @pytest.mark.parametrize("case", _load_cases(), ids=lambda c: c["id"]) + def test_two_parses_produce_byte_equal_output(self, case: dict) -> None: + a = parse_problem(case["problem"]) + b = parse_problem(case["problem"]) + assert a.canonical_bytes() == b.canonical_bytes() + assert a == b + + +class TestPhase1_3ExitCriterion: + """The exit criterion for Phase 1.3 is parse correctness ≥ 0.90. + + Today the dev set is the 5 seed cases (5/5 = 1.00). When Codex's + Phase 1.2 PR lands with 45 more cases, this same gate runs against + 50 and must produce ≥ 45 passes to ship Phase 1.3. + """ + + def test_parse_correctness_meets_exit_criterion(self) -> None: + cases = _load_cases() + passed, total, failed = _parse_correctness(cases) + ratio = passed / total + assert ratio >= 0.90, ( + f"parse correctness {passed}/{total} = {ratio:.2%} below 0.90 " + f"exit criterion; failing cases: {failed}" + ) + + +class TestParserRejectsMalformed: + def test_empty_input_raises(self) -> None: + with pytest.raises(ParseError): + parse_problem("") + + def test_whitespace_only_raises(self) -> None: + with pytest.raises(ParseError): + parse_problem(" \n\t ") + + def test_no_question_raises(self) -> None: + with pytest.raises(ParseError, match="exactly one question"): + parse_problem("Sam has 5 apples. He buys 3 more.") + + def test_multiple_questions_raises(self) -> None: + with pytest.raises(ParseError, match="exactly one question"): + parse_problem( + "Sam has 5 apples. How many does he have? How many does Tom have?" + ) + + def test_question_referencing_undefined_entity_raises(self) -> None: + with pytest.raises(ParseError, match="undefined entity"): + parse_problem("Sam has 5 apples. How many apples does Tom have?") + + def test_unsupported_construction_raises(self) -> None: + # Conditional / time-modal is out of scope per ADR-0115 Phase 1.1. + with pytest.raises(ParseError): + parse_problem( + "If Sam had 5 apples, how many apples does Sam have?" + ) + + +class TestParserIsPure: + def test_does_not_mutate_input_string(self) -> None: + text = "Sam has 5 apples. He buys 3 more. How many apples does Sam have?" + before = text + parse_problem(text) + assert text == before + + def test_does_not_mutate_cases_file(self) -> None: + before = _CASES_PATH.read_bytes() + for c in _load_cases(): + try: + parse_problem(c["problem"]) + except ParseError: + pass + after = _CASES_PATH.read_bytes() + assert before == after + + +class TestParserOutputIsTyped: + def test_returns_math_problem_graph(self) -> None: + result = parse_problem( + "Sam has 5 apples. He buys 3 more. How many apples does Sam have?" + ) + assert isinstance(result, MathProblemGraph) + assert "Sam" in result.entities + assert result.unknown.entity == "Sam" + assert result.unknown.unit == "apples"