From b448657c158240fc6c1ea8f1077e643c0e481b35 Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 23 May 2026 21:58:55 -0700 Subject: [PATCH] =?UTF-8?q?feat(ADR-0136.S.3):=20compound=20initial-mutati?= =?UTF-8?q?on=20extractor=20=E2=80=94=20one=20shape,=20gsm8k-0010=20barrie?= =?UTF-8?q?r=20shift,=20wrong=3D=3D0=20(#207)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closed-verb init-mutation extractor for "Entity had N unit, but then verb M" canonical compound form. Produces derived InitialPossession (N ± M) through existing graph machinery (no short-circuit). Admission delta: 0 (gsm8k-0010 sentence 1 now extracts but sentence 2 fraction_operand blocks). Barrier shifted: 1 case (0010: compound_statement → fraction_operand). Axis lane: 24/24 pass, wrong=0. S.1 lane: unchanged. GSM8K admission set: {0014, 0018, 0042} unchanged. --- .../ADR-0136.S3-compound-initial-mutation.md | 108 ++++++++ .../S3_compound_initial_mutation/__init__.py | 0 .../v1/__init__.py | 0 .../v1/cases.jsonl | 24 ++ .../v1/report.json | 234 ++++++++++++++++++ .../S3_compound_initial_mutation/v1/runner.py | 125 ++++++++++ generate/math_candidate_parser.py | 84 +++++++ ...t_adr_0136_S3_compound_initial_mutation.py | 177 +++++++++++++ 8 files changed, 752 insertions(+) create mode 100644 docs/decisions/ADR-0136.S3-compound-initial-mutation.md create mode 100644 evals/math_capability_axes/S3_compound_initial_mutation/__init__.py create mode 100644 evals/math_capability_axes/S3_compound_initial_mutation/v1/__init__.py create mode 100644 evals/math_capability_axes/S3_compound_initial_mutation/v1/cases.jsonl create mode 100644 evals/math_capability_axes/S3_compound_initial_mutation/v1/report.json create mode 100644 evals/math_capability_axes/S3_compound_initial_mutation/v1/runner.py create mode 100644 tests/test_adr_0136_S3_compound_initial_mutation.py diff --git a/docs/decisions/ADR-0136.S3-compound-initial-mutation.md b/docs/decisions/ADR-0136.S3-compound-initial-mutation.md new file mode 100644 index 00000000..ea02e4bf --- /dev/null +++ b/docs/decisions/ADR-0136.S3-compound-initial-mutation.md @@ -0,0 +1,108 @@ +# ADR-0136.S.3 — Compound Initial-Mutation Extractor + +**Status:** Accepted +**Parent:** ADR-0136 (Statement Layer Corridor) +**Date:** 2026-05-23 + +## Context + +The v2 refusal taxonomy (ADR-0136.S.2) identified 6 cases with `compound_statement` +as primary barrier. These cases contain two operations fused into a single sentence. +S.3 targets exactly one canonical shape — the initial-mutation form: + +``` + had [initially], but [then] . +``` + +This is the simplest compound shape: a single derived `InitialPossession(entity, N ± M, unit)` +candidate, same topology as `_CONJ_EMBEDDED_RE`'s SUM-emission pattern. Other compound +shapes (fraction-of-remainder, percentage-of, age-multiplier, verb-conjunction) are +structurally different and out of scope. + +## Decision + +### One shape, not "all of compound_statement" + +The 6 compound_statement cases represent at least 5 distinct structural shapes. S.3 +ships one extractor for the initial-mutation form only. Bundling multiple shapes +would conflate distinct parser risks and violate the one-shape-per-PR discipline. + +### Closed verb sets + +Past-tense only (the "had N, but then verb M" form requires past tense): + +**Subtract:** lost, gave, gave away, used, spent, ate, dropped, sold + +**Add:** gained, got, received, found, earned, picked up, bought + +No wildcards. Verbs not in either set cause the sentence to refuse (not wrong). + +### No short-circuit path + +Per ADR-0137's subsumption directive, the extractor produces a `CandidateInitial` +that flows through the existing graph machinery. No new bypass route was added. + +### Negative-result refusal + +If N - M < 0, the extractor refuses. Emitting a negative quantity would violate +the solver's non-negative invariant. + +## Per-Case Impact + +| Case | v2 Barrier | S.3 Effect | Post-S.3 Barrier | Why | +|---|---|---|---|---| +| 0010 | compound_statement | **Shifted** | fraction_operand | Sentence 1 now parses (Yun, 8, paperclips); sentence 2 "1/4 more than" blocks | +| 0012 | compound_statement | No change | compound_statement | "his fish ate half of them" = compound + fraction + coreference | +| 0013 | compound_statement | No change | compound_statement | "uploads 10 one-hour videos each day" = nested embedded quantifier | +| 0016 | compound_statement | No change | compound_statement | "2 more than 5 miles" = comparative arithmetic, not init-mutation | +| 0032 | compound_statement | No change | compound_statement | "draws and colors 10 pictures" = verb conjunction, not init-mutation | +| 0033 | compound_statement | No change | compound_statement | "7 times her age" = multiplier, not init-mutation | + +## GSM8K Admission Count + +| Metric | Pre-S.3 | Post-S.3 | +|---|---|---| +| Admitted (correct) | 3 | 3 | +| Wrong | 0 | 0 | +| Refused | 47 | 47 | +| Barrier shifted (v2 → v3) | — | 1 (gsm8k-0010) | + +Direct admission delta: **0**. gsm8k-0010's sentence 1 now extracts cleanly but +sentence 2's fraction_operand blocks the case from admitting. + +## Axis Lane + +24 curated cases at `evals/math_capability_axes/S3_compound_initial_mutation/v1/`: + +| Category | Count | Gate | +|---|---|---| +| subtract_canonical | 4 | answer == expected | +| subtract_synonym | 7 | answer == expected | +| add_canonical | 4 | answer == expected | +| add_synonym | 5 | answer == expected | +| refusal_negative_result | 2 | answer is None | +| refusal_verb_miss | 2 | answer is None | + +All 24 pass, wrong = 0. + +## Cross-Lane Regression + +- S.1 rate_events: 20/20 pass, wrong = 0 +- S.3 axis lane: 24/24 pass, wrong = 0 +- GSM8K admission set: {0014, 0018, 0042} (unchanged) + +## Deferred + +Every other compound shape from the 6 cases: + +- **0012**: "his fish ate half of them" — compound + fraction + coreference +- **0013**: "uploads 10 one-hour videos each day" — nested embedded quantifier + temporal +- **0016**: "2 more than 5 miles" — comparative arithmetic expression +- **0032**: "draws and colors 10 pictures" — verb conjunction (single object, multiple verbs) +- **0033**: "7 times her age" — age-multiplier compound + +## Evidence + +- Parser addition: `generate/math_candidate_parser.py` (`_INIT_MUTATION_RE`, `_init_mutation_candidates`) +- Axis lane: `evals/math_capability_axes/S3_compound_initial_mutation/v1/` +- Tests: `tests/test_adr_0136_S3_compound_initial_mutation.py` diff --git a/evals/math_capability_axes/S3_compound_initial_mutation/__init__.py b/evals/math_capability_axes/S3_compound_initial_mutation/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/math_capability_axes/S3_compound_initial_mutation/v1/__init__.py b/evals/math_capability_axes/S3_compound_initial_mutation/v1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/math_capability_axes/S3_compound_initial_mutation/v1/cases.jsonl b/evals/math_capability_axes/S3_compound_initial_mutation/v1/cases.jsonl new file mode 100644 index 00000000..789db148 --- /dev/null +++ b/evals/math_capability_axes/S3_compound_initial_mutation/v1/cases.jsonl @@ -0,0 +1,24 @@ +{"case_id": "S3-sub-canon-001", "category": "subtract_canonical", "problem": "Tom had 20 marbles, but then lost 8. How many marbles does Tom have?", "expected_answer": 12.0} +{"case_id": "S3-sub-canon-002", "category": "subtract_canonical", "problem": "Sara had 50 stickers initially, but lost 15. How many stickers does Sara have?", "expected_answer": 35.0} +{"case_id": "S3-sub-canon-003", "category": "subtract_canonical", "problem": "Mike had 100 coins, but then lost 37. How many coins does Mike have?", "expected_answer": 63.0} +{"case_id": "S3-sub-canon-004", "category": "subtract_canonical", "problem": "Yun had 20 paperclips initially, but then lost 12. How many paperclips does Yun have?", "expected_answer": 8.0} +{"case_id": "S3-sub-syn-001", "category": "subtract_synonym", "problem": "Alice had 30 apples, but then gave 12. How many apples does Alice have?", "expected_answer": 18.0} +{"case_id": "S3-sub-syn-002", "category": "subtract_synonym", "problem": "Bob had 25 pencils initially, but then used 9. How many pencils does Bob have?", "expected_answer": 16.0} +{"case_id": "S3-sub-syn-003", "category": "subtract_synonym", "problem": "Carol had 40 cookies, but then spent 18. How many cookies does Carol have?", "expected_answer": 22.0} +{"case_id": "S3-sub-syn-004", "category": "subtract_synonym", "problem": "Dave had 15 balls, but then ate 3. How many balls does Dave have?", "expected_answer": 12.0} +{"case_id": "S3-sub-syn-005", "category": "subtract_synonym", "problem": "Eve had 60 stamps initially, but then sold 25. How many stamps does Eve have?", "expected_answer": 35.0} +{"case_id": "S3-sub-syn-006", "category": "subtract_synonym", "problem": "Frank had 45 cards, but then dropped 10. How many cards does Frank have?", "expected_answer": 35.0} +{"case_id": "S3-add-canon-001", "category": "add_canonical", "problem": "Gina had 10 books, but then gained 5. How many books does Gina have?", "expected_answer": 15.0} +{"case_id": "S3-add-canon-002", "category": "add_canonical", "problem": "Hank had 30 tokens initially, but then gained 12. How many tokens does Hank have?", "expected_answer": 42.0} +{"case_id": "S3-add-canon-003", "category": "add_canonical", "problem": "Iris had 22 gems, but gained 8. How many gems does Iris have?", "expected_answer": 30.0} +{"case_id": "S3-add-canon-004", "category": "add_canonical", "problem": "Jake had 55 points initially, but then gained 20. How many points does Jake have?", "expected_answer": 75.0} +{"case_id": "S3-add-syn-001", "category": "add_synonym", "problem": "Kim had 14 shells, but then got 6. How many shells does Kim have?", "expected_answer": 20.0} +{"case_id": "S3-add-syn-002", "category": "add_synonym", "problem": "Leo had 40 tickets initially, but then received 15. How many tickets does Leo have?", "expected_answer": 55.0} +{"case_id": "S3-add-syn-003", "category": "add_synonym", "problem": "Mia had 18 rings, but then found 7. How many rings does Mia have?", "expected_answer": 25.0} +{"case_id": "S3-add-syn-004", "category": "add_synonym", "problem": "Ned had 35 coins, but then earned 10. How many coins does Ned have?", "expected_answer": 45.0} +{"case_id": "S3-ref-negative-001", "category": "refusal_negative_result", "problem": "Pat had 10 toys, but then lost 20. How many toys does Pat have?", "expected_answer": null} +{"case_id": "S3-ref-negative-002", "category": "refusal_negative_result", "problem": "Quinn had 5 pens initially, but then gave 15. How many pens does Quinn have?", "expected_answer": null} +{"case_id": "S3-ref-verb-miss-001", "category": "refusal_verb_miss", "problem": "Rob had 10 balls, but then juggled 5. How many balls does Rob have?", "expected_answer": null} +{"case_id": "S3-ref-verb-miss-002", "category": "refusal_verb_miss", "problem": "Sam had 20 cards, but then shuffled 8. How many cards does Sam have?", "expected_answer": null} +{"case_id": "S3-sub-syn-007", "category": "subtract_synonym", "problem": "Tina had 80 beads initially, but gave away 33. How many beads does Tina have?", "expected_answer": 47.0} +{"case_id": "S3-add-syn-005", "category": "add_synonym", "problem": "Uma had 12 dolls, but then bought 4. How many dolls does Uma have?", "expected_answer": 16.0} diff --git a/evals/math_capability_axes/S3_compound_initial_mutation/v1/report.json b/evals/math_capability_axes/S3_compound_initial_mutation/v1/report.json new file mode 100644 index 00000000..173f13d5 --- /dev/null +++ b/evals/math_capability_axes/S3_compound_initial_mutation/v1/report.json @@ -0,0 +1,234 @@ +{ + "adr": "0136.S.3", + "axis": "compound_initial_mutation", + "cases_path": "evals/math_capability_axes/S3_compound_initial_mutation/v1/cases.jsonl", + "metrics": { + "cases_total": 24, + "pass_rate": 1.0, + "passed": 24, + "wrong": 0, + "wrong_count_is_zero": true, + "wrong_rate": 0.0 + }, + "per_case": [ + { + "answer": 12.0, + "case_id": "S3-sub-canon-001", + "category": "subtract_canonical", + "expected_answer": 12.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 35.0, + "case_id": "S3-sub-canon-002", + "category": "subtract_canonical", + "expected_answer": 35.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 63.0, + "case_id": "S3-sub-canon-003", + "category": "subtract_canonical", + "expected_answer": 63.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 8.0, + "case_id": "S3-sub-canon-004", + "category": "subtract_canonical", + "expected_answer": 8.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 18.0, + "case_id": "S3-sub-syn-001", + "category": "subtract_synonym", + "expected_answer": 18.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 16.0, + "case_id": "S3-sub-syn-002", + "category": "subtract_synonym", + "expected_answer": 16.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 22.0, + "case_id": "S3-sub-syn-003", + "category": "subtract_synonym", + "expected_answer": 22.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 12.0, + "case_id": "S3-sub-syn-004", + "category": "subtract_synonym", + "expected_answer": 12.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 35.0, + "case_id": "S3-sub-syn-005", + "category": "subtract_synonym", + "expected_answer": 35.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 35.0, + "case_id": "S3-sub-syn-006", + "category": "subtract_synonym", + "expected_answer": 35.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 15.0, + "case_id": "S3-add-canon-001", + "category": "add_canonical", + "expected_answer": 15.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 42.0, + "case_id": "S3-add-canon-002", + "category": "add_canonical", + "expected_answer": 42.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 30.0, + "case_id": "S3-add-canon-003", + "category": "add_canonical", + "expected_answer": 30.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 75.0, + "case_id": "S3-add-canon-004", + "category": "add_canonical", + "expected_answer": 75.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 20.0, + "case_id": "S3-add-syn-001", + "category": "add_synonym", + "expected_answer": 20.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 55.0, + "case_id": "S3-add-syn-002", + "category": "add_synonym", + "expected_answer": 55.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 25.0, + "case_id": "S3-add-syn-003", + "category": "add_synonym", + "expected_answer": 25.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 45.0, + "case_id": "S3-add-syn-004", + "category": "add_synonym", + "expected_answer": 45.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": null, + "case_id": "S3-ref-negative-001", + "category": "refusal_negative_result", + "expected_answer": null, + "outcome": "pass", + "reason": "" + }, + { + "answer": null, + "case_id": "S3-ref-negative-002", + "category": "refusal_negative_result", + "expected_answer": null, + "outcome": "pass", + "reason": "" + }, + { + "answer": null, + "case_id": "S3-ref-verb-miss-001", + "category": "refusal_verb_miss", + "expected_answer": null, + "outcome": "pass", + "reason": "" + }, + { + "answer": null, + "case_id": "S3-ref-verb-miss-002", + "category": "refusal_verb_miss", + "expected_answer": null, + "outcome": "pass", + "reason": "" + }, + { + "answer": 47.0, + "case_id": "S3-sub-syn-007", + "category": "subtract_synonym", + "expected_answer": 47.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 16.0, + "case_id": "S3-add-syn-005", + "category": "add_synonym", + "expected_answer": 16.0, + "outcome": "pass", + "reason": "" + } + ], + "per_category": { + "add_canonical": { + "pass": 4, + "wrong": 0 + }, + "add_synonym": { + "pass": 5, + "wrong": 0 + }, + "refusal_negative_result": { + "pass": 2, + "wrong": 0 + }, + "refusal_verb_miss": { + "pass": 2, + "wrong": 0 + }, + "subtract_canonical": { + "pass": 4, + "wrong": 0 + }, + "subtract_synonym": { + "pass": 7, + "wrong": 0 + } + }, + "schema_version": 1 +} diff --git a/evals/math_capability_axes/S3_compound_initial_mutation/v1/runner.py b/evals/math_capability_axes/S3_compound_initial_mutation/v1/runner.py new file mode 100644 index 00000000..a661561b --- /dev/null +++ b/evals/math_capability_axes/S3_compound_initial_mutation/v1/runner.py @@ -0,0 +1,125 @@ +"""ADR-0136.S.3 — Capability axis runner for compound initial-mutation parsing. + +Exercises the init-mutation extractor ("Entity had N unit, but then verb M") +against curated coverage cases independent of GSM8K. + +Per-case classification: + +| Case category | pass criterion | +|-----------------------------|-------------------------------------------| +| subtract_canonical | answer == expected_answer (exact float) | +| subtract_synonym | answer == expected_answer | +| add_canonical | answer == expected_answer | +| add_synonym | answer == expected_answer | +| refusal_negative_result | answer is None (question not admitted) | +| refusal_verb_miss | answer is None (question not admitted) | + +``wrong == 0`` is the load-bearing gate (ADR-0114a Obligation #4). + +Determinism: case order in ``cases.jsonl`` is the report order; same +input file -> byte-equal report. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from generate.math_candidate_graph import parse_and_solve + +_HERE = Path(__file__).resolve().parent +_CASES_PATH = _HERE / "cases.jsonl" +_REPORT_PATH = _HERE / "report.json" + + +def _load_cases() -> list[dict[str, Any]]: + return [ + json.loads(line) + for line in _CASES_PATH.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def _score_case(case: dict[str, Any]) -> dict[str, Any]: + r = parse_and_solve(case["problem"]) + exp = case["expected_answer"] + + if exp is not None: + if r.answer == exp: + outcome, reason = "pass", "" + elif r.answer is None: + outcome = "wrong" + reason = f"expected {exp} but got refusal: {r.refusal_reason}" + else: + outcome = "wrong" + reason = f"expected {exp} but got {r.answer}" + else: + if r.answer is None: + outcome, reason = "pass", "" + else: + outcome = "wrong" + reason = f"expected refusal but got answer {r.answer}" + + return { + "case_id": case["case_id"], + "category": case["category"], + "outcome": outcome, + "reason": reason, + "answer": r.answer, + "expected_answer": exp, + } + + +def build_report() -> dict[str, Any]: + cases = _load_cases() + per_case = [_score_case(c) for c in cases] + total = len(per_case) + passed = sum(1 for d in per_case if d["outcome"] == "pass") + wrong = sum(1 for d in per_case if d["outcome"] == "wrong") + by_category: dict[str, dict[str, int]] = {} + for d in per_case: + slot = by_category.setdefault(d["category"], {"pass": 0, "wrong": 0}) + slot[d["outcome"]] = slot.get(d["outcome"], 0) + 1 + return { + "schema_version": 1, + "adr": "0136.S.3", + "axis": "compound_initial_mutation", + "cases_path": "evals/math_capability_axes/S3_compound_initial_mutation/v1/cases.jsonl", + "metrics": { + "cases_total": total, + "passed": passed, + "wrong": wrong, + "pass_rate": (passed / total) if total else 0.0, + "wrong_rate": (wrong / total) if total else 0.0, + "wrong_count_is_zero": wrong == 0, + }, + "per_category": { + k: dict(sorted(v.items())) for k, v in sorted(by_category.items()) + }, + "per_case": per_case, + } + + +def write_report(report: dict[str, Any]) -> None: + _REPORT_PATH.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def main() -> int: + report = build_report() + write_report(report) + m = report["metrics"] + print( + f"ADR-0136.S.3 compound_initial_mutation: passed {m['passed']}/{m['cases_total']} " + f"({m['pass_rate']:.1%}); wrong={m['wrong']} (gate: must be 0)" + ) + for cat, counts in report["per_category"].items(): + print(f" {cat:30s} {counts}") + return 0 if m["wrong_count_is_zero"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/generate/math_candidate_parser.py b/generate/math_candidate_parser.py index 9f059807..76942af9 100644 --- a/generate/math_candidate_parser.py +++ b/generate/math_candidate_parser.py @@ -491,6 +491,9 @@ def extract_initial_candidates(sentence: str) -> list[CandidateInitial]: out.extend(_conj_object_candidates(sentence)) out.extend(_embedded_quantifier_candidates(sentence)) + # ADR-0136.S.3 — compound initial-mutation: "Entity had N unit, but then verb M" + out.extend(_init_mutation_candidates(sentence)) + m2 = _INITIAL_THERE_ARE_RE.match(s) if m2 is not None: value_raw = m2.group("value") @@ -1571,6 +1574,87 @@ def _build_conj_embedded_sum( return [] +# --------------------------------------------------------------------------- +# ADR-0136.S.3 — Compound initial-mutation extractor +# --------------------------------------------------------------------------- + +_INIT_MUT_SUBTRACT_VERBS: Final[frozenset[str]] = frozenset({ + "lost", "gave", "gave away", "used", "spent", "ate", "dropped", "sold", +}) + +_INIT_MUT_ADD_VERBS: Final[frozenset[str]] = frozenset({ + "gained", "got", "received", "found", "earned", "picked up", "bought", +}) + +_INIT_MUT_VERB_PATTERN: Final[str] = ( + r"(?:" + "|".join( + re.escape(v) + for v in sorted( + _INIT_MUT_SUBTRACT_VERBS | _INIT_MUT_ADD_VERBS, + key=len, reverse=True, + ) + ) + r")" +) + +_INIT_MUTATION_RE: Final[re.Pattern[str]] = re.compile( + rf"^(?P{_ENTITY})\s+(?:has|have|had)\s+" + rf"(?P{_VALUE})\s+(?P\w+)" + r"(?:\s+initially)?" + r",?\s+but(?:\s+then)?\s+" + rf"(?P{_INIT_MUT_VERB_PATTERN})\s+" + rf"(?P{_VALUE})" + r"\s*\.?\s*$", + flags=re.IGNORECASE, +) + + +def _init_mutation_candidates(sentence: str) -> list[CandidateInitial]: + s = sentence.strip().rstrip(".") + m = _INIT_MUTATION_RE.match(s) + if m is None: + return [] + n_raw = m.group("n") + m_raw = m.group("m") + if _is_indefinite_quantifier(n_raw) or _is_indefinite_quantifier(m_raw): + return [] + rv_n = _resolve_value(n_raw) + rv_m = _resolve_value(m_raw) + if rv_n is None or rv_m is None: + return [] + verb = m.group("verb").lower() + if verb in _INIT_MUT_SUBTRACT_VERBS: + derived = rv_n.value - rv_m.value + elif verb in _INIT_MUT_ADD_VERBS: + derived = rv_n.value + rv_m.value + else: + return [] + if derived < 0: + return [] + unit_raw = m.group("unit") + unit = rv_n.unit_override if rv_n.unit_override is not None else _canonicalize_unit(unit_raw) + entity = _normalize_entity(m.group("entity")) + try: + return [ + CandidateInitial( + initial=InitialPossession( + entity=entity, + quantity=Quantity(value=derived, unit=unit), + ), + source_span=sentence, + matched_anchor="had", + matched_value_token=n_raw, + matched_unit_token=unit_raw, + matched_entity_token=m.group("entity"), + ) + ] + except Exception: + return [] + + +def _init_mutation_admitted(sentence: str) -> list[CandidateInitial]: + return _admit(_init_mutation_candidates(sentence)) + + # --------------------------------------------------------------------------- # ADR-0136.S.0 — Context-sentence classifier # --------------------------------------------------------------------------- diff --git a/tests/test_adr_0136_S3_compound_initial_mutation.py b/tests/test_adr_0136_S3_compound_initial_mutation.py new file mode 100644 index 00000000..1a57d49a --- /dev/null +++ b/tests/test_adr_0136_S3_compound_initial_mutation.py @@ -0,0 +1,177 @@ +"""ADR-0136.S.3 — compound initial-mutation extractor tests. + +Covers: regex matching, closed-verb enforcement, negative-result refusal, +indefinite-quantifier refusal, gsm8k-0010 sentence-1 extraction, +gsm8k-0010 barrier shift, and cross-lane regression gates. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from generate.math_candidate_parser import ( + _INIT_MUTATION_RE, + _init_mutation_candidates, + extract_initial_candidates, +) +from generate.math_candidate_graph import parse_and_solve + + +class TestInitMutationRegex: + def test_canonical_subtract(self) -> None: + assert _INIT_MUTATION_RE.match("Tom had 20 marbles, but then lost 8") + + def test_canonical_add(self) -> None: + assert _INIT_MUTATION_RE.match("Gina had 10 books, but then gained 5") + + def test_with_initially(self) -> None: + assert _INIT_MUTATION_RE.match( + "Yun had 20 paperclips initially, but then lost 12" + ) + + def test_without_then(self) -> None: + assert _INIT_MUTATION_RE.match("Bob had 30 coins, but lost 10") + + def test_multi_word_verb_gave_away(self) -> None: + assert _INIT_MUTATION_RE.match("Alice had 15 toys, but then gave away 5") + + def test_multi_word_verb_picked_up(self) -> None: + assert _INIT_MUTATION_RE.match("Bob had 10 rocks, but then picked up 3") + + def test_non_closed_verb_rejects(self) -> None: + assert _INIT_MUTATION_RE.match("Bob had 10 balls, but then juggled 5") is None + + def test_non_closed_verb_shuffled(self) -> None: + assert _INIT_MUTATION_RE.match( + "Sam had 20 cards, but then shuffled 8" + ) is None + + +class TestInitMutationCandidates: + def test_subtract_produces_correct_value(self) -> None: + cands = _init_mutation_candidates("Tom had 20 marbles, but then lost 8.") + assert len(cands) == 1 + assert cands[0].initial.entity == "Tom" + assert cands[0].initial.quantity.value == 12 + assert cands[0].initial.quantity.unit == "marbles" + + def test_add_produces_correct_value(self) -> None: + cands = _init_mutation_candidates("Gina had 10 books, but then gained 5.") + assert len(cands) == 1 + assert cands[0].initial.quantity.value == 15 + + def test_negative_result_refuses(self) -> None: + cands = _init_mutation_candidates("Pat had 10 toys, but then lost 20.") + assert cands == [] + + def test_indefinite_quantifier_refuses(self) -> None: + cands = _init_mutation_candidates( + "Bob had several marbles, but then lost 3." + ) + assert cands == [] + + def test_indefinite_mutation_refuses(self) -> None: + cands = _init_mutation_candidates( + "Bob had 10 marbles, but then lost some." + ) + assert cands == [] + + def test_closed_verb_miss_refuses(self) -> None: + cands = _init_mutation_candidates( + "Rob had 10 balls, but then juggled 5." + ) + assert cands == [] + + def test_gave_away_subtract(self) -> None: + cands = _init_mutation_candidates( + "Tina had 80 beads initially, but gave away 33." + ) + assert len(cands) == 1 + assert cands[0].initial.quantity.value == 47 + + def test_picked_up_add(self) -> None: + cands = _init_mutation_candidates( + "Bob had 10 rocks, but then picked up 3." + ) + assert len(cands) == 1 + assert cands[0].initial.quantity.value == 13 + + def test_zero_result_allowed(self) -> None: + cands = _init_mutation_candidates("Bob had 10 balls, but then lost 10.") + assert len(cands) == 1 + assert cands[0].initial.quantity.value == 0 + + +class TestExtractInitialCandidatesWiring: + def test_gsm8k_0010_sentence_1(self) -> None: + cands = extract_initial_candidates( + "Yun had 20 paperclips initially, but then lost 12." + ) + assert len(cands) == 1 + assert cands[0].initial.entity == "Yun" + assert cands[0].initial.quantity.value == 8 + assert cands[0].initial.quantity.unit == "paperclips" + + +class TestGsm8k0010BarrierShift: + def test_barrier_shifts_to_sentence_2(self) -> None: + r = parse_and_solve( + "Yun had 20 paperclips initially, but then lost 12. " + "Marion has 1/4 more than what Yun currently has, plus 7. " + "How many paperclips does Marion have?" + ) + assert r.answer is None + assert r.refusal_reason is not None + assert "Marion" in r.refusal_reason + assert "1/4" in r.refusal_reason + + +class TestCrossLaneRegression: + def test_s3_lane_wrong_is_zero(self) -> None: + from evals.math_capability_axes.S3_compound_initial_mutation.v1.runner import ( + build_report, + ) + + report = build_report() + assert report["metrics"]["wrong"] == 0 + assert report["metrics"]["passed"] == report["metrics"]["cases_total"] + + def test_s1_lane_wrong_is_zero(self) -> None: + from evals.math_capability_axes.S1_rate_events.v1.runner import build_report + + report = build_report() + assert report["metrics"]["wrong"] == 0 + + def test_gsm8k_admission_superset(self) -> None: + cases_path = Path("evals/gsm8k_math/train_sample/v1/cases.jsonl") + cases = [ + json.loads(line) + for line in cases_path.read_text().splitlines() + if line.strip() + ] + admitted = set() + wrong_count = 0 + for c in cases: + r = parse_and_solve(c["question"]) + if r.answer is not None: + if r.answer == c["answer_numeric"]: + admitted.add(c["case_id"]) + else: + wrong_count += 1 + assert wrong_count == 0 + required = { + "gsm8k-train-sample-v1-0014", + "gsm8k-train-sample-v1-0018", + "gsm8k-train-sample-v1-0042", + } + assert admitted >= required, f"Missing: {required - admitted}" + + def test_s3_deterministic(self) -> None: + from evals.math_capability_axes.S3_compound_initial_mutation.v1.runner import ( + build_report, + ) + + r1 = json.dumps(build_report(), indent=2, sort_keys=True) + r2 = json.dumps(build_report(), indent=2, sort_keys=True) + assert r1 == r2