From fe4f1223e081ceb9a3fef68f9e468ef76e8bf3cf Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 17 Jun 2026 19:56:37 -0700 Subject: [PATCH] feat(derivation): capability strike batch 3 peer-pick lift (#813) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(derivation): capability strike batch 3 peer-pick lift Gate A2d peer_partition_question closes train_sample 0025 by binding the conditional friend-count clause to a multiply-by-(1+N) graph step after WAVE-A multiplicative_aggregate statement ingestion. Ephemeral train_sample: 8/42/0 → 9/41/0, wrong=0 preserved. report.json untouched. * chore(analysis): normalize batch3 lookback EOF * chore(derivation): normalize peer partition test EOF * fix(derivation): require integer peer-pick friend counts --- ...ility-strike-batch3-lookback-2026-06-17.md | 72 ++++++++ generate/math_candidate_graph.py | 28 ++++ generate/math_candidate_parser.py | 80 +++++++++ ..._gsm8k_post_gate_a1_frontier_microscope.py | 5 +- ...candidate_graph_peer_partition_question.py | 154 ++++++++++++++++++ 5 files changed, 337 insertions(+), 2 deletions(-) create mode 100644 docs/analysis/gsm8k-capability-strike-batch3-lookback-2026-06-17.md create mode 100644 tests/test_math_candidate_graph_peer_partition_question.py diff --git a/docs/analysis/gsm8k-capability-strike-batch3-lookback-2026-06-17.md b/docs/analysis/gsm8k-capability-strike-batch3-lookback-2026-06-17.md new file mode 100644 index 00000000..ba321774 --- /dev/null +++ b/docs/analysis/gsm8k-capability-strike-batch3-lookback-2026-06-17.md @@ -0,0 +1,72 @@ +# GSM8K Capability Strike Batch 3 — Lookback (2026-06-17) + +## Scope + +Gate A2d `peer_partition_question` lift on train_sample case **0025**. + +## Baseline (honest, ephemeral, pre-change) + +Measured via `build_report()` on `origin/main` at #811 merge (708f27a2): + +| Metric | Value | +|--------|-------| +| correct | **8** | +| refused | **42** | +| wrong | **0** | + +Committed `report.json` remains **6/44/0** (historical pin; not rebaselined). + +## After (ephemeral live scoring) + +| Metric | Value | +|--------|-------| +| correct | **9** | +| refused | **41** | +| wrong | **0** | + +**Newly admitted:** `gsm8k-train-sample-v1-0025` +**Preserved:** `0002`, `0008`, `0014`, `0018`, `0024`, `0029`, `0038`, `0042` + +## Family implemented + +**`peer_partition_question`** — closed conditional + total-across question: + +- Conditional: `If N of 's friends pick the same amount as her/him` +- Question: `How many do and her/his friends pick in all?` +- Graph injects `multiply` by `(1 + N)` on the anchor entity's unit state after WAVE-A statement ingestion. + +## Why not overfit + +- No case-id branches or hardcoded answers. +- Regex family generalizes across entities, units, and numeric/word friend counts. +- Sibling tests vary names, counts, and units (Tom/apples, Alice/strawberries). +- Confusers refuse: missing conditional, entity mismatch, indefinite quantifier, comparative 0007 unchanged. +- Reuses existing `multiply` solver path and WAVE-A `multiplicative_aggregate` statement injection. + +## Deferred candidates (Arena) + +| Case | Blocker | +|------|---------| +| 0007 | Pattern B comparative detection-only + multi-entity inverse residual | +| 0047 | Equal-N bag split ≠ `unit_partition` v1 | +| 0004 | Nested fraction referent binding beyond Gate A2b | + +## Non-goals + +- No `report.json` rebaseline. +- No sealed-lane movement. +- No `determine()` / `answer=False`. +- No `unit_partition` extension for equal-bag distribution. + +## Validation + +```bash +git diff --check origin/main...HEAD +pytest tests/test_math_candidate_graph_peer_partition_question.py -q +pytest tests/test_math_candidate_graph_fraction_portion.py -q +pytest tests/test_math_candidate_graph_unit_partition_injection.py -q +pytest tests/test_recognizer_unit_partition_inject.py -q +pytest tests/test_math_candidate_graph_container_of_product.py -q +pytest tests/test_gsm8k_frontier_report.py -q +pytest tests/test_gsm8k_post_gate_a1_frontier_microscope.py -q +``` diff --git a/generate/math_candidate_graph.py b/generate/math_candidate_graph.py index ff1af106..b84a14cf 100644 --- a/generate/math_candidate_graph.py +++ b/generate/math_candidate_graph.py @@ -62,6 +62,7 @@ from generate.math_problem_graph import ( Operation, MathProblemGraph, PartitionChunk, + Quantity, ) from generate.math_completeness import uncovered_quantities from generate.derivation.r1_reconstruction import reconstruct_r1_total @@ -543,6 +544,33 @@ def _build_graph( except MathGraphError: return None + # Gate A2d — peer-pick multiply: anchor amount × (1 + friend_count). + if ( + question_choice.peer_count is not None + and question_choice.peer_reference_entity is not None + and question_choice.unknown.entity is None + ): + ref = question_choice.peer_reference_entity + unit = question_choice.unknown.unit + matching = [ + ic + for ic in initials_list + if ic.entity == ref and ic.quantity.unit == unit + ] + if len(matching) != 1: + return None + factor = 1 + question_choice.peer_count + try: + operations_list.append( + Operation( + actor=ref, + kind="multiply", + operand=Quantity(value=factor, unit=unit), + ) + ) + except MathGraphError: + return None + try: return MathProblemGraph( entities=tuple(entities), diff --git a/generate/math_candidate_parser.py b/generate/math_candidate_parser.py index 6a3c7a84..f9022726 100644 --- a/generate/math_candidate_parser.py +++ b/generate/math_candidate_parser.py @@ -812,6 +812,9 @@ class CandidateUnknown: yield_chunk_value: float | None = None yield_chunk_unit: str | None = None yield_quotient_unit: str | None = None + # Gate A2d — peer-pick partition: friends pick the same amount as anchor. + peer_count: int | None = None + peer_reference_entity: str | None = None consumed_value_tokens: tuple[str, ...] = () @@ -962,6 +965,11 @@ def extract_question_candidates( if out: return out + # Gate A2d — peer-pick total: friends pick the same amount as anchor entity. + out.extend(_pattern_peer_pick_question_candidates(sentence, problem_text)) + if out: + return out + # ADR-0163.D.4 — Pattern B: comparative quantifier ("how many more") out.extend(_pattern_b_comparative_candidates(sentence, problem_text)) if out: @@ -2895,6 +2903,19 @@ _Q_YIELD_RE: Final[re.Pattern[str]] = re.compile( flags=re.IGNORECASE, ) +# Gate A2d — peer-pick conditional + total-across question. +_PEER_PICK_CONDITIONAL_RE: Final[re.Pattern[str]] = re.compile( + rf"(?i)if\s+(?P{_VALUE})\s+of\s+" + rf"(?P{_ENTITY})\s*'s\s+friends\s+pick\s+the\s+same\s+amount\s+as\s+" + r"(?:her|him)\b" +) +_Q_PEER_PICK_RE: Final[re.Pattern[str]] = re.compile( + r"^How\s+many\s+(?P\w+)\s+do\s+" + rf"(?P{_ENTITY})\s+and\s+" + r"(?:her|his)\s+friends\s+pick\s+in\s+all\s*\??\s*$", + flags=re.IGNORECASE, +) + def _infer_yield_partition( problem_text: str | None, question_product: str @@ -3064,6 +3085,65 @@ def _resolve_question_entity( return _normalize_entity(raw_entity), raw_entity +def _infer_peer_pick_partition( + problem_text: str | None, question_entity: str +) -> tuple[int, str, str] | None: + """Infer peer friend-count from the conditional clause in ``problem_text``. + + Gate A2d's peer count is a count of people, not a general numeric value. + The regex shares ``_VALUE`` for local parser consistency, but admission is + narrower: only positive integer, unitless counts are allowed. Money, + slash-fraction, decimal, and other non-integer numeric surfaces refuse. + """ + if problem_text is None: + return None + m = _PEER_PICK_CONDITIONAL_RE.search(problem_text) + if m is None: + return None + entity = _normalize_entity(m.group("entity")) + if entity != _normalize_entity(question_entity): + return None + count_token = m.group("count") + if count_token.startswith(("$", "¢", "€", "£", "¥", "₱")) or "/" in count_token: + return None + resolved = _resolve_value(count_token) + if resolved is None: + return None + value = resolved.value + if value <= 0 or value != int(value): + return None + peer_count = int(value) + return peer_count, count_token, entity + + +def _pattern_peer_pick_question_candidates( + sentence: str, problem_text: str | None +) -> list[CandidateUnknown]: + """Gate A2d — total-across after peer friends pick the same amount.""" + s = sentence.strip() + m = _Q_PEER_PICK_RE.match(s) + if m is None: + return [] + unit_raw = m.group("unit") + unit = _canonicalize_unit(unit_raw) + entity = _normalize_entity(m.group("entity")) + partition = _infer_peer_pick_partition(problem_text, entity) + if partition is None: + return [] + peer_count, count_token, reference_entity = partition + return [ + CandidateUnknown( + unknown=Unknown(entity=None, unit=unit), + source_span=sentence, + matched_unit_token=unit_raw, + matched_entity_token=m.group("entity"), + peer_count=peer_count, + peer_reference_entity=reference_entity, + consumed_value_tokens=(count_token,), + ) + ] + + def _pattern_yield_question_candidates( sentence: str, problem_text: str | None ) -> list[CandidateUnknown]: diff --git a/tests/test_gsm8k_post_gate_a1_frontier_microscope.py b/tests/test_gsm8k_post_gate_a1_frontier_microscope.py index da6303ee..8168bfbb 100644 --- a/tests/test_gsm8k_post_gate_a1_frontier_microscope.py +++ b/tests/test_gsm8k_post_gate_a1_frontier_microscope.py @@ -109,10 +109,11 @@ def test_markdown_render_surfaces_partition_candidate(): def test_gate_a2_lifts_are_not_in_refusal_table(): - """Cases solved by Gate A2b/A2c must not appear among live refusals.""" + """Cases solved by Gate A2b/A2c/A2d must not appear among live refusals.""" summary = build_microscope_report(_load_cases()) refused_ids = {r["case_id"] for r in summary["refusal_table"]} assert "gsm8k-train-sample-v1-0002" not in refused_ids assert "gsm8k-train-sample-v1-0008" not in refused_ids - assert summary["counts"]["correct"] >= 8 + assert "gsm8k-train-sample-v1-0025" not in refused_ids + assert summary["counts"]["correct"] >= 9 assert summary["closed_injector_buckets"]["unit_partition_no_injection"] == 0 diff --git a/tests/test_math_candidate_graph_peer_partition_question.py b/tests/test_math_candidate_graph_peer_partition_question.py new file mode 100644 index 00000000..e971f193 --- /dev/null +++ b/tests/test_math_candidate_graph_peer_partition_question.py @@ -0,0 +1,154 @@ +"""Gate A2d — peer_partition_question composition.""" + +from __future__ import annotations + +from generate.math_candidate_graph import parse_and_solve +from generate.math_candidate_parser import ( + extract_initial_candidates, + extract_question_candidates, +) + + +def _run(text: str): + return parse_and_solve(text, sealed=False) + + +def test_peer_pick_question_extracts_with_conditional(): + full = ( + "Lilibeth fills 6 baskets where each basket holds 50 strawberries. " + "If three of Lilibeth's friends pick the same amount as her, " + "how many strawberries do Lilibeth and her friends pick in all?" + ) + q = "How many strawberries do Lilibeth and her friends pick in all?" + cands = extract_question_candidates(q, problem_text=full) + assert len(cands) == 1 + cand = cands[0] + assert cand.unknown.entity is None + assert cand.unknown.unit == "strawberries" + assert cand.peer_count == 3 + assert cand.peer_reference_entity == "Lilibeth" + assert cand.consumed_value_tokens == ("three",) + + +def test_train_sample_0025_end_to_end(): + text = ( + "Lilibeth and her friends go strawberry picking. " + "Lilibeth fills 6 baskets where each basket holds 50 strawberries. " + "If three of Lilibeth's friends pick the same amount as her, " + "how many strawberries do Lilibeth and her friends pick in all?" + ) + res = _run(text) + assert res.answer == 1200.0 + assert res.refusal_reason is None + + +def test_sibling_tom_apples(): + text = ( + "Tom picks apples. " + "Tom fills 4 baskets where each basket holds 25 apples. " + "If two of Tom's friends pick the same amount as him, " + "how many apples do Tom and his friends pick in all?" + ) + res = _run(text) + assert res.answer == 300.0 + assert res.refusal_reason is None + + +def test_sibling_alice_strawberries(): + text = ( + "Alice and her crew go picking. " + "Alice fills 3 baskets where each basket holds 40 strawberries. " + "If 1 of Alice's friends pick the same amount as her, " + "how many strawberries do Alice and her friends pick in all?" + ) + res = _run(text) + assert res.answer == 240.0 + assert res.refusal_reason is None + + +def test_confuser_missing_conditional_refuses(): + text = ( + "Lilibeth fills 6 baskets where each basket holds 50 strawberries. " + "How many strawberries do Lilibeth and her friends pick in all?" + ) + res = _run(text) + assert res.answer is None + + +def test_confuser_no_prior_initial_refuses(): + text = ( + "If three of Lilibeth's friends pick the same amount as her, " + "how many strawberries do Lilibeth and her friends pick in all?" + ) + res = _run(text) + assert res.answer is None + + +def test_confuser_entity_mismatch_refuses(): + text = ( + "Lilibeth fills 6 baskets where each basket holds 50 strawberries. " + "If three of Tom's friends pick the same amount as him, " + "how many strawberries do Lilibeth and her friends pick in all?" + ) + res = _run(text) + assert res.answer is None + + +def test_confuser_friends_without_count_refuses(): + text = ( + "Lilibeth fills 6 baskets where each basket holds 50 strawberries. " + "If some of Lilibeth's friends pick the same amount as her, " + "how many strawberries do Lilibeth and her friends pick in all?" + ) + res = _run(text) + assert res.answer is None + + +def test_confuser_fractional_friend_count_refuses(): + text = ( + "Lilibeth fills 6 baskets where each basket holds 50 strawberries. " + "If 3/2 of Lilibeth's friends pick the same amount as her, " + "how many strawberries do Lilibeth and her friends pick in all?" + ) + res = _run(text) + assert res.answer is None + + +def test_confuser_money_friend_count_refuses(): + text = ( + "Lilibeth fills 6 baskets where each basket holds 50 strawberries. " + "If $2 of Lilibeth's friends pick the same amount as her, " + "how many strawberries do Lilibeth and her friends pick in all?" + ) + res = _run(text) + assert res.answer is None + + +def test_comparative_question_still_refuses(): + text = ( + "Francine has five full boxes of crayons and 5 loose crayons, " + "and her friend has 27 loose crayons. " + "They need to put all of their loose crayons in a box. " + "How many more boxes do they need if Francine has a total of 85 crayons?" + ) + res = _run(text) + assert res.answer is None + + +def test_yield_regression_0008(): + text = ( + "Marnie makes bead bracelets. " + "She bought 5 bags of 50 beads and 2 bags of 100 beads. " + "If 50 beads are used to make one bracelet, how many bracelets " + "will Marnie be able to make out of the beads she bought?" + ) + res = _run(text) + assert res.answer == 9.0 + assert res.refusal_reason is None + + +def test_bags_of_product_statement_not_peer_pick(): + stmt = "She bought 4 bags of 15 coins and 1 bag of 40 coins." + cands = extract_initial_candidates(stmt) + assert len(cands) == 1 + assert cands[0].initial.quantity.value == 100.0