diff --git a/docs/analysis/gsm8k-capability-strike-batch2-lookback-2026-06-17.md b/docs/analysis/gsm8k-capability-strike-batch2-lookback-2026-06-17.md new file mode 100644 index 00000000..e5f7da28 --- /dev/null +++ b/docs/analysis/gsm8k-capability-strike-batch2-lookback-2026-06-17.md @@ -0,0 +1,54 @@ +# GSM8K Capability Strike Batch 2 — Lookback (2026-06-17) + +## Selected target + +- **Primary case:** `gsm8k-train-sample-v1-0008` (Marnie bead bracelets) +- **Family:** `container_of_product` (stmt) + `yield_question` (question) composing into existing `unit_partition` + +## Before / after (live ephemeral) + +| Metric | Before (#810) | After (this branch) | +|--------|---------------|---------------------| +| correct | 7 | **8** | +| refused | 43 | **42** | +| wrong | 0 | **0** | + +**Newly admitted:** `0008` +**Preserved admissions:** `0002`, `0014`, `0018`, `0024`, `0029`, `0038`, `0042` + +## Implementation slice + +1. **`_bags_of_product_candidates`** — `N of M ` under closed acquisition verbs; conjoined sum when units match. +2. **`_pattern_yield_question_candidates`** — `how many will be able to make` with rate inferred from `If N are used to make one `. +3. **`CandidateUnknown` yield fields** — graph-build injects `unit_partition` (reuses Gate A2a solver path; no new op kind). +4. **`_bind_parser_pronoun_actor`** — extended to bind pronoun **entities** on `CandidateInitial` (She → Marnie via discourse prior). + +## Anti-overfit evidence + +- Sibling synthetics: Tom/marbles→displays (6), Alice/coins→charms (5). +- Confusers refuse: mismatched conjunct units, missing rate clause, product/rate mismatch, non-integer quotient. +- Regression: `0042` embedded-quantifier conditional-op path still admits 30. +- No case-id branches; no hardcoded answers. + +## Hazards reviewed + +| Hazard | Mitigation | +|--------|------------| +| Confuse bags-of with embedded-quantifier (in each) | Separate regex; `0042` regression test | +| Ingredient vs product count | `unit_partition` requires exact integer quotient; mismatched units refuse | +| Pronoun entity vs question entity | Discourse binding on initials + named entity in question | +| Completeness false-positive on "one" | Rate tokens `(n, "one")` on question candidate | + +## Non-goals + +- `report.json` untouched +- Sealed lanes untouched +- No `determine()` / FrameVerdict paths +- No broad DCS widening + +## Files changed + +- `generate/math_candidate_parser.py` +- `generate/math_candidate_graph.py` +- `tests/test_math_candidate_graph_container_of_product.py` (new) +- `tests/test_gsm8k_post_gate_a1_frontier_microscope.py` (live-count fixture update) \ No newline at end of file diff --git a/generate/math_candidate_graph.py b/generate/math_candidate_graph.py index 5f5eecb0..ff1af106 100644 --- a/generate/math_candidate_graph.py +++ b/generate/math_candidate_graph.py @@ -57,9 +57,11 @@ from generate.math_candidate_parser import ( _to_seconds, ) from generate.math_problem_graph import ( + InitialPossession, MathGraphError, Operation, MathProblemGraph, + PartitionChunk, ) from generate.math_completeness import uncovered_quantities from generate.derivation.r1_reconstruction import reconstruct_r1_total @@ -202,37 +204,67 @@ def _bind_parser_pronoun_actor( antecedent: str | None, multi_actor_ambiguous: bool, ) -> SentenceChoice | None: - """Bind parser-emitted pronoun actors to a discourse antecedent.""" - if not isinstance(choice, CandidateOperation): - return choice - if choice.matched_actor_token.lower() not in _PARSER_PRONOUN_ACTORS: - return choice + """Bind parser-emitted pronoun actors/entities to a discourse antecedent.""" if multi_actor_ambiguous or not antecedent: - return None + if isinstance(choice, CandidateOperation) and ( + choice.matched_actor_token.lower() in _PARSER_PRONOUN_ACTORS + ): + return None + if isinstance(choice, CandidateInitial) and ( + choice.matched_entity_token.lower() in _PARSER_PRONOUN_ACTORS + ): + return None + return choice from generate.math_candidate_parser import _normalize_entity bound_actor = _normalize_entity(antecedent) - if bound_actor == choice.op.actor: - return choice - try: - rebound = Operation( - actor=bound_actor, - kind=choice.op.kind, - operand=choice.op.operand, - target=choice.op.target, + if isinstance(choice, CandidateOperation): + if choice.matched_actor_token.lower() not in _PARSER_PRONOUN_ACTORS: + return choice + if bound_actor == choice.op.actor: + return choice + try: + rebound = Operation( + actor=bound_actor, + kind=choice.op.kind, + operand=choice.op.operand, + target=choice.op.target, + ) + except MathGraphError: + return None + return CandidateOperation( + op=rebound, + source_span=choice.source_span, + matched_verb=choice.matched_verb, + matched_value_token=choice.matched_value_token, + matched_unit_token=choice.matched_unit_token, + matched_actor_token=choice.matched_actor_token, + matched_target_token=choice.matched_target_token, + matched_reference_actor_token=choice.matched_reference_actor_token, ) - except MathGraphError: - return None - return CandidateOperation( - op=rebound, - source_span=choice.source_span, - matched_verb=choice.matched_verb, - matched_value_token=choice.matched_value_token, - matched_unit_token=choice.matched_unit_token, - matched_actor_token=choice.matched_actor_token, - matched_target_token=choice.matched_target_token, - matched_reference_actor_token=choice.matched_reference_actor_token, - ) + if isinstance(choice, CandidateInitial): + if choice.matched_entity_token.lower() not in _PARSER_PRONOUN_ACTORS: + return choice + if bound_actor == choice.initial.entity: + return choice + try: + rebound_initial = InitialPossession( + entity=bound_actor, + quantity=choice.initial.quantity, + ) + except MathGraphError: + return None + return CandidateInitial( + initial=rebound_initial, + source_span=choice.source_span, + matched_anchor=choice.matched_anchor, + matched_value_token=choice.matched_value_token, + matched_unit_token=choice.matched_unit_token, + matched_entity_token=choice.matched_entity_token, + composition_evidence=choice.composition_evidence, + consumed_value_tokens=choice.consumed_value_tokens, + ) + return choice def _filtered_statement_choices(sentence: str) -> list[SentenceChoice]: @@ -488,6 +520,29 @@ def _build_graph( if question_choice.unknown.entity not in seen_entities: return None # question references unknown entity + # Gate A2c — inject yield partition before solve when the question + # carries a typed per-unit consumption rate from the conditional clause. + if ( + question_choice.yield_chunk_value is not None + and question_choice.yield_chunk_unit is not None + and question_choice.yield_quotient_unit is not None + and question_choice.unknown.entity is not None + ): + try: + operations_list.append( + Operation( + actor=question_choice.unknown.entity, + kind="unit_partition", + operand=PartitionChunk( + value=question_choice.yield_chunk_value, + unit=question_choice.yield_chunk_unit, + result_unit=question_choice.yield_quotient_unit, + ), + ) + ) + except MathGraphError: + return None + try: return MathProblemGraph( entities=tuple(entities), diff --git a/generate/math_candidate_parser.py b/generate/math_candidate_parser.py index 53d73ea8..6a3c7a84 100644 --- a/generate/math_candidate_parser.py +++ b/generate/math_candidate_parser.py @@ -575,6 +575,8 @@ def extract_initial_candidates(sentence: str) -> list[CandidateInitial]: out.extend(_conj_subject_each_candidates(sentence)) out.extend(_conj_object_candidates(sentence)) out.extend(_embedded_quantifier_candidates(sentence)) + # Gate A2c — "N bags of M " acquisition composition. + out.extend(_bags_of_product_candidates(sentence)) # ADR-0189a — day-of-week count enumeration → summed initial. out.extend(_day_enumeration_candidates(sentence)) @@ -804,6 +806,13 @@ class CandidateUnknown: # ADR-0163.D.4 — Pattern B comparative marker ("how many more X"). # Default False keeps existing constructions byte-identical. comparative_marker: bool = False + # Gate A2c — production-yield partition injected at graph-build time. + # When all three are set, ``math_candidate_graph._build_graph`` appends + # a ``unit_partition`` step (inventory unit ÷ chunk size → product unit). + yield_chunk_value: float | None = None + yield_chunk_unit: str | None = None + yield_quotient_unit: str | None = None + consumed_value_tokens: tuple[str, ...] = () _Q_ENTITY_RE: Final[re.Pattern[str]] = re.compile( @@ -948,6 +957,11 @@ def extract_question_candidates( if out: return out + # Gate A2c — production yield: "how many will be able to make" + out.extend(_pattern_yield_question_candidates(sentence, problem_text)) + if out: + return out + # ADR-0163.D.4 — Pattern B: comparative quantifier ("how many more") out.extend(_pattern_b_comparative_candidates(sentence, problem_text)) if out: @@ -1916,6 +1930,113 @@ def _embedded_quantifier_candidates(sentence: str) -> list[CandidateInitial]: return [] +# Gate A2c — "N of M " acquisition composition. +# Distinct from G.4 embedded quantifier ("with M in each"); this shape +# uses "of" without a per-container "in each" tail. +_ACQUIRE_BAGS_OF_VERBS: Final[str] = ( + r"(?:bought|buys|buy|got|gets|get|received|receives|receive)" +) +_BAGS_OF_PRODUCT_RE: Final[re.Pattern[str]] = re.compile( + rf"^(?P{_ACTOR_OR_PRONOUN})\s+" + rf"(?P{_ACQUIRE_BAGS_OF_VERBS})\s+" + rf"(?P{_VALUE})\s+(?P\w+)\s+of\s+" + rf"(?P{_VALUE})\s+(?P\w+)" + r"\s*\.?$", + flags=re.IGNORECASE, +) +_CONJ_BAGS_OF_RE: Final[re.Pattern[str]] = re.compile( + rf"^(?P{_ACTOR_OR_PRONOUN})\s+" + rf"(?P{_ACQUIRE_BAGS_OF_VERBS})\s+" + rf"(?P{_VALUE})\s+(?P\w+)\s+of\s+(?P{_VALUE})\s+(?P\w+)\s+and\s+" + rf"(?P{_VALUE})\s+(?P\w+)\s+of\s+(?P{_VALUE})\s+(?P\w+)" + r"\s*\.?$", + flags=re.IGNORECASE, +) + + +def _build_conj_bags_of_sum( + m: re.Match[str], sentence: str +) -> list[CandidateInitial]: + """Single SUM candidate for conjoined bags-of-product.""" + n1_raw, m1_raw = m.group("n1"), m.group("m1") + n2_raw, m2_raw = m.group("n2"), m.group("m2") + for raw in (n1_raw, m1_raw, n2_raw, m2_raw): + if _is_indefinite_quantifier(raw): + return [] + u1 = _canonicalize_unit(m.group("u1")) + u2 = _canonicalize_unit(m.group("u2")) + if u1 != u2: + return [] + rv_n1 = _resolve_value(n1_raw) + rv_m1 = _resolve_value(m1_raw) + rv_n2 = _resolve_value(n2_raw) + rv_m2 = _resolve_value(m2_raw) + if any(rv is None for rv in (rv_n1, rv_m1, rv_n2, rv_m2)): + return [] + total = (rv_n1.value * rv_m1.value) + (rv_n2.value * rv_m2.value) # type: ignore[union-attr] + entity = _normalize_entity(m.group("entity")) + anchor = m.group("anchor").lower() + try: + return [ + CandidateInitial( + initial=InitialPossession( + entity=entity, + quantity=Quantity(value=total, unit=u1), + ), + source_span=sentence, + matched_anchor=anchor, + matched_value_token=m1_raw, + matched_unit_token=m.group("u1"), + matched_entity_token=m.group("entity"), + consumed_value_tokens=(n1_raw, m1_raw, n2_raw, m2_raw), + ) + ] + except Exception: + return [] + + +def _bags_of_product_candidates(sentence: str) -> list[CandidateInitial]: + """``N bags of M beads`` acquisition → derived total inventory.""" + s = sentence.strip().rstrip(".") + + m = _CONJ_BAGS_OF_RE.match(s) + if m is not None: + return _build_conj_bags_of_sum(m, sentence) + + m = _BAGS_OF_PRODUCT_RE.match(s) + if m is None: + return [] + n_raw, m_raw = m.group("n"), m.group("m") + if _is_indefinite_quantifier(n_raw) or _is_indefinite_quantifier(m_raw): + return [] + rv_n = _resolve_value(n_raw) + rv_per = _resolve_value(m_raw) + if rv_n is None or rv_per is None: + return [] + total = rv_n.value * rv_per.value + entity = _normalize_entity(m.group("entity")) + unit_raw = m.group("unit") + unit = _canonicalize_unit(unit_raw) + anchor = m.group("anchor").lower() + try: + return [ + CandidateInitial( + initial=InitialPossession( + entity=entity, + quantity=Quantity(value=total, unit=unit), + ), + source_span=sentence, + matched_anchor=anchor, + matched_value_token=m_raw, + matched_unit_token=unit_raw, + matched_entity_token=m.group("entity"), + consumed_value_tokens=(n_raw, m_raw), + ) + ] + except Exception: + return [] + + # --------------------------------------------------------------------------- # Per-shape admitted-only wrappers (used by the G4 runner). # Each filters its extractor's output through _initial_admissible from @@ -2761,6 +2882,41 @@ def _infer_partition_count_unit(problem_text: str | None) -> str | None: return _canonicalize_unit(m.group(1)) +# Gate A2c — production yield question + conditional rate clause. +_YIELD_RATE_RE: Final[re.Pattern[str]] = re.compile( + r"(?i)if\s+(?P\d+(?:\.\d+)?)\s+(?P\w+)\s+" + r"(?:are|is)\s+used\s+to\s+make\s+one\s+(?P\w+)" +) +_Q_YIELD_RE: Final[re.Pattern[str]] = re.compile( + r"^How\s+many\s+(?P\w+)\s+will\s+" + rf"(?P{_ENTITY})\s+" + r"be\s+able\s+to\s+make" + r"(?:\s+out\s+of\s+.*?)?\??\s*$", + flags=re.IGNORECASE, +) + + +def _infer_yield_partition( + problem_text: str | None, question_product: str +) -> tuple[float, str, str, tuple[str, ...]] | None: + """Infer yield partition + consumed rate tokens from a conditional clause.""" + if problem_text is None: + return None + m = _YIELD_RATE_RE.search(problem_text) + if m is None: + return None + product_unit = _canonicalize_unit(m.group("product")) + if product_unit != _canonicalize_unit(question_product): + return None + inventory_unit = _canonicalize_unit(m.group("unit")) + if inventory_unit == product_unit: + return None + chunk_size = float(m.group("n")) + if chunk_size <= 0: + return None + return chunk_size, inventory_unit, product_unit, (m.group("n"), "one") + + _Q_MASS_NOUN_RE: Final[re.Pattern[str]] = re.compile( r"^How\s+much\s+" rf"(?P{_MASS_NOUN_PATTERN})" @@ -2908,6 +3064,34 @@ def _resolve_question_entity( return _normalize_entity(raw_entity), raw_entity +def _pattern_yield_question_candidates( + sentence: str, problem_text: str | None +) -> list[CandidateUnknown]: + """Gate A2c — quotient question after inventory + per-unit yield rate.""" + s = sentence.strip() + m = _Q_YIELD_RE.match(s) + if m is None: + return [] + product_raw = m.group("product") + partition = _infer_yield_partition(problem_text, product_raw) + if partition is None: + return [] + chunk_size, inventory_unit, product_unit, rate_tokens = partition + entity = _normalize_entity(m.group("entity")) + return [ + CandidateUnknown( + unknown=Unknown(entity=entity, unit=product_unit), + source_span=sentence, + matched_unit_token=product_raw, + matched_entity_token=m.group("entity"), + yield_chunk_value=chunk_size, + yield_chunk_unit=inventory_unit, + yield_quotient_unit=product_unit, + consumed_value_tokens=rate_tokens, + ) + ] + + def _pattern_d_keep_on_hand_candidates( sentence: str, problem_text: str | None ) -> list[CandidateUnknown]: diff --git a/tests/test_gsm8k_post_gate_a1_frontier_microscope.py b/tests/test_gsm8k_post_gate_a1_frontier_microscope.py index ca4633c0..da6303ee 100644 --- a/tests/test_gsm8k_post_gate_a1_frontier_microscope.py +++ b/tests/test_gsm8k_post_gate_a1_frontier_microscope.py @@ -77,7 +77,7 @@ def test_live_microscope_refusal_partition_is_complete(): def test_live_microscope_partition_seed_case_is_tagged(): summary = build_microscope_report(_load_cases()) assert ( - "gsm8k-train-sample-v1-0002" + "gsm8k-train-sample-v1-0003" in summary["implementation_slice_candidates"]["partition_chunking"]["case_ids"] ) @@ -104,16 +104,15 @@ def test_markdown_render_surfaces_partition_candidate(): summary = build_microscope_report(_load_cases()) md = render_markdown(summary) assert "partition_chunking" in md - assert "| 0002 |" in md + assert "| 0003 |" in md assert "Gate A2a unit_partition" in md -def test_case_0002_post_gate_a2a_reclassified_off_partition_misroute(): - """After Gate A2a, 0002 refuses downstream (fraction give), not partition no-injection.""" +def test_gate_a2_lifts_are_not_in_refusal_table(): + """Cases solved by Gate A2b/A2c must not appear among live refusals.""" summary = build_microscope_report(_load_cases()) - row = next( - r for r in summary["refusal_table"] if r["case_id"].endswith("0002") - ) - assert "25-foot sections" not in (row.get("reason") or "") + refused_ids = {r["case_id"] for r in summary["refusal_table"]} + assert "gsm8k-train-sample-v1-0002" not in refused_ids + assert "gsm8k-train-sample-v1-0008" not in refused_ids + assert summary["counts"]["correct"] >= 8 assert summary["closed_injector_buckets"]["unit_partition_no_injection"] == 0 - assert row["top_refusal_bucket"] == "no_admissible_statement" diff --git a/tests/test_math_candidate_graph_container_of_product.py b/tests/test_math_candidate_graph_container_of_product.py new file mode 100644 index 00000000..d5e35e09 --- /dev/null +++ b/tests/test_math_candidate_graph_container_of_product.py @@ -0,0 +1,127 @@ +"""Gate A2c — container_of_product + yield_question composition.""" + +from __future__ import annotations + +from generate.math_candidate_graph import parse_and_solve +from generate.math_candidate_parser import ( + extract_initial_candidates, + extract_question_candidates, +) +def _run(text: str): + return parse_and_solve(text, sealed=False) + + +def test_bags_of_product_single_extracts(): + stmt = "Tom bought 3 bags of 20 marbles." + cands = extract_initial_candidates(stmt) + assert len(cands) == 1 + cand = cands[0] + assert cand.initial.quantity.value == 60.0 + assert cand.initial.quantity.unit == "marbles" + assert cand.matched_anchor == "bought" + + +def test_bags_of_product_conj_extracts_sum(): + stmt = "She bought 4 bags of 15 coins and 1 bag of 40 coins." + cands = extract_initial_candidates(stmt) + assert len(cands) == 1 + assert cands[0].initial.quantity.value == 100.0 + assert cands[0].initial.quantity.unit == "coins" + + +def test_yield_question_extracts_with_rate(): + full = ( + "If 10 marbles are used to make one necklace, " + "how many necklaces will Alice be able to make?" + ) + q = "How many necklaces will Alice be able to make?" + cands = extract_question_candidates(q, problem_text=full) + assert len(cands) == 1 + cand = cands[0] + assert cand.unknown.entity == "Alice" + assert cand.unknown.unit == "necklaces" + assert cand.yield_chunk_value == 10.0 + assert cand.yield_chunk_unit == "marbles" + assert cand.yield_quotient_unit == "necklaces" + + +def test_train_sample_0008_end_to_end(): + text = ( + "Marnie makes bead bracelets. " + "She bought 5 bags of 50 beads and 2 bags of 100 beads. " + "If 50 beads are used to make one bracelet, how many bracelets " + "will Marnie be able to make out of the beads she bought?" + ) + res = _run(text) + assert res.answer == 9.0 + assert res.refusal_reason is None + + +def test_sibling_tom_marbles(): + text = ( + "Tom collects marbles. " + "He bought 3 bags of 20 marbles. " + "If 10 marbles are used to make one display, " + "how many displays will Tom be able to make?" + ) + res = _run(text) + assert res.answer == 6.0 + assert res.refusal_reason is None + + +def test_sibling_alice_coins(): + text = ( + "Alice runs a craft shop. " + "She bought 4 bags of 15 coins and 1 bag of 40 coins. " + "If 20 coins are used to make one charm, " + "how many charms will Alice be able to make?" + ) + res = _run(text) + assert res.answer == 5.0 + assert res.refusal_reason is None + + +def test_confuser_mismatched_units_in_conj_refuses(): + stmt = "She bought 3 bags of 20 beads and 2 boxes of 10 marbles." + assert extract_initial_candidates(stmt) == [] + + +def test_confuser_bags_of_without_numeric_product_refuses(): + stmt = "She bought bags of beads." + assert extract_initial_candidates(stmt) == [] + + +def test_confuser_yield_without_rate_clause_refuses(): + q = "How many bracelets will Marnie be able to make?" + assert extract_question_candidates(q, problem_text=q) == [] + + +def test_confuser_rate_product_mismatch_refuses(): + q = ( + "If 50 beads are used to make one bracelet, " + "how many necklaces will Marnie be able to make?" + ) + assert extract_question_candidates(q, problem_text=q) == [] + + +def test_confuser_non_integer_quotient_refuses(): + text = ( + "Marnie makes bead bracelets. " + "She bought 3 bags of 10 beads. " + "If 7 beads are used to make one bracelet, " + "how many bracelets will Marnie be able to make?" + ) + res = _run(text) + assert res.answer is None + assert res.refusal_reason is not None + + +def test_regression_0042_embedded_quantifier_still_solves(): + text = ( + "Ella has 4 bags with 20 apples in each bag and " + "six bags with 25 apples in each bag. " + "If Ella sells 200 apples, how many apples does Ella has left?" + ) + res = _run(text) + assert res.answer == 30.0 + assert res.refusal_reason is None \ No newline at end of file