feat(derivation): capability strike batch 3 peer-pick lift (#813)
* feat(derivation): capability strike batch 3 peer-pick lift Gate A2d peer_partition_question closes train_sample 0025 by binding the conditional friend-count clause to a multiply-by-(1+N) graph step after WAVE-A multiplicative_aggregate statement ingestion. Ephemeral train_sample: 8/42/0 → 9/41/0, wrong=0 preserved. report.json untouched. * chore(analysis): normalize batch3 lookback EOF * chore(derivation): normalize peer partition test EOF * fix(derivation): require integer peer-pick friend counts
This commit is contained in:
parent
0215c30bbe
commit
fe4f1223e0
5 changed files with 337 additions and 2 deletions
|
|
@ -0,0 +1,72 @@
|
|||
# GSM8K Capability Strike Batch 3 — Lookback (2026-06-17)
|
||||
|
||||
## Scope
|
||||
|
||||
Gate A2d `peer_partition_question` lift on train_sample case **0025**.
|
||||
|
||||
## Baseline (honest, ephemeral, pre-change)
|
||||
|
||||
Measured via `build_report()` on `origin/main` at #811 merge (708f27a2):
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| correct | **8** |
|
||||
| refused | **42** |
|
||||
| wrong | **0** |
|
||||
|
||||
Committed `report.json` remains **6/44/0** (historical pin; not rebaselined).
|
||||
|
||||
## After (ephemeral live scoring)
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| correct | **9** |
|
||||
| refused | **41** |
|
||||
| wrong | **0** |
|
||||
|
||||
**Newly admitted:** `gsm8k-train-sample-v1-0025`
|
||||
**Preserved:** `0002`, `0008`, `0014`, `0018`, `0024`, `0029`, `0038`, `0042`
|
||||
|
||||
## Family implemented
|
||||
|
||||
**`peer_partition_question`** — closed conditional + total-across question:
|
||||
|
||||
- Conditional: `If N of <Entity>'s friends pick the same amount as her/him`
|
||||
- Question: `How many <unit> do <Entity> and her/his friends pick in all?`
|
||||
- Graph injects `multiply` by `(1 + N)` on the anchor entity's unit state after WAVE-A statement ingestion.
|
||||
|
||||
## Why not overfit
|
||||
|
||||
- No case-id branches or hardcoded answers.
|
||||
- Regex family generalizes across entities, units, and numeric/word friend counts.
|
||||
- Sibling tests vary names, counts, and units (Tom/apples, Alice/strawberries).
|
||||
- Confusers refuse: missing conditional, entity mismatch, indefinite quantifier, comparative 0007 unchanged.
|
||||
- Reuses existing `multiply` solver path and WAVE-A `multiplicative_aggregate` statement injection.
|
||||
|
||||
## Deferred candidates (Arena)
|
||||
|
||||
| Case | Blocker |
|
||||
|------|---------|
|
||||
| 0007 | Pattern B comparative detection-only + multi-entity inverse residual |
|
||||
| 0047 | Equal-N bag split ≠ `unit_partition` v1 |
|
||||
| 0004 | Nested fraction referent binding beyond Gate A2b |
|
||||
|
||||
## Non-goals
|
||||
|
||||
- No `report.json` rebaseline.
|
||||
- No sealed-lane movement.
|
||||
- No `determine()` / `answer=False`.
|
||||
- No `unit_partition` extension for equal-bag distribution.
|
||||
|
||||
## Validation
|
||||
|
||||
```bash
|
||||
git diff --check origin/main...HEAD
|
||||
pytest tests/test_math_candidate_graph_peer_partition_question.py -q
|
||||
pytest tests/test_math_candidate_graph_fraction_portion.py -q
|
||||
pytest tests/test_math_candidate_graph_unit_partition_injection.py -q
|
||||
pytest tests/test_recognizer_unit_partition_inject.py -q
|
||||
pytest tests/test_math_candidate_graph_container_of_product.py -q
|
||||
pytest tests/test_gsm8k_frontier_report.py -q
|
||||
pytest tests/test_gsm8k_post_gate_a1_frontier_microscope.py -q
|
||||
```
|
||||
|
|
@ -62,6 +62,7 @@ from generate.math_problem_graph import (
|
|||
Operation,
|
||||
MathProblemGraph,
|
||||
PartitionChunk,
|
||||
Quantity,
|
||||
)
|
||||
from generate.math_completeness import uncovered_quantities
|
||||
from generate.derivation.r1_reconstruction import reconstruct_r1_total
|
||||
|
|
@ -543,6 +544,33 @@ def _build_graph(
|
|||
except MathGraphError:
|
||||
return None
|
||||
|
||||
# Gate A2d — peer-pick multiply: anchor amount × (1 + friend_count).
|
||||
if (
|
||||
question_choice.peer_count is not None
|
||||
and question_choice.peer_reference_entity is not None
|
||||
and question_choice.unknown.entity is None
|
||||
):
|
||||
ref = question_choice.peer_reference_entity
|
||||
unit = question_choice.unknown.unit
|
||||
matching = [
|
||||
ic
|
||||
for ic in initials_list
|
||||
if ic.entity == ref and ic.quantity.unit == unit
|
||||
]
|
||||
if len(matching) != 1:
|
||||
return None
|
||||
factor = 1 + question_choice.peer_count
|
||||
try:
|
||||
operations_list.append(
|
||||
Operation(
|
||||
actor=ref,
|
||||
kind="multiply",
|
||||
operand=Quantity(value=factor, unit=unit),
|
||||
)
|
||||
)
|
||||
except MathGraphError:
|
||||
return None
|
||||
|
||||
try:
|
||||
return MathProblemGraph(
|
||||
entities=tuple(entities),
|
||||
|
|
|
|||
|
|
@ -812,6 +812,9 @@ class CandidateUnknown:
|
|||
yield_chunk_value: float | None = None
|
||||
yield_chunk_unit: str | None = None
|
||||
yield_quotient_unit: str | None = None
|
||||
# Gate A2d — peer-pick partition: friends pick the same amount as anchor.
|
||||
peer_count: int | None = None
|
||||
peer_reference_entity: str | None = None
|
||||
consumed_value_tokens: tuple[str, ...] = ()
|
||||
|
||||
|
||||
|
|
@ -962,6 +965,11 @@ def extract_question_candidates(
|
|||
if out:
|
||||
return out
|
||||
|
||||
# Gate A2d — peer-pick total: friends pick the same amount as anchor entity.
|
||||
out.extend(_pattern_peer_pick_question_candidates(sentence, problem_text))
|
||||
if out:
|
||||
return out
|
||||
|
||||
# ADR-0163.D.4 — Pattern B: comparative quantifier ("how many more")
|
||||
out.extend(_pattern_b_comparative_candidates(sentence, problem_text))
|
||||
if out:
|
||||
|
|
@ -2895,6 +2903,19 @@ _Q_YIELD_RE: Final[re.Pattern[str]] = re.compile(
|
|||
flags=re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Gate A2d — peer-pick conditional + total-across question.
|
||||
_PEER_PICK_CONDITIONAL_RE: Final[re.Pattern[str]] = re.compile(
|
||||
rf"(?i)if\s+(?P<count>{_VALUE})\s+of\s+"
|
||||
rf"(?P<entity>{_ENTITY})\s*'s\s+friends\s+pick\s+the\s+same\s+amount\s+as\s+"
|
||||
r"(?:her|him)\b"
|
||||
)
|
||||
_Q_PEER_PICK_RE: Final[re.Pattern[str]] = re.compile(
|
||||
r"^How\s+many\s+(?P<unit>\w+)\s+do\s+"
|
||||
rf"(?P<entity>{_ENTITY})\s+and\s+"
|
||||
r"(?:her|his)\s+friends\s+pick\s+in\s+all\s*\??\s*$",
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _infer_yield_partition(
|
||||
problem_text: str | None, question_product: str
|
||||
|
|
@ -3064,6 +3085,65 @@ def _resolve_question_entity(
|
|||
return _normalize_entity(raw_entity), raw_entity
|
||||
|
||||
|
||||
def _infer_peer_pick_partition(
|
||||
problem_text: str | None, question_entity: str
|
||||
) -> tuple[int, str, str] | None:
|
||||
"""Infer peer friend-count from the conditional clause in ``problem_text``.
|
||||
|
||||
Gate A2d's peer count is a count of people, not a general numeric value.
|
||||
The regex shares ``_VALUE`` for local parser consistency, but admission is
|
||||
narrower: only positive integer, unitless counts are allowed. Money,
|
||||
slash-fraction, decimal, and other non-integer numeric surfaces refuse.
|
||||
"""
|
||||
if problem_text is None:
|
||||
return None
|
||||
m = _PEER_PICK_CONDITIONAL_RE.search(problem_text)
|
||||
if m is None:
|
||||
return None
|
||||
entity = _normalize_entity(m.group("entity"))
|
||||
if entity != _normalize_entity(question_entity):
|
||||
return None
|
||||
count_token = m.group("count")
|
||||
if count_token.startswith(("$", "¢", "€", "£", "¥", "₱")) or "/" in count_token:
|
||||
return None
|
||||
resolved = _resolve_value(count_token)
|
||||
if resolved is None:
|
||||
return None
|
||||
value = resolved.value
|
||||
if value <= 0 or value != int(value):
|
||||
return None
|
||||
peer_count = int(value)
|
||||
return peer_count, count_token, entity
|
||||
|
||||
|
||||
def _pattern_peer_pick_question_candidates(
|
||||
sentence: str, problem_text: str | None
|
||||
) -> list[CandidateUnknown]:
|
||||
"""Gate A2d — total-across after peer friends pick the same amount."""
|
||||
s = sentence.strip()
|
||||
m = _Q_PEER_PICK_RE.match(s)
|
||||
if m is None:
|
||||
return []
|
||||
unit_raw = m.group("unit")
|
||||
unit = _canonicalize_unit(unit_raw)
|
||||
entity = _normalize_entity(m.group("entity"))
|
||||
partition = _infer_peer_pick_partition(problem_text, entity)
|
||||
if partition is None:
|
||||
return []
|
||||
peer_count, count_token, reference_entity = partition
|
||||
return [
|
||||
CandidateUnknown(
|
||||
unknown=Unknown(entity=None, unit=unit),
|
||||
source_span=sentence,
|
||||
matched_unit_token=unit_raw,
|
||||
matched_entity_token=m.group("entity"),
|
||||
peer_count=peer_count,
|
||||
peer_reference_entity=reference_entity,
|
||||
consumed_value_tokens=(count_token,),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _pattern_yield_question_candidates(
|
||||
sentence: str, problem_text: str | None
|
||||
) -> list[CandidateUnknown]:
|
||||
|
|
|
|||
|
|
@ -109,10 +109,11 @@ def test_markdown_render_surfaces_partition_candidate():
|
|||
|
||||
|
||||
def test_gate_a2_lifts_are_not_in_refusal_table():
|
||||
"""Cases solved by Gate A2b/A2c must not appear among live refusals."""
|
||||
"""Cases solved by Gate A2b/A2c/A2d must not appear among live refusals."""
|
||||
summary = build_microscope_report(_load_cases())
|
||||
refused_ids = {r["case_id"] for r in summary["refusal_table"]}
|
||||
assert "gsm8k-train-sample-v1-0002" not in refused_ids
|
||||
assert "gsm8k-train-sample-v1-0008" not in refused_ids
|
||||
assert summary["counts"]["correct"] >= 8
|
||||
assert "gsm8k-train-sample-v1-0025" not in refused_ids
|
||||
assert summary["counts"]["correct"] >= 9
|
||||
assert summary["closed_injector_buckets"]["unit_partition_no_injection"] == 0
|
||||
|
|
|
|||
154
tests/test_math_candidate_graph_peer_partition_question.py
Normal file
154
tests/test_math_candidate_graph_peer_partition_question.py
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
"""Gate A2d — peer_partition_question composition."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from generate.math_candidate_graph import parse_and_solve
|
||||
from generate.math_candidate_parser import (
|
||||
extract_initial_candidates,
|
||||
extract_question_candidates,
|
||||
)
|
||||
|
||||
|
||||
def _run(text: str):
|
||||
return parse_and_solve(text, sealed=False)
|
||||
|
||||
|
||||
def test_peer_pick_question_extracts_with_conditional():
|
||||
full = (
|
||||
"Lilibeth fills 6 baskets where each basket holds 50 strawberries. "
|
||||
"If three of Lilibeth's friends pick the same amount as her, "
|
||||
"how many strawberries do Lilibeth and her friends pick in all?"
|
||||
)
|
||||
q = "How many strawberries do Lilibeth and her friends pick in all?"
|
||||
cands = extract_question_candidates(q, problem_text=full)
|
||||
assert len(cands) == 1
|
||||
cand = cands[0]
|
||||
assert cand.unknown.entity is None
|
||||
assert cand.unknown.unit == "strawberries"
|
||||
assert cand.peer_count == 3
|
||||
assert cand.peer_reference_entity == "Lilibeth"
|
||||
assert cand.consumed_value_tokens == ("three",)
|
||||
|
||||
|
||||
def test_train_sample_0025_end_to_end():
|
||||
text = (
|
||||
"Lilibeth and her friends go strawberry picking. "
|
||||
"Lilibeth fills 6 baskets where each basket holds 50 strawberries. "
|
||||
"If three of Lilibeth's friends pick the same amount as her, "
|
||||
"how many strawberries do Lilibeth and her friends pick in all?"
|
||||
)
|
||||
res = _run(text)
|
||||
assert res.answer == 1200.0
|
||||
assert res.refusal_reason is None
|
||||
|
||||
|
||||
def test_sibling_tom_apples():
|
||||
text = (
|
||||
"Tom picks apples. "
|
||||
"Tom fills 4 baskets where each basket holds 25 apples. "
|
||||
"If two of Tom's friends pick the same amount as him, "
|
||||
"how many apples do Tom and his friends pick in all?"
|
||||
)
|
||||
res = _run(text)
|
||||
assert res.answer == 300.0
|
||||
assert res.refusal_reason is None
|
||||
|
||||
|
||||
def test_sibling_alice_strawberries():
|
||||
text = (
|
||||
"Alice and her crew go picking. "
|
||||
"Alice fills 3 baskets where each basket holds 40 strawberries. "
|
||||
"If 1 of Alice's friends pick the same amount as her, "
|
||||
"how many strawberries do Alice and her friends pick in all?"
|
||||
)
|
||||
res = _run(text)
|
||||
assert res.answer == 240.0
|
||||
assert res.refusal_reason is None
|
||||
|
||||
|
||||
def test_confuser_missing_conditional_refuses():
|
||||
text = (
|
||||
"Lilibeth fills 6 baskets where each basket holds 50 strawberries. "
|
||||
"How many strawberries do Lilibeth and her friends pick in all?"
|
||||
)
|
||||
res = _run(text)
|
||||
assert res.answer is None
|
||||
|
||||
|
||||
def test_confuser_no_prior_initial_refuses():
|
||||
text = (
|
||||
"If three of Lilibeth's friends pick the same amount as her, "
|
||||
"how many strawberries do Lilibeth and her friends pick in all?"
|
||||
)
|
||||
res = _run(text)
|
||||
assert res.answer is None
|
||||
|
||||
|
||||
def test_confuser_entity_mismatch_refuses():
|
||||
text = (
|
||||
"Lilibeth fills 6 baskets where each basket holds 50 strawberries. "
|
||||
"If three of Tom's friends pick the same amount as him, "
|
||||
"how many strawberries do Lilibeth and her friends pick in all?"
|
||||
)
|
||||
res = _run(text)
|
||||
assert res.answer is None
|
||||
|
||||
|
||||
def test_confuser_friends_without_count_refuses():
|
||||
text = (
|
||||
"Lilibeth fills 6 baskets where each basket holds 50 strawberries. "
|
||||
"If some of Lilibeth's friends pick the same amount as her, "
|
||||
"how many strawberries do Lilibeth and her friends pick in all?"
|
||||
)
|
||||
res = _run(text)
|
||||
assert res.answer is None
|
||||
|
||||
|
||||
def test_confuser_fractional_friend_count_refuses():
|
||||
text = (
|
||||
"Lilibeth fills 6 baskets where each basket holds 50 strawberries. "
|
||||
"If 3/2 of Lilibeth's friends pick the same amount as her, "
|
||||
"how many strawberries do Lilibeth and her friends pick in all?"
|
||||
)
|
||||
res = _run(text)
|
||||
assert res.answer is None
|
||||
|
||||
|
||||
def test_confuser_money_friend_count_refuses():
|
||||
text = (
|
||||
"Lilibeth fills 6 baskets where each basket holds 50 strawberries. "
|
||||
"If $2 of Lilibeth's friends pick the same amount as her, "
|
||||
"how many strawberries do Lilibeth and her friends pick in all?"
|
||||
)
|
||||
res = _run(text)
|
||||
assert res.answer is None
|
||||
|
||||
|
||||
def test_comparative_question_still_refuses():
|
||||
text = (
|
||||
"Francine has five full boxes of crayons and 5 loose crayons, "
|
||||
"and her friend has 27 loose crayons. "
|
||||
"They need to put all of their loose crayons in a box. "
|
||||
"How many more boxes do they need if Francine has a total of 85 crayons?"
|
||||
)
|
||||
res = _run(text)
|
||||
assert res.answer is None
|
||||
|
||||
|
||||
def test_yield_regression_0008():
|
||||
text = (
|
||||
"Marnie makes bead bracelets. "
|
||||
"She bought 5 bags of 50 beads and 2 bags of 100 beads. "
|
||||
"If 50 beads are used to make one bracelet, how many bracelets "
|
||||
"will Marnie be able to make out of the beads she bought?"
|
||||
)
|
||||
res = _run(text)
|
||||
assert res.answer == 9.0
|
||||
assert res.refusal_reason is None
|
||||
|
||||
|
||||
def test_bags_of_product_statement_not_peer_pick():
|
||||
stmt = "She bought 4 bags of 15 coins and 1 bag of 40 coins."
|
||||
cands = extract_initial_candidates(stmt)
|
||||
assert len(cands) == 1
|
||||
assert cands[0].initial.quantity.value == 100.0
|
||||
Loading…
Reference in a new issue