diff --git a/docs/analysis/gsm8k-capability-paradigm-sprint7-lookback-2026-06-17.md b/docs/analysis/gsm8k-capability-paradigm-sprint7-lookback-2026-06-17.md new file mode 100644 index 00000000..7911676d --- /dev/null +++ b/docs/analysis/gsm8k-capability-paradigm-sprint7-lookback-2026-06-17.md @@ -0,0 +1,161 @@ +# GSM8K Capability Paradigm Sprint 7 — Lookback (2026-06-17) + +## 1. Starting baseline + +| Metric | Value | +|--------|------:| +| correct | **14** | +| refused | **36** | +| wrong | **0** | + +After #817 (Sprint 6: duration_segment_total + survey_rate_earnings). Preserved solved: 0002, 0003, 0008, 0014, 0015, 0018, 0021, 0024, 0025, 0029, 0037, 0038, 0042, 0045. + +## 2. Experience Flywheel output summary + +`scripts/gsm8k_experience_flywheel.py --limit 50 --out /tmp/gsm8k-experience-sprint7.json` + +| Field | Value | +|-------|------:| +| retained_record_count | 50 | +| compacted_record_count | 50 | +| lift_refused_to_correct | **0** | +| promotion_candidates | 4 families, **all** `blocked_by_wrong_risk` | + +Top missing primitives: `relation_hypothesis` (DCS cluster), `diagnostic_hold`, `multiplicative_aggregate`, `temporal_tariff`. + +**Key signal:** No scout refused→correct delta (Sprint 6 already promoted easy lifts). Flywheel ranked `joint_sealed_no_resolution` cluster (0004, 0005, 0007, 0009, 0010, **0030**, 0035, 0048, 0050) plus composition-validation pins for decomposition. + +## 3. Scout/frontier output summary + +| Regime | correct | refused | wrong | +|--------|--------:|--------:|------:| +| Serving | 14 | 36 | 0 | +| Sealed (resolve_pooled) | 3 | 39 | 8 | + +- `lift_refused_to_correct`: **0** +- `lift_recommendations`: **[]** +- Dominant bottleneck: joint refusal (30/50) +- Sealed-wrong unchanged: 0011, 0018, 0019, 0025, 0026, 0028, 0032, 0047 + +## 4. Candidate families ranked + +| Rank | Family | Cases | Lift signal | Wrong-risk | Sprint 7 verdict | +|------|--------|-------|-------------|------------|------------------| +| 1 | `joint_sealed_no_resolution` + cv-0006 | **0030** | typed R5 round-trip chain | low | **Selected (A2i)** | +| 2 | `joint_sealed_no_resolution` + cv-0021 | **0035** | typed R4 giveaway-residual | low | **Selected (A2j)** | +| 3 | `temporal_tariff:temporal_aggregation` | 0001, 0017 | joint only | medium | Deferred — tariff/overtime | +| 4 | `multiplicative_aggregate` | 0006, 0013 | joint only | low surface | Deferred — chained age/video shapes | +| 5 | `relation_hypothesis:DCS` | 14 joint + 2 wrong | none | **blocked** (0032, 0047) | Deferred | +| 6 | `diagnostic_hold:currency_amount` | 0019, 0028 | sealed-wrong | **blocked** | Deferred | + +## 5. Blocked families and why + +| Family | Blocked because | +|--------|-----------------| +| `relation_hypothesis:DCS` | 0032, 0047 sealed-wrong share recognizer surface | +| `conservative_boundary` | 0018, 0025 sealed-wrong on served-correct cases | +| `sealed_elimination` | 0011, 0026 hard elimination | +| `diagnostic_hold:currency_amount` | 0019, 0028 sealed-wrong | + +## 6. Selected paradigm + +**Dual narrow organs** (composition-validation decomposition, not broad family promotion): + +1. **Gate A2i `round_trip_trip_duration`** — one-way drive ×2 (each way) + comparative activity duration vs total driving; trip-time question (`cv-0006` / 0030). +2. **Gate A2j `giveaway_target_residual`** — possession − stated remainder − Σ giveaways (including `N more than` prior recipient); distinct from goal-residual (`cv-0021` / 0035). + +Rejected wholesale DCS/MA/temporal_tariff promotion: flywheel blocked or joint-only without typed end-to-end chains. + +## 7. Typed chains for target cases + +**0030 (gold 14):** `2 + 2 = 4` driving → `4 × 2.5 = 10` beach → `4 + 10 = 14` trip total. + +**0035 (gold 4):** `20 − 4 = 16` to give → `−5` Jane → `−5 − 2` James (2 more than Jane) → `4` more needed. + +## 8. Implemented organs + +| Module | Promotion entry | +|--------|-----------------| +| `generate/derivation/round_trip_trip_duration.py` | `resolve_promotable_round_trip_trip_duration` | +| `generate/derivation/giveaway_target_residual.py` | `resolve_promotable_giveaway_target_residual` | + +Wired in `generate/math_candidate_graph.py` after Gate A2h, before rate short-circuits. + +## 9. Before / after score + +| | Before | After | +|---|------:|------:| +| correct | 14 | **16** | +| refused | 36 | **34** | +| wrong | 0 | **0** | + +## 10. Newly solved IDs + +- `gsm8k-train-sample-v1-0030` +- `gsm8k-train-sample-v1-0035` + +## 11. Preserved solved IDs + +0002, 0003, 0008, 0014, 0015, 0018, 0021, 0024, 0025, 0029, 0037, 0038, 0042, 0045 + +## 12. Wrong=0 proof + +Ephemeral train_sample: `16 / 34 / 0`. Holdout_dev scan: **0 admissions** for either new gate. + +## 13. Confuser matrix + +| Confuser | Result | +|----------|--------| +| Round trip without `each way` | refuse | +| Fraction/percent duration surface | refuse | +| Goal-language giveaway (cv-0005) | refuse giveaway organ; goal_residual still serves | +| Comparative giveaway question (`more than Jane`) | refuse | +| Duration segment total (0015) | refuse round-trip organ | +| 0032 percent-time shape | refuse | +| 0047 partition weight shape | refuse | + +## 14. Sibling generality proof + +- Round-trip sibling: 3h each way, 2× driving at lake → 18h trip +- Giveaway sibling: 30 oranges, 8 + (8+3) given, leave 6 → 5 more + +## 15. Sealed-wrong negative evidence used + +Flywheel `blocked_by_wrong_risk` on DCS/currency families informed refusal to promote broad injectors. Tests refuse 0032/0047 shapes on new gates. + +## 16–21. Confirmations + +- `report.json` untouched (historical pin) +- Sealed-lane artifacts untouched +- No case-id logic +- No hardcoded final answers +- No direct-answer extraction +- **Non-goals:** broad `product_bridge`, broad DCS/MA injectors, temporal_tariff organ, `resolve_pooled` re-wire, report rebaseline, sealed movement + +## Sprint 8 handoff recommendation + +**Recommended track: Track A — Capability Sprint 8** + +| Field | Value | +|-------|-------| +| Evidence | Flywheel still shows dense `joint_sealed_no_resolution` cluster; composition-validation pins **cv-0007** (0005 fraction temp), **cv-0008** (0046 percent partition), and **cv-0021**-sibling **cv-0005** already served — next clean typed chains are affine/fraction (0005, 0046) and temporal_tariff (0001, 0017) after confuser design | +| Target families | `affine_equation` / R6 fraction-decrease (0005), percent-partition (0046); secondary: `temporal_tariff` (0001, 0017) | +| Blocked families | DCS (0032/0047), currency_amount (0019/0028), sealed_elimination (0011/0026) | +| Expected movement | +1–2 correct if R6 fraction or percent-partition organ passes wrong=0 adversary | +| Required modules | `generate/derivation/.py`, `tests/test_math_candidate_graph__sprint8_lift.py` | +| Non-goals | Experience Flywheel PR-2 until lift_refused_to_correct > 0 or promotion_candidates unblock; no broad DCS until confusers pass | + +Track B (Flywheel PR-2) deferred: no `promotion_candidates` with safe lift signal yet. Track C (Diagnostic Hardening) not warranted — two clean organs lifted this sprint. + +## Validation + +```bash +git diff --check origin/main...HEAD +pytest tests/test_math_candidate_graph_sprint7_experience_guided_lift.py -q +pytest tests/test_gsm8k_experience_flywheel.py -q +pytest tests/test_gsm8k_sealed_attempt_scout.py -q +pytest tests/test_composition_validation_corpus.py -q +core test --suite smoke -q +``` + +Composition-validation snapshot: **12 solve / 10 refuse / 0 wrong** (`cv-0006`, `cv-0021` flipped). \ No newline at end of file diff --git a/generate/derivation/giveaway_target_residual.py b/generate/derivation/giveaway_target_residual.py new file mode 100644 index 00000000..4bb47bcf --- /dev/null +++ b/generate/derivation/giveaway_target_residual.py @@ -0,0 +1,194 @@ +"""Gate A2j — possession giveaway target-residual (R4 sibling, not goal-residual). + +Experience Flywheel Sprint 7 selected this family from decomposition of +composition-validation **cv-0021** (train_sample **0035**): anchor possession, +progress giveaways to named recipients (including ``N more than`` a prior +giveaway), question asks how many *more* to give away to reach a stated +remainder. + + total_to_give = possession − target_remainder + more_needed = total_to_give − Σ giveaways (with comparative increment) + +Distinct from Gate A2e ``goal_residual``: + +- anchor is possession language (``has``), not goal intent (``wants``/``needs``); +- target remainder is stated in the question (``left with only N``), not a goal + quantity in the anchor clause; +- comparative ``more than `` licenses an extra subtract step. + +Promotion requires: + +- question binds ``more``, ``give``, and ``left``; +- exactly one possession quantity in the anchor clause; +- at least one giveaway progress clause with licensed giveaway cues; +- hazard refusal (goal language, comparative questions, competing possessions). + +Deterministic; sealed module (no ``chat/`` import). +""" + +from __future__ import annotations + +import re +from typing import Final + +from generate.derivation.clauses import segment_clauses +from generate.derivation.extract import extract_quantities +from generate.derivation.model import GroundedDerivation, Quantity, Step +from generate.derivation.state.bind import leading_subject_token +from generate.derivation.target import _question_clause +from generate.derivation.verify import Resolution, select_self_verified +from generate.math_roundtrip import _tokens + +_AND_SPLIT_RE: Final[re.Pattern[str]] = re.compile(r",\s*and\s+", re.IGNORECASE) + +_GOAL_INTENT: Final[frozenset[str]] = frozenset( + {"want", "wants", "wanted", "need", "needs", "hoping", "hopes", "plans", "aims", "goal"} +) +_COMPARATIVE_TARGET_CUES: Final[frozenset[str]] = frozenset({"than"}) +_GIVEAWAY_CUES: Final[frozenset[str]] = frozenset({"got", "gave", "gives", "give"}) +_POSSESSION_CUES: Final[frozenset[str]] = frozenset({"has", "have", "had"}) + + +def _asks_giveaway_residual(question_clause: str) -> bool: + tokens = _tokens(question_clause) + if _COMPARATIVE_TARGET_CUES & tokens: + return False + return "more" in tokens and "give" in tokens and "left" in tokens + + +def _target_remainder(question_clause: str) -> Quantity | None: + """The ``only N`` remainder stated in the question.""" + quantities = [q for q in extract_quantities(question_clause) if not q.unit] + if len(quantities) != 1: + return None + return quantities[0] + + +def _giveaway_subclauses(clause: str) -> tuple[str, ...]: + return tuple(part.strip() for part in _AND_SPLIT_RE.split(clause) if part.strip()) + + +def _competing_possession_hazard(subclause: str) -> bool: + tokens = _tokens(subclause) + if not (_POSSESSION_CUES & tokens): + return False + return not (_GIVEAWAY_CUES & tokens) + + +def build_giveaway_target_residual(problem_text: str) -> GroundedDerivation | None: + """Construct ``possession − remainder − Σgiveaways``, or ``None``.""" + question_clause = _question_clause(problem_text) + if not _asks_giveaway_residual(question_clause): + return None + + remainder = _target_remainder(question_clause) + if remainder is None: + return None + + clauses = segment_clauses(problem_text) + body_clauses = [c for c in clauses if c != question_clause] + quantity_clauses = [c for c in body_clauses if extract_quantities(c)] + if len(quantity_clauses) < 2: + return None + + anchor_clause = quantity_clauses[0] + anchor_tokens = _tokens(anchor_clause) + if not (_POSSESSION_CUES & anchor_tokens): + return None + if _GOAL_INTENT & anchor_tokens: + return None + + anchor_quantities = extract_quantities(anchor_clause) + if len(anchor_quantities) != 1: + return None + possession = anchor_quantities[0] + + steps: list[Step] = [ + Step( + op="subtract", + operand=Quantity( + value=remainder.value, + unit=possession.unit, + source_token=remainder.source_token, + ), + cue="left", + ) + ] + + first_giveaway: Quantity | None = None + giveaway_steps = 0 + + for clause in quantity_clauses[1:]: + for subclause in _giveaway_subclauses(clause): + if _competing_possession_hazard(subclause): + return None + clause_tokens = _tokens(subclause) + if not (_GIVEAWAY_CUES & clause_tokens): + continue + cue = next(c for c in ("got", "gave", "gives", "give") if c in clause_tokens) + + progress = [ + q + for q in extract_quantities(subclause) + if (not q.unit) or q.unit == possession.unit + ] + if not progress: + return None + + if "more" in clause_tokens and "than" in clause_tokens: + if len(progress) != 1: + return None + if first_giveaway is None: + return None + increment = progress[0] + base = first_giveaway + steps.append( + Step( + op="subtract", + operand=Quantity( + value=base.value, + unit=possession.unit, + source_token=base.source_token, + ), + cue=leading_subject_token(subclause) or cue, + ) + ) + steps.append( + Step( + op="subtract", + operand=Quantity( + value=increment.value, + unit=possession.unit, + source_token=increment.source_token, + ), + cue="more", + ) + ) + giveaway_steps += 2 + continue + + for q in progress: + operand = Quantity( + value=q.value, unit=possession.unit, source_token=q.source_token + ) + steps.append(Step(op="subtract", operand=operand, cue=cue)) + giveaway_steps += 1 + if first_giveaway is None: + first_giveaway = operand + + if giveaway_steps < 2: + return None + return GroundedDerivation(start=possession, steps=tuple(steps)) + + +def compose_giveaway_target_residual(problem_text: str) -> Resolution | None: + """Gate the typed giveaway-residual chain through self-verification.""" + derivation = build_giveaway_target_residual(problem_text) + if derivation is None: + return None + return select_self_verified([derivation], problem_text, target_units=()) + + +def resolve_promotable_giveaway_target_residual(problem_text: str) -> Resolution | None: + """Serving promotion bridge (Gate A2j).""" + return compose_giveaway_target_residual(problem_text) \ No newline at end of file diff --git a/generate/derivation/round_trip_trip_duration.py b/generate/derivation/round_trip_trip_duration.py new file mode 100644 index 00000000..91d6c1b8 --- /dev/null +++ b/generate/derivation/round_trip_trip_duration.py @@ -0,0 +1,166 @@ +"""Gate A2i — round-trip drive + comparative activity duration total. + +Experience Flywheel Sprint 7 selected this family from the +``joint_sealed_no_resolution`` cluster after decomposition of composition- +validation **cv-0006** (train_sample **0030**): one-way drive duration, round +trip via ``each way``, activity duration as a comparative scalar of total +driving time, question binds trip time. + + driving_total = one_way × 2 + activity = driving_total × comparative_scalar + trip_total = driving_total + activity + = one_way × 2 × (1 + comparative_scalar) + +Fold (grounded operands only): + + start one_way → add one_way (second leg) → × comparative_scalar + → +one_way → +one_way + +This is distinct from Gate A2g ``duration_segment_total`` (fixed leg + +comparative middle + trailing leg). Promotion requires: + +- ``each`` and ``way`` in the body; +- exactly one duration quantity and one comparative scalar; +- comparative references driving time (``driving`` token present); +- question binds ``time`` and ``trip``; +- hazard refusal (fractions, percent, money, goal/residual giveaway shapes). + +Deterministic; sealed module (no ``chat/`` import). +""" + +from __future__ import annotations + +import re +from typing import Final + +from generate.derivation.clauses import segment_clauses +from generate.derivation.comparatives import comparative_step, extract_comparative_scalars +from generate.derivation.extract import extract_quantities +from generate.derivation.model import GroundedDerivation, Quantity, Step +from generate.derivation.target import _question_clause +from generate.derivation.verify import Resolution, select_self_verified +from generate.math_roundtrip import _tokens + +_FRACTION_RE: Final[re.Pattern[str]] = re.compile(r"\d+/\d+") +_DURATION_UNITS: Final[frozenset[str]] = frozenset( + {"hour", "hours", "hr", "hrs", "minute", "minutes", "min", "mins"} +) +_TEXT_BLOCKERS: Final[frozenset[str]] = frozenset( + { + "insurance", + "money", + "overtime", + "percent", + "percentage", + "profit", + "salary", + "wage", + } +) +_QUESTION_BLOCKERS: Final[frozenset[str]] = frozenset( + {"longer", "less", "per", "profit", "remaining"} +) + + +def _asks_trip_time(question_clause: str) -> bool: + tokens = _tokens(question_clause) + return "time" in tokens and "trip" in tokens + + +def _has_hazard_surface(problem_text: str, question_clause: str) -> bool: + if _FRACTION_RE.search(problem_text): + return True + text_tokens = _tokens(problem_text) + question_tokens = _tokens(question_clause) + if "%" in problem_text or "percent" in text_tokens or "percentage" in text_tokens: + return True + if text_tokens & _TEXT_BLOCKERS: + return True + if "give" in text_tokens and "away" in text_tokens: + return True + if question_tokens & _QUESTION_BLOCKERS: + return True + return False + + +def _duration_quantities(clause: str) -> tuple[Quantity, ...]: + return tuple( + q + for q in extract_quantities(clause) + if q.unit in _DURATION_UNITS + or q.unit.rstrip("s") in {"hour", "hr", "minute", "min"} + ) + + +def _duration_family(unit: str) -> str | None: + stem = unit.rstrip("s") + if stem in {"hour", "hr"}: + return "hour" + if stem in {"minute", "min"}: + return "minute" + return None + + +def build_round_trip_trip_duration(problem_text: str) -> GroundedDerivation | None: + """Construct the ungated round-trip trip-duration chain, or ``None``.""" + question_clause = _question_clause(problem_text) + if not _asks_trip_time(question_clause): + return None + if _has_hazard_surface(problem_text, question_clause): + return None + + text_tokens = _tokens(problem_text) + if "each" not in text_tokens or "way" not in text_tokens: + return None + if "driving" not in text_tokens: + return None + + comparatives = extract_comparative_scalars(problem_text) + if len(comparatives) != 1: + return None + comparative = comparatives[0] + if comparative.scalar <= 0: + return None + + body_clauses = [ + clause + for clause in segment_clauses(problem_text) + if clause != question_clause + ] + durations = [ + q + for clause in body_clauses + for q in _duration_quantities(clause) + ] + if len(durations) != 1: + return None + + one_way = durations[0] + if _duration_family(one_way.unit) is None: + return None + + leg = Quantity( + value=one_way.value, + unit=one_way.unit, + source_token=one_way.source_token, + ) + steps: list[Step] = [ + Step(op="add", operand=leg, cue="way"), + comparative_step(comparative), + Step(op="add", operand=leg, cue="drive"), + Step(op="add", operand=leg, cue="each"), + ] + return GroundedDerivation(start=leg, steps=tuple(steps)) + + +def compose_round_trip_trip_duration(problem_text: str) -> Resolution | None: + """Gate the typed round-trip chain through self-verification.""" + derivation = build_round_trip_trip_duration(problem_text) + if derivation is None: + return None + return select_self_verified([derivation], problem_text, target_units=()) + + +def resolve_promotable_round_trip_trip_duration(problem_text: str) -> Resolution | None: + """Serving promotion bridge (Gate A2i).""" + return compose_round_trip_trip_duration(problem_text) \ No newline at end of file diff --git a/generate/math_candidate_graph.py b/generate/math_candidate_graph.py index 5fd8827a..4d31bdfd 100644 --- a/generate/math_candidate_graph.py +++ b/generate/math_candidate_graph.py @@ -759,6 +759,36 @@ def parse_and_solve(text: str, *, sealed: bool = False) -> CandidateGraphResult: branches_admissible=1, ) + # Gate A2i — round-trip trip duration (Sprint 7 experience-guided lift). + from generate.derivation.round_trip_trip_duration import ( + resolve_promotable_round_trip_trip_duration, + ) + + round_trip_resolution = resolve_promotable_round_trip_trip_duration(text) + if round_trip_resolution is not None: + return CandidateGraphResult( + answer=round_trip_resolution.answer, + selected_graph=None, + refusal_reason=None, + branches_enumerated=1, + branches_admissible=1, + ) + + # Gate A2j — giveaway target residual (Sprint 7 experience-guided lift). + from generate.derivation.giveaway_target_residual import ( + resolve_promotable_giveaway_target_residual, + ) + + giveaway_resolution = resolve_promotable_giveaway_target_residual(text) + if giveaway_resolution is not None: + return CandidateGraphResult( + answer=giveaway_resolution.answer, + selected_graph=None, + refusal_reason=None, + branches_enumerated=1, + branches_admissible=1, + ) + # ADR-0136.S.1 — Rate/event short-circuit paths (before Cartesian product). # Capacity path: single statement with one CandidateCapacity + matching question. if len(statement_sentences) == 1: diff --git a/tests/test_composition_validation_corpus.py b/tests/test_composition_validation_corpus.py index 92bf6f83..96fc2a50 100644 --- a/tests/test_composition_validation_corpus.py +++ b/tests/test_composition_validation_corpus.py @@ -166,6 +166,8 @@ def test_current_baseline_snapshot() -> None: Batch 5 (2026-06-17): Gate A2f question_bound_product admits cv-0020 (0021); Batch 4 goal_residual admits cv-0005; prior Gate A1/A2 lifts admit others. Sprint 6 (2026-06-17): Gate A2g duration_segment_total admits cv-0022 (0015). + Sprint 7 (2026-06-17): Gate A2i round_trip_trip_duration admits cv-0006 (0030); + Gate A2j giveaway_target_residual admits cv-0021 (0035). """ solve = refuse = wrong = 0 for case in _CASES: @@ -177,7 +179,7 @@ def test_current_baseline_snapshot() -> None: else: refuse += 1 assert wrong == 0 - assert (solve, refuse) == (10, 12), ( + assert (solve, refuse) == (12, 10), ( f"snapshot moved to {solve} solve / {refuse} refuse — if a Phase 5b " f"slice landed, update this expectation and the affected rows' " f"baseline fields in lockstep" diff --git a/tests/test_gsm8k_post_gate_a1_frontier_microscope.py b/tests/test_gsm8k_post_gate_a1_frontier_microscope.py index 82450ff6..e31a1ca3 100644 --- a/tests/test_gsm8k_post_gate_a1_frontier_microscope.py +++ b/tests/test_gsm8k_post_gate_a1_frontier_microscope.py @@ -74,12 +74,12 @@ def test_live_microscope_refusal_partition_is_complete(): assert row["evidence_snippet"] -def test_live_microscope_partition_seed_case_is_tagged(): - """0003 is solved by Gate A2f; partition_chunking still tags refused peers.""" +def test_live_microscope_solved_lift_not_in_refusal_table(): + """Gate A2f lift 0003 must not appear among live refusals; frontier slices remain.""" summary = build_microscope_report(_load_cases()) - partition_ids = summary["implementation_slice_candidates"]["partition_chunking"]["case_ids"] + rate_ids = summary["implementation_slice_candidates"]["rate_follow_up"]["case_ids"] assert "gsm8k-train-sample-v1-0003" not in {r["case_id"] for r in summary["refusal_table"]} - assert len(partition_ids) >= 1 + assert len(rate_ids) >= 1 def test_microscope_output_is_deterministic(): @@ -104,12 +104,12 @@ def test_markdown_render_surfaces_partition_candidate(): summary = build_microscope_report(_load_cases()) md = render_markdown(summary) assert "partition_chunking" in md - assert "correct: 14" in md + assert "correct: 16" in md assert "Gate A2a unit_partition" in md def test_gate_a2_lifts_are_not_in_refusal_table(): - """Cases solved by Gate A2b–A2h must not appear among live refusals.""" + """Cases solved by Gate A2b–A2j must not appear among live refusals.""" summary = build_microscope_report(_load_cases()) refused_ids = {r["case_id"] for r in summary["refusal_table"]} assert "gsm8k-train-sample-v1-0002" not in refused_ids @@ -118,7 +118,9 @@ def test_gate_a2_lifts_are_not_in_refusal_table(): assert "gsm8k-train-sample-v1-0015" not in refused_ids assert "gsm8k-train-sample-v1-0021" not in refused_ids assert "gsm8k-train-sample-v1-0025" not in refused_ids + assert "gsm8k-train-sample-v1-0030" not in refused_ids + assert "gsm8k-train-sample-v1-0035" not in refused_ids assert "gsm8k-train-sample-v1-0037" not in refused_ids assert "gsm8k-train-sample-v1-0045" not in refused_ids - assert summary["counts"]["correct"] >= 14 + assert summary["counts"]["correct"] >= 16 assert summary["closed_injector_buckets"]["unit_partition_no_injection"] == 0 diff --git a/tests/test_math_candidate_graph_question_bound_product_lift.py b/tests/test_math_candidate_graph_question_bound_product_lift.py index d5be27e4..a8d46c38 100644 --- a/tests/test_math_candidate_graph_question_bound_product_lift.py +++ b/tests/test_math_candidate_graph_question_bound_product_lift.py @@ -163,8 +163,8 @@ def test_full_train_sample_wrong_zero_and_chunk_lift(): report = build_report(_load_cases(_CASES_PATH)) counts = report["counts"] assert counts["wrong"] == 0 - assert counts["correct"] == 14 - assert counts["refused"] == 36 + assert counts["correct"] == 16 + assert counts["refused"] == 34 by_case = {row["case_id"]: row for row in report["per_case"]} assert by_case["gsm8k-train-sample-v1-0003"]["verdict"] == "correct" assert by_case["gsm8k-train-sample-v1-0021"]["verdict"] == "correct" diff --git a/tests/test_math_candidate_graph_sprint6_experience_guided_lift.py b/tests/test_math_candidate_graph_sprint6_experience_guided_lift.py index e1938476..31d33b41 100644 --- a/tests/test_math_candidate_graph_sprint6_experience_guided_lift.py +++ b/tests/test_math_candidate_graph_sprint6_experience_guided_lift.py @@ -173,7 +173,7 @@ class TestTrainSampleScore: wrong += 1 assert wrong == 0 - def test_aggregate_14_36_0(self): + def test_aggregate_16_34_0(self): correct = refused = wrong = 0 for case in _load_train_cases(): res = _run(case["question"]) @@ -184,8 +184,8 @@ class TestTrainSampleScore: correct += 1 else: wrong += 1 - assert correct == 14 - assert refused == 36 + assert correct == 16 + assert refused == 34 assert wrong == 0 diff --git a/tests/test_math_candidate_graph_sprint7_experience_guided_lift.py b/tests/test_math_candidate_graph_sprint7_experience_guided_lift.py new file mode 100644 index 00000000..310d0e41 --- /dev/null +++ b/tests/test_math_candidate_graph_sprint7_experience_guided_lift.py @@ -0,0 +1,238 @@ +"""Gate A2i/A2j — Capability Paradigm Sprint 7 experience-guided lift.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from generate.math_candidate_graph import parse_and_solve +from generate.derivation.round_trip_trip_duration import ( + compose_round_trip_trip_duration, + resolve_promotable_round_trip_trip_duration, +) +from generate.derivation.giveaway_target_residual import ( + compose_giveaway_target_residual, + resolve_promotable_giveaway_target_residual, +) + +_CASES_PATH = ( + Path(__file__).resolve().parents[1] + / "evals" + / "gsm8k_math" + / "train_sample" + / "v1" + / "cases.jsonl" +) +_HOLDOUT_PATH = ( + Path(__file__).resolve().parents[1] + / "evals" + / "gsm8k_math" + / "holdout_dev" + / "v1" + / "cases.jsonl" +) + +CASE_0030 = ( + "Jake decides to go to the beach for a fun day. It is a 2-hour drive each way. " + "He then spends 2.5 times at long at the beach as his total driving time. " + "How much time does the trip take?" +) + +CASE_0035 = ( + "Martha has 20 apples. She decided to split them among her friends. " + "Jane got 5 apples from her, and James got 2 more than Jane. " + "How many more apples would Martha need to give away to be left with only 4 of them?" +) + +SIBLING_ROUND_TRIP = ( + "Mia plans a lake visit. It is a 3-hour drive each way. She then spends " + "2 times as long at the lake as her total driving time. How much time does " + "the trip take?" +) + +SIBLING_GIVEAWAY = ( + "Paul has 30 oranges. He shared them with friends. Amy got 8 oranges from him, " + "and Ben got 3 more than Amy. How many more oranges would Paul need to give away " + "to be left with only 6 of them?" +) + +GOAL_RESIDUAL_CV0005 = ( + "Michael wants to lose 10 pounds by June. He lost 3 pounds in March and 4 pounds " + "in April. How much weight does he have to lose in May to meet his goal?" +) + +DURATION_SEGMENT_0015 = ( + "Traveling from Manhattan to the Bronx, Andrew rides the subway for 10 hours, " + "takes the train and rides for twice as much time as the subway ride, and then " + "bikes the remaining distance for 8 hours. What's the total time he takes to " + "reach the Bronx from Manhattan?" +) + +SEALED_WRONG_0032_SHAPE = ( + "He draws and colors 10 pictures. He draws and colors 7 pictures, but each " + "picture takes 30% less time than the previous one." +) + +SEALED_WRONG_0047_SHAPE = ( + "John bakes 12 coconut macaroons, each weighing 5 ounces. He then packs an equal " + "number of the macaroons in 4 different brown bags, ready for delivery. When he " + "briefly leaves the kitchen to pick the phone, his little brother Steve eats the " + "entire contents of one of the brown bags. What is the total weight, in ounces, " + "of the remaining coconut macaroons?" +) + +PRESERVED_SOLVED = ( + "0002", + "0003", + "0008", + "0014", + "0015", + "0018", + "0021", + "0024", + "0025", + "0029", + "0037", + "0038", + "0042", + "0045", +) + + +def _run(text: str): + return parse_and_solve(text, sealed=False) + + +def _load_train_cases() -> list[dict]: + return [json.loads(line) for line in _CASES_PATH.read_text().splitlines() if line.strip()] + + +class TestTargetCases: + def test_train_sample_0030_end_to_end(self): + res = _run(CASE_0030) + assert res.answer == 14.0 + assert res.refusal_reason is None + + def test_train_sample_0035_end_to_end(self): + res = _run(CASE_0035) + assert res.answer == 4.0 + assert res.refusal_reason is None + + +class TestSiblingGeneralization: + def test_round_trip_sibling(self): + res = _run(SIBLING_ROUND_TRIP) + assert res.answer == 18.0 + assert res.refusal_reason is None + + def test_giveaway_sibling(self): + res = _run(SIBLING_GIVEAWAY) + assert res.answer == 5.0 + assert res.refusal_reason is None + + +class TestConfuserRefusals: + def test_round_trip_without_each_way_refuses(self): + text = ( + "She drove 2 hours to the beach and spent 2.5 times that long swimming. " + "How much time does the trip take?" + ) + assert resolve_promotable_round_trip_trip_duration(text) is None + assert _run(text).answer is None + + def test_round_trip_fraction_surface_refuses(self): + text = ( + "It is a 1/2-hour drive each way. He spends 2 times as long at the park " + "as his total driving time. How much time does the trip take?" + ) + assert resolve_promotable_round_trip_trip_duration(text) is None + + def test_giveaway_goal_language_refuses(self): + assert resolve_promotable_giveaway_target_residual(GOAL_RESIDUAL_CV0005) is None + assert _run(GOAL_RESIDUAL_CV0005).answer == 3.0 + + def test_giveaway_comparative_question_refuses(self): + text = ( + "Martha has 20 apples. Jane got 5 apples from her, and James got 2 more " + "than Jane. How many more apples did James get than Jane?" + ) + assert resolve_promotable_giveaway_target_residual(text) is None + + def test_duration_segment_not_round_trip(self): + assert resolve_promotable_round_trip_trip_duration(DURATION_SEGMENT_0015) is None + res = _run(DURATION_SEGMENT_0015) + assert res.answer == 38.0 + + +class TestSealedWrongPatternRefusal: + def test_0032_percent_time_not_promoted(self): + assert resolve_promotable_round_trip_trip_duration(SEALED_WRONG_0032_SHAPE) is None + assert _run(SEALED_WRONG_0032_SHAPE).answer is None + + def test_0047_partition_not_promoted(self): + assert resolve_promotable_giveaway_target_residual(SEALED_WRONG_0047_SHAPE) is None + assert _run(SEALED_WRONG_0047_SHAPE).answer is None + + +class TestTrainSampleScore: + def test_full_train_sample_wrong_zero(self): + wrong = 0 + for case in _load_train_cases(): + res = _run(case["question"]) + if res.answer is not None and res.answer != float(case["answer_numeric"]): + wrong += 1 + assert wrong == 0 + + def test_aggregate_16_34_0(self): + correct = refused = wrong = 0 + for case in _load_train_cases(): + res = _run(case["question"]) + gold = float(case["answer_numeric"]) + if res.answer is None: + refused += 1 + elif res.answer == gold: + correct += 1 + else: + wrong += 1 + assert correct == 16 + assert refused == 34 + assert wrong == 0 + + +class TestPriorSolvedRegression: + @pytest.mark.parametrize("case_suffix", PRESERVED_SOLVED) + def test_prior_solved_still_correct(self, case_suffix: str): + cases = {c["case_id"].split("-")[-1]: c for c in _load_train_cases()} + case = cases[case_suffix] + res = _run(case["question"]) + assert res.answer == float(case["answer_numeric"]) + + +class TestHoldoutDevSafety: + def test_holdout_dev_no_new_admissions(self): + if not _HOLDOUT_PATH.exists(): + pytest.skip("holdout_dev corpus unavailable") + admitted = 0 + for line in _HOLDOUT_PATH.read_text().splitlines(): + if not line.strip(): + continue + case = json.loads(line) + text = case["problem"] + if ( + compose_round_trip_trip_duration(text) is not None + or compose_giveaway_target_residual(text) is not None + ): + admitted += 1 + assert admitted == 0 + + +class TestComposeAPI: + def test_round_trip_compose_matches_promote(self): + assert compose_round_trip_trip_duration(CASE_0030) is not None + assert resolve_promotable_round_trip_trip_duration(CASE_0030) is not None + + def test_giveaway_compose_matches_promote(self): + assert compose_giveaway_target_residual(CASE_0035) is not None + assert resolve_promotable_giveaway_target_residual(CASE_0035) is not None \ No newline at end of file