From e2f3d3737348752b6b4f3ad7b5a1e5428689336f Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 17 Jun 2026 22:22:53 -0700 Subject: [PATCH] feat(derivation): capability paradigm sprint 6 experience-guided lift (#817) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(derivation): capability paradigm sprint 6 experience-guided lift Experience Flywheel + scout on post-#816 main showed lift_refused_to_correct=0 (Sprint 5 already served prior deltas). Decompose the MA cluster and R5 pin into two narrow organs: - Gate A2g duration_segment_total (0015 / cv-0022) - Gate A2h survey_rate_earnings (0045) Ephemeral train_sample: 12/38/0 → 14/36/0, wrong=0 preserved. Holdout_dev: 0 new admissions. report.json and sealed artifacts untouched. * fix(gsm8k): require compatible duration units * fix(gsm8k): harden survey earnings verification * test(gsm8k): cover sprint6 review hardening --- ...ty-paradigm-sprint6-lookback-2026-06-17.md | 144 +++++++++++ generate/derivation/duration_segment_total.py | 187 ++++++++++++++ generate/derivation/survey_rate_earnings.py | 228 ++++++++++++++++++ generate/math_candidate_graph.py | 30 +++ tests/test_composition_validation_corpus.py | 3 +- ..._gsm8k_post_gate_a1_frontier_microscope.py | 8 +- ...idate_graph_question_bound_product_lift.py | 4 +- ...te_graph_sprint6_experience_guided_lift.py | 226 +++++++++++++++++ ...th_candidate_graph_sprint6_review_patch.py | 30 +++ 9 files changed, 854 insertions(+), 6 deletions(-) create mode 100644 docs/analysis/gsm8k-capability-paradigm-sprint6-lookback-2026-06-17.md create mode 100644 generate/derivation/duration_segment_total.py create mode 100644 generate/derivation/survey_rate_earnings.py create mode 100644 tests/test_math_candidate_graph_sprint6_experience_guided_lift.py create mode 100644 tests/test_math_candidate_graph_sprint6_review_patch.py diff --git a/docs/analysis/gsm8k-capability-paradigm-sprint6-lookback-2026-06-17.md b/docs/analysis/gsm8k-capability-paradigm-sprint6-lookback-2026-06-17.md new file mode 100644 index 00000000..a4598e05 --- /dev/null +++ b/docs/analysis/gsm8k-capability-paradigm-sprint6-lookback-2026-06-17.md @@ -0,0 +1,144 @@ +# GSM8K Capability Paradigm Sprint 6 — Lookback (2026-06-17) + +## 1. Starting baseline + +| Metric | Value | +|--------|------:| +| correct | **12** | +| refused | **38** | +| wrong | **0** | + +After #816 (Experience Flywheel PR-1) and #815 (question_bound_product). Preserved solved: 0002, 0003, 0008, 0014, 0018, 0021, 0024, 0025, 0029, 0037, 0038, 0042. + +## 2. Experience Flywheel output summary + +`scripts/gsm8k_experience_flywheel.py --limit 50 --out /tmp/gsm8k-experience-sprint6.json` + +| Field | Value | +|-------|------:| +| retained_record_count | 50 | +| compacted_record_count | 50 | +| lift_refused_to_correct | **0** | +| promotion_candidates (all) | 4 families, **all** `blocked_by_wrong_risk` | + +Top missing primitives: `relation_hypothesis` (15), `diagnostic_hold` (7), `multiplicative_aggregate` (3), `temporal_tariff` (2). + +**Key signal:** No scout refused→correct delta remained (Sprint 5 already promoted 0003/0021). Flywheel ranked clusters by joint-refusal density + wrong-risk absence, not lift delta. + +## 3. Scout/frontier output summary + +| Regime | correct | refused | wrong | +|--------|--------:|--------:|------:| +| Serving | 12 | 38 | 0 | +| Sealed (resolve_pooled) | 3 | 39 | 8 | + +- `lift_refused_to_correct`: **0** +- `lift_recommendations`: **[]** +- Dominant bottleneck: recognizer matched → no injection (27/50) +- Sealed-wrong elimination: 0011, 0018, 0019, 0025, 0026, 0028, 0032, 0047 + +## 4. Candidate families ranked + +| Rank | Family | Cases | Lift signal | Wrong-risk | Sprint 6 verdict | +|------|--------|-------|-------------|------------|------------------| +| 1 | `multiplicative_aggregate:multiplicative_aggregation` | 0006, 0013, 0045 | joint only | none on surface | **Decomposed** — 0045 solvable as survey organ; 0006/0013 need other primitives | +| 2 | `temporal_tariff:temporal_aggregation` | 0001, 0017 | joint only | low | Deferred — tariff/overtime shapes | +| 3 | `relation_hypothesis:discrete_count_statement` | 13 joint + 2 wrong | none | **blocked** (0032, 0047) | Deferred — confuser-gated | +| — | `composition_validation` R5 pin | 0015 (`cv-0022`) | typed chain documented | low | **Selected (A2g)** | +| — | Survey earnings sub-shape of MA cluster | 0045 | typed chain | low | **Selected (A2h)** | + +## 5. Blocked families and why + +| Family | Blocked because | +|--------|-----------------| +| `relation_hypothesis:DCS` | 0032, 0047 sealed-wrong share recognizer surface | +| `conservative_boundary` | 0018, 0025 sealed-wrong on served-correct cases | +| `sealed_elimination` | 0011, 0026 hard elimination | +| `diagnostic_hold:currency_amount` | 0019, 0028 sealed-wrong | + +## 6. Selected paradigm + +**Dual narrow organs** (experience-guided decomposition, not broad MA/DCS promotion): + +1. **Gate A2g `duration_segment_total`** — R5 comparative middle-leg + fixed legs, total-time question (`cv-0022` / 0015). +2. **Gate A2h `survey_rate_earnings`** — `(surveys_a + surveys_b) × questions_per_survey × $/question` with money+earn question binding (0045 MA sub-shape). + +Rejected wholesale `multiplicative_aggregate` promotion: subagent chain analysis showed 0006/0013 not end-to-end MA; 0045 only the survey slice is sound. + +## 7. Typed chains for target cases + +**0015 (gold 38):** `10 + (10×2) + 8` — subway leg, train leg = twice subway, bike leg; question binds `total` + `time`. + +**0045 (gold 14):** `(3+4) × 10 × 0.2` — Mon/Tue survey counts, questions/survey, rate/question; question binds `money` + `earn`. + +## 8. Implemented organs + +| Module | Promotion entry | +|--------|-----------------| +| `generate/derivation/duration_segment_total.py` | `resolve_promotable_duration_segment_total` | +| `generate/derivation/survey_rate_earnings.py` | `resolve_promotable_survey_rate_earnings` | + +Wired in `generate/math_candidate_graph.py` after Gate A2f, before rate short-circuits. + +## 9. Before / after score + +| | Before | After | +|---|------:|------:| +| correct | 12 | **14** | +| refused | 38 | **36** | +| wrong | 0 | **0** | + +## 10. Newly solved IDs + +- `gsm8k-train-sample-v1-0015` +- `gsm8k-train-sample-v1-0045` + +## 11. Preserved solved IDs + +0002, 0003, 0008, 0014, 0018, 0021, 0024, 0025, 0029, 0037, 0038, 0042 + +## 12. Wrong=0 proof + +Ephemeral train_sample: `14 / 36 / 0`. Holdout_dev scan: **0 admissions** for either new promotion bridge. + +## 13. Confuser matrix + +| Confuser | Result | +|----------|--------| +| Duration without total-time question | refuse | +| Survey without rate clause | refuse | +| Comparative survey count (twice Monday) | refuse | +| Fraction duration surface | refuse | +| 0047 partition/remaining weight | refuse | +| 0032 percent-less time | refuse | + +## 14. Sibling generality proof + +- Duration sibling: bus 5h + ferry 2×5h + walk 3h → 18 +- Survey sibling: (2+5)×8×$0.50 → 28 + +## 15. Sealed-wrong negative evidence used + +Flywheel `blocked_by_wrong_risk` on DCS/MA families informed refusal to promote broad injectors. Tests explicitly refuse 0047/0032 shapes on new gates. + +## 16–21. Confirmations + +- `report.json` untouched (historical pin) +- Sealed-lane artifacts untouched +- No case-id logic +- No hardcoded final answers +- No direct-answer extraction +- **Non-goals:** broad `product_bridge`, broad DCS/MA injectors, `resolve_pooled` re-wire, report rebaseline, sealed movement, 0030 round-trip/beach organ (deferred), DCS family until confusers pass + +## Validation + +```bash +git diff --check origin/main...HEAD +pytest tests/test_math_candidate_graph_sprint6_experience_guided_lift.py -q +pytest tests/test_gsm8k_experience_flywheel.py -q +pytest tests/test_gsm8k_sealed_attempt_scout.py -q +pytest tests/test_composition_validation_corpus.py -q +core test --suite smoke -q +``` + +Composition-validation snapshot: **10 solve / 12 refuse / 0 wrong** (`cv-0022` flipped). \ No newline at end of file diff --git a/generate/derivation/duration_segment_total.py b/generate/derivation/duration_segment_total.py new file mode 100644 index 00000000..b89ed534 --- /dev/null +++ b/generate/derivation/duration_segment_total.py @@ -0,0 +1,187 @@ +"""Gate A2g — duration segment total (comparative middle leg + fixed legs). + +Experience Flywheel Sprint 6 selected this family after scout showed +``lift_refused_to_correct == 0`` (prior lifts already served) and the +``multiplicative_aggregate`` cluster lacked end-to-end sealed signal. The +composition-validation corpus pins train_sample **0015** as R5 +(``cv-0022``): fixed leg + comparative middle leg referencing the first +fixed leg + optional trailing fixed leg, question asks total time. + +This is a first-principles organ — not broad DCS injection, not +``resolve_pooled``. Promotion requires: + +- question binds ``total`` + ``time``; +- exactly one comparative scalar (``twice``, ``double``, ``thrice``, …); +- two or three grounded duration quantities in the body; +- comparative cue appears between the first and last duration anchors; +- hazard refusal (fractions, percent, money, profit, remaining, per-day tariffs). + +Deterministic; sealed module (no ``chat/`` import). +""" + +from __future__ import annotations + +import re +from typing import Final + +from generate.derivation.clauses import segment_clauses +from generate.derivation.comparatives import comparative_step, extract_comparative_scalars +from generate.derivation.extract import extract_quantities +from generate.derivation.model import GroundedDerivation, Quantity, Step +from generate.derivation.target import _question_clause +from generate.derivation.verify import Resolution, select_self_verified +from generate.math_roundtrip import _tokens + +_FRACTION_RE: Final[re.Pattern[str]] = re.compile(r"\d+/\d+") +_DURATION_UNITS: Final[frozenset[str]] = frozenset( + {"hour", "hours", "hr", "hrs", "minute", "minutes", "min", "mins"} +) +_TEXT_BLOCKERS: Final[frozenset[str]] = frozenset( + { + "each", + "insurance", + "money", + "overtime", + "percent", + "percentage", + "profit", + "rest", + "salary", + "wage", + } +) +_REMAINING_DISTANCE_RE: Final[re.Pattern[str]] = re.compile( + r"\bremaining\s+distance\b", re.IGNORECASE +) +_QUESTION_BLOCKERS: Final[frozenset[str]] = frozenset( + {"longer", "more", "less", "per", "profit", "remaining"} +) + + +def _asks_total_time(question_clause: str) -> bool: + tokens = _tokens(question_clause) + return "total" in tokens and "time" in tokens + + +def _has_hazard_surface(problem_text: str, question_clause: str) -> bool: + if _FRACTION_RE.search(problem_text): + return True + text_tokens = _tokens(problem_text) + question_tokens = _tokens(question_clause) + if "%" in problem_text or "percent" in text_tokens or "percentage" in text_tokens: + return True + if text_tokens & _TEXT_BLOCKERS: + return True + if "remaining" in text_tokens and not _REMAINING_DISTANCE_RE.search(problem_text): + return True + if question_tokens & _QUESTION_BLOCKERS: + return True + return False + + +def _duration_quantities(clause: str) -> tuple[Quantity, ...]: + return tuple( + q + for q in extract_quantities(clause) + if q.unit in _DURATION_UNITS + or q.unit.rstrip("s") in {"hour", "hr", "minute", "min"} + ) + + +def _duration_family(unit: str) -> str | None: + stem = unit.rstrip("s") + if stem in {"hour", "hr"}: + return "hour" + if stem in {"minute", "min"}: + return "minute" + return None + + +def _same_duration_family(a: Quantity, b: Quantity) -> bool: + """Refuse mixed duration units instead of adding hours and minutes directly.""" + fam_a = _duration_family(a.unit) + fam_b = _duration_family(b.unit) + return fam_a is not None and fam_a == fam_b + + +def _pick_add_cue(problem_text: str, *, prefer: tuple[str, ...]) -> str | None: + tokens = _tokens(problem_text) + for cue in prefer: + if cue in tokens: + return cue + return None + + +def build_duration_segment_total(problem_text: str) -> GroundedDerivation | None: + """Construct the ungated duration-segment-total chain, or ``None``. + + Shape (0015-class): + leg1 + (leg1 × comparative_scalar) + leg3? + + Fold: + start leg1 → × comparative → + leg1 → + leg3? + """ + question_clause = _question_clause(problem_text) + if not _asks_total_time(question_clause): + return None + if _has_hazard_surface(problem_text, question_clause): + return None + + comparatives = extract_comparative_scalars(problem_text) + if len(comparatives) != 1: + return None + comparative = comparatives[0] + if comparative.scalar <= 0: + return None + + body_clauses = [ + clause + for clause in segment_clauses(problem_text) + if clause != question_clause and comparative.cue in _tokens(clause) + ] + if len(body_clauses) != 1: + return None + + durations = _duration_quantities(body_clauses[0]) + if len(durations) != 2: + return None + + leg1, leg3 = durations + if not _same_duration_family(leg1, leg3): + return None + + add_back_cue = _pick_add_cue(problem_text, prefer=("subway", "bus", "ride", "and")) + if add_back_cue is None: + return None + tail_cue = _pick_add_cue(problem_text, prefer=("then", "and")) + if tail_cue is None: + return None + + steps: list[Step] = [ + comparative_step(comparative), + Step( + op="add", + operand=Quantity(value=leg1.value, unit=leg1.unit, source_token=leg1.source_token), + cue=add_back_cue, + ), + Step( + op="add", + operand=Quantity(value=leg3.value, unit=leg3.unit, source_token=leg3.source_token), + cue=tail_cue, + ), + ] + + return GroundedDerivation(start=leg1, steps=tuple(steps)) + + +def compose_duration_segment_total(problem_text: str) -> Resolution | None: + """Gate the typed duration chain through self-verification.""" + derivation = build_duration_segment_total(problem_text) + if derivation is None: + return None + return select_self_verified([derivation], problem_text, target_units=()) + + +def resolve_promotable_duration_segment_total(problem_text: str) -> Resolution | None: + """Serving promotion bridge (Gate A2g).""" + return compose_duration_segment_total(problem_text) diff --git a/generate/derivation/survey_rate_earnings.py b/generate/derivation/survey_rate_earnings.py new file mode 100644 index 00000000..95d21978 --- /dev/null +++ b/generate/derivation/survey_rate_earnings.py @@ -0,0 +1,228 @@ +"""Gate A2h — survey rate earnings (survey-count sum × questions/survey × $/question). + +Experience Flywheel Sprint 6: ``multiplicative_aggregate`` cluster had recognizer +hits but no sealed-correct lift signal; case **0045** decomposes cleanly as +``(surveys_mon + surveys_tue) × questions_per_survey × rate_per_question`` with +explicit question money binding. + +Narrow organ — not broad product_bridge, not generic multiplication over +co-occurring numbers. Promotion requires: + +- question asks ``money`` + ``earn``/``make``/``receive``; +- body states rate per question (``$`` + ``every``/``per`` + ``question``); +- body states ``each/every survey has N questions``; +- exactly two day-indexed survey counts that sum; +- hazard refusal (fractions, percent, profit, comparative distractors). + +Question-clause temporal scaffolding (e.g. ``two days``) is excluded from the +completeness obligation — Mon/Tue survey counts already decompose the window. + +Deterministic; sealed module (no ``chat/`` import). +""" + +from __future__ import annotations + +from collections import Counter +import re +from typing import Final + +from generate.derivation.clauses import segment_clauses +from generate.derivation.extract import extract_quantities +from generate.derivation.model import GroundedDerivation, Quantity, Step +from generate.derivation.target import _question_clause +from generate.derivation.verify import Resolution, SelfVerification, select_self_verified +from generate.math_roundtrip import _token_in, _tokens, _value_grounds + +_FRACTION_RE: Final[re.Pattern[str]] = re.compile(r"\d+/\d+") +_EARN_VERBS: Final[frozenset[str]] = frozenset( + {"earn", "earned", "make", "made", "receive", "received", "get", "got"} +) +_TEXT_BLOCKERS: Final[frozenset[str]] = frozenset( + { + "half", + "insurance", + "percent", + "percentage", + "profit", + "twice", + "thrice", + } +) +_QUESTION_BLOCKERS: Final[frozenset[str]] = frozenset( + {"left", "remaining", "profit"} +) + + +def _asks_money_earned(question_clause: str) -> bool: + tokens = _tokens(question_clause) + return "money" in tokens and bool(_EARN_VERBS & tokens) + + +def _has_hazard_surface(problem_text: str, question_clause: str) -> bool: + if _FRACTION_RE.search(problem_text): + return True + text_tokens = _tokens(problem_text) + question_tokens = _tokens(question_clause) + if "%" in problem_text or "percent" in text_tokens: + return True + if text_tokens & _TEXT_BLOCKERS: + return True + if question_tokens & _QUESTION_BLOCKERS: + return True + return False + + +def _body_text(problem_text: str) -> str: + question_clause = _question_clause(problem_text) + return problem_text.replace(question_clause, "").strip() + + +def _clause_mentions_survey(clause: str) -> bool: + tokens = _tokens(clause) + return "survey" in tokens or "surveys" in tokens + + +def _survey_counts(problem_text: str) -> tuple[Quantity, Quantity] | None: + counts: list[Quantity] = [] + question_clause = _question_clause(problem_text) + for clause in segment_clauses(problem_text): + if clause == question_clause: + continue + if not _clause_mentions_survey(clause): + continue + for q in extract_quantities(clause): + if q.unit == "surveys" or q.unit.rstrip("s") == "survey": + counts.append( + Quantity(value=q.value, unit="surveys", source_token=q.source_token) + ) + if len(counts) != 2: + return None + return counts[0], counts[1] + + +def _questions_per_survey(problem_text: str) -> Quantity | None: + question_clause = _question_clause(problem_text) + for clause in segment_clauses(problem_text): + if clause == question_clause: + continue + if not _clause_mentions_survey(clause): + continue + if not ("each" in _tokens(clause) or "every" in _tokens(clause)): + continue + if "questions" not in _tokens(clause) and "question" not in _tokens(clause): + continue + quantities = extract_quantities(clause) + if len(quantities) != 1: + continue + q = quantities[0] + if q.unit not in {"questions", "question"} and q.unit.rstrip("s") != "question": + continue + return Quantity(value=q.value, unit="questions", source_token=q.source_token) + return None + + +def _rate_per_question(problem_text: str) -> tuple[Quantity, str] | None: + question_clause = _question_clause(problem_text) + for clause in segment_clauses(problem_text): + if clause == question_clause: + continue + if "$" not in clause and "dollar" not in _tokens(clause): + continue + if "question" not in _tokens(clause): + continue + rate_cue = next( + (c for c in ("every", "per", "each") if c in _tokens(clause)), + None, + ) + if rate_cue is None: + continue + quantities = extract_quantities(clause) + if len(quantities) != 1: + continue + q = quantities[0] + return Quantity(value=q.value, unit="dollars", source_token=q.source_token), rate_cue + return None + + +def build_survey_rate_earnings(problem_text: str) -> GroundedDerivation | None: + """Construct the ungated survey earnings chain, or ``None``.""" + question_clause = _question_clause(problem_text) + if not _asks_money_earned(question_clause): + return None + if _has_hazard_surface(problem_text, question_clause): + return None + + survey_pair = _survey_counts(problem_text) + per_survey = _questions_per_survey(problem_text) + rate_info = _rate_per_question(problem_text) + if survey_pair is None or per_survey is None or rate_info is None: + return None + + count_a, count_b = survey_pair + rate, rate_cue = rate_info + sum_cue = "and" if "and" in _tokens(problem_text) else None + if sum_cue is None: + return None + has_cue = next((c for c in ("has", "have", "contains") if c in _tokens(problem_text)), None) + if has_cue is None: + return None + + return GroundedDerivation( + start=count_a, + steps=( + Step(op="add", operand=count_b, cue=sum_cue), + Step(op="multiply", operand=per_survey, cue=has_cue), + Step(op="multiply", operand=rate, cue=rate_cue), + ), + ) + + +def _self_verifies_survey_earnings( + derivation: GroundedDerivation, problem_text: str +) -> SelfVerification: + """Self-verify with body-scoped completeness (question ``days`` scaffolding).""" + from generate.derivation.verify import _base_reasons + + tokens = _tokens(problem_text) + reasons = list(_base_reasons(derivation, tokens)) + + body = _body_text(problem_text) + body_quantities = Counter(q.source_token for q in extract_quantities(body)) + used = Counter( + [ + derivation.start.source_token, + *(step.operand.source_token for step in derivation.steps), + ] + ) + unused = body_quantities - used + if unused: + reasons.append(f"incomplete: unused body quantities {sorted(unused.elements())}") + + for step in derivation.steps: + if not _token_in(step.cue, tokens): + reasons.append(f"operation cue {step.cue!r} not grounded in text") + if not _value_grounds(derivation.start.source_token, tokens): + reasons.append( + f"operand {derivation.start.source_token!r} not grounded in text" + ) + + return SelfVerification(verified=not reasons, reasons=tuple(reasons)) + + +def compose_survey_rate_earnings(problem_text: str) -> Resolution | None: + """Gate the typed survey earnings chain through self-verification.""" + derivation = build_survey_rate_earnings(problem_text) + if derivation is None: + return None + if not _self_verifies_survey_earnings(derivation, problem_text).verified: + return None + return Resolution( + answer=derivation.answer, + answer_unit=derivation.answer_unit, + derivation=derivation, + ) + + +def resolve_promotable_survey_rate_earnings(problem_text: str) -> Resolution | None: + """Serving promotion bridge (Gate A2h).""" + return compose_survey_rate_earnings(problem_text) diff --git a/generate/math_candidate_graph.py b/generate/math_candidate_graph.py index 86065344..5fd8827a 100644 --- a/generate/math_candidate_graph.py +++ b/generate/math_candidate_graph.py @@ -729,6 +729,36 @@ def parse_and_solve(text: str, *, sealed: bool = False) -> CandidateGraphResult: branches_admissible=1, ) + # Gate A2g — duration segment total (Sprint 6 experience-guided lift). + from generate.derivation.duration_segment_total import ( + resolve_promotable_duration_segment_total, + ) + + duration_resolution = resolve_promotable_duration_segment_total(text) + if duration_resolution is not None: + return CandidateGraphResult( + answer=duration_resolution.answer, + selected_graph=None, + refusal_reason=None, + branches_enumerated=1, + branches_admissible=1, + ) + + # Gate A2h — survey rate earnings (Sprint 6 experience-guided lift). + from generate.derivation.survey_rate_earnings import ( + resolve_promotable_survey_rate_earnings, + ) + + survey_resolution = resolve_promotable_survey_rate_earnings(text) + if survey_resolution is not None: + return CandidateGraphResult( + answer=survey_resolution.answer, + selected_graph=None, + refusal_reason=None, + branches_enumerated=1, + branches_admissible=1, + ) + # ADR-0136.S.1 — Rate/event short-circuit paths (before Cartesian product). # Capacity path: single statement with one CandidateCapacity + matching question. if len(statement_sentences) == 1: diff --git a/tests/test_composition_validation_corpus.py b/tests/test_composition_validation_corpus.py index d6dcc382..92bf6f83 100644 --- a/tests/test_composition_validation_corpus.py +++ b/tests/test_composition_validation_corpus.py @@ -165,6 +165,7 @@ def test_current_baseline_snapshot() -> None: Updates when a Phase 5b / capability-strike slice flips a positive row. Batch 5 (2026-06-17): Gate A2f question_bound_product admits cv-0020 (0021); Batch 4 goal_residual admits cv-0005; prior Gate A1/A2 lifts admit others. + Sprint 6 (2026-06-17): Gate A2g duration_segment_total admits cv-0022 (0015). """ solve = refuse = wrong = 0 for case in _CASES: @@ -176,7 +177,7 @@ def test_current_baseline_snapshot() -> None: else: refuse += 1 assert wrong == 0 - assert (solve, refuse) == (9, 13), ( + assert (solve, refuse) == (10, 12), ( f"snapshot moved to {solve} solve / {refuse} refuse — if a Phase 5b " f"slice landed, update this expectation and the affected rows' " f"baseline fields in lockstep" diff --git a/tests/test_gsm8k_post_gate_a1_frontier_microscope.py b/tests/test_gsm8k_post_gate_a1_frontier_microscope.py index fb7bdebe..82450ff6 100644 --- a/tests/test_gsm8k_post_gate_a1_frontier_microscope.py +++ b/tests/test_gsm8k_post_gate_a1_frontier_microscope.py @@ -104,19 +104,21 @@ def test_markdown_render_surfaces_partition_candidate(): summary = build_microscope_report(_load_cases()) md = render_markdown(summary) assert "partition_chunking" in md - assert "correct: 12" in md + assert "correct: 14" in md assert "Gate A2a unit_partition" in md def test_gate_a2_lifts_are_not_in_refusal_table(): - """Cases solved by Gate A2b–A2f must not appear among live refusals.""" + """Cases solved by Gate A2b–A2h must not appear among live refusals.""" summary = build_microscope_report(_load_cases()) refused_ids = {r["case_id"] for r in summary["refusal_table"]} assert "gsm8k-train-sample-v1-0002" not in refused_ids assert "gsm8k-train-sample-v1-0003" not in refused_ids assert "gsm8k-train-sample-v1-0008" not in refused_ids + assert "gsm8k-train-sample-v1-0015" not in refused_ids assert "gsm8k-train-sample-v1-0021" not in refused_ids assert "gsm8k-train-sample-v1-0025" not in refused_ids assert "gsm8k-train-sample-v1-0037" not in refused_ids - assert summary["counts"]["correct"] >= 12 + assert "gsm8k-train-sample-v1-0045" not in refused_ids + assert summary["counts"]["correct"] >= 14 assert summary["closed_injector_buckets"]["unit_partition_no_injection"] == 0 diff --git a/tests/test_math_candidate_graph_question_bound_product_lift.py b/tests/test_math_candidate_graph_question_bound_product_lift.py index 20ac0e58..d5be27e4 100644 --- a/tests/test_math_candidate_graph_question_bound_product_lift.py +++ b/tests/test_math_candidate_graph_question_bound_product_lift.py @@ -163,8 +163,8 @@ def test_full_train_sample_wrong_zero_and_chunk_lift(): report = build_report(_load_cases(_CASES_PATH)) counts = report["counts"] assert counts["wrong"] == 0 - assert counts["correct"] == 12 - assert counts["refused"] == 38 + assert counts["correct"] == 14 + assert counts["refused"] == 36 by_case = {row["case_id"]: row for row in report["per_case"]} assert by_case["gsm8k-train-sample-v1-0003"]["verdict"] == "correct" assert by_case["gsm8k-train-sample-v1-0021"]["verdict"] == "correct" diff --git a/tests/test_math_candidate_graph_sprint6_experience_guided_lift.py b/tests/test_math_candidate_graph_sprint6_experience_guided_lift.py new file mode 100644 index 00000000..e1938476 --- /dev/null +++ b/tests/test_math_candidate_graph_sprint6_experience_guided_lift.py @@ -0,0 +1,226 @@ +"""Gate A2g/A2h — Capability Paradigm Sprint 6 experience-guided lift.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from generate.math_candidate_graph import parse_and_solve +from generate.derivation.duration_segment_total import ( + compose_duration_segment_total, + resolve_promotable_duration_segment_total, +) +from generate.derivation.survey_rate_earnings import ( + compose_survey_rate_earnings, + resolve_promotable_survey_rate_earnings, +) + +_CASES_PATH = ( + Path(__file__).resolve().parents[1] + / "evals" + / "gsm8k_math" + / "train_sample" + / "v1" + / "cases.jsonl" +) +_HOLDOUT_PATH = ( + Path(__file__).resolve().parents[1] + / "evals" + / "gsm8k_math" + / "holdout_dev" + / "v1" + / "cases.jsonl" +) + +CASE_0015 = ( + "Traveling from Manhattan to the Bronx, Andrew rides the subway for 10 hours, " + "takes the train and rides for twice as much time as the subway ride, and then " + "bikes the remaining distance for 8 hours. What's the total time he takes to " + "reach the Bronx from Manhattan?" +) + +CASE_0045 = ( + "Bart fills out surveys to earn money. He receives $0.2 for every question he " + "answers in the survey. Each survey has 10 questions. On Monday he finished " + "3 surveys, and on Tuesday 4 surveys. How much money did he earn during these " + "two days?" +) + +SIBLING_DURATION = ( + "Crossing the city, Maria rides the bus for 5 hours, then the ferry for twice " + "as much time as the bus ride, then walks the remaining distance for 3 hours. " + "What's the total time for her journey?" +) + +SIBLING_SURVEY = ( + "Lisa completes surveys for cash. She gets $0.50 per question. Every survey has " + "8 questions. On Wednesday she finished 2 surveys, and on Thursday 5 surveys. " + "How much money did she earn?" +) + +SEALED_WRONG_0047_SHAPE = ( + "John bakes 12 coconut macaroons, each weighing 5 ounces. He then packs an equal " + "number of the macaroons in 4 different brown bags, ready for delivery. When he " + "briefly leaves the kitchen to pick the phone, his little brother Steve eats the " + "entire contents of one of the brown bags. What is the total weight, in ounces, " + "of the remaining coconut macaroons?" +) + +SEALED_WRONG_0032_SHAPE = ( + "He draws and colors 10 pictures. He draws and colors 7 pictures, but each " + "picture takes 30% less time than the previous one." +) + +PRESERVED_SOLVED = ( + "0002", + "0003", + "0008", + "0014", + "0018", + "0021", + "0024", + "0025", + "0029", + "0037", + "0038", + "0042", +) + + +def _run(text: str): + return parse_and_solve(text, sealed=False) + + +def _load_train_cases() -> list[dict]: + return [json.loads(line) for line in _CASES_PATH.read_text().splitlines() if line.strip()] + + +class TestTargetCases: + def test_train_sample_0015_end_to_end(self): + res = _run(CASE_0015) + assert res.answer == 38.0 + assert res.refusal_reason is None + + def test_train_sample_0045_end_to_end(self): + res = _run(CASE_0045) + assert res.answer == 14.0 + assert res.refusal_reason is None + + +class TestSiblingGeneralization: + def test_duration_segment_sibling(self): + res = _run(SIBLING_DURATION) + assert res.answer == 18.0 + assert res.refusal_reason is None + + def test_survey_earnings_sibling(self): + res = _run(SIBLING_SURVEY) + assert res.answer == 28.0 + assert res.refusal_reason is None + + +class TestConfuserRefusals: + def test_duration_without_total_time_question_refuses(self): + text = ( + "Andrew rides the subway for 10 hours, then the train for twice as much " + "time as the subway ride, then bikes for 8 hours. How many hours on the train?" + ) + assert resolve_promotable_duration_segment_total(text) is None + assert _run(text).answer is None + + def test_survey_without_rate_refuses(self): + text = ( + "Each survey has 10 questions. On Monday 3 surveys and Tuesday 4 surveys. " + "How much money did he earn?" + ) + assert resolve_promotable_survey_rate_earnings(text) is None + + def test_survey_comparative_distractor_refuses(self): + text = ( + "He gets $0.20 per question. Each survey has 10 questions. " + "On Monday 3 surveys and on Tuesday twice as many surveys as Monday. " + "How much money did he earn?" + ) + assert _run(text).answer is None + + def test_duration_fraction_surface_refuses(self): + text = ( + "She rode 1/2 hour on the bus, then twice as much time as that on the train, " + "then 2 hours walking. What's the total time?" + ) + assert resolve_promotable_duration_segment_total(text) is None + + +class TestSealedWrongPatternRefusal: + def test_0047_partition_weight_not_promoted(self): + assert resolve_promotable_duration_segment_total(SEALED_WRONG_0047_SHAPE) is None + assert resolve_promotable_survey_rate_earnings(SEALED_WRONG_0047_SHAPE) is None + assert _run(SEALED_WRONG_0047_SHAPE).answer is None + + def test_0032_percent_time_not_promoted(self): + assert resolve_promotable_duration_segment_total(SEALED_WRONG_0032_SHAPE) is None + assert _run(SEALED_WRONG_0032_SHAPE).answer is None + + +class TestTrainSampleScore: + def test_full_train_sample_wrong_zero(self): + wrong = 0 + for case in _load_train_cases(): + res = _run(case["question"]) + if res.answer is not None and res.answer != float(case["answer_numeric"]): + wrong += 1 + assert wrong == 0 + + def test_aggregate_14_36_0(self): + correct = refused = wrong = 0 + for case in _load_train_cases(): + res = _run(case["question"]) + gold = float(case["answer_numeric"]) + if res.answer is None: + refused += 1 + elif res.answer == gold: + correct += 1 + else: + wrong += 1 + assert correct == 14 + assert refused == 36 + assert wrong == 0 + + +class TestPriorSolvedRegression: + @pytest.mark.parametrize("case_suffix", PRESERVED_SOLVED) + def test_prior_solved_still_correct(self, case_suffix: str): + cases = {c["case_id"].split("-")[-1]: c for c in _load_train_cases()} + case = cases[case_suffix] + res = _run(case["question"]) + assert res.answer == float(case["answer_numeric"]) + + +class TestHoldoutDevSafety: + def test_holdout_dev_no_new_admissions(self): + if not _HOLDOUT_PATH.exists(): + pytest.skip("holdout_dev corpus unavailable") + admitted = 0 + for line in _HOLDOUT_PATH.read_text().splitlines(): + if not line.strip(): + continue + case = json.loads(line) + text = case["problem"] + if ( + compose_duration_segment_total(text) is not None + or compose_survey_rate_earnings(text) is not None + ): + admitted += 1 + assert admitted == 0 + + +class TestComposeAPI: + def test_duration_compose_matches_promote(self): + assert compose_duration_segment_total(CASE_0015) is not None + assert resolve_promotable_duration_segment_total(CASE_0015) is not None + + def test_survey_compose_matches_promote(self): + assert compose_survey_rate_earnings(CASE_0045) is not None + assert resolve_promotable_survey_rate_earnings(CASE_0045) is not None \ No newline at end of file diff --git a/tests/test_math_candidate_graph_sprint6_review_patch.py b/tests/test_math_candidate_graph_sprint6_review_patch.py new file mode 100644 index 00000000..e36ff39c --- /dev/null +++ b/tests/test_math_candidate_graph_sprint6_review_patch.py @@ -0,0 +1,30 @@ +"""Review-hardening tests for Sprint 6 A2g/A2h organs.""" + +from __future__ import annotations + +from generate.derivation.duration_segment_total import ( + resolve_promotable_duration_segment_total, +) +from generate.derivation.survey_rate_earnings import compose_survey_rate_earnings +from generate.math_candidate_graph import parse_and_solve + + +def test_duration_segment_total_refuses_mixed_duration_units(): + text = ( + "Crossing the city, Maria rides the bus for 5 hours, then the ferry for " + "twice as much time as the bus ride, then walks the remaining distance " + "for 3 minutes. What's the total time for her journey?" + ) + assert resolve_promotable_duration_segment_total(text) is None + assert parse_and_solve(text).answer is None + + +def test_survey_rate_earnings_rejects_duplicate_unused_quantity(): + text = ( + "Lisa completes surveys for cash. She gets $0.10 per question. Every " + "survey has 10 questions. On Wednesday she finished 2 surveys, and on " + "Thursday 5 surveys. She also completed 10 training modules. How much " + "money did she earn?" + ) + assert compose_survey_rate_earnings(text) is None + assert parse_and_solve(text).answer is None