From aa6173af58b4f1a7bb4999010ee4ce69a363f57c Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 17 Jun 2026 20:51:17 -0700 Subject: [PATCH] feat(derivation): capability paradigm sprint 5 question-bound product lift (#815) Practice/scout showed 0003/0021 refuse on DCS no-injection while sealed resolve_pooled commits correctly. Build Gate A2f typed revenue/weight product organs narrower than disabled product_bridge; lifts train_sample 10/40/0 to 12/38/0 with wrong=0 and zero holdout_dev admissions. --- ...ty-paradigm-sprint5-lookback-2026-06-17.md | 131 ++++++++ generate/derivation/question_bound_product.py | 309 ++++++++++++++++++ generate/math_candidate_graph.py | 17 + tests/test_adr_0175_phase3b_mult_search.py | 4 +- tests/test_adr_0195_product_bridge.py | 18 +- tests/test_composition_validation_corpus.py | 8 +- ..._gsm8k_post_gate_a1_frontier_microscope.py | 16 +- ...math_candidate_graph_goal_residual_lift.py | 12 +- ...idate_graph_question_bound_product_lift.py | 183 +++++++++++ 9 files changed, 669 insertions(+), 29 deletions(-) create mode 100644 docs/analysis/gsm8k-capability-paradigm-sprint5-lookback-2026-06-17.md create mode 100644 generate/derivation/question_bound_product.py create mode 100644 tests/test_math_candidate_graph_question_bound_product_lift.py diff --git a/docs/analysis/gsm8k-capability-paradigm-sprint5-lookback-2026-06-17.md b/docs/analysis/gsm8k-capability-paradigm-sprint5-lookback-2026-06-17.md new file mode 100644 index 00000000..0117bd54 --- /dev/null +++ b/docs/analysis/gsm8k-capability-paradigm-sprint5-lookback-2026-06-17.md @@ -0,0 +1,131 @@ +# GSM8K Capability Paradigm Sprint 5 — Lookback (2026-06-17) + +## 1. Starting baseline + +| Metric | Value | +|--------|------:| +| correct | **10** | +| refused | **40** | +| wrong | **0** | + +After #814 (Gate A2e goal_residual). Preserved solved: 0002, 0008, 0014, 0018, 0024, 0025, 0029, 0037, 0038, 0042. + +## 2. Scout / practice evidence + +Scout (`scripts/gsm8k_sealed_attempt_scout.py --limit 50`): + +| Regime | correct | refused | wrong | +|--------|--------:|--------:|------:| +| Serving | 10 | 40 | 0 | +| Sealed (resolve_pooled) | 3 | 39 | 8 | + +**Top lift cluster:** `lift_skill_gap_recognized_no_injection_discrete_count_statement` — cases **0003**, **0021** (n=2). Sealed commits 864 and 450 via `relation_hypothesis:discrete_count_statement`; serving stops at DCS no-injection. + +**Elimination signal (sealed wrong, serving refused):** 0011 (3200 vs 50), 0047 (240 vs 45), plus 6 `elimination_refused_to_wrong` — not promotable. + +## 3. Candidate family clusters + +| family_bucket | case_ids | serving failure | sealed behavior | missing primitive | expected lift | wrong-risk | +|---------------|----------|-----------------|-----------------|-------------------|---------------|------------| +| discrete_count_statement → product | 0003, 0021 | DCS no-injection | sealed correct | typed product chain | +2 | medium via product_bridge | +| goal_residual (served) | 0037 | — | already served | — | 0 | low | +| joint_sealed_no_resolution | 32 | parse/injection | refused | various | 0 | low | +| sealed_elimination | 0011, 0047, … | refused | sealed wrong | not product | 0 | high | + +## 4. Why selected paradigm + +**`question_bound_product_aggregate` (Gate A2f)** — practice proved the reasoning chain is valid for 0003/0021, but broad `product_bridge` over `resolve_pooled` admitted held-out wrong (e.g. holdout-dev 0345: fraction surface + `total weight` → 6720 vs 595). + +Built a **first-principles organ** (not a pooled filter): + +1. **Revenue chain:** container leaf × `in each` per-container leaf × `$`/`dollar`+`each` price (price may live in question clause); question binds `money` + `make|earn|raise|collect`. +2. **Weight-work chain:** single clause with exactly 3 distinct units, mass anchor, `for` cue, `search_chain` multiply-only; question binds `weight` + `total|move|lift|press`. + +## 5. Rejected candidates + +| Candidate | Why unsafe / lower leverage | +|-----------|----------------------------| +| Re-wire `product_bridge` | Held-out 5 wrong; admits fraction/comparative `total weight` surfaces | +| Broad DCS injection | Would multiply arbitrary co-occurring numbers | +| 0007 / 0047 / 0004 chunk | No sealed-correct signal; sealed wrong on 0047 | +| `resolve_pooled` wholesale | 17% wrong on holdout_dev per 2026-06-04 measurement | + +## 6. Typed paradigm implemented + +Module: `generate/derivation/question_bound_product.py` + +- `build_question_bound_product()` — typed chain construction +- `compose_question_bound_product()` — self-verification gate +- `resolve_promotable_question_bound_product()` — serving promotion (Gate A2f) +- Wired in `generate/math_candidate_graph.py` after goal_residual, before rate short-circuits + +## 7. Before / after score + +| | Before | After | +|---|------:|------:| +| correct | 10 | **12** | +| refused | 40 | **38** | +| wrong | 0 | **0** | + +## 8. Newly solved IDs + +- `gsm8k-train-sample-v1-0003` +- `gsm8k-train-sample-v1-0021` + +## 9. Preserved solved IDs + +0002, 0008, 0014, 0018, 0024, 0025, 0029, 0037, 0038, 0042 + +## 10. Wrong=0 proof + +Full ephemeral train_sample: `12 / 38 / 0`. Holdout_dev scan: **0 admissions** for `resolve_promotable_question_bound_product`. + +## 11. Confuser matrix + +| Confuser | Result | +|----------|--------| +| Fraction surface (holdout 0345 shape) | refuse | +| Tom donates / Sam sells | refuse (seller≠supplier) | +| Additive distractor (loose pens) | refuse | +| Rate without money/weight target | refuse | +| `more than` comparative | refuse | +| Revenue without `in each` | refuse | +| Weight without `for` cue | refuse | +| Known product_bridge held-out hazard | refuse | + +## 12. Sibling generality proof + +- Revenue sibling: 12 cartons × 30 in each × $1.25 → 450 +- Weight sibling: 20 kg × 8 reps × 4 sets → 640 +- No case-id logic; no hardcoded answers + +## 13. product_bridge status + +**Broad `product_bridge` stays disabled** in `math_candidate_graph.py`. Function still resolves 0003/0021 in isolation; serving uses Gate A2f only. + +## 14–19. Confirmations + +- `report.json` untouched (historical 6/44/0 pin) +- Sealed-lane artifacts untouched +- No case-id logic +- No hardcoded final answers +- No direct-answer extraction +- **Non-goals:** broad product_bridge, report rebaseline, sealed movement, determine()/CLOSE, generic multiplication of co-occurring numbers + +## Validation + +```bash +git diff --check origin/main +pytest tests/test_math_candidate_graph_question_bound_product_lift.py -q +pytest tests/test_gsm8k_sealed_attempt_scout.py -q +pytest tests/test_gsm8k_frontier_report.py -q +pytest tests/test_gsm8k_post_gate_a1_frontier_microscope.py -q +pytest tests/test_math_candidate_graph_fraction_portion.py -q +pytest tests/test_math_candidate_graph_unit_partition_injection.py -q +pytest tests/test_math_candidate_graph_container_of_product.py -q +pytest tests/test_math_candidate_graph_peer_partition_question.py -q +pytest tests/test_math_candidate_graph_goal_residual_lift.py -q +pytest tests/test_adr_0195_product_bridge.py -q +pytest tests/test_composition_validation_corpus.py -q +# 151 passed +``` \ No newline at end of file diff --git a/generate/derivation/question_bound_product.py b/generate/derivation/question_bound_product.py new file mode 100644 index 00000000..8bca2439 --- /dev/null +++ b/generate/derivation/question_bound_product.py @@ -0,0 +1,309 @@ +"""Gate A2f — question-bound product aggregate (typed cross-clause / in-clause chain). + +Practice/scout evidence (Batch 5) showed train_sample **0003** and **0021** refuse on +serving because discrete_count_statement injection stops before a product chain can +commit, while sealed ``resolve_pooled`` already derives the correct readings. The +broad ``product_bridge`` wrapper over ``resolve_pooled`` was disabled after it +committed **5 wrong** on the sealed 1,319 — it admitted structurally similar but +non-product problems (fraction/comparative surfaces with a ``total weight`` target). + +This module is a **first-principles organ**, not a filter over the pooled reader. +It constructs only two explicit shapes: + +1. **Revenue chain** — container count × per-container count (``in each``) × unit + price (``$``/``dollar`` + ``each``, price may live in the question clause). +2. **Weight-work chain** — mass-per-rep × reps × sets in a **single** statement + clause, licensed by ``for``, question asks ``total``/``move`` + ``weight``. + +Promotion requires question-target binding, distinct non-empty units across factors +(where known), hazard refusal (fractions, comparatives, residual/profit cues), and +the unchanged self-verification gate (grounding ∧ cue ∧ unit ∧ completeness ∧ +uniqueness). Deterministic; sealed module (no ``chat/`` import). +""" + +from __future__ import annotations + +import re +from typing import Final + +from generate.derivation.clauses import segment_clauses +from generate.derivation.extract import extract_quantities +from generate.derivation.model import GroundedDerivation, Quantity, Step +from generate.derivation.multistep import search_chain +from generate.derivation.state.bind import leading_subject_token +from generate.derivation.target import _question_clause +from generate.derivation.verify import Resolution, select_self_verified +from generate.math_roundtrip import _tokens + +_FRACTION_RE: Final[re.Pattern[str]] = re.compile(r"\d+/\d+") +_COMPARATIVE_CUES: Final[frozenset[str]] = frozenset( + {"twice", "thrice", "double", "doubled", "half", "triple", "tripled"} +) +_MASS_UNITS: Final[frozenset[str]] = frozenset( + { + "gram", + "grams", + "kg", + "kilogram", + "kilograms", + "ounce", + "ounces", + "pound", + "pounds", + } +) +_REVENUE_QUESTION_VERBS: Final[frozenset[str]] = frozenset( + {"make", "earn", "raise", "collect"} +) +_WEIGHT_QUESTION_VERBS: Final[frozenset[str]] = frozenset( + {"total", "move", "lift", "press"} +) +_TEXT_BLOCKERS: Final[frozenset[str]] = frozenset( + { + "back", + "cover", + "covered", + "covers", + "eat", + "eats", + "gave", + "give", + "given", + "gives", + "insurance", + "kept", + "left", + "less", + "long", + "more", + "profit", + "remain", + "remaining", + "rest", + "same", + "spent", + "spend", + "subsequent", + } +) +_QUESTION_BLOCKERS: Final[frozenset[str]] = frozenset( + { + "after", + "before", + "left", + "long", + "per", + "profit", + "remaining", + } +) + + +def _asks_revenue_total(question_clause: str) -> bool: + tokens = _tokens(question_clause) + return "money" in tokens and bool(_REVENUE_QUESTION_VERBS & tokens) + + +def _asks_weight_total(question_clause: str) -> bool: + tokens = _tokens(question_clause) + return "weight" in tokens and bool(_WEIGHT_QUESTION_VERBS & tokens) + + +def _has_hazard_surface(problem_text: str, question_clause: str) -> bool: + if _FRACTION_RE.search(problem_text): + return True + text_tokens = _tokens(problem_text) + question_tokens = _tokens(question_clause) + if _COMPARATIVE_CUES & text_tokens: + return True + if "%" in problem_text or "percent" in text_tokens or "percentage" in text_tokens: + return True + if text_tokens & _TEXT_BLOCKERS: + return True + if question_tokens & _QUESTION_BLOCKERS: + return True + return False + + +def _distinct_known_units(derivation: GroundedDerivation) -> bool: + units = [derivation.start.unit, *(step.operand.unit for step in derivation.steps)] + non_empty = [unit for unit in units if unit] + return len(non_empty) == len(set(non_empty)) + + +_IMPERSONAL_SUBJECTS: Final[frozenset[str]] = frozenset( + { + "a", + "an", + "each", + "every", + "if", + "it", + "the", + "there", + "this", + "that", + "when", + "while", + } +) + + +def _named_actor_subjects(problem_text: str) -> set[str]: + """Collect likely named-actor subjects from quantity-bearing clauses.""" + subjects: set[str] = set() + for clause in segment_clauses(problem_text): + if not extract_quantities(clause): + continue + subject = leading_subject_token(clause) + if subject is None: + continue + key = subject.lower() + if key in _IMPERSONAL_SUBJECTS: + continue + subjects.add(key) + return subjects + + +_SELL_VERBS: Final[frozenset[str]] = frozenset({"sell", "sells", "sold", "selling"}) + + +def _question_named_seller(question_clause: str) -> str | None: + """Named seller in a conditional question clause, if present.""" + tokens = _tokens(question_clause) + if "if" not in tokens: + return None + if not (_SELL_VERBS & tokens): + return None + # ``If Sam sells ...`` — the token immediately after ``if``. + token_list = question_clause.split() + for index, word in enumerate(token_list): + if word.lower() == "if" and index + 1 < len(token_list): + candidate = token_list[index + 1].strip("',.") + key = candidate.lower() + if key not in _IMPERSONAL_SUBJECTS: + return key + return None + + +def _has_conflicting_actor_subjects(problem_text: str) -> bool: + """Refuse when named suppliers and sellers disagree across clauses.""" + question_clause = _question_clause(problem_text) + statement_actors = set() + for clause in segment_clauses(problem_text): + if clause == question_clause: + continue + statement_actors |= _named_actor_subjects(clause) + seller = _question_named_seller(question_clause) + if seller is not None and statement_actors and seller not in statement_actors: + return True + all_actors = _named_actor_subjects(problem_text) + return len(all_actors) > 1 + + +def _build_revenue_product_chain(problem_text: str) -> GroundedDerivation | None: + """Container × per-container (``in each``) × unit price — revenue target only.""" + question_clause = _question_clause(problem_text) + if not _asks_revenue_total(question_clause): + return None + + factors: list[Quantity] = [] + in_each_seen = False + price_seen = False + + for clause in segment_clauses(problem_text): + if clause == question_clause: + continue + quantities = extract_quantities(clause) + if len(quantities) != 1: + continue + factors.append(quantities[0]) + if "in each" in clause.lower(): + in_each_seen = True + + question_quantities = extract_quantities(question_clause) + if ( + len(question_quantities) == 1 + and ("$" in question_clause or "dollar" in _tokens(question_clause)) + and "each" in _tokens(question_clause) + ): + factors.append(question_quantities[0]) + price_seen = True + + if len(factors) != 3 or not in_each_seen or not price_seen: + return None + + start, *rest = factors + return GroundedDerivation( + start=start, + steps=tuple(Step(op="multiply", operand=q, cue="each") for q in rest), + ) + + +def _build_weight_product_chain(problem_text: str) -> GroundedDerivation | None: + """Mass-per-rep × reps × sets in one clause — total-weight target only.""" + question_clause = _question_clause(problem_text) + if not _asks_weight_total(question_clause): + return None + + statement_clauses = [ + clause + for clause in segment_clauses(problem_text) + if clause != question_clause and not clause.rstrip().endswith("?") + ] + quantity_clauses = [ + clause for clause in statement_clauses if len(extract_quantities(clause)) >= 2 + ] + if len(quantity_clauses) != 1: + return None + + clause = quantity_clauses[0] + quantities = extract_quantities(clause) + if len(quantities) != 3: + return None + if not (_MASS_UNITS & {quantities[0].unit}): + return None + if "for" not in _tokens(clause): + return None + + resolution = search_chain(clause) + if resolution is None: + return None + + derivation = resolution.derivation + if not all(step.op == "multiply" and step.cue == "for" for step in derivation.steps): + return None + if not _distinct_known_units(derivation): + return None + return derivation + + +def build_question_bound_product(problem_text: str) -> GroundedDerivation | None: + """Construct the ungated question-bound product chain, or ``None``.""" + question_clause = _question_clause(problem_text) + if _has_hazard_surface(problem_text, question_clause): + return None + if _has_conflicting_actor_subjects(problem_text): + return None + return _build_revenue_product_chain(problem_text) or _build_weight_product_chain( + problem_text + ) + + +def compose_question_bound_product(problem_text: str) -> Resolution | None: + """Gate the typed product chain through self-verification.""" + derivation = build_question_bound_product(problem_text) + if derivation is None: + return None + return select_self_verified([derivation], problem_text, target_units=()) + + +def resolve_promotable_question_bound_product(problem_text: str) -> Resolution | None: + """Serving promotion bridge (Gate A2f). + + Narrower than :func:`generate.derivation.product_bridge.resolve_promotable_product`: + does not call ``resolve_pooled``; admits only the two typed shapes above with + explicit question binding and hazard refusal. Held-out dev scan (2026-06-17): + 0 admissions on 500 cases; train_sample lifts 0003 + 0021 only. + """ + return compose_question_bound_product(problem_text) \ No newline at end of file diff --git a/generate/math_candidate_graph.py b/generate/math_candidate_graph.py index 716e84c7..86065344 100644 --- a/generate/math_candidate_graph.py +++ b/generate/math_candidate_graph.py @@ -712,6 +712,23 @@ def parse_and_solve(text: str, *, sealed: bool = False) -> CandidateGraphResult: branches_admissible=1, ) + # Gate A2f — question-bound product aggregate (typed revenue/weight chains). + # Paradigm Sprint 5: practice/scout lift for 0003/0021 without re-wiring the + # broad product_bridge over resolve_pooled (sealed 1,319 had 5 wrong). + from generate.derivation.question_bound_product import ( + resolve_promotable_question_bound_product, + ) + + product_resolution = resolve_promotable_question_bound_product(text) + if product_resolution is not None: + return CandidateGraphResult( + answer=product_resolution.answer, + selected_graph=None, + refusal_reason=None, + branches_enumerated=1, + branches_admissible=1, + ) + # ADR-0136.S.1 — Rate/event short-circuit paths (before Cartesian product). # Capacity path: single statement with one CandidateCapacity + matching question. if len(statement_sentences) == 1: diff --git a/tests/test_adr_0175_phase3b_mult_search.py b/tests/test_adr_0175_phase3b_mult_search.py index 6b326c33..6c414c96 100644 --- a/tests/test_adr_0175_phase3b_mult_search.py +++ b/tests/test_adr_0175_phase3b_mult_search.py @@ -114,8 +114,8 @@ class TestSealInvariant: build_search_report() # run practice with the search live counts = serving_report(_load_cases(_CASES_PATH))["counts"] assert counts["wrong"] == 0 - assert counts["correct"] >= 10 - assert counts["refused"] <= 40 + assert counts["correct"] >= 12 + assert counts["refused"] <= 38 def test_0050_canary_refuses_in_serving_and_is_not_attempted_wrong(self) -> None: from generate.math_candidate_graph import parse_and_solve diff --git a/tests/test_adr_0195_product_bridge.py b/tests/test_adr_0195_product_bridge.py index fe092ab5..702ff474 100644 --- a/tests/test_adr_0195_product_bridge.py +++ b/tests/test_adr_0195_product_bridge.py @@ -41,11 +41,10 @@ def test_promotable_product_cases_resolve(case_suffix: str, expected: float) -> assert resolution is not None assert resolution.answer == expected - # ...but its SERVING promotion is DISABLED (2026-06-04): the bridge committed - # 0 correct / 5 WRONG on the sealed 1,319, so serving now REFUSES rather than - # promote a product reading unsound on held-out. wrong=0 over train "correct". + # ...but broad product_bridge SERVING promotion stays DISABLED (2026-06-04). + # Gate A2f question_bound_product lifts 0003/0021 via a narrower typed organ. result = parse_and_solve(row["question"]) - assert result.answer is None + assert result.answer == expected @pytest.mark.parametrize( @@ -67,16 +66,13 @@ def test_known_pooled_wrong_commits_are_not_promotable(case_suffix: str) -> None def test_product_bridge_serving_promotion_is_disabled() -> None: - """DISABLED 2026-06-04: product_bridge commits 0 correct / 5 WRONG on held-out. - - Goal-residual (ADR-0207 R4) is re-wired separately; product_bridge stays off. - Train-sample 0003/0021 refuse; 0037 lifts via goal_residual only.""" + """Broad product_bridge stays off; Gate A2f typed organ promotes 0003/0021.""" report = build_report(_load_cases(_CASES_PATH)) counts = report["counts"] assert counts["wrong"] == 0 - assert counts["correct"] >= 10 + assert counts["correct"] >= 12 by_case = {row["case_id"]: row for row in report["per_case"]} - assert by_case["gsm8k-train-sample-v1-0003"]["verdict"] == "refused" - assert by_case["gsm8k-train-sample-v1-0021"]["verdict"] == "refused" + assert by_case["gsm8k-train-sample-v1-0003"]["verdict"] == "correct" + assert by_case["gsm8k-train-sample-v1-0021"]["verdict"] == "correct" assert by_case["gsm8k-train-sample-v1-0037"]["verdict"] == "correct" assert by_case["gsm8k-train-sample-v1-0050"]["verdict"] == "refused" diff --git a/tests/test_composition_validation_corpus.py b/tests/test_composition_validation_corpus.py index ab48c4ea..d6dcc382 100644 --- a/tests/test_composition_validation_corpus.py +++ b/tests/test_composition_validation_corpus.py @@ -15,7 +15,7 @@ It enforces two kinds of obligation: - frozen baseline snapshot for those non-positive rows matches the live tree. * **Current snapshot** (the one assertion a Phase 5b slice updates when it - flips a positive): the aggregate is ``8 solve / 14 refuse / 0 wrong`` today. + flips a positive): the aggregate is ``9 solve / 13 refuse / 0 wrong`` today. A future positive (``gate`` like ``5b-R1``) is *expected* to flip refuse -> solve when its slice lands; that flip must still satisfy the firewall, @@ -163,8 +163,8 @@ def test_current_baseline_snapshot() -> None: """Current aggregate snapshot for the 22-row composition-validation corpus. Updates when a Phase 5b / capability-strike slice flips a positive row. - Batch 4 (2026-06-17): goal_residual serving promotion re-wired for cv-0005; - prior Gate A1/A2 lifts already admit cv-0001/0002/0003/0009 on live code. + Batch 5 (2026-06-17): Gate A2f question_bound_product admits cv-0020 (0021); + Batch 4 goal_residual admits cv-0005; prior Gate A1/A2 lifts admit others. """ solve = refuse = wrong = 0 for case in _CASES: @@ -176,7 +176,7 @@ def test_current_baseline_snapshot() -> None: else: refuse += 1 assert wrong == 0 - assert (solve, refuse) == (8, 14), ( + assert (solve, refuse) == (9, 13), ( f"snapshot moved to {solve} solve / {refuse} refuse — if a Phase 5b " f"slice landed, update this expectation and the affected rows' " f"baseline fields in lockstep" diff --git a/tests/test_gsm8k_post_gate_a1_frontier_microscope.py b/tests/test_gsm8k_post_gate_a1_frontier_microscope.py index 25686504..fb7bdebe 100644 --- a/tests/test_gsm8k_post_gate_a1_frontier_microscope.py +++ b/tests/test_gsm8k_post_gate_a1_frontier_microscope.py @@ -75,11 +75,11 @@ def test_live_microscope_refusal_partition_is_complete(): def test_live_microscope_partition_seed_case_is_tagged(): + """0003 is solved by Gate A2f; partition_chunking still tags refused peers.""" summary = build_microscope_report(_load_cases()) - assert ( - "gsm8k-train-sample-v1-0003" - in summary["implementation_slice_candidates"]["partition_chunking"]["case_ids"] - ) + partition_ids = summary["implementation_slice_candidates"]["partition_chunking"]["case_ids"] + assert "gsm8k-train-sample-v1-0003" not in {r["case_id"] for r in summary["refusal_table"]} + assert len(partition_ids) >= 1 def test_microscope_output_is_deterministic(): @@ -104,17 +104,19 @@ def test_markdown_render_surfaces_partition_candidate(): summary = build_microscope_report(_load_cases()) md = render_markdown(summary) assert "partition_chunking" in md - assert "| 0003 |" in md + assert "correct: 12" in md assert "Gate A2a unit_partition" in md def test_gate_a2_lifts_are_not_in_refusal_table(): - """Cases solved by Gate A2b/A2c/A2d/A2e must not appear among live refusals.""" + """Cases solved by Gate A2b–A2f must not appear among live refusals.""" summary = build_microscope_report(_load_cases()) refused_ids = {r["case_id"] for r in summary["refusal_table"]} assert "gsm8k-train-sample-v1-0002" not in refused_ids + assert "gsm8k-train-sample-v1-0003" not in refused_ids assert "gsm8k-train-sample-v1-0008" not in refused_ids + assert "gsm8k-train-sample-v1-0021" not in refused_ids assert "gsm8k-train-sample-v1-0025" not in refused_ids assert "gsm8k-train-sample-v1-0037" not in refused_ids - assert summary["counts"]["correct"] >= 10 + assert summary["counts"]["correct"] >= 12 assert summary["closed_injector_buckets"]["unit_partition_no_injection"] == 0 diff --git a/tests/test_math_candidate_graph_goal_residual_lift.py b/tests/test_math_candidate_graph_goal_residual_lift.py index ca164cce..b4ac0c6a 100644 --- a/tests/test_math_candidate_graph_goal_residual_lift.py +++ b/tests/test_math_candidate_graph_goal_residual_lift.py @@ -70,9 +70,9 @@ def test_confuser_comparative_more_than_refuses(): assert compose_goal_residual(text) is None -def test_product_bridge_cases_still_refuse(): - """product_bridge stays disabled — 0003/0021 must not lift via this path.""" - for text in ( +def test_product_bridge_broad_path_not_serving_promoted(): + """Broad product_bridge stays disabled; Gate A2f lifts 0003/0021 instead.""" + texts = ( ( "The student council sells scented erasers in the morning before school starts " "to help raise money for school dances. The local bookstore donated 48 boxes of " @@ -83,9 +83,11 @@ def test_product_bridge_cases_still_refuse(): "John is lifting weights. He bench presses 15 pounds for 10 reps and does 3 sets. " "How much total weight does he move?" ), - ): + ) + expected = (864.0, 450.0) + for text, ans in zip(texts, expected, strict=True): res = _run(text) - assert res.answer is None + assert res.answer == ans def test_peer_pick_regression_0025(): diff --git a/tests/test_math_candidate_graph_question_bound_product_lift.py b/tests/test_math_candidate_graph_question_bound_product_lift.py new file mode 100644 index 00000000..20ac0e58 --- /dev/null +++ b/tests/test_math_candidate_graph_question_bound_product_lift.py @@ -0,0 +1,183 @@ +"""Gate A2f — question_bound_product_aggregate serving lift (Paradigm Sprint 5).""" + +from __future__ import annotations + +import json +from pathlib import Path + +from generate.math_candidate_graph import parse_and_solve +from generate.derivation.question_bound_product import ( + compose_question_bound_product, + resolve_promotable_question_bound_product, +) + +_CASES_PATH = ( + Path(__file__).resolve().parents[1] + / "evals" + / "gsm8k_math" + / "train_sample" + / "v1" + / "cases.jsonl" +) + +CASE_0003 = ( + "The student council sells scented erasers in the morning before school starts to help " + "raise money for school dances. The local bookstore donated 48 boxes of erasers. " + "There are 24 erasers in each box. If the student council sells the erasers for " + "$0.75 each, how much money will they make?" +) + +CASE_0021 = ( + "John is lifting weights. He bench presses 15 pounds for 10 reps and does 3 sets. " + "How much total weight does he move?" +) + +HOLDOUT_WRONG_SHAPE = ( + "Penny's canoe can carry 6 people, but if she wants to take her dog, she will only " + "fit 2/3 of that number inside. If every person in a trip where Penny had her dog " + "inside the canoe weighed 140 pounds, and the dog 1/4 as much weight, calculate the " + "total weight the canoe was carrying?" +) + + +def _run(text: str): + return parse_and_solve(text, sealed=False) + + +def test_train_sample_0003_end_to_end(): + res = _run(CASE_0003) + assert res.answer == 864.0 + assert res.refusal_reason is None + + +def test_train_sample_0021_end_to_end(): + res = _run(CASE_0021) + assert res.answer == 450.0 + assert res.refusal_reason is None + + +def test_sibling_revenue_chain_varied_surface(): + text = ( + "The library donated 12 cartons of markers for a fundraiser. " + "There are 30 markers in each carton. " + "If the club sells every marker for $1.25 each, how much money will they raise?" + ) + res = _run(text) + assert res.answer == 450.0 + assert res.refusal_reason is None + + +def test_sibling_weight_chain_varied_surface(): + text = ( + "Ana is training. She squats 20 kilograms for 8 reps and completes 4 sets. " + "How much total weight does she lift?" + ) + res = _run(text) + assert res.answer == 640.0 + assert res.refusal_reason is None + + +def test_confuser_fraction_surface_refuses(): + assert resolve_promotable_question_bound_product(HOLDOUT_WRONG_SHAPE) is None + assert _run(HOLDOUT_WRONG_SHAPE).answer is None + + +def test_confuser_ambiguous_actor_refuses(): + text = ( + "Tom donated 10 crates of books. There are 5 books in each crate. " + "If Sam sells the books for $2 each, how much money will they make?" + ) + assert _run(text).answer is None + + +def test_confuser_additive_distractor_refuses(): + text = ( + "The shop has 20 boxes of pens and 3 loose pens. " + "There are 10 pens in each box. " + "If pens sell for $1 each, how much money will they make?" + ) + assert _run(text).answer is None + + +def test_confuser_rate_without_target_refuses(): + text = ( + "She runs 6 miles per hour for 2 hours. " + "How many miles does she run?" + ) + assert _run(text).answer is None + + +def test_confuser_reversed_relation_more_than_refuses(): + text = ( + "Lisa has 5 more boxes than Tom. Tom has 10 boxes. " + "There are 8 toys in each box. " + "If toys sell for $3 each, how much money will they make?" + ) + assert _run(text).answer is None + + +def test_confuser_money_without_in_each_refuses(): + text = ( + "The store has 40 items. Items sell for $2 each. " + "How much money will they make?" + ) + assert _run(text).answer is None + + +def test_confuser_weight_without_for_cue_refuses(): + text = ( + "Kai lifts 12 pounds, 8 reps, and 2 sets during practice. " + "How much total weight does he move?" + ) + assert _run(text).answer is None + + +def test_product_bridge_stays_disabled(): + """Broad product_bridge is not re-wired; this organ is narrower.""" + from generate.derivation.product_bridge import resolve_promotable_product + + assert resolve_promotable_product(CASE_0003) is not None + assert resolve_promotable_product(CASE_0021) is not None + assert compose_question_bound_product(CASE_0003) is not None + assert compose_question_bound_product(CASE_0021) is not None + + +def test_goal_residual_and_peer_partition_regressions(): + goal = ( + "Michael wants to lose 10 pounds by June. He lost 3 pounds in March and 4 pounds " + "in April. How much weight does he have to lose in May to meet his goal?" + ) + peer = ( + "Lilibeth and her friends go strawberry picking. " + "Lilibeth fills 6 baskets where each basket holds 50 strawberries. " + "If three of Lilibeth's friends pick the same amount as her, " + "how many strawberries do Lilibeth and her friends pick in all?" + ) + assert _run(goal).answer == 3.0 + assert _run(peer).answer == 1200.0 + + +def test_full_train_sample_wrong_zero_and_chunk_lift(): + from evals.gsm8k_math.train_sample.v1.runner import build_report, _load_cases + + report = build_report(_load_cases(_CASES_PATH)) + counts = report["counts"] + assert counts["wrong"] == 0 + assert counts["correct"] == 12 + assert counts["refused"] == 38 + by_case = {row["case_id"]: row for row in report["per_case"]} + assert by_case["gsm8k-train-sample-v1-0003"]["verdict"] == "correct" + assert by_case["gsm8k-train-sample-v1-0021"]["verdict"] == "correct" + for case_id in ( + "gsm8k-train-sample-v1-0002", + "gsm8k-train-sample-v1-0008", + "gsm8k-train-sample-v1-0014", + "gsm8k-train-sample-v1-0018", + "gsm8k-train-sample-v1-0024", + "gsm8k-train-sample-v1-0025", + "gsm8k-train-sample-v1-0029", + "gsm8k-train-sample-v1-0037", + "gsm8k-train-sample-v1-0038", + "gsm8k-train-sample-v1-0042", + ): + assert by_case[case_id]["verdict"] == "correct", case_id \ No newline at end of file