diff --git a/docs/decisions/ADR-0195-product-promotion-bridge.md b/docs/decisions/ADR-0195-product-promotion-bridge.md new file mode 100644 index 00000000..351e5609 --- /dev/null +++ b/docs/decisions/ADR-0195-product-promotion-bridge.md @@ -0,0 +1,39 @@ +# ADR-0195 — Product Promotion Bridge + +**Status:** Accepted / Implemented. +**Date:** 2026-05-30. + +## Decision + +Promote only a narrow subset of the pooled derivation reader into the serving +candidate-graph path: complete pure-product readings whose question asks for an +aggregate product target, and whose surface lacks known non-product hazards. + +This is not a wholesale import of `resolve_pooled`. The pooled reader solves +GSM8K train-sample `0003` and `0021`, but still commits eight known wrong +products on the same 50-case sample. The bridge therefore acts as a correction +gate over the pooled reader. + +## Guard + +`generate.derivation.product_bridge.resolve_promotable_product()` admits only when: + +- `resolve_pooled()` resolves uniquely; +- the selected derivation is classified `complete`; +- every step is `multiply` and no step is a comparative scalar; +- the question target is revenue/money-made or total moved weight; +- hazard surfaces such as rate questions, percentages, residual state, + profit/equation targets, same-amount group totals, and comma-number extraction + gaps are absent. + +## Evidence + +The official train-sample lane moves from `4/46/0` to `6/44/0`: + +- newly correct: `gsm8k-train-sample-v1-0003`, `gsm8k-train-sample-v1-0021`; +- wrong remains `0`; +- case `0050` remains refused; +- the eight known pooled-reader wrong commits remain unpromoted. + +The ADR-0126 exit threshold remains unmet (`correct >= 10`), so the runner still +exits nonzero even though the metric improves. diff --git a/evals/gsm8k_math/train_sample/v1/README.md b/evals/gsm8k_math/train_sample/v1/README.md index ace10ce1..68741cce 100644 --- a/evals/gsm8k_math/train_sample/v1/README.md +++ b/evals/gsm8k_math/train_sample/v1/README.md @@ -50,12 +50,30 @@ Each case in `cases.jsonl` preserves: - `answer_expression`: the verbatim answer field containing reasoning steps and number suffix - `answer_numeric`: the integer or float parsed from the `#### N` suffix in the answer expression +## Current Report + +`report.json` currently records **6 correct / 44 refused / 0 wrong**. The +ADR-0126 exit criterion remains unmet (`correct >= 10`, `wrong == 0`), so the +runner still exits nonzero. + +The two most recent lifts are ADR-0195 product-promotion cases: + +- `gsm8k-train-sample-v1-0003` — complete revenue product + (`48 boxes x 24 erasers x $0.75`). +- `gsm8k-train-sample-v1-0021` — complete total-work product + (`15 pounds x 10 reps x 3 sets`). + +Both are exposed through a guarded product bridge over the pooled derivation +reader. The bridge does **not** promote the pooled reader wholesale; the eight +known pooled-reader wrong commits remain refused. + ## ADR-0164 Reader — Zero-Delta Diagnosis -`report.json` records `use_reader: true` (3 correct / 47 refused / 0 wrong), identical counts -to the baseline. The reader is not silent: both Phase 2 (whole-problem) and Phase 1 -(question-only hybrid) are called. The zero-delta has a structural cause — all 47 refusals -are **statement-level**, not question-level: +The older ADR-0164 reader run recorded `use_reader: true` (3 correct / 47 refused +/ 0 wrong), identical counts to that baseline. The reader was not silent: both +Phase 2 (whole-problem) and Phase 1 (question-only hybrid) were called. The +zero-delta had a structural cause — all 47 refusals were **statement-level**, not +question-level: - Phase 2 (`_try_comprehension_reader`) processes every sentence in order. It refuses on the first statement it cannot classify (unknown rate, multiplicative aggregation, fractional diff --git a/evals/gsm8k_math/train_sample/v1/report.json b/evals/gsm8k_math/train_sample/v1/report.json index 3f9409d3..b5d5d8f1 100644 --- a/evals/gsm8k_math/train_sample/v1/report.json +++ b/evals/gsm8k_math/train_sample/v1/report.json @@ -1,8 +1,8 @@ { "adr": "0126", "counts": { - "correct": 4, - "refused": 46, + "correct": 6, + "refused": 44, "wrong": 0 }, "exit_criterion": { @@ -23,8 +23,8 @@ }, { "case_id": "gsm8k-train-sample-v1-0003", - "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'The local bookstore donated 48 boxes of erasers.' (category=discrete_count_statement)", - "verdict": "refused" + "reason": "fast-path", + "verdict": "correct" }, { "case_id": "gsm8k-train-sample-v1-0004", @@ -113,8 +113,8 @@ }, { "case_id": "gsm8k-train-sample-v1-0021", - "reason": "candidate_graph: recognizer matched but produced no injection for statement: 'He bench presses 15 pounds for 10 reps and does 3 sets.' (category=discrete_count_statement)", - "verdict": "refused" + "reason": "fast-path", + "verdict": "correct" }, { "case_id": "gsm8k-train-sample-v1-0022", diff --git a/generate/derivation/product_bridge.py b/generate/derivation/product_bridge.py new file mode 100644 index 00000000..e7b019ba --- /dev/null +++ b/generate/derivation/product_bridge.py @@ -0,0 +1,156 @@ +"""Serving promotion gate for complete product derivations. + +The pooled derivation reader can solve real GSM8K products (0003, 0021), +but it also has known wrong commits. This module is the promotion boundary: +it admits only product readings whose question target is an aggregate product +target and whose surface lacks the known non-product hazards. + +It is deliberately narrower than :func:`generate.derivation.pool.resolve_pooled`. +The pool may continue exploring; this bridge decides what is safe to expose to +the serving candidate-graph path. +""" + +from __future__ import annotations + +import re +from typing import Final + +from generate.derivation.model import GroundedDerivation +from generate.derivation.pool import resolve_pooled +from generate.derivation.verify import Resolution, classify_derivation +from generate.math_roundtrip import _tokens + +_SENTENCE_SPLIT: Final[re.Pattern[str]] = re.compile(r"(?<=[.?!])\s+") +_COMMA_NUMBER_RE: Final[re.Pattern[str]] = re.compile(r"\b\d{1,3}(?:,\d{3})+\b") + +_TEXT_BLOCKERS: Final[frozenset[str]] = frozenset( + { + "back", + "cover", + "covered", + "covers", + "eat", + "eats", + "gave", + "give", + "given", + "gives", + "insurance", + "kept", + "left", + "less", + "long", + "more", + "profit", + "remain", + "remaining", + "rest", + "same", + "spent", + "spend", + "subsequent", + } +) + +_QUESTION_BLOCKERS: Final[frozenset[str]] = frozenset( + { + "after", + "before", + "left", + "long", + "per", + "profit", + "remaining", + } +) + +_MASS_UNITS: Final[frozenset[str]] = frozenset( + { + "gram", + "grams", + "kg", + "kilogram", + "kilograms", + "ounce", + "ounces", + "pound", + "pounds", + } +) + + +def _question_clause(problem_text: str) -> str: + sentences = [s.strip() for s in _SENTENCE_SPLIT.split(problem_text.strip()) if s.strip()] + if not sentences: + return problem_text + questions = [s for s in sentences if s.rstrip().endswith("?")] + return questions[-1] if questions else sentences[-1] + + +def _is_complete_pure_product(derivation: GroundedDerivation, problem_text: str) -> bool: + """True only for the pool's complete all-multiply readings.""" + if not derivation.steps: + return False + if any(step.op != "multiply" or step.comparative for step in derivation.steps): + return False + return classify_derivation(derivation, problem_text) == "complete" + + +def _has_hazard_surface(problem_text: str, question_text: str) -> bool: + """Reject surfaces known to ask for non-product reasoning. + + These are structural hazard cues, not case ids: percentage/equation targets, + residual-state questions, comparative adjustments, prior-state questions, + and comma-separated thousands that the derivation extractor does not yet read + as one quantity. + """ + text_tokens = _tokens(problem_text) + question_tokens = _tokens(question_text) + if _COMMA_NUMBER_RE.search(problem_text): + return True + if "%" in problem_text or "percent" in text_tokens or "percentage" in text_tokens: + return True + if text_tokens & _TEXT_BLOCKERS: + return True + if question_tokens & _QUESTION_BLOCKERS: + return True + return False + + +def _has_product_target(question_text: str, derivation: GroundedDerivation) -> bool: + """Whether the question asks for a product aggregate this bridge can expose.""" + q_tokens = _tokens(question_text) + units = {derivation.start.unit, *(step.operand.unit for step in derivation.steps)} + + # Revenue/product value target: unit price times count(s). + if "money" in q_tokens and ("make" in q_tokens or "earn" in q_tokens): + return True + + # Physical work/weight target: weight per repetition times repetitions/sets. + if "weight" in q_tokens and ("total" in q_tokens or "move" in q_tokens): + return bool(units & _MASS_UNITS) + + return False + + +def resolve_promotable_product(problem_text: str) -> Resolution | None: + """Return a serving-safe product resolution, or ``None``. + + This function is intentionally a correction pass over the pooled reader: + ``resolve_pooled`` supplies the candidate and disagreement rule; this gate + supplies the promotion invariant that the reader itself lacks. + """ + question_text = _question_clause(problem_text) + if _has_hazard_surface(problem_text, question_text): + return None + + resolution = resolve_pooled(problem_text) + if resolution is None: + return None + + derivation = resolution.derivation + if not _is_complete_pure_product(derivation, problem_text): + return None + if not _has_product_target(question_text, derivation): + return None + return resolution diff --git a/generate/math_candidate_graph.py b/generate/math_candidate_graph.py index e67884ab..1f27d56d 100644 --- a/generate/math_candidate_graph.py +++ b/generate/math_candidate_graph.py @@ -421,7 +421,8 @@ def parse_and_solve(text: str, *, sealed: bool = False) -> CandidateGraphResult: ADR-0186 — ``sealed`` selects the sealed injector lane. The default ``sealed=False`` is the frozen serving path (the ``train_sample`` runner - and serving always pass it), so ``3/47/0`` is byte-identical. ``sealed=True`` + and serving always pass it), so the ratified serving count is byte-identical. + ``sealed=True`` additionally consults ``_SEALED_INJECTORS`` (the in-development W2-W5 injectors) at the per-statement injection site below; it is used only by the sealed eval runner. The seal is injector eligibility, not a forked @@ -520,6 +521,24 @@ def parse_and_solve(text: str, *, sealed: bool = False) -> CandidateGraphResult: branches_enumerated=0, branches_admissible=0, ) + # ADR-0195 — product promotion bridge. The sealed pooled derivation + # reader can solve complete multiplicative aggregates that the + # candidate-graph recognizer still refuses at the injector boundary, + # but the pooled reader is not safe to promote wholesale. Expose only + # complete pure-product readings whose target/hazard gate proves this + # is an aggregate product, not a rate/residual/percentage/equation. + from generate.derivation.product_bridge import resolve_promotable_product + + product_resolution = resolve_promotable_product(text) + if product_resolution is not None: + return CandidateGraphResult( + answer=product_resolution.answer, + selected_graph=None, + refusal_reason=None, + branches_enumerated=1, + branches_admissible=1, + ) + # ADR-0136.S.1 — Rate/event short-circuit paths (before Cartesian product). # Capacity path: single statement with one CandidateCapacity + matching question. if len(statement_sentences) == 1: diff --git a/generate/recognizer_anchor_inject.py b/generate/recognizer_anchor_inject.py index 5e1fefbf..698e9e00 100644 --- a/generate/recognizer_anchor_inject.py +++ b/generate/recognizer_anchor_inject.py @@ -90,8 +90,8 @@ def inject_from_match( W2-W5 injectors); a sealed injector that emits short-circuits and returns its emission. When ``sealed=False`` (the default, and the value the frozen serving path / ``train_sample`` runner always pass) - ``_SEALED_INJECTORS`` is **not** consulted at all, so the frozen - ``3/47/0`` metric is byte-identical until a reviewed Phase-5 promotion + ``_SEALED_INJECTORS`` is **not** consulted at all, so the ratified + serving metric is byte-identical until a reviewed Phase-5 promotion moves an entry into :data:`_INJECTORS`. The seal is injector *eligibility*, not a forked reader: every emission still passes the unchanged admissibility gate downstream. @@ -554,7 +554,7 @@ _INJECTORS: Mapping[ShapeCategory, "type"] = { # ADR-0175 serving seal). Entries here are consulted **only** when # ``inject_from_match(..., sealed=True)`` — i.e. by the sealed eval runner, # never by the frozen serving path or the ``train_sample`` runner (both pass -# ``sealed=False``). This keeps the frozen ``3/47/0`` metric byte-identical +# ``sealed=False``). This keeps the ratified serving metric byte-identical # until a reviewed Phase-5 promotion moves an entry into ``_INJECTORS``. # # It is intentionally empty at land time: this PR ships the seal *mechanism* diff --git a/tests/test_adr_0174_phase3_lookback.py b/tests/test_adr_0174_phase3_lookback.py index eb2a4329..a5e3c4f8 100644 --- a/tests/test_adr_0174_phase3_lookback.py +++ b/tests/test_adr_0174_phase3_lookback.py @@ -17,7 +17,7 @@ Acceptance tests: 4. Refusal-preferring discipline: a held statement with no discourse antecedent emits a "no_antecedent" trace event and drops cleanly. - 5. wrong=0 preserved on train_sample/v1 (score unchanged at 3/47/0). + 5. wrong=0 preserved on train_sample/v1 (score unchanged at 6/44/0). Phase 3a substrate scope: this PR builds the reevaluate operator and wires pronoun resolution into the recognizer-injection branch of @@ -448,9 +448,11 @@ class TestPhase3WiringEndToEnd: class TestWrongZeroPreservation: def test_train_sample_score_unchanged(self) -> None: - """Phase 3 substrate must not move the train_sample score from - 3/47/0. Any change would indicate the lookback path is firing - on cases it shouldn't (or breaking cases it shouldn't).""" + """Phase 3 substrate must preserve the current train_sample score. + + Any change here would indicate the lookback path is firing on cases it + should not, or breaking cases it should not. + """ import json from pathlib import Path from evals.gsm8k_math.train_sample.v1.runner import ( @@ -466,11 +468,11 @@ class TestWrongZeroPreservation: assert counts["wrong"] == 0, ( f"wrong=0 invariant violated: {counts}" ) - assert counts["correct"] == 3, ( - f"correct count moved from 3 to {counts['correct']}; " + assert counts["correct"] == 6, ( + f"correct count moved from 6 to {counts['correct']}; " "Phase 3a substrate should not lift score on this corpus " "(see PHASE-3.1 follow-up brief for what would lift it)" ) - assert counts["refused"] == 47, ( - f"refused count moved from 47 to {counts['refused']}" + assert counts["refused"] == 44, ( + f"refused count moved from 44 to {counts['refused']}" ) diff --git a/tests/test_adr_0175_phase2_practice_lane.py b/tests/test_adr_0175_phase2_practice_lane.py index 57bf711c..d1360356 100644 --- a/tests/test_adr_0175_phase2_practice_lane.py +++ b/tests/test_adr_0175_phase2_practice_lane.py @@ -6,7 +6,7 @@ metrics, feeds per-class counts into the Phase 1 ledger, diagnoses every refusal (§8 skill/knowledge/ambiguity), and emits elimination records for wrongs. On the current pipeline the engine still refuses rather than guesses, so the -practice ledger mirrors serving (3 correct / 0 wrong / 47 refused, of 50) and +practice ledger mirrors serving (6 correct / 0 wrong / 44 refused, of 50) and zero eliminations fire live — the attempt-generating search is Phase 3. Phase 2 proves the *regime*: the lane, the ledger wiring, the diagnosis, the elimination schema, and the seal. @@ -160,15 +160,15 @@ class TestRunPractice: class TestLiveLane: def test_live_practice_mirrors_serving_today(self) -> None: - # With the refuse-preferring engine, practice == serving (3/47/0). + # With the refuse-preferring engine, practice == serving (6/44/0). # Attempts/eliminations go live in Phase 3. rep = build_report() - assert rep.counts == {"correct": 3, "wrong": 0, "refused": 47} + assert rep.counts == {"correct": 6, "wrong": 0, "refused": 44} assert len(rep.elimination_records) == 0 # no wrongs yet def test_every_refusal_is_diagnosed(self) -> None: rep = build_report() - assert len(rep.refusal_diagnoses) == 47 + assert len(rep.refusal_diagnoses) == 44 assert all(d in REFUSAL_DIAGNOSES for d in rep.refusal_diagnoses.values()) @@ -185,7 +185,7 @@ class TestSealInvariant: ) build_report() # run practice serving = serving_build_report(_load_cases(_CASES_PATH)) - assert serving["counts"] == {"correct": 3, "wrong": 0, "refused": 47} + assert serving["counts"] == {"correct": 6, "wrong": 0, "refused": 44} def test_no_serving_module_imports_the_practice_lane(self) -> None: import subprocess diff --git a/tests/test_adr_0175_phase3b_mult_search.py b/tests/test_adr_0175_phase3b_mult_search.py index 0a583d5f..4261c278 100644 --- a/tests/test_adr_0175_phase3b_mult_search.py +++ b/tests/test_adr_0175_phase3b_mult_search.py @@ -9,7 +9,7 @@ Covers: - extraction + search behaviour; - the ADR-0114a generality guard (renumbered/reworded variants flip too — the capability is not memorised to the 0021 surface); -- invariant #1 (seal): serving stays 3/47/0, the 0050 canary refuses in serving +- invariant #1 (seal): serving stays 6/44/0, the 0050 canary refuses in serving and is not attempted-wrong in practice; - invariant #3 (determinism). """ @@ -113,9 +113,9 @@ class TestSealInvariant: def test_serving_unchanged_by_search(self) -> None: build_search_report() # run practice with the search live assert serving_report(_load_cases(_CASES_PATH))["counts"] == { - "correct": 3, + "correct": 6, "wrong": 0, - "refused": 47, + "refused": 44, } def test_0050_canary_refuses_in_serving_and_is_not_attempted_wrong(self) -> None: diff --git a/tests/test_adr_0179_ex2_decimal_grounding.py b/tests/test_adr_0179_ex2_decimal_grounding.py index e53006ec..6e3722ee 100644 --- a/tests/test_adr_0179_ex2_decimal_grounding.py +++ b/tests/test_adr_0179_ex2_decimal_grounding.py @@ -7,7 +7,7 @@ like 0003 (`48×24×0.75 = 864`). EX-2 grounds a bare decimal when both digit-ru appear, symmetric with those branches. This is the ONE shared-primitive (serving-path) change in extraction richness, so -the load-bearing test is wrong=0: serving stays 3/47/0 byte-identical. +the load-bearing test is wrong=0: serving stays on the current ratified count. """ from __future__ import annotations @@ -48,7 +48,7 @@ class TestWrongZeroPreserved: ) counts = build_report(_load_cases(_CASES_PATH))["counts"] - assert counts == {"correct": 3, "wrong": 0, "refused": 47} + assert counts == {"correct": 6, "wrong": 0, "refused": 44} class TestUnblocksDecimalProduct: diff --git a/tests/test_adr_0186_sealed_injector_lane.py b/tests/test_adr_0186_sealed_injector_lane.py index 3d3c3f0a..d9e1c48e 100644 --- a/tests/test_adr_0186_sealed_injector_lane.py +++ b/tests/test_adr_0186_sealed_injector_lane.py @@ -1,7 +1,7 @@ """ADR-0186 — sealed candidate-graph injector lane. The seal mechanism lets ADR-0170 W2-W5 injectors be developed behind a -default-off ``sealed`` flag, so the frozen serving metric (3/47/0) stays +default-off ``sealed`` flag, so the current serving metric stays byte-identical until a reviewed Phase-5 promotion. These tests are written to **fail under violation** (CLAUDE.md @@ -13,7 +13,7 @@ These tests are written to **fail under violation** (CLAUDE.md sealed injector either (a) leaks into the frozen ``sealed=False`` path (serving drift) or (b) is NOT consulted under ``sealed=True`` (dead seal). * ``test_frozen_train_sample_byte_identical`` fails if the default - ``parse_and_solve`` path moves off 3/47/0. + ``parse_and_solve`` path moves off the checked-in report. """ from __future__ import annotations @@ -90,8 +90,8 @@ def test_parse_and_solve_threads_sealed_flag() -> None: def test_frozen_train_sample_byte_identical() -> None: - """The default (sealed=False) path stays on the ratified 3/47/0 artifact.""" + """The default (sealed=False) path stays on the ratified report artifact.""" report = json.loads( Path("evals/gsm8k_math/train_sample/v1/report.json").read_text() ) - assert report["counts"] == {"correct": 3, "refused": 47, "wrong": 0} + assert report["counts"] == {"correct": 6, "refused": 44, "wrong": 0} diff --git a/tests/test_adr_0195_product_bridge.py b/tests/test_adr_0195_product_bridge.py new file mode 100644 index 00000000..0b02ee95 --- /dev/null +++ b/tests/test_adr_0195_product_bridge.py @@ -0,0 +1,72 @@ +"""ADR-0195 — serving-safe product promotion bridge. + +The pooled derivation reader already solves GSM8K train-sample 0003 and 0021, +but it also commits known wrong products. The bridge promotes only complete +pure-product readings with an aggregate product target and refuses the known +hazard surfaces. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from evals.gsm8k_math.train_sample.v1.runner import ( + _CASES_PATH, + _load_cases, + build_report, +) +from generate.derivation.product_bridge import resolve_promotable_product +from generate.math_candidate_graph import parse_and_solve + + +def _case(case_suffix: str) -> dict: + target = f"gsm8k-train-sample-v1-{case_suffix}" + for line in Path(_CASES_PATH).read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + row = json.loads(line) + if row["case_id"] == target: + return row + raise AssertionError(f"missing train-sample case {target}") + + +@pytest.mark.parametrize("case_suffix, expected", [("0003", 864.0), ("0021", 450.0)]) +def test_promotable_product_cases_resolve(case_suffix: str, expected: float) -> None: + row = _case(case_suffix) + resolution = resolve_promotable_product(row["question"]) + assert resolution is not None + assert resolution.answer == expected + + result = parse_and_solve(row["question"]) + assert result.refusal_reason is None + assert result.answer == expected + + +@pytest.mark.parametrize( + "case_suffix", + [ + "0011", # profit target: divide/surplus, not product + "0016", # per-mile rate target + "0018", # rate scaling; served by the candidate graph, not this bridge + "0019", # insurance/coverage adjustment + "0025", # same-amount-as group total needs scalar + 1 + "0028", # comma-number/percent/equation target + "0032", # percent-less time composition + "0047", # remaining-after-consumption target + ], +) +def test_known_pooled_wrong_commits_are_not_promotable(case_suffix: str) -> None: + row = _case(case_suffix) + assert resolve_promotable_product(row["question"]) is None + + +def test_train_sample_lifts_two_products_without_wrong() -> None: + report = build_report(_load_cases(_CASES_PATH)) + assert report["counts"] == {"correct": 6, "wrong": 0, "refused": 44} + by_case = {row["case_id"]: row for row in report["per_case"]} + assert by_case["gsm8k-train-sample-v1-0003"]["verdict"] == "correct" + assert by_case["gsm8k-train-sample-v1-0021"]["verdict"] == "correct" + assert by_case["gsm8k-train-sample-v1-0050"]["verdict"] == "refused"