From 872ed3b52d8afdcb82465dd5b00487df1e9f6e61 Mon Sep 17 00:00:00 2001 From: Shay Date: Thu, 28 May 2026 15:29:08 -0700 Subject: [PATCH] feat(adr-0175-phase3b): bounded multiplicative search in the sealed practice lane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR-0175 Phase 3b — the first live attempt generator. Runs only in the sealed practice lane, only on cases the engine refused; every proposal is gated by the Phase 3a self-verification gate. generate/derivation/: - extract.py: extract_quantities() — lexeme-level (number + unit word; ADR-0165). - search.py: search_multiplicative() — one in-clause product candidate per sentence with >=2 quantities + a present multiplicative cue; gated by select_self_verified. Per-sentence scope + multi-candidate disagreement give the uniqueness gate real teeth (two qualifying sentences -> refuse). The cue set {each,every,for,per,times} is an explicit PROVISIONAL hypothesis the practice loop refines, not a claimed-correct grammar. evals/gsm8k_math/practice/v1/search_runner.py: search_augmented_scorer + build_search_report — base scorer, then a practice-only attempt on refusals. MEASUREMENT (the deliverable, per the breadth-of-impact test): practice with search: correct=4 wrong=9 refused=37 (baseline 3/0/47) - Flips +1 (0021, the clean in-clause aggregate) and its renumbered/reworded variants (ADR-0114a perturbation guard) -> a real capability, not memorisation. - 9 wrong attempts -> elimination records (§9), the learning signal. The naive full-product cue model over-attempts; the eliminations are exactly the signal that refines it. HONEST FINDING: self-verification (grounding ∧ cue ∧ unit ∧ uniqueness) is NECESSARY but NOT SUFFICIENT — 9/13 self-verified attempts were wrong vs gold. The gap is cue PRECISION / which-quantities-compose (the knowledge axis), not 'can we multiply' (skill). This is why the search runs sealed: gold catches the 9, and case 0050 (canary) attempted-and-failed IN PRACTICE without touching serving -> validates the seal. Invariants: #1 seal (serving still 3/47/0; 0050 refuses in serving; no generate/chat import of the lane), #3 determinism. Serving wrong=0 untouched. Verified: 3a+3b 31/31; ruff clean; serving lane 4/4; smoke 67/67. --- evals/gsm8k_math/practice/v1/search_runner.py | 60 +++++++ generate/derivation/__init__.py | 5 + generate/derivation/extract.py | 39 +++++ generate/derivation/search.py | 75 +++++++++ tests/test_adr_0175_phase3b_mult_search.py | 147 ++++++++++++++++++ 5 files changed, 326 insertions(+) create mode 100644 evals/gsm8k_math/practice/v1/search_runner.py create mode 100644 generate/derivation/extract.py create mode 100644 generate/derivation/search.py create mode 100644 tests/test_adr_0175_phase3b_mult_search.py diff --git a/evals/gsm8k_math/practice/v1/search_runner.py b/evals/gsm8k_math/practice/v1/search_runner.py new file mode 100644 index 00000000..fdc771b8 --- /dev/null +++ b/evals/gsm8k_math/practice/v1/search_runner.py @@ -0,0 +1,60 @@ +"""ADR-0175 Phase 3b — wire the multiplicative search into the sealed practice lane. + +The practice lane (Phase 2) runs the base candidate-graph scorer. Here we augment +it: when the base engine *refuses*, the practice regime is allowed to *attempt* — +it runs the Phase 3b multiplicative search and checks the result against gold +(Tier-1, available in practice). Correct attempts flip; wrong attempts become +elimination records (§9). The base (serving) outcome is never altered — the search +only fires on cases the engine already declined, and only inside the sealed lane. + +This is the first phase where attempts and eliminations go live. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from evals.gsm8k_math.practice.v1.runner import PracticeReport, run_practice +from evals.gsm8k_math.runner import _score_one_candidate_graph +from evals.gsm8k_math.train_sample.v1.runner import _CASES_PATH, _load_cases +from generate.derivation.search import search_multiplicative + +_TOL = 1e-6 + + +@dataclass(frozen=True, slots=True) +class _SearchOutcome: + case_id: str + outcome: str + reason: str | None + actual_answer: float | None + + +def search_augmented_scorer(adapted: dict) -> object: + """Base scorer, then a practice-only multiplicative attempt on refusals.""" + base = _score_one_candidate_graph(adapted) + if base.outcome != "refused": + return base # the serving path already committed — leave it untouched + + resolution = search_multiplicative(adapted["problem"]) + if resolution is None: + return base # search also declined -> still refused + + attempted = resolution.answer + gold = float(adapted["expected_answer"]) + correct = abs(attempted - gold) <= _TOL + return _SearchOutcome( + case_id=adapted["id"], + outcome="correct" if correct else "wrong", + reason=( + f"search_multiplicative -> {attempted:g}" + if correct + else f"search_multiplicative wrong: got {attempted:g}, gold {gold:g}" + ), + actual_answer=attempted, + ) + + +def build_search_report() -> PracticeReport: + """Practice report with the multiplicative search enabled (attempts live).""" + return run_practice(_load_cases(_CASES_PATH), scorer=search_augmented_scorer) diff --git a/generate/derivation/__init__.py b/generate/derivation/__init__.py index 881a5dd3..b8cfa540 100644 --- a/generate/derivation/__init__.py +++ b/generate/derivation/__init__.py @@ -7,7 +7,9 @@ guard that keeps the (Phase 3b) bounded search honest. from __future__ import annotations +from generate.derivation.extract import extract_quantities from generate.derivation.model import GroundedDerivation, Quantity, Step, VALID_OPS +from generate.derivation.search import MULTIPLICATIVE_CUES, search_multiplicative from generate.derivation.verify import ( Resolution, SelfVerification, @@ -17,11 +19,14 @@ from generate.derivation.verify import ( __all__ = [ "GroundedDerivation", + "MULTIPLICATIVE_CUES", "Quantity", "Resolution", "SelfVerification", "Step", "VALID_OPS", + "extract_quantities", + "search_multiplicative", "select_self_verified", "self_verifies", ] diff --git a/generate/derivation/extract.py b/generate/derivation/extract.py new file mode 100644 index 00000000..b7ed6896 --- /dev/null +++ b/generate/derivation/extract.py @@ -0,0 +1,39 @@ +"""ADR-0175 Phase 3b — lexeme-level quantity extraction. + +Pulls ``(value, unit, source_token)`` triples from a problem using a single +orthographic pattern: a number immediately followed by a unit word. Per +ADR-0165 this is a *lexeme* pattern ("what this piece looks like: a number, a +unit word") — not a grammar template ("how words combine to mean X"). The +*combining* is the search's job (search.py) gated by self-verification. +""" + +from __future__ import annotations + +import re +from typing import Final + +from generate.derivation.model import Quantity + +# Number (int or decimal) immediately followed by a unit word. Lexeme-level. +_QTY_RE: Final[re.Pattern[str]] = re.compile( + r"(? tuple[Quantity, ...]: + """Extract ``(value, unit, source_token)`` quantities in left-to-right order. + + Deterministic. ``source_token`` is the surface number string (used by the + self-verification gate to prove the value is grounded in the text). Units + are lowercased; the value's surface token is preserved verbatim. + """ + out: list[Quantity] = [] + for match in _QTY_RE.finditer(problem_text): + value_token = match.group(1) + unit = match.group(2).lower() + try: + value = float(value_token) + except ValueError: # pragma: no cover - regex guarantees numeric + continue + out.append(Quantity(value=value, unit=unit, source_token=value_token)) + return tuple(out) diff --git a/generate/derivation/search.py b/generate/derivation/search.py new file mode 100644 index 00000000..c3d8d321 --- /dev/null +++ b/generate/derivation/search.py @@ -0,0 +1,75 @@ +"""ADR-0175 Phase 3b — bounded, deterministic multiplicative derivation search. + +The first attempt generator. Conservative by design: it proposes a single +candidate — the **full product of all extracted quantities** — and only when a +multiplicative-relation cue lexeme is present in the text. The proposal is then +passed through the Phase 3a self-verification gate (grounding ∧ unit ∧ unique), +so nothing ungrounded can resolve. + +The cue set is an explicit **provisional hypothesis**: it is the search's first +guess at which lexemes license multiplication. It is *not* claimed correct — the +sealed practice lane checks every attempt against gold, and wrong attempts become +elimination records (§9) that refine the hypothesis over time. That refinement is +the compounding loop; this module only stands up the first, gated attempt. + +wrong=0 posture: the search runs only in the sealed practice lane (never serving), +every proposal is gated by self-verification, and a non-unique or ungrounded +proposal refuses. Bounded by :data:`MAX_QUANTITIES` (refuse rather than enumerate +an unbounded product). +""" + +from __future__ import annotations + +import re +from typing import Final + +from generate.derivation.extract import extract_quantities +from generate.derivation.model import GroundedDerivation, Step +from generate.derivation.verify import Resolution, select_self_verified +from generate.math_roundtrip import _tokens + +# Provisional multiplicative-cue lexemes (the search's first hypothesis; refined +# by practice elimination, not asserted correct). Sorted use for determinism. +MULTIPLICATIVE_CUES: Final[tuple[str, ...]] = ("each", "every", "for", "per", "times") +MAX_QUANTITIES: Final[int] = 6 + +_SENTENCE_SPLIT: Final[re.Pattern[str]] = re.compile(r"(?<=[.?!])\s+") + + +def _sentence_candidates(problem_text: str) -> list[GroundedDerivation]: + """One in-clause product candidate per sentence that has ≥2 quantities and a + present multiplicative cue. + + Per-sentence (in-clause) scope is deliberate: it targets the multiplicative + *aggregate* and avoids multiplying quantities that merely co-occur across + sentences. When two sentences each yield a product, they disagree and the + uniqueness gate refuses — so the disagreement rule does real safety work + instead of being trivially satisfied by a single whole-text candidate. + """ + candidates: list[GroundedDerivation] = [] + for sentence in _SENTENCE_SPLIT.split(problem_text): + quantities = extract_quantities(sentence) + if not 2 <= len(quantities) <= MAX_QUANTITIES: + continue + present = [c for c in MULTIPLICATIVE_CUES if c in _tokens(sentence)] + if not present: + continue + cue = present[0] # deterministic (MULTIPLICATIVE_CUES is sorted-by-design) + start, *rest = quantities + candidates.append( + GroundedDerivation( + start=start, + steps=tuple(Step(op="multiply", operand=q, cue=cue) for q in rest), + ) + ) + return candidates + + +def search_multiplicative(problem_text: str) -> Resolution | None: + """Attempt a grounded in-clause multiplicative product. + + Builds one product candidate per qualifying sentence and runs them through + the Phase 3a gate: a single self-verifying candidate resolves; zero (no + grounded product) or several that disagree refuse. Deterministic and bounded. + """ + return select_self_verified(_sentence_candidates(problem_text), problem_text) diff --git a/tests/test_adr_0175_phase3b_mult_search.py b/tests/test_adr_0175_phase3b_mult_search.py new file mode 100644 index 00000000..0a583d5f --- /dev/null +++ b/tests/test_adr_0175_phase3b_mult_search.py @@ -0,0 +1,147 @@ +"""ADR-0175 Phase 3b — bounded multiplicative derivation search + practice wiring. + +The first live attempt generator. Runs only in the sealed practice lane and only +on cases the engine already refused; every proposal is gated by Phase 3a +self-verification. Wrong attempts are tolerated — they are the elimination +signal, not a lane failure. + +Covers: +- extraction + search behaviour; +- the ADR-0114a generality guard (renumbered/reworded variants flip too — the + capability is not memorised to the 0021 surface); +- invariant #1 (seal): serving stays 3/47/0, the 0050 canary refuses in serving + and is not attempted-wrong in practice; +- invariant #3 (determinism). +""" + +from __future__ import annotations + +import json + +from evals.gsm8k_math.practice.v1.search_runner import build_search_report +from evals.gsm8k_math.train_sample.v1.runner import _CASES_PATH, _load_cases +from evals.gsm8k_math.train_sample.v1.runner import build_report as serving_report +from generate.derivation.search import search_multiplicative +from generate.derivation import extract_quantities + +_T0021 = "He bench presses 15 pounds for 10 reps and does 3 sets." + + +# --------------------------------------------------------------------------- +# extraction +# --------------------------------------------------------------------------- + +class TestExtractQuantities: + def test_extracts_number_unit_pairs(self) -> None: + qs = extract_quantities(_T0021) + assert [(q.value, q.unit) for q in qs] == [ + (15.0, "pounds"), + (10.0, "reps"), + (3.0, "sets"), + ] + + def test_handles_decimals(self) -> None: + qs = extract_quantities("It moves 2.5 times the 4 kg load.") + assert (2.5, "times") in [(q.value, q.unit) for q in qs] + + +# --------------------------------------------------------------------------- +# search behaviour +# --------------------------------------------------------------------------- + +class TestSearchMultiplicative: + def test_flips_0021(self) -> None: + res = search_multiplicative(_T0021) + assert res is not None + assert res.answer == 450.0 + assert res.answer_unit == "pounds" + + def test_refuses_without_a_cue(self) -> None: + # two quantities, same sentence, but no multiplicative cue lexeme + assert search_multiplicative("She has 5 apples and 3 oranges.") is None + + def test_refuses_single_quantity(self) -> None: + assert search_multiplicative("He lifts 15 pounds for the workout.") is None + + def test_disagreeing_sentences_refuse(self) -> None: + # two qualifying sentences -> two distinct products -> disagreement -> refuse + text = "He does 15 pounds for 10 reps. She bakes 4 trays for 6 batches." + assert search_multiplicative(text) is None + + +# --------------------------------------------------------------------------- +# ADR-0114a generality guard — the flip is a capability, not memorisation +# --------------------------------------------------------------------------- + +class TestPerturbationGenerality: + def test_renumbered_variant_flips(self) -> None: + res = search_multiplicative("He bench presses 20 pounds for 5 reps and does 4 sets.") + assert res is not None and res.answer == 400.0 + + def test_reworded_same_shape_flips(self) -> None: + # different domain, same in-clause multiplicative shape + res = search_multiplicative("She bakes 4 trays for 6 batches and does 2 rounds.") + assert res is not None and res.answer == 48.0 + + def test_two_factor_variant_flips(self) -> None: + res = search_multiplicative("Each shelf holds 7 books for 9 shelves.") + assert res is not None and res.answer == 63.0 + + +# --------------------------------------------------------------------------- +# live practice measurement — attempts + eliminations go live +# --------------------------------------------------------------------------- + +class TestLiveSearchPractice: + def test_search_flips_at_least_one_beyond_baseline(self) -> None: + rep = build_search_report() + # baseline practice is 3 correct; the search must add at least one flip + assert rep.counts["correct"] >= 4 + + def test_wrong_attempts_are_recorded_as_eliminations(self) -> None: + rep = build_search_report() + # practice tolerates wrong — they are the learning signal (§9) + assert rep.counts["wrong"] == len(rep.elimination_records) + assert rep.counts["wrong"] >= 1 # the naive cue model over-attempts (expected) + + +# --------------------------------------------------------------------------- +# invariant #1 — the seal +# --------------------------------------------------------------------------- + +class TestSealInvariant: + def test_serving_unchanged_by_search(self) -> None: + build_search_report() # run practice with the search live + assert serving_report(_load_cases(_CASES_PATH))["counts"] == { + "correct": 3, + "wrong": 0, + "refused": 47, + } + + def test_0050_canary_refuses_in_serving_and_is_not_attempted_wrong(self) -> None: + from generate.math_candidate_graph import parse_and_solve + + c0050 = next( + json.loads(line) + for line in _CASES_PATH.read_text().splitlines() + if "0050" in line + ) + # serving still refuses the canary + assert parse_and_solve(c0050["question"]).answer is None + # and the search did not attempt-wrong on it in practice + rep = build_search_report() + assert not any(r.case_id.endswith("0050") for r in rep.elimination_records) + + +# --------------------------------------------------------------------------- +# invariant #3 — determinism +# --------------------------------------------------------------------------- + +class TestDeterminism: + def test_search_is_deterministic(self) -> None: + assert search_multiplicative(_T0021) == search_multiplicative(_T0021) + + def test_report_byte_identical(self) -> None: + a = json.dumps(build_search_report().as_dict(), sort_keys=True) + b = json.dumps(build_search_report().as_dict(), sort_keys=True) + assert a == b