From 309f3fc10c28b42bb0f347c2e7230c7bdb5708c6 Mon Sep 17 00:00:00 2001 From: Shay Date: Thu, 28 May 2026 16:51:43 -0700 Subject: [PATCH] feat(adr-0176-ms3): target-guided bounded multi-step search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MS-3 composes MS-1 (Target) + MS-2 (comparative chains + completeness) + the gate. generate/derivation/multistep.py: search_chain(problem_text, target=None). Shape-based, NOT blind enumeration: enumerates a small principled candidate set (product-of-all if a multiplicative cue is present; sum-of-all if an aggregation hint is present; each optionally + comparative scalars), each using all quantities, routed through select_self_verified (grounding ∧ cue ∧ unit ∧ completeness ∧ uniqueness). Bounded (MAX_QUANTITIES, refuse-on-overflow) + deterministic. Target supplies the aggregation cue + question quantities; target- UNIT matching is deferred (answer_unit=start.unit is wrong for cross-unit products -> a unit gate would over-refuse; documented). Honest practice measurement (sealed lane): 4 correct / 9 wrong / 37 refused (baseline 3/0/47). +1 flip is the unambiguous whole-problem product (0021); the 9 wrongs are product-of-all eliminations on multi-step problems (caught by gold, the learning signal). Whole-problem shapes add no coverage beyond the unambiguous product WITHOUT cue precision: when product and sum both self-verify they disagree -> uniqueness refuses (safe-but-low-coverage by design). The lever remains cue precision (the ADR-0175 learning loop). Microscope finding: 0003-class flips (48*24*0.75=864=gold) are blocked by a DECIMAL/currency grounding gap -- '$0.75' tokenizes to 0/75 so '0.75' is not grounded by the shared round-trip primitive. Not a search bug; deferred extraction-richness work (won't casually change the serving round-trip primitive). A test documents the current refusal so the fix is detectable. wrong=0: serving untouched (sealed); ambiguity + no-licensed-cue refuse; routes through the proven gate. 8 MS-3 tests; full derivation surface 77/77; ruff clean; smoke 67. --- generate/derivation/__init__.py | 2 + generate/derivation/multistep.py | 89 +++++++++++++++++++++++++++++++ tests/test_adr_0176_ms3_search.py | 79 +++++++++++++++++++++++++++ 3 files changed, 170 insertions(+) create mode 100644 generate/derivation/multistep.py create mode 100644 tests/test_adr_0176_ms3_search.py diff --git a/generate/derivation/__init__.py b/generate/derivation/__init__.py index d9f023fd..bcd84ca5 100644 --- a/generate/derivation/__init__.py +++ b/generate/derivation/__init__.py @@ -14,6 +14,7 @@ from generate.derivation.comparatives import ( ) from generate.derivation.extract import extract_quantities from generate.derivation.model import GroundedDerivation, Quantity, Step, VALID_OPS +from generate.derivation.multistep import search_chain from generate.derivation.search import MULTIPLICATIVE_CUES, search_multiplicative from generate.derivation.target import Target, extract_target from generate.derivation.verify import ( @@ -37,6 +38,7 @@ __all__ = [ "extract_comparative_scalars", "extract_quantities", "extract_target", + "search_chain", "search_multiplicative", "select_self_verified", "self_verifies", diff --git a/generate/derivation/multistep.py b/generate/derivation/multistep.py new file mode 100644 index 00000000..e0e329bd --- /dev/null +++ b/generate/derivation/multistep.py @@ -0,0 +1,89 @@ +"""ADR-0176 MS-3 — target-guided bounded multi-step search. + +Composes everything: extract body+question quantities (MS-1 ``Target``), +comparative scalars (the pack), enumerate a small, principled set of candidate +**chains** that use *all* the quantities, and route them through the self- +verification gate (grounding ∧ cue ∧ unit ∧ completeness) + ``target_units`` + +uniqueness (MS-2 ``select_self_verified``). + +Deliberately **shape-based, not blind enumeration**: it tries the arithmetic +shapes the gold structures actually use (product-of-all, sum-of-all, each +optionally followed by the comparative scalars), each licensed by a *present* +cue. This is bounded by construction (a handful of candidates) and deterministic. + +Honest posture (wrong=0-first): when more than one shape self-verifies and they +disagree, the uniqueness rule **refuses** — broad cues cannot tell which op the +text licenses, and that ambiguity is a refusal, not a guess. Coverage is therefore +gated on cue *precision* (the ADR-0175 learning loop), exactly as the practice +eliminations will show. Refuse-preferring throughout; runs only in the sealed +practice lane. +""" + +from __future__ import annotations + +from typing import Final + +from generate.derivation.comparatives import comparative_step, extract_comparative_scalars +from generate.derivation.extract import extract_quantities +from generate.derivation.model import GroundedDerivation, Quantity, Step +from generate.derivation.search import MULTIPLICATIVE_CUES +from generate.derivation.target import Target, extract_target +from generate.derivation.verify import Resolution, select_self_verified +from generate.math_roundtrip import _tokens + +MAX_QUANTITIES: Final[int] = 6 + + +def _chain(quantities: list[Quantity], op: str, cue: str, tail: tuple[Step, ...]) -> GroundedDerivation: + """Left-fold all quantities under ``op`` (cue ``cue``), then append ``tail``.""" + start, *rest = quantities + steps = tuple(Step(op=op, operand=q, cue=cue) for q in rest) + tail + return GroundedDerivation(start=start, steps=steps) + + +def _candidate_chains( + quantities: list[Quantity], problem_text: str, target: Target +) -> list[GroundedDerivation]: + """The small, principled candidate set. Bounded + deterministic.""" + tokens = _tokens(problem_text) + comparative_tail = tuple( + comparative_step(cs) for cs in extract_comparative_scalars(problem_text) + ) + + # product-of-all is licensed by a present multiplicative cue; + # sum-of-all by a present aggregation hint (a strong, single-word sum signal). + mult_cue = next((c for c in MULTIPLICATIVE_CUES if c in tokens), None) + add_cue = target.aggregation if (target.aggregation in tokens) else None + + candidates: list[GroundedDerivation] = [] + for op, cue in (("multiply", mult_cue), ("add", add_cue)): + if cue is None: + continue + candidates.append(_chain(quantities, op, cue, ())) + if comparative_tail: + candidates.append(_chain(quantities, op, cue, comparative_tail)) + return candidates + + +def search_chain(problem_text: str, target: Target | None = None) -> Resolution | None: + """Target-guided bounded multi-step search over the problem's quantities. + + Returns a :class:`Resolution` only when a single candidate chain self-verifies + (and, when a target unit is known, matches it); refuses on no candidate, + disagreement, or > :data:`MAX_QUANTITIES` quantities. Deterministic. + """ + quantities = list(extract_quantities(problem_text)) + if not 2 <= len(quantities) <= MAX_QUANTITIES: + return None # refuse-on-overflow / too few to compose + + resolved: Target = target if target is not None else extract_target( + problem_text, known_units=tuple(q.unit for q in quantities) + ) + candidates = _candidate_chains(quantities, problem_text, resolved) + # Target-UNIT matching is deferred: the model's answer_unit is the start + # quantity's unit, which is wrong for cross-unit products (6 boxes x 50 apples + # -> value 300 but unit "boxes"), so a unit gate would over-refuse correct + # answers. The Target still prunes via its aggregation cue + supplies the + # question quantities (above). Unit matching returns once a result-unit model + # exists (a superordinate-units pack + product-unit inference). + return select_self_verified(candidates, problem_text, target_units=()) diff --git a/tests/test_adr_0176_ms3_search.py b/tests/test_adr_0176_ms3_search.py new file mode 100644 index 00000000..7a3ece13 --- /dev/null +++ b/tests/test_adr_0176_ms3_search.py @@ -0,0 +1,79 @@ +"""ADR-0176 MS-3 — target-guided bounded multi-step search. + +Composes MS-1 (Target) + MS-2 (comparative chains + completeness) + the gate. +Shape-based + bounded + deterministic + refuse-on-overflow. Uniqueness refuses +ambiguity (multiple shapes self-verify + disagree -> refuse) — safe-but-low- +coverage by design; coverage is gated on cue precision (the learning lever). + +Honest practice measurement (sealed lane, this build): 4 correct / 9 wrong / 37 +refused (baseline 3/0/47). The +1 flip is the unambiguous whole-problem product +(0021); the 9 wrongs are product-of-all eliminations on multi-step problems +(caught by gold). 0003-class flips are blocked by a decimal-grounding gap +(``$0.75`` tokenizes to ``0``/``75``) in the shared round-trip primitive — +deferred extraction-richness work, not a search bug. +""" + +from __future__ import annotations + +from generate.derivation import search_chain +from generate.derivation.target import Target + + +class TestFlipsCleanProduct: + def test_unambiguous_product_resolves(self) -> None: + # mult cue "each", no aggregation hint -> only the product shape -> unique + res = search_chain("Maria packs 6 boxes with 50 apples each. How many apples?") + assert res is not None and res.answer == 300.0 + + +class TestRefusesAmbiguity: + def test_product_and_sum_disagree_refuses(self) -> None: + # same-unit quantities so both shapes self-verify: "each" licenses product + # (300), "total" licenses sum (56); they disagree -> refuse (wrong=0). + text = "Each box has 6 apples. There are 50 apples in total. How many apples?" + assert search_chain(text) is None + + def test_no_licensed_op_refuses(self) -> None: + # no multiplicative cue and no aggregation hint -> no candidate -> refuse + # (the search does not fabricate an operation it cannot license) + assert search_chain("A theater has 6 rows and 50 seats.") is None + + +class TestBounded: + def test_too_few_quantities_refuses(self) -> None: + assert search_chain("He has 6 boxes each.") is None + + def test_refuse_on_overflow(self) -> None: + # > MAX_QUANTITIES (6) quantities -> refuse rather than enumerate + text = ( + "1 apple, 2 pears, 3 plums, 4 figs, 5 dates, 6 limes, 7 kiwis each." + ) + assert search_chain(text) is None + + +class TestDeterminism: + def test_deterministic(self) -> None: + text = "Maria packs 6 boxes with 50 apples each. How many apples?" + assert search_chain(text) == search_chain(text) + + +class TestTargetThreading: + def test_explicit_target_is_used(self) -> None: + # passing a Target with no aggregation -> sum shape not attempted -> + # unambiguous product resolves + text = "Maria packs 6 boxes with 50 apples each." + target = Target(quantities=(), aggregation=None, units=()) + assert search_chain(text, target).answer == 300.0 + + +class TestDecimalGroundingGapIsDeferred: + def test_decimal_operand_currently_refused(self) -> None: + # Documents the known gap: $0.75 tokenizes to 0/75 so "0.75" is not grounded + # by the shared round-trip primitive -> the (correct, 864) product refuses. + # When decimal/currency grounding lands, this should flip. Asserting the + # CURRENT behaviour so the fix is detectable. + text = ( + "There are 48 boxes with 24 erasers in each box. " + "They sell the erasers for $0.75 each. How much money will they make?" + ) + assert search_chain(text) is None # refused today (decimal grounding gap)