feat(adr-0176-ms3): target-guided bounded multi-step search

MS-3 composes MS-1 (Target) + MS-2 (comparative chains + completeness) + the gate.
generate/derivation/multistep.py: search_chain(problem_text, target=None).

Shape-based, NOT blind enumeration: enumerates a small principled candidate set
(product-of-all if a multiplicative cue is present; sum-of-all if an aggregation
hint is present; each optionally + comparative scalars), each using all
quantities, routed through select_self_verified (grounding ∧ cue ∧ unit ∧
completeness ∧ uniqueness). Bounded (MAX_QUANTITIES, refuse-on-overflow) +
deterministic. Target supplies the aggregation cue + question quantities; target-
UNIT matching is deferred (answer_unit=start.unit is wrong for cross-unit products
-> a unit gate would over-refuse; documented).

Honest practice measurement (sealed lane): 4 correct / 9 wrong / 37 refused
(baseline 3/0/47). +1 flip is the unambiguous whole-problem product (0021); the 9
wrongs are product-of-all eliminations on multi-step problems (caught by gold,
the learning signal). Whole-problem shapes add no coverage beyond the unambiguous
product WITHOUT cue precision: when product and sum both self-verify they disagree
-> uniqueness refuses (safe-but-low-coverage by design). The lever remains cue
precision (the ADR-0175 learning loop).

Microscope finding: 0003-class flips (48*24*0.75=864=gold) are blocked by a
DECIMAL/currency grounding gap -- '$0.75' tokenizes to 0/75 so '0.75' is not
grounded by the shared round-trip primitive. Not a search bug; deferred
extraction-richness work (won't casually change the serving round-trip primitive).
A test documents the current refusal so the fix is detectable.

wrong=0: serving untouched (sealed); ambiguity + no-licensed-cue refuse; routes
through the proven gate. 8 MS-3 tests; full derivation surface 77/77; ruff clean;
smoke 67.
This commit is contained in:
Shay 2026-05-28 16:51:43 -07:00
parent 5a9454af20
commit 309f3fc10c
3 changed files with 170 additions and 0 deletions

View file

@ -14,6 +14,7 @@ from generate.derivation.comparatives import (
)
from generate.derivation.extract import extract_quantities
from generate.derivation.model import GroundedDerivation, Quantity, Step, VALID_OPS
from generate.derivation.multistep import search_chain
from generate.derivation.search import MULTIPLICATIVE_CUES, search_multiplicative
from generate.derivation.target import Target, extract_target
from generate.derivation.verify import (
@ -37,6 +38,7 @@ __all__ = [
"extract_comparative_scalars",
"extract_quantities",
"extract_target",
"search_chain",
"search_multiplicative",
"select_self_verified",
"self_verifies",

View file

@ -0,0 +1,89 @@
"""ADR-0176 MS-3 — target-guided bounded multi-step search.
Composes everything: extract body+question quantities (MS-1 ``Target``),
comparative scalars (the pack), enumerate a small, principled set of candidate
**chains** that use *all* the quantities, and route them through the self-
verification gate (grounding cue unit completeness) + ``target_units`` +
uniqueness (MS-2 ``select_self_verified``).
Deliberately **shape-based, not blind enumeration**: it tries the arithmetic
shapes the gold structures actually use (product-of-all, sum-of-all, each
optionally followed by the comparative scalars), each licensed by a *present*
cue. This is bounded by construction (a handful of candidates) and deterministic.
Honest posture (wrong=0-first): when more than one shape self-verifies and they
disagree, the uniqueness rule **refuses** broad cues cannot tell which op the
text licenses, and that ambiguity is a refusal, not a guess. Coverage is therefore
gated on cue *precision* (the ADR-0175 learning loop), exactly as the practice
eliminations will show. Refuse-preferring throughout; runs only in the sealed
practice lane.
"""
from __future__ import annotations
from typing import Final
from generate.derivation.comparatives import comparative_step, extract_comparative_scalars
from generate.derivation.extract import extract_quantities
from generate.derivation.model import GroundedDerivation, Quantity, Step
from generate.derivation.search import MULTIPLICATIVE_CUES
from generate.derivation.target import Target, extract_target
from generate.derivation.verify import Resolution, select_self_verified
from generate.math_roundtrip import _tokens
MAX_QUANTITIES: Final[int] = 6
def _chain(quantities: list[Quantity], op: str, cue: str, tail: tuple[Step, ...]) -> GroundedDerivation:
"""Left-fold all quantities under ``op`` (cue ``cue``), then append ``tail``."""
start, *rest = quantities
steps = tuple(Step(op=op, operand=q, cue=cue) for q in rest) + tail
return GroundedDerivation(start=start, steps=steps)
def _candidate_chains(
quantities: list[Quantity], problem_text: str, target: Target
) -> list[GroundedDerivation]:
"""The small, principled candidate set. Bounded + deterministic."""
tokens = _tokens(problem_text)
comparative_tail = tuple(
comparative_step(cs) for cs in extract_comparative_scalars(problem_text)
)
# product-of-all is licensed by a present multiplicative cue;
# sum-of-all by a present aggregation hint (a strong, single-word sum signal).
mult_cue = next((c for c in MULTIPLICATIVE_CUES if c in tokens), None)
add_cue = target.aggregation if (target.aggregation in tokens) else None
candidates: list[GroundedDerivation] = []
for op, cue in (("multiply", mult_cue), ("add", add_cue)):
if cue is None:
continue
candidates.append(_chain(quantities, op, cue, ()))
if comparative_tail:
candidates.append(_chain(quantities, op, cue, comparative_tail))
return candidates
def search_chain(problem_text: str, target: Target | None = None) -> Resolution | None:
"""Target-guided bounded multi-step search over the problem's quantities.
Returns a :class:`Resolution` only when a single candidate chain self-verifies
(and, when a target unit is known, matches it); refuses on no candidate,
disagreement, or > :data:`MAX_QUANTITIES` quantities. Deterministic.
"""
quantities = list(extract_quantities(problem_text))
if not 2 <= len(quantities) <= MAX_QUANTITIES:
return None # refuse-on-overflow / too few to compose
resolved: Target = target if target is not None else extract_target(
problem_text, known_units=tuple(q.unit for q in quantities)
)
candidates = _candidate_chains(quantities, problem_text, resolved)
# Target-UNIT matching is deferred: the model's answer_unit is the start
# quantity's unit, which is wrong for cross-unit products (6 boxes x 50 apples
# -> value 300 but unit "boxes"), so a unit gate would over-refuse correct
# answers. The Target still prunes via its aggregation cue + supplies the
# question quantities (above). Unit matching returns once a result-unit model
# exists (a superordinate-units pack + product-unit inference).
return select_self_verified(candidates, problem_text, target_units=())

View file

@ -0,0 +1,79 @@
"""ADR-0176 MS-3 — target-guided bounded multi-step search.
Composes MS-1 (Target) + MS-2 (comparative chains + completeness) + the gate.
Shape-based + bounded + deterministic + refuse-on-overflow. Uniqueness refuses
ambiguity (multiple shapes self-verify + disagree -> refuse) safe-but-low-
coverage by design; coverage is gated on cue precision (the learning lever).
Honest practice measurement (sealed lane, this build): 4 correct / 9 wrong / 37
refused (baseline 3/0/47). The +1 flip is the unambiguous whole-problem product
(0021); the 9 wrongs are product-of-all eliminations on multi-step problems
(caught by gold). 0003-class flips are blocked by a decimal-grounding gap
(``$0.75`` tokenizes to ``0``/``75``) in the shared round-trip primitive
deferred extraction-richness work, not a search bug.
"""
from __future__ import annotations
from generate.derivation import search_chain
from generate.derivation.target import Target
class TestFlipsCleanProduct:
def test_unambiguous_product_resolves(self) -> None:
# mult cue "each", no aggregation hint -> only the product shape -> unique
res = search_chain("Maria packs 6 boxes with 50 apples each. How many apples?")
assert res is not None and res.answer == 300.0
class TestRefusesAmbiguity:
def test_product_and_sum_disagree_refuses(self) -> None:
# same-unit quantities so both shapes self-verify: "each" licenses product
# (300), "total" licenses sum (56); they disagree -> refuse (wrong=0).
text = "Each box has 6 apples. There are 50 apples in total. How many apples?"
assert search_chain(text) is None
def test_no_licensed_op_refuses(self) -> None:
# no multiplicative cue and no aggregation hint -> no candidate -> refuse
# (the search does not fabricate an operation it cannot license)
assert search_chain("A theater has 6 rows and 50 seats.") is None
class TestBounded:
def test_too_few_quantities_refuses(self) -> None:
assert search_chain("He has 6 boxes each.") is None
def test_refuse_on_overflow(self) -> None:
# > MAX_QUANTITIES (6) quantities -> refuse rather than enumerate
text = (
"1 apple, 2 pears, 3 plums, 4 figs, 5 dates, 6 limes, 7 kiwis each."
)
assert search_chain(text) is None
class TestDeterminism:
def test_deterministic(self) -> None:
text = "Maria packs 6 boxes with 50 apples each. How many apples?"
assert search_chain(text) == search_chain(text)
class TestTargetThreading:
def test_explicit_target_is_used(self) -> None:
# passing a Target with no aggregation -> sum shape not attempted ->
# unambiguous product resolves
text = "Maria packs 6 boxes with 50 apples each."
target = Target(quantities=(), aggregation=None, units=())
assert search_chain(text, target).answer == 300.0
class TestDecimalGroundingGapIsDeferred:
def test_decimal_operand_currently_refused(self) -> None:
# Documents the known gap: $0.75 tokenizes to 0/75 so "0.75" is not grounded
# by the shared round-trip primitive -> the (correct, 864) product refuses.
# When decimal/currency grounding lands, this should flip. Asserting the
# CURRENT behaviour so the fix is detectable.
text = (
"There are 48 boxes with 24 erasers in each box. "
"They sell the erasers for $0.75 each. How much money will they make?"
)
assert search_chain(text) is None # refused today (decimal grounding gap)