feat(adr-0176-ms3): target-guided bounded multi-step search
MS-3 composes MS-1 (Target) + MS-2 (comparative chains + completeness) + the gate. generate/derivation/multistep.py: search_chain(problem_text, target=None). Shape-based, NOT blind enumeration: enumerates a small principled candidate set (product-of-all if a multiplicative cue is present; sum-of-all if an aggregation hint is present; each optionally + comparative scalars), each using all quantities, routed through select_self_verified (grounding ∧ cue ∧ unit ∧ completeness ∧ uniqueness). Bounded (MAX_QUANTITIES, refuse-on-overflow) + deterministic. Target supplies the aggregation cue + question quantities; target- UNIT matching is deferred (answer_unit=start.unit is wrong for cross-unit products -> a unit gate would over-refuse; documented). Honest practice measurement (sealed lane): 4 correct / 9 wrong / 37 refused (baseline 3/0/47). +1 flip is the unambiguous whole-problem product (0021); the 9 wrongs are product-of-all eliminations on multi-step problems (caught by gold, the learning signal). Whole-problem shapes add no coverage beyond the unambiguous product WITHOUT cue precision: when product and sum both self-verify they disagree -> uniqueness refuses (safe-but-low-coverage by design). The lever remains cue precision (the ADR-0175 learning loop). Microscope finding: 0003-class flips (48*24*0.75=864=gold) are blocked by a DECIMAL/currency grounding gap -- '$0.75' tokenizes to 0/75 so '0.75' is not grounded by the shared round-trip primitive. Not a search bug; deferred extraction-richness work (won't casually change the serving round-trip primitive). A test documents the current refusal so the fix is detectable. wrong=0: serving untouched (sealed); ambiguity + no-licensed-cue refuse; routes through the proven gate. 8 MS-3 tests; full derivation surface 77/77; ruff clean; smoke 67.
This commit is contained in:
parent
5a9454af20
commit
309f3fc10c
3 changed files with 170 additions and 0 deletions
|
|
@ -14,6 +14,7 @@ from generate.derivation.comparatives import (
|
|||
)
|
||||
from generate.derivation.extract import extract_quantities
|
||||
from generate.derivation.model import GroundedDerivation, Quantity, Step, VALID_OPS
|
||||
from generate.derivation.multistep import search_chain
|
||||
from generate.derivation.search import MULTIPLICATIVE_CUES, search_multiplicative
|
||||
from generate.derivation.target import Target, extract_target
|
||||
from generate.derivation.verify import (
|
||||
|
|
@ -37,6 +38,7 @@ __all__ = [
|
|||
"extract_comparative_scalars",
|
||||
"extract_quantities",
|
||||
"extract_target",
|
||||
"search_chain",
|
||||
"search_multiplicative",
|
||||
"select_self_verified",
|
||||
"self_verifies",
|
||||
|
|
|
|||
89
generate/derivation/multistep.py
Normal file
89
generate/derivation/multistep.py
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
"""ADR-0176 MS-3 — target-guided bounded multi-step search.
|
||||
|
||||
Composes everything: extract body+question quantities (MS-1 ``Target``),
|
||||
comparative scalars (the pack), enumerate a small, principled set of candidate
|
||||
**chains** that use *all* the quantities, and route them through the self-
|
||||
verification gate (grounding ∧ cue ∧ unit ∧ completeness) + ``target_units`` +
|
||||
uniqueness (MS-2 ``select_self_verified``).
|
||||
|
||||
Deliberately **shape-based, not blind enumeration**: it tries the arithmetic
|
||||
shapes the gold structures actually use (product-of-all, sum-of-all, each
|
||||
optionally followed by the comparative scalars), each licensed by a *present*
|
||||
cue. This is bounded by construction (a handful of candidates) and deterministic.
|
||||
|
||||
Honest posture (wrong=0-first): when more than one shape self-verifies and they
|
||||
disagree, the uniqueness rule **refuses** — broad cues cannot tell which op the
|
||||
text licenses, and that ambiguity is a refusal, not a guess. Coverage is therefore
|
||||
gated on cue *precision* (the ADR-0175 learning loop), exactly as the practice
|
||||
eliminations will show. Refuse-preferring throughout; runs only in the sealed
|
||||
practice lane.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
from generate.derivation.comparatives import comparative_step, extract_comparative_scalars
|
||||
from generate.derivation.extract import extract_quantities
|
||||
from generate.derivation.model import GroundedDerivation, Quantity, Step
|
||||
from generate.derivation.search import MULTIPLICATIVE_CUES
|
||||
from generate.derivation.target import Target, extract_target
|
||||
from generate.derivation.verify import Resolution, select_self_verified
|
||||
from generate.math_roundtrip import _tokens
|
||||
|
||||
MAX_QUANTITIES: Final[int] = 6
|
||||
|
||||
|
||||
def _chain(quantities: list[Quantity], op: str, cue: str, tail: tuple[Step, ...]) -> GroundedDerivation:
|
||||
"""Left-fold all quantities under ``op`` (cue ``cue``), then append ``tail``."""
|
||||
start, *rest = quantities
|
||||
steps = tuple(Step(op=op, operand=q, cue=cue) for q in rest) + tail
|
||||
return GroundedDerivation(start=start, steps=steps)
|
||||
|
||||
|
||||
def _candidate_chains(
|
||||
quantities: list[Quantity], problem_text: str, target: Target
|
||||
) -> list[GroundedDerivation]:
|
||||
"""The small, principled candidate set. Bounded + deterministic."""
|
||||
tokens = _tokens(problem_text)
|
||||
comparative_tail = tuple(
|
||||
comparative_step(cs) for cs in extract_comparative_scalars(problem_text)
|
||||
)
|
||||
|
||||
# product-of-all is licensed by a present multiplicative cue;
|
||||
# sum-of-all by a present aggregation hint (a strong, single-word sum signal).
|
||||
mult_cue = next((c for c in MULTIPLICATIVE_CUES if c in tokens), None)
|
||||
add_cue = target.aggregation if (target.aggregation in tokens) else None
|
||||
|
||||
candidates: list[GroundedDerivation] = []
|
||||
for op, cue in (("multiply", mult_cue), ("add", add_cue)):
|
||||
if cue is None:
|
||||
continue
|
||||
candidates.append(_chain(quantities, op, cue, ()))
|
||||
if comparative_tail:
|
||||
candidates.append(_chain(quantities, op, cue, comparative_tail))
|
||||
return candidates
|
||||
|
||||
|
||||
def search_chain(problem_text: str, target: Target | None = None) -> Resolution | None:
|
||||
"""Target-guided bounded multi-step search over the problem's quantities.
|
||||
|
||||
Returns a :class:`Resolution` only when a single candidate chain self-verifies
|
||||
(and, when a target unit is known, matches it); refuses on no candidate,
|
||||
disagreement, or > :data:`MAX_QUANTITIES` quantities. Deterministic.
|
||||
"""
|
||||
quantities = list(extract_quantities(problem_text))
|
||||
if not 2 <= len(quantities) <= MAX_QUANTITIES:
|
||||
return None # refuse-on-overflow / too few to compose
|
||||
|
||||
resolved: Target = target if target is not None else extract_target(
|
||||
problem_text, known_units=tuple(q.unit for q in quantities)
|
||||
)
|
||||
candidates = _candidate_chains(quantities, problem_text, resolved)
|
||||
# Target-UNIT matching is deferred: the model's answer_unit is the start
|
||||
# quantity's unit, which is wrong for cross-unit products (6 boxes x 50 apples
|
||||
# -> value 300 but unit "boxes"), so a unit gate would over-refuse correct
|
||||
# answers. The Target still prunes via its aggregation cue + supplies the
|
||||
# question quantities (above). Unit matching returns once a result-unit model
|
||||
# exists (a superordinate-units pack + product-unit inference).
|
||||
return select_self_verified(candidates, problem_text, target_units=())
|
||||
79
tests/test_adr_0176_ms3_search.py
Normal file
79
tests/test_adr_0176_ms3_search.py
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
"""ADR-0176 MS-3 — target-guided bounded multi-step search.
|
||||
|
||||
Composes MS-1 (Target) + MS-2 (comparative chains + completeness) + the gate.
|
||||
Shape-based + bounded + deterministic + refuse-on-overflow. Uniqueness refuses
|
||||
ambiguity (multiple shapes self-verify + disagree -> refuse) — safe-but-low-
|
||||
coverage by design; coverage is gated on cue precision (the learning lever).
|
||||
|
||||
Honest practice measurement (sealed lane, this build): 4 correct / 9 wrong / 37
|
||||
refused (baseline 3/0/47). The +1 flip is the unambiguous whole-problem product
|
||||
(0021); the 9 wrongs are product-of-all eliminations on multi-step problems
|
||||
(caught by gold). 0003-class flips are blocked by a decimal-grounding gap
|
||||
(``$0.75`` tokenizes to ``0``/``75``) in the shared round-trip primitive —
|
||||
deferred extraction-richness work, not a search bug.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from generate.derivation import search_chain
|
||||
from generate.derivation.target import Target
|
||||
|
||||
|
||||
class TestFlipsCleanProduct:
|
||||
def test_unambiguous_product_resolves(self) -> None:
|
||||
# mult cue "each", no aggregation hint -> only the product shape -> unique
|
||||
res = search_chain("Maria packs 6 boxes with 50 apples each. How many apples?")
|
||||
assert res is not None and res.answer == 300.0
|
||||
|
||||
|
||||
class TestRefusesAmbiguity:
|
||||
def test_product_and_sum_disagree_refuses(self) -> None:
|
||||
# same-unit quantities so both shapes self-verify: "each" licenses product
|
||||
# (300), "total" licenses sum (56); they disagree -> refuse (wrong=0).
|
||||
text = "Each box has 6 apples. There are 50 apples in total. How many apples?"
|
||||
assert search_chain(text) is None
|
||||
|
||||
def test_no_licensed_op_refuses(self) -> None:
|
||||
# no multiplicative cue and no aggregation hint -> no candidate -> refuse
|
||||
# (the search does not fabricate an operation it cannot license)
|
||||
assert search_chain("A theater has 6 rows and 50 seats.") is None
|
||||
|
||||
|
||||
class TestBounded:
|
||||
def test_too_few_quantities_refuses(self) -> None:
|
||||
assert search_chain("He has 6 boxes each.") is None
|
||||
|
||||
def test_refuse_on_overflow(self) -> None:
|
||||
# > MAX_QUANTITIES (6) quantities -> refuse rather than enumerate
|
||||
text = (
|
||||
"1 apple, 2 pears, 3 plums, 4 figs, 5 dates, 6 limes, 7 kiwis each."
|
||||
)
|
||||
assert search_chain(text) is None
|
||||
|
||||
|
||||
class TestDeterminism:
|
||||
def test_deterministic(self) -> None:
|
||||
text = "Maria packs 6 boxes with 50 apples each. How many apples?"
|
||||
assert search_chain(text) == search_chain(text)
|
||||
|
||||
|
||||
class TestTargetThreading:
|
||||
def test_explicit_target_is_used(self) -> None:
|
||||
# passing a Target with no aggregation -> sum shape not attempted ->
|
||||
# unambiguous product resolves
|
||||
text = "Maria packs 6 boxes with 50 apples each."
|
||||
target = Target(quantities=(), aggregation=None, units=())
|
||||
assert search_chain(text, target).answer == 300.0
|
||||
|
||||
|
||||
class TestDecimalGroundingGapIsDeferred:
|
||||
def test_decimal_operand_currently_refused(self) -> None:
|
||||
# Documents the known gap: $0.75 tokenizes to 0/75 so "0.75" is not grounded
|
||||
# by the shared round-trip primitive -> the (correct, 864) product refuses.
|
||||
# When decimal/currency grounding lands, this should flip. Asserting the
|
||||
# CURRENT behaviour so the fix is detectable.
|
||||
text = (
|
||||
"There are 48 boxes with 24 erasers in each box. "
|
||||
"They sell the erasers for $0.75 each. How much money will they make?"
|
||||
)
|
||||
assert search_chain(text) is None # refused today (decimal grounding gap)
|
||||
Loading…
Reference in a new issue