0fbcce429b
Implements ADR-0182's first win on top of EX-6 (#473). The distractor-quantity confuser 0014 misfired (20x3x5=300) because a blunt product-of-all was the *unique* self-verifying reading: the completeness clause forces the distractor into it, and no rival reading existed to trigger the wrong=0 disagreement rule. No tight cue rule separates it from the legitimate cross-unit products (`for` licenses both the 0014 distractor AND the correct train-0021 product) -- that is the deferred cue-precision problem. So instead of a reactive patch, let the disagreement rule do the refusing. Mechanism (sealed lane; chat/ does not import these -> serving 3/47/0 frozen): - verify.classify_derivation: a derivation is `complete` (commit-eligible), `exempt` (verified but for an isolated-foreign unused quantity -> commit- INELIGIBLE), or None. Refactored self_verifies into _base_reasons + _unused_quantities so the two share logic and cannot drift (behavior identical; 385 derivation tests + smoke 67 green). - accumulate.accumulation_candidates: exposes the ungated readings, incl. a distractor-skip reading that drops an isolated-foreign quantity from a multi- quantity change clause (20+5=25). compose_accumulation is byte-identical (drop_isolated_foreign=False + the same gate). - search.multiplicative_candidates / multistep.candidate_chains: ungated candidates. - pool.resolve_pooled: pools every composer's readings; disagreement -> refuse; a single answer commits only if a `complete` candidate produced it (exempt-only -> refuse, so the commit-path completeness guarantee from ADR-0175 is untouched). - confuser runner: _engine_answer now delegates to resolve_pooled (the prior first-composer-wins order could not notice it held two incompatible readings). Result (the microscope): - confuser wrong 5 -> 2. distractor 0014 refuses (product 300 vs additive 25); BONUS: both disguised-polarity cases (0001/0003) refuse -- the spurious "buys X for N coins" product disagrees with the accumulation reading. Remaining wrong: 0016 (distractor in the anchor clause -> needs anchor-skip, separate step) and 0020 (temporal-scope). pair-tells 4 -> 1. - genuine positives still 7 solved, 0 wrong. - train_sample 3/47/0 and practice 3/47/0 byte-identical (they call compose_accumulation/search directly -- unchanged -- not the pool). - smoke 67, architectural invariants 40, lane-SHA freeze 8/8. Tests: - test_adr_0182_pool.py: classify (complete/exempt/None incl. narrow-exemption edges) + resolve_pooled, with the wrong=0 obligation test_exempt_only_never_commits (a distractor with no multiplicative cue must refuse, not commit 25 -- fails loudly if the exemption is made commit-eligible). - test_adr_0163_f2_confusers.py: baseline tightened wrong 5->2, pair_tells 4->1; new test_distractor_0014_refuses_via_pooling + test_disguised_polarity_does_not_misfire. Stacked on #473 (EX-6); merge #473 first. 0016 + the remaining wrongs are follow-ups.
104 lines
5 KiB
Python
104 lines
5 KiB
Python
"""ADR-0182 — cross-composer disagreement pooling (distractor refusal).
|
|
|
|
Two surfaces under test:
|
|
|
|
* :func:`generate.derivation.verify.classify_derivation` — the commit-eligibility
|
|
class (``complete`` / ``exempt`` / ``None``) and, critically, that the
|
|
isolated-foreign exemption is *narrow* (an empty-unit or same-unit unused
|
|
quantity is never exempt — it is real signal, not a distractor).
|
|
* :func:`generate.derivation.pool.resolve_pooled` — the pooled resolution: a clean
|
|
reading commits, a distractor problem's product-vs-additive disagreement refuses,
|
|
and (the wrong=0-critical property) an ``exempt``-only answer **never commits**.
|
|
|
|
Sealed lane: ``chat/`` does not import these; serving ``3/47/0`` cannot move.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from generate.derivation.accumulate import accumulation_candidates
|
|
from generate.derivation.model import GroundedDerivation, Quantity, Step
|
|
from generate.derivation.pool import resolve_pooled
|
|
from generate.derivation.verify import classify_derivation
|
|
|
|
_DISTRACTOR_0014 = (
|
|
"Kate has 20 pencils. She studies for 3 hours and then buys 5 more pencils. "
|
|
"How many pencils does Kate have?"
|
|
)
|
|
# A distractor with NO multiplicative cue: the only candidate is the exempt additive
|
|
# reading (no complete product exists). The commit-ineligibility rule must refuse it.
|
|
_EXEMPT_ONLY = (
|
|
"Kate has 20 pencils. She rests for 3 hours and buys 5 more pencils."
|
|
)
|
|
_CLEAN_ACCUMULATION = "Sam has 14 apples. He buys 9 more apples."
|
|
|
|
|
|
class TestClassifyDerivation:
|
|
def test_complete_reading_is_commit_eligible(self) -> None:
|
|
# uses every quantity -> complete
|
|
derivation = accumulation_candidates(_CLEAN_ACCUMULATION)[0]
|
|
assert classify_derivation(derivation, _CLEAN_ACCUMULATION) == "complete"
|
|
|
|
def test_isolated_foreign_unused_is_exempt(self) -> None:
|
|
# the additive reading of 0014 leaves "3 hours" unused; hours is foreign to
|
|
# the used unit (pencils) -> exempt (commit-ineligible).
|
|
derivation = GroundedDerivation(
|
|
start=Quantity(20.0, "pencils", "20"),
|
|
steps=(Step(op="add", operand=Quantity(5.0, "pencils", "5"), cue="more"),),
|
|
)
|
|
assert classify_derivation(derivation, _DISTRACTOR_0014) == "exempt"
|
|
|
|
def test_same_unit_unused_is_not_exempt(self) -> None:
|
|
# a quantity sharing the reading's unit is real signal, never a distractor:
|
|
# leaving it unused must NOT be exempted (it stays invalid -> None).
|
|
text = "Sam has 14 apples. He buys 9 more apples. He eats 2 apples."
|
|
derivation = GroundedDerivation(
|
|
start=Quantity(14.0, "apples", "14"),
|
|
steps=(Step(op="add", operand=Quantity(9.0, "apples", "9"), cue="more"),),
|
|
)
|
|
assert classify_derivation(derivation, text) is None
|
|
|
|
def test_empty_unit_unused_is_not_exempt(self) -> None:
|
|
# an unused quantity with an unknown (empty) unit cannot be shown foreign,
|
|
# so it is never exempt — completeness still rejects the reading.
|
|
text = "Sam has 14 apples. He buys 9 more apples. He had 2."
|
|
derivation = GroundedDerivation(
|
|
start=Quantity(14.0, "apples", "14"),
|
|
steps=(Step(op="add", operand=Quantity(9.0, "apples", "9"), cue="more"),),
|
|
)
|
|
assert classify_derivation(derivation, text) is None
|
|
|
|
def test_ungrounded_operand_is_invalid(self) -> None:
|
|
derivation = GroundedDerivation(
|
|
start=Quantity(14.0, "apples", "14"),
|
|
steps=(Step(op="add", operand=Quantity(999.0, "apples", "999"), cue="more"),),
|
|
)
|
|
assert classify_derivation(derivation, _CLEAN_ACCUMULATION) is None
|
|
|
|
|
|
class TestResolvePooled:
|
|
def test_clean_accumulation_commits(self) -> None:
|
|
resolution = resolve_pooled(_CLEAN_ACCUMULATION)
|
|
assert resolution is not None
|
|
assert resolution.answer == 23.0
|
|
|
|
def test_distractor_0014_refuses_via_disagreement(self) -> None:
|
|
# product 300 (complete) vs additive 25 (exempt) disagree -> refuse.
|
|
assert resolve_pooled(_DISTRACTOR_0014) is None
|
|
|
|
def test_exempt_only_never_commits(self) -> None:
|
|
# THE wrong=0-critical obligation. The only verifying reading here is the
|
|
# exempt additive one (no multiplicative cue -> no competing product), so a
|
|
# commit would be on an incomplete reading. The commit-ineligibility rule
|
|
# must refuse. Flipping `exempt` to commit-eligible makes this commit 25 and
|
|
# this test fails loudly.
|
|
assert accumulation_candidates(_EXEMPT_ONLY), "expected an exempt candidate"
|
|
assert classify_derivation(
|
|
accumulation_candidates(_EXEMPT_ONLY)[-1], _EXEMPT_ONLY
|
|
) == "exempt"
|
|
assert resolve_pooled(_EXEMPT_ONLY) is None
|
|
|
|
def test_deterministic(self) -> None:
|
|
assert resolve_pooled(_DISTRACTOR_0014) == resolve_pooled(_DISTRACTOR_0014)
|
|
a = resolve_pooled(_CLEAN_ACCUMULATION)
|
|
b = resolve_pooled(_CLEAN_ACCUMULATION)
|
|
assert a is not None and b is not None and a.answer == b.answer
|