The "before/left" reading-rule lever. The microscope showed only the question-time
reading is cleanly achievable: "for N money = spend" is cue-precision-blocked (the
`for` cue is overloaded across train_sample -- `for $2`, `for 14 days`, `for 10 reps`
-- so a spend rule risks regressing train-0021/0003 and is the overfitting trap),
and the disguised-polarity cases already refuse via pooling. "left" is already handled
by loss verbs.
What ships: a question-scope guard. A question asking for a state *before* a stated
change ("How much did Lisa have before lunch?", gold 50) asks for a temporal point
the forward composers do not compute -- they derive the final/net state (50-20=30).
Until a question-time reader exists that is a refusal, never a guess at the wrong
point. target.asks_prior_state detects before/initially/originally/at first/to begin
with/to start with/at the start in the QUESTION CLAUSE only (the last `?`-sentence),
so body narrative does not trip it -- verified safe against train-0003 ("sells before
school starts"), 0010 ("had 20 initially, then lost 12"), 0028. `used to` is excluded
(the purpose infinitive "beads used to make a bracelet" is a false positive).
resolve_pooled refuses when asks_prior_state holds.
Result (sealed lane; chat/ does not import these -> serving 3/47/0 frozen):
- confuser wrong 2 -> 1 (only 0016 distractor-anchor-skip remains). temporal-scope
category cleared (0 wrong / 2 refused). pair-tells 1 -> 0: 0020 ("before lunch")
refuses while its minimal-pair twin 0021 ("left", gold 30) still solves the net --
the discrimination the corpus is built to measure.
- genuine positives still 7 solved, 0 wrong.
- train_sample 3/47/0 and practice 3/47/0 byte-identical.
- 205 derivation/target/pool tests + 40 architectural invariants green.
Tests:
- test_adr_0182_pool.py TestPriorStateQuestionGuard: detector true/false edges
(before-in-question vs before-in-body vs `used to make` false positive vs the
`left` twin) + resolve_pooled refuses the before-question while the left-twin
resolves forward to 30.
- test_adr_0163_f2_confusers.py: baseline wrong 2->1, pair_tells 1->0; new
test_temporal_scope_does_not_misfire + test_before_left_minimal_pair_discriminated.
Stacked on #476 (pooling); merge #476 first. 0016 anchor-skip is the next lever.
Pairs with ADR-0163-F2 graduation amendment (#478).
145 lines
7 KiB
Python
145 lines
7 KiB
Python
"""ADR-0182 — cross-composer disagreement pooling (distractor refusal).
|
|
|
|
Two surfaces under test:
|
|
|
|
* :func:`generate.derivation.verify.classify_derivation` — the commit-eligibility
|
|
class (``complete`` / ``exempt`` / ``None``) and, critically, that the
|
|
isolated-foreign exemption is *narrow* (an empty-unit or same-unit unused
|
|
quantity is never exempt — it is real signal, not a distractor).
|
|
* :func:`generate.derivation.pool.resolve_pooled` — the pooled resolution: a clean
|
|
reading commits, a distractor problem's product-vs-additive disagreement refuses,
|
|
and (the wrong=0-critical property) an ``exempt``-only answer **never commits**.
|
|
|
|
Sealed lane: ``chat/`` does not import these; serving ``3/47/0`` cannot move.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from generate.derivation.accumulate import accumulation_candidates
|
|
from generate.derivation.model import GroundedDerivation, Quantity, Step
|
|
from generate.derivation.pool import resolve_pooled
|
|
from generate.derivation.target import asks_prior_state
|
|
from generate.derivation.verify import classify_derivation
|
|
|
|
_DISTRACTOR_0014 = (
|
|
"Kate has 20 pencils. She studies for 3 hours and then buys 5 more pencils. "
|
|
"How many pencils does Kate have?"
|
|
)
|
|
# A distractor with NO multiplicative cue: the only candidate is the exempt additive
|
|
# reading (no complete product exists). The commit-ineligibility rule must refuse it.
|
|
_EXEMPT_ONLY = (
|
|
"Kate has 20 pencils. She rests for 3 hours and buys 5 more pencils."
|
|
)
|
|
_CLEAN_ACCUMULATION = "Sam has 14 apples. He buys 9 more apples."
|
|
|
|
|
|
class TestClassifyDerivation:
|
|
def test_complete_reading_is_commit_eligible(self) -> None:
|
|
# uses every quantity -> complete
|
|
derivation = accumulation_candidates(_CLEAN_ACCUMULATION)[0]
|
|
assert classify_derivation(derivation, _CLEAN_ACCUMULATION) == "complete"
|
|
|
|
def test_isolated_foreign_unused_is_exempt(self) -> None:
|
|
# the additive reading of 0014 leaves "3 hours" unused; hours is foreign to
|
|
# the used unit (pencils) -> exempt (commit-ineligible).
|
|
derivation = GroundedDerivation(
|
|
start=Quantity(20.0, "pencils", "20"),
|
|
steps=(Step(op="add", operand=Quantity(5.0, "pencils", "5"), cue="more"),),
|
|
)
|
|
assert classify_derivation(derivation, _DISTRACTOR_0014) == "exempt"
|
|
|
|
def test_same_unit_unused_is_not_exempt(self) -> None:
|
|
# a quantity sharing the reading's unit is real signal, never a distractor:
|
|
# leaving it unused must NOT be exempted (it stays invalid -> None).
|
|
text = "Sam has 14 apples. He buys 9 more apples. He eats 2 apples."
|
|
derivation = GroundedDerivation(
|
|
start=Quantity(14.0, "apples", "14"),
|
|
steps=(Step(op="add", operand=Quantity(9.0, "apples", "9"), cue="more"),),
|
|
)
|
|
assert classify_derivation(derivation, text) is None
|
|
|
|
def test_empty_unit_unused_is_not_exempt(self) -> None:
|
|
# an unused quantity with an unknown (empty) unit cannot be shown foreign,
|
|
# so it is never exempt — completeness still rejects the reading.
|
|
text = "Sam has 14 apples. He buys 9 more apples. He had 2."
|
|
derivation = GroundedDerivation(
|
|
start=Quantity(14.0, "apples", "14"),
|
|
steps=(Step(op="add", operand=Quantity(9.0, "apples", "9"), cue="more"),),
|
|
)
|
|
assert classify_derivation(derivation, text) is None
|
|
|
|
def test_ungrounded_operand_is_invalid(self) -> None:
|
|
derivation = GroundedDerivation(
|
|
start=Quantity(14.0, "apples", "14"),
|
|
steps=(Step(op="add", operand=Quantity(999.0, "apples", "999"), cue="more"),),
|
|
)
|
|
assert classify_derivation(derivation, _CLEAN_ACCUMULATION) is None
|
|
|
|
|
|
class TestResolvePooled:
|
|
def test_clean_accumulation_commits(self) -> None:
|
|
resolution = resolve_pooled(_CLEAN_ACCUMULATION)
|
|
assert resolution is not None
|
|
assert resolution.answer == 23.0
|
|
|
|
def test_distractor_0014_refuses_via_disagreement(self) -> None:
|
|
# product 300 (complete) vs additive 25 (exempt) disagree -> refuse.
|
|
assert resolve_pooled(_DISTRACTOR_0014) is None
|
|
|
|
def test_exempt_only_never_commits(self) -> None:
|
|
# THE wrong=0-critical obligation. The only verifying reading here is the
|
|
# exempt additive one (no multiplicative cue -> no competing product), so a
|
|
# commit would be on an incomplete reading. The commit-ineligibility rule
|
|
# must refuse. Flipping `exempt` to commit-eligible makes this commit 25 and
|
|
# this test fails loudly.
|
|
assert accumulation_candidates(_EXEMPT_ONLY), "expected an exempt candidate"
|
|
assert classify_derivation(
|
|
accumulation_candidates(_EXEMPT_ONLY)[-1], _EXEMPT_ONLY
|
|
) == "exempt"
|
|
assert resolve_pooled(_EXEMPT_ONLY) is None
|
|
|
|
def test_deterministic(self) -> None:
|
|
assert resolve_pooled(_DISTRACTOR_0014) == resolve_pooled(_DISTRACTOR_0014)
|
|
a = resolve_pooled(_CLEAN_ACCUMULATION)
|
|
b = resolve_pooled(_CLEAN_ACCUMULATION)
|
|
assert a is not None and b is not None and a.answer == b.answer
|
|
|
|
|
|
_BEFORE_Q = "Lisa had 50 dollars. She spent 20 on lunch. How much money did Lisa have before lunch?"
|
|
_LEFT_TWIN = "Lisa had 50 dollars. She spent 20 on lunch. How much money does Lisa have left?"
|
|
|
|
|
|
class TestPriorStateQuestionGuard:
|
|
"""ADR-0182 — a question asking for a *prior* state is refused (the forward
|
|
composers compute the final state, the wrong temporal point). Question-clause
|
|
scoped, so body narrative ('before school starts') does not trip it."""
|
|
|
|
def test_before_question_detected(self) -> None:
|
|
assert asks_prior_state(_BEFORE_Q) is True
|
|
|
|
def test_left_twin_not_detected(self) -> None:
|
|
# the minimal-pair twin asks for the net ('left') -> forward reading, solvable.
|
|
assert asks_prior_state(_LEFT_TWIN) is False
|
|
|
|
def test_before_in_body_not_detected(self) -> None:
|
|
# 'before' in narrative (not the question clause) must NOT trip the guard,
|
|
# or it would wrongly refuse train-0003 (gold 864, currently committed).
|
|
body_before = (
|
|
"The student council sells erasers in the morning before school starts. "
|
|
"There are 24 erasers in each box. If they sell 48 boxes, how many erasers?"
|
|
)
|
|
assert asks_prior_state(body_before) is False
|
|
|
|
def test_used_to_make_is_not_a_prior_marker(self) -> None:
|
|
# the purpose infinitive 'used to make' is a false positive guarded against.
|
|
assert asks_prior_state("If 50 beads are used to make one bracelet, how many bracelets?") is False
|
|
|
|
def test_prior_state_question_refuses(self) -> None:
|
|
# the forward reading computes 50-20=30 (the net); the question asks the
|
|
# pre-change state -> refuse, not commit 30.
|
|
assert resolve_pooled(_BEFORE_Q) is None
|
|
|
|
def test_left_twin_still_resolves_forward(self) -> None:
|
|
# discrimination: the twin asking 'left' commits the forward net (30).
|
|
resolution = resolve_pooled(_LEFT_TWIN)
|
|
assert resolution is not None and resolution.answer == 30.0
|