feat(adr-0182): prior-state question guard — temporal-scope confuser refuses (confuser wrong 2->1, pair-tells ->0) (#480)
The "before/left" reading-rule lever. The microscope showed only the question-time
reading is cleanly achievable: "for N money = spend" is cue-precision-blocked (the
`for` cue is overloaded across train_sample -- `for $2`, `for 14 days`, `for 10 reps`
-- so a spend rule risks regressing train-0021/0003 and is the overfitting trap),
and the disguised-polarity cases already refuse via pooling. "left" is already handled
by loss verbs.
What ships: a question-scope guard. A question asking for a state *before* a stated
change ("How much did Lisa have before lunch?", gold 50) asks for a temporal point
the forward composers do not compute -- they derive the final/net state (50-20=30).
Until a question-time reader exists that is a refusal, never a guess at the wrong
point. target.asks_prior_state detects before/initially/originally/at first/to begin
with/to start with/at the start in the QUESTION CLAUSE only (the last `?`-sentence),
so body narrative does not trip it -- verified safe against train-0003 ("sells before
school starts"), 0010 ("had 20 initially, then lost 12"), 0028. `used to` is excluded
(the purpose infinitive "beads used to make a bracelet" is a false positive).
resolve_pooled refuses when asks_prior_state holds.
Result (sealed lane; chat/ does not import these -> serving 3/47/0 frozen):
- confuser wrong 2 -> 1 (only 0016 distractor-anchor-skip remains). temporal-scope
category cleared (0 wrong / 2 refused). pair-tells 1 -> 0: 0020 ("before lunch")
refuses while its minimal-pair twin 0021 ("left", gold 30) still solves the net --
the discrimination the corpus is built to measure.
- genuine positives still 7 solved, 0 wrong.
- train_sample 3/47/0 and practice 3/47/0 byte-identical.
- 205 derivation/target/pool tests + 40 architectural invariants green.
Tests:
- test_adr_0182_pool.py TestPriorStateQuestionGuard: detector true/false edges
(before-in-question vs before-in-body vs `used to make` false positive vs the
`left` twin) + resolve_pooled refuses the before-question while the left-twin
resolves forward to 30.
- test_adr_0163_f2_confusers.py: baseline wrong 2->1, pair_tells 1->0; new
test_temporal_scope_does_not_misfire + test_before_left_minimal_pair_discriminated.
Stacked on #476 (pooling); merge #476 first. 0016 anchor-skip is the next lever.
Pairs with ADR-0163-F2 graduation amendment (#478).
This commit is contained in:
parent
ac7bda019b
commit
a862d73084
5 changed files with 109 additions and 3 deletions
|
|
@ -28,7 +28,7 @@ from generate.derivation.search import (
|
|||
multiplicative_candidates,
|
||||
search_multiplicative,
|
||||
)
|
||||
from generate.derivation.target import Target, extract_target
|
||||
from generate.derivation.target import Target, asks_prior_state, extract_target
|
||||
from generate.derivation.verify import (
|
||||
Resolution,
|
||||
SelfVerification,
|
||||
|
|
@ -49,6 +49,7 @@ __all__ = [
|
|||
"Target",
|
||||
"VALID_OPS",
|
||||
"accumulation_candidates",
|
||||
"asks_prior_state",
|
||||
"candidate_chains",
|
||||
"classify_derivation",
|
||||
"clause_local_results",
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ from generate.derivation.accumulate import accumulation_candidates
|
|||
from generate.derivation.model import GroundedDerivation
|
||||
from generate.derivation.multistep import candidate_chains
|
||||
from generate.derivation.search import multiplicative_candidates
|
||||
from generate.derivation.target import asks_prior_state
|
||||
from generate.derivation.verify import Resolution, classify_derivation
|
||||
|
||||
|
||||
|
|
@ -63,7 +64,15 @@ def resolve_pooled(problem_text: str) -> Resolution | None:
|
|||
|
||||
Refuses on no verifying candidate, on disagreement among the pool, or when the
|
||||
sole answer is produced only by commit-ineligible (``exempt``) readings.
|
||||
|
||||
ADR-0182 question-scope guard: a question asking for a *prior* state ("how much
|
||||
did Lisa have **before** lunch?") asks for a temporal point the forward composers
|
||||
do not compute — they derive the final/net state. Until a question-time reader
|
||||
exists, that is a refusal, never a guess at the wrong point.
|
||||
"""
|
||||
if asks_prior_state(problem_text):
|
||||
return None
|
||||
|
||||
classified = [
|
||||
(kind, derivation)
|
||||
for derivation in pooled_candidates(problem_text)
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ weaker prune and leans on completeness, or refuses.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -37,6 +38,40 @@ from generate.math_roundtrip import _tokens
|
|||
_AGG_WORDS: Final[tuple[str, ...]] = ("total", "altogether", "combined", "sum")
|
||||
_AGG_PHRASES: Final[tuple[str, ...]] = ("in all", "in total")
|
||||
|
||||
# ADR-0182 — prior-state question markers. A question that asks for a state *before*
|
||||
# a stated change ("how much did Lisa have **before** lunch?", "...**initially**?")
|
||||
# is asking for a temporal point the forward reader does not compute — it derives
|
||||
# the final/net state. Detected on the **question clause only**: "before"/"initially"
|
||||
# in the problem *body* is narrative ("sells before school starts", "had 20
|
||||
# initially, then lost 12") and must not trip this. ``used to`` is excluded — the
|
||||
# purpose infinitive ("beads used to make a bracelet") is a false positive, not a
|
||||
# prior-state cue. Lexeme-level (ADR-0165): it names markers, it does not parse the
|
||||
# question's grammar.
|
||||
_PRIOR_STATE_RE: Final[re.Pattern[str]] = re.compile(
|
||||
r"\b(before|initially|originally|at first|to begin with|to start with|at the start)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_SENTENCE_SPLIT: Final[re.Pattern[str]] = re.compile(r"(?<=[.?!])\s+")
|
||||
|
||||
|
||||
def _question_clause(problem_text: str) -> str:
|
||||
"""The interrogative clause: the last ``?``-terminated sentence, else the last
|
||||
sentence. Deterministic."""
|
||||
sentences = [s for s in _SENTENCE_SPLIT.split(problem_text.strip()) if s.strip()]
|
||||
if not sentences:
|
||||
return problem_text
|
||||
questions = [s for s in sentences if s.rstrip().endswith("?")]
|
||||
return questions[-1] if questions else sentences[-1]
|
||||
|
||||
|
||||
def asks_prior_state(problem_text: str) -> bool:
|
||||
"""True iff the *question* asks for a state before a stated change (ADR-0182).
|
||||
|
||||
Question-clause-scoped so body narrative ("before school starts") does not trip
|
||||
it. The forward reader computes the final state, so a prior-state question is a
|
||||
refusal until a question-time reader exists — never a guess at the wrong point."""
|
||||
return bool(_PRIOR_STATE_RE.search(_question_clause(problem_text)))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Target:
|
||||
|
|
|
|||
|
|
@ -30,8 +30,12 @@ from evals.gsm8k_math.confusers.v1.runner import (
|
|||
# the spurious "for N coins" product disagrees with the accumulation reading)
|
||||
# now refuse. wrong 5->2 (0016 needs anchor-skip; 0020 is temporal-scope).
|
||||
# pair_tells 4->1 (0001/0003/0014 clear; 0020 remains).
|
||||
_BASELINE_WRONG = 2
|
||||
_BASELINE_PAIR_TELLS = 1
|
||||
# 2026-05-29 (ADR-0182 prior-state question guard): a question asking for a state
|
||||
# *before* a stated change ("...before lunch?") is refused — the forward reader
|
||||
# computes the net, the wrong temporal point. 0020 refuses; its 'left' twin 0021
|
||||
# still solves. wrong 2->1 (only 0016 remains). pair_tells 1->0.
|
||||
_BASELINE_WRONG = 1
|
||||
_BASELINE_PAIR_TELLS = 0
|
||||
|
||||
|
||||
class TestSchema:
|
||||
|
|
@ -111,5 +115,21 @@ class TestProbeBaseline:
|
|||
assert disguised
|
||||
assert all(r.verdict != "wrong" for r in disguised)
|
||||
|
||||
def test_temporal_scope_does_not_misfire(self) -> None:
|
||||
# ADR-0182 prior-state question guard: a "before <event>?" question is
|
||||
# refused (the forward reader computes the net, the wrong temporal point).
|
||||
# Fails loudly if the guard regresses (0020 would commit 30 again).
|
||||
results = run_probe()
|
||||
temporal = [r for r in results if r.category == "temporal-scope"]
|
||||
assert temporal
|
||||
assert all(r.verdict != "wrong" for r in temporal)
|
||||
|
||||
def test_before_left_minimal_pair_discriminated(self) -> None:
|
||||
# 0020 ("before lunch") refuses; its twin 0021 ("left") solves the net.
|
||||
# This is the pair-consistency win (the last surface-match tell cleared).
|
||||
by_id = {r.case_id: r for r in run_probe()}
|
||||
assert by_id["confuser-v1-0020"].verdict == "refused"
|
||||
assert by_id["confuser-v1-0021"].verdict == "solved"
|
||||
|
||||
def test_deterministic(self) -> None:
|
||||
assert summarize(run_probe()) == summarize(run_probe())
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from __future__ import annotations
|
|||
from generate.derivation.accumulate import accumulation_candidates
|
||||
from generate.derivation.model import GroundedDerivation, Quantity, Step
|
||||
from generate.derivation.pool import resolve_pooled
|
||||
from generate.derivation.target import asks_prior_state
|
||||
from generate.derivation.verify import classify_derivation
|
||||
|
||||
_DISTRACTOR_0014 = (
|
||||
|
|
@ -102,3 +103,43 @@ class TestResolvePooled:
|
|||
a = resolve_pooled(_CLEAN_ACCUMULATION)
|
||||
b = resolve_pooled(_CLEAN_ACCUMULATION)
|
||||
assert a is not None and b is not None and a.answer == b.answer
|
||||
|
||||
|
||||
_BEFORE_Q = "Lisa had 50 dollars. She spent 20 on lunch. How much money did Lisa have before lunch?"
|
||||
_LEFT_TWIN = "Lisa had 50 dollars. She spent 20 on lunch. How much money does Lisa have left?"
|
||||
|
||||
|
||||
class TestPriorStateQuestionGuard:
|
||||
"""ADR-0182 — a question asking for a *prior* state is refused (the forward
|
||||
composers compute the final state, the wrong temporal point). Question-clause
|
||||
scoped, so body narrative ('before school starts') does not trip it."""
|
||||
|
||||
def test_before_question_detected(self) -> None:
|
||||
assert asks_prior_state(_BEFORE_Q) is True
|
||||
|
||||
def test_left_twin_not_detected(self) -> None:
|
||||
# the minimal-pair twin asks for the net ('left') -> forward reading, solvable.
|
||||
assert asks_prior_state(_LEFT_TWIN) is False
|
||||
|
||||
def test_before_in_body_not_detected(self) -> None:
|
||||
# 'before' in narrative (not the question clause) must NOT trip the guard,
|
||||
# or it would wrongly refuse train-0003 (gold 864, currently committed).
|
||||
body_before = (
|
||||
"The student council sells erasers in the morning before school starts. "
|
||||
"There are 24 erasers in each box. If they sell 48 boxes, how many erasers?"
|
||||
)
|
||||
assert asks_prior_state(body_before) is False
|
||||
|
||||
def test_used_to_make_is_not_a_prior_marker(self) -> None:
|
||||
# the purpose infinitive 'used to make' is a false positive guarded against.
|
||||
assert asks_prior_state("If 50 beads are used to make one bracelet, how many bracelets?") is False
|
||||
|
||||
def test_prior_state_question_refuses(self) -> None:
|
||||
# the forward reading computes 50-20=30 (the net); the question asks the
|
||||
# pre-change state -> refuse, not commit 30.
|
||||
assert resolve_pooled(_BEFORE_Q) is None
|
||||
|
||||
def test_left_twin_still_resolves_forward(self) -> None:
|
||||
# discrimination: the twin asking 'left' commits the forward net (30).
|
||||
resolution = resolve_pooled(_LEFT_TWIN)
|
||||
assert resolution is not None and resolution.answer == 30.0
|
||||
|
|
|
|||
Loading…
Reference in a new issue