feat(adr-0182): prior-state question guard — temporal-scope confuser refuses (confuser wrong 2->1, pair-tells ->0) (#480)

The "before/left" reading-rule lever. The microscope showed only the question-time
reading is cleanly achievable: "for N money = spend" is cue-precision-blocked (the
`for` cue is overloaded across train_sample -- `for $2`, `for 14 days`, `for 10 reps`
-- so a spend rule risks regressing train-0021/0003 and is the overfitting trap),
and the disguised-polarity cases already refuse via pooling. "left" is already handled
by loss verbs.

What ships: a question-scope guard. A question asking for a state *before* a stated
change ("How much did Lisa have before lunch?", gold 50) asks for a temporal point
the forward composers do not compute -- they derive the final/net state (50-20=30).
Until a question-time reader exists that is a refusal, never a guess at the wrong
point. target.asks_prior_state detects before/initially/originally/at first/to begin
with/to start with/at the start in the QUESTION CLAUSE only (the last `?`-sentence),
so body narrative does not trip it -- verified safe against train-0003 ("sells before
school starts"), 0010 ("had 20 initially, then lost 12"), 0028. `used to` is excluded
(the purpose infinitive "beads used to make a bracelet" is a false positive).
resolve_pooled refuses when asks_prior_state holds.

Result (sealed lane; chat/ does not import these -> serving 3/47/0 frozen):
- confuser wrong 2 -> 1 (only 0016 distractor-anchor-skip remains). temporal-scope
  category cleared (0 wrong / 2 refused). pair-tells 1 -> 0: 0020 ("before lunch")
  refuses while its minimal-pair twin 0021 ("left", gold 30) still solves the net --
  the discrimination the corpus is built to measure.
- genuine positives still 7 solved, 0 wrong.
- train_sample 3/47/0 and practice 3/47/0 byte-identical.
- 205 derivation/target/pool tests + 40 architectural invariants green.

Tests:
- test_adr_0182_pool.py TestPriorStateQuestionGuard: detector true/false edges
  (before-in-question vs before-in-body vs `used to make` false positive vs the
  `left` twin) + resolve_pooled refuses the before-question while the left-twin
  resolves forward to 30.
- test_adr_0163_f2_confusers.py: baseline wrong 2->1, pair_tells 1->0; new
  test_temporal_scope_does_not_misfire + test_before_left_minimal_pair_discriminated.

Stacked on #476 (pooling); merge #476 first. 0016 anchor-skip is the next lever.
Pairs with ADR-0163-F2 graduation amendment (#478).
This commit is contained in:
Shay 2026-05-29 14:22:28 -07:00 committed by GitHub
parent ac7bda019b
commit a862d73084
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 109 additions and 3 deletions

View file

@ -28,7 +28,7 @@ from generate.derivation.search import (
multiplicative_candidates, multiplicative_candidates,
search_multiplicative, search_multiplicative,
) )
from generate.derivation.target import Target, extract_target from generate.derivation.target import Target, asks_prior_state, extract_target
from generate.derivation.verify import ( from generate.derivation.verify import (
Resolution, Resolution,
SelfVerification, SelfVerification,
@ -49,6 +49,7 @@ __all__ = [
"Target", "Target",
"VALID_OPS", "VALID_OPS",
"accumulation_candidates", "accumulation_candidates",
"asks_prior_state",
"candidate_chains", "candidate_chains",
"classify_derivation", "classify_derivation",
"clause_local_results", "clause_local_results",

View file

@ -33,6 +33,7 @@ from generate.derivation.accumulate import accumulation_candidates
from generate.derivation.model import GroundedDerivation from generate.derivation.model import GroundedDerivation
from generate.derivation.multistep import candidate_chains from generate.derivation.multistep import candidate_chains
from generate.derivation.search import multiplicative_candidates from generate.derivation.search import multiplicative_candidates
from generate.derivation.target import asks_prior_state
from generate.derivation.verify import Resolution, classify_derivation from generate.derivation.verify import Resolution, classify_derivation
@ -63,7 +64,15 @@ def resolve_pooled(problem_text: str) -> Resolution | None:
Refuses on no verifying candidate, on disagreement among the pool, or when the Refuses on no verifying candidate, on disagreement among the pool, or when the
sole answer is produced only by commit-ineligible (``exempt``) readings. sole answer is produced only by commit-ineligible (``exempt``) readings.
ADR-0182 question-scope guard: a question asking for a *prior* state ("how much
did Lisa have **before** lunch?") asks for a temporal point the forward composers
do not compute they derive the final/net state. Until a question-time reader
exists, that is a refusal, never a guess at the wrong point.
""" """
if asks_prior_state(problem_text):
return None
classified = [ classified = [
(kind, derivation) (kind, derivation)
for derivation in pooled_candidates(problem_text) for derivation in pooled_candidates(problem_text)

View file

@ -25,6 +25,7 @@ weaker prune and leans on completeness, or refuses.
from __future__ import annotations from __future__ import annotations
import re
from dataclasses import dataclass from dataclasses import dataclass
from typing import Final from typing import Final
@ -37,6 +38,40 @@ from generate.math_roundtrip import _tokens
_AGG_WORDS: Final[tuple[str, ...]] = ("total", "altogether", "combined", "sum") _AGG_WORDS: Final[tuple[str, ...]] = ("total", "altogether", "combined", "sum")
_AGG_PHRASES: Final[tuple[str, ...]] = ("in all", "in total") _AGG_PHRASES: Final[tuple[str, ...]] = ("in all", "in total")
# ADR-0182 — prior-state question markers. A question that asks for a state *before*
# a stated change ("how much did Lisa have **before** lunch?", "...**initially**?")
# is asking for a temporal point the forward reader does not compute — it derives
# the final/net state. Detected on the **question clause only**: "before"/"initially"
# in the problem *body* is narrative ("sells before school starts", "had 20
# initially, then lost 12") and must not trip this. ``used to`` is excluded — the
# purpose infinitive ("beads used to make a bracelet") is a false positive, not a
# prior-state cue. Lexeme-level (ADR-0165): it names markers, it does not parse the
# question's grammar.
_PRIOR_STATE_RE: Final[re.Pattern[str]] = re.compile(
r"\b(before|initially|originally|at first|to begin with|to start with|at the start)\b",
re.IGNORECASE,
)
_SENTENCE_SPLIT: Final[re.Pattern[str]] = re.compile(r"(?<=[.?!])\s+")
def _question_clause(problem_text: str) -> str:
"""The interrogative clause: the last ``?``-terminated sentence, else the last
sentence. Deterministic."""
sentences = [s for s in _SENTENCE_SPLIT.split(problem_text.strip()) if s.strip()]
if not sentences:
return problem_text
questions = [s for s in sentences if s.rstrip().endswith("?")]
return questions[-1] if questions else sentences[-1]
def asks_prior_state(problem_text: str) -> bool:
"""True iff the *question* asks for a state before a stated change (ADR-0182).
Question-clause-scoped so body narrative ("before school starts") does not trip
it. The forward reader computes the final state, so a prior-state question is a
refusal until a question-time reader exists never a guess at the wrong point."""
return bool(_PRIOR_STATE_RE.search(_question_clause(problem_text)))
@dataclass(frozen=True, slots=True) @dataclass(frozen=True, slots=True)
class Target: class Target:

View file

@ -30,8 +30,12 @@ from evals.gsm8k_math.confusers.v1.runner import (
# the spurious "for N coins" product disagrees with the accumulation reading) # the spurious "for N coins" product disagrees with the accumulation reading)
# now refuse. wrong 5->2 (0016 needs anchor-skip; 0020 is temporal-scope). # now refuse. wrong 5->2 (0016 needs anchor-skip; 0020 is temporal-scope).
# pair_tells 4->1 (0001/0003/0014 clear; 0020 remains). # pair_tells 4->1 (0001/0003/0014 clear; 0020 remains).
_BASELINE_WRONG = 2 # 2026-05-29 (ADR-0182 prior-state question guard): a question asking for a state
_BASELINE_PAIR_TELLS = 1 # *before* a stated change ("...before lunch?") is refused — the forward reader
# computes the net, the wrong temporal point. 0020 refuses; its 'left' twin 0021
# still solves. wrong 2->1 (only 0016 remains). pair_tells 1->0.
_BASELINE_WRONG = 1
_BASELINE_PAIR_TELLS = 0
class TestSchema: class TestSchema:
@ -111,5 +115,21 @@ class TestProbeBaseline:
assert disguised assert disguised
assert all(r.verdict != "wrong" for r in disguised) assert all(r.verdict != "wrong" for r in disguised)
def test_temporal_scope_does_not_misfire(self) -> None:
# ADR-0182 prior-state question guard: a "before <event>?" question is
# refused (the forward reader computes the net, the wrong temporal point).
# Fails loudly if the guard regresses (0020 would commit 30 again).
results = run_probe()
temporal = [r for r in results if r.category == "temporal-scope"]
assert temporal
assert all(r.verdict != "wrong" for r in temporal)
def test_before_left_minimal_pair_discriminated(self) -> None:
# 0020 ("before lunch") refuses; its twin 0021 ("left") solves the net.
# This is the pair-consistency win (the last surface-match tell cleared).
by_id = {r.case_id: r for r in run_probe()}
assert by_id["confuser-v1-0020"].verdict == "refused"
assert by_id["confuser-v1-0021"].verdict == "solved"
def test_deterministic(self) -> None: def test_deterministic(self) -> None:
assert summarize(run_probe()) == summarize(run_probe()) assert summarize(run_probe()) == summarize(run_probe())

View file

@ -18,6 +18,7 @@ from __future__ import annotations
from generate.derivation.accumulate import accumulation_candidates from generate.derivation.accumulate import accumulation_candidates
from generate.derivation.model import GroundedDerivation, Quantity, Step from generate.derivation.model import GroundedDerivation, Quantity, Step
from generate.derivation.pool import resolve_pooled from generate.derivation.pool import resolve_pooled
from generate.derivation.target import asks_prior_state
from generate.derivation.verify import classify_derivation from generate.derivation.verify import classify_derivation
_DISTRACTOR_0014 = ( _DISTRACTOR_0014 = (
@ -102,3 +103,43 @@ class TestResolvePooled:
a = resolve_pooled(_CLEAN_ACCUMULATION) a = resolve_pooled(_CLEAN_ACCUMULATION)
b = resolve_pooled(_CLEAN_ACCUMULATION) b = resolve_pooled(_CLEAN_ACCUMULATION)
assert a is not None and b is not None and a.answer == b.answer assert a is not None and b is not None and a.answer == b.answer
_BEFORE_Q = "Lisa had 50 dollars. She spent 20 on lunch. How much money did Lisa have before lunch?"
_LEFT_TWIN = "Lisa had 50 dollars. She spent 20 on lunch. How much money does Lisa have left?"
class TestPriorStateQuestionGuard:
"""ADR-0182 — a question asking for a *prior* state is refused (the forward
composers compute the final state, the wrong temporal point). Question-clause
scoped, so body narrative ('before school starts') does not trip it."""
def test_before_question_detected(self) -> None:
assert asks_prior_state(_BEFORE_Q) is True
def test_left_twin_not_detected(self) -> None:
# the minimal-pair twin asks for the net ('left') -> forward reading, solvable.
assert asks_prior_state(_LEFT_TWIN) is False
def test_before_in_body_not_detected(self) -> None:
# 'before' in narrative (not the question clause) must NOT trip the guard,
# or it would wrongly refuse train-0003 (gold 864, currently committed).
body_before = (
"The student council sells erasers in the morning before school starts. "
"There are 24 erasers in each box. If they sell 48 boxes, how many erasers?"
)
assert asks_prior_state(body_before) is False
def test_used_to_make_is_not_a_prior_marker(self) -> None:
# the purpose infinitive 'used to make' is a false positive guarded against.
assert asks_prior_state("If 50 beads are used to make one bracelet, how many bracelets?") is False
def test_prior_state_question_refuses(self) -> None:
# the forward reading computes 50-20=30 (the net); the question asks the
# pre-change state -> refuse, not commit 30.
assert resolve_pooled(_BEFORE_Q) is None
def test_left_twin_still_resolves_forward(self) -> None:
# discrimination: the twin asking 'left' commits the forward net (30).
resolution = resolve_pooled(_LEFT_TWIN)
assert resolution is not None and resolution.answer == 30.0