feat(adr-0182): prior-state question guard — temporal-scope confuser refuses (confuser wrong 2->1, pair-tells ->0) (#480)

The "before/left" reading-rule lever. The microscope showed only the question-time
reading is cleanly achievable: "for N money = spend" is cue-precision-blocked (the
`for` cue is overloaded across train_sample -- `for $2`, `for 14 days`, `for 10 reps`
-- so a spend rule risks regressing train-0021/0003 and is the overfitting trap),
and the disguised-polarity cases already refuse via pooling. "left" is already handled
by loss verbs.

What ships: a question-scope guard. A question asking for a state *before* a stated
change ("How much did Lisa have before lunch?", gold 50) asks for a temporal point
the forward composers do not compute -- they derive the final/net state (50-20=30).
Until a question-time reader exists that is a refusal, never a guess at the wrong
point. target.asks_prior_state detects before/initially/originally/at first/to begin
with/to start with/at the start in the QUESTION CLAUSE only (the last `?`-sentence),
so body narrative does not trip it -- verified safe against train-0003 ("sells before
school starts"), 0010 ("had 20 initially, then lost 12"), 0028. `used to` is excluded
(the purpose infinitive "beads used to make a bracelet" is a false positive).
resolve_pooled refuses when asks_prior_state holds.

Result (sealed lane; chat/ does not import these -> serving 3/47/0 frozen):
- confuser wrong 2 -> 1 (only 0016 distractor-anchor-skip remains). temporal-scope
  category cleared (0 wrong / 2 refused). pair-tells 1 -> 0: 0020 ("before lunch")
  refuses while its minimal-pair twin 0021 ("left", gold 30) still solves the net --
  the discrimination the corpus is built to measure.
- genuine positives still 7 solved, 0 wrong.
- train_sample 3/47/0 and practice 3/47/0 byte-identical.
- 205 derivation/target/pool tests + 40 architectural invariants green.

Tests:
- test_adr_0182_pool.py TestPriorStateQuestionGuard: detector true/false edges
  (before-in-question vs before-in-body vs `used to make` false positive vs the
  `left` twin) + resolve_pooled refuses the before-question while the left-twin
  resolves forward to 30.
- test_adr_0163_f2_confusers.py: baseline wrong 2->1, pair_tells 1->0; new
  test_temporal_scope_does_not_misfire + test_before_left_minimal_pair_discriminated.

Stacked on #476 (pooling); merge #476 first. 0016 anchor-skip is the next lever.
Pairs with ADR-0163-F2 graduation amendment (#478).
This commit is contained in:
Shay 2026-05-29 14:22:28 -07:00 committed by GitHub
parent ac7bda019b
commit a862d73084
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 109 additions and 3 deletions

View file

@ -28,7 +28,7 @@ from generate.derivation.search import (
multiplicative_candidates,
search_multiplicative,
)
from generate.derivation.target import Target, extract_target
from generate.derivation.target import Target, asks_prior_state, extract_target
from generate.derivation.verify import (
Resolution,
SelfVerification,
@ -49,6 +49,7 @@ __all__ = [
"Target",
"VALID_OPS",
"accumulation_candidates",
"asks_prior_state",
"candidate_chains",
"classify_derivation",
"clause_local_results",

View file

@ -33,6 +33,7 @@ from generate.derivation.accumulate import accumulation_candidates
from generate.derivation.model import GroundedDerivation
from generate.derivation.multistep import candidate_chains
from generate.derivation.search import multiplicative_candidates
from generate.derivation.target import asks_prior_state
from generate.derivation.verify import Resolution, classify_derivation
@ -63,7 +64,15 @@ def resolve_pooled(problem_text: str) -> Resolution | None:
Refuses on no verifying candidate, on disagreement among the pool, or when the
sole answer is produced only by commit-ineligible (``exempt``) readings.
ADR-0182 question-scope guard: a question asking for a *prior* state ("how much
did Lisa have **before** lunch?") asks for a temporal point the forward composers
do not compute they derive the final/net state. Until a question-time reader
exists, that is a refusal, never a guess at the wrong point.
"""
if asks_prior_state(problem_text):
return None
classified = [
(kind, derivation)
for derivation in pooled_candidates(problem_text)

View file

@ -25,6 +25,7 @@ weaker prune and leans on completeness, or refuses.
from __future__ import annotations
import re
from dataclasses import dataclass
from typing import Final
@ -37,6 +38,40 @@ from generate.math_roundtrip import _tokens
_AGG_WORDS: Final[tuple[str, ...]] = ("total", "altogether", "combined", "sum")
_AGG_PHRASES: Final[tuple[str, ...]] = ("in all", "in total")
# ADR-0182 — prior-state question markers. A question that asks for a state *before*
# a stated change ("how much did Lisa have **before** lunch?", "...**initially**?")
# is asking for a temporal point the forward reader does not compute — it derives
# the final/net state. Detected on the **question clause only**: "before"/"initially"
# in the problem *body* is narrative ("sells before school starts", "had 20
# initially, then lost 12") and must not trip this. ``used to`` is excluded — the
# purpose infinitive ("beads used to make a bracelet") is a false positive, not a
# prior-state cue. Lexeme-level (ADR-0165): it names markers, it does not parse the
# question's grammar.
_PRIOR_STATE_RE: Final[re.Pattern[str]] = re.compile(
r"\b(before|initially|originally|at first|to begin with|to start with|at the start)\b",
re.IGNORECASE,
)
_SENTENCE_SPLIT: Final[re.Pattern[str]] = re.compile(r"(?<=[.?!])\s+")
def _question_clause(problem_text: str) -> str:
"""The interrogative clause: the last ``?``-terminated sentence, else the last
sentence. Deterministic."""
sentences = [s for s in _SENTENCE_SPLIT.split(problem_text.strip()) if s.strip()]
if not sentences:
return problem_text
questions = [s for s in sentences if s.rstrip().endswith("?")]
return questions[-1] if questions else sentences[-1]
def asks_prior_state(problem_text: str) -> bool:
"""True iff the *question* asks for a state before a stated change (ADR-0182).
Question-clause-scoped so body narrative ("before school starts") does not trip
it. The forward reader computes the final state, so a prior-state question is a
refusal until a question-time reader exists never a guess at the wrong point."""
return bool(_PRIOR_STATE_RE.search(_question_clause(problem_text)))
@dataclass(frozen=True, slots=True)
class Target:

View file

@ -30,8 +30,12 @@ from evals.gsm8k_math.confusers.v1.runner import (
# the spurious "for N coins" product disagrees with the accumulation reading)
# now refuse. wrong 5->2 (0016 needs anchor-skip; 0020 is temporal-scope).
# pair_tells 4->1 (0001/0003/0014 clear; 0020 remains).
_BASELINE_WRONG = 2
_BASELINE_PAIR_TELLS = 1
# 2026-05-29 (ADR-0182 prior-state question guard): a question asking for a state
# *before* a stated change ("...before lunch?") is refused — the forward reader
# computes the net, the wrong temporal point. 0020 refuses; its 'left' twin 0021
# still solves. wrong 2->1 (only 0016 remains). pair_tells 1->0.
_BASELINE_WRONG = 1
_BASELINE_PAIR_TELLS = 0
class TestSchema:
@ -111,5 +115,21 @@ class TestProbeBaseline:
assert disguised
assert all(r.verdict != "wrong" for r in disguised)
def test_temporal_scope_does_not_misfire(self) -> None:
# ADR-0182 prior-state question guard: a "before <event>?" question is
# refused (the forward reader computes the net, the wrong temporal point).
# Fails loudly if the guard regresses (0020 would commit 30 again).
results = run_probe()
temporal = [r for r in results if r.category == "temporal-scope"]
assert temporal
assert all(r.verdict != "wrong" for r in temporal)
def test_before_left_minimal_pair_discriminated(self) -> None:
# 0020 ("before lunch") refuses; its twin 0021 ("left") solves the net.
# This is the pair-consistency win (the last surface-match tell cleared).
by_id = {r.case_id: r for r in run_probe()}
assert by_id["confuser-v1-0020"].verdict == "refused"
assert by_id["confuser-v1-0021"].verdict == "solved"
def test_deterministic(self) -> None:
assert summarize(run_probe()) == summarize(run_probe())

View file

@ -18,6 +18,7 @@ from __future__ import annotations
from generate.derivation.accumulate import accumulation_candidates
from generate.derivation.model import GroundedDerivation, Quantity, Step
from generate.derivation.pool import resolve_pooled
from generate.derivation.target import asks_prior_state
from generate.derivation.verify import classify_derivation
_DISTRACTOR_0014 = (
@ -102,3 +103,43 @@ class TestResolvePooled:
a = resolve_pooled(_CLEAN_ACCUMULATION)
b = resolve_pooled(_CLEAN_ACCUMULATION)
assert a is not None and b is not None and a.answer == b.answer
_BEFORE_Q = "Lisa had 50 dollars. She spent 20 on lunch. How much money did Lisa have before lunch?"
_LEFT_TWIN = "Lisa had 50 dollars. She spent 20 on lunch. How much money does Lisa have left?"
class TestPriorStateQuestionGuard:
"""ADR-0182 — a question asking for a *prior* state is refused (the forward
composers compute the final state, the wrong temporal point). Question-clause
scoped, so body narrative ('before school starts') does not trip it."""
def test_before_question_detected(self) -> None:
assert asks_prior_state(_BEFORE_Q) is True
def test_left_twin_not_detected(self) -> None:
# the minimal-pair twin asks for the net ('left') -> forward reading, solvable.
assert asks_prior_state(_LEFT_TWIN) is False
def test_before_in_body_not_detected(self) -> None:
# 'before' in narrative (not the question clause) must NOT trip the guard,
# or it would wrongly refuse train-0003 (gold 864, currently committed).
body_before = (
"The student council sells erasers in the morning before school starts. "
"There are 24 erasers in each box. If they sell 48 boxes, how many erasers?"
)
assert asks_prior_state(body_before) is False
def test_used_to_make_is_not_a_prior_marker(self) -> None:
# the purpose infinitive 'used to make' is a false positive guarded against.
assert asks_prior_state("If 50 beads are used to make one bracelet, how many bracelets?") is False
def test_prior_state_question_refuses(self) -> None:
# the forward reading computes 50-20=30 (the net); the question asks the
# pre-change state -> refuse, not commit 30.
assert resolve_pooled(_BEFORE_Q) is None
def test_left_twin_still_resolves_forward(self) -> None:
# discrimination: the twin asking 'left' commits the forward net (30).
resolution = resolve_pooled(_LEFT_TWIN)
assert resolution is not None and resolution.answer == 30.0