feat(adr-0182): prior-state question guard — temporal-scope confuser refuses (confuser wrong 2->1, pair-tells ->0) (#480)
The "before/left" reading-rule lever. The microscope showed only the question-time
reading is cleanly achievable: "for N money = spend" is cue-precision-blocked (the
`for` cue is overloaded across train_sample -- `for $2`, `for 14 days`, `for 10 reps`
-- so a spend rule risks regressing train-0021/0003 and is the overfitting trap),
and the disguised-polarity cases already refuse via pooling. "left" is already handled
by loss verbs.
What ships: a question-scope guard. A question asking for a state *before* a stated
change ("How much did Lisa have before lunch?", gold 50) asks for a temporal point
the forward composers do not compute -- they derive the final/net state (50-20=30).
Until a question-time reader exists that is a refusal, never a guess at the wrong
point. target.asks_prior_state detects before/initially/originally/at first/to begin
with/to start with/at the start in the QUESTION CLAUSE only (the last `?`-sentence),
so body narrative does not trip it -- verified safe against train-0003 ("sells before
school starts"), 0010 ("had 20 initially, then lost 12"), 0028. `used to` is excluded
(the purpose infinitive "beads used to make a bracelet" is a false positive).
resolve_pooled refuses when asks_prior_state holds.
Result (sealed lane; chat/ does not import these -> serving 3/47/0 frozen):
- confuser wrong 2 -> 1 (only 0016 distractor-anchor-skip remains). temporal-scope
category cleared (0 wrong / 2 refused). pair-tells 1 -> 0: 0020 ("before lunch")
refuses while its minimal-pair twin 0021 ("left", gold 30) still solves the net --
the discrimination the corpus is built to measure.
- genuine positives still 7 solved, 0 wrong.
- train_sample 3/47/0 and practice 3/47/0 byte-identical.
- 205 derivation/target/pool tests + 40 architectural invariants green.
Tests:
- test_adr_0182_pool.py TestPriorStateQuestionGuard: detector true/false edges
(before-in-question vs before-in-body vs `used to make` false positive vs the
`left` twin) + resolve_pooled refuses the before-question while the left-twin
resolves forward to 30.
- test_adr_0163_f2_confusers.py: baseline wrong 2->1, pair_tells 1->0; new
test_temporal_scope_does_not_misfire + test_before_left_minimal_pair_discriminated.
Stacked on #476 (pooling); merge #476 first. 0016 anchor-skip is the next lever.
Pairs with ADR-0163-F2 graduation amendment (#478).
This commit is contained in:
parent
ac7bda019b
commit
a862d73084
5 changed files with 109 additions and 3 deletions
|
|
@ -28,7 +28,7 @@ from generate.derivation.search import (
|
||||||
multiplicative_candidates,
|
multiplicative_candidates,
|
||||||
search_multiplicative,
|
search_multiplicative,
|
||||||
)
|
)
|
||||||
from generate.derivation.target import Target, extract_target
|
from generate.derivation.target import Target, asks_prior_state, extract_target
|
||||||
from generate.derivation.verify import (
|
from generate.derivation.verify import (
|
||||||
Resolution,
|
Resolution,
|
||||||
SelfVerification,
|
SelfVerification,
|
||||||
|
|
@ -49,6 +49,7 @@ __all__ = [
|
||||||
"Target",
|
"Target",
|
||||||
"VALID_OPS",
|
"VALID_OPS",
|
||||||
"accumulation_candidates",
|
"accumulation_candidates",
|
||||||
|
"asks_prior_state",
|
||||||
"candidate_chains",
|
"candidate_chains",
|
||||||
"classify_derivation",
|
"classify_derivation",
|
||||||
"clause_local_results",
|
"clause_local_results",
|
||||||
|
|
|
||||||
|
|
@ -33,6 +33,7 @@ from generate.derivation.accumulate import accumulation_candidates
|
||||||
from generate.derivation.model import GroundedDerivation
|
from generate.derivation.model import GroundedDerivation
|
||||||
from generate.derivation.multistep import candidate_chains
|
from generate.derivation.multistep import candidate_chains
|
||||||
from generate.derivation.search import multiplicative_candidates
|
from generate.derivation.search import multiplicative_candidates
|
||||||
|
from generate.derivation.target import asks_prior_state
|
||||||
from generate.derivation.verify import Resolution, classify_derivation
|
from generate.derivation.verify import Resolution, classify_derivation
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -63,7 +64,15 @@ def resolve_pooled(problem_text: str) -> Resolution | None:
|
||||||
|
|
||||||
Refuses on no verifying candidate, on disagreement among the pool, or when the
|
Refuses on no verifying candidate, on disagreement among the pool, or when the
|
||||||
sole answer is produced only by commit-ineligible (``exempt``) readings.
|
sole answer is produced only by commit-ineligible (``exempt``) readings.
|
||||||
|
|
||||||
|
ADR-0182 question-scope guard: a question asking for a *prior* state ("how much
|
||||||
|
did Lisa have **before** lunch?") asks for a temporal point the forward composers
|
||||||
|
do not compute — they derive the final/net state. Until a question-time reader
|
||||||
|
exists, that is a refusal, never a guess at the wrong point.
|
||||||
"""
|
"""
|
||||||
|
if asks_prior_state(problem_text):
|
||||||
|
return None
|
||||||
|
|
||||||
classified = [
|
classified = [
|
||||||
(kind, derivation)
|
(kind, derivation)
|
||||||
for derivation in pooled_candidates(problem_text)
|
for derivation in pooled_candidates(problem_text)
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,7 @@ weaker prune and leans on completeness, or refuses.
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Final
|
from typing import Final
|
||||||
|
|
||||||
|
|
@ -37,6 +38,40 @@ from generate.math_roundtrip import _tokens
|
||||||
_AGG_WORDS: Final[tuple[str, ...]] = ("total", "altogether", "combined", "sum")
|
_AGG_WORDS: Final[tuple[str, ...]] = ("total", "altogether", "combined", "sum")
|
||||||
_AGG_PHRASES: Final[tuple[str, ...]] = ("in all", "in total")
|
_AGG_PHRASES: Final[tuple[str, ...]] = ("in all", "in total")
|
||||||
|
|
||||||
|
# ADR-0182 — prior-state question markers. A question that asks for a state *before*
|
||||||
|
# a stated change ("how much did Lisa have **before** lunch?", "...**initially**?")
|
||||||
|
# is asking for a temporal point the forward reader does not compute — it derives
|
||||||
|
# the final/net state. Detected on the **question clause only**: "before"/"initially"
|
||||||
|
# in the problem *body* is narrative ("sells before school starts", "had 20
|
||||||
|
# initially, then lost 12") and must not trip this. ``used to`` is excluded — the
|
||||||
|
# purpose infinitive ("beads used to make a bracelet") is a false positive, not a
|
||||||
|
# prior-state cue. Lexeme-level (ADR-0165): it names markers, it does not parse the
|
||||||
|
# question's grammar.
|
||||||
|
_PRIOR_STATE_RE: Final[re.Pattern[str]] = re.compile(
|
||||||
|
r"\b(before|initially|originally|at first|to begin with|to start with|at the start)\b",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
_SENTENCE_SPLIT: Final[re.Pattern[str]] = re.compile(r"(?<=[.?!])\s+")
|
||||||
|
|
||||||
|
|
||||||
|
def _question_clause(problem_text: str) -> str:
|
||||||
|
"""The interrogative clause: the last ``?``-terminated sentence, else the last
|
||||||
|
sentence. Deterministic."""
|
||||||
|
sentences = [s for s in _SENTENCE_SPLIT.split(problem_text.strip()) if s.strip()]
|
||||||
|
if not sentences:
|
||||||
|
return problem_text
|
||||||
|
questions = [s for s in sentences if s.rstrip().endswith("?")]
|
||||||
|
return questions[-1] if questions else sentences[-1]
|
||||||
|
|
||||||
|
|
||||||
|
def asks_prior_state(problem_text: str) -> bool:
|
||||||
|
"""True iff the *question* asks for a state before a stated change (ADR-0182).
|
||||||
|
|
||||||
|
Question-clause-scoped so body narrative ("before school starts") does not trip
|
||||||
|
it. The forward reader computes the final state, so a prior-state question is a
|
||||||
|
refusal until a question-time reader exists — never a guess at the wrong point."""
|
||||||
|
return bool(_PRIOR_STATE_RE.search(_question_clause(problem_text)))
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class Target:
|
class Target:
|
||||||
|
|
|
||||||
|
|
@ -30,8 +30,12 @@ from evals.gsm8k_math.confusers.v1.runner import (
|
||||||
# the spurious "for N coins" product disagrees with the accumulation reading)
|
# the spurious "for N coins" product disagrees with the accumulation reading)
|
||||||
# now refuse. wrong 5->2 (0016 needs anchor-skip; 0020 is temporal-scope).
|
# now refuse. wrong 5->2 (0016 needs anchor-skip; 0020 is temporal-scope).
|
||||||
# pair_tells 4->1 (0001/0003/0014 clear; 0020 remains).
|
# pair_tells 4->1 (0001/0003/0014 clear; 0020 remains).
|
||||||
_BASELINE_WRONG = 2
|
# 2026-05-29 (ADR-0182 prior-state question guard): a question asking for a state
|
||||||
_BASELINE_PAIR_TELLS = 1
|
# *before* a stated change ("...before lunch?") is refused — the forward reader
|
||||||
|
# computes the net, the wrong temporal point. 0020 refuses; its 'left' twin 0021
|
||||||
|
# still solves. wrong 2->1 (only 0016 remains). pair_tells 1->0.
|
||||||
|
_BASELINE_WRONG = 1
|
||||||
|
_BASELINE_PAIR_TELLS = 0
|
||||||
|
|
||||||
|
|
||||||
class TestSchema:
|
class TestSchema:
|
||||||
|
|
@ -111,5 +115,21 @@ class TestProbeBaseline:
|
||||||
assert disguised
|
assert disguised
|
||||||
assert all(r.verdict != "wrong" for r in disguised)
|
assert all(r.verdict != "wrong" for r in disguised)
|
||||||
|
|
||||||
|
def test_temporal_scope_does_not_misfire(self) -> None:
|
||||||
|
# ADR-0182 prior-state question guard: a "before <event>?" question is
|
||||||
|
# refused (the forward reader computes the net, the wrong temporal point).
|
||||||
|
# Fails loudly if the guard regresses (0020 would commit 30 again).
|
||||||
|
results = run_probe()
|
||||||
|
temporal = [r for r in results if r.category == "temporal-scope"]
|
||||||
|
assert temporal
|
||||||
|
assert all(r.verdict != "wrong" for r in temporal)
|
||||||
|
|
||||||
|
def test_before_left_minimal_pair_discriminated(self) -> None:
|
||||||
|
# 0020 ("before lunch") refuses; its twin 0021 ("left") solves the net.
|
||||||
|
# This is the pair-consistency win (the last surface-match tell cleared).
|
||||||
|
by_id = {r.case_id: r for r in run_probe()}
|
||||||
|
assert by_id["confuser-v1-0020"].verdict == "refused"
|
||||||
|
assert by_id["confuser-v1-0021"].verdict == "solved"
|
||||||
|
|
||||||
def test_deterministic(self) -> None:
|
def test_deterministic(self) -> None:
|
||||||
assert summarize(run_probe()) == summarize(run_probe())
|
assert summarize(run_probe()) == summarize(run_probe())
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,7 @@ from __future__ import annotations
|
||||||
from generate.derivation.accumulate import accumulation_candidates
|
from generate.derivation.accumulate import accumulation_candidates
|
||||||
from generate.derivation.model import GroundedDerivation, Quantity, Step
|
from generate.derivation.model import GroundedDerivation, Quantity, Step
|
||||||
from generate.derivation.pool import resolve_pooled
|
from generate.derivation.pool import resolve_pooled
|
||||||
|
from generate.derivation.target import asks_prior_state
|
||||||
from generate.derivation.verify import classify_derivation
|
from generate.derivation.verify import classify_derivation
|
||||||
|
|
||||||
_DISTRACTOR_0014 = (
|
_DISTRACTOR_0014 = (
|
||||||
|
|
@ -102,3 +103,43 @@ class TestResolvePooled:
|
||||||
a = resolve_pooled(_CLEAN_ACCUMULATION)
|
a = resolve_pooled(_CLEAN_ACCUMULATION)
|
||||||
b = resolve_pooled(_CLEAN_ACCUMULATION)
|
b = resolve_pooled(_CLEAN_ACCUMULATION)
|
||||||
assert a is not None and b is not None and a.answer == b.answer
|
assert a is not None and b is not None and a.answer == b.answer
|
||||||
|
|
||||||
|
|
||||||
|
_BEFORE_Q = "Lisa had 50 dollars. She spent 20 on lunch. How much money did Lisa have before lunch?"
|
||||||
|
_LEFT_TWIN = "Lisa had 50 dollars. She spent 20 on lunch. How much money does Lisa have left?"
|
||||||
|
|
||||||
|
|
||||||
|
class TestPriorStateQuestionGuard:
|
||||||
|
"""ADR-0182 — a question asking for a *prior* state is refused (the forward
|
||||||
|
composers compute the final state, the wrong temporal point). Question-clause
|
||||||
|
scoped, so body narrative ('before school starts') does not trip it."""
|
||||||
|
|
||||||
|
def test_before_question_detected(self) -> None:
|
||||||
|
assert asks_prior_state(_BEFORE_Q) is True
|
||||||
|
|
||||||
|
def test_left_twin_not_detected(self) -> None:
|
||||||
|
# the minimal-pair twin asks for the net ('left') -> forward reading, solvable.
|
||||||
|
assert asks_prior_state(_LEFT_TWIN) is False
|
||||||
|
|
||||||
|
def test_before_in_body_not_detected(self) -> None:
|
||||||
|
# 'before' in narrative (not the question clause) must NOT trip the guard,
|
||||||
|
# or it would wrongly refuse train-0003 (gold 864, currently committed).
|
||||||
|
body_before = (
|
||||||
|
"The student council sells erasers in the morning before school starts. "
|
||||||
|
"There are 24 erasers in each box. If they sell 48 boxes, how many erasers?"
|
||||||
|
)
|
||||||
|
assert asks_prior_state(body_before) is False
|
||||||
|
|
||||||
|
def test_used_to_make_is_not_a_prior_marker(self) -> None:
|
||||||
|
# the purpose infinitive 'used to make' is a false positive guarded against.
|
||||||
|
assert asks_prior_state("If 50 beads are used to make one bracelet, how many bracelets?") is False
|
||||||
|
|
||||||
|
def test_prior_state_question_refuses(self) -> None:
|
||||||
|
# the forward reading computes 50-20=30 (the net); the question asks the
|
||||||
|
# pre-change state -> refuse, not commit 30.
|
||||||
|
assert resolve_pooled(_BEFORE_Q) is None
|
||||||
|
|
||||||
|
def test_left_twin_still_resolves_forward(self) -> None:
|
||||||
|
# discrimination: the twin asking 'left' commits the forward net (30).
|
||||||
|
resolution = resolve_pooled(_LEFT_TWIN)
|
||||||
|
assert resolution is not None and resolution.answer == 30.0
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue