diff --git a/generate/derivation/__init__.py b/generate/derivation/__init__.py index c8c3b580..3d5121e1 100644 --- a/generate/derivation/__init__.py +++ b/generate/derivation/__init__.py @@ -28,7 +28,7 @@ from generate.derivation.search import ( multiplicative_candidates, search_multiplicative, ) -from generate.derivation.target import Target, extract_target +from generate.derivation.target import Target, asks_prior_state, extract_target from generate.derivation.verify import ( Resolution, SelfVerification, @@ -49,6 +49,7 @@ __all__ = [ "Target", "VALID_OPS", "accumulation_candidates", + "asks_prior_state", "candidate_chains", "classify_derivation", "clause_local_results", diff --git a/generate/derivation/pool.py b/generate/derivation/pool.py index b458aeb9..cd21712c 100644 --- a/generate/derivation/pool.py +++ b/generate/derivation/pool.py @@ -33,6 +33,7 @@ from generate.derivation.accumulate import accumulation_candidates from generate.derivation.model import GroundedDerivation from generate.derivation.multistep import candidate_chains from generate.derivation.search import multiplicative_candidates +from generate.derivation.target import asks_prior_state from generate.derivation.verify import Resolution, classify_derivation @@ -63,7 +64,15 @@ def resolve_pooled(problem_text: str) -> Resolution | None: Refuses on no verifying candidate, on disagreement among the pool, or when the sole answer is produced only by commit-ineligible (``exempt``) readings. + + ADR-0182 question-scope guard: a question asking for a *prior* state ("how much + did Lisa have **before** lunch?") asks for a temporal point the forward composers + do not compute — they derive the final/net state. Until a question-time reader + exists, that is a refusal, never a guess at the wrong point. """ + if asks_prior_state(problem_text): + return None + classified = [ (kind, derivation) for derivation in pooled_candidates(problem_text) diff --git a/generate/derivation/target.py b/generate/derivation/target.py index 8f9c474a..2f9f6bd2 100644 --- a/generate/derivation/target.py +++ b/generate/derivation/target.py @@ -25,6 +25,7 @@ weaker prune and leans on completeness, or refuses. from __future__ import annotations +import re from dataclasses import dataclass from typing import Final @@ -37,6 +38,40 @@ from generate.math_roundtrip import _tokens _AGG_WORDS: Final[tuple[str, ...]] = ("total", "altogether", "combined", "sum") _AGG_PHRASES: Final[tuple[str, ...]] = ("in all", "in total") +# ADR-0182 — prior-state question markers. A question that asks for a state *before* +# a stated change ("how much did Lisa have **before** lunch?", "...**initially**?") +# is asking for a temporal point the forward reader does not compute — it derives +# the final/net state. Detected on the **question clause only**: "before"/"initially" +# in the problem *body* is narrative ("sells before school starts", "had 20 +# initially, then lost 12") and must not trip this. ``used to`` is excluded — the +# purpose infinitive ("beads used to make a bracelet") is a false positive, not a +# prior-state cue. Lexeme-level (ADR-0165): it names markers, it does not parse the +# question's grammar. +_PRIOR_STATE_RE: Final[re.Pattern[str]] = re.compile( + r"\b(before|initially|originally|at first|to begin with|to start with|at the start)\b", + re.IGNORECASE, +) +_SENTENCE_SPLIT: Final[re.Pattern[str]] = re.compile(r"(?<=[.?!])\s+") + + +def _question_clause(problem_text: str) -> str: + """The interrogative clause: the last ``?``-terminated sentence, else the last + sentence. Deterministic.""" + sentences = [s for s in _SENTENCE_SPLIT.split(problem_text.strip()) if s.strip()] + if not sentences: + return problem_text + questions = [s for s in sentences if s.rstrip().endswith("?")] + return questions[-1] if questions else sentences[-1] + + +def asks_prior_state(problem_text: str) -> bool: + """True iff the *question* asks for a state before a stated change (ADR-0182). + + Question-clause-scoped so body narrative ("before school starts") does not trip + it. The forward reader computes the final state, so a prior-state question is a + refusal until a question-time reader exists — never a guess at the wrong point.""" + return bool(_PRIOR_STATE_RE.search(_question_clause(problem_text))) + @dataclass(frozen=True, slots=True) class Target: diff --git a/tests/test_adr_0163_f2_confusers.py b/tests/test_adr_0163_f2_confusers.py index be1f40d3..6b4b617d 100644 --- a/tests/test_adr_0163_f2_confusers.py +++ b/tests/test_adr_0163_f2_confusers.py @@ -30,8 +30,12 @@ from evals.gsm8k_math.confusers.v1.runner import ( # the spurious "for N coins" product disagrees with the accumulation reading) # now refuse. wrong 5->2 (0016 needs anchor-skip; 0020 is temporal-scope). # pair_tells 4->1 (0001/0003/0014 clear; 0020 remains). -_BASELINE_WRONG = 2 -_BASELINE_PAIR_TELLS = 1 +# 2026-05-29 (ADR-0182 prior-state question guard): a question asking for a state +# *before* a stated change ("...before lunch?") is refused — the forward reader +# computes the net, the wrong temporal point. 0020 refuses; its 'left' twin 0021 +# still solves. wrong 2->1 (only 0016 remains). pair_tells 1->0. +_BASELINE_WRONG = 1 +_BASELINE_PAIR_TELLS = 0 class TestSchema: @@ -111,5 +115,21 @@ class TestProbeBaseline: assert disguised assert all(r.verdict != "wrong" for r in disguised) + def test_temporal_scope_does_not_misfire(self) -> None: + # ADR-0182 prior-state question guard: a "before ?" question is + # refused (the forward reader computes the net, the wrong temporal point). + # Fails loudly if the guard regresses (0020 would commit 30 again). + results = run_probe() + temporal = [r for r in results if r.category == "temporal-scope"] + assert temporal + assert all(r.verdict != "wrong" for r in temporal) + + def test_before_left_minimal_pair_discriminated(self) -> None: + # 0020 ("before lunch") refuses; its twin 0021 ("left") solves the net. + # This is the pair-consistency win (the last surface-match tell cleared). + by_id = {r.case_id: r for r in run_probe()} + assert by_id["confuser-v1-0020"].verdict == "refused" + assert by_id["confuser-v1-0021"].verdict == "solved" + def test_deterministic(self) -> None: assert summarize(run_probe()) == summarize(run_probe()) diff --git a/tests/test_adr_0182_pool.py b/tests/test_adr_0182_pool.py index 8a717e33..36d5592d 100644 --- a/tests/test_adr_0182_pool.py +++ b/tests/test_adr_0182_pool.py @@ -18,6 +18,7 @@ from __future__ import annotations from generate.derivation.accumulate import accumulation_candidates from generate.derivation.model import GroundedDerivation, Quantity, Step from generate.derivation.pool import resolve_pooled +from generate.derivation.target import asks_prior_state from generate.derivation.verify import classify_derivation _DISTRACTOR_0014 = ( @@ -102,3 +103,43 @@ class TestResolvePooled: a = resolve_pooled(_CLEAN_ACCUMULATION) b = resolve_pooled(_CLEAN_ACCUMULATION) assert a is not None and b is not None and a.answer == b.answer + + +_BEFORE_Q = "Lisa had 50 dollars. She spent 20 on lunch. How much money did Lisa have before lunch?" +_LEFT_TWIN = "Lisa had 50 dollars. She spent 20 on lunch. How much money does Lisa have left?" + + +class TestPriorStateQuestionGuard: + """ADR-0182 — a question asking for a *prior* state is refused (the forward + composers compute the final state, the wrong temporal point). Question-clause + scoped, so body narrative ('before school starts') does not trip it.""" + + def test_before_question_detected(self) -> None: + assert asks_prior_state(_BEFORE_Q) is True + + def test_left_twin_not_detected(self) -> None: + # the minimal-pair twin asks for the net ('left') -> forward reading, solvable. + assert asks_prior_state(_LEFT_TWIN) is False + + def test_before_in_body_not_detected(self) -> None: + # 'before' in narrative (not the question clause) must NOT trip the guard, + # or it would wrongly refuse train-0003 (gold 864, currently committed). + body_before = ( + "The student council sells erasers in the morning before school starts. " + "There are 24 erasers in each box. If they sell 48 boxes, how many erasers?" + ) + assert asks_prior_state(body_before) is False + + def test_used_to_make_is_not_a_prior_marker(self) -> None: + # the purpose infinitive 'used to make' is a false positive guarded against. + assert asks_prior_state("If 50 beads are used to make one bracelet, how many bracelets?") is False + + def test_prior_state_question_refuses(self) -> None: + # the forward reading computes 50-20=30 (the net); the question asks the + # pre-change state -> refuse, not commit 30. + assert resolve_pooled(_BEFORE_Q) is None + + def test_left_twin_still_resolves_forward(self) -> None: + # discrimination: the twin asking 'left' commits the forward net (30). + resolution = resolve_pooled(_LEFT_TWIN) + assert resolution is not None and resolution.answer == 30.0