From a862d73084f82e0a8a2b4c337cc7664c1861375b Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 29 May 2026 14:22:28 -0700 Subject: [PATCH] =?UTF-8?q?feat(adr-0182):=20prior-state=20question=20guar?= =?UTF-8?q?d=20=E2=80=94=20temporal-scope=20confuser=20refuses=20(confuser?= =?UTF-8?q?=20wrong=202->1,=20pair-tells=20->0)=20(#480)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "before/left" reading-rule lever. The microscope showed only the question-time reading is cleanly achievable: "for N money = spend" is cue-precision-blocked (the `for` cue is overloaded across train_sample -- `for $2`, `for 14 days`, `for 10 reps` -- so a spend rule risks regressing train-0021/0003 and is the overfitting trap), and the disguised-polarity cases already refuse via pooling. "left" is already handled by loss verbs. What ships: a question-scope guard. A question asking for a state *before* a stated change ("How much did Lisa have before lunch?", gold 50) asks for a temporal point the forward composers do not compute -- they derive the final/net state (50-20=30). Until a question-time reader exists that is a refusal, never a guess at the wrong point. target.asks_prior_state detects before/initially/originally/at first/to begin with/to start with/at the start in the QUESTION CLAUSE only (the last `?`-sentence), so body narrative does not trip it -- verified safe against train-0003 ("sells before school starts"), 0010 ("had 20 initially, then lost 12"), 0028. `used to` is excluded (the purpose infinitive "beads used to make a bracelet" is a false positive). resolve_pooled refuses when asks_prior_state holds. Result (sealed lane; chat/ does not import these -> serving 3/47/0 frozen): - confuser wrong 2 -> 1 (only 0016 distractor-anchor-skip remains). temporal-scope category cleared (0 wrong / 2 refused). pair-tells 1 -> 0: 0020 ("before lunch") refuses while its minimal-pair twin 0021 ("left", gold 30) still solves the net -- the discrimination the corpus is built to measure. - genuine positives still 7 solved, 0 wrong. - train_sample 3/47/0 and practice 3/47/0 byte-identical. - 205 derivation/target/pool tests + 40 architectural invariants green. Tests: - test_adr_0182_pool.py TestPriorStateQuestionGuard: detector true/false edges (before-in-question vs before-in-body vs `used to make` false positive vs the `left` twin) + resolve_pooled refuses the before-question while the left-twin resolves forward to 30. - test_adr_0163_f2_confusers.py: baseline wrong 2->1, pair_tells 1->0; new test_temporal_scope_does_not_misfire + test_before_left_minimal_pair_discriminated. Stacked on #476 (pooling); merge #476 first. 0016 anchor-skip is the next lever. Pairs with ADR-0163-F2 graduation amendment (#478). --- generate/derivation/__init__.py | 3 ++- generate/derivation/pool.py | 9 +++++++ generate/derivation/target.py | 35 ++++++++++++++++++++++++ tests/test_adr_0163_f2_confusers.py | 24 +++++++++++++++-- tests/test_adr_0182_pool.py | 41 +++++++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 3 deletions(-) diff --git a/generate/derivation/__init__.py b/generate/derivation/__init__.py index c8c3b580..3d5121e1 100644 --- a/generate/derivation/__init__.py +++ b/generate/derivation/__init__.py @@ -28,7 +28,7 @@ from generate.derivation.search import ( multiplicative_candidates, search_multiplicative, ) -from generate.derivation.target import Target, extract_target +from generate.derivation.target import Target, asks_prior_state, extract_target from generate.derivation.verify import ( Resolution, SelfVerification, @@ -49,6 +49,7 @@ __all__ = [ "Target", "VALID_OPS", "accumulation_candidates", + "asks_prior_state", "candidate_chains", "classify_derivation", "clause_local_results", diff --git a/generate/derivation/pool.py b/generate/derivation/pool.py index b458aeb9..cd21712c 100644 --- a/generate/derivation/pool.py +++ b/generate/derivation/pool.py @@ -33,6 +33,7 @@ from generate.derivation.accumulate import accumulation_candidates from generate.derivation.model import GroundedDerivation from generate.derivation.multistep import candidate_chains from generate.derivation.search import multiplicative_candidates +from generate.derivation.target import asks_prior_state from generate.derivation.verify import Resolution, classify_derivation @@ -63,7 +64,15 @@ def resolve_pooled(problem_text: str) -> Resolution | None: Refuses on no verifying candidate, on disagreement among the pool, or when the sole answer is produced only by commit-ineligible (``exempt``) readings. + + ADR-0182 question-scope guard: a question asking for a *prior* state ("how much + did Lisa have **before** lunch?") asks for a temporal point the forward composers + do not compute — they derive the final/net state. Until a question-time reader + exists, that is a refusal, never a guess at the wrong point. """ + if asks_prior_state(problem_text): + return None + classified = [ (kind, derivation) for derivation in pooled_candidates(problem_text) diff --git a/generate/derivation/target.py b/generate/derivation/target.py index 8f9c474a..2f9f6bd2 100644 --- a/generate/derivation/target.py +++ b/generate/derivation/target.py @@ -25,6 +25,7 @@ weaker prune and leans on completeness, or refuses. from __future__ import annotations +import re from dataclasses import dataclass from typing import Final @@ -37,6 +38,40 @@ from generate.math_roundtrip import _tokens _AGG_WORDS: Final[tuple[str, ...]] = ("total", "altogether", "combined", "sum") _AGG_PHRASES: Final[tuple[str, ...]] = ("in all", "in total") +# ADR-0182 — prior-state question markers. A question that asks for a state *before* +# a stated change ("how much did Lisa have **before** lunch?", "...**initially**?") +# is asking for a temporal point the forward reader does not compute — it derives +# the final/net state. Detected on the **question clause only**: "before"/"initially" +# in the problem *body* is narrative ("sells before school starts", "had 20 +# initially, then lost 12") and must not trip this. ``used to`` is excluded — the +# purpose infinitive ("beads used to make a bracelet") is a false positive, not a +# prior-state cue. Lexeme-level (ADR-0165): it names markers, it does not parse the +# question's grammar. +_PRIOR_STATE_RE: Final[re.Pattern[str]] = re.compile( + r"\b(before|initially|originally|at first|to begin with|to start with|at the start)\b", + re.IGNORECASE, +) +_SENTENCE_SPLIT: Final[re.Pattern[str]] = re.compile(r"(?<=[.?!])\s+") + + +def _question_clause(problem_text: str) -> str: + """The interrogative clause: the last ``?``-terminated sentence, else the last + sentence. Deterministic.""" + sentences = [s for s in _SENTENCE_SPLIT.split(problem_text.strip()) if s.strip()] + if not sentences: + return problem_text + questions = [s for s in sentences if s.rstrip().endswith("?")] + return questions[-1] if questions else sentences[-1] + + +def asks_prior_state(problem_text: str) -> bool: + """True iff the *question* asks for a state before a stated change (ADR-0182). + + Question-clause-scoped so body narrative ("before school starts") does not trip + it. The forward reader computes the final state, so a prior-state question is a + refusal until a question-time reader exists — never a guess at the wrong point.""" + return bool(_PRIOR_STATE_RE.search(_question_clause(problem_text))) + @dataclass(frozen=True, slots=True) class Target: diff --git a/tests/test_adr_0163_f2_confusers.py b/tests/test_adr_0163_f2_confusers.py index be1f40d3..6b4b617d 100644 --- a/tests/test_adr_0163_f2_confusers.py +++ b/tests/test_adr_0163_f2_confusers.py @@ -30,8 +30,12 @@ from evals.gsm8k_math.confusers.v1.runner import ( # the spurious "for N coins" product disagrees with the accumulation reading) # now refuse. wrong 5->2 (0016 needs anchor-skip; 0020 is temporal-scope). # pair_tells 4->1 (0001/0003/0014 clear; 0020 remains). -_BASELINE_WRONG = 2 -_BASELINE_PAIR_TELLS = 1 +# 2026-05-29 (ADR-0182 prior-state question guard): a question asking for a state +# *before* a stated change ("...before lunch?") is refused — the forward reader +# computes the net, the wrong temporal point. 0020 refuses; its 'left' twin 0021 +# still solves. wrong 2->1 (only 0016 remains). pair_tells 1->0. +_BASELINE_WRONG = 1 +_BASELINE_PAIR_TELLS = 0 class TestSchema: @@ -111,5 +115,21 @@ class TestProbeBaseline: assert disguised assert all(r.verdict != "wrong" for r in disguised) + def test_temporal_scope_does_not_misfire(self) -> None: + # ADR-0182 prior-state question guard: a "before ?" question is + # refused (the forward reader computes the net, the wrong temporal point). + # Fails loudly if the guard regresses (0020 would commit 30 again). + results = run_probe() + temporal = [r for r in results if r.category == "temporal-scope"] + assert temporal + assert all(r.verdict != "wrong" for r in temporal) + + def test_before_left_minimal_pair_discriminated(self) -> None: + # 0020 ("before lunch") refuses; its twin 0021 ("left") solves the net. + # This is the pair-consistency win (the last surface-match tell cleared). + by_id = {r.case_id: r for r in run_probe()} + assert by_id["confuser-v1-0020"].verdict == "refused" + assert by_id["confuser-v1-0021"].verdict == "solved" + def test_deterministic(self) -> None: assert summarize(run_probe()) == summarize(run_probe()) diff --git a/tests/test_adr_0182_pool.py b/tests/test_adr_0182_pool.py index 8a717e33..36d5592d 100644 --- a/tests/test_adr_0182_pool.py +++ b/tests/test_adr_0182_pool.py @@ -18,6 +18,7 @@ from __future__ import annotations from generate.derivation.accumulate import accumulation_candidates from generate.derivation.model import GroundedDerivation, Quantity, Step from generate.derivation.pool import resolve_pooled +from generate.derivation.target import asks_prior_state from generate.derivation.verify import classify_derivation _DISTRACTOR_0014 = ( @@ -102,3 +103,43 @@ class TestResolvePooled: a = resolve_pooled(_CLEAN_ACCUMULATION) b = resolve_pooled(_CLEAN_ACCUMULATION) assert a is not None and b is not None and a.answer == b.answer + + +_BEFORE_Q = "Lisa had 50 dollars. She spent 20 on lunch. How much money did Lisa have before lunch?" +_LEFT_TWIN = "Lisa had 50 dollars. She spent 20 on lunch. How much money does Lisa have left?" + + +class TestPriorStateQuestionGuard: + """ADR-0182 — a question asking for a *prior* state is refused (the forward + composers compute the final state, the wrong temporal point). Question-clause + scoped, so body narrative ('before school starts') does not trip it.""" + + def test_before_question_detected(self) -> None: + assert asks_prior_state(_BEFORE_Q) is True + + def test_left_twin_not_detected(self) -> None: + # the minimal-pair twin asks for the net ('left') -> forward reading, solvable. + assert asks_prior_state(_LEFT_TWIN) is False + + def test_before_in_body_not_detected(self) -> None: + # 'before' in narrative (not the question clause) must NOT trip the guard, + # or it would wrongly refuse train-0003 (gold 864, currently committed). + body_before = ( + "The student council sells erasers in the morning before school starts. " + "There are 24 erasers in each box. If they sell 48 boxes, how many erasers?" + ) + assert asks_prior_state(body_before) is False + + def test_used_to_make_is_not_a_prior_marker(self) -> None: + # the purpose infinitive 'used to make' is a false positive guarded against. + assert asks_prior_state("If 50 beads are used to make one bracelet, how many bracelets?") is False + + def test_prior_state_question_refuses(self) -> None: + # the forward reading computes 50-20=30 (the net); the question asks the + # pre-change state -> refuse, not commit 30. + assert resolve_pooled(_BEFORE_Q) is None + + def test_left_twin_still_resolves_forward(self) -> None: + # discrimination: the twin asking 'left' commits the forward net (30). + resolution = resolve_pooled(_LEFT_TWIN) + assert resolution is not None and resolution.answer == 30.0