From f008550d7901e31a25d4b347b269b5f49b534699 Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 29 May 2026 15:05:31 -0700 Subject: [PATCH] =?UTF-8?q?feat(adr-0182):=20anchor-skip=20+=20intra-claus?= =?UTF-8?q?e=20accumulation=20=E2=80=94=20distractor=200016=20refuses,=20t?= =?UTF-8?q?win=200017=20solves=20(confuser=20wrong=201=E2=86=920)=20(#481)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(adr-0182): anchor-skip + intra-clause accumulation — distractor 0016 refuses, twin 0017 solves (confuser wrong 1->0) The last confuser wrong. 0016 ("A train travels 60 miles per hour for 2 hours. Tom has 8 tickets and buys 4 more tickets. How many tickets?") committed the blunt product 60x2x8x4=3840 because it was the unique complete reading: the train sentence is an all-foreign anchor-position block (2 quantities, can't seed an anchor) and the Tom sentence packs state+change in ONE sentence ("has 8 ... and buys 4 more"), which the sentence-level reader couldn't decompose -> accumulation_candidates was empty -> no rival reading -> product committed. The microscope confirmed intra-clause state+change is a REAL GSM8K pattern (train-0010 "Yun had 20 paperclips initially, but then lost 12"; practice-0121 "Sam has 30 apples and gives 10 to Anna") -- so this is genuine comprehension, not a 0016-only patch. Mechanism (added to accumulation_candidates ONLY -> feeds only the pool; train_sample serving + practice use compose_accumulation, unchanged -> 3/47/0 byte-identical by construction): - _sub_clauses: sentence clauses further split on coordinating conjunctions (and / then / and then). LOCAL to the ungated candidate generator -- the global segmenter (GB-1/GB-2/serving) is untouched. - _build_accumulation_anchor_skip: anchor = first single-quantity sub-clause (leading non-anchorable all-foreign blocks are skipped); chain the conjunction-mate change ("buys 4 more" -> +4). Referent guard + polarity-cue requirement still gate it (a no-cue sub-clause -> refuse), so GB-2 same-unit lists ("6 apples and 4 apples") produce no spurious candidate. Result (sealed lane): - confuser wrong 1 -> 0. distractor-quantity 0 wrong / 2 refused: 0016 refuses (product 3840 [complete] vs additive 12 [exempt] disagree). BONUS: the clean twin 0017 ("Tom has 8 tickets and buys 4 more tickets", no distractor) now *solves* 12 -- genuine comprehension of a real positive, the comprehension-vs-surface-match discrimination the corpus exists to measure. genuine positives 7 -> 8 solved. - the only non-clean verdict left is 1 spurious (0010 multi-referent "altogether" -- a separate H1 graduation question, not this lever). - train_sample 3/47/0 and practice 3/47/0 byte-identical; 243 derivation/pool tests + 40 architectural invariants green. Tests: - test_adr_0182_pool.py TestAnchorSkipIntraClause: intra-clause twin resolves to 12; the 0016 anchor-skip candidate classifies `exempt`; 0016 refuses via disagreement; no spurious extra candidate without a conjunction. - test_adr_0163_f2_confusers.py: baseline wrong 1->0, positives_solved 7->8; renamed test_distractor_quantity_refuses asserts BOTH 0014 and 0016; new test_intra_clause_twin_0017_solves. Stacked on #480 (prior-state guard); merge #480 first. Confuser arc wrong 7->5->2->1->0. * test(adr-0182): close anchor-skip refuse-branch coverage gap (lookback finding) The four-PR lookback review (EX-6/pooling/prior-state/anchor-skip) found the anchor-skip refuse branches in _build_accumulation_anchor_skip untested: the referent guard (new named actor -> refuse) and the polarity-cue requirement were asserted but not proven (no test would fail if removed). Per the schema-obligation discipline, add NON-VACUOUS failing-under-violation tests: - test_anchor_skip_referent_guard_discriminates_actor: identical structure, change sub-clause subject is a pronoun ('he', same referent -> 12 IS produced) vs a new name ('Sara' -> guard suppresses it). Removing _same_referent makes the new-actor case also produce 12 -> the second assertion fails. (Non-vacuous: the positive control proves the path is reachable, so the negative isn't trivially-empty.) - test_anchor_skip_requires_a_polarity_cue: 'gets 4 more' (cue -> 12) vs 'owns 4' (no cue -> not guessed). Removing the polarity gate makes the no-cue case produce a reading -> fails. - test_anchor_skip_refuses_without_single_quantity_anchor: no single-quantity sub-clause -> () (does not force a multi-quantity clause to anchor). No code change; behavior unchanged. Confuser wrong stays 0; 281 derivation tests + 40 invariants green. --- generate/derivation/accumulate.py | 73 +++++++++++++++++++++++-- tests/test_adr_0163_f2_confusers.py | 34 +++++++++--- tests/test_adr_0182_pool.py | 82 +++++++++++++++++++++++++++++ 3 files changed, 178 insertions(+), 11 deletions(-) diff --git a/generate/derivation/accumulate.py b/generate/derivation/accumulate.py index cc1d5fbb..9e7fb592 100644 --- a/generate/derivation/accumulate.py +++ b/generate/derivation/accumulate.py @@ -178,6 +178,67 @@ def _build_accumulation( return GroundedDerivation(start=start, steps=tuple(steps)) +# ADR-0182 anchor-skip: sub-clause split on conjunctions. A single sentence can pack +# a state and its change ("Tom has 8 tickets and buys 4 more tickets") — the +# sentence-level segmenter (used everywhere; not changed) keeps them together. This +# finer split is *local* to the ungated candidate generator, so it cannot move +# GB-1/GB-2/serving/practice (which never call it). Lexeme-level (ADR-0165): it names +# coordinating conjunctions, it does not parse grammar. +_CONJUNCTION_SPLIT: Final[re.Pattern[str]] = re.compile(r",?\s+(?:and then|and|then)\s+") + + +def _sub_clauses(problem_text: str) -> list[str]: + """Sentence clauses, each further split on coordinating conjunctions.""" + parts: list[str] = [] + for clause in segment_clauses(problem_text): + parts.extend(p.strip() for p in _CONJUNCTION_SPLIT.split(clause) if p.strip()) + return parts + + +def _build_accumulation_anchor_skip(problem_text: str) -> GroundedDerivation | None: + """ADR-0182 — accumulation over sub-clauses, skipping a leading all-foreign block. + + Reads ``A train travels 60 mph for 2 hours. Tom has 8 tickets and buys 4 more + tickets.`` by skipping the (anchor-position) train block — its quantities cannot + seed an anchor (≠1 quantity) — and anchoring on the first single-quantity + sub-clause (``Tom has 8 tickets``), then chaining its conjunction-mate change + (``buys 4 more`` → +4). The skipped block's quantities go unused; the pool's + isolated-foreign exemption then classifies the reading ``exempt`` (commit- + ineligible), so it can only force a disagreement refusal, never commit. Ungated. + """ + sub_clauses = [(s, extract_quantities(s)) for s in _sub_clauses(problem_text)] + quantity_subs = [(s, qs) for s, qs in sub_clauses if qs] + if len(quantity_subs) < 2: + return None + + # Anchor = first single-quantity sub-clause; leading non-anchorable (≠1 + # quantity) sub-clauses are skipped (candidate distractor blocks). + anchor_idx = next((i for i, (_, qs) in enumerate(quantity_subs) if len(qs) == 1), None) + if anchor_idx is None: + return None + anchor_sub, anchor_qs = quantity_subs[anchor_idx] + start = anchor_qs[0] + anchor_subject = _subject_token(anchor_sub) + + steps: list[Step] = [] + for sub, qs in quantity_subs[anchor_idx + 1:]: + if not _same_referent(sub, anchor_subject): + return None # new named actor -> referent hazard -> refuse + if len(qs) != 1: + return None # one change per sub-clause (multi-change is GB-3b.2) + polarity = _polarity(sub) + if polarity is None: + return None # no unambiguous licensed change cue -> refuse + change = qs[0] + operand = Quantity(value=change.value, unit=start.unit, source_token=change.source_token) + op = "add" if polarity > 0 else "subtract" + steps.append(Step(op=op, operand=operand, cue=_cue(sub, polarity))) + + if not steps: + return None + return GroundedDerivation(start=start, steps=tuple(steps)) + + def compose_accumulation(problem_text: str) -> Resolution | None: """GB-3b.1 composer — single-referent gain/loss accumulation. Refuse-preferring. @@ -193,13 +254,19 @@ def compose_accumulation(problem_text: str) -> Resolution | None: def accumulation_candidates(problem_text: str) -> tuple[GroundedDerivation, ...]: """ADR-0182 — the ungated accumulation readings for cross-composer pooling. - The strict reading and (when it differs) the distractor-skip reading. Ungated: - the pool classifies each (``complete`` commits, ``exempt`` refuses-only) and the - disagreement rule does the wrong=0 work. Deterministic; de-dup is the pool's job. + Three readings: the strict GB-3b.1 reading, the distractor-skip reading (drops an + isolated-foreign quantity in a multi-quantity change clause — 0014), and the + anchor-skip reading (skips a leading all-foreign block + reads a conjunction-mate + intra-sentence change — 0016). Ungated: the pool classifies each (``complete`` + commits, ``exempt`` refuses-only) and the disagreement rule does the wrong=0 work. + Deterministic; de-dup is the pool's job. """ candidates: list[GroundedDerivation] = [] for drop in (False, True): derivation = _build_accumulation(problem_text, drop_isolated_foreign=drop) if derivation is not None: candidates.append(derivation) + anchor_skip = _build_accumulation_anchor_skip(problem_text) + if anchor_skip is not None: + candidates.append(anchor_skip) return tuple(candidates) diff --git a/tests/test_adr_0163_f2_confusers.py b/tests/test_adr_0163_f2_confusers.py index 6b4b617d..3fbb2d3b 100644 --- a/tests/test_adr_0163_f2_confusers.py +++ b/tests/test_adr_0163_f2_confusers.py @@ -34,8 +34,17 @@ from evals.gsm8k_math.confusers.v1.runner import ( # *before* a stated change ("...before lunch?") is refused — the forward reader # computes the net, the wrong temporal point. 0020 refuses; its 'left' twin 0021 # still solves. wrong 2->1 (only 0016 remains). pair_tells 1->0. -_BASELINE_WRONG = 1 +# 2026-05-29 (ADR-0182 anchor-skip + intra-clause): a sentence packing state+change +# ("Tom has 8 tickets and buys 4 more tickets") is read via conjunction split, and +# a leading all-foreign block ("A train travels 60 miles ... for 2 hours") is +# skipped from anchor selection. 0016 refuses (product 3840 vs additive 12). BONUS: +# the clean twin 0017 ("Tom has 8 ... and buys 4 more", no distractor) now *solves* +# 12 — genuine comprehension, not just refusal. wrong 1->0; genuine positives 7->8 +# solved. The 1 remaining spurious is 0010 (multi-referent "altogether", a separate +# H1 graduation question). +_BASELINE_WRONG = 0 _BASELINE_PAIR_TELLS = 0 +_BASELINE_POSITIVES_SOLVED = 8 class TestSchema: @@ -76,7 +85,10 @@ class TestProbeBaseline: results = run_probe() positives = [r for r in results if r.expected == "solve"] solved = sum(1 for r in positives if r.verdict == "solved") - assert solved >= 7, f"genuine positives solving dropped to {solved}" + assert solved >= _BASELINE_POSITIVES_SOLVED, ( + f"genuine positives solving dropped to {solved} (baseline " + f"{_BASELINE_POSITIVES_SOLVED})" + ) # and a positive must never be answered WRONG (that would be a real defect). assert all(r.verdict != "wrong" for r in positives) @@ -97,14 +109,20 @@ class TestProbeBaseline: "divisor is no longer reaching the completeness/polarity gate." ) - def test_distractor_0014_refuses_via_pooling(self) -> None: - # ADR-0182: 0014's blunt product (300) and its competing additive reading - # (25) disagree under cross-composer pooling -> refuse. Fails loudly if - # pooling regresses (the unique product would commit 300 again). 0016 is - # NOT asserted here — its distractor sits in the anchor clause and needs - # anchor-skip, a separate step (see ADR-0182 §3b). + def test_distractor_quantity_refuses(self) -> None: + # ADR-0182: both distractor confusers refuse. 0014 via the change-clause + # drop (product 300 vs additive 25); 0016 via anchor-skip + intra-clause + # (product 3840 vs additive 12). Fails loudly if either regresses. by_id = {r.case_id: r for r in run_probe()} assert by_id["confuser-v1-0014"].verdict == "refused" + assert by_id["confuser-v1-0016"].verdict == "refused" + + def test_intra_clause_twin_0017_solves(self) -> None: + # the discrimination: 0017 ("Tom has 8 tickets and buys 4 more tickets", the + # clean twin of 0016) is genuinely *read* to 12 — comprehension, not refusal. + by_id = {r.case_id: r for r in run_probe()} + assert by_id["confuser-v1-0017"].verdict == "solved" + assert by_id["confuser-v1-0017"].answered == 12.0 def test_disguised_polarity_does_not_misfire(self) -> None: # ADR-0182 bonus: the spurious "buys X for N " product disagrees with diff --git a/tests/test_adr_0182_pool.py b/tests/test_adr_0182_pool.py index 7eebe1d9..0185403b 100644 --- a/tests/test_adr_0182_pool.py +++ b/tests/test_adr_0182_pool.py @@ -167,3 +167,85 @@ class TestPriorStateQuestionGuard: # discrimination: the twin asking 'left' commits the forward net (30). resolution = resolve_pooled(_LEFT_TWIN) assert resolution is not None and resolution.answer == 30.0 + + +_ANCHOR_SKIP_0016 = ( + "A train travels at 60 miles per hour for 2 hours. Tom has 8 tickets and " + "buys 4 more tickets. How many tickets does Tom have?" +) +_INTRACLAUSE_TWIN = "Tom has 8 tickets and buys 4 more tickets. How many tickets does Tom have?" + + +class TestAnchorSkipIntraClause: + """ADR-0182 anchor-skip: a sentence packing state+change ("Tom has 8 tickets and + buys 4 more tickets") is read by splitting on the conjunction, and a leading + all-foreign block ("A train travels 60 miles ... for 2 hours") is skipped from + anchor selection. Its quantities go unused -> the pool's isolated-foreign + exemption makes the reading commit-ineligible -> it forces a disagreement refusal + on the distractor case, while the clean twin commits.""" + + def test_intra_clause_state_and_change_resolves(self) -> None: + # the clean twin: "has 8 ... and buys 4 more" -> 12, committed. + resolution = resolve_pooled(_INTRACLAUSE_TWIN) + assert resolution is not None and resolution.answer == 12.0 + + def test_anchor_skip_candidate_is_exempt(self) -> None: + # the 0016 reading skips the train block; 8+4=12 leaves 60/2 unused-foreign. + cands = accumulation_candidates(_ANCHOR_SKIP_0016) + assert cands, "expected an anchor-skip accumulation candidate" + twelve = [d for d in cands if d.answer == 12.0] + assert twelve, "expected the 8+4=12 reading" + assert classify_derivation(twelve[0], _ANCHOR_SKIP_0016) == "exempt" + + def test_distractor_0016_refuses_via_disagreement(self) -> None: + # product 3840 (complete) vs additive 12 (exempt) disagree -> refuse. + assert resolve_pooled(_ANCHOR_SKIP_0016) is None + + def test_no_anchor_skip_candidate_without_conjunction(self) -> None: + # a plain single-quantity sentence yields no spurious extra reading. + cands = accumulation_candidates("Sam has 14 apples. He buys 9 more apples.") + assert all(d.answer == 23.0 for d in cands) + + # --- ADR-0182 lookback: the anchor-skip refuse branches (failing-under-violation) --- + + def test_anchor_skip_referent_guard_discriminates_actor(self) -> None: + # NON-VACUOUS failing-under-violation: the ONLY difference between these two + # is the change sub-clause's subject (pronoun 'he' vs new name 'Sara'). + # Same referent -> the 8+4=12 anchor-skip reading IS produced; new actor -> + # the referent guard suppresses it. If `_same_referent` were removed, the + # new-actor case would also produce 12 and the second assertion fails. + same_referent = ( + "A train travels at 60 miles per hour for 2 hours. Tom has 8 tickets and " + "he buys 4 more tickets. How many tickets does Tom have?" + ) + new_actor = ( + "A train travels at 60 miles per hour for 2 hours. Tom has 8 tickets and " + "Sara buys 4 more tickets. How many tickets does Tom have?" + ) + assert any(d.answer == 12.0 for d in accumulation_candidates(same_referent)), ( + "same-referent anchor-skip should produce the 8+4=12 reading" + ) + assert all(d.answer != 12.0 for d in accumulation_candidates(new_actor)), ( + "anchor-skip chained across a new actor (Sara onto Tom)" + ) + + def test_anchor_skip_requires_a_polarity_cue(self) -> None: + # NON-VACUOUS: same structure, change sub-clause with vs without a licensed + # cue. "gets 4 more" -> +4 -> 12 produced; "owns 4" (no gain/loss/more cue) + # -> polarity None -> not guessed -> no candidate. + with_cue = ( + "A train travels at 60 miles per hour for 2 hours. Tom has 8 tickets and " + "gets 4 more tickets. How many tickets does Tom have?" + ) + no_cue = ( + "A train travels at 60 miles per hour for 2 hours. Tom has 8 tickets and " + "owns 4 tickets. How many tickets does Tom have?" + ) + assert any(d.answer == 12.0 for d in accumulation_candidates(with_cue)) + assert all(d.answer != 12.0 for d in accumulation_candidates(no_cue)) + + def test_anchor_skip_refuses_without_single_quantity_anchor(self) -> None: + # No sub-clause yields a single-quantity anchor -> the anchor-skip path + # produces nothing (it does not force a multi-quantity clause to anchor). + no_anchor = "A train travels at 60 miles per hour for 2 hours and covers 5 towns." + assert accumulation_candidates(no_anchor) == ()