From 6611a7017d2ac083b84341ed2f7f78f378815e18 Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 29 May 2026 10:41:51 -0700 Subject: [PATCH] feat(adr-0178-gb3b1): single-referent accumulation chaining (practice 0 -> 55) (#465) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first cross-clause comprehension reading: one actor's quantity changes over successive clauses ("Sam has 14 apples. He buys 9 more." -> 14 + 9). It is the safe specialisation of the cross-clause sum that GB-3a refuses wholesale (the Alice/Tom hazard) — we chain only when (same referent) AND (a licensed change cue of unambiguous polarity), else refuse. generate/derivation/accumulate.py — compose_accumulation: - anchor on clause 1's single quantity; apply +M (gain) / -M (loss) per later change clause, operand taken in the anchor's unit (accumulation is same-dimension); routed through the unchanged self-verification gate. - polarity (ordered, so ambiguous "gives" is resolved not guessed): "more" -> gain; else unambiguous loss verb -> loss; else gives/gave + to/away -> loss; else unambiguous gain verb -> gain; else REFUSE. - referent guard (the ADR-0174 multi-actor hazard's defensive fix, built minimally in the clean lane — NOT the retired gender-blind resolver): a later clause's subject token must be a pronoun or the anchor's name; a NEW named subject (Tom) -> refuse. Pronoun gender/number is not matched; a new name is the only signal. evals/.../accumulation_runner.py — practice scorer: on a base refusal, attempt compose_accumulation and gold-check (mirrors search_runner). Sealed: fires only on already-refused cases, never alters serving. Measured (sealed practice additive lane): 0 -> 55 correct, wrong unchanged at 1 (the base scorer's pre-existing one; accumulation added 55 correct, 0 wrong). The 36 still-refused are multi-change (GB-3b.2) or unrecognised verbs (vocab growth) — conservative, never wrong. Proof obligations (tests fail under the violation): new-named-actor refuses (H1), no/ambiguous change cue refuses, list anchor refuses, multi-change refuses, determinism. 136 targeted tests + architectural invariants green; serving 3/47/0 byte-identical (lane-SHA 8/8, claims --check OK). --- .../practice/v1/accumulation_runner.py | 51 ++++++ generate/derivation/__init__.py | 2 + generate/derivation/accumulate.py | 163 ++++++++++++++++++ tests/test_adr_0178_gb3b1_accumulation.py | 76 ++++++++ 4 files changed, 292 insertions(+) create mode 100644 evals/gsm8k_math/practice/v1/accumulation_runner.py create mode 100644 generate/derivation/accumulate.py create mode 100644 tests/test_adr_0178_gb3b1_accumulation.py diff --git a/evals/gsm8k_math/practice/v1/accumulation_runner.py b/evals/gsm8k_math/practice/v1/accumulation_runner.py new file mode 100644 index 00000000..c194aaff --- /dev/null +++ b/evals/gsm8k_math/practice/v1/accumulation_runner.py @@ -0,0 +1,51 @@ +"""ADR-0178 GB-3b.1 — wire the accumulation composer into the sealed practice lane. + +Mirrors :mod:`evals.gsm8k_math.practice.v1.search_runner`: when the base engine +*refuses*, the practice regime is allowed to *attempt* the single-referent +accumulation reading (:func:`generate.derivation.accumulate.compose_accumulation`) +and checks it against gold (Tier-1, available in practice). Correct attempts flip; +wrong attempts become elimination records. The base (serving) outcome is never +altered — the composer only fires on cases the engine already declined, inside the +sealed lane. Serving stays ``3/47/0``. +""" + +from __future__ import annotations + +from evals.gsm8k_math.practice.v1.runner import ( + PracticeReport, + _load_practice_cases, + run_practice, +) +from evals.gsm8k_math.practice.v1.search_runner import _TOL, _SearchOutcome +from evals.gsm8k_math.runner import _score_one_candidate_graph +from generate.derivation.accumulate import compose_accumulation + + +def accumulation_augmented_scorer(adapted: dict) -> object: + """Base scorer, then a practice-only accumulation attempt on refusals.""" + base = _score_one_candidate_graph(adapted) + if base.outcome != "refused": + return base # the serving path already committed — leave it untouched + + resolution = compose_accumulation(adapted["problem"]) + if resolution is None: + return base # accumulation also declined -> still refused + + attempted = resolution.answer + gold = float(adapted["expected_answer"]) + correct = abs(attempted - gold) <= _TOL + return _SearchOutcome( + case_id=adapted["id"], + outcome="correct" if correct else "wrong", + reason=( + f"compose_accumulation -> {attempted:g}" + if correct + else f"compose_accumulation wrong: got {attempted:g}, gold {gold:g}" + ), + actual_answer=attempted, + ) + + +def build_accumulation_report() -> PracticeReport: + """Practice report with the accumulation reading enabled (attempts live).""" + return run_practice(_load_practice_cases(), scorer=accumulation_augmented_scorer) diff --git a/generate/derivation/__init__.py b/generate/derivation/__init__.py index 4bc4b77a..caa5c86e 100644 --- a/generate/derivation/__init__.py +++ b/generate/derivation/__init__.py @@ -12,6 +12,7 @@ from generate.derivation.clauses import ( clause_local_results, segment_clauses, ) +from generate.derivation.accumulate import compose_accumulation from generate.derivation.compose import compose_sequential from generate.derivation.comparatives import ( ComparativeScalar, @@ -42,6 +43,7 @@ __all__ = [ "Target", "VALID_OPS", "clause_local_results", + "compose_accumulation", "compose_sequential", "comparative_step", "extract_comparative_scalars", diff --git a/generate/derivation/accumulate.py b/generate/derivation/accumulate.py new file mode 100644 index 00000000..5ee8c2a5 --- /dev/null +++ b/generate/derivation/accumulate.py @@ -0,0 +1,163 @@ +"""ADR-0178 GB-3b.1 — single-referent accumulation chaining. + +The first cross-clause *comprehension* reading: one actor's quantity changes over +successive clauses (``Sam has 14 apples. He buys 9 more.`` -> ``14 + 9``). It is +the safe specialisation of the cross-clause sum that GB-3a's referent guard +correctly refuses wholesale (the ``Alice has 6 … Tom has 2 …`` hazard): we chain +across clauses **only** when (a) the later clause stays on the **same referent** +and (b) it carries a **licensed change cue** whose polarity is unambiguous. +Otherwise we refuse — the guard is generalised, never weakened. + +Reading: + +1. **Anchor** — clause 1 must establish exactly one quantity ``(actor, N, unit)``. +2. **Change steps** — each later quantity-bearing clause applies ``+ M`` (gain) or + ``- M`` (loss) to the running total, where ``M`` is the clause's single grounded + quantity, taken **in the anchor's unit** (``9 more`` = 9 more *apples*; the unit + is inherited from the running total, which is what accumulation means). +3. **Gate** — the constructed chain runs through the unchanged self-verification + gate (grounding ∧ cue ∧ unit ∧ completeness ∧ uniqueness). The gate keeps + wrong=0; this only proposes a structurally-licensed candidate. + +Polarity (ordered, so the ambiguous ``gives`` is resolved, never guessed): + +* ``more`` present -> **gain** (covers ``buys/gets/… + N more`` and ``gives her N more`` — the subject is the recipient); +* else an unambiguous **loss** verb -> **loss**; +* else ``gives``/``gave`` with ``to``/``away`` -> **loss** (gives N *to* someone); +* else an unambiguous **gain** verb -> **gain**; +* else -> **refuse** (no guessing). + +Referent guard (wrong=0-critical; the ADR-0174 multi-actor hazard's defensive fix, +built minimally in the clean lane rather than resurrecting the retired resolver): +a later clause stays on the anchor's referent iff its **subject token** is a +pronoun (``He/She/They/…``) or the same name as the anchor's subject. A **new named +subject** (a different capitalised non-pronoun first token, e.g. ``Tom``) -> refuse. +Pronoun gender/number is **not** matched (that was the old resolver's trap); a new +*name* is the only signal, and it triggers refusal, not resolution. + +Sealed (no ``chat/`` import); deterministic; refuse-preferring. +""" + +from __future__ import annotations + +import re +from typing import Final + +from generate.derivation.clauses import segment_clauses +from generate.derivation.extract import extract_quantities +from generate.derivation.model import GroundedDerivation, Quantity, Step +from generate.derivation.verify import Resolution, select_self_verified +from generate.math_roundtrip import _tokens + +# Closed change-cue lexeme sets (ADR-0165: lexemes, not grammar templates; refined +# by the CP ledger, not asserted complete). Sorted use keeps cue selection stable. +_GAIN_VERBS: Final[frozenset[str]] = frozenset( + { + "buys", "bought", "gets", "got", "finds", "found", "picks", "picked", + "earns", "earned", "receives", "received", "collects", "collected", + "wins", "won", "makes", "made", "gains", "gained", "adds", "added", + } +) +_LOSS_VERBS: Final[frozenset[str]] = frozenset( + { + "loses", "lost", "spends", "spent", "uses", "used", "eats", "ate", + "sells", "sold", "donates", "donated", "drops", "dropped", "removes", + "removed", "breaks", "broke", + } +) +_PRONOUNS: Final[frozenset[str]] = frozenset( + {"he", "she", "they", "it", "him", "her", "them", "his", "hers", "its", "their", "we", "i", "you"} +) + +_WORD_RE: Final[re.Pattern[str]] = re.compile(r"[A-Za-z]+") + + +def _subject_token(clause: str) -> str | None: + """The clause's leading word token (its surface subject), or None if wordless.""" + match = _WORD_RE.search(clause) + return match.group(0) if match is not None else None + + +def _same_referent(clause: str, anchor_subject: str | None) -> bool: + """True iff ``clause`` does not introduce a new *named* subject. + + Conservative: a leading pronoun continues the referent; a leading token equal + to the anchor's subject continues it; any other capitalised (named) leading + token is a *new actor* and breaks the referent (-> caller refuses). + """ + subject = _subject_token(clause) + if subject is None: + return True # wordless fragment carries no new actor + if subject.lower() in _PRONOUNS: + return True + if anchor_subject is not None and subject == anchor_subject: + return True + # A new capitalised, non-pronoun leading token is a new named actor. + return not subject[:1].isupper() + + +def _polarity(clause: str) -> int | None: + """+1 (gain), -1 (loss), or None (ambiguous / no licensed change cue -> refuse).""" + tokens = set(_tokens(clause)) + if "more" in tokens: + return +1 + loss = bool(_LOSS_VERBS & tokens) + gain = bool(_GAIN_VERBS & tokens) + gives = "gives" in tokens or "gave" in tokens + directional = "to" in tokens or "away" in tokens + if loss and not gain: + return -1 + if gives and directional and not gain and not loss: + return -1 + if gain and not loss: + return +1 + return None + + +def _cue(clause: str, polarity: int) -> str: + """A grounded cue lexeme present in the clause (for the gate's cue check).""" + tokens = set(_tokens(clause)) + if "more" in tokens and polarity > 0: + return "more" + verbs = _GAIN_VERBS if polarity > 0 else _LOSS_VERBS + present = sorted(verbs & tokens) + if present: + return present[0] + return "gives" # the only remaining licensed loss path (gives … to/away) + + +def compose_accumulation(problem_text: str) -> Resolution | None: + """GB-3b.1 composer — single-referent gain/loss accumulation. Refuse-preferring.""" + clauses = segment_clauses(problem_text) + quantity_clauses = [c for c in clauses if extract_quantities(c)] + if len(quantity_clauses) < 2: + return None + + anchor_clause, *change_clauses = quantity_clauses + anchor_quantities = extract_quantities(anchor_clause) + if len(anchor_quantities) != 1: + return None # the anchor must establish exactly one quantity (GB-3b.1 scope) + start = anchor_quantities[0] + anchor_subject = _subject_token(anchor_clause) + + steps: list[Step] = [] + for clause in change_clauses: + if not _same_referent(clause, anchor_subject): + return None # new named actor -> referent hazard -> refuse + change_quantities = extract_quantities(clause) + if len(change_quantities) != 1: + return None # one change per clause (multi-change is GB-3b.2) + polarity = _polarity(clause) + if polarity is None: + return None # no unambiguous licensed change cue -> refuse + change = change_quantities[0] + # The change is in the running total's dimension ("9 more" = 9 more apples). + operand = Quantity(value=change.value, unit=start.unit, source_token=change.source_token) + op = "add" if polarity > 0 else "subtract" + steps.append(Step(op=op, operand=operand, cue=_cue(clause, polarity))) + + if not steps: + return None + derivation = GroundedDerivation(start=start, steps=tuple(steps)) + return select_self_verified([derivation], problem_text, target_units=()) diff --git a/tests/test_adr_0178_gb3b1_accumulation.py b/tests/test_adr_0178_gb3b1_accumulation.py new file mode 100644 index 00000000..a42be113 --- /dev/null +++ b/tests/test_adr_0178_gb3b1_accumulation.py @@ -0,0 +1,76 @@ +"""ADR-0178 GB-3b.1 — single-referent accumulation chaining. + +The first cross-clause comprehension reading. These tests prove the obligations +from the GB-3b scope: the accumulation flips the gain/loss cases, and the wrong=0 +guards (multi-actor, absent/ambiguous change cue, the GB-3a hazards) each REFUSE. +Every refusal test would *fail* if the guard it covers were removed. +""" + +from __future__ import annotations + +from generate.derivation.accumulate import compose_accumulation + + +def _answer(text: str): + res = compose_accumulation(text) + return None if res is None else res.answer + + +class TestAccumulationFlips: + def test_gain_with_more(self) -> None: + assert _answer("Sam has 14 apples. He buys 9 more. How many apples does Sam have now?") == 23.0 + + def test_gain_verb_no_more(self) -> None: + assert _answer("Ben has 20 marbles. He finds 7 on the playground. How many marbles?") == 27.0 + + def test_gain_giver_to_subject_reads_as_more(self) -> None: + # "Her teacher gives her 5 more" — subject (pronoun) is the recipient -> +5. + assert _answer("Kate has 18 pencils. Her teacher gives her 5 more. How many pencils?") == 23.0 + + def test_loss_verb(self) -> None: + assert _answer("Sam has 30 apples. He eats 8. How many apples does Sam have left?") == 22.0 + + def test_loss_gives_to_recipient(self) -> None: + assert _answer("Anna has 25 stickers. She gives 10 to her friend. How many stickers left?") == 15.0 + + +class TestReferentGuardRefuses: + def test_new_named_actor_refuses(self) -> None: + # The Alice/Tom hazard: a new named subject -> refuse (not Sam's 14+9). + assert _answer("Sam has 14 apples. Tom buys 9 more. How many apples does Sam have?") is None + + def test_h1_unrelated_same_unit_across_sentences(self) -> None: + assert _answer("Alice has 6 apples. Tom has 2 apples. How many apples does Alice have?") is None + + +class TestChangeCueGuardRefuses: + def test_no_change_cue_refuses(self) -> None: + # two quantities, no gain/loss verb and no "more" -> no licensed change -> refuse. + assert _answer("Lisa has 30 coins. She has 15 stickers. How many coins does Lisa have?") is None + + def test_anchor_must_be_single_quantity(self) -> None: + # a list anchor is GB-2a's job, not accumulation -> refuse here. + assert _answer("Sam has 6 apples and 4 apples. He buys 5 more. How many?") is None + + def test_multi_change_in_one_clause_refuses(self) -> None: + # "gets 5 more from Tom and 3 more from Lisa" is multi-change (GB-3b.2) -> refuse. + assert _answer("Sam has 10 apples. He gets 5 more and 3 more. How many?") is None + + +class TestDeterminism: + def test_deterministic(self) -> None: + t = "Sam has 14 apples. He buys 9 more. How many?" + assert compose_accumulation(t) == compose_accumulation(t) + + +class TestPracticeLaneFlip: + def test_accumulation_flips_a_chunk_with_no_new_wrong(self) -> None: + from evals.gsm8k_math.practice.v1.accumulation_runner import build_accumulation_report + from evals.gsm8k_math.practice.v1.runner import build_practice_report + + before = build_practice_report().counts + after = build_accumulation_report().counts + # accumulation fires only on refusals: it adds correct, never a new wrong. + assert after["correct"] >= 55 + assert after["wrong"] == before["wrong"] + assert after["correct"] + after["wrong"] + after["refused"] == sum(before.values())