feat(adr-0178-gb3b1): single-referent accumulation chaining (practice 0 -> 55) (#465)
The first cross-clause comprehension reading: one actor's quantity changes over
successive clauses ("Sam has 14 apples. He buys 9 more." -> 14 + 9). It is the
safe specialisation of the cross-clause sum that GB-3a refuses wholesale (the
Alice/Tom hazard) — we chain only when (same referent) AND (a licensed change
cue of unambiguous polarity), else refuse.
generate/derivation/accumulate.py — compose_accumulation:
- anchor on clause 1's single quantity; apply +M (gain) / -M (loss) per later
change clause, operand taken in the anchor's unit (accumulation is same-dimension);
routed through the unchanged self-verification gate.
- polarity (ordered, so ambiguous "gives" is resolved not guessed): "more" -> gain;
else unambiguous loss verb -> loss; else gives/gave + to/away -> loss; else
unambiguous gain verb -> gain; else REFUSE.
- referent guard (the ADR-0174 multi-actor hazard's defensive fix, built minimally
in the clean lane — NOT the retired gender-blind resolver): a later clause's
subject token must be a pronoun or the anchor's name; a NEW named subject (Tom)
-> refuse. Pronoun gender/number is not matched; a new name is the only signal.
evals/.../accumulation_runner.py — practice scorer: on a base refusal, attempt
compose_accumulation and gold-check (mirrors search_runner). Sealed: fires only
on already-refused cases, never alters serving.
Measured (sealed practice additive lane): 0 -> 55 correct, wrong unchanged at 1
(the base scorer's pre-existing one; accumulation added 55 correct, 0 wrong). The
36 still-refused are multi-change (GB-3b.2) or unrecognised verbs (vocab growth) —
conservative, never wrong.
Proof obligations (tests fail under the violation): new-named-actor refuses (H1),
no/ambiguous change cue refuses, list anchor refuses, multi-change refuses,
determinism. 136 targeted tests + architectural invariants green; serving 3/47/0
byte-identical (lane-SHA 8/8, claims --check OK).
This commit is contained in:
parent
cecf7b82cc
commit
6611a7017d
4 changed files with 292 additions and 0 deletions
51
evals/gsm8k_math/practice/v1/accumulation_runner.py
Normal file
51
evals/gsm8k_math/practice/v1/accumulation_runner.py
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
"""ADR-0178 GB-3b.1 — wire the accumulation composer into the sealed practice lane.
|
||||
|
||||
Mirrors :mod:`evals.gsm8k_math.practice.v1.search_runner`: when the base engine
|
||||
*refuses*, the practice regime is allowed to *attempt* the single-referent
|
||||
accumulation reading (:func:`generate.derivation.accumulate.compose_accumulation`)
|
||||
and checks it against gold (Tier-1, available in practice). Correct attempts flip;
|
||||
wrong attempts become elimination records. The base (serving) outcome is never
|
||||
altered — the composer only fires on cases the engine already declined, inside the
|
||||
sealed lane. Serving stays ``3/47/0``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from evals.gsm8k_math.practice.v1.runner import (
|
||||
PracticeReport,
|
||||
_load_practice_cases,
|
||||
run_practice,
|
||||
)
|
||||
from evals.gsm8k_math.practice.v1.search_runner import _TOL, _SearchOutcome
|
||||
from evals.gsm8k_math.runner import _score_one_candidate_graph
|
||||
from generate.derivation.accumulate import compose_accumulation
|
||||
|
||||
|
||||
def accumulation_augmented_scorer(adapted: dict) -> object:
|
||||
"""Base scorer, then a practice-only accumulation attempt on refusals."""
|
||||
base = _score_one_candidate_graph(adapted)
|
||||
if base.outcome != "refused":
|
||||
return base # the serving path already committed — leave it untouched
|
||||
|
||||
resolution = compose_accumulation(adapted["problem"])
|
||||
if resolution is None:
|
||||
return base # accumulation also declined -> still refused
|
||||
|
||||
attempted = resolution.answer
|
||||
gold = float(adapted["expected_answer"])
|
||||
correct = abs(attempted - gold) <= _TOL
|
||||
return _SearchOutcome(
|
||||
case_id=adapted["id"],
|
||||
outcome="correct" if correct else "wrong",
|
||||
reason=(
|
||||
f"compose_accumulation -> {attempted:g}"
|
||||
if correct
|
||||
else f"compose_accumulation wrong: got {attempted:g}, gold {gold:g}"
|
||||
),
|
||||
actual_answer=attempted,
|
||||
)
|
||||
|
||||
|
||||
def build_accumulation_report() -> PracticeReport:
|
||||
"""Practice report with the accumulation reading enabled (attempts live)."""
|
||||
return run_practice(_load_practice_cases(), scorer=accumulation_augmented_scorer)
|
||||
|
|
@ -12,6 +12,7 @@ from generate.derivation.clauses import (
|
|||
clause_local_results,
|
||||
segment_clauses,
|
||||
)
|
||||
from generate.derivation.accumulate import compose_accumulation
|
||||
from generate.derivation.compose import compose_sequential
|
||||
from generate.derivation.comparatives import (
|
||||
ComparativeScalar,
|
||||
|
|
@ -42,6 +43,7 @@ __all__ = [
|
|||
"Target",
|
||||
"VALID_OPS",
|
||||
"clause_local_results",
|
||||
"compose_accumulation",
|
||||
"compose_sequential",
|
||||
"comparative_step",
|
||||
"extract_comparative_scalars",
|
||||
|
|
|
|||
163
generate/derivation/accumulate.py
Normal file
163
generate/derivation/accumulate.py
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
"""ADR-0178 GB-3b.1 — single-referent accumulation chaining.
|
||||
|
||||
The first cross-clause *comprehension* reading: one actor's quantity changes over
|
||||
successive clauses (``Sam has 14 apples. He buys 9 more.`` -> ``14 + 9``). It is
|
||||
the safe specialisation of the cross-clause sum that GB-3a's referent guard
|
||||
correctly refuses wholesale (the ``Alice has 6 … Tom has 2 …`` hazard): we chain
|
||||
across clauses **only** when (a) the later clause stays on the **same referent**
|
||||
and (b) it carries a **licensed change cue** whose polarity is unambiguous.
|
||||
Otherwise we refuse — the guard is generalised, never weakened.
|
||||
|
||||
Reading:
|
||||
|
||||
1. **Anchor** — clause 1 must establish exactly one quantity ``(actor, N, unit)``.
|
||||
2. **Change steps** — each later quantity-bearing clause applies ``+ M`` (gain) or
|
||||
``- M`` (loss) to the running total, where ``M`` is the clause's single grounded
|
||||
quantity, taken **in the anchor's unit** (``9 more`` = 9 more *apples*; the unit
|
||||
is inherited from the running total, which is what accumulation means).
|
||||
3. **Gate** — the constructed chain runs through the unchanged self-verification
|
||||
gate (grounding ∧ cue ∧ unit ∧ completeness ∧ uniqueness). The gate keeps
|
||||
wrong=0; this only proposes a structurally-licensed candidate.
|
||||
|
||||
Polarity (ordered, so the ambiguous ``gives`` is resolved, never guessed):
|
||||
|
||||
* ``more`` present -> **gain** (covers ``buys/gets/…
|
||||
N more`` and ``gives her N more`` — the subject is the recipient);
|
||||
* else an unambiguous **loss** verb -> **loss**;
|
||||
* else ``gives``/``gave`` with ``to``/``away`` -> **loss** (gives N *to* someone);
|
||||
* else an unambiguous **gain** verb -> **gain**;
|
||||
* else -> **refuse** (no guessing).
|
||||
|
||||
Referent guard (wrong=0-critical; the ADR-0174 multi-actor hazard's defensive fix,
|
||||
built minimally in the clean lane rather than resurrecting the retired resolver):
|
||||
a later clause stays on the anchor's referent iff its **subject token** is a
|
||||
pronoun (``He/She/They/…``) or the same name as the anchor's subject. A **new named
|
||||
subject** (a different capitalised non-pronoun first token, e.g. ``Tom``) -> refuse.
|
||||
Pronoun gender/number is **not** matched (that was the old resolver's trap); a new
|
||||
*name* is the only signal, and it triggers refusal, not resolution.
|
||||
|
||||
Sealed (no ``chat/`` import); deterministic; refuse-preferring.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Final
|
||||
|
||||
from generate.derivation.clauses import segment_clauses
|
||||
from generate.derivation.extract import extract_quantities
|
||||
from generate.derivation.model import GroundedDerivation, Quantity, Step
|
||||
from generate.derivation.verify import Resolution, select_self_verified
|
||||
from generate.math_roundtrip import _tokens
|
||||
|
||||
# Closed change-cue lexeme sets (ADR-0165: lexemes, not grammar templates; refined
|
||||
# by the CP ledger, not asserted complete). Sorted use keeps cue selection stable.
|
||||
_GAIN_VERBS: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
"buys", "bought", "gets", "got", "finds", "found", "picks", "picked",
|
||||
"earns", "earned", "receives", "received", "collects", "collected",
|
||||
"wins", "won", "makes", "made", "gains", "gained", "adds", "added",
|
||||
}
|
||||
)
|
||||
_LOSS_VERBS: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
"loses", "lost", "spends", "spent", "uses", "used", "eats", "ate",
|
||||
"sells", "sold", "donates", "donated", "drops", "dropped", "removes",
|
||||
"removed", "breaks", "broke",
|
||||
}
|
||||
)
|
||||
_PRONOUNS: Final[frozenset[str]] = frozenset(
|
||||
{"he", "she", "they", "it", "him", "her", "them", "his", "hers", "its", "their", "we", "i", "you"}
|
||||
)
|
||||
|
||||
_WORD_RE: Final[re.Pattern[str]] = re.compile(r"[A-Za-z]+")
|
||||
|
||||
|
||||
def _subject_token(clause: str) -> str | None:
|
||||
"""The clause's leading word token (its surface subject), or None if wordless."""
|
||||
match = _WORD_RE.search(clause)
|
||||
return match.group(0) if match is not None else None
|
||||
|
||||
|
||||
def _same_referent(clause: str, anchor_subject: str | None) -> bool:
|
||||
"""True iff ``clause`` does not introduce a new *named* subject.
|
||||
|
||||
Conservative: a leading pronoun continues the referent; a leading token equal
|
||||
to the anchor's subject continues it; any other capitalised (named) leading
|
||||
token is a *new actor* and breaks the referent (-> caller refuses).
|
||||
"""
|
||||
subject = _subject_token(clause)
|
||||
if subject is None:
|
||||
return True # wordless fragment carries no new actor
|
||||
if subject.lower() in _PRONOUNS:
|
||||
return True
|
||||
if anchor_subject is not None and subject == anchor_subject:
|
||||
return True
|
||||
# A new capitalised, non-pronoun leading token is a new named actor.
|
||||
return not subject[:1].isupper()
|
||||
|
||||
|
||||
def _polarity(clause: str) -> int | None:
|
||||
"""+1 (gain), -1 (loss), or None (ambiguous / no licensed change cue -> refuse)."""
|
||||
tokens = set(_tokens(clause))
|
||||
if "more" in tokens:
|
||||
return +1
|
||||
loss = bool(_LOSS_VERBS & tokens)
|
||||
gain = bool(_GAIN_VERBS & tokens)
|
||||
gives = "gives" in tokens or "gave" in tokens
|
||||
directional = "to" in tokens or "away" in tokens
|
||||
if loss and not gain:
|
||||
return -1
|
||||
if gives and directional and not gain and not loss:
|
||||
return -1
|
||||
if gain and not loss:
|
||||
return +1
|
||||
return None
|
||||
|
||||
|
||||
def _cue(clause: str, polarity: int) -> str:
|
||||
"""A grounded cue lexeme present in the clause (for the gate's cue check)."""
|
||||
tokens = set(_tokens(clause))
|
||||
if "more" in tokens and polarity > 0:
|
||||
return "more"
|
||||
verbs = _GAIN_VERBS if polarity > 0 else _LOSS_VERBS
|
||||
present = sorted(verbs & tokens)
|
||||
if present:
|
||||
return present[0]
|
||||
return "gives" # the only remaining licensed loss path (gives … to/away)
|
||||
|
||||
|
||||
def compose_accumulation(problem_text: str) -> Resolution | None:
|
||||
"""GB-3b.1 composer — single-referent gain/loss accumulation. Refuse-preferring."""
|
||||
clauses = segment_clauses(problem_text)
|
||||
quantity_clauses = [c for c in clauses if extract_quantities(c)]
|
||||
if len(quantity_clauses) < 2:
|
||||
return None
|
||||
|
||||
anchor_clause, *change_clauses = quantity_clauses
|
||||
anchor_quantities = extract_quantities(anchor_clause)
|
||||
if len(anchor_quantities) != 1:
|
||||
return None # the anchor must establish exactly one quantity (GB-3b.1 scope)
|
||||
start = anchor_quantities[0]
|
||||
anchor_subject = _subject_token(anchor_clause)
|
||||
|
||||
steps: list[Step] = []
|
||||
for clause in change_clauses:
|
||||
if not _same_referent(clause, anchor_subject):
|
||||
return None # new named actor -> referent hazard -> refuse
|
||||
change_quantities = extract_quantities(clause)
|
||||
if len(change_quantities) != 1:
|
||||
return None # one change per clause (multi-change is GB-3b.2)
|
||||
polarity = _polarity(clause)
|
||||
if polarity is None:
|
||||
return None # no unambiguous licensed change cue -> refuse
|
||||
change = change_quantities[0]
|
||||
# The change is in the running total's dimension ("9 more" = 9 more apples).
|
||||
operand = Quantity(value=change.value, unit=start.unit, source_token=change.source_token)
|
||||
op = "add" if polarity > 0 else "subtract"
|
||||
steps.append(Step(op=op, operand=operand, cue=_cue(clause, polarity)))
|
||||
|
||||
if not steps:
|
||||
return None
|
||||
derivation = GroundedDerivation(start=start, steps=tuple(steps))
|
||||
return select_self_verified([derivation], problem_text, target_units=())
|
||||
76
tests/test_adr_0178_gb3b1_accumulation.py
Normal file
76
tests/test_adr_0178_gb3b1_accumulation.py
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
"""ADR-0178 GB-3b.1 — single-referent accumulation chaining.
|
||||
|
||||
The first cross-clause comprehension reading. These tests prove the obligations
|
||||
from the GB-3b scope: the accumulation flips the gain/loss cases, and the wrong=0
|
||||
guards (multi-actor, absent/ambiguous change cue, the GB-3a hazards) each REFUSE.
|
||||
Every refusal test would *fail* if the guard it covers were removed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from generate.derivation.accumulate import compose_accumulation
|
||||
|
||||
|
||||
def _answer(text: str):
|
||||
res = compose_accumulation(text)
|
||||
return None if res is None else res.answer
|
||||
|
||||
|
||||
class TestAccumulationFlips:
|
||||
def test_gain_with_more(self) -> None:
|
||||
assert _answer("Sam has 14 apples. He buys 9 more. How many apples does Sam have now?") == 23.0
|
||||
|
||||
def test_gain_verb_no_more(self) -> None:
|
||||
assert _answer("Ben has 20 marbles. He finds 7 on the playground. How many marbles?") == 27.0
|
||||
|
||||
def test_gain_giver_to_subject_reads_as_more(self) -> None:
|
||||
# "Her teacher gives her 5 more" — subject (pronoun) is the recipient -> +5.
|
||||
assert _answer("Kate has 18 pencils. Her teacher gives her 5 more. How many pencils?") == 23.0
|
||||
|
||||
def test_loss_verb(self) -> None:
|
||||
assert _answer("Sam has 30 apples. He eats 8. How many apples does Sam have left?") == 22.0
|
||||
|
||||
def test_loss_gives_to_recipient(self) -> None:
|
||||
assert _answer("Anna has 25 stickers. She gives 10 to her friend. How many stickers left?") == 15.0
|
||||
|
||||
|
||||
class TestReferentGuardRefuses:
|
||||
def test_new_named_actor_refuses(self) -> None:
|
||||
# The Alice/Tom hazard: a new named subject -> refuse (not Sam's 14+9).
|
||||
assert _answer("Sam has 14 apples. Tom buys 9 more. How many apples does Sam have?") is None
|
||||
|
||||
def test_h1_unrelated_same_unit_across_sentences(self) -> None:
|
||||
assert _answer("Alice has 6 apples. Tom has 2 apples. How many apples does Alice have?") is None
|
||||
|
||||
|
||||
class TestChangeCueGuardRefuses:
|
||||
def test_no_change_cue_refuses(self) -> None:
|
||||
# two quantities, no gain/loss verb and no "more" -> no licensed change -> refuse.
|
||||
assert _answer("Lisa has 30 coins. She has 15 stickers. How many coins does Lisa have?") is None
|
||||
|
||||
def test_anchor_must_be_single_quantity(self) -> None:
|
||||
# a list anchor is GB-2a's job, not accumulation -> refuse here.
|
||||
assert _answer("Sam has 6 apples and 4 apples. He buys 5 more. How many?") is None
|
||||
|
||||
def test_multi_change_in_one_clause_refuses(self) -> None:
|
||||
# "gets 5 more from Tom and 3 more from Lisa" is multi-change (GB-3b.2) -> refuse.
|
||||
assert _answer("Sam has 10 apples. He gets 5 more and 3 more. How many?") is None
|
||||
|
||||
|
||||
class TestDeterminism:
|
||||
def test_deterministic(self) -> None:
|
||||
t = "Sam has 14 apples. He buys 9 more. How many?"
|
||||
assert compose_accumulation(t) == compose_accumulation(t)
|
||||
|
||||
|
||||
class TestPracticeLaneFlip:
|
||||
def test_accumulation_flips_a_chunk_with_no_new_wrong(self) -> None:
|
||||
from evals.gsm8k_math.practice.v1.accumulation_runner import build_accumulation_report
|
||||
from evals.gsm8k_math.practice.v1.runner import build_practice_report
|
||||
|
||||
before = build_practice_report().counts
|
||||
after = build_accumulation_report().counts
|
||||
# accumulation fires only on refusals: it adds correct, never a new wrong.
|
||||
assert after["correct"] >= 55
|
||||
assert after["wrong"] == before["wrong"]
|
||||
assert after["correct"] + after["wrong"] + after["refused"] == sum(before.values())
|
||||
Loading…
Reference in a new issue