diff --git a/evals/gsm8k_math/confusers/v1/runner.py b/evals/gsm8k_math/confusers/v1/runner.py index f8ebf91d..a728c992 100644 --- a/evals/gsm8k_math/confusers/v1/runner.py +++ b/evals/gsm8k_math/confusers/v1/runner.py @@ -22,21 +22,23 @@ from dataclasses import dataclass from pathlib import Path from typing import Final -from generate.derivation.accumulate import compose_accumulation -from generate.derivation.multistep import search_chain -from generate.derivation.search import search_multiplicative +from generate.derivation.pool import resolve_pooled _CASES_PATH: Final[Path] = Path(__file__).resolve().parent / "cases.jsonl" _TOL: Final[float] = 1e-6 def _engine_answer(problem_text: str) -> float | None: - """The sealed engine's attempt: first composer to resolve wins (fixed order).""" - for composer in (compose_accumulation, search_multiplicative, search_chain): - resolution = composer(problem_text) - if resolution is not None: - return resolution.answer - return None + """The sealed engine's attempt (ADR-0182 cross-composer pooling). + + Pools the ungated readings of every composer (accumulation, multiplicative, + chain) and resolves them together: a single ``complete`` answer commits; + disagreement among the pool — e.g. a distractor problem's blunt product vs its + competing additive reading — refuses. Replaces the prior first-composer-wins + order, which had no way to notice it held two incompatible readings. + """ + resolution = resolve_pooled(problem_text) + return resolution.answer if resolution is not None else None @dataclass(frozen=True, slots=True) diff --git a/generate/derivation/__init__.py b/generate/derivation/__init__.py index caa5c86e..c8c3b580 100644 --- a/generate/derivation/__init__.py +++ b/generate/derivation/__init__.py @@ -12,7 +12,7 @@ from generate.derivation.clauses import ( clause_local_results, segment_clauses, ) -from generate.derivation.accumulate import compose_accumulation +from generate.derivation.accumulate import accumulation_candidates, compose_accumulation from generate.derivation.compose import compose_sequential from generate.derivation.comparatives import ( ComparativeScalar, @@ -21,12 +21,18 @@ from generate.derivation.comparatives import ( ) from generate.derivation.extract import extract_quantities from generate.derivation.model import GroundedDerivation, Quantity, Step, VALID_OPS -from generate.derivation.multistep import search_chain -from generate.derivation.search import MULTIPLICATIVE_CUES, search_multiplicative +from generate.derivation.multistep import candidate_chains, search_chain +from generate.derivation.pool import pooled_candidates, resolve_pooled +from generate.derivation.search import ( + MULTIPLICATIVE_CUES, + multiplicative_candidates, + search_multiplicative, +) from generate.derivation.target import Target, extract_target from generate.derivation.verify import ( Resolution, SelfVerification, + classify_derivation, select_self_verified, self_verifies, ) @@ -42,6 +48,9 @@ __all__ = [ "Step", "Target", "VALID_OPS", + "accumulation_candidates", + "candidate_chains", + "classify_derivation", "clause_local_results", "compose_accumulation", "compose_sequential", @@ -49,6 +58,9 @@ __all__ = [ "extract_comparative_scalars", "extract_quantities", "extract_target", + "multiplicative_candidates", + "pooled_candidates", + "resolve_pooled", "search_chain", "search_multiplicative", "segment_clauses", diff --git a/generate/derivation/accumulate.py b/generate/derivation/accumulate.py index 5ee8c2a5..cc1d5fbb 100644 --- a/generate/derivation/accumulate.py +++ b/generate/derivation/accumulate.py @@ -127,8 +127,20 @@ def _cue(clause: str, polarity: int) -> str: return "gives" # the only remaining licensed loss path (gives … to/away) -def compose_accumulation(problem_text: str) -> Resolution | None: - """GB-3b.1 composer — single-referent gain/loss accumulation. Refuse-preferring.""" +def _build_accumulation( + problem_text: str, *, drop_isolated_foreign: bool +) -> GroundedDerivation | None: + """Construct the single-referent accumulation chain (ungated). + + ``drop_isolated_foreign`` (ADR-0182): when a change clause carries more than + one quantity, drop those with a **non-empty unit foreign to the anchor's unit** + (a candidate distractor — ``studies for 3 hours`` among ``pencils``) and proceed + if exactly one same-unit/unitless change remains. With the flag off this is the + strict GB-3b.1 reading (a multi-quantity change clause refuses), so + :func:`compose_accumulation` is byte-identical to its pre-ADR-0182 behavior. + The distractor-skip reading is **never committed alone** — it only ever enters + the pool to force a disagreement refusal (see :mod:`generate.derivation.pool`). + """ clauses = segment_clauses(problem_text) quantity_clauses = [c for c in clauses if extract_quantities(c)] if len(quantity_clauses) < 2: @@ -145,7 +157,11 @@ def compose_accumulation(problem_text: str) -> Resolution | None: for clause in change_clauses: if not _same_referent(clause, anchor_subject): return None # new named actor -> referent hazard -> refuse - change_quantities = extract_quantities(clause) + change_quantities = list(extract_quantities(clause)) + if drop_isolated_foreign and len(change_quantities) > 1: + change_quantities = [ + q for q in change_quantities if not (q.unit and q.unit != start.unit) + ] if len(change_quantities) != 1: return None # one change per clause (multi-change is GB-3b.2) polarity = _polarity(clause) @@ -159,5 +175,31 @@ def compose_accumulation(problem_text: str) -> Resolution | None: if not steps: return None - derivation = GroundedDerivation(start=start, steps=tuple(steps)) + return GroundedDerivation(start=start, steps=tuple(steps)) + + +def compose_accumulation(problem_text: str) -> Resolution | None: + """GB-3b.1 composer — single-referent gain/loss accumulation. Refuse-preferring. + + The strict (commit) reading: it gates the no-distractor-skip derivation through + the unchanged self-verification gate. Behavior is byte-identical to pre-ADR-0182. + """ + derivation = _build_accumulation(problem_text, drop_isolated_foreign=False) + if derivation is None: + return None return select_self_verified([derivation], problem_text, target_units=()) + + +def accumulation_candidates(problem_text: str) -> tuple[GroundedDerivation, ...]: + """ADR-0182 — the ungated accumulation readings for cross-composer pooling. + + The strict reading and (when it differs) the distractor-skip reading. Ungated: + the pool classifies each (``complete`` commits, ``exempt`` refuses-only) and the + disagreement rule does the wrong=0 work. Deterministic; de-dup is the pool's job. + """ + candidates: list[GroundedDerivation] = [] + for drop in (False, True): + derivation = _build_accumulation(problem_text, drop_isolated_foreign=drop) + if derivation is not None: + candidates.append(derivation) + return tuple(candidates) diff --git a/generate/derivation/pool.py b/generate/derivation/pool.py new file mode 100644 index 00000000..b458aeb9 --- /dev/null +++ b/generate/derivation/pool.py @@ -0,0 +1,88 @@ +"""ADR-0182 — cross-composer disagreement pooling. + +The distractor-quantity confusers (``confuser-v1-0014`` → 300, ``0016`` → 3840) +misfire because a blunt product-of-all is the *unique* self-verifying reading: the +completeness clause forces the distractor into it, and no rival reading exists to +trigger the wrong=0 disagreement rule. The microscope showed no tight cue/gate rule +separates these from the legitimate cross-unit products (``for`` licenses both the +0014 distractor and the correct ``train-0021`` product) — that is the deferred +cue-precision problem (ADR-0177). + +This module sidesteps cue precision: it pools the **ungated candidate readings of +every composer** (accumulation, in-clause multiplicative, target-guided chain) for +one problem and resolves them together. A reading is classified +(:func:`generate.derivation.verify.classify_derivation`) as ``complete`` +(commit-eligible) or ``exempt`` (commit-INELIGIBLE — verified but for an +isolated-foreign *distractor* quantity it leaves unused). The rule: + +* candidates disagree (more than one distinct answer across the pool) → **refuse** + (the wrong=0 disagreement rule does the work); +* a single distinct answer → commit **only if** a ``complete`` candidate produced it + (an ``exempt``-only answer never commits — full completeness is still required to + resolve, so ADR-0175's multi-step-incomplete defence is untouched). + +For a distractor problem this gives the product (``complete``) a competing additive +reading (``exempt``) to disagree with → refusal; for a genuine product (0021/0003) +there is no rival reading, so it still commits. Deterministic; sealed (serving is +never invoked here). +""" + +from __future__ import annotations + +from generate.derivation.accumulate import accumulation_candidates +from generate.derivation.model import GroundedDerivation +from generate.derivation.multistep import candidate_chains +from generate.derivation.search import multiplicative_candidates +from generate.derivation.verify import Resolution, classify_derivation + + +def pooled_candidates(problem_text: str) -> list[GroundedDerivation]: + """Every composer's ungated candidate readings, de-duplicated. Deterministic + order (accumulation, then multiplicative, then chain).""" + seen: set[tuple[object, ...]] = set() + pooled: list[GroundedDerivation] = [] + for derivation in ( + *accumulation_candidates(problem_text), + *multiplicative_candidates(problem_text), + *candidate_chains(problem_text), + ): + key = ( + round(derivation.answer, 9), + derivation.start.source_token, + tuple((step.op, step.operand.source_token) for step in derivation.steps), + ) + if key in seen: + continue + seen.add(key) + pooled.append(derivation) + return pooled + + +def resolve_pooled(problem_text: str) -> Resolution | None: + """Resolve a problem by pooling every composer's readings. Refuse-preferring. + + Refuses on no verifying candidate, on disagreement among the pool, or when the + sole answer is produced only by commit-ineligible (``exempt``) readings. + """ + classified = [ + (kind, derivation) + for derivation in pooled_candidates(problem_text) + if (kind := classify_derivation(derivation, problem_text)) is not None + ] + if not classified: + return None + + distinct = {round(derivation.answer, 9) for _, derivation in classified} + if len(distinct) != 1: + return None # disagreement -> refuse (wrong=0) + + committable = [d for kind, d in classified if kind == "complete"] + if not committable: + return None # exempt-only -> refuse (a commit requires full completeness) + + chosen = committable[0] + return Resolution( + answer=chosen.answer, + answer_unit=chosen.answer_unit, + derivation=chosen, + ) diff --git a/generate/derivation/search.py b/generate/derivation/search.py index c3d8d321..486712f2 100644 --- a/generate/derivation/search.py +++ b/generate/derivation/search.py @@ -65,6 +65,13 @@ def _sentence_candidates(problem_text: str) -> list[GroundedDerivation]: return candidates +def multiplicative_candidates(problem_text: str) -> list[GroundedDerivation]: + """ADR-0182 — the ungated in-clause product candidates, for cross-composer + pooling. Same construction :func:`search_multiplicative` gates, exposed so the + pool can weigh products against the other composers' readings.""" + return _sentence_candidates(problem_text) + + def search_multiplicative(problem_text: str) -> Resolution | None: """Attempt a grounded in-clause multiplicative product. diff --git a/generate/derivation/verify.py b/generate/derivation/verify.py index 7696e21c..97f51dbf 100644 --- a/generate/derivation/verify.py +++ b/generate/derivation/verify.py @@ -50,9 +50,10 @@ class Resolution: derivation: GroundedDerivation -def self_verifies(derivation: GroundedDerivation, problem_text: str) -> SelfVerification: - """Decide whether ``derivation`` self-verifies against ``problem_text``.""" - tokens = _tokens(problem_text) +def _base_reasons(derivation: GroundedDerivation, tokens: frozenset[str]) -> list[str]: + """The grounding ∧ cue ∧ unit ∧ divide-by-zero clauses (everything *but* + completeness). Shared by :func:`self_verifies` and :func:`classify_derivation` + so the two cannot drift.""" reasons: list[str] = [] # 1. operand grounding — every TEXT operand value must be sourced from the @@ -82,6 +83,24 @@ def self_verifies(derivation: GroundedDerivation, problem_text: str) -> SelfVeri if step.op == "divide" and step.operand.value == 0: reasons.append("division by zero") + return reasons + + +def _unused_quantities(derivation: GroundedDerivation, problem_text: str) -> Counter[str]: + """Problem quantities (by source token) the derivation does not consume.""" + problem_quantities = Counter(q.source_token for q in extract_quantities(problem_text)) + used = Counter( + [derivation.start.source_token] + + [step.operand.source_token for step in derivation.steps] + ) + return problem_quantities - used + + +def self_verifies(derivation: GroundedDerivation, problem_text: str) -> SelfVerification: + """Decide whether ``derivation`` self-verifies against ``problem_text``.""" + tokens = _tokens(problem_text) + reasons = _base_reasons(derivation, tokens) + # 5. completeness — a trustworthy derivation must account for every quantity # the problem states. A derivation that ignores given numbers is an # incomplete reading (typically a correct *first step* of a multi-step @@ -90,15 +109,54 @@ def self_verifies(derivation: GroundedDerivation, problem_text: str) -> SelfVeri # microscope identified (ADR-0175 self-verification strengthening): it # catches the multi-step-incomplete attempts the cue/grounding clauses # cannot, because their operands ARE grounded. - problem_quantities = Counter(q.source_token for q in extract_quantities(problem_text)) - used = Counter([derivation.start.source_token] + [step.operand.source_token for step in derivation.steps]) - unused = problem_quantities - used + unused = _unused_quantities(derivation, problem_text) if unused: reasons.append(f"incomplete: unused problem quantities {sorted(unused.keys())}") return SelfVerification(verified=not reasons, reasons=tuple(reasons)) +def classify_derivation(derivation: GroundedDerivation, problem_text: str) -> str | None: + """ADR-0182 — the commit-eligibility class of a derivation, for pooling. + + Returns: + + * ``"complete"`` — passes every clause *including* full completeness; + **commit-eligible** (may resolve as an answer). + * ``"exempt"`` — passes every base clause and the only unused quantities are + **isolated-foreign** (unit non-empty ∧ equal to no *used* operand's unit, i.e. + a candidate distractor standing alone in a dimension the reading never + touches); **commit-INELIGIBLE**. It exists only to enter the pool and force a + disagreement → refusal, never to be committed alone (see :mod:`generate. + derivation.pool`). This keeps the commit-path completeness guarantee + (ADR-0175's multi-step-incomplete defence) intact — the exemption widens only + what can *refuse*. + * ``None`` — fails a base clause, or an unused quantity is not + isolated-foreign (empty unit, or a unit shared with a used operand → real + signal the reading dropped). + """ + tokens = _tokens(problem_text) + if _base_reasons(derivation, tokens): + return None + unused = _unused_quantities(derivation, problem_text) + if not unused: + return "complete" + + used_units = {derivation.start.unit, *(step.operand.unit for step in derivation.steps)} + units_by_token: dict[str, set[str]] = {} + for q in extract_quantities(problem_text): + units_by_token.setdefault(q.source_token, set()).add(q.unit) + + for token in unused: + token_units = units_by_token.get(token, {""}) + # isolated-foreign iff *every* occurrence has a non-empty unit not shared + # with any used operand. An empty unit, or a unit a used operand carries, + # disqualifies the exemption — that quantity is real signal, not a distractor. + if any((not unit) or (unit in used_units) for unit in token_units): + return None + return "exempt" + + def select_self_verified( derivations: list[GroundedDerivation], problem_text: str, diff --git a/tests/test_adr_0163_f2_confusers.py b/tests/test_adr_0163_f2_confusers.py index eb1058f5..be1f40d3 100644 --- a/tests/test_adr_0163_f2_confusers.py +++ b/tests/test_adr_0163_f2_confusers.py @@ -24,8 +24,14 @@ from evals.gsm8k_math.confusers.v1.runner import ( # misfires (0005->796, 0007->996) now refuse — the 25-foot/20-inch divisor is # finally visible to the completeness/polarity gate. wrong 7->5. pair_tells # unchanged (the pseudo-accumulation cases carry no positive twin). -_BASELINE_WRONG = 5 -_BASELINE_PAIR_TELLS = 4 +# 2026-05-29 (ADR-0182 cross-composer pooling): the engine now pools every +# composer's readings and refuses on disagreement. The distractor 0014 +# (product 300 vs additive 25) and BOTH disguised-polarity cases (0001/0003: +# the spurious "for N coins" product disagrees with the accumulation reading) +# now refuse. wrong 5->2 (0016 needs anchor-skip; 0020 is temporal-scope). +# pair_tells 4->1 (0001/0003/0014 clear; 0020 remains). +_BASELINE_WRONG = 2 +_BASELINE_PAIR_TELLS = 1 class TestSchema: @@ -87,5 +93,23 @@ class TestProbeBaseline: "divisor is no longer reaching the completeness/polarity gate." ) + def test_distractor_0014_refuses_via_pooling(self) -> None: + # ADR-0182: 0014's blunt product (300) and its competing additive reading + # (25) disagree under cross-composer pooling -> refuse. Fails loudly if + # pooling regresses (the unique product would commit 300 again). 0016 is + # NOT asserted here — its distractor sits in the anchor clause and needs + # anchor-skip, a separate step (see ADR-0182 §3b). + by_id = {r.case_id: r for r in run_probe()} + assert by_id["confuser-v1-0014"].verdict == "refused" + + def test_disguised_polarity_does_not_misfire(self) -> None: + # ADR-0182 bonus: the spurious "buys X for N " product disagrees with + # the accumulation reading, so both disguised-polarity cases refuse instead + # of committing the gain-polarity sum. + results = run_probe() + disguised = [r for r in results if r.category == "disguised-polarity"] + assert disguised + assert all(r.verdict != "wrong" for r in disguised) + def test_deterministic(self) -> None: assert summarize(run_probe()) == summarize(run_probe()) diff --git a/tests/test_adr_0182_pool.py b/tests/test_adr_0182_pool.py new file mode 100644 index 00000000..8a717e33 --- /dev/null +++ b/tests/test_adr_0182_pool.py @@ -0,0 +1,104 @@ +"""ADR-0182 — cross-composer disagreement pooling (distractor refusal). + +Two surfaces under test: + +* :func:`generate.derivation.verify.classify_derivation` — the commit-eligibility + class (``complete`` / ``exempt`` / ``None``) and, critically, that the + isolated-foreign exemption is *narrow* (an empty-unit or same-unit unused + quantity is never exempt — it is real signal, not a distractor). +* :func:`generate.derivation.pool.resolve_pooled` — the pooled resolution: a clean + reading commits, a distractor problem's product-vs-additive disagreement refuses, + and (the wrong=0-critical property) an ``exempt``-only answer **never commits**. + +Sealed lane: ``chat/`` does not import these; serving ``3/47/0`` cannot move. +""" + +from __future__ import annotations + +from generate.derivation.accumulate import accumulation_candidates +from generate.derivation.model import GroundedDerivation, Quantity, Step +from generate.derivation.pool import resolve_pooled +from generate.derivation.verify import classify_derivation + +_DISTRACTOR_0014 = ( + "Kate has 20 pencils. She studies for 3 hours and then buys 5 more pencils. " + "How many pencils does Kate have?" +) +# A distractor with NO multiplicative cue: the only candidate is the exempt additive +# reading (no complete product exists). The commit-ineligibility rule must refuse it. +_EXEMPT_ONLY = ( + "Kate has 20 pencils. She rests for 3 hours and buys 5 more pencils." +) +_CLEAN_ACCUMULATION = "Sam has 14 apples. He buys 9 more apples." + + +class TestClassifyDerivation: + def test_complete_reading_is_commit_eligible(self) -> None: + # uses every quantity -> complete + derivation = accumulation_candidates(_CLEAN_ACCUMULATION)[0] + assert classify_derivation(derivation, _CLEAN_ACCUMULATION) == "complete" + + def test_isolated_foreign_unused_is_exempt(self) -> None: + # the additive reading of 0014 leaves "3 hours" unused; hours is foreign to + # the used unit (pencils) -> exempt (commit-ineligible). + derivation = GroundedDerivation( + start=Quantity(20.0, "pencils", "20"), + steps=(Step(op="add", operand=Quantity(5.0, "pencils", "5"), cue="more"),), + ) + assert classify_derivation(derivation, _DISTRACTOR_0014) == "exempt" + + def test_same_unit_unused_is_not_exempt(self) -> None: + # a quantity sharing the reading's unit is real signal, never a distractor: + # leaving it unused must NOT be exempted (it stays invalid -> None). + text = "Sam has 14 apples. He buys 9 more apples. He eats 2 apples." + derivation = GroundedDerivation( + start=Quantity(14.0, "apples", "14"), + steps=(Step(op="add", operand=Quantity(9.0, "apples", "9"), cue="more"),), + ) + assert classify_derivation(derivation, text) is None + + def test_empty_unit_unused_is_not_exempt(self) -> None: + # an unused quantity with an unknown (empty) unit cannot be shown foreign, + # so it is never exempt — completeness still rejects the reading. + text = "Sam has 14 apples. He buys 9 more apples. He had 2." + derivation = GroundedDerivation( + start=Quantity(14.0, "apples", "14"), + steps=(Step(op="add", operand=Quantity(9.0, "apples", "9"), cue="more"),), + ) + assert classify_derivation(derivation, text) is None + + def test_ungrounded_operand_is_invalid(self) -> None: + derivation = GroundedDerivation( + start=Quantity(14.0, "apples", "14"), + steps=(Step(op="add", operand=Quantity(999.0, "apples", "999"), cue="more"),), + ) + assert classify_derivation(derivation, _CLEAN_ACCUMULATION) is None + + +class TestResolvePooled: + def test_clean_accumulation_commits(self) -> None: + resolution = resolve_pooled(_CLEAN_ACCUMULATION) + assert resolution is not None + assert resolution.answer == 23.0 + + def test_distractor_0014_refuses_via_disagreement(self) -> None: + # product 300 (complete) vs additive 25 (exempt) disagree -> refuse. + assert resolve_pooled(_DISTRACTOR_0014) is None + + def test_exempt_only_never_commits(self) -> None: + # THE wrong=0-critical obligation. The only verifying reading here is the + # exempt additive one (no multiplicative cue -> no competing product), so a + # commit would be on an incomplete reading. The commit-ineligibility rule + # must refuse. Flipping `exempt` to commit-eligible makes this commit 25 and + # this test fails loudly. + assert accumulation_candidates(_EXEMPT_ONLY), "expected an exempt candidate" + assert classify_derivation( + accumulation_candidates(_EXEMPT_ONLY)[-1], _EXEMPT_ONLY + ) == "exempt" + assert resolve_pooled(_EXEMPT_ONLY) is None + + def test_deterministic(self) -> None: + assert resolve_pooled(_DISTRACTOR_0014) == resolve_pooled(_DISTRACTOR_0014) + a = resolve_pooled(_CLEAN_ACCUMULATION) + b = resolve_pooled(_CLEAN_ACCUMULATION) + assert a is not None and b is not None and a.answer == b.answer