feat(adr-0182): cross-composer disagreement pooling — distractor 0014 + disguised-polarity refuse (confuser wrong 5->2) (#476)
Implements ADR-0182's first win on top of EX-6 (#473). The distractor-quantity confuser 0014 misfired (20x3x5=300) because a blunt product-of-all was the *unique* self-verifying reading: the completeness clause forces the distractor into it, and no rival reading existed to trigger the wrong=0 disagreement rule. No tight cue rule separates it from the legitimate cross-unit products (`for` licenses both the 0014 distractor AND the correct train-0021 product) -- that is the deferred cue-precision problem. So instead of a reactive patch, let the disagreement rule do the refusing. Mechanism (sealed lane; chat/ does not import these -> serving 3/47/0 frozen): - verify.classify_derivation: a derivation is `complete` (commit-eligible), `exempt` (verified but for an isolated-foreign unused quantity -> commit- INELIGIBLE), or None. Refactored self_verifies into _base_reasons + _unused_quantities so the two share logic and cannot drift (behavior identical; 385 derivation tests + smoke 67 green). - accumulate.accumulation_candidates: exposes the ungated readings, incl. a distractor-skip reading that drops an isolated-foreign quantity from a multi- quantity change clause (20+5=25). compose_accumulation is byte-identical (drop_isolated_foreign=False + the same gate). - search.multiplicative_candidates / multistep.candidate_chains: ungated candidates. - pool.resolve_pooled: pools every composer's readings; disagreement -> refuse; a single answer commits only if a `complete` candidate produced it (exempt-only -> refuse, so the commit-path completeness guarantee from ADR-0175 is untouched). - confuser runner: _engine_answer now delegates to resolve_pooled (the prior first-composer-wins order could not notice it held two incompatible readings). Result (the microscope): - confuser wrong 5 -> 2. distractor 0014 refuses (product 300 vs additive 25); BONUS: both disguised-polarity cases (0001/0003) refuse -- the spurious "buys X for N coins" product disagrees with the accumulation reading. Remaining wrong: 0016 (distractor in the anchor clause -> needs anchor-skip, separate step) and 0020 (temporal-scope). pair-tells 4 -> 1. - genuine positives still 7 solved, 0 wrong. - train_sample 3/47/0 and practice 3/47/0 byte-identical (they call compose_accumulation/search directly -- unchanged -- not the pool). - smoke 67, architectural invariants 40, lane-SHA freeze 8/8. Tests: - test_adr_0182_pool.py: classify (complete/exempt/None incl. narrow-exemption edges) + resolve_pooled, with the wrong=0 obligation test_exempt_only_never_commits (a distractor with no multiplicative cue must refuse, not commit 25 -- fails loudly if the exemption is made commit-eligible). - test_adr_0163_f2_confusers.py: baseline tightened wrong 5->2, pair_tells 4->1; new test_distractor_0014_refuses_via_pooling + test_disguised_polarity_does_not_misfire. Stacked on #473 (EX-6); merge #473 first. 0016 + the remaining wrongs are follow-ups.
This commit is contained in:
parent
3f9edd06da
commit
0fbcce429b
8 changed files with 361 additions and 24 deletions
|
|
@ -22,21 +22,23 @@ from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Final
|
from typing import Final
|
||||||
|
|
||||||
from generate.derivation.accumulate import compose_accumulation
|
from generate.derivation.pool import resolve_pooled
|
||||||
from generate.derivation.multistep import search_chain
|
|
||||||
from generate.derivation.search import search_multiplicative
|
|
||||||
|
|
||||||
_CASES_PATH: Final[Path] = Path(__file__).resolve().parent / "cases.jsonl"
|
_CASES_PATH: Final[Path] = Path(__file__).resolve().parent / "cases.jsonl"
|
||||||
_TOL: Final[float] = 1e-6
|
_TOL: Final[float] = 1e-6
|
||||||
|
|
||||||
|
|
||||||
def _engine_answer(problem_text: str) -> float | None:
|
def _engine_answer(problem_text: str) -> float | None:
|
||||||
"""The sealed engine's attempt: first composer to resolve wins (fixed order)."""
|
"""The sealed engine's attempt (ADR-0182 cross-composer pooling).
|
||||||
for composer in (compose_accumulation, search_multiplicative, search_chain):
|
|
||||||
resolution = composer(problem_text)
|
Pools the ungated readings of every composer (accumulation, multiplicative,
|
||||||
if resolution is not None:
|
chain) and resolves them together: a single ``complete`` answer commits;
|
||||||
return resolution.answer
|
disagreement among the pool — e.g. a distractor problem's blunt product vs its
|
||||||
return None
|
competing additive reading — refuses. Replaces the prior first-composer-wins
|
||||||
|
order, which had no way to notice it held two incompatible readings.
|
||||||
|
"""
|
||||||
|
resolution = resolve_pooled(problem_text)
|
||||||
|
return resolution.answer if resolution is not None else None
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,7 @@ from generate.derivation.clauses import (
|
||||||
clause_local_results,
|
clause_local_results,
|
||||||
segment_clauses,
|
segment_clauses,
|
||||||
)
|
)
|
||||||
from generate.derivation.accumulate import compose_accumulation
|
from generate.derivation.accumulate import accumulation_candidates, compose_accumulation
|
||||||
from generate.derivation.compose import compose_sequential
|
from generate.derivation.compose import compose_sequential
|
||||||
from generate.derivation.comparatives import (
|
from generate.derivation.comparatives import (
|
||||||
ComparativeScalar,
|
ComparativeScalar,
|
||||||
|
|
@ -21,12 +21,18 @@ from generate.derivation.comparatives import (
|
||||||
)
|
)
|
||||||
from generate.derivation.extract import extract_quantities
|
from generate.derivation.extract import extract_quantities
|
||||||
from generate.derivation.model import GroundedDerivation, Quantity, Step, VALID_OPS
|
from generate.derivation.model import GroundedDerivation, Quantity, Step, VALID_OPS
|
||||||
from generate.derivation.multistep import search_chain
|
from generate.derivation.multistep import candidate_chains, search_chain
|
||||||
from generate.derivation.search import MULTIPLICATIVE_CUES, search_multiplicative
|
from generate.derivation.pool import pooled_candidates, resolve_pooled
|
||||||
|
from generate.derivation.search import (
|
||||||
|
MULTIPLICATIVE_CUES,
|
||||||
|
multiplicative_candidates,
|
||||||
|
search_multiplicative,
|
||||||
|
)
|
||||||
from generate.derivation.target import Target, extract_target
|
from generate.derivation.target import Target, extract_target
|
||||||
from generate.derivation.verify import (
|
from generate.derivation.verify import (
|
||||||
Resolution,
|
Resolution,
|
||||||
SelfVerification,
|
SelfVerification,
|
||||||
|
classify_derivation,
|
||||||
select_self_verified,
|
select_self_verified,
|
||||||
self_verifies,
|
self_verifies,
|
||||||
)
|
)
|
||||||
|
|
@ -42,6 +48,9 @@ __all__ = [
|
||||||
"Step",
|
"Step",
|
||||||
"Target",
|
"Target",
|
||||||
"VALID_OPS",
|
"VALID_OPS",
|
||||||
|
"accumulation_candidates",
|
||||||
|
"candidate_chains",
|
||||||
|
"classify_derivation",
|
||||||
"clause_local_results",
|
"clause_local_results",
|
||||||
"compose_accumulation",
|
"compose_accumulation",
|
||||||
"compose_sequential",
|
"compose_sequential",
|
||||||
|
|
@ -49,6 +58,9 @@ __all__ = [
|
||||||
"extract_comparative_scalars",
|
"extract_comparative_scalars",
|
||||||
"extract_quantities",
|
"extract_quantities",
|
||||||
"extract_target",
|
"extract_target",
|
||||||
|
"multiplicative_candidates",
|
||||||
|
"pooled_candidates",
|
||||||
|
"resolve_pooled",
|
||||||
"search_chain",
|
"search_chain",
|
||||||
"search_multiplicative",
|
"search_multiplicative",
|
||||||
"segment_clauses",
|
"segment_clauses",
|
||||||
|
|
|
||||||
|
|
@ -127,8 +127,20 @@ def _cue(clause: str, polarity: int) -> str:
|
||||||
return "gives" # the only remaining licensed loss path (gives … to/away)
|
return "gives" # the only remaining licensed loss path (gives … to/away)
|
||||||
|
|
||||||
|
|
||||||
def compose_accumulation(problem_text: str) -> Resolution | None:
|
def _build_accumulation(
|
||||||
"""GB-3b.1 composer — single-referent gain/loss accumulation. Refuse-preferring."""
|
problem_text: str, *, drop_isolated_foreign: bool
|
||||||
|
) -> GroundedDerivation | None:
|
||||||
|
"""Construct the single-referent accumulation chain (ungated).
|
||||||
|
|
||||||
|
``drop_isolated_foreign`` (ADR-0182): when a change clause carries more than
|
||||||
|
one quantity, drop those with a **non-empty unit foreign to the anchor's unit**
|
||||||
|
(a candidate distractor — ``studies for 3 hours`` among ``pencils``) and proceed
|
||||||
|
if exactly one same-unit/unitless change remains. With the flag off this is the
|
||||||
|
strict GB-3b.1 reading (a multi-quantity change clause refuses), so
|
||||||
|
:func:`compose_accumulation` is byte-identical to its pre-ADR-0182 behavior.
|
||||||
|
The distractor-skip reading is **never committed alone** — it only ever enters
|
||||||
|
the pool to force a disagreement refusal (see :mod:`generate.derivation.pool`).
|
||||||
|
"""
|
||||||
clauses = segment_clauses(problem_text)
|
clauses = segment_clauses(problem_text)
|
||||||
quantity_clauses = [c for c in clauses if extract_quantities(c)]
|
quantity_clauses = [c for c in clauses if extract_quantities(c)]
|
||||||
if len(quantity_clauses) < 2:
|
if len(quantity_clauses) < 2:
|
||||||
|
|
@ -145,7 +157,11 @@ def compose_accumulation(problem_text: str) -> Resolution | None:
|
||||||
for clause in change_clauses:
|
for clause in change_clauses:
|
||||||
if not _same_referent(clause, anchor_subject):
|
if not _same_referent(clause, anchor_subject):
|
||||||
return None # new named actor -> referent hazard -> refuse
|
return None # new named actor -> referent hazard -> refuse
|
||||||
change_quantities = extract_quantities(clause)
|
change_quantities = list(extract_quantities(clause))
|
||||||
|
if drop_isolated_foreign and len(change_quantities) > 1:
|
||||||
|
change_quantities = [
|
||||||
|
q for q in change_quantities if not (q.unit and q.unit != start.unit)
|
||||||
|
]
|
||||||
if len(change_quantities) != 1:
|
if len(change_quantities) != 1:
|
||||||
return None # one change per clause (multi-change is GB-3b.2)
|
return None # one change per clause (multi-change is GB-3b.2)
|
||||||
polarity = _polarity(clause)
|
polarity = _polarity(clause)
|
||||||
|
|
@ -159,5 +175,31 @@ def compose_accumulation(problem_text: str) -> Resolution | None:
|
||||||
|
|
||||||
if not steps:
|
if not steps:
|
||||||
return None
|
return None
|
||||||
derivation = GroundedDerivation(start=start, steps=tuple(steps))
|
return GroundedDerivation(start=start, steps=tuple(steps))
|
||||||
|
|
||||||
|
|
||||||
|
def compose_accumulation(problem_text: str) -> Resolution | None:
|
||||||
|
"""GB-3b.1 composer — single-referent gain/loss accumulation. Refuse-preferring.
|
||||||
|
|
||||||
|
The strict (commit) reading: it gates the no-distractor-skip derivation through
|
||||||
|
the unchanged self-verification gate. Behavior is byte-identical to pre-ADR-0182.
|
||||||
|
"""
|
||||||
|
derivation = _build_accumulation(problem_text, drop_isolated_foreign=False)
|
||||||
|
if derivation is None:
|
||||||
|
return None
|
||||||
return select_self_verified([derivation], problem_text, target_units=())
|
return select_self_verified([derivation], problem_text, target_units=())
|
||||||
|
|
||||||
|
|
||||||
|
def accumulation_candidates(problem_text: str) -> tuple[GroundedDerivation, ...]:
|
||||||
|
"""ADR-0182 — the ungated accumulation readings for cross-composer pooling.
|
||||||
|
|
||||||
|
The strict reading and (when it differs) the distractor-skip reading. Ungated:
|
||||||
|
the pool classifies each (``complete`` commits, ``exempt`` refuses-only) and the
|
||||||
|
disagreement rule does the wrong=0 work. Deterministic; de-dup is the pool's job.
|
||||||
|
"""
|
||||||
|
candidates: list[GroundedDerivation] = []
|
||||||
|
for drop in (False, True):
|
||||||
|
derivation = _build_accumulation(problem_text, drop_isolated_foreign=drop)
|
||||||
|
if derivation is not None:
|
||||||
|
candidates.append(derivation)
|
||||||
|
return tuple(candidates)
|
||||||
|
|
|
||||||
88
generate/derivation/pool.py
Normal file
88
generate/derivation/pool.py
Normal file
|
|
@ -0,0 +1,88 @@
|
||||||
|
"""ADR-0182 — cross-composer disagreement pooling.
|
||||||
|
|
||||||
|
The distractor-quantity confusers (``confuser-v1-0014`` → 300, ``0016`` → 3840)
|
||||||
|
misfire because a blunt product-of-all is the *unique* self-verifying reading: the
|
||||||
|
completeness clause forces the distractor into it, and no rival reading exists to
|
||||||
|
trigger the wrong=0 disagreement rule. The microscope showed no tight cue/gate rule
|
||||||
|
separates these from the legitimate cross-unit products (``for`` licenses both the
|
||||||
|
0014 distractor and the correct ``train-0021`` product) — that is the deferred
|
||||||
|
cue-precision problem (ADR-0177).
|
||||||
|
|
||||||
|
This module sidesteps cue precision: it pools the **ungated candidate readings of
|
||||||
|
every composer** (accumulation, in-clause multiplicative, target-guided chain) for
|
||||||
|
one problem and resolves them together. A reading is classified
|
||||||
|
(:func:`generate.derivation.verify.classify_derivation`) as ``complete``
|
||||||
|
(commit-eligible) or ``exempt`` (commit-INELIGIBLE — verified but for an
|
||||||
|
isolated-foreign *distractor* quantity it leaves unused). The rule:
|
||||||
|
|
||||||
|
* candidates disagree (more than one distinct answer across the pool) → **refuse**
|
||||||
|
(the wrong=0 disagreement rule does the work);
|
||||||
|
* a single distinct answer → commit **only if** a ``complete`` candidate produced it
|
||||||
|
(an ``exempt``-only answer never commits — full completeness is still required to
|
||||||
|
resolve, so ADR-0175's multi-step-incomplete defence is untouched).
|
||||||
|
|
||||||
|
For a distractor problem this gives the product (``complete``) a competing additive
|
||||||
|
reading (``exempt``) to disagree with → refusal; for a genuine product (0021/0003)
|
||||||
|
there is no rival reading, so it still commits. Deterministic; sealed (serving is
|
||||||
|
never invoked here).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from generate.derivation.accumulate import accumulation_candidates
|
||||||
|
from generate.derivation.model import GroundedDerivation
|
||||||
|
from generate.derivation.multistep import candidate_chains
|
||||||
|
from generate.derivation.search import multiplicative_candidates
|
||||||
|
from generate.derivation.verify import Resolution, classify_derivation
|
||||||
|
|
||||||
|
|
||||||
|
def pooled_candidates(problem_text: str) -> list[GroundedDerivation]:
|
||||||
|
"""Every composer's ungated candidate readings, de-duplicated. Deterministic
|
||||||
|
order (accumulation, then multiplicative, then chain)."""
|
||||||
|
seen: set[tuple[object, ...]] = set()
|
||||||
|
pooled: list[GroundedDerivation] = []
|
||||||
|
for derivation in (
|
||||||
|
*accumulation_candidates(problem_text),
|
||||||
|
*multiplicative_candidates(problem_text),
|
||||||
|
*candidate_chains(problem_text),
|
||||||
|
):
|
||||||
|
key = (
|
||||||
|
round(derivation.answer, 9),
|
||||||
|
derivation.start.source_token,
|
||||||
|
tuple((step.op, step.operand.source_token) for step in derivation.steps),
|
||||||
|
)
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
pooled.append(derivation)
|
||||||
|
return pooled
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_pooled(problem_text: str) -> Resolution | None:
|
||||||
|
"""Resolve a problem by pooling every composer's readings. Refuse-preferring.
|
||||||
|
|
||||||
|
Refuses on no verifying candidate, on disagreement among the pool, or when the
|
||||||
|
sole answer is produced only by commit-ineligible (``exempt``) readings.
|
||||||
|
"""
|
||||||
|
classified = [
|
||||||
|
(kind, derivation)
|
||||||
|
for derivation in pooled_candidates(problem_text)
|
||||||
|
if (kind := classify_derivation(derivation, problem_text)) is not None
|
||||||
|
]
|
||||||
|
if not classified:
|
||||||
|
return None
|
||||||
|
|
||||||
|
distinct = {round(derivation.answer, 9) for _, derivation in classified}
|
||||||
|
if len(distinct) != 1:
|
||||||
|
return None # disagreement -> refuse (wrong=0)
|
||||||
|
|
||||||
|
committable = [d for kind, d in classified if kind == "complete"]
|
||||||
|
if not committable:
|
||||||
|
return None # exempt-only -> refuse (a commit requires full completeness)
|
||||||
|
|
||||||
|
chosen = committable[0]
|
||||||
|
return Resolution(
|
||||||
|
answer=chosen.answer,
|
||||||
|
answer_unit=chosen.answer_unit,
|
||||||
|
derivation=chosen,
|
||||||
|
)
|
||||||
|
|
@ -65,6 +65,13 @@ def _sentence_candidates(problem_text: str) -> list[GroundedDerivation]:
|
||||||
return candidates
|
return candidates
|
||||||
|
|
||||||
|
|
||||||
|
def multiplicative_candidates(problem_text: str) -> list[GroundedDerivation]:
|
||||||
|
"""ADR-0182 — the ungated in-clause product candidates, for cross-composer
|
||||||
|
pooling. Same construction :func:`search_multiplicative` gates, exposed so the
|
||||||
|
pool can weigh products against the other composers' readings."""
|
||||||
|
return _sentence_candidates(problem_text)
|
||||||
|
|
||||||
|
|
||||||
def search_multiplicative(problem_text: str) -> Resolution | None:
|
def search_multiplicative(problem_text: str) -> Resolution | None:
|
||||||
"""Attempt a grounded in-clause multiplicative product.
|
"""Attempt a grounded in-clause multiplicative product.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -50,9 +50,10 @@ class Resolution:
|
||||||
derivation: GroundedDerivation
|
derivation: GroundedDerivation
|
||||||
|
|
||||||
|
|
||||||
def self_verifies(derivation: GroundedDerivation, problem_text: str) -> SelfVerification:
|
def _base_reasons(derivation: GroundedDerivation, tokens: frozenset[str]) -> list[str]:
|
||||||
"""Decide whether ``derivation`` self-verifies against ``problem_text``."""
|
"""The grounding ∧ cue ∧ unit ∧ divide-by-zero clauses (everything *but*
|
||||||
tokens = _tokens(problem_text)
|
completeness). Shared by :func:`self_verifies` and :func:`classify_derivation`
|
||||||
|
so the two cannot drift."""
|
||||||
reasons: list[str] = []
|
reasons: list[str] = []
|
||||||
|
|
||||||
# 1. operand grounding — every TEXT operand value must be sourced from the
|
# 1. operand grounding — every TEXT operand value must be sourced from the
|
||||||
|
|
@ -82,6 +83,24 @@ def self_verifies(derivation: GroundedDerivation, problem_text: str) -> SelfVeri
|
||||||
if step.op == "divide" and step.operand.value == 0:
|
if step.op == "divide" and step.operand.value == 0:
|
||||||
reasons.append("division by zero")
|
reasons.append("division by zero")
|
||||||
|
|
||||||
|
return reasons
|
||||||
|
|
||||||
|
|
||||||
|
def _unused_quantities(derivation: GroundedDerivation, problem_text: str) -> Counter[str]:
|
||||||
|
"""Problem quantities (by source token) the derivation does not consume."""
|
||||||
|
problem_quantities = Counter(q.source_token for q in extract_quantities(problem_text))
|
||||||
|
used = Counter(
|
||||||
|
[derivation.start.source_token]
|
||||||
|
+ [step.operand.source_token for step in derivation.steps]
|
||||||
|
)
|
||||||
|
return problem_quantities - used
|
||||||
|
|
||||||
|
|
||||||
|
def self_verifies(derivation: GroundedDerivation, problem_text: str) -> SelfVerification:
|
||||||
|
"""Decide whether ``derivation`` self-verifies against ``problem_text``."""
|
||||||
|
tokens = _tokens(problem_text)
|
||||||
|
reasons = _base_reasons(derivation, tokens)
|
||||||
|
|
||||||
# 5. completeness — a trustworthy derivation must account for every quantity
|
# 5. completeness — a trustworthy derivation must account for every quantity
|
||||||
# the problem states. A derivation that ignores given numbers is an
|
# the problem states. A derivation that ignores given numbers is an
|
||||||
# incomplete reading (typically a correct *first step* of a multi-step
|
# incomplete reading (typically a correct *first step* of a multi-step
|
||||||
|
|
@ -90,15 +109,54 @@ def self_verifies(derivation: GroundedDerivation, problem_text: str) -> SelfVeri
|
||||||
# microscope identified (ADR-0175 self-verification strengthening): it
|
# microscope identified (ADR-0175 self-verification strengthening): it
|
||||||
# catches the multi-step-incomplete attempts the cue/grounding clauses
|
# catches the multi-step-incomplete attempts the cue/grounding clauses
|
||||||
# cannot, because their operands ARE grounded.
|
# cannot, because their operands ARE grounded.
|
||||||
problem_quantities = Counter(q.source_token for q in extract_quantities(problem_text))
|
unused = _unused_quantities(derivation, problem_text)
|
||||||
used = Counter([derivation.start.source_token] + [step.operand.source_token for step in derivation.steps])
|
|
||||||
unused = problem_quantities - used
|
|
||||||
if unused:
|
if unused:
|
||||||
reasons.append(f"incomplete: unused problem quantities {sorted(unused.keys())}")
|
reasons.append(f"incomplete: unused problem quantities {sorted(unused.keys())}")
|
||||||
|
|
||||||
return SelfVerification(verified=not reasons, reasons=tuple(reasons))
|
return SelfVerification(verified=not reasons, reasons=tuple(reasons))
|
||||||
|
|
||||||
|
|
||||||
|
def classify_derivation(derivation: GroundedDerivation, problem_text: str) -> str | None:
|
||||||
|
"""ADR-0182 — the commit-eligibility class of a derivation, for pooling.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
|
||||||
|
* ``"complete"`` — passes every clause *including* full completeness;
|
||||||
|
**commit-eligible** (may resolve as an answer).
|
||||||
|
* ``"exempt"`` — passes every base clause and the only unused quantities are
|
||||||
|
**isolated-foreign** (unit non-empty ∧ equal to no *used* operand's unit, i.e.
|
||||||
|
a candidate distractor standing alone in a dimension the reading never
|
||||||
|
touches); **commit-INELIGIBLE**. It exists only to enter the pool and force a
|
||||||
|
disagreement → refusal, never to be committed alone (see :mod:`generate.
|
||||||
|
derivation.pool`). This keeps the commit-path completeness guarantee
|
||||||
|
(ADR-0175's multi-step-incomplete defence) intact — the exemption widens only
|
||||||
|
what can *refuse*.
|
||||||
|
* ``None`` — fails a base clause, or an unused quantity is not
|
||||||
|
isolated-foreign (empty unit, or a unit shared with a used operand → real
|
||||||
|
signal the reading dropped).
|
||||||
|
"""
|
||||||
|
tokens = _tokens(problem_text)
|
||||||
|
if _base_reasons(derivation, tokens):
|
||||||
|
return None
|
||||||
|
unused = _unused_quantities(derivation, problem_text)
|
||||||
|
if not unused:
|
||||||
|
return "complete"
|
||||||
|
|
||||||
|
used_units = {derivation.start.unit, *(step.operand.unit for step in derivation.steps)}
|
||||||
|
units_by_token: dict[str, set[str]] = {}
|
||||||
|
for q in extract_quantities(problem_text):
|
||||||
|
units_by_token.setdefault(q.source_token, set()).add(q.unit)
|
||||||
|
|
||||||
|
for token in unused:
|
||||||
|
token_units = units_by_token.get(token, {""})
|
||||||
|
# isolated-foreign iff *every* occurrence has a non-empty unit not shared
|
||||||
|
# with any used operand. An empty unit, or a unit a used operand carries,
|
||||||
|
# disqualifies the exemption — that quantity is real signal, not a distractor.
|
||||||
|
if any((not unit) or (unit in used_units) for unit in token_units):
|
||||||
|
return None
|
||||||
|
return "exempt"
|
||||||
|
|
||||||
|
|
||||||
def select_self_verified(
|
def select_self_verified(
|
||||||
derivations: list[GroundedDerivation],
|
derivations: list[GroundedDerivation],
|
||||||
problem_text: str,
|
problem_text: str,
|
||||||
|
|
|
||||||
|
|
@ -24,8 +24,14 @@ from evals.gsm8k_math.confusers.v1.runner import (
|
||||||
# misfires (0005->796, 0007->996) now refuse — the 25-foot/20-inch divisor is
|
# misfires (0005->796, 0007->996) now refuse — the 25-foot/20-inch divisor is
|
||||||
# finally visible to the completeness/polarity gate. wrong 7->5. pair_tells
|
# finally visible to the completeness/polarity gate. wrong 7->5. pair_tells
|
||||||
# unchanged (the pseudo-accumulation cases carry no positive twin).
|
# unchanged (the pseudo-accumulation cases carry no positive twin).
|
||||||
_BASELINE_WRONG = 5
|
# 2026-05-29 (ADR-0182 cross-composer pooling): the engine now pools every
|
||||||
_BASELINE_PAIR_TELLS = 4
|
# composer's readings and refuses on disagreement. The distractor 0014
|
||||||
|
# (product 300 vs additive 25) and BOTH disguised-polarity cases (0001/0003:
|
||||||
|
# the spurious "for N coins" product disagrees with the accumulation reading)
|
||||||
|
# now refuse. wrong 5->2 (0016 needs anchor-skip; 0020 is temporal-scope).
|
||||||
|
# pair_tells 4->1 (0001/0003/0014 clear; 0020 remains).
|
||||||
|
_BASELINE_WRONG = 2
|
||||||
|
_BASELINE_PAIR_TELLS = 1
|
||||||
|
|
||||||
|
|
||||||
class TestSchema:
|
class TestSchema:
|
||||||
|
|
@ -87,5 +93,23 @@ class TestProbeBaseline:
|
||||||
"divisor is no longer reaching the completeness/polarity gate."
|
"divisor is no longer reaching the completeness/polarity gate."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_distractor_0014_refuses_via_pooling(self) -> None:
|
||||||
|
# ADR-0182: 0014's blunt product (300) and its competing additive reading
|
||||||
|
# (25) disagree under cross-composer pooling -> refuse. Fails loudly if
|
||||||
|
# pooling regresses (the unique product would commit 300 again). 0016 is
|
||||||
|
# NOT asserted here — its distractor sits in the anchor clause and needs
|
||||||
|
# anchor-skip, a separate step (see ADR-0182 §3b).
|
||||||
|
by_id = {r.case_id: r for r in run_probe()}
|
||||||
|
assert by_id["confuser-v1-0014"].verdict == "refused"
|
||||||
|
|
||||||
|
def test_disguised_polarity_does_not_misfire(self) -> None:
|
||||||
|
# ADR-0182 bonus: the spurious "buys X for N <unit>" product disagrees with
|
||||||
|
# the accumulation reading, so both disguised-polarity cases refuse instead
|
||||||
|
# of committing the gain-polarity sum.
|
||||||
|
results = run_probe()
|
||||||
|
disguised = [r for r in results if r.category == "disguised-polarity"]
|
||||||
|
assert disguised
|
||||||
|
assert all(r.verdict != "wrong" for r in disguised)
|
||||||
|
|
||||||
def test_deterministic(self) -> None:
|
def test_deterministic(self) -> None:
|
||||||
assert summarize(run_probe()) == summarize(run_probe())
|
assert summarize(run_probe()) == summarize(run_probe())
|
||||||
|
|
|
||||||
104
tests/test_adr_0182_pool.py
Normal file
104
tests/test_adr_0182_pool.py
Normal file
|
|
@ -0,0 +1,104 @@
|
||||||
|
"""ADR-0182 — cross-composer disagreement pooling (distractor refusal).
|
||||||
|
|
||||||
|
Two surfaces under test:
|
||||||
|
|
||||||
|
* :func:`generate.derivation.verify.classify_derivation` — the commit-eligibility
|
||||||
|
class (``complete`` / ``exempt`` / ``None``) and, critically, that the
|
||||||
|
isolated-foreign exemption is *narrow* (an empty-unit or same-unit unused
|
||||||
|
quantity is never exempt — it is real signal, not a distractor).
|
||||||
|
* :func:`generate.derivation.pool.resolve_pooled` — the pooled resolution: a clean
|
||||||
|
reading commits, a distractor problem's product-vs-additive disagreement refuses,
|
||||||
|
and (the wrong=0-critical property) an ``exempt``-only answer **never commits**.
|
||||||
|
|
||||||
|
Sealed lane: ``chat/`` does not import these; serving ``3/47/0`` cannot move.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from generate.derivation.accumulate import accumulation_candidates
|
||||||
|
from generate.derivation.model import GroundedDerivation, Quantity, Step
|
||||||
|
from generate.derivation.pool import resolve_pooled
|
||||||
|
from generate.derivation.verify import classify_derivation
|
||||||
|
|
||||||
|
_DISTRACTOR_0014 = (
|
||||||
|
"Kate has 20 pencils. She studies for 3 hours and then buys 5 more pencils. "
|
||||||
|
"How many pencils does Kate have?"
|
||||||
|
)
|
||||||
|
# A distractor with NO multiplicative cue: the only candidate is the exempt additive
|
||||||
|
# reading (no complete product exists). The commit-ineligibility rule must refuse it.
|
||||||
|
_EXEMPT_ONLY = (
|
||||||
|
"Kate has 20 pencils. She rests for 3 hours and buys 5 more pencils."
|
||||||
|
)
|
||||||
|
_CLEAN_ACCUMULATION = "Sam has 14 apples. He buys 9 more apples."
|
||||||
|
|
||||||
|
|
||||||
|
class TestClassifyDerivation:
|
||||||
|
def test_complete_reading_is_commit_eligible(self) -> None:
|
||||||
|
# uses every quantity -> complete
|
||||||
|
derivation = accumulation_candidates(_CLEAN_ACCUMULATION)[0]
|
||||||
|
assert classify_derivation(derivation, _CLEAN_ACCUMULATION) == "complete"
|
||||||
|
|
||||||
|
def test_isolated_foreign_unused_is_exempt(self) -> None:
|
||||||
|
# the additive reading of 0014 leaves "3 hours" unused; hours is foreign to
|
||||||
|
# the used unit (pencils) -> exempt (commit-ineligible).
|
||||||
|
derivation = GroundedDerivation(
|
||||||
|
start=Quantity(20.0, "pencils", "20"),
|
||||||
|
steps=(Step(op="add", operand=Quantity(5.0, "pencils", "5"), cue="more"),),
|
||||||
|
)
|
||||||
|
assert classify_derivation(derivation, _DISTRACTOR_0014) == "exempt"
|
||||||
|
|
||||||
|
def test_same_unit_unused_is_not_exempt(self) -> None:
|
||||||
|
# a quantity sharing the reading's unit is real signal, never a distractor:
|
||||||
|
# leaving it unused must NOT be exempted (it stays invalid -> None).
|
||||||
|
text = "Sam has 14 apples. He buys 9 more apples. He eats 2 apples."
|
||||||
|
derivation = GroundedDerivation(
|
||||||
|
start=Quantity(14.0, "apples", "14"),
|
||||||
|
steps=(Step(op="add", operand=Quantity(9.0, "apples", "9"), cue="more"),),
|
||||||
|
)
|
||||||
|
assert classify_derivation(derivation, text) is None
|
||||||
|
|
||||||
|
def test_empty_unit_unused_is_not_exempt(self) -> None:
|
||||||
|
# an unused quantity with an unknown (empty) unit cannot be shown foreign,
|
||||||
|
# so it is never exempt — completeness still rejects the reading.
|
||||||
|
text = "Sam has 14 apples. He buys 9 more apples. He had 2."
|
||||||
|
derivation = GroundedDerivation(
|
||||||
|
start=Quantity(14.0, "apples", "14"),
|
||||||
|
steps=(Step(op="add", operand=Quantity(9.0, "apples", "9"), cue="more"),),
|
||||||
|
)
|
||||||
|
assert classify_derivation(derivation, text) is None
|
||||||
|
|
||||||
|
def test_ungrounded_operand_is_invalid(self) -> None:
|
||||||
|
derivation = GroundedDerivation(
|
||||||
|
start=Quantity(14.0, "apples", "14"),
|
||||||
|
steps=(Step(op="add", operand=Quantity(999.0, "apples", "999"), cue="more"),),
|
||||||
|
)
|
||||||
|
assert classify_derivation(derivation, _CLEAN_ACCUMULATION) is None
|
||||||
|
|
||||||
|
|
||||||
|
class TestResolvePooled:
|
||||||
|
def test_clean_accumulation_commits(self) -> None:
|
||||||
|
resolution = resolve_pooled(_CLEAN_ACCUMULATION)
|
||||||
|
assert resolution is not None
|
||||||
|
assert resolution.answer == 23.0
|
||||||
|
|
||||||
|
def test_distractor_0014_refuses_via_disagreement(self) -> None:
|
||||||
|
# product 300 (complete) vs additive 25 (exempt) disagree -> refuse.
|
||||||
|
assert resolve_pooled(_DISTRACTOR_0014) is None
|
||||||
|
|
||||||
|
def test_exempt_only_never_commits(self) -> None:
|
||||||
|
# THE wrong=0-critical obligation. The only verifying reading here is the
|
||||||
|
# exempt additive one (no multiplicative cue -> no competing product), so a
|
||||||
|
# commit would be on an incomplete reading. The commit-ineligibility rule
|
||||||
|
# must refuse. Flipping `exempt` to commit-eligible makes this commit 25 and
|
||||||
|
# this test fails loudly.
|
||||||
|
assert accumulation_candidates(_EXEMPT_ONLY), "expected an exempt candidate"
|
||||||
|
assert classify_derivation(
|
||||||
|
accumulation_candidates(_EXEMPT_ONLY)[-1], _EXEMPT_ONLY
|
||||||
|
) == "exempt"
|
||||||
|
assert resolve_pooled(_EXEMPT_ONLY) is None
|
||||||
|
|
||||||
|
def test_deterministic(self) -> None:
|
||||||
|
assert resolve_pooled(_DISTRACTOR_0014) == resolve_pooled(_DISTRACTOR_0014)
|
||||||
|
a = resolve_pooled(_CLEAN_ACCUMULATION)
|
||||||
|
b = resolve_pooled(_CLEAN_ACCUMULATION)
|
||||||
|
assert a is not None and b is not None and a.answer == b.answer
|
||||||
Loading…
Reference in a new issue