feat(adr-0182): cross-composer disagreement pooling — distractor 0014 + disguised-polarity refuse (confuser wrong 5->2) (#476)
Implements ADR-0182's first win on top of EX-6 (#473). The distractor-quantity confuser 0014 misfired (20x3x5=300) because a blunt product-of-all was the *unique* self-verifying reading: the completeness clause forces the distractor into it, and no rival reading existed to trigger the wrong=0 disagreement rule. No tight cue rule separates it from the legitimate cross-unit products (`for` licenses both the 0014 distractor AND the correct train-0021 product) -- that is the deferred cue-precision problem. So instead of a reactive patch, let the disagreement rule do the refusing. Mechanism (sealed lane; chat/ does not import these -> serving 3/47/0 frozen): - verify.classify_derivation: a derivation is `complete` (commit-eligible), `exempt` (verified but for an isolated-foreign unused quantity -> commit- INELIGIBLE), or None. Refactored self_verifies into _base_reasons + _unused_quantities so the two share logic and cannot drift (behavior identical; 385 derivation tests + smoke 67 green). - accumulate.accumulation_candidates: exposes the ungated readings, incl. a distractor-skip reading that drops an isolated-foreign quantity from a multi- quantity change clause (20+5=25). compose_accumulation is byte-identical (drop_isolated_foreign=False + the same gate). - search.multiplicative_candidates / multistep.candidate_chains: ungated candidates. - pool.resolve_pooled: pools every composer's readings; disagreement -> refuse; a single answer commits only if a `complete` candidate produced it (exempt-only -> refuse, so the commit-path completeness guarantee from ADR-0175 is untouched). - confuser runner: _engine_answer now delegates to resolve_pooled (the prior first-composer-wins order could not notice it held two incompatible readings). Result (the microscope): - confuser wrong 5 -> 2. distractor 0014 refuses (product 300 vs additive 25); BONUS: both disguised-polarity cases (0001/0003) refuse -- the spurious "buys X for N coins" product disagrees with the accumulation reading. Remaining wrong: 0016 (distractor in the anchor clause -> needs anchor-skip, separate step) and 0020 (temporal-scope). pair-tells 4 -> 1. - genuine positives still 7 solved, 0 wrong. - train_sample 3/47/0 and practice 3/47/0 byte-identical (they call compose_accumulation/search directly -- unchanged -- not the pool). - smoke 67, architectural invariants 40, lane-SHA freeze 8/8. Tests: - test_adr_0182_pool.py: classify (complete/exempt/None incl. narrow-exemption edges) + resolve_pooled, with the wrong=0 obligation test_exempt_only_never_commits (a distractor with no multiplicative cue must refuse, not commit 25 -- fails loudly if the exemption is made commit-eligible). - test_adr_0163_f2_confusers.py: baseline tightened wrong 5->2, pair_tells 4->1; new test_distractor_0014_refuses_via_pooling + test_disguised_polarity_does_not_misfire. Stacked on #473 (EX-6); merge #473 first. 0016 + the remaining wrongs are follow-ups.
This commit is contained in:
Shay
GitHub
parent
3f9edd06da
commit
0fbcce429b
8 changed files with 361 additions and 24 deletions
|
|
@ -22,21 +22,23 @@ from dataclasses import dataclass
|
|||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from generate.derivation.accumulate import compose_accumulation
|
||||
from generate.derivation.multistep import search_chain
|
||||
from generate.derivation.search import search_multiplicative
|
||||
from generate.derivation.pool import resolve_pooled
|
||||
|
||||
_CASES_PATH: Final[Path] = Path(__file__).resolve().parent / "cases.jsonl"
|
||||
_TOL: Final[float] = 1e-6
|
||||
|
||||
|
||||
def _engine_answer(problem_text: str) -> float | None:
|
||||
"""The sealed engine's attempt: first composer to resolve wins (fixed order)."""
|
||||
for composer in (compose_accumulation, search_multiplicative, search_chain):
|
||||
resolution = composer(problem_text)
|
||||
if resolution is not None:
|
||||
return resolution.answer
|
||||
return None
|
||||
"""The sealed engine's attempt (ADR-0182 cross-composer pooling).
|
||||
|
||||
Pools the ungated readings of every composer (accumulation, multiplicative,
|
||||
chain) and resolves them together: a single ``complete`` answer commits;
|
||||
disagreement among the pool — e.g. a distractor problem's blunt product vs its
|
||||
competing additive reading — refuses. Replaces the prior first-composer-wins
|
||||
order, which had no way to notice it held two incompatible readings.
|
||||
"""
|
||||
resolution = resolve_pooled(problem_text)
|
||||
return resolution.answer if resolution is not None else None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from generate.derivation.clauses import (
|
|||
clause_local_results,
|
||||
segment_clauses,
|
||||
)
|
||||
from generate.derivation.accumulate import compose_accumulation
|
||||
from generate.derivation.accumulate import accumulation_candidates, compose_accumulation
|
||||
from generate.derivation.compose import compose_sequential
|
||||
from generate.derivation.comparatives import (
|
||||
ComparativeScalar,
|
||||
|
|
@ -21,12 +21,18 @@ from generate.derivation.comparatives import (
|
|||
)
|
||||
from generate.derivation.extract import extract_quantities
|
||||
from generate.derivation.model import GroundedDerivation, Quantity, Step, VALID_OPS
|
||||
from generate.derivation.multistep import search_chain
|
||||
from generate.derivation.search import MULTIPLICATIVE_CUES, search_multiplicative
|
||||
from generate.derivation.multistep import candidate_chains, search_chain
|
||||
from generate.derivation.pool import pooled_candidates, resolve_pooled
|
||||
from generate.derivation.search import (
|
||||
MULTIPLICATIVE_CUES,
|
||||
multiplicative_candidates,
|
||||
search_multiplicative,
|
||||
)
|
||||
from generate.derivation.target import Target, extract_target
|
||||
from generate.derivation.verify import (
|
||||
Resolution,
|
||||
SelfVerification,
|
||||
classify_derivation,
|
||||
select_self_verified,
|
||||
self_verifies,
|
||||
)
|
||||
|
|
@ -42,6 +48,9 @@ __all__ = [
|
|||
"Step",
|
||||
"Target",
|
||||
"VALID_OPS",
|
||||
"accumulation_candidates",
|
||||
"candidate_chains",
|
||||
"classify_derivation",
|
||||
"clause_local_results",
|
||||
"compose_accumulation",
|
||||
"compose_sequential",
|
||||
|
|
@ -49,6 +58,9 @@ __all__ = [
|
|||
"extract_comparative_scalars",
|
||||
"extract_quantities",
|
||||
"extract_target",
|
||||
"multiplicative_candidates",
|
||||
"pooled_candidates",
|
||||
"resolve_pooled",
|
||||
"search_chain",
|
||||
"search_multiplicative",
|
||||
"segment_clauses",
|
||||
|
|
|
|||
|
|
@ -127,8 +127,20 @@ def _cue(clause: str, polarity: int) -> str:
|
|||
return "gives" # the only remaining licensed loss path (gives … to/away)
|
||||
|
||||
|
||||
def compose_accumulation(problem_text: str) -> Resolution | None:
|
||||
"""GB-3b.1 composer — single-referent gain/loss accumulation. Refuse-preferring."""
|
||||
def _build_accumulation(
|
||||
problem_text: str, *, drop_isolated_foreign: bool
|
||||
) -> GroundedDerivation | None:
|
||||
"""Construct the single-referent accumulation chain (ungated).
|
||||
|
||||
``drop_isolated_foreign`` (ADR-0182): when a change clause carries more than
|
||||
one quantity, drop those with a **non-empty unit foreign to the anchor's unit**
|
||||
(a candidate distractor — ``studies for 3 hours`` among ``pencils``) and proceed
|
||||
if exactly one same-unit/unitless change remains. With the flag off this is the
|
||||
strict GB-3b.1 reading (a multi-quantity change clause refuses), so
|
||||
:func:`compose_accumulation` is byte-identical to its pre-ADR-0182 behavior.
|
||||
The distractor-skip reading is **never committed alone** — it only ever enters
|
||||
the pool to force a disagreement refusal (see :mod:`generate.derivation.pool`).
|
||||
"""
|
||||
clauses = segment_clauses(problem_text)
|
||||
quantity_clauses = [c for c in clauses if extract_quantities(c)]
|
||||
if len(quantity_clauses) < 2:
|
||||
|
|
@ -145,7 +157,11 @@ def compose_accumulation(problem_text: str) -> Resolution | None:
|
|||
for clause in change_clauses:
|
||||
if not _same_referent(clause, anchor_subject):
|
||||
return None # new named actor -> referent hazard -> refuse
|
||||
change_quantities = extract_quantities(clause)
|
||||
change_quantities = list(extract_quantities(clause))
|
||||
if drop_isolated_foreign and len(change_quantities) > 1:
|
||||
change_quantities = [
|
||||
q for q in change_quantities if not (q.unit and q.unit != start.unit)
|
||||
]
|
||||
if len(change_quantities) != 1:
|
||||
return None # one change per clause (multi-change is GB-3b.2)
|
||||
polarity = _polarity(clause)
|
||||
|
|
@ -159,5 +175,31 @@ def compose_accumulation(problem_text: str) -> Resolution | None:
|
|||
|
||||
if not steps:
|
||||
return None
|
||||
derivation = GroundedDerivation(start=start, steps=tuple(steps))
|
||||
return GroundedDerivation(start=start, steps=tuple(steps))
|
||||
|
||||
|
||||
def compose_accumulation(problem_text: str) -> Resolution | None:
|
||||
"""GB-3b.1 composer — single-referent gain/loss accumulation. Refuse-preferring.
|
||||
|
||||
The strict (commit) reading: it gates the no-distractor-skip derivation through
|
||||
the unchanged self-verification gate. Behavior is byte-identical to pre-ADR-0182.
|
||||
"""
|
||||
derivation = _build_accumulation(problem_text, drop_isolated_foreign=False)
|
||||
if derivation is None:
|
||||
return None
|
||||
return select_self_verified([derivation], problem_text, target_units=())
|
||||
|
||||
|
||||
def accumulation_candidates(problem_text: str) -> tuple[GroundedDerivation, ...]:
|
||||
"""ADR-0182 — the ungated accumulation readings for cross-composer pooling.
|
||||
|
||||
The strict reading and (when it differs) the distractor-skip reading. Ungated:
|
||||
the pool classifies each (``complete`` commits, ``exempt`` refuses-only) and the
|
||||
disagreement rule does the wrong=0 work. Deterministic; de-dup is the pool's job.
|
||||
"""
|
||||
candidates: list[GroundedDerivation] = []
|
||||
for drop in (False, True):
|
||||
derivation = _build_accumulation(problem_text, drop_isolated_foreign=drop)
|
||||
if derivation is not None:
|
||||
candidates.append(derivation)
|
||||
return tuple(candidates)
|
||||
|
|
|
|||
88
generate/derivation/pool.py
Normal file
88
generate/derivation/pool.py
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
"""ADR-0182 — cross-composer disagreement pooling.
|
||||
|
||||
The distractor-quantity confusers (``confuser-v1-0014`` → 300, ``0016`` → 3840)
|
||||
misfire because a blunt product-of-all is the *unique* self-verifying reading: the
|
||||
completeness clause forces the distractor into it, and no rival reading exists to
|
||||
trigger the wrong=0 disagreement rule. The microscope showed no tight cue/gate rule
|
||||
separates these from the legitimate cross-unit products (``for`` licenses both the
|
||||
0014 distractor and the correct ``train-0021`` product) — that is the deferred
|
||||
cue-precision problem (ADR-0177).
|
||||
|
||||
This module sidesteps cue precision: it pools the **ungated candidate readings of
|
||||
every composer** (accumulation, in-clause multiplicative, target-guided chain) for
|
||||
one problem and resolves them together. A reading is classified
|
||||
(:func:`generate.derivation.verify.classify_derivation`) as ``complete``
|
||||
(commit-eligible) or ``exempt`` (commit-INELIGIBLE — verified but for an
|
||||
isolated-foreign *distractor* quantity it leaves unused). The rule:
|
||||
|
||||
* candidates disagree (more than one distinct answer across the pool) → **refuse**
|
||||
(the wrong=0 disagreement rule does the work);
|
||||
* a single distinct answer → commit **only if** a ``complete`` candidate produced it
|
||||
(an ``exempt``-only answer never commits — full completeness is still required to
|
||||
resolve, so ADR-0175's multi-step-incomplete defence is untouched).
|
||||
|
||||
For a distractor problem this gives the product (``complete``) a competing additive
|
||||
reading (``exempt``) to disagree with → refusal; for a genuine product (0021/0003)
|
||||
there is no rival reading, so it still commits. Deterministic; sealed (serving is
|
||||
never invoked here).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from generate.derivation.accumulate import accumulation_candidates
|
||||
from generate.derivation.model import GroundedDerivation
|
||||
from generate.derivation.multistep import candidate_chains
|
||||
from generate.derivation.search import multiplicative_candidates
|
||||
from generate.derivation.verify import Resolution, classify_derivation
|
||||
|
||||
|
||||
def pooled_candidates(problem_text: str) -> list[GroundedDerivation]:
|
||||
"""Every composer's ungated candidate readings, de-duplicated. Deterministic
|
||||
order (accumulation, then multiplicative, then chain)."""
|
||||
seen: set[tuple[object, ...]] = set()
|
||||
pooled: list[GroundedDerivation] = []
|
||||
for derivation in (
|
||||
*accumulation_candidates(problem_text),
|
||||
*multiplicative_candidates(problem_text),
|
||||
*candidate_chains(problem_text),
|
||||
):
|
||||
key = (
|
||||
round(derivation.answer, 9),
|
||||
derivation.start.source_token,
|
||||
tuple((step.op, step.operand.source_token) for step in derivation.steps),
|
||||
)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
pooled.append(derivation)
|
||||
return pooled
|
||||
|
||||
|
||||
def resolve_pooled(problem_text: str) -> Resolution | None:
|
||||
"""Resolve a problem by pooling every composer's readings. Refuse-preferring.
|
||||
|
||||
Refuses on no verifying candidate, on disagreement among the pool, or when the
|
||||
sole answer is produced only by commit-ineligible (``exempt``) readings.
|
||||
"""
|
||||
classified = [
|
||||
(kind, derivation)
|
||||
for derivation in pooled_candidates(problem_text)
|
||||
if (kind := classify_derivation(derivation, problem_text)) is not None
|
||||
]
|
||||
if not classified:
|
||||
return None
|
||||
|
||||
distinct = {round(derivation.answer, 9) for _, derivation in classified}
|
||||
if len(distinct) != 1:
|
||||
return None # disagreement -> refuse (wrong=0)
|
||||
|
||||
committable = [d for kind, d in classified if kind == "complete"]
|
||||
if not committable:
|
||||
return None # exempt-only -> refuse (a commit requires full completeness)
|
||||
|
||||
chosen = committable[0]
|
||||
return Resolution(
|
||||
answer=chosen.answer,
|
||||
answer_unit=chosen.answer_unit,
|
||||
derivation=chosen,
|
||||
)
|
||||
|
|
@ -65,6 +65,13 @@ def _sentence_candidates(problem_text: str) -> list[GroundedDerivation]:
|
|||
return candidates
|
||||
|
||||
|
||||
def multiplicative_candidates(problem_text: str) -> list[GroundedDerivation]:
|
||||
"""ADR-0182 — the ungated in-clause product candidates, for cross-composer
|
||||
pooling. Same construction :func:`search_multiplicative` gates, exposed so the
|
||||
pool can weigh products against the other composers' readings."""
|
||||
return _sentence_candidates(problem_text)
|
||||
|
||||
|
||||
def search_multiplicative(problem_text: str) -> Resolution | None:
|
||||
"""Attempt a grounded in-clause multiplicative product.
|
||||
|
||||
|
|
|
|||
|
|
@ -50,9 +50,10 @@ class Resolution:
|
|||
derivation: GroundedDerivation
|
||||
|
||||
|
||||
def self_verifies(derivation: GroundedDerivation, problem_text: str) -> SelfVerification:
|
||||
"""Decide whether ``derivation`` self-verifies against ``problem_text``."""
|
||||
tokens = _tokens(problem_text)
|
||||
def _base_reasons(derivation: GroundedDerivation, tokens: frozenset[str]) -> list[str]:
|
||||
"""The grounding ∧ cue ∧ unit ∧ divide-by-zero clauses (everything *but*
|
||||
completeness). Shared by :func:`self_verifies` and :func:`classify_derivation`
|
||||
so the two cannot drift."""
|
||||
reasons: list[str] = []
|
||||
|
||||
# 1. operand grounding — every TEXT operand value must be sourced from the
|
||||
|
|
@ -82,6 +83,24 @@ def self_verifies(derivation: GroundedDerivation, problem_text: str) -> SelfVeri
|
|||
if step.op == "divide" and step.operand.value == 0:
|
||||
reasons.append("division by zero")
|
||||
|
||||
return reasons
|
||||
|
||||
|
||||
def _unused_quantities(derivation: GroundedDerivation, problem_text: str) -> Counter[str]:
|
||||
"""Problem quantities (by source token) the derivation does not consume."""
|
||||
problem_quantities = Counter(q.source_token for q in extract_quantities(problem_text))
|
||||
used = Counter(
|
||||
[derivation.start.source_token]
|
||||
+ [step.operand.source_token for step in derivation.steps]
|
||||
)
|
||||
return problem_quantities - used
|
||||
|
||||
|
||||
def self_verifies(derivation: GroundedDerivation, problem_text: str) -> SelfVerification:
|
||||
"""Decide whether ``derivation`` self-verifies against ``problem_text``."""
|
||||
tokens = _tokens(problem_text)
|
||||
reasons = _base_reasons(derivation, tokens)
|
||||
|
||||
# 5. completeness — a trustworthy derivation must account for every quantity
|
||||
# the problem states. A derivation that ignores given numbers is an
|
||||
# incomplete reading (typically a correct *first step* of a multi-step
|
||||
|
|
@ -90,15 +109,54 @@ def self_verifies(derivation: GroundedDerivation, problem_text: str) -> SelfVeri
|
|||
# microscope identified (ADR-0175 self-verification strengthening): it
|
||||
# catches the multi-step-incomplete attempts the cue/grounding clauses
|
||||
# cannot, because their operands ARE grounded.
|
||||
problem_quantities = Counter(q.source_token for q in extract_quantities(problem_text))
|
||||
used = Counter([derivation.start.source_token] + [step.operand.source_token for step in derivation.steps])
|
||||
unused = problem_quantities - used
|
||||
unused = _unused_quantities(derivation, problem_text)
|
||||
if unused:
|
||||
reasons.append(f"incomplete: unused problem quantities {sorted(unused.keys())}")
|
||||
|
||||
return SelfVerification(verified=not reasons, reasons=tuple(reasons))
|
||||
|
||||
|
||||
def classify_derivation(derivation: GroundedDerivation, problem_text: str) -> str | None:
|
||||
"""ADR-0182 — the commit-eligibility class of a derivation, for pooling.
|
||||
|
||||
Returns:
|
||||
|
||||
* ``"complete"`` — passes every clause *including* full completeness;
|
||||
**commit-eligible** (may resolve as an answer).
|
||||
* ``"exempt"`` — passes every base clause and the only unused quantities are
|
||||
**isolated-foreign** (unit non-empty ∧ equal to no *used* operand's unit, i.e.
|
||||
a candidate distractor standing alone in a dimension the reading never
|
||||
touches); **commit-INELIGIBLE**. It exists only to enter the pool and force a
|
||||
disagreement → refusal, never to be committed alone (see :mod:`generate.
|
||||
derivation.pool`). This keeps the commit-path completeness guarantee
|
||||
(ADR-0175's multi-step-incomplete defence) intact — the exemption widens only
|
||||
what can *refuse*.
|
||||
* ``None`` — fails a base clause, or an unused quantity is not
|
||||
isolated-foreign (empty unit, or a unit shared with a used operand → real
|
||||
signal the reading dropped).
|
||||
"""
|
||||
tokens = _tokens(problem_text)
|
||||
if _base_reasons(derivation, tokens):
|
||||
return None
|
||||
unused = _unused_quantities(derivation, problem_text)
|
||||
if not unused:
|
||||
return "complete"
|
||||
|
||||
used_units = {derivation.start.unit, *(step.operand.unit for step in derivation.steps)}
|
||||
units_by_token: dict[str, set[str]] = {}
|
||||
for q in extract_quantities(problem_text):
|
||||
units_by_token.setdefault(q.source_token, set()).add(q.unit)
|
||||
|
||||
for token in unused:
|
||||
token_units = units_by_token.get(token, {""})
|
||||
# isolated-foreign iff *every* occurrence has a non-empty unit not shared
|
||||
# with any used operand. An empty unit, or a unit a used operand carries,
|
||||
# disqualifies the exemption — that quantity is real signal, not a distractor.
|
||||
if any((not unit) or (unit in used_units) for unit in token_units):
|
||||
return None
|
||||
return "exempt"
|
||||
|
||||
|
||||
def select_self_verified(
|
||||
derivations: list[GroundedDerivation],
|
||||
problem_text: str,
|
||||
|
|
|
|||
|
|
@ -24,8 +24,14 @@ from evals.gsm8k_math.confusers.v1.runner import (
|
|||
# misfires (0005->796, 0007->996) now refuse — the 25-foot/20-inch divisor is
|
||||
# finally visible to the completeness/polarity gate. wrong 7->5. pair_tells
|
||||
# unchanged (the pseudo-accumulation cases carry no positive twin).
|
||||
_BASELINE_WRONG = 5
|
||||
_BASELINE_PAIR_TELLS = 4
|
||||
# 2026-05-29 (ADR-0182 cross-composer pooling): the engine now pools every
|
||||
# composer's readings and refuses on disagreement. The distractor 0014
|
||||
# (product 300 vs additive 25) and BOTH disguised-polarity cases (0001/0003:
|
||||
# the spurious "for N coins" product disagrees with the accumulation reading)
|
||||
# now refuse. wrong 5->2 (0016 needs anchor-skip; 0020 is temporal-scope).
|
||||
# pair_tells 4->1 (0001/0003/0014 clear; 0020 remains).
|
||||
_BASELINE_WRONG = 2
|
||||
_BASELINE_PAIR_TELLS = 1
|
||||
|
||||
|
||||
class TestSchema:
|
||||
|
|
@ -87,5 +93,23 @@ class TestProbeBaseline:
|
|||
"divisor is no longer reaching the completeness/polarity gate."
|
||||
)
|
||||
|
||||
def test_distractor_0014_refuses_via_pooling(self) -> None:
|
||||
# ADR-0182: 0014's blunt product (300) and its competing additive reading
|
||||
# (25) disagree under cross-composer pooling -> refuse. Fails loudly if
|
||||
# pooling regresses (the unique product would commit 300 again). 0016 is
|
||||
# NOT asserted here — its distractor sits in the anchor clause and needs
|
||||
# anchor-skip, a separate step (see ADR-0182 §3b).
|
||||
by_id = {r.case_id: r for r in run_probe()}
|
||||
assert by_id["confuser-v1-0014"].verdict == "refused"
|
||||
|
||||
def test_disguised_polarity_does_not_misfire(self) -> None:
|
||||
# ADR-0182 bonus: the spurious "buys X for N <unit>" product disagrees with
|
||||
# the accumulation reading, so both disguised-polarity cases refuse instead
|
||||
# of committing the gain-polarity sum.
|
||||
results = run_probe()
|
||||
disguised = [r for r in results if r.category == "disguised-polarity"]
|
||||
assert disguised
|
||||
assert all(r.verdict != "wrong" for r in disguised)
|
||||
|
||||
def test_deterministic(self) -> None:
|
||||
assert summarize(run_probe()) == summarize(run_probe())
|
||||
|
|
|
|||
104
tests/test_adr_0182_pool.py
Normal file
104
tests/test_adr_0182_pool.py
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
"""ADR-0182 — cross-composer disagreement pooling (distractor refusal).
|
||||
|
||||
Two surfaces under test:
|
||||
|
||||
* :func:`generate.derivation.verify.classify_derivation` — the commit-eligibility
|
||||
class (``complete`` / ``exempt`` / ``None``) and, critically, that the
|
||||
isolated-foreign exemption is *narrow* (an empty-unit or same-unit unused
|
||||
quantity is never exempt — it is real signal, not a distractor).
|
||||
* :func:`generate.derivation.pool.resolve_pooled` — the pooled resolution: a clean
|
||||
reading commits, a distractor problem's product-vs-additive disagreement refuses,
|
||||
and (the wrong=0-critical property) an ``exempt``-only answer **never commits**.
|
||||
|
||||
Sealed lane: ``chat/`` does not import these; serving ``3/47/0`` cannot move.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from generate.derivation.accumulate import accumulation_candidates
|
||||
from generate.derivation.model import GroundedDerivation, Quantity, Step
|
||||
from generate.derivation.pool import resolve_pooled
|
||||
from generate.derivation.verify import classify_derivation
|
||||
|
||||
_DISTRACTOR_0014 = (
|
||||
"Kate has 20 pencils. She studies for 3 hours and then buys 5 more pencils. "
|
||||
"How many pencils does Kate have?"
|
||||
)
|
||||
# A distractor with NO multiplicative cue: the only candidate is the exempt additive
|
||||
# reading (no complete product exists). The commit-ineligibility rule must refuse it.
|
||||
_EXEMPT_ONLY = (
|
||||
"Kate has 20 pencils. She rests for 3 hours and buys 5 more pencils."
|
||||
)
|
||||
_CLEAN_ACCUMULATION = "Sam has 14 apples. He buys 9 more apples."
|
||||
|
||||
|
||||
class TestClassifyDerivation:
|
||||
def test_complete_reading_is_commit_eligible(self) -> None:
|
||||
# uses every quantity -> complete
|
||||
derivation = accumulation_candidates(_CLEAN_ACCUMULATION)[0]
|
||||
assert classify_derivation(derivation, _CLEAN_ACCUMULATION) == "complete"
|
||||
|
||||
def test_isolated_foreign_unused_is_exempt(self) -> None:
|
||||
# the additive reading of 0014 leaves "3 hours" unused; hours is foreign to
|
||||
# the used unit (pencils) -> exempt (commit-ineligible).
|
||||
derivation = GroundedDerivation(
|
||||
start=Quantity(20.0, "pencils", "20"),
|
||||
steps=(Step(op="add", operand=Quantity(5.0, "pencils", "5"), cue="more"),),
|
||||
)
|
||||
assert classify_derivation(derivation, _DISTRACTOR_0014) == "exempt"
|
||||
|
||||
def test_same_unit_unused_is_not_exempt(self) -> None:
|
||||
# a quantity sharing the reading's unit is real signal, never a distractor:
|
||||
# leaving it unused must NOT be exempted (it stays invalid -> None).
|
||||
text = "Sam has 14 apples. He buys 9 more apples. He eats 2 apples."
|
||||
derivation = GroundedDerivation(
|
||||
start=Quantity(14.0, "apples", "14"),
|
||||
steps=(Step(op="add", operand=Quantity(9.0, "apples", "9"), cue="more"),),
|
||||
)
|
||||
assert classify_derivation(derivation, text) is None
|
||||
|
||||
def test_empty_unit_unused_is_not_exempt(self) -> None:
|
||||
# an unused quantity with an unknown (empty) unit cannot be shown foreign,
|
||||
# so it is never exempt — completeness still rejects the reading.
|
||||
text = "Sam has 14 apples. He buys 9 more apples. He had 2."
|
||||
derivation = GroundedDerivation(
|
||||
start=Quantity(14.0, "apples", "14"),
|
||||
steps=(Step(op="add", operand=Quantity(9.0, "apples", "9"), cue="more"),),
|
||||
)
|
||||
assert classify_derivation(derivation, text) is None
|
||||
|
||||
def test_ungrounded_operand_is_invalid(self) -> None:
|
||||
derivation = GroundedDerivation(
|
||||
start=Quantity(14.0, "apples", "14"),
|
||||
steps=(Step(op="add", operand=Quantity(999.0, "apples", "999"), cue="more"),),
|
||||
)
|
||||
assert classify_derivation(derivation, _CLEAN_ACCUMULATION) is None
|
||||
|
||||
|
||||
class TestResolvePooled:
|
||||
def test_clean_accumulation_commits(self) -> None:
|
||||
resolution = resolve_pooled(_CLEAN_ACCUMULATION)
|
||||
assert resolution is not None
|
||||
assert resolution.answer == 23.0
|
||||
|
||||
def test_distractor_0014_refuses_via_disagreement(self) -> None:
|
||||
# product 300 (complete) vs additive 25 (exempt) disagree -> refuse.
|
||||
assert resolve_pooled(_DISTRACTOR_0014) is None
|
||||
|
||||
def test_exempt_only_never_commits(self) -> None:
|
||||
# THE wrong=0-critical obligation. The only verifying reading here is the
|
||||
# exempt additive one (no multiplicative cue -> no competing product), so a
|
||||
# commit would be on an incomplete reading. The commit-ineligibility rule
|
||||
# must refuse. Flipping `exempt` to commit-eligible makes this commit 25 and
|
||||
# this test fails loudly.
|
||||
assert accumulation_candidates(_EXEMPT_ONLY), "expected an exempt candidate"
|
||||
assert classify_derivation(
|
||||
accumulation_candidates(_EXEMPT_ONLY)[-1], _EXEMPT_ONLY
|
||||
) == "exempt"
|
||||
assert resolve_pooled(_EXEMPT_ONLY) is None
|
||||
|
||||
def test_deterministic(self) -> None:
|
||||
assert resolve_pooled(_DISTRACTOR_0014) == resolve_pooled(_DISTRACTOR_0014)
|
||||
a = resolve_pooled(_CLEAN_ACCUMULATION)
|
||||
b = resolve_pooled(_CLEAN_ACCUMULATION)
|
||||
assert a is not None and b is not None and a.answer == b.answer
|
||||
Loading…
Reference in a new issue