feat(adr-0175): strengthen self-verification with a completeness clause
Self-verification strengthening (microscope-driven). The Phase 3b measurement
showed self-verification was necessary-but-not-sufficient: 9/13 self-verified
attempts were wrong. Inspecting them deterministically revealed most were
correct FIRST STEPS of multi-step problems that ignored numbers stated elsewhere.
Adds clause 5 to self_verifies: a derivation must account for every quantity the
problem states (problem quantities subset of used). Refuse-preferring: unused
quantities -> not self-verified. This catches the multi-step-incomplete attempts
the grounding/cue/unit clauses cannot (their operands ARE grounded).
Practice measurement: wrongs 9 -> 2 (4 correct / 2 wrong / 44 refused). The 2
survivors (0015, 0025) are COMPLETE but wrong due to missed WORD-quantities
('twice', 'her friends') -> the microscope points the next change at extraction.
Updated the disagreement test to use two complete derivations; added an
incomplete-refusal test. 32 tests pass; smoke green; serving untouched (sealed).
This commit is contained in:
parent
dd6064065f
commit
6212943c5a
2 changed files with 38 additions and 6 deletions
|
|
@ -30,6 +30,7 @@ from typing import Final
|
|||
# Canonical grounding primitives — reused so this gate stays identical to the
|
||||
# round-trip filter's notion of "appears in the problem text".
|
||||
from generate.math_roundtrip import _token_in, _tokens, _value_grounds
|
||||
from generate.derivation.extract import extract_quantities
|
||||
from generate.derivation.model import GroundedDerivation
|
||||
|
||||
_SAME_UNIT_REQUIRED: Final[frozenset[str]] = frozenset({"add", "subtract"})
|
||||
|
|
@ -77,6 +78,21 @@ def self_verifies(derivation: GroundedDerivation, problem_text: str) -> SelfVeri
|
|||
if step.op == "divide" and step.operand.value == 0:
|
||||
reasons.append("division by zero")
|
||||
|
||||
# 5. completeness — a trustworthy derivation must account for every quantity
|
||||
# the problem states. A derivation that ignores given numbers is an
|
||||
# incomplete reading (typically a correct *first step* of a multi-step
|
||||
# problem, mistaken for the whole answer). Refuse-preferring: unused
|
||||
# quantities -> not self-verified. This is the clause the practice-lane
|
||||
# microscope identified (ADR-0175 self-verification strengthening): it
|
||||
# catches the multi-step-incomplete attempts the cue/grounding clauses
|
||||
# cannot, because their operands ARE grounded.
|
||||
problem_quantities = {q.source_token for q in extract_quantities(problem_text)}
|
||||
used = {derivation.start.source_token}
|
||||
used.update(step.operand.source_token for step in derivation.steps)
|
||||
unused = problem_quantities - used
|
||||
if unused:
|
||||
reasons.append(f"incomplete: unused problem quantities {sorted(unused)}")
|
||||
|
||||
return SelfVerification(verified=not reasons, reasons=tuple(reasons))
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -162,15 +162,31 @@ class TestSelectUnique:
|
|||
assert select_self_verified([spurious], "Martha has 20 apples and 5 friends.") is None
|
||||
|
||||
def test_disagreeing_self_verified_refuses(self) -> None:
|
||||
# two grounded derivations that disagree on the answer -> refuse (wrong=0)
|
||||
# two COMPLETE grounded derivations that disagree -> refuse (wrong=0).
|
||||
# Both use all problem quantities {5, 3}; different ops -> different answers.
|
||||
text = "She has 5 apples and 3 apples."
|
||||
d_add = GroundedDerivation(
|
||||
start=_q(5, "apples", "5"),
|
||||
steps=(Step(op="add", operand=_q(3, "apples", "3"), cue="and"),),
|
||||
) # 8
|
||||
d_mul = GroundedDerivation(
|
||||
start=_q(5, "apples", "5"),
|
||||
steps=(Step(op="multiply", operand=_q(3, "apples", "3"), cue="and"),),
|
||||
) # 15
|
||||
assert d_add.answer != d_mul.answer
|
||||
assert select_self_verified([d_add, d_mul], text) is None
|
||||
|
||||
def test_incomplete_derivation_refuses(self) -> None:
|
||||
# completeness clause: a derivation that ignores a stated quantity is an
|
||||
# incomplete reading -> not self-verified (catches multi-step first-steps).
|
||||
text = "He bench presses 15 pounds for 10 reps and does 3 sets."
|
||||
d1 = _mult_0021() # 450
|
||||
d2 = GroundedDerivation( # 15 x 10 = 150 (grounded but different answer)
|
||||
partial = GroundedDerivation( # uses 15,10 but ignores the stated 3
|
||||
start=_q(15, "pounds", "15"),
|
||||
steps=(Step(op="multiply", operand=_q(10, "reps", "10"), cue="reps"),),
|
||||
steps=(Step(op="multiply", operand=_q(10, "reps", "10"), cue="for"),),
|
||||
)
|
||||
assert d1.answer != d2.answer
|
||||
assert select_self_verified([d1, d2], text) is None
|
||||
sv = self_verifies(partial, text)
|
||||
assert sv.verified is False
|
||||
assert any("incomplete" in r for r in sv.reasons)
|
||||
|
||||
def test_agreeing_self_verified_resolves(self) -> None:
|
||||
# two self-verifying derivations that AGREE -> resolve (convergent evidence)
|
||||
|
|
|
|||
Loading…
Reference in a new issue