From 6212943c5abb6440855fc171dbbb929b70f2bb3c Mon Sep 17 00:00:00 2001 From: Shay Date: Thu, 28 May 2026 15:53:11 -0700 Subject: [PATCH] feat(adr-0175): strengthen self-verification with a completeness clause Self-verification strengthening (microscope-driven). The Phase 3b measurement showed self-verification was necessary-but-not-sufficient: 9/13 self-verified attempts were wrong. Inspecting them deterministically revealed most were correct FIRST STEPS of multi-step problems that ignored numbers stated elsewhere. Adds clause 5 to self_verifies: a derivation must account for every quantity the problem states (problem quantities subset of used). Refuse-preferring: unused quantities -> not self-verified. This catches the multi-step-incomplete attempts the grounding/cue/unit clauses cannot (their operands ARE grounded). Practice measurement: wrongs 9 -> 2 (4 correct / 2 wrong / 44 refused). The 2 survivors (0015, 0025) are COMPLETE but wrong due to missed WORD-quantities ('twice', 'her friends') -> the microscope points the next change at extraction. Updated the disagreement test to use two complete derivations; added an incomplete-refusal test. 32 tests pass; smoke green; serving untouched (sealed). --- generate/derivation/verify.py | 16 +++++++++++ .../test_adr_0175_phase3a_selfverify_gate.py | 28 +++++++++++++++---- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/generate/derivation/verify.py b/generate/derivation/verify.py index 420d25b6..fa579dcd 100644 --- a/generate/derivation/verify.py +++ b/generate/derivation/verify.py @@ -30,6 +30,7 @@ from typing import Final # Canonical grounding primitives — reused so this gate stays identical to the # round-trip filter's notion of "appears in the problem text". from generate.math_roundtrip import _token_in, _tokens, _value_grounds +from generate.derivation.extract import extract_quantities from generate.derivation.model import GroundedDerivation _SAME_UNIT_REQUIRED: Final[frozenset[str]] = frozenset({"add", "subtract"}) @@ -77,6 +78,21 @@ def self_verifies(derivation: GroundedDerivation, problem_text: str) -> SelfVeri if step.op == "divide" and step.operand.value == 0: reasons.append("division by zero") + # 5. completeness — a trustworthy derivation must account for every quantity + # the problem states. A derivation that ignores given numbers is an + # incomplete reading (typically a correct *first step* of a multi-step + # problem, mistaken for the whole answer). Refuse-preferring: unused + # quantities -> not self-verified. This is the clause the practice-lane + # microscope identified (ADR-0175 self-verification strengthening): it + # catches the multi-step-incomplete attempts the cue/grounding clauses + # cannot, because their operands ARE grounded. + problem_quantities = {q.source_token for q in extract_quantities(problem_text)} + used = {derivation.start.source_token} + used.update(step.operand.source_token for step in derivation.steps) + unused = problem_quantities - used + if unused: + reasons.append(f"incomplete: unused problem quantities {sorted(unused)}") + return SelfVerification(verified=not reasons, reasons=tuple(reasons)) diff --git a/tests/test_adr_0175_phase3a_selfverify_gate.py b/tests/test_adr_0175_phase3a_selfverify_gate.py index 440c2345..e8dc37c4 100644 --- a/tests/test_adr_0175_phase3a_selfverify_gate.py +++ b/tests/test_adr_0175_phase3a_selfverify_gate.py @@ -162,15 +162,31 @@ class TestSelectUnique: assert select_self_verified([spurious], "Martha has 20 apples and 5 friends.") is None def test_disagreeing_self_verified_refuses(self) -> None: - # two grounded derivations that disagree on the answer -> refuse (wrong=0) + # two COMPLETE grounded derivations that disagree -> refuse (wrong=0). + # Both use all problem quantities {5, 3}; different ops -> different answers. + text = "She has 5 apples and 3 apples." + d_add = GroundedDerivation( + start=_q(5, "apples", "5"), + steps=(Step(op="add", operand=_q(3, "apples", "3"), cue="and"),), + ) # 8 + d_mul = GroundedDerivation( + start=_q(5, "apples", "5"), + steps=(Step(op="multiply", operand=_q(3, "apples", "3"), cue="and"),), + ) # 15 + assert d_add.answer != d_mul.answer + assert select_self_verified([d_add, d_mul], text) is None + + def test_incomplete_derivation_refuses(self) -> None: + # completeness clause: a derivation that ignores a stated quantity is an + # incomplete reading -> not self-verified (catches multi-step first-steps). text = "He bench presses 15 pounds for 10 reps and does 3 sets." - d1 = _mult_0021() # 450 - d2 = GroundedDerivation( # 15 x 10 = 150 (grounded but different answer) + partial = GroundedDerivation( # uses 15,10 but ignores the stated 3 start=_q(15, "pounds", "15"), - steps=(Step(op="multiply", operand=_q(10, "reps", "10"), cue="reps"),), + steps=(Step(op="multiply", operand=_q(10, "reps", "10"), cue="for"),), ) - assert d1.answer != d2.answer - assert select_self_verified([d1, d2], text) is None + sv = self_verifies(partial, text) + assert sv.verified is False + assert any("incomplete" in r for r in sv.reasons) def test_agreeing_self_verified_resolves(self) -> None: # two self-verifying derivations that AGREE -> resolve (convergent evidence)