chore(gsm8k): make practice-lane live counts monotonic

Replace exact 6/44/0 equality with monotonic capability contract:
wrong=0 hard, correct>=floor, refused<=ceiling, full refusal diagnosis,
elimination_records tied to wrongs. Seal test compares serving before/after
practice without freezing serving capability.
This commit is contained in:
Shay 2026-06-17 12:39:51 -07:00
parent 525793abcc
commit 73156617a1

View file

@ -1,15 +1,19 @@
"""ADR-0175 Phase 2 — sealed practice lane. """ADR-0175 Phase 2 — sealed practice lane.
A NEW lane (never the wrong=0-pinned train_sample serving runner) that runs the A NEW lane (never the wrong=0-pinned train_sample serving runner) that runs the
47 train cases in *practice* mode: scores correct/wrong/refused as practice train-sample cases in *practice* mode: scores correct/wrong/refused as practice
metrics, feeds per-class counts into the Phase 1 ledger, diagnoses every refusal metrics, feeds per-class counts into the Phase 1 ledger, diagnoses every refusal
(§8 skill/knowledge/ambiguity), and emits elimination records for wrongs. (§8 skill/knowledge/ambiguity), and emits elimination records for wrongs.
On the current pipeline the engine still refuses rather than guesses, so the Live capability contract (monotonic, not frozen):
practice ledger mirrors serving (6 correct / 0 wrong / 44 refused, of 50) and - wrong == 0 is hard;
zero eliminations fire live the attempt-generating search is Phase 3. Phase 2 - correct >= historical floor, refused <= historical ceiling;
proves the *regime*: the lane, the ledger wiring, the diagnosis, the elimination - total count fixed; every refusal diagnosed; eliminations track wrongs only.
schema, and the seal.
The historical floor/ceiling (6 correct / 44 refused / 0 wrong of 50) is the
Inc1-era rebaseline snapshot not a design target or ceiling on capability lift.
Phase 2 proves the *regime*: lane, ledger wiring, diagnosis, elimination schema,
and the seal. Attempt-generating search is Phase 3.
Invariants exercised by failing-under-violation tests: Invariants exercised by failing-under-violation tests:
- #1 seal -> TestSealInvariant - #1 seal -> TestSealInvariant
@ -23,6 +27,7 @@ import dataclasses
import pytest import pytest
from core.reliability_gate import conservative_floor from core.reliability_gate import conservative_floor
from core.learning_arena.report import PracticeReport
from evals.gsm8k_math.practice.v1.runner import ( from evals.gsm8k_math.practice.v1.runner import (
OPERATION_CLASSES, OPERATION_CLASSES,
REFUSAL_DIAGNOSES, REFUSAL_DIAGNOSES,
@ -33,6 +38,30 @@ from evals.gsm8k_math.practice.v1.runner import (
run_practice, run_practice,
) )
# Historical floor/ceiling from Inc1 rebaseline (2026-06-17). Monotonic contract:
# correct may rise, refused may fall, wrong must stay 0. Not a capability ceiling.
BASELINE_CORRECT = 6
BASELINE_WRONG = 0
BASELINE_REFUSED = 44
TRAIN_SAMPLE_COUNT = 50
def _assert_monotonic_capability_contract(rep: PracticeReport) -> None:
"""CORE practice-lane live contract: wrong=0 hard; counts monotonic vs floor."""
counts = rep.counts
assert counts["wrong"] == BASELINE_WRONG
assert counts["correct"] >= BASELINE_CORRECT
assert counts["refused"] <= BASELINE_REFUSED
assert (
counts["correct"] + counts["wrong"] + counts["refused"]
== TRAIN_SAMPLE_COUNT
)
assert len(rep.elimination_records) == counts["wrong"]
assert len(rep.refusal_diagnoses) == counts["refused"]
assert all(d in REFUSAL_DIAGNOSES for d in rep.refusal_diagnoses.values())
diagnosis_counts = rep.as_dict()["diagnosis_counts"]
assert sum(diagnosis_counts.values()) == counts["refused"]
# A stub CaseOutcome shape compatible with what the practice lane reads. # A stub CaseOutcome shape compatible with what the practice lane reads.
@dataclasses.dataclass(frozen=True) @dataclasses.dataclass(frozen=True)
@ -155,22 +184,22 @@ class TestRunPractice:
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Live lane over the real 47 — mirrors serving on the current pipeline # Live lane over train sample — monotonic capability contract
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
class TestLiveLane: class TestLiveLane:
def test_live_practice_mirrors_serving_today(self) -> None: def test_live_practice_meets_monotonic_capability_contract(self) -> None:
# With the refuse-preferring engine, practice == serving (6/44/0 after # Historical floor 6/44/0; future correct-count lift must not fail this test.
# the current serving lane lifted two additional cases). # Eliminations remain tied to wrong attempts only (zero at current baseline).
# Attempts/eliminations go live in Phase 3.
rep = build_report() rep = build_report()
assert rep.counts == {"correct": 6, "wrong": 0, "refused": 44} _assert_monotonic_capability_contract(rep)
assert len(rep.elimination_records) == 0 # no wrongs yet
def test_every_refusal_is_diagnosed(self) -> None: def test_every_refusal_is_diagnosed(self) -> None:
rep = build_report() rep = build_report()
assert len(rep.refusal_diagnoses) == 44 _assert_monotonic_capability_contract(rep)
assert all(d in REFUSAL_DIAGNOSES for d in rep.refusal_diagnoses.values()) # Redundant pins kept explicit: no refusal silently drops from diagnostics.
assert len(rep.refusal_diagnoses) == rep.counts["refused"]
assert sum(rep.as_dict()["diagnosis_counts"].values()) == rep.counts["refused"]
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@ -184,9 +213,14 @@ class TestSealInvariant:
_load_cases, _load_cases,
build_report as serving_build_report, build_report as serving_build_report,
) )
build_report() # run practice cases = _load_cases(_CASES_PATH)
serving = serving_build_report(_load_cases(_CASES_PATH)) serving_before = serving_build_report(cases)
assert serving["counts"] == {"correct": 6, "wrong": 0, "refused": 44} build_report() # run practice over same cases
serving_after = serving_build_report(cases)
assert serving_before["counts"] == serving_after["counts"]
assert serving_after["counts"]["wrong"] == BASELINE_WRONG
assert serving_after["counts"]["correct"] >= BASELINE_CORRECT
assert serving_after["counts"]["refused"] <= BASELINE_REFUSED
def test_no_serving_module_imports_the_practice_lane(self) -> None: def test_no_serving_module_imports_the_practice_lane(self) -> None:
import subprocess import subprocess