chore(gsm8k): make practice-lane live counts monotonic (#802)
Replace exact 6/44/0 equality with monotonic capability contract: wrong=0 hard, correct>=floor, refused<=ceiling, full refusal diagnosis, elimination_records tied to wrongs. Seal test compares serving before/after practice without freezing serving capability.
This commit is contained in:
parent
525793abcc
commit
17f474a8d7
1 changed files with 52 additions and 18 deletions
|
|
@ -1,15 +1,19 @@
|
|||
"""ADR-0175 Phase 2 — sealed practice lane.
|
||||
|
||||
A NEW lane (never the wrong=0-pinned train_sample serving runner) that runs the
|
||||
47 train cases in *practice* mode: scores correct/wrong/refused as practice
|
||||
train-sample cases in *practice* mode: scores correct/wrong/refused as practice
|
||||
metrics, feeds per-class counts into the Phase 1 ledger, diagnoses every refusal
|
||||
(§8 skill/knowledge/ambiguity), and emits elimination records for wrongs.
|
||||
|
||||
On the current pipeline the engine still refuses rather than guesses, so the
|
||||
practice ledger mirrors serving (6 correct / 0 wrong / 44 refused, of 50) and
|
||||
zero eliminations fire live — the attempt-generating search is Phase 3. Phase 2
|
||||
proves the *regime*: the lane, the ledger wiring, the diagnosis, the elimination
|
||||
schema, and the seal.
|
||||
Live capability contract (monotonic, not frozen):
|
||||
- wrong == 0 is hard;
|
||||
- correct >= historical floor, refused <= historical ceiling;
|
||||
- total count fixed; every refusal diagnosed; eliminations track wrongs only.
|
||||
|
||||
The historical floor/ceiling (6 correct / 44 refused / 0 wrong of 50) is the
|
||||
Inc1-era rebaseline snapshot — not a design target or ceiling on capability lift.
|
||||
Phase 2 proves the *regime*: lane, ledger wiring, diagnosis, elimination schema,
|
||||
and the seal. Attempt-generating search is Phase 3.
|
||||
|
||||
Invariants exercised by failing-under-violation tests:
|
||||
- #1 seal -> TestSealInvariant
|
||||
|
|
@ -23,6 +27,7 @@ import dataclasses
|
|||
import pytest
|
||||
|
||||
from core.reliability_gate import conservative_floor
|
||||
from core.learning_arena.report import PracticeReport
|
||||
from evals.gsm8k_math.practice.v1.runner import (
|
||||
OPERATION_CLASSES,
|
||||
REFUSAL_DIAGNOSES,
|
||||
|
|
@ -33,6 +38,30 @@ from evals.gsm8k_math.practice.v1.runner import (
|
|||
run_practice,
|
||||
)
|
||||
|
||||
# Historical floor/ceiling from Inc1 rebaseline (2026-06-17). Monotonic contract:
|
||||
# correct may rise, refused may fall, wrong must stay 0. Not a capability ceiling.
|
||||
BASELINE_CORRECT = 6
|
||||
BASELINE_WRONG = 0
|
||||
BASELINE_REFUSED = 44
|
||||
TRAIN_SAMPLE_COUNT = 50
|
||||
|
||||
|
||||
def _assert_monotonic_capability_contract(rep: PracticeReport) -> None:
|
||||
"""CORE practice-lane live contract: wrong=0 hard; counts monotonic vs floor."""
|
||||
counts = rep.counts
|
||||
assert counts["wrong"] == BASELINE_WRONG
|
||||
assert counts["correct"] >= BASELINE_CORRECT
|
||||
assert counts["refused"] <= BASELINE_REFUSED
|
||||
assert (
|
||||
counts["correct"] + counts["wrong"] + counts["refused"]
|
||||
== TRAIN_SAMPLE_COUNT
|
||||
)
|
||||
assert len(rep.elimination_records) == counts["wrong"]
|
||||
assert len(rep.refusal_diagnoses) == counts["refused"]
|
||||
assert all(d in REFUSAL_DIAGNOSES for d in rep.refusal_diagnoses.values())
|
||||
diagnosis_counts = rep.as_dict()["diagnosis_counts"]
|
||||
assert sum(diagnosis_counts.values()) == counts["refused"]
|
||||
|
||||
|
||||
# A stub CaseOutcome shape compatible with what the practice lane reads.
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
|
|
@ -155,22 +184,22 @@ class TestRunPractice:
|
|||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Live lane over the real 47 — mirrors serving on the current pipeline
|
||||
# Live lane over train sample — monotonic capability contract
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestLiveLane:
|
||||
def test_live_practice_mirrors_serving_today(self) -> None:
|
||||
# With the refuse-preferring engine, practice == serving (6/44/0 after
|
||||
# the current serving lane lifted two additional cases).
|
||||
# Attempts/eliminations go live in Phase 3.
|
||||
def test_live_practice_meets_monotonic_capability_contract(self) -> None:
|
||||
# Historical floor 6/44/0; future correct-count lift must not fail this test.
|
||||
# Eliminations remain tied to wrong attempts only (zero at current baseline).
|
||||
rep = build_report()
|
||||
assert rep.counts == {"correct": 6, "wrong": 0, "refused": 44}
|
||||
assert len(rep.elimination_records) == 0 # no wrongs yet
|
||||
_assert_monotonic_capability_contract(rep)
|
||||
|
||||
def test_every_refusal_is_diagnosed(self) -> None:
|
||||
rep = build_report()
|
||||
assert len(rep.refusal_diagnoses) == 44
|
||||
assert all(d in REFUSAL_DIAGNOSES for d in rep.refusal_diagnoses.values())
|
||||
_assert_monotonic_capability_contract(rep)
|
||||
# Redundant pins kept explicit: no refusal silently drops from diagnostics.
|
||||
assert len(rep.refusal_diagnoses) == rep.counts["refused"]
|
||||
assert sum(rep.as_dict()["diagnosis_counts"].values()) == rep.counts["refused"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -184,9 +213,14 @@ class TestSealInvariant:
|
|||
_load_cases,
|
||||
build_report as serving_build_report,
|
||||
)
|
||||
build_report() # run practice
|
||||
serving = serving_build_report(_load_cases(_CASES_PATH))
|
||||
assert serving["counts"] == {"correct": 6, "wrong": 0, "refused": 44}
|
||||
cases = _load_cases(_CASES_PATH)
|
||||
serving_before = serving_build_report(cases)
|
||||
build_report() # run practice over same cases
|
||||
serving_after = serving_build_report(cases)
|
||||
assert serving_before["counts"] == serving_after["counts"]
|
||||
assert serving_after["counts"]["wrong"] == BASELINE_WRONG
|
||||
assert serving_after["counts"]["correct"] >= BASELINE_CORRECT
|
||||
assert serving_after["counts"]["refused"] <= BASELINE_REFUSED
|
||||
|
||||
def test_no_serving_module_imports_the_practice_lane(self) -> None:
|
||||
import subprocess
|
||||
|
|
|
|||
Loading…
Reference in a new issue