From ed2d04c99e90f96d04689d0d94c825fc9a48d5b1 Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 17 Jun 2026 13:30:18 -0700 Subject: [PATCH] chore(gsm8k): split score preservation from truth preservation in tests (#804) * chore(gsm8k): split score preservation from truth preservation in tests Exact 6/44/0 pins remain only on the committed report.json historical fixture. Live serving and practice seal tests use a shared monotonic contract (wrong==0 hard; correct>=6; refused<=44) so capability lift does not fight the suite. * chore(gsm8k): polish monotonic baseline test hygiene --- tests/gsm8k_train_sample_baseline.py | 31 +++++++++++++++ tests/test_adr_0174_phase3_lookback.py | 24 ++++------- tests/test_adr_0175_phase2_practice_lane.py | 17 ++++---- tests/test_gsm8k_frontier_report.py | 44 +++++++++++++++------ 4 files changed, 77 insertions(+), 39 deletions(-) create mode 100644 tests/gsm8k_train_sample_baseline.py diff --git a/tests/gsm8k_train_sample_baseline.py b/tests/gsm8k_train_sample_baseline.py new file mode 100644 index 00000000..aca3113b --- /dev/null +++ b/tests/gsm8k_train_sample_baseline.py @@ -0,0 +1,31 @@ +"""Train-sample serving capability baseline (Inc1 rebaseline snapshot). + +The 6/44/0 counts are a historical measurement receipt — not a design target +or ceiling on capability lift. Live serving tests use the monotonic contract: + + wrong == 0 (hard safety invariant) + correct >= BASELINE_CORRECT + refused <= BASELINE_REFUSED + +Exact-count equality is reserved for explicitly named historical fixture tests +over the committed ``report.json`` artifact until a separate rebaseline ratifies +a new pin. +""" + +from __future__ import annotations + +BASELINE_CORRECT = 6 +BASELINE_WRONG = 0 +BASELINE_REFUSED = 44 +TRAIN_SAMPLE_COUNT = 50 + + +def assert_monotonic_serving_counts(counts: dict[str, int]) -> None: + """Serving lane live contract: wrong=0 hard; counts monotonic vs floor.""" + assert counts["wrong"] == BASELINE_WRONG + assert counts["correct"] >= BASELINE_CORRECT + assert counts["refused"] <= BASELINE_REFUSED + total = counts["correct"] + counts["wrong"] + counts["refused"] + assert total == TRAIN_SAMPLE_COUNT, ( + f"expected {TRAIN_SAMPLE_COUNT} scored cases, got {total}" + ) diff --git a/tests/test_adr_0174_phase3_lookback.py b/tests/test_adr_0174_phase3_lookback.py index d61e9680..24fde755 100644 --- a/tests/test_adr_0174_phase3_lookback.py +++ b/tests/test_adr_0174_phase3_lookback.py @@ -447,32 +447,24 @@ class TestPhase3WiringEndToEnd: class TestWrongZeroPreservation: - def test_train_sample_score_unchanged(self) -> None: - """Phase 3 substrate must preserve the current train_sample score. + def test_train_sample_serving_meets_monotonic_contract(self) -> None: + """Phase 3 substrate must not violate serving safety or regress below baseline. - Any change here would indicate the lookback path is firing on cases it - should not, or breaking cases it should not. + Capability lift (correct climbing, refusals falling) is allowed and + expected as injectors land; wrong answers and sub-baseline correct counts + indicate the lookback path is firing unsafely or breaking solved cases. """ import json from pathlib import Path from evals.gsm8k_math.train_sample.v1.runner import ( build_report, _CASES_PATH, ) + from tests.gsm8k_train_sample_baseline import assert_monotonic_serving_counts + cases = [ json.loads(line) for line in Path(_CASES_PATH).open(encoding="utf-8") if line.strip() ] report = build_report(cases) - counts = report["counts"] - assert counts["wrong"] == 0, ( - f"wrong=0 invariant violated: {counts}" - ) - assert counts["correct"] == 6, ( - f"correct count moved from 6 to {counts['correct']}; " - "Phase 3a substrate should not lift score on this corpus " - "(see PHASE-3.1 follow-up brief for what would lift it)" - ) - assert counts["refused"] == 44, ( - f"refused count moved from 44 to {counts['refused']}" - ) + assert_monotonic_serving_counts(report["counts"]) diff --git a/tests/test_adr_0175_phase2_practice_lane.py b/tests/test_adr_0175_phase2_practice_lane.py index 59049d0b..5993068d 100644 --- a/tests/test_adr_0175_phase2_practice_lane.py +++ b/tests/test_adr_0175_phase2_practice_lane.py @@ -37,13 +37,12 @@ from evals.gsm8k_math.practice.v1.runner import ( diagnose_refusal, run_practice, ) - -# Historical floor/ceiling from Inc1 rebaseline (2026-06-17). Monotonic contract: -# correct may rise, refused may fall, wrong must stay 0. Not a capability ceiling. -BASELINE_CORRECT = 6 -BASELINE_WRONG = 0 -BASELINE_REFUSED = 44 -TRAIN_SAMPLE_COUNT = 50 +from tests.gsm8k_train_sample_baseline import ( + BASELINE_CORRECT, + BASELINE_REFUSED, + BASELINE_WRONG, + TRAIN_SAMPLE_COUNT, +) def _assert_monotonic_capability_contract(rep: PracticeReport) -> None: @@ -217,10 +216,8 @@ class TestSealInvariant: serving_before = serving_build_report(cases) build_report() # run practice over same cases serving_after = serving_build_report(cases) - assert serving_before["counts"] == serving_after["counts"] + assert serving_after == serving_before assert serving_after["counts"]["wrong"] == BASELINE_WRONG - assert serving_after["counts"]["correct"] >= BASELINE_CORRECT - assert serving_after["counts"]["refused"] <= BASELINE_REFUSED def test_no_serving_module_imports_the_practice_lane(self) -> None: import subprocess diff --git a/tests/test_gsm8k_frontier_report.py b/tests/test_gsm8k_frontier_report.py index 24154787..24789dc9 100644 --- a/tests/test_gsm8k_frontier_report.py +++ b/tests/test_gsm8k_frontier_report.py @@ -1,13 +1,12 @@ """Tests for the deterministic GSM8K frontier report analyzer (Inc 2 + Inc3 evidence). -These tests pin: -- Stable bucketing of the exact refusal reasons emitted by the candidate graph. -- Correct extraction of category=... from "recognizer matched but produced no injection" strings. -- **Pinned historical artifact:** committed report.json still encodes pre-Inc2/Inc3 rate - no-injection (Inc 2 measurement target); not rebaselined by policy. -- **Live post-Inc3 behavior:** ephemeral train_sample runner shows rate_with_currency - no-injection = 0 after #799 connector support. -- Fully deterministic output (sorted keys, no timestamps, repeatable across runs). +Test classes: +- **Historical fixture:** committed ``report.json`` exact-count pin until rebaselined. +- **Live capability:** monotonic serving contract (wrong=0; correct may climb). +- **Frontier diagnostics:** bucketing, category extraction, deterministic analyzer output. + +Score preservation and truth preservation are separate: frozen counts protect replay +of committed measurement artifacts; live tests must not block capability lift. """ from __future__ import annotations @@ -20,13 +19,14 @@ from scripts.gsm8k_frontier_report import ( analyze_report, render_markdown, ) +from tests.gsm8k_train_sample_baseline import assert_monotonic_serving_counts _REPO_ROOT = Path(__file__).resolve().parents[1] _REPORT = _REPO_ROOT / "evals/gsm8k_math/train_sample/v1/report.json" -def test_pinned_report_json_is_historical_pre_inc3_artifact(): - """Committed report.json is the stale Inc1-era measurement artifact. +def test_committed_frontier_fixture_is_6_44_0_until_rebaselined(): + """Historical artifact: committed report.json stays at 6/44/0 until rebaselined. Inc2 used this file to surface rate_with_currency no-injection (3 cases). Inc3 (#799) changed live code but did not rebaseline report.json — this test @@ -58,6 +58,26 @@ def test_pinned_report_json_is_historical_pre_inc3_artifact(): assert json.dumps(summary, sort_keys=True) == json.dumps(summary2, sort_keys=True) +def test_live_serving_meets_monotonic_capability_contract(): + """Live train_sample: wrong=0 hard; counts monotonic; refusals carry reasons.""" + from evals.gsm8k_math.train_sample.v1.runner import build_report + + cases_path = _REPO_ROOT / "evals/gsm8k_math/train_sample/v1/cases.jsonl" + cases = [ + json.loads(line) + for line in cases_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + report = build_report(cases) + + assert_monotonic_serving_counts(report["counts"]) + + refused_rows = [r for r in report["per_case"] if r.get("verdict") == "refused"] + assert len(refused_rows) == report["counts"]["refused"] + for row in refused_rows: + assert row.get("reason"), f"refused case {row.get('case_id')} lacks reason" + + def test_post_inc3_live_runner_has_zero_rate_no_injection(): """Live train_sample scoring on current code: rate bucket no longer at injector.""" import re @@ -73,9 +93,7 @@ def test_post_inc3_live_runner_has_zero_rate_no_injection(): ] report = build_report(cases) - assert report["counts"]["correct"] == 6 - assert report["counts"]["refused"] == 44 - assert report["counts"]["wrong"] == 0 + assert_monotonic_serving_counts(report["counts"]) cats: Counter[str] = Counter() for row in report["per_case"]: