a7feda3c19
Re-runs parse_and_solve on the 50-case GSM8K train sample on current main (post-S.3) and compares to v2. Result: admitted=3/50 (unchanged), wrong=0, exactly 1 barrier shifted v2→v3. Shift: gsm8k-0010 (compound_statement → fraction_operand). S.3's _INIT_MUTATION_RE resolves "Yun had 20 paperclips initially, but then lost 12" to InitialPossession(Yun, 8, paperclips). First refusal moved to sentence 2: "Marion has 1/4 more than what Yun currently has, plus 7" — needs fraction-operand + coreference-quantity + comparative-additive arithmetic. Top blockers (v3): compound_statement 5 (was 6) novel_initial_form 5 (unchanged) fraction_operand 4 (was 3 — gsm8k-0010 moved here) novel_initial_verb 4 (unchanged) Artifacts: - evals/gsm8k_math/train_sample/v1/rescan_v3.py - evals/gsm8k_math/train_sample/v1/refusal_rescan_v3.json - evals/gsm8k_math/train_sample/v1/refusal_taxonomy_v3.json - docs/decisions/ADR-0136.S3-post-rescan.md - tests/test_rescan_v3_invariants.py (7 tests; determinism + admission set unchanged + exactly-one-shift + 0010-specific shift assertions)
76 lines
2.7 KiB
Python
76 lines
2.7 KiB
Python
"""ADR-0136.S.3-post-rescan — invariant tests for refusal rescan v3."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from evals.gsm8k_math.train_sample.v1.rescan_v3 import build_rescan
|
|
|
|
_HERE = Path(__file__).resolve().parent.parent
|
|
_RESCAN_V3 = _HERE / "evals/gsm8k_math/train_sample/v1/refusal_rescan_v3.json"
|
|
_TAXONOMY_V3 = _HERE / "evals/gsm8k_math/train_sample/v1/refusal_taxonomy_v3.json"
|
|
|
|
|
|
def test_wrong_is_zero() -> None:
|
|
"""GSM8K probe under v3 must hold wrong == 0."""
|
|
rescan, _ = build_rescan()
|
|
assert rescan["summary"]["wrong"] == 0
|
|
|
|
|
|
def test_admission_set_unchanged() -> None:
|
|
"""S.3 did not change the admission set: {0014, 0018, 0042}."""
|
|
rescan, _ = build_rescan()
|
|
admitted = {
|
|
c["case_id"] for c in rescan["per_case"] if c["current_outcome"] == "admitted"
|
|
}
|
|
assert admitted == {
|
|
"gsm8k-train-sample-v1-0014",
|
|
"gsm8k-train-sample-v1-0018",
|
|
"gsm8k-train-sample-v1-0042",
|
|
}
|
|
|
|
|
|
def test_exactly_one_shift_v2_to_v3() -> None:
|
|
"""S.3 shifted exactly one case (gsm8k-0010)."""
|
|
rescan, _ = build_rescan()
|
|
assert rescan["summary"]["barrier_shifted_v2_to_v3"] == 1
|
|
shifted = [c["case_id"] for c in rescan["per_case"] if c["barrier_shifted"]]
|
|
assert shifted == ["gsm8k-train-sample-v1-0010"]
|
|
|
|
|
|
def test_gsm8k_0010_now_refuses_on_sentence_2() -> None:
|
|
"""gsm8k-0010 first-refusal moved from sentence 1 to sentence 2."""
|
|
rescan, _ = build_rescan()
|
|
case = next(
|
|
c for c in rescan["per_case"]
|
|
if c["case_id"] == "gsm8k-train-sample-v1-0010"
|
|
)
|
|
assert case["previous_first_refusal"].startswith("Yun had 20 paperclips")
|
|
assert case["current_first_refusal"].startswith("Marion has 1/4")
|
|
assert case["previous_primary_barrier"] == "compound_statement"
|
|
assert case["current_primary_barrier"] == "fraction_operand"
|
|
|
|
|
|
def test_artifacts_deterministic() -> None:
|
|
"""Two runs of build_rescan() produce byte-equal artifacts."""
|
|
r1, t1 = build_rescan()
|
|
r2, t2 = build_rescan()
|
|
assert json.dumps(r1, sort_keys=True) == json.dumps(r2, sort_keys=True)
|
|
assert json.dumps(t1, sort_keys=True) == json.dumps(t2, sort_keys=True)
|
|
|
|
|
|
def test_taxonomy_v3_has_50_entries() -> None:
|
|
_, taxonomy = build_rescan()
|
|
assert len(taxonomy["per_case"]) == 50
|
|
|
|
|
|
def test_disk_artifacts_match_freshly_computed() -> None:
|
|
"""If rescan v3 was committed, disk artifacts must match a fresh run."""
|
|
if not _RESCAN_V3.exists() or not _TAXONOMY_V3.exists():
|
|
return
|
|
rescan, taxonomy = build_rescan()
|
|
on_disk_rescan = json.loads(_RESCAN_V3.read_text(encoding="utf-8"))
|
|
on_disk_taxonomy = json.loads(_TAXONOMY_V3.read_text(encoding="utf-8"))
|
|
assert on_disk_rescan == rescan
|
|
assert on_disk_taxonomy == taxonomy
|