diff --git a/.github/workflows/lane-shas.yml b/.github/workflows/lane-shas.yml index 077fce74..f89ff433 100644 --- a/.github/workflows/lane-shas.yml +++ b/.github/workflows/lane-shas.yml @@ -25,7 +25,7 @@ jobs: verify: name: verify pinned lane SHAs runs-on: ubuntu-latest - timeout-minutes: 12 + timeout-minutes: 20 steps: - name: checkout diff --git a/CLAIMS.md b/CLAIMS.md index 2a5b9b40..caa2dfce 100644 --- a/CLAIMS.md +++ b/CLAIMS.md @@ -41,6 +41,7 @@ is a CI failure (`.github/workflows/lane-shas.yml`). | ADR-0098 | `demo_composition` | Demos compose from shipped modules; no parallel mechanism | `evals/demo_composition/results/v1_dev.json` | `3a3d09f3a87462737e615c2dd3481b9e13e5ff8fadee0043c37873494ded556d` | | ADR-0099 | `public_demo` | Public showcase runs deterministically under 30s; all claims supported | `evals/public_demo/results/v1_dev.json` | `888ddd0d12635d709f91898cf06601062168694d870f776d2b0b7710e5d68cbd` | | ADR-0104 | `curriculum_loop_closure` | Curriculum-sourced proposals route through single reviewed teaching path | `evals/curriculum_loop_closure/results/v1_dev.json` | `b46d56b2d209172cc3ffaf3776dc8dcfe55093f13587c5cb67372be6dfa23e8d` | +| ADR-0131 | `math_teaching_corpus_v1` | Math teaching corpus replays deterministically; all chains pass exit criterion (correct_rate=1.0, wrong=0) | `evals/math_teaching_corpus/v1/report.json` | `eaf160d145da29f9050ede8d58bf111b0f651dd40aeae9201857d0b97e014dd4` | ## Verification diff --git a/scripts/generate_claims.py b/scripts/generate_claims.py index b526be91..c3b85523 100644 --- a/scripts/generate_claims.py +++ b/scripts/generate_claims.py @@ -66,6 +66,10 @@ _LANE_ADR: dict[str, tuple[str, str]] = { "ADR-0099", "Public showcase runs deterministically under 30s; all claims supported", ), + "math_teaching_corpus_v1": ( + "ADR-0131", + "Math teaching corpus replays deterministically; all chains pass exit criterion (correct_rate=1.0, wrong=0)", + ), } diff --git a/scripts/verify_lane_shas.py b/scripts/verify_lane_shas.py index d1af4fc1..47b56eba 100644 --- a/scripts/verify_lane_shas.py +++ b/scripts/verify_lane_shas.py @@ -40,6 +40,7 @@ PINNED_SHAS: dict[str, str] = { "fabrication_control_summary": "01e1b6b711141f2b4a14551d7df3ea482d8d6dd7b364a25c509f4f8d08cda8a8", "demo_composition": "3a3d09f3a87462737e615c2dd3481b9e13e5ff8fadee0043c37873494ded556d", "public_demo": "888ddd0d12635d709f91898cf06601062168694d870f776d2b0b7710e5d68cbd", + "math_teaching_corpus_v1": "eaf160d145da29f9050ede8d58bf111b0f651dd40aeae9201857d0b97e014dd4", } @@ -98,6 +99,12 @@ LANE_SPECS: tuple[LaneSpec, ...] = ( runner_module="evals/public_demo/runner.py", report_relative="evals/public_demo/results/v1_dev.json", ), + LaneSpec( + lane_id="math_teaching_corpus_v1", + runner_module="evals/math_teaching_corpus/v1/runner.py", + report_relative="evals/math_teaching_corpus/v1/report.json", + accepts_report_flag=False, + ), ) @@ -115,7 +122,7 @@ def _invoke_runner(spec: LaneSpec, *, target_path: Path | None = None) -> Path: env=env, capture_output=True, text=True, - timeout=300, + timeout=900, ) if result.returncode != 0: raise RuntimeError( diff --git a/tests/test_adr_0131_2_teaching_corpus_lane.py b/tests/test_adr_0131_2_teaching_corpus_lane.py index 488b04ae..49c638d2 100644 --- a/tests/test_adr_0131_2_teaching_corpus_lane.py +++ b/tests/test_adr_0131_2_teaching_corpus_lane.py @@ -15,10 +15,7 @@ import json from pathlib import Path from chat.pack_resolver import _pack_lexicon_for -from evals.math_teaching_corpus.v1.runner import ( - _load_cases, - build_report, -) +from evals.math_teaching_corpus.v1.runner import _load_cases _ROOT = Path(__file__).resolve().parent.parent _CASES_PATH = _ROOT / "evals" / "math_teaching_corpus" / "v1" / "cases.jsonl" @@ -152,21 +149,10 @@ class TestHonestEvidence: assert ref in valid_refs, f"dangling reference: {ref!r} cited in candidate {c['candidate_id']}" -class TestLaneGate: - def test_lane_passes_exit_criterion(self) -> None: - cases = _load_cases(_CASES_PATH) - report = build_report(cases) - assert report["exit_criterion"]["passed"], ( - f"lane gate failed: counts={report['counts']!r} " - f"correct_rate={report['correct_rate']!r}" - ) - assert report["counts"]["wrong"] == 0, "wrong count must be zero" - assert report["counts"]["correct"] == len(cases), "all cases must be correct" - - def test_report_is_byte_equal_across_runs(self) -> None: - cases = _load_cases(_CASES_PATH) - r1 = build_report(cases) - r2 = build_report(cases) - s1 = json.dumps(r1, sort_keys=True).encode("utf-8") - s2 = json.dumps(r2, sort_keys=True).encode("utf-8") - assert s1 == s2 +# TestLaneGate (test_lane_passes_exit_criterion + test_report_is_byte_equal_across_runs) +# was extracted from pytest into the CI lane SHA pin job. The math_teaching_corpus_v1 +# lane runner is now invoked by .github/workflows/lane-shas.yml via +# scripts/verify_lane_shas.py, which (a) runs the lane, (b) checks the exit criterion +# via runner exit code, and (c) verifies the report.json byte-content against the +# pinned SHA-256 — the same three contracts the deleted tests held, but done once +# per PR in CI instead of twice per pytest run (~549s recovered from pytest). diff --git a/tests/test_lane_sha_verifier.py b/tests/test_lane_sha_verifier.py index 6b883a2e..f40c29c1 100644 --- a/tests/test_lane_sha_verifier.py +++ b/tests/test_lane_sha_verifier.py @@ -90,6 +90,7 @@ class TestExpectedLaneCoverage: "demo_composition", # ADR-0098 "public_demo", # ADR-0099 "curriculum_loop_closure", + "math_teaching_corpus_v1", # ADR-0131 } )