From 0ad97e5ef7e43686b2b2ccbe178fe1b4d73dfa17 Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 25 May 2026 05:42:12 -0700 Subject: [PATCH] perf(tests): extract math_teaching_corpus lane from pytest into CI lane SHAs (-9m suite time) (#261) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(tests): extract math_teaching_corpus lane from pytest into CI lane SHAs The two slowest tests in the pytest suite were: 388s test_adr_0131_2_teaching_corpus_lane::test_report_is_byte_equal_across_runs 161s test_adr_0131_2_teaching_corpus_lane::test_lane_passes_exit_criterion Both invoked build_report() from evals.math_teaching_corpus.v1.runner — the canonical math-teaching-corpus lane runner — once for the exit criterion and again for byte-equality. Together: 549s = 9m 9s, 30% of the full pytest suite, recomputed on every developer run. This is the exact 'lane runner invoked from pytest' anti-pattern that the existing scripts/verify_lane_shas.py CI job is designed to absorb. The other 7 lanes (reviewer_registry, miner_loop_closure, etc.) all run in CI via SHA pinning rather than in pytest. Changes: scripts/verify_lane_shas.py — add math_teaching_corpus_v1 spec + PINNED_SHAS entry (eaf160d145da29f9..., computed locally from a clean run of the lane in this commit's tree). scripts/generate_claims.py — add _LANE_ADR entry (ADR-0131) + claim text. Failing fast on missing lanes is by design. CLAIMS.md — regenerated; one new row. tests/test_adr_0131_2_teaching_corpus_lane.py — delete TestLaneGate class (2 tests, 549s). Retain TestDatasetIntegrity (5 tests), TestBoundedDomain (2), TestHonestEvidence (1) — these are fast (0.26s total) and pin contracts the lane runner does not cover (dataset shape, lemma boundedness, evidence reachability). Replace deletion with an explanatory comment block. The deleted contracts are still enforced — just in CI instead of pytest: exit criterion → runner exit code (returns 1 on failure) byte-equality → PINNED_SHAS verification (SHA-256 of report.json) Verified locally: scripts/verify_lane_shas.py — 8/8 lanes match pinned SHAs pytest tests/test_adr_0131_2_teaching_corpus_lane.py — 8/8 pass in 0.26s Expected full-suite delta: -549s (from ~30m to ~21m). Further speedup will come from the upcoming full-pytest CI gate with pytest-xdist -n4. * ci: bump lane-shas timeout 12m → 20m for new math_teaching_corpus lane The math_teaching_corpus_v1 lane added in this PR runs in ~5-6 min, pushing the total lane-shas job over the previous 12-min timeout. First CI run cancelled at 12m17s. Bumping to 20m gives ~8m headroom. * fix(ci): bump lane subprocess timeout 300s→900s + add math_teaching_corpus to test_lane_sha_verifier EXPECTED_LANES Two issues surfaced by CI run on the prior commit: 1. The math_teaching_corpus lane takes ~142s wall-clock locally (3.79 cores × ~538s CPU). On CI's single/dual-core runner that translates to ~5-9 min, exceeding the 300s subprocess timeout in scripts/verify_lane_shas.py. Bumping to 900s gives ~60% headroom. 2. tests/test_lane_sha_verifier.py::TestExpectedLaneCoverage::test_all_expected_lanes_covered hardcodes the expected lane set. Adding math_teaching_corpus_v1 to LANE_SPECS triggered the 'extra lanes' assertion. Adding it to EXPECTED_LANES (the file's own contract: 'if intentional, add here'). --- .github/workflows/lane-shas.yml | 2 +- CLAIMS.md | 1 + scripts/generate_claims.py | 4 +++ scripts/verify_lane_shas.py | 9 +++++- tests/test_adr_0131_2_teaching_corpus_lane.py | 30 +++++-------------- tests/test_lane_sha_verifier.py | 1 + 6 files changed, 23 insertions(+), 24 deletions(-) diff --git a/.github/workflows/lane-shas.yml b/.github/workflows/lane-shas.yml index 077fce74..f89ff433 100644 --- a/.github/workflows/lane-shas.yml +++ b/.github/workflows/lane-shas.yml @@ -25,7 +25,7 @@ jobs: verify: name: verify pinned lane SHAs runs-on: ubuntu-latest - timeout-minutes: 12 + timeout-minutes: 20 steps: - name: checkout diff --git a/CLAIMS.md b/CLAIMS.md index 2a5b9b40..caa2dfce 100644 --- a/CLAIMS.md +++ b/CLAIMS.md @@ -41,6 +41,7 @@ is a CI failure (`.github/workflows/lane-shas.yml`). | ADR-0098 | `demo_composition` | Demos compose from shipped modules; no parallel mechanism | `evals/demo_composition/results/v1_dev.json` | `3a3d09f3a87462737e615c2dd3481b9e13e5ff8fadee0043c37873494ded556d` | | ADR-0099 | `public_demo` | Public showcase runs deterministically under 30s; all claims supported | `evals/public_demo/results/v1_dev.json` | `888ddd0d12635d709f91898cf06601062168694d870f776d2b0b7710e5d68cbd` | | ADR-0104 | `curriculum_loop_closure` | Curriculum-sourced proposals route through single reviewed teaching path | `evals/curriculum_loop_closure/results/v1_dev.json` | `b46d56b2d209172cc3ffaf3776dc8dcfe55093f13587c5cb67372be6dfa23e8d` | +| ADR-0131 | `math_teaching_corpus_v1` | Math teaching corpus replays deterministically; all chains pass exit criterion (correct_rate=1.0, wrong=0) | `evals/math_teaching_corpus/v1/report.json` | `eaf160d145da29f9050ede8d58bf111b0f651dd40aeae9201857d0b97e014dd4` | ## Verification diff --git a/scripts/generate_claims.py b/scripts/generate_claims.py index b526be91..c3b85523 100644 --- a/scripts/generate_claims.py +++ b/scripts/generate_claims.py @@ -66,6 +66,10 @@ _LANE_ADR: dict[str, tuple[str, str]] = { "ADR-0099", "Public showcase runs deterministically under 30s; all claims supported", ), + "math_teaching_corpus_v1": ( + "ADR-0131", + "Math teaching corpus replays deterministically; all chains pass exit criterion (correct_rate=1.0, wrong=0)", + ), } diff --git a/scripts/verify_lane_shas.py b/scripts/verify_lane_shas.py index d1af4fc1..47b56eba 100644 --- a/scripts/verify_lane_shas.py +++ b/scripts/verify_lane_shas.py @@ -40,6 +40,7 @@ PINNED_SHAS: dict[str, str] = { "fabrication_control_summary": "01e1b6b711141f2b4a14551d7df3ea482d8d6dd7b364a25c509f4f8d08cda8a8", "demo_composition": "3a3d09f3a87462737e615c2dd3481b9e13e5ff8fadee0043c37873494ded556d", "public_demo": "888ddd0d12635d709f91898cf06601062168694d870f776d2b0b7710e5d68cbd", + "math_teaching_corpus_v1": "eaf160d145da29f9050ede8d58bf111b0f651dd40aeae9201857d0b97e014dd4", } @@ -98,6 +99,12 @@ LANE_SPECS: tuple[LaneSpec, ...] = ( runner_module="evals/public_demo/runner.py", report_relative="evals/public_demo/results/v1_dev.json", ), + LaneSpec( + lane_id="math_teaching_corpus_v1", + runner_module="evals/math_teaching_corpus/v1/runner.py", + report_relative="evals/math_teaching_corpus/v1/report.json", + accepts_report_flag=False, + ), ) @@ -115,7 +122,7 @@ def _invoke_runner(spec: LaneSpec, *, target_path: Path | None = None) -> Path: env=env, capture_output=True, text=True, - timeout=300, + timeout=900, ) if result.returncode != 0: raise RuntimeError( diff --git a/tests/test_adr_0131_2_teaching_corpus_lane.py b/tests/test_adr_0131_2_teaching_corpus_lane.py index 488b04ae..49c638d2 100644 --- a/tests/test_adr_0131_2_teaching_corpus_lane.py +++ b/tests/test_adr_0131_2_teaching_corpus_lane.py @@ -15,10 +15,7 @@ import json from pathlib import Path from chat.pack_resolver import _pack_lexicon_for -from evals.math_teaching_corpus.v1.runner import ( - _load_cases, - build_report, -) +from evals.math_teaching_corpus.v1.runner import _load_cases _ROOT = Path(__file__).resolve().parent.parent _CASES_PATH = _ROOT / "evals" / "math_teaching_corpus" / "v1" / "cases.jsonl" @@ -152,21 +149,10 @@ class TestHonestEvidence: assert ref in valid_refs, f"dangling reference: {ref!r} cited in candidate {c['candidate_id']}" -class TestLaneGate: - def test_lane_passes_exit_criterion(self) -> None: - cases = _load_cases(_CASES_PATH) - report = build_report(cases) - assert report["exit_criterion"]["passed"], ( - f"lane gate failed: counts={report['counts']!r} " - f"correct_rate={report['correct_rate']!r}" - ) - assert report["counts"]["wrong"] == 0, "wrong count must be zero" - assert report["counts"]["correct"] == len(cases), "all cases must be correct" - - def test_report_is_byte_equal_across_runs(self) -> None: - cases = _load_cases(_CASES_PATH) - r1 = build_report(cases) - r2 = build_report(cases) - s1 = json.dumps(r1, sort_keys=True).encode("utf-8") - s2 = json.dumps(r2, sort_keys=True).encode("utf-8") - assert s1 == s2 +# TestLaneGate (test_lane_passes_exit_criterion + test_report_is_byte_equal_across_runs) +# was extracted from pytest into the CI lane SHA pin job. The math_teaching_corpus_v1 +# lane runner is now invoked by .github/workflows/lane-shas.yml via +# scripts/verify_lane_shas.py, which (a) runs the lane, (b) checks the exit criterion +# via runner exit code, and (c) verifies the report.json byte-content against the +# pinned SHA-256 — the same three contracts the deleted tests held, but done once +# per PR in CI instead of twice per pytest run (~549s recovered from pytest). diff --git a/tests/test_lane_sha_verifier.py b/tests/test_lane_sha_verifier.py index 6b883a2e..f40c29c1 100644 --- a/tests/test_lane_sha_verifier.py +++ b/tests/test_lane_sha_verifier.py @@ -90,6 +90,7 @@ class TestExpectedLaneCoverage: "demo_composition", # ADR-0098 "public_demo", # ADR-0099 "curriculum_loop_closure", + "math_teaching_corpus_v1", # ADR-0131 } )