diff --git a/evals/gsm8k_math/scoring/depth_curve.py b/evals/gsm8k_math/scoring/depth_curve.py index edcd7def..8661f4eb 100644 --- a/evals/gsm8k_math/scoring/depth_curve.py +++ b/evals/gsm8k_math/scoring/depth_curve.py @@ -2,6 +2,16 @@ Buckets the correct rate of a lane run by reasoning depth (number of operations in the ground-truth graph). + +Documented buckets per ADR-0119.6 contract: +``depth_1``, ``depth_2-3``, ``depth_4-5``, ``depth_6-8``. + +Depth outside ``1..8`` raises :class:`DepthCurveError` rather than +silently creating a new bucket — the schema is documented and any +extension requires an explicit ADR amendment. + +Missing case_id in the lane report also raises rather than silently +treating as ``refused`` — that would mask runner bugs. """ from __future__ import annotations @@ -12,12 +22,20 @@ import sys from pathlib import Path from typing import Any -# Ensure workspace root is in sys.path when running as script -sys.path.insert(0, str(Path(__file__).resolve().parents[3])) - from evals.gsm8k_math.runner import LaneReport, run_lane +class DepthCurveError(ValueError): + """Raised when a case's depth or outcome cannot be classified. + + Reasons: + - depth ≥ 9 (out of documented bucket range) + - depth == 0 (degenerate) + - case_id present in input list but missing from lane_report + (runner-output integrity violation) + """ + + def compute_depth_curve(cases: list[dict[str, Any]], lane_report: LaneReport) -> dict[str, Any]: """Pure, deterministic function bucketizing cases by reasoning depth. @@ -54,11 +72,20 @@ def compute_depth_curve(cases: list[dict[str, Any]], lane_report: LaneReport) -> for case in cases: case_id = case["id"] depth = case_depths[case_id] - outcome = outcomes.get(case_id, "refused") + + # Missing case_id in lane_report is a runner-integrity violation, + # not a silent refusal. Fail loud. + if case_id not in outcomes: + raise DepthCurveError( + f"case_id {case_id!r} present in input but missing from " + f"lane_report.case_details — runner-output integrity violation" + ) + outcome = outcomes[case_id] is_correct = 1 if outcome == "correct" else 0 - # Resolve bucket key + # Resolve bucket key. Depth outside documented range raises rather + # than silently extending the schema. if depth == 1: b_key = "depth_1" elif 2 <= depth <= 3: @@ -68,16 +95,15 @@ def compute_depth_curve(cases: list[dict[str, Any]], lane_report: LaneReport) -> elif 6 <= depth <= 8: b_key = "depth_6-8" else: - b_key = f"depth_{depth}" - - if b_key not in buckets: - buckets[b_key] = {"total": 0, "correct": 0, "rate": 0.0} + raise DepthCurveError( + f"case {case_id!r} has depth {depth}, outside documented " + f"range 1..8 (extending the bucket schema requires an ADR " + f"amendment to ADR-0119.6)" + ) buckets[b_key]["total"] += 1 buckets[b_key]["correct"] += is_correct - if depth not in raw_depths: - raw_depths[depth] = {"total": 0, "correct": 0} raw_depths[depth]["total"] += 1 raw_depths[depth]["correct"] += is_correct diff --git a/tests/test_adr_0119_4_frontier_baseline.py b/tests/test_adr_0119_4_frontier_baseline.py index 6e01c07c..1f6718f9 100644 --- a/tests/test_adr_0119_4_frontier_baseline.py +++ b/tests/test_adr_0119_4_frontier_baseline.py @@ -111,3 +111,51 @@ def test_disclaimer_content() -> None: assert "public split" in disclaimer_lower or "original" in disclaimer_lower assert "actual gsm8k" in disclaimer_lower or "actual-gsm8k-test" in disclaimer_lower assert "caveat" in disclaimer_lower or "not strictly comparable" in disclaimer_lower or "different" in disclaimer_lower + + +# ------------------------------------------------------------------- +# ADR-0119.4 follow-up: numeric + freshness asserts +# ------------------------------------------------------------------- + +import re + + +def test_comparison_core_measurement_reflects_wrong_zero_discipline() -> None: + """The comparison file must report wrong == 0 — the load-bearing + differentiator the disclaimer names. A tampered file with wrong > 0 + would still be syntactically valid; this gates the substance.""" + root = Path(__file__).resolve().parents[1] + comparison_path = root / "evals/gsm8k_math/baselines/comparison_v1.json" + with open(comparison_path, "r", encoding="utf-8") as f: + data = json.load(f) + core = data["core_measurement"] + assert core["wrong"] == 0, ( + f"comparison_v1 reports wrong={core['wrong']}; the disclaimer " + f"explicitly names zero-confabulation as the differentiator, " + f"so wrong must be 0" + ) + # And correct + refused == cases_total (no missing accounting) + assert core["correct"] + core["refused"] == core["cases_total"] + + +def test_frontier_citations_are_recent_enough() -> None: + """Baseline freshness: every citation source must mention a year >= 2023. + Citations older than that should be refreshed before the lane gates + anything for an `expert` promotion (ADR-0120).""" + root = Path(__file__).resolve().parents[1] + frontier_path = root / "evals/gsm8k_math/baselines/frontier.json" + with open(frontier_path, "r", encoding="utf-8") as f: + data = json.load(f) + year_re = re.compile(r"\b(19|20)\d{2}\b") + for entry in data["baselines"]: + years = year_re.findall(entry["source"]) + # findall returns the first-group matches; re-extract full years + full_years = [int(s) for s in re.findall(r"\b(20\d{2})\b", entry["source"])] + assert full_years, ( + f"citation for {entry['system']!r} has no year in source: " + f"{entry['source']!r}" + ) + assert max(full_years) >= 2023, ( + f"citation for {entry['system']!r} is older than 2023: " + f"{entry['source']!r}" + ) diff --git a/tests/test_adr_0119_6_depth_curve.py b/tests/test_adr_0119_6_depth_curve.py index c2f0606e..cde894a0 100644 --- a/tests/test_adr_0119_6_depth_curve.py +++ b/tests/test_adr_0119_6_depth_curve.py @@ -3,11 +3,16 @@ from __future__ import annotations import json +from dataclasses import replace from pathlib import Path + import pytest -from evals.gsm8k_math.runner import run_lane -from evals.gsm8k_math.scoring.depth_curve import compute_depth_curve +from evals.gsm8k_math.runner import LaneReport, run_lane +from evals.gsm8k_math.scoring.depth_curve import ( + DepthCurveError, + compute_depth_curve, +) _REPO_ROOT = Path(__file__).resolve().parent.parent @@ -128,3 +133,47 @@ def test_curve_flatness_ratio() -> None: # The test confirms that the harness CAN compute and report the ratio assert isinstance(ratio, float) assert ratio >= 0.0 + + +# ------------------------------------------------------------------- +# ADR-0119.6 follow-up: explicit-refusal tests +# ------------------------------------------------------------------- + + +def test_depth_curve_refuses_case_missing_from_lane_report() -> None: + """Runner-output integrity: every input case_id must appear in lane_report.""" + cases = _DEV_CASES[:3] + real_report = run_lane(cases) + # Drop one case from the report → harness must raise, not silently fallback + stripped = LaneReport() + stripped.metrics = real_report.metrics + stripped.case_details = real_report.case_details[1:] # drop first + with pytest.raises(DepthCurveError, match="missing from lane_report"): + compute_depth_curve(cases, stripped) + + +def test_depth_curve_refuses_depth_outside_documented_range() -> None: + """Depth >= 9 raises rather than silently extending the bucket schema.""" + # Synthesize a case with depth 9 + deep_case = { + "id": "synthetic-deep-01", + "problem": "synthetic", + "expected_answer": 0, + "expected_unit": "x", + "ground_truth_graph": { + "entities": ["X"], + "initial_state": [], + "operations": [ + {"actor": "X", "kind": "add", "operand": {"unit": "x", "value": 1}} + for _ in range(9) + ], + "unknown": {"entity": "X", "unit": "x"}, + }, + } + report = LaneReport() + report.metrics = {"cases_total": 1, "correct": 0, "wrong": 0, "refused": 1, + "correct_rate": 0.0, "wrong_rate": 0.0, "refused_rate": 1.0, + "wrong_count_is_zero": True, "overall_pass": True} + report.case_details = [{"case_id": "synthetic-deep-01", "outcome": "refused"}] + with pytest.raises(DepthCurveError, match="outside documented range"): + compute_depth_curve([deep_case], report)