Merge pull request #147 from AssetOverflow/feat/phase5-cleanup

chore: ADR-0119.4 + ADR-0119.6 cleanup — typed refusals + numeric/freshness asserts

evals/gsm8k_math/scoring/depth_curve.py

@@ -2,6 +2,16 @@
 
 Buckets the correct rate of a lane run by reasoning depth
 (number of operations in the ground-truth graph).
+
+Documented buckets per ADR-0119.6 contract:
+``depth_1``, ``depth_2-3``, ``depth_4-5``, ``depth_6-8``.
+
+Depth outside ``1..8`` raises :class:`DepthCurveError` rather than
+silently creating a new bucket — the schema is documented and any
+extension requires an explicit ADR amendment.
+
+Missing case_id in the lane report also raises rather than silently
+treating as ``refused`` — that would mask runner bugs.
 """
 
 from __future__ import annotations
@@ -12,12 +22,20 @@ import sys
 from pathlib import Path
 from typing import Any
 
-# Ensure workspace root is in sys.path when running as script
-sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
-
 from evals.gsm8k_math.runner import LaneReport, run_lane
 
 
+class DepthCurveError(ValueError):
+    """Raised when a case's depth or outcome cannot be classified.
+
+    Reasons:
+        - depth ≥ 9 (out of documented bucket range)
+        - depth == 0 (degenerate)
+        - case_id present in input list but missing from lane_report
+          (runner-output integrity violation)
+    """
+
+
 def compute_depth_curve(cases: list[dict[str, Any]], lane_report: LaneReport) -> dict[str, Any]:
     """Pure, deterministic function bucketizing cases by reasoning depth.
 
@@ -54,11 +72,20 @@ def compute_depth_curve(cases: list[dict[str, Any]], lane_report: LaneReport) ->
     for case in cases:
         case_id = case["id"]
         depth = case_depths[case_id]
-        outcome = outcomes.get(case_id, "refused")
+
+        # Missing case_id in lane_report is a runner-integrity violation,
+        # not a silent refusal. Fail loud.
+        if case_id not in outcomes:
+            raise DepthCurveError(
+                f"case_id {case_id!r} present in input but missing from "
+                f"lane_report.case_details — runner-output integrity violation"
+            )
+        outcome = outcomes[case_id]
 
         is_correct = 1 if outcome == "correct" else 0
 
-        # Resolve bucket key
+        # Resolve bucket key. Depth outside documented range raises rather
+        # than silently extending the schema.
         if depth == 1:
             b_key = "depth_1"
         elif 2 <= depth <= 3:
@@ -68,16 +95,15 @@ def compute_depth_curve(cases: list[dict[str, Any]], lane_report: LaneReport) ->
         elif 6 <= depth <= 8:
             b_key = "depth_6-8"
         else:
-            b_key = f"depth_{depth}"
+            raise DepthCurveError(
-
+                f"case {case_id!r} has depth {depth}, outside documented "
-        if b_key not in buckets:
+                f"range 1..8 (extending the bucket schema requires an ADR "
-            buckets[b_key] = {"total": 0, "correct": 0, "rate": 0.0}
+                f"amendment to ADR-0119.6)"
+            )
 
         buckets[b_key]["total"] += 1
         buckets[b_key]["correct"] += is_correct
 
-        if depth not in raw_depths:
-            raw_depths[depth] = {"total": 0, "correct": 0}
         raw_depths[depth]["total"] += 1
         raw_depths[depth]["correct"] += is_correct
 

tests/test_adr_0119_4_frontier_baseline.py

@@ -111,3 +111,51 @@ def test_disclaimer_content() -> None:
     assert "public split" in disclaimer_lower or "original" in disclaimer_lower
     assert "actual gsm8k" in disclaimer_lower or "actual-gsm8k-test" in disclaimer_lower
     assert "caveat" in disclaimer_lower or "not strictly comparable" in disclaimer_lower or "different" in disclaimer_lower
+
+
+# -------------------------------------------------------------------
+# ADR-0119.4 follow-up: numeric + freshness asserts
+# -------------------------------------------------------------------
+
+import re
+
+
+def test_comparison_core_measurement_reflects_wrong_zero_discipline() -> None:
+    """The comparison file must report wrong == 0 — the load-bearing
+    differentiator the disclaimer names. A tampered file with wrong > 0
+    would still be syntactically valid; this gates the substance."""
+    root = Path(__file__).resolve().parents[1]
+    comparison_path = root / "evals/gsm8k_math/baselines/comparison_v1.json"
+    with open(comparison_path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    core = data["core_measurement"]
+    assert core["wrong"] == 0, (
+        f"comparison_v1 reports wrong={core['wrong']}; the disclaimer "
+        f"explicitly names zero-confabulation as the differentiator, "
+        f"so wrong must be 0"
+    )
+    # And correct + refused == cases_total (no missing accounting)
+    assert core["correct"] + core["refused"] == core["cases_total"]
+
+
+def test_frontier_citations_are_recent_enough() -> None:
+    """Baseline freshness: every citation source must mention a year >= 2023.
+    Citations older than that should be refreshed before the lane gates
+    anything for an `expert` promotion (ADR-0120)."""
+    root = Path(__file__).resolve().parents[1]
+    frontier_path = root / "evals/gsm8k_math/baselines/frontier.json"
+    with open(frontier_path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    year_re = re.compile(r"\b(19|20)\d{2}\b")
+    for entry in data["baselines"]:
+        years = year_re.findall(entry["source"])
+        # findall returns the first-group matches; re-extract full years
+        full_years = [int(s) for s in re.findall(r"\b(20\d{2})\b", entry["source"])]
+        assert full_years, (
+            f"citation for {entry['system']!r} has no year in source: "
+            f"{entry['source']!r}"
+        )
+        assert max(full_years) >= 2023, (
+            f"citation for {entry['system']!r} is older than 2023: "
+            f"{entry['source']!r}"
+        )

tests/test_adr_0119_6_depth_curve.py

@@ -3,11 +3,16 @@
 from __future__ import annotations
 
 import json
+from dataclasses import replace
 from pathlib import Path
+
 import pytest
 
-from evals.gsm8k_math.runner import run_lane
+from evals.gsm8k_math.runner import LaneReport, run_lane
-from evals.gsm8k_math.scoring.depth_curve import compute_depth_curve
+from evals.gsm8k_math.scoring.depth_curve import (
+    DepthCurveError,
+    compute_depth_curve,
+)
 
 _REPO_ROOT = Path(__file__).resolve().parent.parent
 
@@ -128,3 +133,47 @@ def test_curve_flatness_ratio() -> None:
     # The test confirms that the harness CAN compute and report the ratio
     assert isinstance(ratio, float)
     assert ratio >= 0.0
+
+
+# -------------------------------------------------------------------
+# ADR-0119.6 follow-up: explicit-refusal tests
+# -------------------------------------------------------------------
+
+
+def test_depth_curve_refuses_case_missing_from_lane_report() -> None:
+    """Runner-output integrity: every input case_id must appear in lane_report."""
+    cases = _DEV_CASES[:3]
+    real_report = run_lane(cases)
+    # Drop one case from the report → harness must raise, not silently fallback
+    stripped = LaneReport()
+    stripped.metrics = real_report.metrics
+    stripped.case_details = real_report.case_details[1:]  # drop first
+    with pytest.raises(DepthCurveError, match="missing from lane_report"):
+        compute_depth_curve(cases, stripped)
+
+
+def test_depth_curve_refuses_depth_outside_documented_range() -> None:
+    """Depth >= 9 raises rather than silently extending the bucket schema."""
+    # Synthesize a case with depth 9
+    deep_case = {
+        "id": "synthetic-deep-01",
+        "problem": "synthetic",
+        "expected_answer": 0,
+        "expected_unit": "x",
+        "ground_truth_graph": {
+            "entities": ["X"],
+            "initial_state": [],
+            "operations": [
+                {"actor": "X", "kind": "add", "operand": {"unit": "x", "value": 1}}
+                for _ in range(9)
+            ],
+            "unknown": {"entity": "X", "unit": "x"},
+        },
+    }
+    report = LaneReport()
+    report.metrics = {"cases_total": 1, "correct": 0, "wrong": 0, "refused": 1,
+                      "correct_rate": 0.0, "wrong_rate": 0.0, "refused_rate": 1.0,
+                      "wrong_count_is_zero": True, "overall_pass": True}
+    report.case_details = [{"case_id": "synthetic-deep-01", "outcome": "refused"}]
+    with pytest.raises(DepthCurveError, match="outside documented range"):
+        compute_depth_curve([deep_case], report)