evals: freeze capability index baseline

evals/capability_index/baseline.json

@@ -0,0 +1,38 @@
+{
+  "capability_score": 0.919641,
+  "coverage_geomean": 0.919641,
+  "coverage_micro": 0.995962,
+  "accuracy_micro": 1.0,
+  "breadth": 3,
+  "min_domain_coverage": 0.833333,
+  "wrong_total": 0,
+  "assert_mode_valid": true,
+  "deterministic_digest": "115389d285babb5980b8b31c40d3a1c58ded1e1f9ae08ec428090bba18d42ed0",
+  "domains": [
+    {
+      "domain": "deductive_logic",
+      "correct": 716,
+      "wrong": 0,
+      "refused": 0,
+      "coverage": 1.0,
+      "accuracy": 1.0
+    },
+    {
+      "domain": "dimensional",
+      "correct": 10,
+      "wrong": 0,
+      "refused": 2,
+      "coverage": 0.833333,
+      "accuracy": 1.0
+    },
+    {
+      "domain": "relational_metric",
+      "correct": 14,
+      "wrong": 0,
+      "refused": 1,
+      "coverage": 0.933333,
+      "accuracy": 1.0
+    }
+  ],
+  "not_covered": []
+}

tests/test_capability_baseline.py

@@ -0,0 +1,65 @@
+"""Capability-index baseline freeze.
+
+The committed baseline is the monotonic handle for the autonomous-improvement
+loop. A digest change is intentional only when the baseline is deliberately
+re-frozen after an accepted improvement.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from evals.capability_index.adapters import collect_domain_results
+from evals.capability_index.index import (
+    DomainResult,
+    aggregate,
+    deterministic_digest,
+    index_to_dict,
+)
+
+
+_BASELINE = (
+    Path(__file__).resolve().parent.parent
+    / "evals"
+    / "capability_index"
+    / "baseline.json"
+)
+
+
+def _live_report() -> dict:
+    collection = collect_domain_results()
+    index = aggregate(list(collection.results))
+    report = index_to_dict(index)
+    report["not_covered"] = [
+        {"adapter": name, "error": err} for name, err in collection.not_covered
+    ]
+    return report
+
+
+def test_capability_index_matches_frozen_baseline_digest() -> None:
+    baseline = json.loads(_BASELINE.read_text(encoding="utf-8"))
+    live = _live_report()
+
+    assert live["wrong_total"] == 0
+    assert live["assert_mode_valid"] is True
+    assert live["not_covered"] == []
+    assert live["deterministic_digest"] == baseline["deterministic_digest"]
+
+
+def test_capability_baseline_digest_is_the_index_digest() -> None:
+    baseline = json.loads(_BASELINE.read_text(encoding="utf-8"))
+    rebuilt = aggregate([
+        # Rebuild only from the frozen domain counts; rounded presentation
+        # fields are intentionally ignored by deterministic_digest.
+        DomainResult(
+            domain=d["domain"],
+            correct=int(d["correct"]),
+            wrong=int(d["wrong"]),
+            refused=int(d["refused"]),
+        )
+        for d in baseline["domains"]
+    ])
+
+    assert baseline["wrong_total"] == 0
+    assert baseline["deterministic_digest"] == deterministic_digest(rebuilt)