diff --git a/evals/capability_index/baseline.json b/evals/capability_index/baseline.json new file mode 100644 index 00000000..31d5c81b --- /dev/null +++ b/evals/capability_index/baseline.json @@ -0,0 +1,38 @@ +{ + "capability_score": 0.919641, + "coverage_geomean": 0.919641, + "coverage_micro": 0.995962, + "accuracy_micro": 1.0, + "breadth": 3, + "min_domain_coverage": 0.833333, + "wrong_total": 0, + "assert_mode_valid": true, + "deterministic_digest": "115389d285babb5980b8b31c40d3a1c58ded1e1f9ae08ec428090bba18d42ed0", + "domains": [ + { + "domain": "deductive_logic", + "correct": 716, + "wrong": 0, + "refused": 0, + "coverage": 1.0, + "accuracy": 1.0 + }, + { + "domain": "dimensional", + "correct": 10, + "wrong": 0, + "refused": 2, + "coverage": 0.833333, + "accuracy": 1.0 + }, + { + "domain": "relational_metric", + "correct": 14, + "wrong": 0, + "refused": 1, + "coverage": 0.933333, + "accuracy": 1.0 + } + ], + "not_covered": [] +} diff --git a/tests/test_capability_baseline.py b/tests/test_capability_baseline.py new file mode 100644 index 00000000..6866ce22 --- /dev/null +++ b/tests/test_capability_baseline.py @@ -0,0 +1,65 @@ +"""Capability-index baseline freeze. + +The committed baseline is the monotonic handle for the autonomous-improvement +loop. A digest change is intentional only when the baseline is deliberately +re-frozen after an accepted improvement. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from evals.capability_index.adapters import collect_domain_results +from evals.capability_index.index import ( + DomainResult, + aggregate, + deterministic_digest, + index_to_dict, +) + + +_BASELINE = ( + Path(__file__).resolve().parent.parent + / "evals" + / "capability_index" + / "baseline.json" +) + + +def _live_report() -> dict: + collection = collect_domain_results() + index = aggregate(list(collection.results)) + report = index_to_dict(index) + report["not_covered"] = [ + {"adapter": name, "error": err} for name, err in collection.not_covered + ] + return report + + +def test_capability_index_matches_frozen_baseline_digest() -> None: + baseline = json.loads(_BASELINE.read_text(encoding="utf-8")) + live = _live_report() + + assert live["wrong_total"] == 0 + assert live["assert_mode_valid"] is True + assert live["not_covered"] == [] + assert live["deterministic_digest"] == baseline["deterministic_digest"] + + +def test_capability_baseline_digest_is_the_index_digest() -> None: + baseline = json.loads(_BASELINE.read_text(encoding="utf-8")) + rebuilt = aggregate([ + # Rebuild only from the frozen domain counts; rounded presentation + # fields are intentionally ignored by deterministic_digest. + DomainResult( + domain=d["domain"], + correct=int(d["correct"]), + wrong=int(d["wrong"]), + refused=int(d["refused"]), + ) + for d in baseline["domains"] + ]) + + assert baseline["wrong_total"] == 0 + assert baseline["deterministic_digest"] == deterministic_digest(rebuilt)