Merge pull request #575 from AssetOverflow/feat/capability-index

feat(evals): AGI-roadmap Phase 1 — cross-domain capability index (MEASURE yardstick)

evals/capability_index/__init__.py

@@ -0,0 +1,18 @@
+"""Cross-domain capability index — the AGI-roadmap MEASURE step (Phase 1).
+
+The yardstick that gates every later "more capable" claim. It composes the
+independent-gold reasoning lanes into one report with honest, un-gameable axes:
+
+- **accuracy** — of *committed* answers; wrong stays 0 in assert mode.
+- **coverage** — attempted (not refused) fraction.
+- **coverage_geomean** — the headline: the geometric mean of per-domain coverage,
+  which only rises if EVERY domain rises. A narrow per-domain hack leaves it ~0.
+- **capability_score** — `coverage_geomean × accuracy`, hard-gated to 0 if any
+  domain committed a wrong answer (assert-mode invariant).
+
+This makes "general, not narrow" a number, and makes self-deception (gaming one
+lane) structurally visible. See
+``docs/analysis/AGI-candidacy-autonomous-improvement-roadmap-2026-06-05.md``.
+"""
+
+from __future__ import annotations

evals/capability_index/__main__.py

@@ -0,0 +1,31 @@
+"""On-demand: run the capability index over the composed lanes and print it.
+
+Run:  PYTHONPATH=. .venv/bin/python -m evals.capability_index
+
+Exits non-zero if the assert-mode invariant is violated (any domain committed a
+wrong answer). The printed ``deterministic_digest`` is the freeze handle — the
+baseline the autonomous-improvement loop must climb.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+
+from evals.capability_index.adapters import collect_domain_results
+from evals.capability_index.index import aggregate, index_to_dict
+
+
+def main() -> int:
+    collection = collect_domain_results()
+    index = aggregate(list(collection.results))
+    report = index_to_dict(index)
+    report["not_covered"] = [
+        {"adapter": name, "error": err} for name, err in collection.not_covered
+    ]
+    print(json.dumps(report, indent=2))
+    return 0 if index.assert_mode_valid else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

evals/capability_index/adapters.py

@@ -0,0 +1,68 @@
+"""Per-lane adapters — normalize each independent-gold lane to a DomainResult.
+
+These are thin COUNT extractors, not capability logic: each calls a lane's own
+self-loading runner and reads its correct/wrong/refused counts. A lane that fails
+to run is recorded as ``not_covered`` (no silent drop), never faked.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from evals.capability_index.index import DomainResult
+
+
+def _counts(report: dict) -> tuple[int, int, int]:
+    c = report.get("counts", report)
+    return int(c["correct"]), int(c["wrong"]), int(c["refused"])
+
+
+def deductive_logic_result() -> DomainResult:
+    from evals.deductive_logic.runner import build_combined_report
+
+    agg = build_combined_report()["aggregate"]  # {n, correct, wrong, refused}
+    return DomainResult(
+        "deductive_logic", int(agg["correct"]), int(agg["wrong"]), int(agg["refused"])
+    )
+
+
+def relational_metric_result() -> DomainResult:
+    from evals.relational_metric.runner import run
+
+    r = run()
+    return DomainResult(
+        "relational_metric", int(r["correct"]), int(r["wrong"]), int(r["refused"])
+    )
+
+
+def dimensional_result() -> DomainResult:
+    from evals.dimensional.runner import _ROOT, _load, build_report
+
+    correct, wrong, refused = _counts(build_report(_load(_ROOT / "v1" / "cases.jsonl")))
+    return DomainResult("dimensional", correct, wrong, refused)
+
+
+#: The reasoning domains currently composed into the index (self-loading lanes).
+ADAPTERS = (
+    deductive_logic_result,
+    relational_metric_result,
+    dimensional_result,
+)
+
+
+@dataclass(frozen=True, slots=True)
+class Collection:
+    results: tuple[DomainResult, ...]
+    not_covered: tuple[tuple[str, str], ...]  # (adapter_name, error) — no silent drop
+
+
+def collect_domain_results() -> Collection:
+    """Run every adapter; surface any that fail rather than dropping them."""
+    results: list[DomainResult] = []
+    not_covered: list[tuple[str, str]] = []
+    for adapter in ADAPTERS:
+        try:
+            results.append(adapter())
+        except Exception as exc:  # noqa: BLE001 — surfacing is the contract
+            not_covered.append((adapter.__name__, repr(exc)))
+    return Collection(results=tuple(results), not_covered=tuple(not_covered))

evals/capability_index/index.py

@@ -0,0 +1,153 @@
+"""The capability-index schema + pure aggregation (Phase 1 core).
+
+Pure functions over per-domain counts — no lane execution here (that is
+``adapters.py``), so the math is trivially testable and the anti-gaming property
+is provable in isolation.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class DomainResult:
+    """One domain's outcome counts on its independent-gold lane."""
+
+    domain: str
+    correct: int
+    wrong: int
+    refused: int
+
+    @property
+    def total(self) -> int:
+        return self.correct + self.wrong + self.refused
+
+    @property
+    def attempted(self) -> int:
+        """Committed an answer (not refused)."""
+        return self.correct + self.wrong
+
+    @property
+    def coverage(self) -> float:
+        """Fraction it was willing to answer."""
+        return self.attempted / self.total if self.total else 0.0
+
+    @property
+    def accuracy(self) -> float:
+        """Accuracy OF COMMITTED answers (1.0 when it commits nothing wrong)."""
+        return self.correct / self.attempted if self.attempted else 1.0
+
+
+@dataclass(frozen=True, slots=True)
+class CapabilityIndex:
+    domains: tuple[DomainResult, ...]
+
+    @property
+    def wrong_total(self) -> int:
+        return sum(d.wrong for d in self.domains)
+
+    @property
+    def assert_mode_valid(self) -> bool:
+        """Assert-mode invariant: zero wrong commits across all domains."""
+        return self.wrong_total == 0
+
+    @property
+    def _attempted(self) -> int:
+        return sum(d.attempted for d in self.domains)
+
+    @property
+    def _total(self) -> int:
+        return sum(d.total for d in self.domains)
+
+    @property
+    def coverage(self) -> float:
+        """Micro coverage across all cases."""
+        return self._attempted / self._total if self._total else 0.0
+
+    @property
+    def accuracy(self) -> float:
+        """Micro accuracy of committed answers."""
+        correct = sum(d.correct for d in self.domains)
+        return correct / self._attempted if self._attempted else 1.0
+
+    @property
+    def coverage_geomean(self) -> float:
+        """Geometric mean of per-domain coverage — the anti-gaming headline.
+
+        Zero if ANY domain has zero coverage, so a narrow per-domain win cannot
+        move it; it rises only when breadth rises. This is "general, not narrow"
+        as a number.
+        """
+        if not self.domains:
+            return 0.0
+        # geomean = exp(mean(log(coverage))); any 0 -> 0.
+        if any(d.coverage <= 0.0 for d in self.domains):
+            return 0.0
+        log_sum = sum(math.log(d.coverage) for d in self.domains)
+        return math.exp(log_sum / len(self.domains))
+
+    @property
+    def breadth(self) -> int:
+        """How many domains the engine covers at all."""
+        return sum(1 for d in self.domains if d.coverage > 0.0)
+
+    @property
+    def min_domain_coverage(self) -> float:
+        return min((d.coverage for d in self.domains), default=0.0)
+
+    @property
+    def capability_score(self) -> float:
+        """The single number: breadth-aware coverage × accuracy, hard-gated on
+        the assert-mode invariant (any wrong commit zeroes it)."""
+        if not self.assert_mode_valid:
+            return 0.0
+        return self.coverage_geomean * self.accuracy
+
+
+def aggregate(results: list[DomainResult]) -> CapabilityIndex:
+    """Aggregate per-domain results into the cross-domain index."""
+    return CapabilityIndex(domains=tuple(results))
+
+
+def deterministic_digest(index: CapabilityIndex) -> str:
+    """SHA-256 over the per-domain counts + verdict axes (reproducible)."""
+    payload = {
+        "domains": [
+            {"domain": d.domain, "correct": d.correct, "wrong": d.wrong, "refused": d.refused}
+            for d in sorted(index.domains, key=lambda d: d.domain)
+        ],
+        "wrong_total": index.wrong_total,
+        "assert_mode_valid": index.assert_mode_valid,
+    }
+    serialized = json.dumps(payload, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(serialized.encode("utf-8")).hexdigest()
+
+
+def index_to_dict(index: CapabilityIndex) -> dict:
+    """JSON-safe report view of the index."""
+    return {
+        "capability_score": round(index.capability_score, 6),
+        "coverage_geomean": round(index.coverage_geomean, 6),
+        "coverage_micro": round(index.coverage, 6),
+        "accuracy_micro": round(index.accuracy, 6),
+        "breadth": index.breadth,
+        "min_domain_coverage": round(index.min_domain_coverage, 6),
+        "wrong_total": index.wrong_total,
+        "assert_mode_valid": index.assert_mode_valid,
+        "deterministic_digest": deterministic_digest(index),
+        "domains": [
+            {
+                "domain": d.domain,
+                "correct": d.correct,
+                "wrong": d.wrong,
+                "refused": d.refused,
+                "coverage": round(d.coverage, 6),
+                "accuracy": round(d.accuracy, 6),
+            }
+            for d in sorted(index.domains, key=lambda d: d.domain)
+        ],
+    }

tests/test_capability_index.py

@@ -0,0 +1,114 @@
+"""Cross-domain capability index — AGI-roadmap Phase 1 (MEASURE).
+
+The yardstick that gates every later "more capable" claim. Two honest axes —
+**accuracy** (of committed answers; wrong stays 0 in assert mode) and **coverage**
+(attempted-not-refused) — aggregated across domains so it CANNOT be gamed by a
+narrow per-domain win: the headline coverage is the GEOMETRIC MEAN across domains,
+which only rises if *every* domain rises. A hack that maxes one lane and leaves
+the rest at zero leaves the geomean ~0.
+"""
+
+from __future__ import annotations
+
+from evals.capability_index.index import (
+    DomainResult,
+    aggregate,
+    deterministic_digest,
+)
+
+
+def _d(domain: str, correct: int, wrong: int, refused: int) -> DomainResult:
+    return DomainResult(domain=domain, correct=correct, wrong=wrong, refused=refused)
+
+
+def test_domain_result_axes() -> None:
+    r = _d("logic", correct=8, wrong=0, refused=2)
+    assert r.total == 10
+    assert r.attempted == 8
+    assert r.coverage == 0.8
+    assert r.accuracy == 1.0  # of committed answers
+
+
+def test_aggregate_axes_micro() -> None:
+    idx = aggregate([_d("a", 6, 0, 4), _d("b", 2, 0, 8)])
+    assert idx.wrong_total == 0
+    assert idx.coverage == 0.4  # (6+2)/(10+10) micro
+    assert idx.accuracy == 1.0  # no wrong
+    assert idx.breadth == 2  # both domains have some coverage
+
+
+def test_geomean_coverage_resists_narrow_gaming() -> None:
+    # A NARROW hack: one domain maxed, the rest at zero coverage.
+    narrow = aggregate(
+        [_d("gamed", 10, 0, 0), _d("x", 0, 0, 10), _d("y", 0, 0, 10)]
+    )
+    # A BALANCED engine: every domain partially covered.
+    balanced = aggregate(
+        [_d("gamed", 4, 0, 6), _d("x", 4, 0, 6), _d("y", 4, 0, 6)]
+    )
+    # Micro-coverage is similar (~0.33 vs 0.40), but the geomean exposes the hack:
+    assert narrow.coverage_geomean == 0.0  # any zero-coverage domain -> geomean 0
+    assert balanced.coverage_geomean > 0.39
+    # The capability score (geomean × accuracy) refuses to reward the narrow hack.
+    assert narrow.capability_score == 0.0
+    assert balanced.capability_score > 0.39
+
+
+def test_balanced_progress_moves_the_score_monotonically() -> None:
+    low = aggregate([_d("a", 2, 0, 8), _d("b", 2, 0, 8)])
+    high = aggregate([_d("a", 6, 0, 4), _d("b", 6, 0, 4)])
+    assert high.coverage_geomean > low.coverage_geomean
+    assert high.capability_score > low.capability_score
+
+
+def test_wrong_is_a_hard_gate() -> None:
+    # In assert mode wrong MUST be 0; any wrong invalidates the index (score 0)
+    # and is surfaced — never averaged away.
+    idx = aggregate([_d("a", 8, 1, 1), _d("b", 5, 0, 5)])
+    assert idx.wrong_total == 1
+    assert idx.assert_mode_valid is False
+    assert idx.capability_score == 0.0  # wrong=0 is non-negotiable in assert mode
+
+
+def test_digest_is_deterministic_and_bites() -> None:
+    a = aggregate([_d("a", 6, 0, 4), _d("b", 2, 0, 8)])
+    b = aggregate([_d("a", 6, 0, 4), _d("b", 2, 0, 8)])
+    assert deterministic_digest(a) == deterministic_digest(b)
+    moved = aggregate([_d("a", 7, 0, 3), _d("b", 2, 0, 8)])
+    assert deterministic_digest(moved) != deterministic_digest(a)
+
+
+def test_empty_index_is_well_defined() -> None:
+    idx = aggregate([])
+    assert idx.coverage == 0.0
+    assert idx.coverage_geomean == 0.0
+    assert idx.breadth == 0
+    assert idx.capability_score == 0.0
+
+
+def test_real_lanes_compose_into_the_index_with_wrong_zero() -> None:
+    # The Phase-1b baseline: the three self-loading independent-gold reasoning
+    # lanes compose into the cross-domain index with zero wrong commits.
+    from evals.capability_index.adapters import collect_domain_results
+
+    collection = collect_domain_results()
+    assert collection.not_covered == ()  # every adapter ran (no silent drop)
+    idx = aggregate(list(collection.results))
+    assert idx.wrong_total == 0
+    assert idx.assert_mode_valid
+    assert idx.breadth == 3  # deductive_logic + dimensional + relational_metric
+    assert {d.domain for d in idx.domains} == {
+        "deductive_logic",
+        "dimensional",
+        "relational_metric",
+    }
+    assert idx.capability_score > 0.5  # real, non-trivial cross-domain capability
+
+
+def test_index_report_is_deterministic_across_runs() -> None:
+    # The capability number is reproducible — improvement is a replayable curve.
+    from evals.capability_index.adapters import collect_domain_results
+
+    a = deterministic_digest(aggregate(list(collect_domain_results().results)))
+    b = deterministic_digest(aggregate(list(collect_domain_results().results)))
+    assert a == b