diff --git a/evals/capability_index/__init__.py b/evals/capability_index/__init__.py new file mode 100644 index 00000000..bd162234 --- /dev/null +++ b/evals/capability_index/__init__.py @@ -0,0 +1,18 @@ +"""Cross-domain capability index — the AGI-roadmap MEASURE step (Phase 1). + +The yardstick that gates every later "more capable" claim. It composes the +independent-gold reasoning lanes into one report with honest, un-gameable axes: + +- **accuracy** — of *committed* answers; wrong stays 0 in assert mode. +- **coverage** — attempted (not refused) fraction. +- **coverage_geomean** — the headline: the geometric mean of per-domain coverage, + which only rises if EVERY domain rises. A narrow per-domain hack leaves it ~0. +- **capability_score** — `coverage_geomean × accuracy`, hard-gated to 0 if any + domain committed a wrong answer (assert-mode invariant). + +This makes "general, not narrow" a number, and makes self-deception (gaming one +lane) structurally visible. See +``docs/analysis/AGI-candidacy-autonomous-improvement-roadmap-2026-06-05.md``. +""" + +from __future__ import annotations diff --git a/evals/capability_index/__main__.py b/evals/capability_index/__main__.py new file mode 100644 index 00000000..89f57805 --- /dev/null +++ b/evals/capability_index/__main__.py @@ -0,0 +1,31 @@ +"""On-demand: run the capability index over the composed lanes and print it. + +Run: PYTHONPATH=. .venv/bin/python -m evals.capability_index + +Exits non-zero if the assert-mode invariant is violated (any domain committed a +wrong answer). The printed ``deterministic_digest`` is the freeze handle — the +baseline the autonomous-improvement loop must climb. +""" + +from __future__ import annotations + +import json +import sys + +from evals.capability_index.adapters import collect_domain_results +from evals.capability_index.index import aggregate, index_to_dict + + +def main() -> int: + collection = collect_domain_results() + index = aggregate(list(collection.results)) + report = index_to_dict(index) + report["not_covered"] = [ + {"adapter": name, "error": err} for name, err in collection.not_covered + ] + print(json.dumps(report, indent=2)) + return 0 if index.assert_mode_valid else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/capability_index/adapters.py b/evals/capability_index/adapters.py new file mode 100644 index 00000000..dc9fa6fb --- /dev/null +++ b/evals/capability_index/adapters.py @@ -0,0 +1,68 @@ +"""Per-lane adapters — normalize each independent-gold lane to a DomainResult. + +These are thin COUNT extractors, not capability logic: each calls a lane's own +self-loading runner and reads its correct/wrong/refused counts. A lane that fails +to run is recorded as ``not_covered`` (no silent drop), never faked. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from evals.capability_index.index import DomainResult + + +def _counts(report: dict) -> tuple[int, int, int]: + c = report.get("counts", report) + return int(c["correct"]), int(c["wrong"]), int(c["refused"]) + + +def deductive_logic_result() -> DomainResult: + from evals.deductive_logic.runner import build_combined_report + + agg = build_combined_report()["aggregate"] # {n, correct, wrong, refused} + return DomainResult( + "deductive_logic", int(agg["correct"]), int(agg["wrong"]), int(agg["refused"]) + ) + + +def relational_metric_result() -> DomainResult: + from evals.relational_metric.runner import run + + r = run() + return DomainResult( + "relational_metric", int(r["correct"]), int(r["wrong"]), int(r["refused"]) + ) + + +def dimensional_result() -> DomainResult: + from evals.dimensional.runner import _ROOT, _load, build_report + + correct, wrong, refused = _counts(build_report(_load(_ROOT / "v1" / "cases.jsonl"))) + return DomainResult("dimensional", correct, wrong, refused) + + +#: The reasoning domains currently composed into the index (self-loading lanes). +ADAPTERS = ( + deductive_logic_result, + relational_metric_result, + dimensional_result, +) + + +@dataclass(frozen=True, slots=True) +class Collection: + results: tuple[DomainResult, ...] + not_covered: tuple[tuple[str, str], ...] # (adapter_name, error) — no silent drop + + +def collect_domain_results() -> Collection: + """Run every adapter; surface any that fail rather than dropping them.""" + results: list[DomainResult] = [] + not_covered: list[tuple[str, str]] = [] + for adapter in ADAPTERS: + try: + results.append(adapter()) + except Exception as exc: # noqa: BLE001 — surfacing is the contract + not_covered.append((adapter.__name__, repr(exc))) + return Collection(results=tuple(results), not_covered=tuple(not_covered)) diff --git a/evals/capability_index/index.py b/evals/capability_index/index.py new file mode 100644 index 00000000..79bb054a --- /dev/null +++ b/evals/capability_index/index.py @@ -0,0 +1,153 @@ +"""The capability-index schema + pure aggregation (Phase 1 core). + +Pure functions over per-domain counts — no lane execution here (that is +``adapters.py``), so the math is trivially testable and the anti-gaming property +is provable in isolation. +""" + +from __future__ import annotations + +import hashlib +import json +import math +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class DomainResult: + """One domain's outcome counts on its independent-gold lane.""" + + domain: str + correct: int + wrong: int + refused: int + + @property + def total(self) -> int: + return self.correct + self.wrong + self.refused + + @property + def attempted(self) -> int: + """Committed an answer (not refused).""" + return self.correct + self.wrong + + @property + def coverage(self) -> float: + """Fraction it was willing to answer.""" + return self.attempted / self.total if self.total else 0.0 + + @property + def accuracy(self) -> float: + """Accuracy OF COMMITTED answers (1.0 when it commits nothing wrong).""" + return self.correct / self.attempted if self.attempted else 1.0 + + +@dataclass(frozen=True, slots=True) +class CapabilityIndex: + domains: tuple[DomainResult, ...] + + @property + def wrong_total(self) -> int: + return sum(d.wrong for d in self.domains) + + @property + def assert_mode_valid(self) -> bool: + """Assert-mode invariant: zero wrong commits across all domains.""" + return self.wrong_total == 0 + + @property + def _attempted(self) -> int: + return sum(d.attempted for d in self.domains) + + @property + def _total(self) -> int: + return sum(d.total for d in self.domains) + + @property + def coverage(self) -> float: + """Micro coverage across all cases.""" + return self._attempted / self._total if self._total else 0.0 + + @property + def accuracy(self) -> float: + """Micro accuracy of committed answers.""" + correct = sum(d.correct for d in self.domains) + return correct / self._attempted if self._attempted else 1.0 + + @property + def coverage_geomean(self) -> float: + """Geometric mean of per-domain coverage — the anti-gaming headline. + + Zero if ANY domain has zero coverage, so a narrow per-domain win cannot + move it; it rises only when breadth rises. This is "general, not narrow" + as a number. + """ + if not self.domains: + return 0.0 + # geomean = exp(mean(log(coverage))); any 0 -> 0. + if any(d.coverage <= 0.0 for d in self.domains): + return 0.0 + log_sum = sum(math.log(d.coverage) for d in self.domains) + return math.exp(log_sum / len(self.domains)) + + @property + def breadth(self) -> int: + """How many domains the engine covers at all.""" + return sum(1 for d in self.domains if d.coverage > 0.0) + + @property + def min_domain_coverage(self) -> float: + return min((d.coverage for d in self.domains), default=0.0) + + @property + def capability_score(self) -> float: + """The single number: breadth-aware coverage × accuracy, hard-gated on + the assert-mode invariant (any wrong commit zeroes it).""" + if not self.assert_mode_valid: + return 0.0 + return self.coverage_geomean * self.accuracy + + +def aggregate(results: list[DomainResult]) -> CapabilityIndex: + """Aggregate per-domain results into the cross-domain index.""" + return CapabilityIndex(domains=tuple(results)) + + +def deterministic_digest(index: CapabilityIndex) -> str: + """SHA-256 over the per-domain counts + verdict axes (reproducible).""" + payload = { + "domains": [ + {"domain": d.domain, "correct": d.correct, "wrong": d.wrong, "refused": d.refused} + for d in sorted(index.domains, key=lambda d: d.domain) + ], + "wrong_total": index.wrong_total, + "assert_mode_valid": index.assert_mode_valid, + } + serialized = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(serialized.encode("utf-8")).hexdigest() + + +def index_to_dict(index: CapabilityIndex) -> dict: + """JSON-safe report view of the index.""" + return { + "capability_score": round(index.capability_score, 6), + "coverage_geomean": round(index.coverage_geomean, 6), + "coverage_micro": round(index.coverage, 6), + "accuracy_micro": round(index.accuracy, 6), + "breadth": index.breadth, + "min_domain_coverage": round(index.min_domain_coverage, 6), + "wrong_total": index.wrong_total, + "assert_mode_valid": index.assert_mode_valid, + "deterministic_digest": deterministic_digest(index), + "domains": [ + { + "domain": d.domain, + "correct": d.correct, + "wrong": d.wrong, + "refused": d.refused, + "coverage": round(d.coverage, 6), + "accuracy": round(d.accuracy, 6), + } + for d in sorted(index.domains, key=lambda d: d.domain) + ], + } diff --git a/tests/test_capability_index.py b/tests/test_capability_index.py new file mode 100644 index 00000000..ac03754e --- /dev/null +++ b/tests/test_capability_index.py @@ -0,0 +1,114 @@ +"""Cross-domain capability index — AGI-roadmap Phase 1 (MEASURE). + +The yardstick that gates every later "more capable" claim. Two honest axes — +**accuracy** (of committed answers; wrong stays 0 in assert mode) and **coverage** +(attempted-not-refused) — aggregated across domains so it CANNOT be gamed by a +narrow per-domain win: the headline coverage is the GEOMETRIC MEAN across domains, +which only rises if *every* domain rises. A hack that maxes one lane and leaves +the rest at zero leaves the geomean ~0. +""" + +from __future__ import annotations + +from evals.capability_index.index import ( + DomainResult, + aggregate, + deterministic_digest, +) + + +def _d(domain: str, correct: int, wrong: int, refused: int) -> DomainResult: + return DomainResult(domain=domain, correct=correct, wrong=wrong, refused=refused) + + +def test_domain_result_axes() -> None: + r = _d("logic", correct=8, wrong=0, refused=2) + assert r.total == 10 + assert r.attempted == 8 + assert r.coverage == 0.8 + assert r.accuracy == 1.0 # of committed answers + + +def test_aggregate_axes_micro() -> None: + idx = aggregate([_d("a", 6, 0, 4), _d("b", 2, 0, 8)]) + assert idx.wrong_total == 0 + assert idx.coverage == 0.4 # (6+2)/(10+10) micro + assert idx.accuracy == 1.0 # no wrong + assert idx.breadth == 2 # both domains have some coverage + + +def test_geomean_coverage_resists_narrow_gaming() -> None: + # A NARROW hack: one domain maxed, the rest at zero coverage. + narrow = aggregate( + [_d("gamed", 10, 0, 0), _d("x", 0, 0, 10), _d("y", 0, 0, 10)] + ) + # A BALANCED engine: every domain partially covered. + balanced = aggregate( + [_d("gamed", 4, 0, 6), _d("x", 4, 0, 6), _d("y", 4, 0, 6)] + ) + # Micro-coverage is similar (~0.33 vs 0.40), but the geomean exposes the hack: + assert narrow.coverage_geomean == 0.0 # any zero-coverage domain -> geomean 0 + assert balanced.coverage_geomean > 0.39 + # The capability score (geomean × accuracy) refuses to reward the narrow hack. + assert narrow.capability_score == 0.0 + assert balanced.capability_score > 0.39 + + +def test_balanced_progress_moves_the_score_monotonically() -> None: + low = aggregate([_d("a", 2, 0, 8), _d("b", 2, 0, 8)]) + high = aggregate([_d("a", 6, 0, 4), _d("b", 6, 0, 4)]) + assert high.coverage_geomean > low.coverage_geomean + assert high.capability_score > low.capability_score + + +def test_wrong_is_a_hard_gate() -> None: + # In assert mode wrong MUST be 0; any wrong invalidates the index (score 0) + # and is surfaced — never averaged away. + idx = aggregate([_d("a", 8, 1, 1), _d("b", 5, 0, 5)]) + assert idx.wrong_total == 1 + assert idx.assert_mode_valid is False + assert idx.capability_score == 0.0 # wrong=0 is non-negotiable in assert mode + + +def test_digest_is_deterministic_and_bites() -> None: + a = aggregate([_d("a", 6, 0, 4), _d("b", 2, 0, 8)]) + b = aggregate([_d("a", 6, 0, 4), _d("b", 2, 0, 8)]) + assert deterministic_digest(a) == deterministic_digest(b) + moved = aggregate([_d("a", 7, 0, 3), _d("b", 2, 0, 8)]) + assert deterministic_digest(moved) != deterministic_digest(a) + + +def test_empty_index_is_well_defined() -> None: + idx = aggregate([]) + assert idx.coverage == 0.0 + assert idx.coverage_geomean == 0.0 + assert idx.breadth == 0 + assert idx.capability_score == 0.0 + + +def test_real_lanes_compose_into_the_index_with_wrong_zero() -> None: + # The Phase-1b baseline: the three self-loading independent-gold reasoning + # lanes compose into the cross-domain index with zero wrong commits. + from evals.capability_index.adapters import collect_domain_results + + collection = collect_domain_results() + assert collection.not_covered == () # every adapter ran (no silent drop) + idx = aggregate(list(collection.results)) + assert idx.wrong_total == 0 + assert idx.assert_mode_valid + assert idx.breadth == 3 # deductive_logic + dimensional + relational_metric + assert {d.domain for d in idx.domains} == { + "deductive_logic", + "dimensional", + "relational_metric", + } + assert idx.capability_score > 0.5 # real, non-trivial cross-domain capability + + +def test_index_report_is_deterministic_across_runs() -> None: + # The capability number is reproducible — improvement is a replayable curve. + from evals.capability_index.adapters import collect_domain_results + + a = deterministic_digest(aggregate(list(collect_domain_results().results))) + b = deterministic_digest(aggregate(list(collect_domain_results().results))) + assert a == b