514c6c52ca
The instrument that gates every later "more capable" claim and makes "general, not narrow" a number. evals/capability_index/ composes the self-loading independent-gold reasoning lanes (deductive_logic, dimensional, relational_metric) into one report with honest, un-gameable axes: - accuracy (of committed answers; wrong stays 0 in assert mode), - coverage (attempted-not-refused), - coverage_geomean — the headline: geometric mean of per-domain coverage, which is 0 if ANY domain has zero coverage, so a narrow per-domain win cannot move it; it rises only when breadth rises, - capability_score = coverage_geomean × accuracy, HARD-GATED to 0 if any domain committed a wrong answer (assert-mode invariant), - a deterministic digest (the replayable baseline the autonomous loop must climb). Baseline (today): score 0.9196, accuracy 1.0, breadth 3, wrong_total 0 — high because all three composed lanes are formal/structured; when comprehension-gated NL domains join, the geomean will honestly drop to expose the breadth gap (the instrument working). Adapters surface any lane that fails to run as not_covered — no silent drop (proven: it caught a deductive-report shape mismatch mid-build). Pure aggregation + the geomean anti-gaming property + the wrong=0 hard gate are unit-tested; a real-composition integration test asserts wrong=0 + breadth=3. 10 tests + 52 architectural invariants pass. Additive (new evals/ package). Part of docs/analysis/AGI-candidacy-autonomous-improvement-roadmap-2026-06-05.md (Phase 1).
68 lines
2.2 KiB
Python
68 lines
2.2 KiB
Python
"""Per-lane adapters — normalize each independent-gold lane to a DomainResult.
|
|
|
|
These are thin COUNT extractors, not capability logic: each calls a lane's own
|
|
self-loading runner and reads its correct/wrong/refused counts. A lane that fails
|
|
to run is recorded as ``not_covered`` (no silent drop), never faked.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
|
|
from evals.capability_index.index import DomainResult
|
|
|
|
|
|
def _counts(report: dict) -> tuple[int, int, int]:
|
|
c = report.get("counts", report)
|
|
return int(c["correct"]), int(c["wrong"]), int(c["refused"])
|
|
|
|
|
|
def deductive_logic_result() -> DomainResult:
|
|
from evals.deductive_logic.runner import build_combined_report
|
|
|
|
agg = build_combined_report()["aggregate"] # {n, correct, wrong, refused}
|
|
return DomainResult(
|
|
"deductive_logic", int(agg["correct"]), int(agg["wrong"]), int(agg["refused"])
|
|
)
|
|
|
|
|
|
def relational_metric_result() -> DomainResult:
|
|
from evals.relational_metric.runner import run
|
|
|
|
r = run()
|
|
return DomainResult(
|
|
"relational_metric", int(r["correct"]), int(r["wrong"]), int(r["refused"])
|
|
)
|
|
|
|
|
|
def dimensional_result() -> DomainResult:
|
|
from evals.dimensional.runner import _ROOT, _load, build_report
|
|
|
|
correct, wrong, refused = _counts(build_report(_load(_ROOT / "v1" / "cases.jsonl")))
|
|
return DomainResult("dimensional", correct, wrong, refused)
|
|
|
|
|
|
#: The reasoning domains currently composed into the index (self-loading lanes).
|
|
ADAPTERS = (
|
|
deductive_logic_result,
|
|
relational_metric_result,
|
|
dimensional_result,
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class Collection:
|
|
results: tuple[DomainResult, ...]
|
|
not_covered: tuple[tuple[str, str], ...] # (adapter_name, error) — no silent drop
|
|
|
|
|
|
def collect_domain_results() -> Collection:
|
|
"""Run every adapter; surface any that fail rather than dropping them."""
|
|
results: list[DomainResult] = []
|
|
not_covered: list[tuple[str, str]] = []
|
|
for adapter in ADAPTERS:
|
|
try:
|
|
results.append(adapter())
|
|
except Exception as exc: # noqa: BLE001 — surfacing is the contract
|
|
not_covered.append((adapter.__name__, repr(exc)))
|
|
return Collection(results=tuple(results), not_covered=tuple(not_covered))
|