core/evals/capability_index/adapters.py

"""Per-lane adapters — normalize each independent-gold lane to a DomainResult.

These are thin COUNT extractors, not capability logic: each calls a lane's own
self-loading runner and reads its correct/wrong/refused counts. A lane that fails
to run is recorded as ``not_covered`` (no silent drop), never faked.
"""

from __future__ import annotations

from dataclasses import dataclass

from evals.capability_index.index import DomainResult


def _counts(report: dict) -> tuple[int, int, int]:
    c = report.get("counts", report)
    return int(c["correct"]), int(c["wrong"]), int(c["refused"])


def deductive_logic_result() -> DomainResult:
    from evals.deductive_logic.runner import build_combined_report

    agg = build_combined_report()["aggregate"]  # {n, correct, wrong, refused}
    return DomainResult(
        "deductive_logic", int(agg["correct"]), int(agg["wrong"]), int(agg["refused"])
    )


def relational_metric_result() -> DomainResult:
    from evals.relational_metric.runner import run

    r = run()
    return DomainResult(
        "relational_metric", int(r["correct"]), int(r["wrong"]), int(r["refused"])
    )


def dimensional_result() -> DomainResult:
    from evals.dimensional.runner import _ROOT, _load, build_report

    correct, wrong, refused = _counts(build_report(_load(_ROOT / "v1" / "cases.jsonl")))
    return DomainResult("dimensional", correct, wrong, refused)


def comprehension_set_membership_result() -> DomainResult:
    from evals.comprehension.set_membership_runner import run

    c, w, r = _counts(run())
    return DomainResult("comprehension_set_membership", c, w, r)


def comprehension_syllogism_result() -> DomainResult:
    from evals.comprehension.syllogism_runner import run

    c, w, r = _counts(run())
    return DomainResult("comprehension_syllogism", c, w, r)


def comprehension_total_ordering_result() -> DomainResult:
    from evals.comprehension.total_ordering_runner import run

    c, w, r = _counts(run())
    return DomainResult("comprehension_total_ordering", c, w, r)


def comprehension_propositional_result() -> DomainResult:
    from evals.comprehension.propositional_runner import run

    c, w, r = _counts(run())
    return DomainResult("comprehension_propositional", c, w, r)


def comprehension_relational_metric_result() -> DomainResult:
    from evals.comprehension.relational_metric_runner import run

    c, w, r = _counts(run())
    return DomainResult("comprehension_relational_metric", c, w, r)


def comprehension_relational_predicate_result() -> DomainResult:
    from evals.comprehension.relational_predicate_runner import run

    c, w, r = _counts(run())
    return DomainResult("comprehension_relational_predicate", c, w, r)


def comprehension_relational_inference_result() -> DomainResult:
    from evals.comprehension.relational_inference_runner import run

    c, w, r = _counts(run())
    return DomainResult("comprehension_relational_inference", c, w, r)


def comprehension_relational_transitive_result() -> DomainResult:
    from evals.relational_transitive.runner import run

    c, w, r = _counts(run())
    return DomainResult("comprehension_relational_transitive", c, w, r)


def deduction_serve_existential_result() -> DomainResult:
    """Band v6-EX (ADR-0261) alone — the existential-argument split of the
    deduction-serve lane, scored against its own independently-authored
    gold. Deliberately just this one split, not the full six-band lane: the
    other bands (v1/v1b propositional+categorical, v2-EN/v3-MEM/v4-CM/v5-VP)
    are pure grammar widenings over the SAME propositional-entailment
    substrate the ``deductive_logic`` adapter above already scores; the
    existential band is the one that changes the DECISION PROCEDURE itself
    (domain widening with Skolem witnesses / an arbitrary element — no
    existential import), so it is the one genuinely new reasoning capability
    among the six worth its own index domain."""
    from evals.deduction_serve.runner import build_combined_report

    counts = build_combined_report()["splits"]["v2_exist"]["counts"]
    return DomainResult(
        "deduction_serve_existential",
        int(counts["correct"]),
        int(counts["wrong"]),
        int(counts["declined"]),
    )


def curriculum_serve_result() -> DomainResult:
    """The curriculum-grounded serving lane (ADR-0262): exam questions
    answered from the ratified domain-chain curriculum alone, scored against
    an independent closed-world reachability oracle (``evals.curriculum_serve.oracle``,
    a different decision procedure than the serving path it checks)."""
    from evals.curriculum_serve.runner import build_combined_report

    agg = build_combined_report()["aggregate"]
    return DomainResult(
        "curriculum_serve", int(agg["correct"]), int(agg["wrong"]), int(agg["declined"])
    )


#: The reasoning domains currently composed into the index (self-loading lanes).
#: The six ``comprehension_*`` lanes score the GENERAL comprehension reader; the
#: relational_metric one reads arithmetic prose into the binding-graph quantity
#: substrate (admissibility-checked) and projects to the arithmetic oracle, and the
#: relational_predicate one (#596) reads binary-relation prose into pack-named
#: predicates, the relational_inference one DETERMINES one-hop inverse/symmetric
#: entailments (mastery-v2 Step 3), and the relational_transitive one DETERMINES
#: same-predicate transitive closure over the declared strict orders (mastery-v2 Step-3
#: Brief B2) — so the index measures comprehension breadth across categorical, ordering,
#: propositional, quantitative, relational-reading, one-hop relational-inference, AND
#: transitive relational-inference reasoning. ``deduction_serve_existential`` (ADR-0261)
#: and ``curriculum_serve`` (ADR-0262, generalization arc Tier S) add the PRODUCTION
#: serving deciders (not bare-engine lanes) for existential-domain widening and
#: curriculum-grounded exam answering.
ADAPTERS = (
    deductive_logic_result,
    relational_metric_result,
    dimensional_result,
    comprehension_set_membership_result,
    comprehension_syllogism_result,
    comprehension_total_ordering_result,
    comprehension_propositional_result,
    comprehension_relational_metric_result,
    comprehension_relational_predicate_result,
    comprehension_relational_inference_result,
    comprehension_relational_transitive_result,
    deduction_serve_existential_result,
    curriculum_serve_result,
)


@dataclass(frozen=True, slots=True)
class Collection:
    results: tuple[DomainResult, ...]
    not_covered: tuple[tuple[str, str], ...]  # (adapter_name, error) — no silent drop


def collect_domain_results() -> Collection:
    """Run every adapter; surface any that fail rather than dropping them."""
    results: list[DomainResult] = []
    not_covered: list[tuple[str, str]] = []
    for adapter in ADAPTERS:
        try:
            results.append(adapter())
        except Exception as exc:  # noqa: BLE001 — surfacing is the contract
            not_covered.append((adapter.__name__, repr(exc)))
    return Collection(results=tuple(results), not_covered=tuple(not_covered))