a005a92fed
The binding-graph's FIRST comprehension consumer (doctrine-aligned: quantities live in binding_graph, NOT the MeaningGraph). generate/quantitative_comprehension.py reads arithmetic prose into SymbolBinding/BoundFact/BoundEquation and runs the REAL check_admissibility (shell -> verify -> rebuild with the actual UnitProof) — there is NO stamped "admitted": an equation is admitted only if its operand units verify. Then to_relational_metric projects the binding-graph to the independent relational_metric oracle for the verdict. Templates (digits only; non-digit quantity REFUSES): "<X> has <N> <unit>" -> BoundFact(X = N) "<Y> has <N> more <unit> than <X>" -> BoundEquation(Y = X + N) op=add "<Y> has <N> fewer <unit> than <X>" -> BoundEquation(Y = X - N) op=subtract "How many <unit> does <Y> have" -> ask Y "How many <unit> do <X> and <Y> have"-> total = X + Y; ask total Unit modelling (honest, not faked): a noun the closed en_units_v1 pack knows is used verbatim (dollars -> dollar/money); an UNKNOWN sortal noun (stickers, coins) is a count of discrete objects -> the existing 'item' lemma (dimension count). So admissibility stays a REAL check: count+count admits, count+money (a mixed-unit sum) REFUSES with unit_mismatch — verified to bite. comprehension_relational_metric: 15/15 wrong=0 (full coverage). Located OUTSIDE generate/meaning_graph (it targets binding_graph, not the MeaningGraph) so INV-28 neutrality stays intact; oracle imports none of the SUT (new INV-25 lane). Capability index breadth 7->8, score 0.928622 -> 0.937258, wrong_total 0, digest 50e0675b… Tests: reader templates + count/known-unit modelling + admissibility-bite (mixed unit refuses) + non-digit refusal; end-to-end full-coverage wrong=0; arithmetic added to the structure-preservation generative panel (projected relations+query == ground truth); capability breadth 7->8; INV-25 arithmetic lane. 93 targeted + 90 smoke green; lane SHAs 8/9 (sole miss = public_demo env flake; deductive_logic + math_teaching unchanged -> no GSM8K coupling).
113 lines
3.8 KiB
Python
113 lines
3.8 KiB
Python
"""Per-lane adapters — normalize each independent-gold lane to a DomainResult.
|
|
|
|
These are thin COUNT extractors, not capability logic: each calls a lane's own
|
|
self-loading runner and reads its correct/wrong/refused counts. A lane that fails
|
|
to run is recorded as ``not_covered`` (no silent drop), never faked.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
|
|
from evals.capability_index.index import DomainResult
|
|
|
|
|
|
def _counts(report: dict) -> tuple[int, int, int]:
|
|
c = report.get("counts", report)
|
|
return int(c["correct"]), int(c["wrong"]), int(c["refused"])
|
|
|
|
|
|
def deductive_logic_result() -> DomainResult:
|
|
from evals.deductive_logic.runner import build_combined_report
|
|
|
|
agg = build_combined_report()["aggregate"] # {n, correct, wrong, refused}
|
|
return DomainResult(
|
|
"deductive_logic", int(agg["correct"]), int(agg["wrong"]), int(agg["refused"])
|
|
)
|
|
|
|
|
|
def relational_metric_result() -> DomainResult:
|
|
from evals.relational_metric.runner import run
|
|
|
|
r = run()
|
|
return DomainResult(
|
|
"relational_metric", int(r["correct"]), int(r["wrong"]), int(r["refused"])
|
|
)
|
|
|
|
|
|
def dimensional_result() -> DomainResult:
|
|
from evals.dimensional.runner import _ROOT, _load, build_report
|
|
|
|
correct, wrong, refused = _counts(build_report(_load(_ROOT / "v1" / "cases.jsonl")))
|
|
return DomainResult("dimensional", correct, wrong, refused)
|
|
|
|
|
|
def comprehension_set_membership_result() -> DomainResult:
|
|
from evals.comprehension.set_membership_runner import run
|
|
|
|
c, w, r = _counts(run())
|
|
return DomainResult("comprehension_set_membership", c, w, r)
|
|
|
|
|
|
def comprehension_syllogism_result() -> DomainResult:
|
|
from evals.comprehension.syllogism_runner import run
|
|
|
|
c, w, r = _counts(run())
|
|
return DomainResult("comprehension_syllogism", c, w, r)
|
|
|
|
|
|
def comprehension_total_ordering_result() -> DomainResult:
|
|
from evals.comprehension.total_ordering_runner import run
|
|
|
|
c, w, r = _counts(run())
|
|
return DomainResult("comprehension_total_ordering", c, w, r)
|
|
|
|
|
|
def comprehension_propositional_result() -> DomainResult:
|
|
from evals.comprehension.propositional_runner import run
|
|
|
|
c, w, r = _counts(run())
|
|
return DomainResult("comprehension_propositional", c, w, r)
|
|
|
|
|
|
def comprehension_relational_metric_result() -> DomainResult:
|
|
from evals.comprehension.relational_metric_runner import run
|
|
|
|
c, w, r = _counts(run())
|
|
return DomainResult("comprehension_relational_metric", c, w, r)
|
|
|
|
|
|
#: The reasoning domains currently composed into the index (self-loading lanes).
|
|
#: The five ``comprehension_*`` lanes score the GENERAL comprehension reader; the
|
|
#: relational_metric one reads arithmetic prose into the binding-graph quantity
|
|
#: substrate (admissibility-checked) and projects to the arithmetic oracle, so the
|
|
#: index now measures comprehension breadth across categorical, ordering,
|
|
#: propositional, AND quantitative reasoning.
|
|
ADAPTERS = (
|
|
deductive_logic_result,
|
|
relational_metric_result,
|
|
dimensional_result,
|
|
comprehension_set_membership_result,
|
|
comprehension_syllogism_result,
|
|
comprehension_total_ordering_result,
|
|
comprehension_propositional_result,
|
|
comprehension_relational_metric_result,
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class Collection:
|
|
results: tuple[DomainResult, ...]
|
|
not_covered: tuple[tuple[str, str], ...] # (adapter_name, error) — no silent drop
|
|
|
|
|
|
def collect_domain_results() -> Collection:
|
|
"""Run every adapter; surface any that fail rather than dropping them."""
|
|
results: list[DomainResult] = []
|
|
not_covered: list[tuple[str, str]] = []
|
|
for adapter in ADAPTERS:
|
|
try:
|
|
results.append(adapter())
|
|
except Exception as exc: # noqa: BLE001 — surfacing is the contract
|
|
not_covered.append((adapter.__name__, repr(exc)))
|
|
return Collection(results=tuple(results), not_covered=tuple(not_covered))
|