f66f2ee47f
Adds comprehension_propositional — the comprehension organ now reads the classic
propositional ARGUMENT FORMS end-to-end into the flagship deductive_logic ROBDD
oracle (the most robustly independent gold in the repo). The neutral MeaningGraph
now feeds FOUR independent oracles (set-membership, syllogism-validity,
total-ordering, propositional-entailment) from one interlingua — the Option-B
interlingua thesis validated.
reader.py: propositional templates (atoms are chunked NP ids; fits the existing
entities + n-ary relations + negation model — NO interlingua change, propositional
is not arithmetic-quantities):
- "if <P> then <Q>" -> implies(P, Q)
- "not <P>" -> asserted(P, negated=True)
- "<P> or <Q>" -> or(P, Q)
- "<P>" (single token) -> asserted(P) (bare-atom, single-token only to
keep the parse-or-refuse floor)
- "therefore <prop>" -> query of the same predicate
Relations now carry a negated flag end-to-end (asserted negation).
projectors.py: to_deductive_logic serializes propositional relations/query into
formula strings (keyword operators the oracle tokenizer accepts); returns None
(refusal) unless the comprehension is purely propositional, so categorical/ordering
comprehensions never leak into the entailment oracle.
evals: new evals/propositional_logic/v1 (12 cases — modus ponens/tollens,
hypothetical & disjunctive syllogism, the affirming-consequent / denying-antecedent
fallacies which the oracle marks "unknown"; gold = oracle verdict) + gold-only
runner + evals/comprehension/propositional_runner.py. Oracle "refused" (formula
unevaluable) is treated as a decline, never a wrong.
Scores: comprehension_propositional 12/12 wrong=0 (full coverage); no regression on
the 3 existing lanes (8/8, 7/8, 7/8). Capability index breadth 6->7, score
0.917231 -> 0.928622, wrong_total 0, digest 51df7bba…
Tests: reader propositional templates; to_deductive_logic projector tests;
end-to-end full-coverage wrong=0; propositional generative round-trip added to the
wrong=0 property suite (verified to BITE under a reversed-implies mutation);
capability breadth 6->7. 115 targeted + 87 smoke green. Lane SHAs 8/9 (sole miss =
public_demo env wall-clock flake; deductive_logic_v1 unchanged).
103 lines
3.4 KiB
Python
103 lines
3.4 KiB
Python
"""Per-lane adapters — normalize each independent-gold lane to a DomainResult.
|
|
|
|
These are thin COUNT extractors, not capability logic: each calls a lane's own
|
|
self-loading runner and reads its correct/wrong/refused counts. A lane that fails
|
|
to run is recorded as ``not_covered`` (no silent drop), never faked.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
|
|
from evals.capability_index.index import DomainResult
|
|
|
|
|
|
def _counts(report: dict) -> tuple[int, int, int]:
|
|
c = report.get("counts", report)
|
|
return int(c["correct"]), int(c["wrong"]), int(c["refused"])
|
|
|
|
|
|
def deductive_logic_result() -> DomainResult:
|
|
from evals.deductive_logic.runner import build_combined_report
|
|
|
|
agg = build_combined_report()["aggregate"] # {n, correct, wrong, refused}
|
|
return DomainResult(
|
|
"deductive_logic", int(agg["correct"]), int(agg["wrong"]), int(agg["refused"])
|
|
)
|
|
|
|
|
|
def relational_metric_result() -> DomainResult:
|
|
from evals.relational_metric.runner import run
|
|
|
|
r = run()
|
|
return DomainResult(
|
|
"relational_metric", int(r["correct"]), int(r["wrong"]), int(r["refused"])
|
|
)
|
|
|
|
|
|
def dimensional_result() -> DomainResult:
|
|
from evals.dimensional.runner import _ROOT, _load, build_report
|
|
|
|
correct, wrong, refused = _counts(build_report(_load(_ROOT / "v1" / "cases.jsonl")))
|
|
return DomainResult("dimensional", correct, wrong, refused)
|
|
|
|
|
|
def comprehension_set_membership_result() -> DomainResult:
|
|
from evals.comprehension.set_membership_runner import run
|
|
|
|
c, w, r = _counts(run())
|
|
return DomainResult("comprehension_set_membership", c, w, r)
|
|
|
|
|
|
def comprehension_syllogism_result() -> DomainResult:
|
|
from evals.comprehension.syllogism_runner import run
|
|
|
|
c, w, r = _counts(run())
|
|
return DomainResult("comprehension_syllogism", c, w, r)
|
|
|
|
|
|
def comprehension_total_ordering_result() -> DomainResult:
|
|
from evals.comprehension.total_ordering_runner import run
|
|
|
|
c, w, r = _counts(run())
|
|
return DomainResult("comprehension_total_ordering", c, w, r)
|
|
|
|
|
|
def comprehension_propositional_result() -> DomainResult:
|
|
from evals.comprehension.propositional_runner import run
|
|
|
|
c, w, r = _counts(run())
|
|
return DomainResult("comprehension_propositional", c, w, r)
|
|
|
|
|
|
#: The reasoning domains currently composed into the index (self-loading lanes).
|
|
#: The four ``comprehension_*`` lanes score the GENERAL comprehension reader
|
|
#: (prose -> MeaningGraph -> projection -> independent oracle), so the index now
|
|
#: measures comprehension breadth, not just structured-input reasoning.
|
|
ADAPTERS = (
|
|
deductive_logic_result,
|
|
relational_metric_result,
|
|
dimensional_result,
|
|
comprehension_set_membership_result,
|
|
comprehension_syllogism_result,
|
|
comprehension_total_ordering_result,
|
|
comprehension_propositional_result,
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class Collection:
|
|
results: tuple[DomainResult, ...]
|
|
not_covered: tuple[tuple[str, str], ...] # (adapter_name, error) — no silent drop
|
|
|
|
|
|
def collect_domain_results() -> Collection:
|
|
"""Run every adapter; surface any that fail rather than dropping them."""
|
|
results: list[DomainResult] = []
|
|
not_covered: list[tuple[str, str]] = []
|
|
for adapter in ADAPTERS:
|
|
try:
|
|
results.append(adapter())
|
|
except Exception as exc: # noqa: BLE001 — surfacing is the contract
|
|
not_covered.append((adapter.__name__, repr(exc)))
|
|
return Collection(results=tuple(results), not_covered=tuple(not_covered))
|