Merge pull request #582 from AssetOverflow/feat/comprehension-organ-three-domains
feat(comprehend): complete 3-domain comprehension organ (syllogism + total_ordering)
This commit is contained in:
Shay
GitHub
commit
14fa922116
12 changed files with 1092 additions and 108 deletions
|
|
@ -42,11 +42,38 @@ def dimensional_result() -> DomainResult:
|
|||
return DomainResult("dimensional", correct, wrong, refused)
|
||||
|
||||
|
||||
def comprehension_set_membership_result() -> DomainResult:
|
||||
from evals.comprehension.set_membership_runner import run
|
||||
|
||||
c, w, r = _counts(run())
|
||||
return DomainResult("comprehension_set_membership", c, w, r)
|
||||
|
||||
|
||||
def comprehension_syllogism_result() -> DomainResult:
|
||||
from evals.comprehension.syllogism_runner import run
|
||||
|
||||
c, w, r = _counts(run())
|
||||
return DomainResult("comprehension_syllogism", c, w, r)
|
||||
|
||||
|
||||
def comprehension_total_ordering_result() -> DomainResult:
|
||||
from evals.comprehension.total_ordering_runner import run
|
||||
|
||||
c, w, r = _counts(run())
|
||||
return DomainResult("comprehension_total_ordering", c, w, r)
|
||||
|
||||
|
||||
#: The reasoning domains currently composed into the index (self-loading lanes).
|
||||
#: The three ``comprehension_*`` lanes score the GENERAL comprehension reader
|
||||
#: (prose -> MeaningGraph -> projection -> independent oracle), so the index now
|
||||
#: measures comprehension breadth, not just structured-input reasoning.
|
||||
ADAPTERS = (
|
||||
deductive_logic_result,
|
||||
relational_metric_result,
|
||||
dimensional_result,
|
||||
comprehension_set_membership_result,
|
||||
comprehension_syllogism_result,
|
||||
comprehension_total_ordering_result,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +1,38 @@
|
|||
{
|
||||
"capability_score": 0.919641,
|
||||
"coverage_geomean": 0.919641,
|
||||
"coverage_micro": 0.995962,
|
||||
"capability_score": 0.814356,
|
||||
"coverage_geomean": 0.814356,
|
||||
"coverage_micro": 0.988266,
|
||||
"accuracy_micro": 1.0,
|
||||
"breadth": 3,
|
||||
"min_domain_coverage": 0.833333,
|
||||
"breadth": 6,
|
||||
"min_domain_coverage": 0.5,
|
||||
"wrong_total": 0,
|
||||
"assert_mode_valid": true,
|
||||
"deterministic_digest": "115389d285babb5980b8b31c40d3a1c58ded1e1f9ae08ec428090bba18d42ed0",
|
||||
"deterministic_digest": "0a98b9b41c23f5166d7e23bc194207ead53a89d5316dddc03194bd2739bb9753",
|
||||
"domains": [
|
||||
{
|
||||
"domain": "comprehension_set_membership",
|
||||
"correct": 8,
|
||||
"wrong": 0,
|
||||
"refused": 0,
|
||||
"coverage": 1.0,
|
||||
"accuracy": 1.0
|
||||
},
|
||||
{
|
||||
"domain": "comprehension_syllogism",
|
||||
"correct": 6,
|
||||
"wrong": 0,
|
||||
"refused": 2,
|
||||
"coverage": 0.75,
|
||||
"accuracy": 1.0
|
||||
},
|
||||
{
|
||||
"domain": "comprehension_total_ordering",
|
||||
"correct": 4,
|
||||
"wrong": 0,
|
||||
"refused": 4,
|
||||
"coverage": 0.5,
|
||||
"accuracy": 1.0
|
||||
},
|
||||
{
|
||||
"domain": "deductive_logic",
|
||||
"correct": 716,
|
||||
|
|
|
|||
70
evals/comprehension/syllogism_runner.py
Normal file
70
evals/comprehension/syllogism_runner.py
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
"""Score the general comprehension reader on the syllogism gold lane.
|
||||
|
||||
prose -> comprehend() -> to_syllogism() -> independent oracle -> answer vs gold.
|
||||
A refusal (unreadable prose, unprojectable, or oracle-refused projection) is NOT a
|
||||
wrong; only a committed answer that disagrees with gold is wrong (must stay 0).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from evals.syllogism.oracle import OracleError, oracle_answer
|
||||
from evals.syllogism.runner import _load_cases
|
||||
from generate.meaning_graph.projectors import to_syllogism
|
||||
from generate.meaning_graph.reader import Refusal, comprehend
|
||||
|
||||
|
||||
def run() -> dict[str, Any]:
|
||||
cases = _load_cases()
|
||||
correct = wrong = refused = 0
|
||||
wrongs: list[dict[str, Any]] = []
|
||||
|
||||
for case in cases:
|
||||
comp = comprehend(case["text"])
|
||||
if isinstance(comp, Refusal):
|
||||
refused += 1
|
||||
continue
|
||||
projected = to_syllogism(comp)
|
||||
if projected is None:
|
||||
refused += 1
|
||||
continue
|
||||
structure, query = projected
|
||||
try:
|
||||
got = oracle_answer(structure, query)
|
||||
except OracleError:
|
||||
refused += 1
|
||||
continue
|
||||
if got == case.get("gold"):
|
||||
correct += 1
|
||||
else:
|
||||
wrong += 1
|
||||
wrongs.append(
|
||||
{"id": case.get("id"), "got": got, "gold": case.get("gold"), "text": case["text"]}
|
||||
)
|
||||
|
||||
return {
|
||||
"domain": "comprehension_syllogism",
|
||||
"total": len(cases),
|
||||
"correct": correct,
|
||||
"wrong": wrong,
|
||||
"refused": refused,
|
||||
"wrongs": wrongs,
|
||||
"counts": {"correct": correct, "wrong": wrong, "refused": refused},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
report = run()
|
||||
print(json.dumps({k: v for k, v in report.items() if k != "wrongs"}, indent=2, sort_keys=True))
|
||||
if report["wrong"]:
|
||||
print("WRONG > 0 — comprehension produced a wrong committed answer:", file=sys.stderr)
|
||||
print(json.dumps(report["wrongs"], indent=2), file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
70
evals/comprehension/total_ordering_runner.py
Normal file
70
evals/comprehension/total_ordering_runner.py
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
"""Score the general comprehension reader on the total_ordering gold lane.
|
||||
|
||||
prose -> comprehend() -> to_total_ordering() -> independent oracle -> answer vs gold.
|
||||
A refusal (unreadable prose, unprojectable, or oracle-refused projection) is NOT a
|
||||
wrong; only a committed answer that disagrees with gold is wrong (must stay 0).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from evals.total_ordering.oracle import OracleError, oracle_answer
|
||||
from evals.total_ordering.runner import _load_cases
|
||||
from generate.meaning_graph.projectors import to_total_ordering
|
||||
from generate.meaning_graph.reader import Refusal, comprehend
|
||||
|
||||
|
||||
def run() -> dict[str, Any]:
|
||||
cases = _load_cases()
|
||||
correct = wrong = refused = 0
|
||||
wrongs: list[dict[str, Any]] = []
|
||||
|
||||
for case in cases:
|
||||
comp = comprehend(case["text"])
|
||||
if isinstance(comp, Refusal):
|
||||
refused += 1
|
||||
continue
|
||||
projected = to_total_ordering(comp)
|
||||
if projected is None:
|
||||
refused += 1
|
||||
continue
|
||||
structure, query = projected
|
||||
try:
|
||||
got = oracle_answer(structure, query)
|
||||
except OracleError:
|
||||
refused += 1
|
||||
continue
|
||||
if got == case.get("gold"):
|
||||
correct += 1
|
||||
else:
|
||||
wrong += 1
|
||||
wrongs.append(
|
||||
{"id": case.get("id"), "got": got, "gold": case.get("gold"), "text": case["text"]}
|
||||
)
|
||||
|
||||
return {
|
||||
"domain": "comprehension_total_ordering",
|
||||
"total": len(cases),
|
||||
"correct": correct,
|
||||
"wrong": wrong,
|
||||
"refused": refused,
|
||||
"wrongs": wrongs,
|
||||
"counts": {"correct": correct, "wrong": wrong, "refused": refused},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
report = run()
|
||||
print(json.dumps({k: v for k, v in report.items() if k != "wrongs"}, indent=2, sort_keys=True))
|
||||
if report["wrong"]:
|
||||
print("WRONG > 0 — comprehension produced a wrong committed answer:", file=sys.stderr)
|
||||
print(json.dumps(report["wrongs"], indent=2), file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
|
@ -55,3 +55,80 @@ def to_set_membership(comp: Comprehension) -> tuple[dict[str, Any], dict[str, An
|
|||
q = subset_queries[0]
|
||||
query = {"kind": "subset", "subset": q.arguments[0], "superset": q.arguments[1]}
|
||||
return structure, query
|
||||
|
||||
|
||||
#: Categorical predicate -> Aristotelian form for the syllogism oracle.
|
||||
_PRED_FORM = {"subset": "A", "disjoint": "E", "intersects": "I", "some_not": "O"}
|
||||
|
||||
#: Finite-model domain size for syllogism validity (matches the gold lane).
|
||||
_SYLLOGISM_DOMAIN_SIZE = 3
|
||||
|
||||
|
||||
def to_syllogism(comp: Comprehension) -> tuple[dict[str, Any], dict[str, Any]] | None:
|
||||
"""Project into ``(structure, query)`` for ``evals.syllogism.oracle``.
|
||||
|
||||
Premises are the categorical relations (subset/disjoint/intersects/some_not);
|
||||
the single categorical query is the conclusion. Returns ``None`` when the
|
||||
comprehension is not exactly one categorical conclusion over >=1 premise — the
|
||||
caller treats that as a refusal (nothing honestly askable of this oracle).
|
||||
"""
|
||||
graph = comp.meaning_graph
|
||||
premises = [
|
||||
{"form": _PRED_FORM[r.predicate], "subject": r.arguments[0], "predicate": r.arguments[1]}
|
||||
for r in graph.relations
|
||||
if r.predicate in _PRED_FORM and not r.negated
|
||||
]
|
||||
conclusions = [q for q in comp.queries if q.predicate in _PRED_FORM and not q.negated]
|
||||
if not premises or len(comp.queries) != 1 or len(conclusions) != 1:
|
||||
return None
|
||||
|
||||
c = conclusions[0]
|
||||
terms = sorted({e.entity_id for e in graph.entities if e.kind == "class"})
|
||||
if len(terms) < 2:
|
||||
return None
|
||||
structure = {
|
||||
"terms": terms,
|
||||
"domain_size": _SYLLOGISM_DOMAIN_SIZE,
|
||||
"premises": premises,
|
||||
}
|
||||
query = {
|
||||
"kind": "validity",
|
||||
"conclusion": {
|
||||
"form": _PRED_FORM[c.predicate],
|
||||
"subject": c.arguments[0],
|
||||
"predicate": c.arguments[1],
|
||||
},
|
||||
}
|
||||
return structure, query
|
||||
|
||||
|
||||
def to_total_ordering(comp: Comprehension) -> tuple[dict[str, Any], dict[str, Any]] | None:
|
||||
"""Project into ``(structure, query)`` for ``evals.total_ordering.oracle``.
|
||||
|
||||
Facts are ``less(lo, hi)`` relations; the single query is a sort or compare.
|
||||
Returns ``None`` when the comprehension does not carry exactly one ordering
|
||||
query — the caller treats that as a refusal.
|
||||
"""
|
||||
graph = comp.meaning_graph
|
||||
less_facts = [
|
||||
(r.arguments[0], r.arguments[1])
|
||||
for r in graph.relations
|
||||
if r.predicate == "less" and not r.negated
|
||||
]
|
||||
order_queries = [q for q in comp.queries if q.predicate in ("sort", "compare")]
|
||||
if len(comp.queries) != 1 or len(order_queries) != 1:
|
||||
return None
|
||||
|
||||
q = order_queries[0]
|
||||
item_set = {item for pair in less_facts for item in pair}
|
||||
if q.predicate == "compare":
|
||||
item_set.update(q.arguments)
|
||||
structure = {
|
||||
"items": sorted(item_set),
|
||||
"relations": [{"less": lo, "greater": hi} for lo, hi in less_facts],
|
||||
}
|
||||
if q.predicate == "sort":
|
||||
query = {"kind": "sort", "order": q.arguments[0]}
|
||||
else:
|
||||
query = {"kind": "compare", "left": q.arguments[0], "right": q.arguments[1]}
|
||||
return structure, query
|
||||
|
|
|
|||
|
|
@ -1,24 +1,45 @@
|
|||
"""The general comprehension reader — disciplined Path β (Phase 2a).
|
||||
"""The general comprehension reader — disciplined Path β (Phase 2a / 2b).
|
||||
|
||||
Reads subject/relation/object STRUCTURE symbolically from the token sequence via
|
||||
domain-agnostic templates keyed on FUNCTION WORDS + ORDER, then mints content as
|
||||
``MeaningGraph`` entities/relations. The field provably cannot recover this
|
||||
structure (the holonomy fold is lossy/non-invertible — see the α-falsification in
|
||||
``docs/analysis/phase2-general-comprehension-organ-scope-2026-06-05.md``); the
|
||||
field's honest role is grounding/coherence, not composition.
|
||||
``MeaningGraph`` entities/relations + ``Query`` objects. The field provably cannot
|
||||
recover this structure (the holonomy fold is lossy/non-invertible — see the
|
||||
α-falsification in ``docs/analysis/phase2-general-comprehension-organ-scope-2026-06-05.md``);
|
||||
the field's honest role is grounding/coherence, not composition.
|
||||
|
||||
Refusal-first: a clause that matches no template, a filler that is not a clean
|
||||
identifier, an unrecognized plural, or a role conflict all REFUSE — never guess.
|
||||
That keeps ``wrong=0`` at the comprehension layer (refusal is success; a fabricated
|
||||
reading is the only failure).
|
||||
identifier, an unrecognized plural, a MULTI-WORD noun phrase, or a role conflict
|
||||
all REFUSE — never guess. That keeps ``wrong=0`` at the comprehension layer
|
||||
(refusal is success; a fabricated reading is the only failure).
|
||||
|
||||
Templates (this increment):
|
||||
- ``<X> is a|an <Y>`` -> member(individual=X, class=Y)
|
||||
- ``all <Xs> are <Ys>`` -> subset(subclass=sing(X), superclass=sing(Y))
|
||||
- ``is <X> a|an <Y>?`` -> Query member(X, Y)
|
||||
Templates (function-word + order; domain-agnostic — the same templates read
|
||||
animals, professions, geography, kin, metals, ranks):
|
||||
|
||||
Overfit-trap mitigation: templates key on function words + order (general), never
|
||||
on domain content; the same templates read animals, professions, geography, kin.
|
||||
membership / subsumption (set_membership shapes)
|
||||
- ``<X> is a|an <Y>`` -> member(individual=X, class=Y)
|
||||
- ``the <X> is a|an <Y>`` -> member(X, Y)
|
||||
- ``is [the] <X> a|an <Y>?`` -> Query member(X, Y)
|
||||
- ``are all <Xs> <Ys>?`` -> Query subset(sing X, sing Y)
|
||||
|
||||
categorical (syllogism shapes); subject/predicate are plural classes
|
||||
- ``all <Xs> are <Ys>`` -> subset(sing X, sing Y) (A)
|
||||
- ``no <Xs> are <Ys>`` -> disjoint(sing X, sing Y) (E)
|
||||
- ``some <Xs> are <Ys>`` -> intersects(sing X, sing Y) (I)
|
||||
- ``some <Xs> are not <Ys>`` -> some_not(sing X, sing Y) (O)
|
||||
- ``therefore <categorical>`` -> Query of the same predicate (the conclusion)
|
||||
|
||||
ordering (total_ordering shapes); X/Y are single items
|
||||
- ``<X> [is] <less-comp> [than] <Y>`` -> less(X, Y)
|
||||
- ``<X> [is] <greater-comp> [than] <Y>`` -> less(Y, X)
|
||||
- ``compare <X> with <Y>`` -> Query compare(X, Y)
|
||||
- ``sort [ascending|descending]`` /
|
||||
``... order from <low> to <high>`` -> Query sort(order)
|
||||
|
||||
Multi-word noun phrases are REFUSED on purpose: the staged gold lanes canonicalize
|
||||
multi-word NPs three contradictory ways ("North station"->"north", "Level one"->
|
||||
"level_one", "metal objects"->"metal"), so no single general rule is wrong=0-safe.
|
||||
Until the gold lanes carry a canonicalization contract, the only honest reading of
|
||||
a multi-word NP is refusal.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -49,6 +70,34 @@ _IRREGULAR_PLURALS = {
|
|||
"geese": "goose",
|
||||
}
|
||||
|
||||
# Categorical quantifier -> the MeaningGraph predicate it mints. The predicate
|
||||
# vocabulary is shared between facts and the "therefore" conclusion query, and is
|
||||
# neutral: the syllogism projector maps {subset:A, disjoint:E, intersects:I,
|
||||
# some_not:O}, while the set_membership projector reads subset/member only.
|
||||
_QUANTIFIER_PREDICATE = {
|
||||
"all": "subset",
|
||||
"no": "disjoint",
|
||||
"some": "intersects", # "some ... not" is special-cased to some_not
|
||||
}
|
||||
|
||||
# Pairwise-order comparators. "less" means X < Y -> less(X, Y); "greater" means
|
||||
# X > Y -> less(Y, X). Closed sets; an unknown comparator falls through to refusal.
|
||||
_COMP_LESS = frozenset(
|
||||
{"below", "under", "beneath", "lower", "shorter", "smaller", "earlier", "closer", "before"}
|
||||
)
|
||||
_COMP_GREATER = frozenset(
|
||||
{"above", "over", "higher", "taller", "larger", "bigger", "greater", "later", "after", "farther", "further"}
|
||||
)
|
||||
|
||||
# Sort-direction endpoints: "from <low> to <high>" is ascending; reversed is
|
||||
# descending. Closed sets; an unrecognized endpoint refuses the sort query.
|
||||
_SORT_LOW = frozenset(
|
||||
{"lowest", "shortest", "smallest", "least", "earliest", "bottom", "first", "low"}
|
||||
)
|
||||
_SORT_HIGH = frozenset(
|
||||
{"highest", "tallest", "largest", "greatest", "most", "latest", "top", "last", "high"}
|
||||
)
|
||||
|
||||
_SENTENCE_RE = re.compile(r"\s*([^.?!]+?)\s*([.?!])")
|
||||
|
||||
|
||||
|
|
@ -78,6 +127,16 @@ class Refusal:
|
|||
detail: str = ""
|
||||
|
||||
|
||||
class _Reject(Exception):
|
||||
"""Internal control-flow signal: a clause matched a template shape but is not
|
||||
honestly readable (multi-word NP, bad morphology, role conflict). Carries the
|
||||
typed ``Refusal`` so ``comprehend`` returns it verbatim."""
|
||||
|
||||
def __init__(self, reason: str, detail: str = "") -> None:
|
||||
super().__init__(reason)
|
||||
self.refusal = Refusal(reason, detail)
|
||||
|
||||
|
||||
def _identifier(word: str) -> str | None:
|
||||
"""Normalize a content token to a clean identifier, or None to refuse."""
|
||||
w = word.strip().lower()
|
||||
|
|
@ -97,6 +156,27 @@ def _singularize(word: str) -> str | None:
|
|||
return None
|
||||
|
||||
|
||||
def _one(words: list[str], detail: str) -> str:
|
||||
"""A single content token -> identifier, or REFUSE (multi-word / non-id)."""
|
||||
if len(words) != 1:
|
||||
raise _Reject("multiword_np", detail)
|
||||
ident = _identifier(words[0])
|
||||
if ident is None:
|
||||
raise _Reject("non_identifier_filler", detail)
|
||||
return ident
|
||||
|
||||
|
||||
def _one_class(words: list[str], detail: str) -> str:
|
||||
"""A single plural content token -> singular class id, or REFUSE."""
|
||||
word = _one(words, detail)
|
||||
singular = _singularize(word)
|
||||
if singular is None:
|
||||
raise _Reject("unknown_morphology", detail)
|
||||
if not singular.isidentifier():
|
||||
raise _Reject("non_identifier_filler", detail)
|
||||
return singular
|
||||
|
||||
|
||||
def _split_sentences(text: str) -> list[tuple[str, str, int, int]]:
|
||||
"""Return (body, terminator, start, end) for each sentence in *text*."""
|
||||
out: list[tuple[str, str, int, int]] = []
|
||||
|
|
@ -114,6 +194,88 @@ def _split_sentences(text: str) -> list[tuple[str, str, int, int]]:
|
|||
return out
|
||||
|
||||
|
||||
def _split_clauses(body: str) -> list[str]:
|
||||
"""Split a sentence body into clauses on commas; strip a leading and/or."""
|
||||
clauses: list[str] = []
|
||||
for part in body.split(","):
|
||||
toks = part.split()
|
||||
if toks and toks[0].lower() in {"and", "or"}:
|
||||
toks = toks[1:]
|
||||
if toks:
|
||||
clauses.append(" ".join(toks))
|
||||
return clauses
|
||||
|
||||
|
||||
def _parse_categorical(toks: list[str], detail: str) -> tuple[str, str, str] | None:
|
||||
"""``<quant> <Xs> are [not] <Ys>`` -> (predicate, sub, sup); None if not it."""
|
||||
if not toks or toks[0] not in _QUANTIFIER_PREDICATE:
|
||||
return None
|
||||
if "are" not in toks:
|
||||
return None
|
||||
quant = toks[0]
|
||||
are_idx = toks.index("are")
|
||||
subject_words = toks[1:are_idx]
|
||||
predicate_words = toks[are_idx + 1:]
|
||||
negated = bool(predicate_words) and predicate_words[0] == "not"
|
||||
if negated:
|
||||
predicate_words = predicate_words[1:]
|
||||
if not subject_words or not predicate_words:
|
||||
raise _Reject("incomplete_categorical", detail)
|
||||
sub = _one_class(subject_words, detail)
|
||||
sup = _one_class(predicate_words, detail)
|
||||
if quant == "some" and negated:
|
||||
predicate = "some_not"
|
||||
elif negated:
|
||||
# "no ... not" / "all ... not" are out of this grammar; refuse, don't guess.
|
||||
raise _Reject("unsupported_negation", detail)
|
||||
else:
|
||||
predicate = _QUANTIFIER_PREDICATE[quant]
|
||||
return predicate, sub, sup
|
||||
|
||||
|
||||
def _parse_comparative(toks: list[str], detail: str) -> tuple[str, str] | None:
|
||||
"""``<X> [is] <comp> [than] <Y>`` -> (less_item, greater_item); None if not it."""
|
||||
comp_idx = next(
|
||||
(i for i, t in enumerate(toks) if t in _COMP_LESS or t in _COMP_GREATER),
|
||||
None,
|
||||
)
|
||||
if comp_idx is None:
|
||||
return None
|
||||
left = toks[:comp_idx]
|
||||
if left and left[-1] == "is":
|
||||
left = left[:-1]
|
||||
right = toks[comp_idx + 1:]
|
||||
if right and right[0] == "than":
|
||||
right = right[1:]
|
||||
if not left or not right:
|
||||
raise _Reject("incomplete_comparative", detail)
|
||||
x = _one(left, detail)
|
||||
y = _one(right, detail)
|
||||
if toks[comp_idx] in _COMP_LESS:
|
||||
return x, y # X < Y
|
||||
return y, x # X > Y -> less(Y, X)
|
||||
|
||||
|
||||
def _parse_sort(toks: list[str], detail: str) -> str | None:
|
||||
"""A sort request -> "ascending"|"descending"; None if not a sort request."""
|
||||
is_sort = toks and toks[0] == "sort"
|
||||
is_order = "order" in toks and "from" in toks
|
||||
if not (is_sort or is_order):
|
||||
return None
|
||||
if "ascending" in toks:
|
||||
return "ascending"
|
||||
if "descending" in toks:
|
||||
return "descending"
|
||||
if "from" in toks and "to" in toks:
|
||||
lo = toks[toks.index("from") + 1] if toks.index("from") + 1 < len(toks) else ""
|
||||
hi = toks[toks.index("to") + 1] if toks.index("to") + 1 < len(toks) else ""
|
||||
if lo in _SORT_LOW and hi in _SORT_HIGH:
|
||||
return "ascending"
|
||||
if lo in _SORT_HIGH and hi in _SORT_LOW:
|
||||
return "descending"
|
||||
raise _Reject("ambiguous_sort_order", detail)
|
||||
|
||||
|
||||
def comprehend(text: str, source_id: str = "input") -> Comprehension | Refusal:
|
||||
"""Comprehend *text* into a MeaningGraph + queries, or a typed Refusal."""
|
||||
if not text or not text.strip():
|
||||
|
|
@ -125,106 +287,133 @@ def comprehend(text: str, source_id: str = "input") -> Comprehension | Refusal:
|
|||
|
||||
role_kind: dict[str, str] = {}
|
||||
span_for: dict[str, MeaningSpan] = {}
|
||||
members: list[tuple[str, str, MeaningSpan]] = []
|
||||
subsets: list[tuple[str, str, MeaningSpan]] = []
|
||||
relations: list[tuple[str, tuple[str, ...], MeaningSpan]] = []
|
||||
queries: list[Query] = []
|
||||
|
||||
def claim(entity_id: str, kind: str, span: MeaningSpan) -> bool:
|
||||
def claim(entity_id: str, kind: str, span: MeaningSpan) -> None:
|
||||
prior = role_kind.get(entity_id)
|
||||
if prior is not None and prior != kind:
|
||||
return False
|
||||
raise _Reject("role_conflict", entity_id)
|
||||
role_kind[entity_id] = kind
|
||||
span_for.setdefault(entity_id, span)
|
||||
return True
|
||||
|
||||
for body, terminator, start, end in sentences:
|
||||
toks = body.lower().split()
|
||||
span = MeaningSpan(source_id=source_id, start=start, end=end, text=text[start:end])
|
||||
is_question = terminator == "?"
|
||||
|
||||
# Template: subset query ``are all <Xs> <Ys>?``
|
||||
if is_question and len(toks) == 4 and toks[0] == "are" and toks[1] == "all":
|
||||
raw_sub, raw_sup = _identifier(toks[2]), _identifier(toks[3])
|
||||
if raw_sub is None or raw_sup is None:
|
||||
return Refusal("non_identifier_filler", body)
|
||||
sub, sup = _singularize(raw_sub), _singularize(raw_sup)
|
||||
if sub is None or sup is None:
|
||||
return Refusal("unknown_morphology", body)
|
||||
if not sub.isidentifier() or not sup.isidentifier():
|
||||
return Refusal("non_identifier_filler", body)
|
||||
if not claim(sub, "class", span) or not claim(sup, "class", span):
|
||||
return Refusal("role_conflict", body)
|
||||
queries.append(Query("subset", (sub, sup), span))
|
||||
continue
|
||||
|
||||
# Template: definite-NP membership query ``is the <X> a|an <Y>?``
|
||||
if is_question and len(toks) == 5 and toks[0] == "is" and toks[1] == "the" and toks[3] in _ARTICLES:
|
||||
name, cls = _identifier(toks[2]), _identifier(toks[4])
|
||||
if name is None or cls is None:
|
||||
return Refusal("non_identifier_filler", body)
|
||||
if not claim(name, "individual", span) or not claim(cls, "class", span):
|
||||
return Refusal("role_conflict", body)
|
||||
queries.append(Query("member", (name, cls), span))
|
||||
continue
|
||||
|
||||
# Template: query ``is <X> a|an <Y>?``
|
||||
if is_question and len(toks) == 4 and toks[0] == "is" and toks[2] in _ARTICLES:
|
||||
name, cls = _identifier(toks[1]), _identifier(toks[3])
|
||||
if name is None or cls is None:
|
||||
return Refusal("non_identifier_filler", body)
|
||||
if not claim(name, "individual", span) or not claim(cls, "class", span):
|
||||
return Refusal("role_conflict", body)
|
||||
queries.append(Query("member", (name, cls), span))
|
||||
continue
|
||||
|
||||
# Template: definite-NP membership ``the <X> is a|an <Y>``
|
||||
if not is_question and len(toks) == 5 and toks[0] == "the" and toks[2] == "is" and toks[3] in _ARTICLES:
|
||||
name, cls = _identifier(toks[1]), _identifier(toks[4])
|
||||
if name is None or cls is None:
|
||||
return Refusal("non_identifier_filler", body)
|
||||
if not claim(name, "individual", span) or not claim(cls, "class", span):
|
||||
return Refusal("role_conflict", body)
|
||||
members.append((name, cls, span))
|
||||
continue
|
||||
|
||||
# Template: membership ``<X> is a|an <Y>``
|
||||
if not is_question and len(toks) == 4 and toks[1] == "is" and toks[2] in _ARTICLES:
|
||||
name, cls = _identifier(toks[0]), _identifier(toks[3])
|
||||
if name is None or cls is None:
|
||||
return Refusal("non_identifier_filler", body)
|
||||
if not claim(name, "individual", span) or not claim(cls, "class", span):
|
||||
return Refusal("role_conflict", body)
|
||||
members.append((name, cls, span))
|
||||
continue
|
||||
|
||||
# Template: subsumption ``all <Xs> are <Ys>``
|
||||
if not is_question and len(toks) == 4 and toks[0] == "all" and toks[2] == "are":
|
||||
raw_sub, raw_sup = _identifier(toks[1]), _identifier(toks[3])
|
||||
if raw_sub is None or raw_sup is None:
|
||||
return Refusal("non_identifier_filler", body)
|
||||
sub, sup = _singularize(raw_sub), _singularize(raw_sup)
|
||||
if sub is None or sup is None:
|
||||
return Refusal("unknown_morphology", body)
|
||||
if not sub.isidentifier() or not sup.isidentifier():
|
||||
return Refusal("non_identifier_filler", body)
|
||||
if not claim(sub, "class", span) or not claim(sup, "class", span):
|
||||
return Refusal("role_conflict", body)
|
||||
subsets.append((sub, sup, span))
|
||||
continue
|
||||
|
||||
return Refusal("no_template_match", body)
|
||||
try:
|
||||
for body, terminator, start, end in sentences:
|
||||
span = MeaningSpan(
|
||||
source_id=source_id, start=start, end=end, text=text[start:end]
|
||||
)
|
||||
is_question = terminator == "?"
|
||||
for clause in _split_clauses(body):
|
||||
_read_clause(clause, is_question, span, claim, relations, queries)
|
||||
except _Reject as rej:
|
||||
return rej.refusal
|
||||
|
||||
try:
|
||||
entities = tuple(
|
||||
Entity(entity_id=eid, name=eid, span=span_for[eid], kind=role_kind[eid])
|
||||
for eid in sorted(role_kind)
|
||||
)
|
||||
relations = tuple(
|
||||
[Relation("member", (ind, cls), sp) for ind, cls, sp in members]
|
||||
+ [Relation("subset", (sub, sup), sp) for sub, sup, sp in subsets]
|
||||
graph = MeaningGraph(
|
||||
entities=entities,
|
||||
relations=tuple(
|
||||
Relation(pred, args, sp) for pred, args, sp in relations
|
||||
),
|
||||
)
|
||||
graph = MeaningGraph(entities=entities, relations=relations)
|
||||
except MeaningGraphError as exc: # defensive — construction invariants
|
||||
return Refusal("invalid_graph", repr(exc))
|
||||
|
||||
return Comprehension(meaning_graph=graph, queries=tuple(queries))
|
||||
|
||||
|
||||
def _read_clause(
|
||||
clause: str,
|
||||
question: bool,
|
||||
span: MeaningSpan,
|
||||
claim,
|
||||
relations: list[tuple[str, tuple[str, ...], MeaningSpan]],
|
||||
queries: list[Query],
|
||||
) -> None:
|
||||
"""Match one clause against the templates; mutate accumulators or REFUSE."""
|
||||
toks = clause.strip().lower().split()
|
||||
if not toks:
|
||||
raise _Reject("empty_clause", clause)
|
||||
|
||||
# --- ordering queries (keyword-led; terminator-independent) ------------- #
|
||||
if toks[0] == "compare":
|
||||
if len(toks) != 4 or toks[2] != "with":
|
||||
raise _Reject("unreadable_compare", clause)
|
||||
left = _one([toks[1]], clause)
|
||||
right = _one([toks[3]], clause)
|
||||
for item in (left, right):
|
||||
claim(item, "item", span)
|
||||
queries.append(Query("compare", (left, right), span))
|
||||
return
|
||||
|
||||
order = _parse_sort(toks, clause)
|
||||
if order is not None:
|
||||
queries.append(Query("sort", (order,), span))
|
||||
return
|
||||
|
||||
# --- syllogism conclusion: "therefore <categorical>" ------------------- #
|
||||
if toks[0] == "therefore":
|
||||
parsed = _parse_categorical(toks[1:], clause)
|
||||
if parsed is None:
|
||||
raise _Reject("unreadable_conclusion", clause)
|
||||
predicate, sub, sup = parsed
|
||||
claim(sub, "class", span)
|
||||
claim(sup, "class", span)
|
||||
queries.append(Query(predicate, (sub, sup), span))
|
||||
return
|
||||
|
||||
# --- membership query: "is [the] <X> a|an <Y>?" ------------------------ #
|
||||
if question and toks[0] == "is":
|
||||
rest = toks[1:]
|
||||
if rest and rest[0] == "the":
|
||||
rest = rest[1:]
|
||||
if len(rest) == 3 and rest[1] in _ARTICLES:
|
||||
name = _one([rest[0]], clause)
|
||||
cls = _one([rest[2]], clause)
|
||||
claim(name, "individual", span)
|
||||
claim(cls, "class", span)
|
||||
queries.append(Query("member", (name, cls), span))
|
||||
return
|
||||
raise _Reject("unreadable_member_query", clause)
|
||||
|
||||
# --- subset query: "are all <Xs> <Ys>?" -------------------------------- #
|
||||
if question and len(toks) == 4 and toks[0] == "are" and toks[1] == "all":
|
||||
sub = _one_class([toks[2]], clause)
|
||||
sup = _one_class([toks[3]], clause)
|
||||
claim(sub, "class", span)
|
||||
claim(sup, "class", span)
|
||||
queries.append(Query("subset", (sub, sup), span))
|
||||
return
|
||||
|
||||
# --- categorical fact: "<quant> <Xs> are [not] <Ys>" ------------------- #
|
||||
categorical = _parse_categorical(toks, clause)
|
||||
if categorical is not None:
|
||||
predicate, sub, sup = categorical
|
||||
claim(sub, "class", span)
|
||||
claim(sup, "class", span)
|
||||
relations.append((predicate, (sub, sup), span))
|
||||
return
|
||||
|
||||
# --- membership fact: "[the] <X> is a|an <Y>" -------------------------- #
|
||||
body_toks = toks[1:] if toks[0] == "the" else toks
|
||||
if len(body_toks) == 4 and body_toks[1] == "is" and body_toks[2] in _ARTICLES:
|
||||
name = _one([body_toks[0]], clause)
|
||||
cls = _one([body_toks[3]], clause)
|
||||
claim(name, "individual", span)
|
||||
claim(cls, "class", span)
|
||||
relations.append(("member", (name, cls), span))
|
||||
return
|
||||
|
||||
# --- ordering fact: "<X> [is] <comp> [than] <Y>" ----------------------- #
|
||||
comparative = _parse_comparative(toks, clause)
|
||||
if comparative is not None:
|
||||
lo, hi = comparative
|
||||
claim(lo, "item", span)
|
||||
claim(hi, "item", span)
|
||||
relations.append(("less", (lo, hi), span))
|
||||
return
|
||||
|
||||
raise _Reject("no_template_match", clause)
|
||||
|
|
|
|||
|
|
@ -87,8 +87,9 @@ def test_empty_index_is_well_defined() -> None:
|
|||
|
||||
|
||||
def test_real_lanes_compose_into_the_index_with_wrong_zero() -> None:
|
||||
# The Phase-1b baseline: the three self-loading independent-gold reasoning
|
||||
# lanes compose into the cross-domain index with zero wrong commits.
|
||||
# The baseline: three structured-input reasoning lanes PLUS the three
|
||||
# comprehension lanes (prose -> MeaningGraph -> projection -> independent
|
||||
# oracle) compose into the cross-domain index with zero wrong commits.
|
||||
from evals.capability_index.adapters import collect_domain_results
|
||||
|
||||
collection = collect_domain_results()
|
||||
|
|
@ -96,11 +97,14 @@ def test_real_lanes_compose_into_the_index_with_wrong_zero() -> None:
|
|||
idx = aggregate(list(collection.results))
|
||||
assert idx.wrong_total == 0
|
||||
assert idx.assert_mode_valid
|
||||
assert idx.breadth == 3 # deductive_logic + dimensional + relational_metric
|
||||
assert idx.breadth == 6
|
||||
assert {d.domain for d in idx.domains} == {
|
||||
"deductive_logic",
|
||||
"dimensional",
|
||||
"relational_metric",
|
||||
"comprehension_set_membership",
|
||||
"comprehension_syllogism",
|
||||
"comprehension_total_ordering",
|
||||
}
|
||||
assert idx.capability_score > 0.5 # real, non-trivial cross-domain capability
|
||||
|
||||
|
|
|
|||
|
|
@ -157,3 +157,163 @@ def test_refuses_non_identifier_filler() -> None:
|
|||
def test_empty_input_refuses() -> None:
|
||||
assert isinstance(comprehend(""), Refusal)
|
||||
assert isinstance(comprehend(" "), Refusal)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Categorical premises — E / I / O forms (syllogism shapes)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_categorical_no_is_disjoint() -> None:
|
||||
comp = comprehend("No reptiles are mammals.")
|
||||
assert isinstance(comp, Comprehension)
|
||||
assert _rel(comp, "disjoint") == (("disjoint", ("reptile", "mammal")),)
|
||||
|
||||
|
||||
def test_categorical_some_is_intersects() -> None:
|
||||
comp = comprehend("Some students are poets.")
|
||||
assert isinstance(comp, Comprehension)
|
||||
assert _rel(comp, "intersects") == (("intersects", ("student", "poet")),)
|
||||
|
||||
|
||||
def test_categorical_some_not_is_some_not() -> None:
|
||||
comp = comprehend("Some pets are not reptiles.")
|
||||
assert isinstance(comp, Comprehension)
|
||||
assert _rel(comp, "some_not") == (("some_not", ("pet", "reptile")),)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# "Therefore <categorical>" -> conclusion QUERY (same neutral predicates)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_therefore_conclusion_is_a_query_not_a_fact() -> None:
|
||||
comp = comprehend("All whales are mammals. Therefore all whales are mammals.")
|
||||
assert isinstance(comp, Comprehension)
|
||||
# the premise is a fact; the conclusion is a query of the same predicate.
|
||||
assert _rel(comp, "subset") == (("subset", ("whale", "mammal")),)
|
||||
assert len(comp.queries) == 1
|
||||
assert comp.queries[0].predicate == "subset"
|
||||
assert comp.queries[0].arguments == ("whale", "mammal")
|
||||
|
||||
|
||||
def test_therefore_maps_each_quantifier_to_its_predicate() -> None:
|
||||
for tail, predicate in [
|
||||
("all dogs are animals", "subset"),
|
||||
("no dogs are cats", "disjoint"),
|
||||
("some dogs are pets", "intersects"),
|
||||
("some dogs are not cats", "some_not"),
|
||||
]:
|
||||
comp = comprehend(f"Therefore {tail}.")
|
||||
assert isinstance(comp, Comprehension), tail
|
||||
assert comp.queries[0].predicate == predicate, tail
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Ordering — comparative facts (total_ordering shapes)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_comparative_less_direction() -> None:
|
||||
comp = comprehend("Bronze is below silver.")
|
||||
assert isinstance(comp, Comprehension)
|
||||
assert _rel(comp, "less") == (("less", ("bronze", "silver")),)
|
||||
assert _entity_kind(comp, "bronze") == "item"
|
||||
|
||||
|
||||
def test_comparative_greater_direction_reverses() -> None:
|
||||
# "X is taller than Y" means X > Y, i.e. less(Y, X).
|
||||
comp = comprehend("Oak is taller than birch.")
|
||||
assert isinstance(comp, Comprehension)
|
||||
assert _rel(comp, "less") == (("less", ("birch", "oak")),)
|
||||
|
||||
|
||||
def test_comparative_elided_verb() -> None:
|
||||
# "Venus closer than Earth" (no copula) still reads.
|
||||
comp = comprehend("Venus closer than Earth.")
|
||||
assert isinstance(comp, Comprehension)
|
||||
assert _rel(comp, "less") == (("less", ("venus", "earth")),)
|
||||
|
||||
|
||||
def test_comparative_clause_splitting_on_comma_and() -> None:
|
||||
comp = comprehend("A is earlier than B, and B is earlier than C.")
|
||||
assert isinstance(comp, Comprehension)
|
||||
assert _rel(comp, "less") == (("less", ("a", "b")), ("less", ("b", "c")))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Ordering — sort / compare QUERIES
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_sort_query_lowest_to_highest_is_ascending() -> None:
|
||||
comp = comprehend("Sort them from lowest to highest.")
|
||||
assert isinstance(comp, Comprehension)
|
||||
assert comp.queries[0].predicate == "sort"
|
||||
assert comp.queries[0].arguments == ("ascending",)
|
||||
|
||||
|
||||
def test_sort_query_explicit_descending() -> None:
|
||||
comp = comprehend("Sort descending.")
|
||||
assert isinstance(comp, Comprehension)
|
||||
assert comp.queries[0].arguments == ("descending",)
|
||||
|
||||
|
||||
def test_sort_query_order_question_form() -> None:
|
||||
comp = comprehend("Which is the height order from shortest to tallest?")
|
||||
assert isinstance(comp, Comprehension)
|
||||
assert comp.queries[0].predicate == "sort"
|
||||
assert comp.queries[0].arguments == ("ascending",)
|
||||
|
||||
|
||||
def test_compare_query() -> None:
|
||||
comp = comprehend("Compare north with south.")
|
||||
assert isinstance(comp, Comprehension)
|
||||
assert comp.queries[0].predicate == "compare"
|
||||
assert comp.queries[0].arguments == ("north", "south")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Generality — SAME comparative template, DISTINCT domains (anti-overfit)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_comparative_template_generalizes_across_domains() -> None:
|
||||
for text, lo, hi in [
|
||||
("Bronze is below silver.", "bronze", "silver"), # metals
|
||||
("Monday is earlier than tuesday.", "monday", "tuesday"), # time
|
||||
("Birch is shorter than oak.", "birch", "oak"), # height
|
||||
]:
|
||||
comp = comprehend(text)
|
||||
assert isinstance(comp, Comprehension), text
|
||||
assert _rel(comp, "less") == (("less", (lo, hi)),), text
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Multi-word NP — REFUSE (the wrong=0 guard: no general canonicalization exists)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_multiword_np_in_categorical_refuses() -> None:
|
||||
comp = comprehend("No metal objects are soft objects.")
|
||||
assert isinstance(comp, Refusal)
|
||||
assert comp.reason == "multiword_np"
|
||||
|
||||
|
||||
def test_multiword_np_in_comparative_refuses() -> None:
|
||||
comp = comprehend("North station comes after central.")
|
||||
assert isinstance(comp, Refusal)
|
||||
assert comp.reason == "multiword_np"
|
||||
|
||||
|
||||
def test_adjectival_predicate_refuses_via_morphology() -> None:
|
||||
# "trained" is an adjective, not a pluralizable noun class -> cannot singularize.
|
||||
comp = comprehend("All pilots are trained.")
|
||||
assert isinstance(comp, Refusal)
|
||||
assert comp.reason == "unknown_morphology"
|
||||
|
||||
|
||||
def test_trailing_tokens_in_compare_refuses() -> None:
|
||||
comp = comprehend("Compare beta with beta in the same order.")
|
||||
assert isinstance(comp, Refusal)
|
||||
assert comp.reason == "unreadable_compare"
|
||||
|
|
|
|||
32
tests/test_comprehension_syllogism.py
Normal file
32
tests/test_comprehension_syllogism.py
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
"""Phase 2a-r3 — end-to-end: the comprehension reader scored on syllogism.
|
||||
|
||||
prose -> comprehend -> to_syllogism -> INDEPENDENT oracle -> answer vs gold.
|
||||
The load-bearing invariant: wrong == 0 (the reader refuses rather than emit a
|
||||
structure that yields a wrong answer). Coverage is allowed to be partial — a
|
||||
refusal is honest, a wrong commit is not.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from evals.comprehension.syllogism_runner import run
|
||||
|
||||
|
||||
def test_comprehension_syllogism_wrong_is_zero() -> None:
|
||||
report = run()
|
||||
assert report["wrong"] == 0, report["wrongs"]
|
||||
|
||||
|
||||
def test_comprehension_syllogism_has_real_coverage() -> None:
|
||||
report = run()
|
||||
assert report["correct"] > 0
|
||||
assert report["correct"] + report["refused"] == report["total"]
|
||||
|
||||
|
||||
def test_comprehension_syllogism_pinned_counts() -> None:
|
||||
# Pins the lane: 6 read end-to-end (Barbara/Celarent/Darii/Ferio/Datisi +
|
||||
# the invalid undistributed-middle, which the oracle correctly rejects);
|
||||
# 2 refused — the existential-import conclusion and the multi-word-NP case.
|
||||
report = run()
|
||||
assert report["correct"] == 6
|
||||
assert report["refused"] == 2
|
||||
assert report["total"] == 8
|
||||
33
tests/test_comprehension_total_ordering.py
Normal file
33
tests/test_comprehension_total_ordering.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
"""Phase 2a-r3 — end-to-end: the comprehension reader scored on total_ordering.
|
||||
|
||||
prose -> comprehend -> to_total_ordering -> INDEPENDENT oracle -> answer vs gold.
|
||||
The load-bearing invariant: wrong == 0 (the reader refuses rather than emit a
|
||||
structure that yields a wrong answer). Coverage is allowed to be partial — a
|
||||
refusal is honest, a wrong commit is not.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from evals.comprehension.total_ordering_runner import run
|
||||
|
||||
|
||||
def test_comprehension_total_ordering_wrong_is_zero() -> None:
|
||||
report = run()
|
||||
assert report["wrong"] == 0, report["wrongs"]
|
||||
|
||||
|
||||
def test_comprehension_total_ordering_has_real_coverage() -> None:
|
||||
report = run()
|
||||
assert report["correct"] > 0
|
||||
assert report["correct"] + report["refused"] == report["total"]
|
||||
|
||||
|
||||
def test_comprehension_total_ordering_pinned_counts() -> None:
|
||||
# Pins the lane: 4 read end-to-end (two sort chains + two transitive
|
||||
# compares); 4 refused — three multi-word-NP cases and one compare with
|
||||
# trailing tokens. No single multi-word-NP rule is wrong=0-safe here because
|
||||
# the gold canonicalizes multi-word NPs three contradictory ways.
|
||||
report = run()
|
||||
assert report["correct"] == 4
|
||||
assert report["refused"] == 4
|
||||
assert report["total"] == 8
|
||||
188
tests/test_comprehension_wrong_zero_property.py
Normal file
188
tests/test_comprehension_wrong_zero_property.py
Normal file
|
|
@ -0,0 +1,188 @@
|
|||
"""Generative wrong=0 hardening for the comprehension organ (anti-overfit).
|
||||
|
||||
The end-to-end lane tests prove wrong=0 on the 8-case staged gold lanes. This file
|
||||
proves the same invariant GENERALIZES: over hundreds of randomly generated
|
||||
single-word problems, the reader is FAITHFUL — it either refuses, or it reproduces
|
||||
the exact verdict the independent oracle gives on the ground-truth structure the
|
||||
prose encodes. It NEVER changes the answer.
|
||||
|
||||
Why this is non-circular: the generated structure ``S`` is the ground truth. We
|
||||
render prose ``P(S)`` with a trivial single-word renderer, then run the
|
||||
comprehension path ``oracle(project(comprehend(P(S))))`` and compare it to
|
||||
``oracle(S)`` computed directly. The oracle is independent of the reader; agreement
|
||||
means the reader carried the meaning losslessly, refusal means it declined — both
|
||||
are wrong=0. A reader that silently mis-read would commit a DIFFERENT verdict and
|
||||
the test would bite.
|
||||
|
||||
Single-word vocab on purpose: multi-word NPs are a known gold-canonicalization wall
|
||||
(the reader refuses them), so the generator stays in the readable single-word
|
||||
regime where faithfulness is the whole claim.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
|
||||
from evals.set_membership.oracle import OracleError as SetError
|
||||
from evals.set_membership.oracle import oracle_answer as set_oracle
|
||||
from evals.syllogism.oracle import OracleError as SylError
|
||||
from evals.syllogism.oracle import oracle_answer as syl_oracle
|
||||
from evals.total_ordering.oracle import OracleError as OrdError
|
||||
from evals.total_ordering.oracle import oracle_answer as ord_oracle
|
||||
from generate.meaning_graph.projectors import (
|
||||
to_set_membership,
|
||||
to_syllogism,
|
||||
to_total_ordering,
|
||||
)
|
||||
from generate.meaning_graph.reader import Refusal, comprehend
|
||||
|
||||
_TERMS = [f"t{i}" for i in range(8)] # single-word identifiers -> trivial plurals
|
||||
|
||||
|
||||
def _committed(comp_or_refusal, projector, oracle, error):
|
||||
"""Run the comprehension path; return the committed oracle answer or None
|
||||
(None == refused / unprojectable / oracle-refused — all wrong=0-safe)."""
|
||||
if isinstance(comp_or_refusal, Refusal):
|
||||
return None
|
||||
projected = projector(comp_or_refusal)
|
||||
if projected is None:
|
||||
return None
|
||||
try:
|
||||
return oracle(*projected)
|
||||
except error:
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Syllogism — the richest grammar (A/E/I/O premises + therefore-conclusion)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
_FORM_PREMISE = {
|
||||
"A": lambda s, p: f"All {s}s are {p}s.",
|
||||
"E": lambda s, p: f"No {s}s are {p}s.",
|
||||
"I": lambda s, p: f"Some {s}s are {p}s.",
|
||||
"O": lambda s, p: f"Some {s}s are not {p}s.",
|
||||
}
|
||||
_FORM_CONCLUSION = {
|
||||
"A": lambda s, p: f"Therefore all {s}s are {p}s.",
|
||||
"E": lambda s, p: f"Therefore no {s}s are {p}s.",
|
||||
"I": lambda s, p: f"Therefore some {s}s are {p}s.",
|
||||
"O": lambda s, p: f"Therefore some {s}s are not {p}s.",
|
||||
}
|
||||
|
||||
|
||||
def test_syllogism_reader_is_faithful_or_refuses() -> None:
|
||||
rng = random.Random(20260605)
|
||||
checked = committed = 0
|
||||
for _ in range(400):
|
||||
a, b, c = rng.sample(_TERMS, 3)
|
||||
# Two premises + a conclusion over the three terms; random forms.
|
||||
(s1, p1), (s2, p2), (sc, pc) = (
|
||||
rng.sample([a, b, c], 2),
|
||||
rng.sample([a, b, c], 2),
|
||||
rng.sample([a, b, c], 2),
|
||||
)
|
||||
f1, f2, fc = (rng.choice("AEIO"), rng.choice("AEIO"), rng.choice("AEIO"))
|
||||
prose = " ".join(
|
||||
[_FORM_PREMISE[f1](s1, p1), _FORM_PREMISE[f2](s2, p2), _FORM_CONCLUSION[fc](sc, pc)]
|
||||
)
|
||||
structure = {
|
||||
"terms": sorted({a, b, c}),
|
||||
"domain_size": 3,
|
||||
"premises": [
|
||||
{"form": f1, "subject": s1, "predicate": p1},
|
||||
{"form": f2, "subject": s2, "predicate": p2},
|
||||
],
|
||||
}
|
||||
query = {"kind": "validity", "conclusion": {"form": fc, "subject": sc, "predicate": pc}}
|
||||
try:
|
||||
expected = syl_oracle(structure, query)
|
||||
except SylError:
|
||||
expected = None # inconsistent premises -> ground truth refuses
|
||||
got = _committed(comprehend(prose), to_syllogism, syl_oracle, SylError)
|
||||
checked += 1
|
||||
if got is not None:
|
||||
committed += 1
|
||||
assert got == expected, (prose, got, expected)
|
||||
assert committed > 50 # the generator actually exercises the committed path
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Total ordering — random strict chains, sort + compare queries
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_total_ordering_reader_is_faithful_or_refuses() -> None:
|
||||
rng = random.Random(7)
|
||||
committed = 0
|
||||
for _ in range(300):
|
||||
n = rng.randint(2, 5)
|
||||
chain = rng.sample(_TERMS, n) # chain[0] < chain[1] < ... (strict)
|
||||
rels = [{"less": chain[i], "greater": chain[i + 1]} for i in range(n - 1)]
|
||||
# Render the chain as comparative clauses joined into one sentence.
|
||||
clauses = [f"{lo} is below {hi}" for lo, hi in zip(chain, chain[1:])]
|
||||
facts = ", and ".join(clauses) + "."
|
||||
if rng.random() < 0.5:
|
||||
order = rng.choice(["ascending", "descending"])
|
||||
prose = f"{facts} Sort {order}."
|
||||
query = {"kind": "sort", "order": order}
|
||||
else:
|
||||
x, y = rng.sample(chain, 2)
|
||||
prose = f"{facts} Compare {x} with {y}."
|
||||
query = {"kind": "compare", "left": x, "right": y}
|
||||
structure = {"items": sorted(set(chain)), "relations": rels}
|
||||
try:
|
||||
expected = ord_oracle(structure, query)
|
||||
except OrdError:
|
||||
expected = None
|
||||
got = _committed(comprehend(prose), to_total_ordering, ord_oracle, OrdError)
|
||||
if got is not None:
|
||||
committed += 1
|
||||
assert got == expected, (prose, got, expected)
|
||||
assert committed > 50
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Set membership — random members + subset chains, member + subset queries
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_set_membership_reader_is_faithful_or_refuses() -> None:
|
||||
rng = random.Random(99)
|
||||
committed = 0
|
||||
for _ in range(300):
|
||||
classes = rng.sample(_TERMS, rng.randint(2, 4))
|
||||
individuals = [f"e{i}" for i in range(rng.randint(1, 3))]
|
||||
member_facts = [(ind, rng.choice(classes)) for ind in individuals]
|
||||
# a short subset chain over a prefix of the classes
|
||||
subset_facts = [
|
||||
(classes[i], classes[i + 1]) for i in range(len(classes) - 1) if rng.random() < 0.7
|
||||
]
|
||||
member_lines = [f"{ind} is a {cls}." for ind, cls in member_facts]
|
||||
subset_lines = [f"All {a}s are {b}s." for a, b in subset_facts]
|
||||
sets = [
|
||||
{"id": c, "members": sorted({i for i, cl in member_facts if cl == c})} for c in classes
|
||||
]
|
||||
structure = {
|
||||
"elements": sorted(individuals),
|
||||
"sets": sets,
|
||||
"subsets": [{"subset": a, "superset": b} for a, b in subset_facts],
|
||||
}
|
||||
if rng.random() < 0.5 and individuals:
|
||||
ind = rng.choice(individuals)
|
||||
target = rng.choice(classes)
|
||||
prose = " ".join(member_lines + subset_lines + [f"Is {ind} a {target}?"])
|
||||
query = {"kind": "member", "element": ind, "set": target}
|
||||
else:
|
||||
a, b = rng.sample(classes, 2)
|
||||
prose = " ".join(member_lines + subset_lines + [f"Are all {a}s {b}s?"])
|
||||
query = {"kind": "subset", "subset": a, "superset": b}
|
||||
try:
|
||||
expected = set_oracle(structure, query)
|
||||
except SetError:
|
||||
expected = None
|
||||
got = _committed(comprehend(prose), to_set_membership, set_oracle, SetError)
|
||||
if got is not None:
|
||||
committed += 1
|
||||
assert got == expected, (prose, got, expected)
|
||||
assert committed > 50
|
||||
110
tests/test_meaning_graph_projectors.py
Normal file
110
tests/test_meaning_graph_projectors.py
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
"""Projector unit tests — MeaningGraph -> each oracle's input shape.
|
||||
|
||||
Projectors hold NO decision logic; they only re-shape. These tests pin the shape
|
||||
(so a drift that mis-maps a form or drops a fact bites) and the None-on-
|
||||
unprojectable contract (so the runner treats "nothing to ask" as a refusal, not a
|
||||
silent wrong).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from generate.meaning_graph.projectors import to_syllogism, to_total_ordering
|
||||
from generate.meaning_graph.reader import Comprehension, comprehend
|
||||
|
||||
|
||||
def _comp(text: str) -> Comprehension:
|
||||
comp = comprehend(text)
|
||||
assert isinstance(comp, Comprehension), text
|
||||
return comp
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# to_syllogism
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_to_syllogism_maps_premises_and_conclusion() -> None:
|
||||
comp = _comp(
|
||||
"All mammals are animals. All whales are mammals. "
|
||||
"Therefore all whales are animals."
|
||||
)
|
||||
projected = to_syllogism(comp)
|
||||
assert projected is not None
|
||||
structure, query = projected
|
||||
assert structure["terms"] == ["animal", "mammal", "whale"]
|
||||
assert structure["domain_size"] == 3
|
||||
assert {"form": "A", "subject": "mammal", "predicate": "animal"} in structure["premises"]
|
||||
assert {"form": "A", "subject": "whale", "predicate": "mammal"} in structure["premises"]
|
||||
assert query == {
|
||||
"kind": "validity",
|
||||
"conclusion": {"form": "A", "subject": "whale", "predicate": "animal"},
|
||||
}
|
||||
|
||||
|
||||
def test_to_syllogism_maps_e_i_o_forms() -> None:
|
||||
comp = _comp(
|
||||
"No reptiles are mammals. Some pets are reptiles. "
|
||||
"Therefore some pets are not mammals."
|
||||
)
|
||||
projected = to_syllogism(comp)
|
||||
assert projected is not None
|
||||
structure, query = projected
|
||||
forms = {(p["form"], p["subject"], p["predicate"]) for p in structure["premises"]}
|
||||
assert ("E", "reptile", "mammal") in forms
|
||||
assert ("I", "pet", "reptile") in forms
|
||||
assert query["conclusion"]["form"] == "O"
|
||||
|
||||
|
||||
def test_to_syllogism_none_when_no_conclusion() -> None:
|
||||
# Premises but no "therefore" conclusion -> nothing askable -> None.
|
||||
assert to_syllogism(_comp("All mammals are animals.")) is None
|
||||
|
||||
|
||||
def test_to_syllogism_none_for_membership_only() -> None:
|
||||
assert to_syllogism(_comp("Rhea is a raven.")) is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# to_total_ordering
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_to_total_ordering_sort_shape() -> None:
|
||||
comp = _comp(
|
||||
"Bronze is below silver, and silver is below gold. "
|
||||
"Sort them from lowest to highest."
|
||||
)
|
||||
projected = to_total_ordering(comp)
|
||||
assert projected is not None
|
||||
structure, query = projected
|
||||
assert structure["items"] == ["bronze", "gold", "silver"]
|
||||
assert {"less": "bronze", "greater": "silver"} in structure["relations"]
|
||||
assert {"less": "silver", "greater": "gold"} in structure["relations"]
|
||||
assert query == {"kind": "sort", "order": "ascending"}
|
||||
|
||||
|
||||
def test_to_total_ordering_compare_shape() -> None:
|
||||
comp = _comp("A is earlier than B. Compare a with b.")
|
||||
projected = to_total_ordering(comp)
|
||||
assert projected is not None
|
||||
structure, query = projected
|
||||
assert structure["items"] == ["a", "b"]
|
||||
assert query == {"kind": "compare", "left": "a", "right": "b"}
|
||||
|
||||
|
||||
def test_to_total_ordering_none_without_query() -> None:
|
||||
assert to_total_ordering(_comp("Bronze is below silver.")) is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Cross-projector neutrality — one MeaningGraph, distinct oracle shapes
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_one_graph_projects_only_to_its_domain() -> None:
|
||||
# A categorical-with-conclusion comprehension projects to syllogism but not to
|
||||
# total_ordering (no ordering query) — the interlingua is neutral, the
|
||||
# projector decides applicability.
|
||||
comp = _comp("All mammals are animals. Therefore all mammals are animals.")
|
||||
assert to_syllogism(comp) is not None
|
||||
assert to_total_ordering(comp) is None
|
||||
Loading…
Reference in a new issue