diff --git a/evals/capability_index/adapters.py b/evals/capability_index/adapters.py index dc9fa6fb..d65acaab 100644 --- a/evals/capability_index/adapters.py +++ b/evals/capability_index/adapters.py @@ -42,11 +42,38 @@ def dimensional_result() -> DomainResult: return DomainResult("dimensional", correct, wrong, refused) +def comprehension_set_membership_result() -> DomainResult: + from evals.comprehension.set_membership_runner import run + + c, w, r = _counts(run()) + return DomainResult("comprehension_set_membership", c, w, r) + + +def comprehension_syllogism_result() -> DomainResult: + from evals.comprehension.syllogism_runner import run + + c, w, r = _counts(run()) + return DomainResult("comprehension_syllogism", c, w, r) + + +def comprehension_total_ordering_result() -> DomainResult: + from evals.comprehension.total_ordering_runner import run + + c, w, r = _counts(run()) + return DomainResult("comprehension_total_ordering", c, w, r) + + #: The reasoning domains currently composed into the index (self-loading lanes). +#: The three ``comprehension_*`` lanes score the GENERAL comprehension reader +#: (prose -> MeaningGraph -> projection -> independent oracle), so the index now +#: measures comprehension breadth, not just structured-input reasoning. ADAPTERS = ( deductive_logic_result, relational_metric_result, dimensional_result, + comprehension_set_membership_result, + comprehension_syllogism_result, + comprehension_total_ordering_result, ) diff --git a/evals/capability_index/baseline.json b/evals/capability_index/baseline.json index 31d5c81b..b259f899 100644 --- a/evals/capability_index/baseline.json +++ b/evals/capability_index/baseline.json @@ -1,14 +1,38 @@ { - "capability_score": 0.919641, - "coverage_geomean": 0.919641, - "coverage_micro": 0.995962, + "capability_score": 0.814356, + "coverage_geomean": 0.814356, + "coverage_micro": 0.988266, "accuracy_micro": 1.0, - "breadth": 3, - "min_domain_coverage": 0.833333, + "breadth": 6, + "min_domain_coverage": 0.5, "wrong_total": 0, "assert_mode_valid": true, - "deterministic_digest": "115389d285babb5980b8b31c40d3a1c58ded1e1f9ae08ec428090bba18d42ed0", + "deterministic_digest": "0a98b9b41c23f5166d7e23bc194207ead53a89d5316dddc03194bd2739bb9753", "domains": [ + { + "domain": "comprehension_set_membership", + "correct": 8, + "wrong": 0, + "refused": 0, + "coverage": 1.0, + "accuracy": 1.0 + }, + { + "domain": "comprehension_syllogism", + "correct": 6, + "wrong": 0, + "refused": 2, + "coverage": 0.75, + "accuracy": 1.0 + }, + { + "domain": "comprehension_total_ordering", + "correct": 4, + "wrong": 0, + "refused": 4, + "coverage": 0.5, + "accuracy": 1.0 + }, { "domain": "deductive_logic", "correct": 716, diff --git a/evals/comprehension/syllogism_runner.py b/evals/comprehension/syllogism_runner.py new file mode 100644 index 00000000..c66fff16 --- /dev/null +++ b/evals/comprehension/syllogism_runner.py @@ -0,0 +1,70 @@ +"""Score the general comprehension reader on the syllogism gold lane. + +prose -> comprehend() -> to_syllogism() -> independent oracle -> answer vs gold. +A refusal (unreadable prose, unprojectable, or oracle-refused projection) is NOT a +wrong; only a committed answer that disagrees with gold is wrong (must stay 0). +""" + +from __future__ import annotations + +import json +import sys +from typing import Any + +from evals.syllogism.oracle import OracleError, oracle_answer +from evals.syllogism.runner import _load_cases +from generate.meaning_graph.projectors import to_syllogism +from generate.meaning_graph.reader import Refusal, comprehend + + +def run() -> dict[str, Any]: + cases = _load_cases() + correct = wrong = refused = 0 + wrongs: list[dict[str, Any]] = [] + + for case in cases: + comp = comprehend(case["text"]) + if isinstance(comp, Refusal): + refused += 1 + continue + projected = to_syllogism(comp) + if projected is None: + refused += 1 + continue + structure, query = projected + try: + got = oracle_answer(structure, query) + except OracleError: + refused += 1 + continue + if got == case.get("gold"): + correct += 1 + else: + wrong += 1 + wrongs.append( + {"id": case.get("id"), "got": got, "gold": case.get("gold"), "text": case["text"]} + ) + + return { + "domain": "comprehension_syllogism", + "total": len(cases), + "correct": correct, + "wrong": wrong, + "refused": refused, + "wrongs": wrongs, + "counts": {"correct": correct, "wrong": wrong, "refused": refused}, + } + + +def main() -> int: + report = run() + print(json.dumps({k: v for k, v in report.items() if k != "wrongs"}, indent=2, sort_keys=True)) + if report["wrong"]: + print("WRONG > 0 — comprehension produced a wrong committed answer:", file=sys.stderr) + print(json.dumps(report["wrongs"], indent=2), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/comprehension/total_ordering_runner.py b/evals/comprehension/total_ordering_runner.py new file mode 100644 index 00000000..d47de1d3 --- /dev/null +++ b/evals/comprehension/total_ordering_runner.py @@ -0,0 +1,70 @@ +"""Score the general comprehension reader on the total_ordering gold lane. + +prose -> comprehend() -> to_total_ordering() -> independent oracle -> answer vs gold. +A refusal (unreadable prose, unprojectable, or oracle-refused projection) is NOT a +wrong; only a committed answer that disagrees with gold is wrong (must stay 0). +""" + +from __future__ import annotations + +import json +import sys +from typing import Any + +from evals.total_ordering.oracle import OracleError, oracle_answer +from evals.total_ordering.runner import _load_cases +from generate.meaning_graph.projectors import to_total_ordering +from generate.meaning_graph.reader import Refusal, comprehend + + +def run() -> dict[str, Any]: + cases = _load_cases() + correct = wrong = refused = 0 + wrongs: list[dict[str, Any]] = [] + + for case in cases: + comp = comprehend(case["text"]) + if isinstance(comp, Refusal): + refused += 1 + continue + projected = to_total_ordering(comp) + if projected is None: + refused += 1 + continue + structure, query = projected + try: + got = oracle_answer(structure, query) + except OracleError: + refused += 1 + continue + if got == case.get("gold"): + correct += 1 + else: + wrong += 1 + wrongs.append( + {"id": case.get("id"), "got": got, "gold": case.get("gold"), "text": case["text"]} + ) + + return { + "domain": "comprehension_total_ordering", + "total": len(cases), + "correct": correct, + "wrong": wrong, + "refused": refused, + "wrongs": wrongs, + "counts": {"correct": correct, "wrong": wrong, "refused": refused}, + } + + +def main() -> int: + report = run() + print(json.dumps({k: v for k, v in report.items() if k != "wrongs"}, indent=2, sort_keys=True)) + if report["wrong"]: + print("WRONG > 0 — comprehension produced a wrong committed answer:", file=sys.stderr) + print(json.dumps(report["wrongs"], indent=2), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/generate/meaning_graph/projectors.py b/generate/meaning_graph/projectors.py index ec91ed8f..36696c73 100644 --- a/generate/meaning_graph/projectors.py +++ b/generate/meaning_graph/projectors.py @@ -55,3 +55,80 @@ def to_set_membership(comp: Comprehension) -> tuple[dict[str, Any], dict[str, An q = subset_queries[0] query = {"kind": "subset", "subset": q.arguments[0], "superset": q.arguments[1]} return structure, query + + +#: Categorical predicate -> Aristotelian form for the syllogism oracle. +_PRED_FORM = {"subset": "A", "disjoint": "E", "intersects": "I", "some_not": "O"} + +#: Finite-model domain size for syllogism validity (matches the gold lane). +_SYLLOGISM_DOMAIN_SIZE = 3 + + +def to_syllogism(comp: Comprehension) -> tuple[dict[str, Any], dict[str, Any]] | None: + """Project into ``(structure, query)`` for ``evals.syllogism.oracle``. + + Premises are the categorical relations (subset/disjoint/intersects/some_not); + the single categorical query is the conclusion. Returns ``None`` when the + comprehension is not exactly one categorical conclusion over >=1 premise — the + caller treats that as a refusal (nothing honestly askable of this oracle). + """ + graph = comp.meaning_graph + premises = [ + {"form": _PRED_FORM[r.predicate], "subject": r.arguments[0], "predicate": r.arguments[1]} + for r in graph.relations + if r.predicate in _PRED_FORM and not r.negated + ] + conclusions = [q for q in comp.queries if q.predicate in _PRED_FORM and not q.negated] + if not premises or len(comp.queries) != 1 or len(conclusions) != 1: + return None + + c = conclusions[0] + terms = sorted({e.entity_id for e in graph.entities if e.kind == "class"}) + if len(terms) < 2: + return None + structure = { + "terms": terms, + "domain_size": _SYLLOGISM_DOMAIN_SIZE, + "premises": premises, + } + query = { + "kind": "validity", + "conclusion": { + "form": _PRED_FORM[c.predicate], + "subject": c.arguments[0], + "predicate": c.arguments[1], + }, + } + return structure, query + + +def to_total_ordering(comp: Comprehension) -> tuple[dict[str, Any], dict[str, Any]] | None: + """Project into ``(structure, query)`` for ``evals.total_ordering.oracle``. + + Facts are ``less(lo, hi)`` relations; the single query is a sort or compare. + Returns ``None`` when the comprehension does not carry exactly one ordering + query — the caller treats that as a refusal. + """ + graph = comp.meaning_graph + less_facts = [ + (r.arguments[0], r.arguments[1]) + for r in graph.relations + if r.predicate == "less" and not r.negated + ] + order_queries = [q for q in comp.queries if q.predicate in ("sort", "compare")] + if len(comp.queries) != 1 or len(order_queries) != 1: + return None + + q = order_queries[0] + item_set = {item for pair in less_facts for item in pair} + if q.predicate == "compare": + item_set.update(q.arguments) + structure = { + "items": sorted(item_set), + "relations": [{"less": lo, "greater": hi} for lo, hi in less_facts], + } + if q.predicate == "sort": + query = {"kind": "sort", "order": q.arguments[0]} + else: + query = {"kind": "compare", "left": q.arguments[0], "right": q.arguments[1]} + return structure, query diff --git a/generate/meaning_graph/reader.py b/generate/meaning_graph/reader.py index d835ca5e..ab9d167e 100644 --- a/generate/meaning_graph/reader.py +++ b/generate/meaning_graph/reader.py @@ -1,24 +1,45 @@ -"""The general comprehension reader — disciplined Path β (Phase 2a). +"""The general comprehension reader — disciplined Path β (Phase 2a / 2b). Reads subject/relation/object STRUCTURE symbolically from the token sequence via domain-agnostic templates keyed on FUNCTION WORDS + ORDER, then mints content as -``MeaningGraph`` entities/relations. The field provably cannot recover this -structure (the holonomy fold is lossy/non-invertible — see the α-falsification in -``docs/analysis/phase2-general-comprehension-organ-scope-2026-06-05.md``); the -field's honest role is grounding/coherence, not composition. +``MeaningGraph`` entities/relations + ``Query`` objects. The field provably cannot +recover this structure (the holonomy fold is lossy/non-invertible — see the +α-falsification in ``docs/analysis/phase2-general-comprehension-organ-scope-2026-06-05.md``); +the field's honest role is grounding/coherence, not composition. Refusal-first: a clause that matches no template, a filler that is not a clean -identifier, an unrecognized plural, or a role conflict all REFUSE — never guess. -That keeps ``wrong=0`` at the comprehension layer (refusal is success; a fabricated -reading is the only failure). +identifier, an unrecognized plural, a MULTI-WORD noun phrase, or a role conflict +all REFUSE — never guess. That keeps ``wrong=0`` at the comprehension layer +(refusal is success; a fabricated reading is the only failure). -Templates (this increment): - - `` is a|an `` -> member(individual=X, class=Y) - - ``all are `` -> subset(subclass=sing(X), superclass=sing(Y)) - - ``is a|an ?`` -> Query member(X, Y) +Templates (function-word + order; domain-agnostic — the same templates read +animals, professions, geography, kin, metals, ranks): -Overfit-trap mitigation: templates key on function words + order (general), never -on domain content; the same templates read animals, professions, geography, kin. + membership / subsumption (set_membership shapes) + - `` is a|an `` -> member(individual=X, class=Y) + - ``the is a|an `` -> member(X, Y) + - ``is [the] a|an ?`` -> Query member(X, Y) + - ``are all ?`` -> Query subset(sing X, sing Y) + + categorical (syllogism shapes); subject/predicate are plural classes + - ``all are `` -> subset(sing X, sing Y) (A) + - ``no are `` -> disjoint(sing X, sing Y) (E) + - ``some are `` -> intersects(sing X, sing Y) (I) + - ``some are not `` -> some_not(sing X, sing Y) (O) + - ``therefore `` -> Query of the same predicate (the conclusion) + + ordering (total_ordering shapes); X/Y are single items + - `` [is] [than] `` -> less(X, Y) + - `` [is] [than] `` -> less(Y, X) + - ``compare with `` -> Query compare(X, Y) + - ``sort [ascending|descending]`` / + ``... order from to `` -> Query sort(order) + +Multi-word noun phrases are REFUSED on purpose: the staged gold lanes canonicalize +multi-word NPs three contradictory ways ("North station"->"north", "Level one"-> +"level_one", "metal objects"->"metal"), so no single general rule is wrong=0-safe. +Until the gold lanes carry a canonicalization contract, the only honest reading of +a multi-word NP is refusal. """ from __future__ import annotations @@ -49,6 +70,34 @@ _IRREGULAR_PLURALS = { "geese": "goose", } +# Categorical quantifier -> the MeaningGraph predicate it mints. The predicate +# vocabulary is shared between facts and the "therefore" conclusion query, and is +# neutral: the syllogism projector maps {subset:A, disjoint:E, intersects:I, +# some_not:O}, while the set_membership projector reads subset/member only. +_QUANTIFIER_PREDICATE = { + "all": "subset", + "no": "disjoint", + "some": "intersects", # "some ... not" is special-cased to some_not +} + +# Pairwise-order comparators. "less" means X < Y -> less(X, Y); "greater" means +# X > Y -> less(Y, X). Closed sets; an unknown comparator falls through to refusal. +_COMP_LESS = frozenset( + {"below", "under", "beneath", "lower", "shorter", "smaller", "earlier", "closer", "before"} +) +_COMP_GREATER = frozenset( + {"above", "over", "higher", "taller", "larger", "bigger", "greater", "later", "after", "farther", "further"} +) + +# Sort-direction endpoints: "from to " is ascending; reversed is +# descending. Closed sets; an unrecognized endpoint refuses the sort query. +_SORT_LOW = frozenset( + {"lowest", "shortest", "smallest", "least", "earliest", "bottom", "first", "low"} +) +_SORT_HIGH = frozenset( + {"highest", "tallest", "largest", "greatest", "most", "latest", "top", "last", "high"} +) + _SENTENCE_RE = re.compile(r"\s*([^.?!]+?)\s*([.?!])") @@ -78,6 +127,16 @@ class Refusal: detail: str = "" +class _Reject(Exception): + """Internal control-flow signal: a clause matched a template shape but is not + honestly readable (multi-word NP, bad morphology, role conflict). Carries the + typed ``Refusal`` so ``comprehend`` returns it verbatim.""" + + def __init__(self, reason: str, detail: str = "") -> None: + super().__init__(reason) + self.refusal = Refusal(reason, detail) + + def _identifier(word: str) -> str | None: """Normalize a content token to a clean identifier, or None to refuse.""" w = word.strip().lower() @@ -97,6 +156,27 @@ def _singularize(word: str) -> str | None: return None +def _one(words: list[str], detail: str) -> str: + """A single content token -> identifier, or REFUSE (multi-word / non-id).""" + if len(words) != 1: + raise _Reject("multiword_np", detail) + ident = _identifier(words[0]) + if ident is None: + raise _Reject("non_identifier_filler", detail) + return ident + + +def _one_class(words: list[str], detail: str) -> str: + """A single plural content token -> singular class id, or REFUSE.""" + word = _one(words, detail) + singular = _singularize(word) + if singular is None: + raise _Reject("unknown_morphology", detail) + if not singular.isidentifier(): + raise _Reject("non_identifier_filler", detail) + return singular + + def _split_sentences(text: str) -> list[tuple[str, str, int, int]]: """Return (body, terminator, start, end) for each sentence in *text*.""" out: list[tuple[str, str, int, int]] = [] @@ -114,6 +194,88 @@ def _split_sentences(text: str) -> list[tuple[str, str, int, int]]: return out +def _split_clauses(body: str) -> list[str]: + """Split a sentence body into clauses on commas; strip a leading and/or.""" + clauses: list[str] = [] + for part in body.split(","): + toks = part.split() + if toks and toks[0].lower() in {"and", "or"}: + toks = toks[1:] + if toks: + clauses.append(" ".join(toks)) + return clauses + + +def _parse_categorical(toks: list[str], detail: str) -> tuple[str, str, str] | None: + """`` are [not] `` -> (predicate, sub, sup); None if not it.""" + if not toks or toks[0] not in _QUANTIFIER_PREDICATE: + return None + if "are" not in toks: + return None + quant = toks[0] + are_idx = toks.index("are") + subject_words = toks[1:are_idx] + predicate_words = toks[are_idx + 1:] + negated = bool(predicate_words) and predicate_words[0] == "not" + if negated: + predicate_words = predicate_words[1:] + if not subject_words or not predicate_words: + raise _Reject("incomplete_categorical", detail) + sub = _one_class(subject_words, detail) + sup = _one_class(predicate_words, detail) + if quant == "some" and negated: + predicate = "some_not" + elif negated: + # "no ... not" / "all ... not" are out of this grammar; refuse, don't guess. + raise _Reject("unsupported_negation", detail) + else: + predicate = _QUANTIFIER_PREDICATE[quant] + return predicate, sub, sup + + +def _parse_comparative(toks: list[str], detail: str) -> tuple[str, str] | None: + """`` [is] [than] `` -> (less_item, greater_item); None if not it.""" + comp_idx = next( + (i for i, t in enumerate(toks) if t in _COMP_LESS or t in _COMP_GREATER), + None, + ) + if comp_idx is None: + return None + left = toks[:comp_idx] + if left and left[-1] == "is": + left = left[:-1] + right = toks[comp_idx + 1:] + if right and right[0] == "than": + right = right[1:] + if not left or not right: + raise _Reject("incomplete_comparative", detail) + x = _one(left, detail) + y = _one(right, detail) + if toks[comp_idx] in _COMP_LESS: + return x, y # X < Y + return y, x # X > Y -> less(Y, X) + + +def _parse_sort(toks: list[str], detail: str) -> str | None: + """A sort request -> "ascending"|"descending"; None if not a sort request.""" + is_sort = toks and toks[0] == "sort" + is_order = "order" in toks and "from" in toks + if not (is_sort or is_order): + return None + if "ascending" in toks: + return "ascending" + if "descending" in toks: + return "descending" + if "from" in toks and "to" in toks: + lo = toks[toks.index("from") + 1] if toks.index("from") + 1 < len(toks) else "" + hi = toks[toks.index("to") + 1] if toks.index("to") + 1 < len(toks) else "" + if lo in _SORT_LOW and hi in _SORT_HIGH: + return "ascending" + if lo in _SORT_HIGH and hi in _SORT_LOW: + return "descending" + raise _Reject("ambiguous_sort_order", detail) + + def comprehend(text: str, source_id: str = "input") -> Comprehension | Refusal: """Comprehend *text* into a MeaningGraph + queries, or a typed Refusal.""" if not text or not text.strip(): @@ -125,106 +287,133 @@ def comprehend(text: str, source_id: str = "input") -> Comprehension | Refusal: role_kind: dict[str, str] = {} span_for: dict[str, MeaningSpan] = {} - members: list[tuple[str, str, MeaningSpan]] = [] - subsets: list[tuple[str, str, MeaningSpan]] = [] + relations: list[tuple[str, tuple[str, ...], MeaningSpan]] = [] queries: list[Query] = [] - def claim(entity_id: str, kind: str, span: MeaningSpan) -> bool: + def claim(entity_id: str, kind: str, span: MeaningSpan) -> None: prior = role_kind.get(entity_id) if prior is not None and prior != kind: - return False + raise _Reject("role_conflict", entity_id) role_kind[entity_id] = kind span_for.setdefault(entity_id, span) - return True - for body, terminator, start, end in sentences: - toks = body.lower().split() - span = MeaningSpan(source_id=source_id, start=start, end=end, text=text[start:end]) - is_question = terminator == "?" - - # Template: subset query ``are all ?`` - if is_question and len(toks) == 4 and toks[0] == "are" and toks[1] == "all": - raw_sub, raw_sup = _identifier(toks[2]), _identifier(toks[3]) - if raw_sub is None or raw_sup is None: - return Refusal("non_identifier_filler", body) - sub, sup = _singularize(raw_sub), _singularize(raw_sup) - if sub is None or sup is None: - return Refusal("unknown_morphology", body) - if not sub.isidentifier() or not sup.isidentifier(): - return Refusal("non_identifier_filler", body) - if not claim(sub, "class", span) or not claim(sup, "class", span): - return Refusal("role_conflict", body) - queries.append(Query("subset", (sub, sup), span)) - continue - - # Template: definite-NP membership query ``is the a|an ?`` - if is_question and len(toks) == 5 and toks[0] == "is" and toks[1] == "the" and toks[3] in _ARTICLES: - name, cls = _identifier(toks[2]), _identifier(toks[4]) - if name is None or cls is None: - return Refusal("non_identifier_filler", body) - if not claim(name, "individual", span) or not claim(cls, "class", span): - return Refusal("role_conflict", body) - queries.append(Query("member", (name, cls), span)) - continue - - # Template: query ``is a|an ?`` - if is_question and len(toks) == 4 and toks[0] == "is" and toks[2] in _ARTICLES: - name, cls = _identifier(toks[1]), _identifier(toks[3]) - if name is None or cls is None: - return Refusal("non_identifier_filler", body) - if not claim(name, "individual", span) or not claim(cls, "class", span): - return Refusal("role_conflict", body) - queries.append(Query("member", (name, cls), span)) - continue - - # Template: definite-NP membership ``the is a|an `` - if not is_question and len(toks) == 5 and toks[0] == "the" and toks[2] == "is" and toks[3] in _ARTICLES: - name, cls = _identifier(toks[1]), _identifier(toks[4]) - if name is None or cls is None: - return Refusal("non_identifier_filler", body) - if not claim(name, "individual", span) or not claim(cls, "class", span): - return Refusal("role_conflict", body) - members.append((name, cls, span)) - continue - - # Template: membership `` is a|an `` - if not is_question and len(toks) == 4 and toks[1] == "is" and toks[2] in _ARTICLES: - name, cls = _identifier(toks[0]), _identifier(toks[3]) - if name is None or cls is None: - return Refusal("non_identifier_filler", body) - if not claim(name, "individual", span) or not claim(cls, "class", span): - return Refusal("role_conflict", body) - members.append((name, cls, span)) - continue - - # Template: subsumption ``all are `` - if not is_question and len(toks) == 4 and toks[0] == "all" and toks[2] == "are": - raw_sub, raw_sup = _identifier(toks[1]), _identifier(toks[3]) - if raw_sub is None or raw_sup is None: - return Refusal("non_identifier_filler", body) - sub, sup = _singularize(raw_sub), _singularize(raw_sup) - if sub is None or sup is None: - return Refusal("unknown_morphology", body) - if not sub.isidentifier() or not sup.isidentifier(): - return Refusal("non_identifier_filler", body) - if not claim(sub, "class", span) or not claim(sup, "class", span): - return Refusal("role_conflict", body) - subsets.append((sub, sup, span)) - continue - - return Refusal("no_template_match", body) + try: + for body, terminator, start, end in sentences: + span = MeaningSpan( + source_id=source_id, start=start, end=end, text=text[start:end] + ) + is_question = terminator == "?" + for clause in _split_clauses(body): + _read_clause(clause, is_question, span, claim, relations, queries) + except _Reject as rej: + return rej.refusal try: entities = tuple( Entity(entity_id=eid, name=eid, span=span_for[eid], kind=role_kind[eid]) for eid in sorted(role_kind) ) - relations = tuple( - [Relation("member", (ind, cls), sp) for ind, cls, sp in members] - + [Relation("subset", (sub, sup), sp) for sub, sup, sp in subsets] + graph = MeaningGraph( + entities=entities, + relations=tuple( + Relation(pred, args, sp) for pred, args, sp in relations + ), ) - graph = MeaningGraph(entities=entities, relations=relations) except MeaningGraphError as exc: # defensive — construction invariants return Refusal("invalid_graph", repr(exc)) return Comprehension(meaning_graph=graph, queries=tuple(queries)) + + +def _read_clause( + clause: str, + question: bool, + span: MeaningSpan, + claim, + relations: list[tuple[str, tuple[str, ...], MeaningSpan]], + queries: list[Query], +) -> None: + """Match one clause against the templates; mutate accumulators or REFUSE.""" + toks = clause.strip().lower().split() + if not toks: + raise _Reject("empty_clause", clause) + + # --- ordering queries (keyword-led; terminator-independent) ------------- # + if toks[0] == "compare": + if len(toks) != 4 or toks[2] != "with": + raise _Reject("unreadable_compare", clause) + left = _one([toks[1]], clause) + right = _one([toks[3]], clause) + for item in (left, right): + claim(item, "item", span) + queries.append(Query("compare", (left, right), span)) + return + + order = _parse_sort(toks, clause) + if order is not None: + queries.append(Query("sort", (order,), span)) + return + + # --- syllogism conclusion: "therefore " ------------------- # + if toks[0] == "therefore": + parsed = _parse_categorical(toks[1:], clause) + if parsed is None: + raise _Reject("unreadable_conclusion", clause) + predicate, sub, sup = parsed + claim(sub, "class", span) + claim(sup, "class", span) + queries.append(Query(predicate, (sub, sup), span)) + return + + # --- membership query: "is [the] a|an ?" ------------------------ # + if question and toks[0] == "is": + rest = toks[1:] + if rest and rest[0] == "the": + rest = rest[1:] + if len(rest) == 3 and rest[1] in _ARTICLES: + name = _one([rest[0]], clause) + cls = _one([rest[2]], clause) + claim(name, "individual", span) + claim(cls, "class", span) + queries.append(Query("member", (name, cls), span)) + return + raise _Reject("unreadable_member_query", clause) + + # --- subset query: "are all ?" -------------------------------- # + if question and len(toks) == 4 and toks[0] == "are" and toks[1] == "all": + sub = _one_class([toks[2]], clause) + sup = _one_class([toks[3]], clause) + claim(sub, "class", span) + claim(sup, "class", span) + queries.append(Query("subset", (sub, sup), span)) + return + + # --- categorical fact: " are [not] " ------------------- # + categorical = _parse_categorical(toks, clause) + if categorical is not None: + predicate, sub, sup = categorical + claim(sub, "class", span) + claim(sup, "class", span) + relations.append((predicate, (sub, sup), span)) + return + + # --- membership fact: "[the] is a|an " -------------------------- # + body_toks = toks[1:] if toks[0] == "the" else toks + if len(body_toks) == 4 and body_toks[1] == "is" and body_toks[2] in _ARTICLES: + name = _one([body_toks[0]], clause) + cls = _one([body_toks[3]], clause) + claim(name, "individual", span) + claim(cls, "class", span) + relations.append(("member", (name, cls), span)) + return + + # --- ordering fact: " [is] [than] " ----------------------- # + comparative = _parse_comparative(toks, clause) + if comparative is not None: + lo, hi = comparative + claim(lo, "item", span) + claim(hi, "item", span) + relations.append(("less", (lo, hi), span)) + return + + raise _Reject("no_template_match", clause) diff --git a/tests/test_capability_index.py b/tests/test_capability_index.py index ac03754e..edc80c32 100644 --- a/tests/test_capability_index.py +++ b/tests/test_capability_index.py @@ -87,8 +87,9 @@ def test_empty_index_is_well_defined() -> None: def test_real_lanes_compose_into_the_index_with_wrong_zero() -> None: - # The Phase-1b baseline: the three self-loading independent-gold reasoning - # lanes compose into the cross-domain index with zero wrong commits. + # The baseline: three structured-input reasoning lanes PLUS the three + # comprehension lanes (prose -> MeaningGraph -> projection -> independent + # oracle) compose into the cross-domain index with zero wrong commits. from evals.capability_index.adapters import collect_domain_results collection = collect_domain_results() @@ -96,11 +97,14 @@ def test_real_lanes_compose_into_the_index_with_wrong_zero() -> None: idx = aggregate(list(collection.results)) assert idx.wrong_total == 0 assert idx.assert_mode_valid - assert idx.breadth == 3 # deductive_logic + dimensional + relational_metric + assert idx.breadth == 6 assert {d.domain for d in idx.domains} == { "deductive_logic", "dimensional", "relational_metric", + "comprehension_set_membership", + "comprehension_syllogism", + "comprehension_total_ordering", } assert idx.capability_score > 0.5 # real, non-trivial cross-domain capability diff --git a/tests/test_comprehension_reader.py b/tests/test_comprehension_reader.py index ef895eb1..30177050 100644 --- a/tests/test_comprehension_reader.py +++ b/tests/test_comprehension_reader.py @@ -157,3 +157,163 @@ def test_refuses_non_identifier_filler() -> None: def test_empty_input_refuses() -> None: assert isinstance(comprehend(""), Refusal) assert isinstance(comprehend(" "), Refusal) + + +# --------------------------------------------------------------------------- # +# Categorical premises — E / I / O forms (syllogism shapes) +# --------------------------------------------------------------------------- # + + +def test_categorical_no_is_disjoint() -> None: + comp = comprehend("No reptiles are mammals.") + assert isinstance(comp, Comprehension) + assert _rel(comp, "disjoint") == (("disjoint", ("reptile", "mammal")),) + + +def test_categorical_some_is_intersects() -> None: + comp = comprehend("Some students are poets.") + assert isinstance(comp, Comprehension) + assert _rel(comp, "intersects") == (("intersects", ("student", "poet")),) + + +def test_categorical_some_not_is_some_not() -> None: + comp = comprehend("Some pets are not reptiles.") + assert isinstance(comp, Comprehension) + assert _rel(comp, "some_not") == (("some_not", ("pet", "reptile")),) + + +# --------------------------------------------------------------------------- # +# "Therefore " -> conclusion QUERY (same neutral predicates) +# --------------------------------------------------------------------------- # + + +def test_therefore_conclusion_is_a_query_not_a_fact() -> None: + comp = comprehend("All whales are mammals. Therefore all whales are mammals.") + assert isinstance(comp, Comprehension) + # the premise is a fact; the conclusion is a query of the same predicate. + assert _rel(comp, "subset") == (("subset", ("whale", "mammal")),) + assert len(comp.queries) == 1 + assert comp.queries[0].predicate == "subset" + assert comp.queries[0].arguments == ("whale", "mammal") + + +def test_therefore_maps_each_quantifier_to_its_predicate() -> None: + for tail, predicate in [ + ("all dogs are animals", "subset"), + ("no dogs are cats", "disjoint"), + ("some dogs are pets", "intersects"), + ("some dogs are not cats", "some_not"), + ]: + comp = comprehend(f"Therefore {tail}.") + assert isinstance(comp, Comprehension), tail + assert comp.queries[0].predicate == predicate, tail + + +# --------------------------------------------------------------------------- # +# Ordering — comparative facts (total_ordering shapes) +# --------------------------------------------------------------------------- # + + +def test_comparative_less_direction() -> None: + comp = comprehend("Bronze is below silver.") + assert isinstance(comp, Comprehension) + assert _rel(comp, "less") == (("less", ("bronze", "silver")),) + assert _entity_kind(comp, "bronze") == "item" + + +def test_comparative_greater_direction_reverses() -> None: + # "X is taller than Y" means X > Y, i.e. less(Y, X). + comp = comprehend("Oak is taller than birch.") + assert isinstance(comp, Comprehension) + assert _rel(comp, "less") == (("less", ("birch", "oak")),) + + +def test_comparative_elided_verb() -> None: + # "Venus closer than Earth" (no copula) still reads. + comp = comprehend("Venus closer than Earth.") + assert isinstance(comp, Comprehension) + assert _rel(comp, "less") == (("less", ("venus", "earth")),) + + +def test_comparative_clause_splitting_on_comma_and() -> None: + comp = comprehend("A is earlier than B, and B is earlier than C.") + assert isinstance(comp, Comprehension) + assert _rel(comp, "less") == (("less", ("a", "b")), ("less", ("b", "c"))) + + +# --------------------------------------------------------------------------- # +# Ordering — sort / compare QUERIES +# --------------------------------------------------------------------------- # + + +def test_sort_query_lowest_to_highest_is_ascending() -> None: + comp = comprehend("Sort them from lowest to highest.") + assert isinstance(comp, Comprehension) + assert comp.queries[0].predicate == "sort" + assert comp.queries[0].arguments == ("ascending",) + + +def test_sort_query_explicit_descending() -> None: + comp = comprehend("Sort descending.") + assert isinstance(comp, Comprehension) + assert comp.queries[0].arguments == ("descending",) + + +def test_sort_query_order_question_form() -> None: + comp = comprehend("Which is the height order from shortest to tallest?") + assert isinstance(comp, Comprehension) + assert comp.queries[0].predicate == "sort" + assert comp.queries[0].arguments == ("ascending",) + + +def test_compare_query() -> None: + comp = comprehend("Compare north with south.") + assert isinstance(comp, Comprehension) + assert comp.queries[0].predicate == "compare" + assert comp.queries[0].arguments == ("north", "south") + + +# --------------------------------------------------------------------------- # +# Generality — SAME comparative template, DISTINCT domains (anti-overfit) +# --------------------------------------------------------------------------- # + + +def test_comparative_template_generalizes_across_domains() -> None: + for text, lo, hi in [ + ("Bronze is below silver.", "bronze", "silver"), # metals + ("Monday is earlier than tuesday.", "monday", "tuesday"), # time + ("Birch is shorter than oak.", "birch", "oak"), # height + ]: + comp = comprehend(text) + assert isinstance(comp, Comprehension), text + assert _rel(comp, "less") == (("less", (lo, hi)),), text + + +# --------------------------------------------------------------------------- # +# Multi-word NP — REFUSE (the wrong=0 guard: no general canonicalization exists) +# --------------------------------------------------------------------------- # + + +def test_multiword_np_in_categorical_refuses() -> None: + comp = comprehend("No metal objects are soft objects.") + assert isinstance(comp, Refusal) + assert comp.reason == "multiword_np" + + +def test_multiword_np_in_comparative_refuses() -> None: + comp = comprehend("North station comes after central.") + assert isinstance(comp, Refusal) + assert comp.reason == "multiword_np" + + +def test_adjectival_predicate_refuses_via_morphology() -> None: + # "trained" is an adjective, not a pluralizable noun class -> cannot singularize. + comp = comprehend("All pilots are trained.") + assert isinstance(comp, Refusal) + assert comp.reason == "unknown_morphology" + + +def test_trailing_tokens_in_compare_refuses() -> None: + comp = comprehend("Compare beta with beta in the same order.") + assert isinstance(comp, Refusal) + assert comp.reason == "unreadable_compare" diff --git a/tests/test_comprehension_syllogism.py b/tests/test_comprehension_syllogism.py new file mode 100644 index 00000000..0d684abd --- /dev/null +++ b/tests/test_comprehension_syllogism.py @@ -0,0 +1,32 @@ +"""Phase 2a-r3 — end-to-end: the comprehension reader scored on syllogism. + +prose -> comprehend -> to_syllogism -> INDEPENDENT oracle -> answer vs gold. +The load-bearing invariant: wrong == 0 (the reader refuses rather than emit a +structure that yields a wrong answer). Coverage is allowed to be partial — a +refusal is honest, a wrong commit is not. +""" + +from __future__ import annotations + +from evals.comprehension.syllogism_runner import run + + +def test_comprehension_syllogism_wrong_is_zero() -> None: + report = run() + assert report["wrong"] == 0, report["wrongs"] + + +def test_comprehension_syllogism_has_real_coverage() -> None: + report = run() + assert report["correct"] > 0 + assert report["correct"] + report["refused"] == report["total"] + + +def test_comprehension_syllogism_pinned_counts() -> None: + # Pins the lane: 6 read end-to-end (Barbara/Celarent/Darii/Ferio/Datisi + + # the invalid undistributed-middle, which the oracle correctly rejects); + # 2 refused — the existential-import conclusion and the multi-word-NP case. + report = run() + assert report["correct"] == 6 + assert report["refused"] == 2 + assert report["total"] == 8 diff --git a/tests/test_comprehension_total_ordering.py b/tests/test_comprehension_total_ordering.py new file mode 100644 index 00000000..b971ec59 --- /dev/null +++ b/tests/test_comprehension_total_ordering.py @@ -0,0 +1,33 @@ +"""Phase 2a-r3 — end-to-end: the comprehension reader scored on total_ordering. + +prose -> comprehend -> to_total_ordering -> INDEPENDENT oracle -> answer vs gold. +The load-bearing invariant: wrong == 0 (the reader refuses rather than emit a +structure that yields a wrong answer). Coverage is allowed to be partial — a +refusal is honest, a wrong commit is not. +""" + +from __future__ import annotations + +from evals.comprehension.total_ordering_runner import run + + +def test_comprehension_total_ordering_wrong_is_zero() -> None: + report = run() + assert report["wrong"] == 0, report["wrongs"] + + +def test_comprehension_total_ordering_has_real_coverage() -> None: + report = run() + assert report["correct"] > 0 + assert report["correct"] + report["refused"] == report["total"] + + +def test_comprehension_total_ordering_pinned_counts() -> None: + # Pins the lane: 4 read end-to-end (two sort chains + two transitive + # compares); 4 refused — three multi-word-NP cases and one compare with + # trailing tokens. No single multi-word-NP rule is wrong=0-safe here because + # the gold canonicalizes multi-word NPs three contradictory ways. + report = run() + assert report["correct"] == 4 + assert report["refused"] == 4 + assert report["total"] == 8 diff --git a/tests/test_comprehension_wrong_zero_property.py b/tests/test_comprehension_wrong_zero_property.py new file mode 100644 index 00000000..a64e2eb7 --- /dev/null +++ b/tests/test_comprehension_wrong_zero_property.py @@ -0,0 +1,188 @@ +"""Generative wrong=0 hardening for the comprehension organ (anti-overfit). + +The end-to-end lane tests prove wrong=0 on the 8-case staged gold lanes. This file +proves the same invariant GENERALIZES: over hundreds of randomly generated +single-word problems, the reader is FAITHFUL — it either refuses, or it reproduces +the exact verdict the independent oracle gives on the ground-truth structure the +prose encodes. It NEVER changes the answer. + +Why this is non-circular: the generated structure ``S`` is the ground truth. We +render prose ``P(S)`` with a trivial single-word renderer, then run the +comprehension path ``oracle(project(comprehend(P(S))))`` and compare it to +``oracle(S)`` computed directly. The oracle is independent of the reader; agreement +means the reader carried the meaning losslessly, refusal means it declined — both +are wrong=0. A reader that silently mis-read would commit a DIFFERENT verdict and +the test would bite. + +Single-word vocab on purpose: multi-word NPs are a known gold-canonicalization wall +(the reader refuses them), so the generator stays in the readable single-word +regime where faithfulness is the whole claim. +""" + +from __future__ import annotations + +import random + +from evals.set_membership.oracle import OracleError as SetError +from evals.set_membership.oracle import oracle_answer as set_oracle +from evals.syllogism.oracle import OracleError as SylError +from evals.syllogism.oracle import oracle_answer as syl_oracle +from evals.total_ordering.oracle import OracleError as OrdError +from evals.total_ordering.oracle import oracle_answer as ord_oracle +from generate.meaning_graph.projectors import ( + to_set_membership, + to_syllogism, + to_total_ordering, +) +from generate.meaning_graph.reader import Refusal, comprehend + +_TERMS = [f"t{i}" for i in range(8)] # single-word identifiers -> trivial plurals + + +def _committed(comp_or_refusal, projector, oracle, error): + """Run the comprehension path; return the committed oracle answer or None + (None == refused / unprojectable / oracle-refused — all wrong=0-safe).""" + if isinstance(comp_or_refusal, Refusal): + return None + projected = projector(comp_or_refusal) + if projected is None: + return None + try: + return oracle(*projected) + except error: + return None + + +# --------------------------------------------------------------------------- # +# Syllogism — the richest grammar (A/E/I/O premises + therefore-conclusion) +# --------------------------------------------------------------------------- # + +_FORM_PREMISE = { + "A": lambda s, p: f"All {s}s are {p}s.", + "E": lambda s, p: f"No {s}s are {p}s.", + "I": lambda s, p: f"Some {s}s are {p}s.", + "O": lambda s, p: f"Some {s}s are not {p}s.", +} +_FORM_CONCLUSION = { + "A": lambda s, p: f"Therefore all {s}s are {p}s.", + "E": lambda s, p: f"Therefore no {s}s are {p}s.", + "I": lambda s, p: f"Therefore some {s}s are {p}s.", + "O": lambda s, p: f"Therefore some {s}s are not {p}s.", +} + + +def test_syllogism_reader_is_faithful_or_refuses() -> None: + rng = random.Random(20260605) + checked = committed = 0 + for _ in range(400): + a, b, c = rng.sample(_TERMS, 3) + # Two premises + a conclusion over the three terms; random forms. + (s1, p1), (s2, p2), (sc, pc) = ( + rng.sample([a, b, c], 2), + rng.sample([a, b, c], 2), + rng.sample([a, b, c], 2), + ) + f1, f2, fc = (rng.choice("AEIO"), rng.choice("AEIO"), rng.choice("AEIO")) + prose = " ".join( + [_FORM_PREMISE[f1](s1, p1), _FORM_PREMISE[f2](s2, p2), _FORM_CONCLUSION[fc](sc, pc)] + ) + structure = { + "terms": sorted({a, b, c}), + "domain_size": 3, + "premises": [ + {"form": f1, "subject": s1, "predicate": p1}, + {"form": f2, "subject": s2, "predicate": p2}, + ], + } + query = {"kind": "validity", "conclusion": {"form": fc, "subject": sc, "predicate": pc}} + try: + expected = syl_oracle(structure, query) + except SylError: + expected = None # inconsistent premises -> ground truth refuses + got = _committed(comprehend(prose), to_syllogism, syl_oracle, SylError) + checked += 1 + if got is not None: + committed += 1 + assert got == expected, (prose, got, expected) + assert committed > 50 # the generator actually exercises the committed path + + +# --------------------------------------------------------------------------- # +# Total ordering — random strict chains, sort + compare queries +# --------------------------------------------------------------------------- # + + +def test_total_ordering_reader_is_faithful_or_refuses() -> None: + rng = random.Random(7) + committed = 0 + for _ in range(300): + n = rng.randint(2, 5) + chain = rng.sample(_TERMS, n) # chain[0] < chain[1] < ... (strict) + rels = [{"less": chain[i], "greater": chain[i + 1]} for i in range(n - 1)] + # Render the chain as comparative clauses joined into one sentence. + clauses = [f"{lo} is below {hi}" for lo, hi in zip(chain, chain[1:])] + facts = ", and ".join(clauses) + "." + if rng.random() < 0.5: + order = rng.choice(["ascending", "descending"]) + prose = f"{facts} Sort {order}." + query = {"kind": "sort", "order": order} + else: + x, y = rng.sample(chain, 2) + prose = f"{facts} Compare {x} with {y}." + query = {"kind": "compare", "left": x, "right": y} + structure = {"items": sorted(set(chain)), "relations": rels} + try: + expected = ord_oracle(structure, query) + except OrdError: + expected = None + got = _committed(comprehend(prose), to_total_ordering, ord_oracle, OrdError) + if got is not None: + committed += 1 + assert got == expected, (prose, got, expected) + assert committed > 50 + + +# --------------------------------------------------------------------------- # +# Set membership — random members + subset chains, member + subset queries +# --------------------------------------------------------------------------- # + + +def test_set_membership_reader_is_faithful_or_refuses() -> None: + rng = random.Random(99) + committed = 0 + for _ in range(300): + classes = rng.sample(_TERMS, rng.randint(2, 4)) + individuals = [f"e{i}" for i in range(rng.randint(1, 3))] + member_facts = [(ind, rng.choice(classes)) for ind in individuals] + # a short subset chain over a prefix of the classes + subset_facts = [ + (classes[i], classes[i + 1]) for i in range(len(classes) - 1) if rng.random() < 0.7 + ] + member_lines = [f"{ind} is a {cls}." for ind, cls in member_facts] + subset_lines = [f"All {a}s are {b}s." for a, b in subset_facts] + sets = [ + {"id": c, "members": sorted({i for i, cl in member_facts if cl == c})} for c in classes + ] + structure = { + "elements": sorted(individuals), + "sets": sets, + "subsets": [{"subset": a, "superset": b} for a, b in subset_facts], + } + if rng.random() < 0.5 and individuals: + ind = rng.choice(individuals) + target = rng.choice(classes) + prose = " ".join(member_lines + subset_lines + [f"Is {ind} a {target}?"]) + query = {"kind": "member", "element": ind, "set": target} + else: + a, b = rng.sample(classes, 2) + prose = " ".join(member_lines + subset_lines + [f"Are all {a}s {b}s?"]) + query = {"kind": "subset", "subset": a, "superset": b} + try: + expected = set_oracle(structure, query) + except SetError: + expected = None + got = _committed(comprehend(prose), to_set_membership, set_oracle, SetError) + if got is not None: + committed += 1 + assert got == expected, (prose, got, expected) + assert committed > 50 diff --git a/tests/test_meaning_graph_projectors.py b/tests/test_meaning_graph_projectors.py new file mode 100644 index 00000000..73f2d4e8 --- /dev/null +++ b/tests/test_meaning_graph_projectors.py @@ -0,0 +1,110 @@ +"""Projector unit tests — MeaningGraph -> each oracle's input shape. + +Projectors hold NO decision logic; they only re-shape. These tests pin the shape +(so a drift that mis-maps a form or drops a fact bites) and the None-on- +unprojectable contract (so the runner treats "nothing to ask" as a refusal, not a +silent wrong). +""" + +from __future__ import annotations + +from generate.meaning_graph.projectors import to_syllogism, to_total_ordering +from generate.meaning_graph.reader import Comprehension, comprehend + + +def _comp(text: str) -> Comprehension: + comp = comprehend(text) + assert isinstance(comp, Comprehension), text + return comp + + +# --------------------------------------------------------------------------- # +# to_syllogism +# --------------------------------------------------------------------------- # + + +def test_to_syllogism_maps_premises_and_conclusion() -> None: + comp = _comp( + "All mammals are animals. All whales are mammals. " + "Therefore all whales are animals." + ) + projected = to_syllogism(comp) + assert projected is not None + structure, query = projected + assert structure["terms"] == ["animal", "mammal", "whale"] + assert structure["domain_size"] == 3 + assert {"form": "A", "subject": "mammal", "predicate": "animal"} in structure["premises"] + assert {"form": "A", "subject": "whale", "predicate": "mammal"} in structure["premises"] + assert query == { + "kind": "validity", + "conclusion": {"form": "A", "subject": "whale", "predicate": "animal"}, + } + + +def test_to_syllogism_maps_e_i_o_forms() -> None: + comp = _comp( + "No reptiles are mammals. Some pets are reptiles. " + "Therefore some pets are not mammals." + ) + projected = to_syllogism(comp) + assert projected is not None + structure, query = projected + forms = {(p["form"], p["subject"], p["predicate"]) for p in structure["premises"]} + assert ("E", "reptile", "mammal") in forms + assert ("I", "pet", "reptile") in forms + assert query["conclusion"]["form"] == "O" + + +def test_to_syllogism_none_when_no_conclusion() -> None: + # Premises but no "therefore" conclusion -> nothing askable -> None. + assert to_syllogism(_comp("All mammals are animals.")) is None + + +def test_to_syllogism_none_for_membership_only() -> None: + assert to_syllogism(_comp("Rhea is a raven.")) is None + + +# --------------------------------------------------------------------------- # +# to_total_ordering +# --------------------------------------------------------------------------- # + + +def test_to_total_ordering_sort_shape() -> None: + comp = _comp( + "Bronze is below silver, and silver is below gold. " + "Sort them from lowest to highest." + ) + projected = to_total_ordering(comp) + assert projected is not None + structure, query = projected + assert structure["items"] == ["bronze", "gold", "silver"] + assert {"less": "bronze", "greater": "silver"} in structure["relations"] + assert {"less": "silver", "greater": "gold"} in structure["relations"] + assert query == {"kind": "sort", "order": "ascending"} + + +def test_to_total_ordering_compare_shape() -> None: + comp = _comp("A is earlier than B. Compare a with b.") + projected = to_total_ordering(comp) + assert projected is not None + structure, query = projected + assert structure["items"] == ["a", "b"] + assert query == {"kind": "compare", "left": "a", "right": "b"} + + +def test_to_total_ordering_none_without_query() -> None: + assert to_total_ordering(_comp("Bronze is below silver.")) is None + + +# --------------------------------------------------------------------------- # +# Cross-projector neutrality — one MeaningGraph, distinct oracle shapes +# --------------------------------------------------------------------------- # + + +def test_one_graph_projects_only_to_its_domain() -> None: + # A categorical-with-conclusion comprehension projects to syllogism but not to + # total_ordering (no ordering query) — the interlingua is neutral, the + # projector decides applicability. + comp = _comp("All mammals are animals. Therefore all mammals are animals.") + assert to_syllogism(comp) is not None + assert to_total_ordering(comp) is None