Merge pull request #581 from AssetOverflow/feat/phase2a-comprehension-reader

feat(comprehend): general comprehension reader + set_membership end-to-end (Phase 2a)

evals/comprehension/__init__.py

@@ -0,0 +1,8 @@
+"""Comprehension scoring lanes — the engine reader under test, scored end-to-end
+against the staged independent-gold lanes (Phase 2a, COMPREHEND).
+
+Unlike the ``evals/<domain>`` gold-only lanes (which verify oracle/gold integrity),
+these runners drive the general comprehension reader over the lane PROSE and score
+its output through the lane's INDEPENDENT oracle. wrong=0 is the floor: the reader
+must refuse rather than emit a structure that yields a wrong answer.
+"""

evals/comprehension/set_membership_runner.py

@@ -0,0 +1,70 @@
+"""Score the general comprehension reader on the set_membership gold lane.
+
+prose -> comprehend() -> to_set_membership() -> independent oracle -> answer vs gold.
+A refusal (unreadable prose, unprojectable, or oracle-refused projection) is NOT a
+wrong; only a committed answer that disagrees with gold is wrong (must stay 0).
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import Any
+
+from evals.set_membership.oracle import OracleError, oracle_answer
+from evals.set_membership.runner import _load_cases
+from generate.meaning_graph.projectors import to_set_membership
+from generate.meaning_graph.reader import Refusal, comprehend
+
+
+def run() -> dict[str, Any]:
+    cases = _load_cases()
+    correct = wrong = refused = 0
+    wrongs: list[dict[str, Any]] = []
+
+    for case in cases:
+        comp = comprehend(case["text"])
+        if isinstance(comp, Refusal):
+            refused += 1
+            continue
+        projected = to_set_membership(comp)
+        if projected is None:
+            refused += 1
+            continue
+        structure, query = projected
+        try:
+            got = oracle_answer(structure, query)
+        except OracleError:
+            refused += 1
+            continue
+        if got == case.get("gold"):
+            correct += 1
+        else:
+            wrong += 1
+            wrongs.append(
+                {"id": case.get("id"), "got": got, "gold": case.get("gold"), "text": case["text"]}
+            )
+
+    return {
+        "domain": "comprehension_set_membership",
+        "total": len(cases),
+        "correct": correct,
+        "wrong": wrong,
+        "refused": refused,
+        "wrongs": wrongs,
+        "counts": {"correct": correct, "wrong": wrong, "refused": refused},
+    }
+
+
+def main() -> int:
+    report = run()
+    print(json.dumps({k: v for k, v in report.items() if k != "wrongs"}, indent=2, sort_keys=True))
+    if report["wrong"]:
+        print("WRONG > 0 — comprehension produced a wrong committed answer:", file=sys.stderr)
+        print(json.dumps(report["wrongs"], indent=2), file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

generate/meaning_graph/projectors.py

@@ -0,0 +1,57 @@
+"""Projectors — map a comprehended ``MeaningGraph`` into a reasoner's input shape.
+
+The comprehension reader produces a neutral ``MeaningGraph``; a projector adapts
+it to a specific independent-gold reasoner so the reader can be scored end-to-end
+(prose -> MeaningGraph -> projection -> oracle -> answer vs gold). Projectors hold
+NO decision logic — they only re-shape; the verdict is the independent oracle's.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from generate.meaning_graph.reader import Comprehension
+
+
+def to_set_membership(comp: Comprehension) -> tuple[dict[str, Any], dict[str, Any]] | None:
+    """Project into ``(structure, query)`` for ``evals.set_membership.oracle``.
+
+    Returns ``None`` when the comprehension does not carry exactly one membership
+    question (nothing to ask the oracle) — the caller treats that as a refusal.
+    """
+    graph = comp.meaning_graph
+    individuals = sorted({e.entity_id for e in graph.entities if e.kind == "individual"})
+    classes = sorted({e.entity_id for e in graph.entities if e.kind == "class"})
+
+    member_facts = [
+        (r.arguments[0], r.arguments[1])
+        for r in graph.relations
+        if r.predicate == "member" and not r.negated
+    ]
+    subset_facts = [
+        (r.arguments[0], r.arguments[1])
+        for r in graph.relations
+        if r.predicate == "subset" and not r.negated
+    ]
+
+    member_queries = [q for q in comp.queries if q.predicate == "member" and not q.negated]
+    subset_queries = [q for q in comp.queries if q.predicate == "subset" and not q.negated]
+    if len(member_queries) + len(subset_queries) != 1:
+        return None
+
+    sets = [
+        {"id": cid, "members": sorted({ind for ind, cls in member_facts if cls == cid})}
+        for cid in classes
+    ]
+    structure = {
+        "elements": individuals,
+        "sets": sets,
+        "subsets": [{"subset": a, "superset": b} for a, b in subset_facts],
+    }
+    if member_queries:
+        q = member_queries[0]
+        query = {"kind": "member", "element": q.arguments[0], "set": q.arguments[1]}
+    else:
+        q = subset_queries[0]
+        query = {"kind": "subset", "subset": q.arguments[0], "superset": q.arguments[1]}
+    return structure, query

generate/meaning_graph/reader.py

@@ -0,0 +1,230 @@
+"""The general comprehension reader — disciplined Path β (Phase 2a).
+
+Reads subject/relation/object STRUCTURE symbolically from the token sequence via
+domain-agnostic templates keyed on FUNCTION WORDS + ORDER, then mints content as
+``MeaningGraph`` entities/relations. The field provably cannot recover this
+structure (the holonomy fold is lossy/non-invertible — see the α-falsification in
+``docs/analysis/phase2-general-comprehension-organ-scope-2026-06-05.md``); the
+field's honest role is grounding/coherence, not composition.
+
+Refusal-first: a clause that matches no template, a filler that is not a clean
+identifier, an unrecognized plural, or a role conflict all REFUSE — never guess.
+That keeps ``wrong=0`` at the comprehension layer (refusal is success; a fabricated
+reading is the only failure).
+
+Templates (this increment):
+  - ``<X> is a|an <Y>``        -> member(individual=X, class=Y)
+  - ``all <Xs> are <Ys>``      -> subset(subclass=sing(X), superclass=sing(Y))
+  - ``is <X> a|an <Y>?``       -> Query member(X, Y)
+
+Overfit-trap mitigation: templates key on function words + order (general), never
+on domain content; the same templates read animals, professions, geography, kin.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+from generate.meaning_graph.model import (
+    Entity,
+    MeaningGraph,
+    MeaningGraphError,
+    MeaningSpan,
+    Relation,
+)
+
+_ARTICLES = frozenset({"a", "an"})
+
+# Common irregular plurals the corpus exercises. Conservative + closed; an
+# unrecognized plural REFUSES rather than guessing a wrong singular (wrong=0).
+_IRREGULAR_PLURALS = {
+    "people": "person",
+    "men": "man",
+    "women": "woman",
+    "children": "child",
+    "feet": "foot",
+    "teeth": "tooth",
+    "mice": "mouse",
+    "geese": "goose",
+}
+
+_SENTENCE_RE = re.compile(r"\s*([^.?!]+?)\s*([.?!])")
+
+
+@dataclass(frozen=True, slots=True)
+class Query:
+    """A question over the comprehended facts (asks whether a relation holds)."""
+
+    predicate: str
+    arguments: tuple[str, ...]
+    span: MeaningSpan
+    negated: bool = False
+
+
+@dataclass(frozen=True, slots=True)
+class Comprehension:
+    """Successful comprehension: a fact graph plus any questions asked."""
+
+    meaning_graph: MeaningGraph
+    queries: tuple[Query, ...] = field(default_factory=tuple)
+
+
+@dataclass(frozen=True, slots=True)
+class Refusal:
+    """A typed refusal — the reader could not honestly read the input."""
+
+    reason: str
+    detail: str = ""
+
+
+def _identifier(word: str) -> str | None:
+    """Normalize a content token to a clean identifier, or None to refuse."""
+    w = word.strip().lower()
+    return w if w.isidentifier() else None
+
+
+def _singularize(word: str) -> str | None:
+    """Conservative plural -> singular. None when not confidently a plural."""
+    if word in _IRREGULAR_PLURALS:
+        return _IRREGULAR_PLURALS[word]
+    if word.endswith("ies") and len(word) > 3:
+        return word[:-3] + "y"
+    if word.endswith(("ses", "xes", "zes", "ches", "shes")):
+        return word[:-2]
+    if word.endswith("s") and not word.endswith("ss") and len(word) > 1:
+        return word[:-1]
+    return None
+
+
+def _split_sentences(text: str) -> list[tuple[str, str, int, int]]:
+    """Return (body, terminator, start, end) for each sentence in *text*."""
+    out: list[tuple[str, str, int, int]] = []
+    matched_any = False
+    for m in _SENTENCE_RE.finditer(text):
+        matched_any = True
+        body = m.group(1)
+        if body.strip():
+            out.append((body, m.group(2), m.start(1), m.end(1)))
+    if not matched_any:
+        body = text.strip()
+        if body:
+            start = text.find(body)
+            out.append((body, ".", start, start + len(body)))
+    return out
+
+
+def comprehend(text: str, source_id: str = "input") -> Comprehension | Refusal:
+    """Comprehend *text* into a MeaningGraph + queries, or a typed Refusal."""
+    if not text or not text.strip():
+        return Refusal("empty")
+
+    sentences = _split_sentences(text)
+    if not sentences:
+        return Refusal("empty")
+
+    role_kind: dict[str, str] = {}
+    span_for: dict[str, MeaningSpan] = {}
+    members: list[tuple[str, str, MeaningSpan]] = []
+    subsets: list[tuple[str, str, MeaningSpan]] = []
+    queries: list[Query] = []
+
+    def claim(entity_id: str, kind: str, span: MeaningSpan) -> bool:
+        prior = role_kind.get(entity_id)
+        if prior is not None and prior != kind:
+            return False
+        role_kind[entity_id] = kind
+        span_for.setdefault(entity_id, span)
+        return True
+
+    for body, terminator, start, end in sentences:
+        toks = body.lower().split()
+        span = MeaningSpan(source_id=source_id, start=start, end=end, text=text[start:end])
+        is_question = terminator == "?"
+
+        # Template: subset query  ``are all <Xs> <Ys>?``
+        if is_question and len(toks) == 4 and toks[0] == "are" and toks[1] == "all":
+            raw_sub, raw_sup = _identifier(toks[2]), _identifier(toks[3])
+            if raw_sub is None or raw_sup is None:
+                return Refusal("non_identifier_filler", body)
+            sub, sup = _singularize(raw_sub), _singularize(raw_sup)
+            if sub is None or sup is None:
+                return Refusal("unknown_morphology", body)
+            if not sub.isidentifier() or not sup.isidentifier():
+                return Refusal("non_identifier_filler", body)
+            if not claim(sub, "class", span) or not claim(sup, "class", span):
+                return Refusal("role_conflict", body)
+            queries.append(Query("subset", (sub, sup), span))
+            continue
+
+        # Template: definite-NP membership query  ``is the <X> a|an <Y>?``
+        if is_question and len(toks) == 5 and toks[0] == "is" and toks[1] == "the" and toks[3] in _ARTICLES:
+            name, cls = _identifier(toks[2]), _identifier(toks[4])
+            if name is None or cls is None:
+                return Refusal("non_identifier_filler", body)
+            if not claim(name, "individual", span) or not claim(cls, "class", span):
+                return Refusal("role_conflict", body)
+            queries.append(Query("member", (name, cls), span))
+            continue
+
+        # Template: query  ``is <X> a|an <Y>?``
+        if is_question and len(toks) == 4 and toks[0] == "is" and toks[2] in _ARTICLES:
+            name, cls = _identifier(toks[1]), _identifier(toks[3])
+            if name is None or cls is None:
+                return Refusal("non_identifier_filler", body)
+            if not claim(name, "individual", span) or not claim(cls, "class", span):
+                return Refusal("role_conflict", body)
+            queries.append(Query("member", (name, cls), span))
+            continue
+
+        # Template: definite-NP membership  ``the <X> is a|an <Y>``
+        if not is_question and len(toks) == 5 and toks[0] == "the" and toks[2] == "is" and toks[3] in _ARTICLES:
+            name, cls = _identifier(toks[1]), _identifier(toks[4])
+            if name is None or cls is None:
+                return Refusal("non_identifier_filler", body)
+            if not claim(name, "individual", span) or not claim(cls, "class", span):
+                return Refusal("role_conflict", body)
+            members.append((name, cls, span))
+            continue
+
+        # Template: membership  ``<X> is a|an <Y>``
+        if not is_question and len(toks) == 4 and toks[1] == "is" and toks[2] in _ARTICLES:
+            name, cls = _identifier(toks[0]), _identifier(toks[3])
+            if name is None or cls is None:
+                return Refusal("non_identifier_filler", body)
+            if not claim(name, "individual", span) or not claim(cls, "class", span):
+                return Refusal("role_conflict", body)
+            members.append((name, cls, span))
+            continue
+
+        # Template: subsumption  ``all <Xs> are <Ys>``
+        if not is_question and len(toks) == 4 and toks[0] == "all" and toks[2] == "are":
+            raw_sub, raw_sup = _identifier(toks[1]), _identifier(toks[3])
+            if raw_sub is None or raw_sup is None:
+                return Refusal("non_identifier_filler", body)
+            sub, sup = _singularize(raw_sub), _singularize(raw_sup)
+            if sub is None or sup is None:
+                return Refusal("unknown_morphology", body)
+            if not sub.isidentifier() or not sup.isidentifier():
+                return Refusal("non_identifier_filler", body)
+            if not claim(sub, "class", span) or not claim(sup, "class", span):
+                return Refusal("role_conflict", body)
+            subsets.append((sub, sup, span))
+            continue
+
+        return Refusal("no_template_match", body)
+
+    try:
+        entities = tuple(
+            Entity(entity_id=eid, name=eid, span=span_for[eid], kind=role_kind[eid])
+            for eid in sorted(role_kind)
+        )
+        relations = tuple(
+            [Relation("member", (ind, cls), sp) for ind, cls, sp in members]
+            + [Relation("subset", (sub, sup), sp) for sub, sup, sp in subsets]
+        )
+        graph = MeaningGraph(entities=entities, relations=relations)
+    except MeaningGraphError as exc:  # defensive — construction invariants
+        return Refusal("invalid_graph", repr(exc))
+
+    return Comprehension(meaning_graph=graph, queries=tuple(queries))

tests/test_comprehension_reader.py

@@ -0,0 +1,159 @@
+"""Phase 2a-r1 — the general comprehension reader (disciplined Path β).
+
+Reads S-P-O structure SYMBOLICALLY from the token sequence via domain-agnostic
+templates keyed on FUNCTION WORDS + ORDER (the field provably cannot recover
+structure — see the α-falsification in the Phase 2 scope doc). Content is the
+filler; structure is the template. Parse-or-refuse → wrong=0 at the comprehension
+layer. Each test bites under its named violation.
+"""
+
+from __future__ import annotations
+
+from generate.meaning_graph.reader import (
+    Comprehension,
+    Query,
+    Refusal,
+    comprehend,
+)
+
+
+def _rel(comp: Comprehension, predicate: str) -> tuple:
+    return tuple(
+        (r.predicate, r.arguments) for r in comp.meaning_graph.relations if r.predicate == predicate
+    )
+
+
+def _entity_kind(comp: Comprehension, entity_id: str) -> str | None:
+    for e in comp.meaning_graph.entities:
+        if e.entity_id == entity_id:
+            return e.kind
+    return None
+
+
+# --------------------------------------------------------------------------- #
+# Single-clause templates
+# --------------------------------------------------------------------------- #
+
+
+def test_membership_clause() -> None:
+    comp = comprehend("Rhea is a raven.")
+    assert isinstance(comp, Comprehension)
+    assert _rel(comp, "member") == (("member", ("rhea", "raven")),)
+    assert _entity_kind(comp, "rhea") == "individual"
+    assert _entity_kind(comp, "raven") == "class"
+    assert comp.queries == ()
+
+
+def test_membership_clause_an() -> None:
+    comp = comprehend("Ada is an engineer.")
+    assert isinstance(comp, Comprehension)
+    assert _rel(comp, "member") == (("member", ("ada", "engineer")),)
+
+
+def test_subsumption_clause_pluralized() -> None:
+    comp = comprehend("All ravens are birds.")
+    assert isinstance(comp, Comprehension)
+    # plural surfaces normalize to the singular class ids the oracle expects.
+    assert _rel(comp, "subset") == (("subset", ("raven", "bird")),)
+    assert _entity_kind(comp, "raven") == "class"
+    assert _entity_kind(comp, "bird") == "class"
+
+
+def test_query_clause() -> None:
+    comp = comprehend("Is Rhea a bird?")
+    assert isinstance(comp, Comprehension)
+    assert len(comp.queries) == 1
+    q = comp.queries[0]
+    assert isinstance(q, Query)
+    assert q.predicate == "member"
+    assert q.arguments == ("rhea", "bird")
+
+
+def test_subset_query_clause() -> None:
+    comp = comprehend("Are all squares polygons?")
+    assert isinstance(comp, Comprehension)
+    assert len(comp.queries) == 1
+    q = comp.queries[0]
+    assert q.predicate == "subset"
+    assert q.arguments == ("square", "polygon")
+
+
+def test_definite_np_membership_clause() -> None:
+    comp = comprehend("The mug is a cup.")
+    assert isinstance(comp, Comprehension)
+    assert _rel(comp, "member") == (("member", ("mug", "cup")),)
+    assert _entity_kind(comp, "mug") == "individual"
+
+
+def test_definite_np_membership_query() -> None:
+    comp = comprehend("Is the mug a tool?")
+    assert isinstance(comp, Comprehension)
+    assert comp.queries[0].predicate == "member"
+    assert comp.queries[0].arguments == ("mug", "tool")
+
+
+# --------------------------------------------------------------------------- #
+# Full multi-clause problem
+# --------------------------------------------------------------------------- #
+
+
+def test_full_membership_problem() -> None:
+    comp = comprehend("Rhea is a raven. All ravens are birds. Is Rhea a bird?")
+    assert isinstance(comp, Comprehension)
+    assert _rel(comp, "member") == (("member", ("rhea", "raven")),)
+    assert _rel(comp, "subset") == (("subset", ("raven", "bird")),)
+    assert len(comp.queries) == 1
+    assert comp.queries[0].arguments == ("rhea", "bird")
+
+
+def test_irregular_plural_people_person_is_consistent() -> None:
+    # The wrong=0 wrinkle: 'people' must normalize to the same id as 'person'.
+    comp = comprehend("All scientists are people. Is Ada a person?")
+    assert isinstance(comp, Comprehension)
+    assert _rel(comp, "subset") == (("subset", ("scientist", "person")),)
+    assert comp.queries[0].arguments == ("ada", "person")
+
+
+# --------------------------------------------------------------------------- #
+# Generality — SAME template, DISTINCT domains (anti-overfit)
+# --------------------------------------------------------------------------- #
+
+
+def test_membership_template_generalizes_across_domains() -> None:
+    # One template, three distinct content domains -> all produce member().
+    for text, ind, cls in [
+        ("Rex is a dog.", "rex", "dog"),          # animals
+        ("Ada is a botanist.", "ada", "botanist"),  # professions
+        ("Paris is a city.", "paris", "city"),    # geography
+    ]:
+        comp = comprehend(text)
+        assert isinstance(comp, Comprehension), text
+        assert _rel(comp, "member") == (("member", (ind, cls)),), text
+
+
+# --------------------------------------------------------------------------- #
+# Parse-or-refuse — refuse, never guess (wrong=0)
+# --------------------------------------------------------------------------- #
+
+
+def test_refuses_unmatched_clause() -> None:
+    comp = comprehend("The weather changed quickly yesterday.")
+    assert isinstance(comp, Refusal)
+    assert comp.reason == "no_template_match"
+
+
+def test_refuses_when_any_clause_unmatched() -> None:
+    # If ONE clause cannot be read, the whole problem refuses (no partial guess).
+    comp = comprehend("Rhea is a raven. Rhea flew over the mountain.")
+    assert isinstance(comp, Refusal)
+
+
+def test_refuses_non_identifier_filler() -> None:
+    # A filler that is not a clean identifier cannot become an entity id -> refuse.
+    comp = comprehend("Rhea is a co-pilot.")
+    assert isinstance(comp, Refusal)
+
+
+def test_empty_input_refuses() -> None:
+    assert isinstance(comprehend(""), Refusal)
+    assert isinstance(comprehend("   "), Refusal)

tests/test_comprehension_set_membership.py

@@ -0,0 +1,31 @@
+"""Phase 2a-r2 — end-to-end: the comprehension reader scored on set_membership.
+
+prose -> comprehend -> project -> INDEPENDENT oracle -> answer vs gold.
+The load-bearing invariant: wrong == 0 (the reader refuses rather than emit a
+structure that yields a wrong answer). Coverage (correct/total) is reported but
+is allowed to be partial — refusal is honest, a wrong commit is not.
+"""
+
+from __future__ import annotations
+
+from evals.comprehension.set_membership_runner import run
+
+
+def test_comprehension_set_membership_wrong_is_zero() -> None:
+    report = run()
+    assert report["wrong"] == 0, report["wrongs"]
+
+
+def test_comprehension_set_membership_has_real_coverage() -> None:
+    report = run()
+    # Real, non-trivial comprehension — not just refuse-everything.
+    assert report["correct"] > 0
+    assert report["correct"] + report["refused"] == report["total"]
+
+
+def test_comprehension_set_membership_full_coverage() -> None:
+    # This increment covers the whole v1 lane (member / subset / both query forms,
+    # definite-NP, irregular plurals) with zero wrong commits.
+    report = run()
+    assert report["refused"] == 0
+    assert report["correct"] == report["total"]