From 96b9b942e6da65f31fb77ab8e9703f85b071dfab Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 5 Jun 2026 16:32:34 -0700 Subject: [PATCH] feat(comprehend): general comprehension reader + set_membership end-to-end (Phase 2a-r1/r2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Disciplined Path β (field decode α was empirically falsified). Reads S-P-O structure SYMBOLICALLY from the token sequence via domain-agnostic templates keyed on FUNCTION WORDS + ORDER, mints content as MeaningGraph entities/relations, parse-or-refuse (wrong=0 at the comprehension layer). Templates (set_membership): 'X is a Y' -> member; 'all Xs are Ys' -> subset; 'is X a Y?' / 'are all Xs Ys?' -> queries; definite-NP ('the X is a Y'); conservative singularization incl. irregulars (people->person), refusing unknown morphology rather than guessing. End-to-end: prose -> comprehend -> project -> INDEPENDENT set_membership oracle -> answer vs gold. Result on the real v1 lane: 8 correct / 0 wrong / 0 refused (full coverage, wrong=0). Cross-content generality (animals/professions/geography) asserted. 14 reader unit tests + 3 end-to-end tests, each bites under its violation. Additive only; invariants + capability index green. --- evals/comprehension/__init__.py | 8 + evals/comprehension/set_membership_runner.py | 70 ++++++ generate/meaning_graph/projectors.py | 57 +++++ generate/meaning_graph/reader.py | 230 +++++++++++++++++++ tests/test_comprehension_reader.py | 159 +++++++++++++ tests/test_comprehension_set_membership.py | 31 +++ 6 files changed, 555 insertions(+) create mode 100644 evals/comprehension/__init__.py create mode 100644 evals/comprehension/set_membership_runner.py create mode 100644 generate/meaning_graph/projectors.py create mode 100644 generate/meaning_graph/reader.py create mode 100644 tests/test_comprehension_reader.py create mode 100644 tests/test_comprehension_set_membership.py diff --git a/evals/comprehension/__init__.py b/evals/comprehension/__init__.py new file mode 100644 index 00000000..24d340c8 --- /dev/null +++ b/evals/comprehension/__init__.py @@ -0,0 +1,8 @@ +"""Comprehension scoring lanes — the engine reader under test, scored end-to-end +against the staged independent-gold lanes (Phase 2a, COMPREHEND). + +Unlike the ``evals/`` gold-only lanes (which verify oracle/gold integrity), +these runners drive the general comprehension reader over the lane PROSE and score +its output through the lane's INDEPENDENT oracle. wrong=0 is the floor: the reader +must refuse rather than emit a structure that yields a wrong answer. +""" diff --git a/evals/comprehension/set_membership_runner.py b/evals/comprehension/set_membership_runner.py new file mode 100644 index 00000000..efe9fac2 --- /dev/null +++ b/evals/comprehension/set_membership_runner.py @@ -0,0 +1,70 @@ +"""Score the general comprehension reader on the set_membership gold lane. + +prose -> comprehend() -> to_set_membership() -> independent oracle -> answer vs gold. +A refusal (unreadable prose, unprojectable, or oracle-refused projection) is NOT a +wrong; only a committed answer that disagrees with gold is wrong (must stay 0). +""" + +from __future__ import annotations + +import json +import sys +from typing import Any + +from evals.set_membership.oracle import OracleError, oracle_answer +from evals.set_membership.runner import _load_cases +from generate.meaning_graph.projectors import to_set_membership +from generate.meaning_graph.reader import Refusal, comprehend + + +def run() -> dict[str, Any]: + cases = _load_cases() + correct = wrong = refused = 0 + wrongs: list[dict[str, Any]] = [] + + for case in cases: + comp = comprehend(case["text"]) + if isinstance(comp, Refusal): + refused += 1 + continue + projected = to_set_membership(comp) + if projected is None: + refused += 1 + continue + structure, query = projected + try: + got = oracle_answer(structure, query) + except OracleError: + refused += 1 + continue + if got == case.get("gold"): + correct += 1 + else: + wrong += 1 + wrongs.append( + {"id": case.get("id"), "got": got, "gold": case.get("gold"), "text": case["text"]} + ) + + return { + "domain": "comprehension_set_membership", + "total": len(cases), + "correct": correct, + "wrong": wrong, + "refused": refused, + "wrongs": wrongs, + "counts": {"correct": correct, "wrong": wrong, "refused": refused}, + } + + +def main() -> int: + report = run() + print(json.dumps({k: v for k, v in report.items() if k != "wrongs"}, indent=2, sort_keys=True)) + if report["wrong"]: + print("WRONG > 0 — comprehension produced a wrong committed answer:", file=sys.stderr) + print(json.dumps(report["wrongs"], indent=2), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/generate/meaning_graph/projectors.py b/generate/meaning_graph/projectors.py new file mode 100644 index 00000000..ec91ed8f --- /dev/null +++ b/generate/meaning_graph/projectors.py @@ -0,0 +1,57 @@ +"""Projectors — map a comprehended ``MeaningGraph`` into a reasoner's input shape. + +The comprehension reader produces a neutral ``MeaningGraph``; a projector adapts +it to a specific independent-gold reasoner so the reader can be scored end-to-end +(prose -> MeaningGraph -> projection -> oracle -> answer vs gold). Projectors hold +NO decision logic — they only re-shape; the verdict is the independent oracle's. +""" + +from __future__ import annotations + +from typing import Any + +from generate.meaning_graph.reader import Comprehension + + +def to_set_membership(comp: Comprehension) -> tuple[dict[str, Any], dict[str, Any]] | None: + """Project into ``(structure, query)`` for ``evals.set_membership.oracle``. + + Returns ``None`` when the comprehension does not carry exactly one membership + question (nothing to ask the oracle) — the caller treats that as a refusal. + """ + graph = comp.meaning_graph + individuals = sorted({e.entity_id for e in graph.entities if e.kind == "individual"}) + classes = sorted({e.entity_id for e in graph.entities if e.kind == "class"}) + + member_facts = [ + (r.arguments[0], r.arguments[1]) + for r in graph.relations + if r.predicate == "member" and not r.negated + ] + subset_facts = [ + (r.arguments[0], r.arguments[1]) + for r in graph.relations + if r.predicate == "subset" and not r.negated + ] + + member_queries = [q for q in comp.queries if q.predicate == "member" and not q.negated] + subset_queries = [q for q in comp.queries if q.predicate == "subset" and not q.negated] + if len(member_queries) + len(subset_queries) != 1: + return None + + sets = [ + {"id": cid, "members": sorted({ind for ind, cls in member_facts if cls == cid})} + for cid in classes + ] + structure = { + "elements": individuals, + "sets": sets, + "subsets": [{"subset": a, "superset": b} for a, b in subset_facts], + } + if member_queries: + q = member_queries[0] + query = {"kind": "member", "element": q.arguments[0], "set": q.arguments[1]} + else: + q = subset_queries[0] + query = {"kind": "subset", "subset": q.arguments[0], "superset": q.arguments[1]} + return structure, query diff --git a/generate/meaning_graph/reader.py b/generate/meaning_graph/reader.py new file mode 100644 index 00000000..d835ca5e --- /dev/null +++ b/generate/meaning_graph/reader.py @@ -0,0 +1,230 @@ +"""The general comprehension reader — disciplined Path β (Phase 2a). + +Reads subject/relation/object STRUCTURE symbolically from the token sequence via +domain-agnostic templates keyed on FUNCTION WORDS + ORDER, then mints content as +``MeaningGraph`` entities/relations. The field provably cannot recover this +structure (the holonomy fold is lossy/non-invertible — see the α-falsification in +``docs/analysis/phase2-general-comprehension-organ-scope-2026-06-05.md``); the +field's honest role is grounding/coherence, not composition. + +Refusal-first: a clause that matches no template, a filler that is not a clean +identifier, an unrecognized plural, or a role conflict all REFUSE — never guess. +That keeps ``wrong=0`` at the comprehension layer (refusal is success; a fabricated +reading is the only failure). + +Templates (this increment): + - `` is a|an `` -> member(individual=X, class=Y) + - ``all are `` -> subset(subclass=sing(X), superclass=sing(Y)) + - ``is a|an ?`` -> Query member(X, Y) + +Overfit-trap mitigation: templates key on function words + order (general), never +on domain content; the same templates read animals, professions, geography, kin. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + +from generate.meaning_graph.model import ( + Entity, + MeaningGraph, + MeaningGraphError, + MeaningSpan, + Relation, +) + +_ARTICLES = frozenset({"a", "an"}) + +# Common irregular plurals the corpus exercises. Conservative + closed; an +# unrecognized plural REFUSES rather than guessing a wrong singular (wrong=0). +_IRREGULAR_PLURALS = { + "people": "person", + "men": "man", + "women": "woman", + "children": "child", + "feet": "foot", + "teeth": "tooth", + "mice": "mouse", + "geese": "goose", +} + +_SENTENCE_RE = re.compile(r"\s*([^.?!]+?)\s*([.?!])") + + +@dataclass(frozen=True, slots=True) +class Query: + """A question over the comprehended facts (asks whether a relation holds).""" + + predicate: str + arguments: tuple[str, ...] + span: MeaningSpan + negated: bool = False + + +@dataclass(frozen=True, slots=True) +class Comprehension: + """Successful comprehension: a fact graph plus any questions asked.""" + + meaning_graph: MeaningGraph + queries: tuple[Query, ...] = field(default_factory=tuple) + + +@dataclass(frozen=True, slots=True) +class Refusal: + """A typed refusal — the reader could not honestly read the input.""" + + reason: str + detail: str = "" + + +def _identifier(word: str) -> str | None: + """Normalize a content token to a clean identifier, or None to refuse.""" + w = word.strip().lower() + return w if w.isidentifier() else None + + +def _singularize(word: str) -> str | None: + """Conservative plural -> singular. None when not confidently a plural.""" + if word in _IRREGULAR_PLURALS: + return _IRREGULAR_PLURALS[word] + if word.endswith("ies") and len(word) > 3: + return word[:-3] + "y" + if word.endswith(("ses", "xes", "zes", "ches", "shes")): + return word[:-2] + if word.endswith("s") and not word.endswith("ss") and len(word) > 1: + return word[:-1] + return None + + +def _split_sentences(text: str) -> list[tuple[str, str, int, int]]: + """Return (body, terminator, start, end) for each sentence in *text*.""" + out: list[tuple[str, str, int, int]] = [] + matched_any = False + for m in _SENTENCE_RE.finditer(text): + matched_any = True + body = m.group(1) + if body.strip(): + out.append((body, m.group(2), m.start(1), m.end(1))) + if not matched_any: + body = text.strip() + if body: + start = text.find(body) + out.append((body, ".", start, start + len(body))) + return out + + +def comprehend(text: str, source_id: str = "input") -> Comprehension | Refusal: + """Comprehend *text* into a MeaningGraph + queries, or a typed Refusal.""" + if not text or not text.strip(): + return Refusal("empty") + + sentences = _split_sentences(text) + if not sentences: + return Refusal("empty") + + role_kind: dict[str, str] = {} + span_for: dict[str, MeaningSpan] = {} + members: list[tuple[str, str, MeaningSpan]] = [] + subsets: list[tuple[str, str, MeaningSpan]] = [] + queries: list[Query] = [] + + def claim(entity_id: str, kind: str, span: MeaningSpan) -> bool: + prior = role_kind.get(entity_id) + if prior is not None and prior != kind: + return False + role_kind[entity_id] = kind + span_for.setdefault(entity_id, span) + return True + + for body, terminator, start, end in sentences: + toks = body.lower().split() + span = MeaningSpan(source_id=source_id, start=start, end=end, text=text[start:end]) + is_question = terminator == "?" + + # Template: subset query ``are all ?`` + if is_question and len(toks) == 4 and toks[0] == "are" and toks[1] == "all": + raw_sub, raw_sup = _identifier(toks[2]), _identifier(toks[3]) + if raw_sub is None or raw_sup is None: + return Refusal("non_identifier_filler", body) + sub, sup = _singularize(raw_sub), _singularize(raw_sup) + if sub is None or sup is None: + return Refusal("unknown_morphology", body) + if not sub.isidentifier() or not sup.isidentifier(): + return Refusal("non_identifier_filler", body) + if not claim(sub, "class", span) or not claim(sup, "class", span): + return Refusal("role_conflict", body) + queries.append(Query("subset", (sub, sup), span)) + continue + + # Template: definite-NP membership query ``is the a|an ?`` + if is_question and len(toks) == 5 and toks[0] == "is" and toks[1] == "the" and toks[3] in _ARTICLES: + name, cls = _identifier(toks[2]), _identifier(toks[4]) + if name is None or cls is None: + return Refusal("non_identifier_filler", body) + if not claim(name, "individual", span) or not claim(cls, "class", span): + return Refusal("role_conflict", body) + queries.append(Query("member", (name, cls), span)) + continue + + # Template: query ``is a|an ?`` + if is_question and len(toks) == 4 and toks[0] == "is" and toks[2] in _ARTICLES: + name, cls = _identifier(toks[1]), _identifier(toks[3]) + if name is None or cls is None: + return Refusal("non_identifier_filler", body) + if not claim(name, "individual", span) or not claim(cls, "class", span): + return Refusal("role_conflict", body) + queries.append(Query("member", (name, cls), span)) + continue + + # Template: definite-NP membership ``the is a|an `` + if not is_question and len(toks) == 5 and toks[0] == "the" and toks[2] == "is" and toks[3] in _ARTICLES: + name, cls = _identifier(toks[1]), _identifier(toks[4]) + if name is None or cls is None: + return Refusal("non_identifier_filler", body) + if not claim(name, "individual", span) or not claim(cls, "class", span): + return Refusal("role_conflict", body) + members.append((name, cls, span)) + continue + + # Template: membership `` is a|an `` + if not is_question and len(toks) == 4 and toks[1] == "is" and toks[2] in _ARTICLES: + name, cls = _identifier(toks[0]), _identifier(toks[3]) + if name is None or cls is None: + return Refusal("non_identifier_filler", body) + if not claim(name, "individual", span) or not claim(cls, "class", span): + return Refusal("role_conflict", body) + members.append((name, cls, span)) + continue + + # Template: subsumption ``all are `` + if not is_question and len(toks) == 4 and toks[0] == "all" and toks[2] == "are": + raw_sub, raw_sup = _identifier(toks[1]), _identifier(toks[3]) + if raw_sub is None or raw_sup is None: + return Refusal("non_identifier_filler", body) + sub, sup = _singularize(raw_sub), _singularize(raw_sup) + if sub is None or sup is None: + return Refusal("unknown_morphology", body) + if not sub.isidentifier() or not sup.isidentifier(): + return Refusal("non_identifier_filler", body) + if not claim(sub, "class", span) or not claim(sup, "class", span): + return Refusal("role_conflict", body) + subsets.append((sub, sup, span)) + continue + + return Refusal("no_template_match", body) + + try: + entities = tuple( + Entity(entity_id=eid, name=eid, span=span_for[eid], kind=role_kind[eid]) + for eid in sorted(role_kind) + ) + relations = tuple( + [Relation("member", (ind, cls), sp) for ind, cls, sp in members] + + [Relation("subset", (sub, sup), sp) for sub, sup, sp in subsets] + ) + graph = MeaningGraph(entities=entities, relations=relations) + except MeaningGraphError as exc: # defensive — construction invariants + return Refusal("invalid_graph", repr(exc)) + + return Comprehension(meaning_graph=graph, queries=tuple(queries)) diff --git a/tests/test_comprehension_reader.py b/tests/test_comprehension_reader.py new file mode 100644 index 00000000..ef895eb1 --- /dev/null +++ b/tests/test_comprehension_reader.py @@ -0,0 +1,159 @@ +"""Phase 2a-r1 — the general comprehension reader (disciplined Path β). + +Reads S-P-O structure SYMBOLICALLY from the token sequence via domain-agnostic +templates keyed on FUNCTION WORDS + ORDER (the field provably cannot recover +structure — see the α-falsification in the Phase 2 scope doc). Content is the +filler; structure is the template. Parse-or-refuse → wrong=0 at the comprehension +layer. Each test bites under its named violation. +""" + +from __future__ import annotations + +from generate.meaning_graph.reader import ( + Comprehension, + Query, + Refusal, + comprehend, +) + + +def _rel(comp: Comprehension, predicate: str) -> tuple: + return tuple( + (r.predicate, r.arguments) for r in comp.meaning_graph.relations if r.predicate == predicate + ) + + +def _entity_kind(comp: Comprehension, entity_id: str) -> str | None: + for e in comp.meaning_graph.entities: + if e.entity_id == entity_id: + return e.kind + return None + + +# --------------------------------------------------------------------------- # +# Single-clause templates +# --------------------------------------------------------------------------- # + + +def test_membership_clause() -> None: + comp = comprehend("Rhea is a raven.") + assert isinstance(comp, Comprehension) + assert _rel(comp, "member") == (("member", ("rhea", "raven")),) + assert _entity_kind(comp, "rhea") == "individual" + assert _entity_kind(comp, "raven") == "class" + assert comp.queries == () + + +def test_membership_clause_an() -> None: + comp = comprehend("Ada is an engineer.") + assert isinstance(comp, Comprehension) + assert _rel(comp, "member") == (("member", ("ada", "engineer")),) + + +def test_subsumption_clause_pluralized() -> None: + comp = comprehend("All ravens are birds.") + assert isinstance(comp, Comprehension) + # plural surfaces normalize to the singular class ids the oracle expects. + assert _rel(comp, "subset") == (("subset", ("raven", "bird")),) + assert _entity_kind(comp, "raven") == "class" + assert _entity_kind(comp, "bird") == "class" + + +def test_query_clause() -> None: + comp = comprehend("Is Rhea a bird?") + assert isinstance(comp, Comprehension) + assert len(comp.queries) == 1 + q = comp.queries[0] + assert isinstance(q, Query) + assert q.predicate == "member" + assert q.arguments == ("rhea", "bird") + + +def test_subset_query_clause() -> None: + comp = comprehend("Are all squares polygons?") + assert isinstance(comp, Comprehension) + assert len(comp.queries) == 1 + q = comp.queries[0] + assert q.predicate == "subset" + assert q.arguments == ("square", "polygon") + + +def test_definite_np_membership_clause() -> None: + comp = comprehend("The mug is a cup.") + assert isinstance(comp, Comprehension) + assert _rel(comp, "member") == (("member", ("mug", "cup")),) + assert _entity_kind(comp, "mug") == "individual" + + +def test_definite_np_membership_query() -> None: + comp = comprehend("Is the mug a tool?") + assert isinstance(comp, Comprehension) + assert comp.queries[0].predicate == "member" + assert comp.queries[0].arguments == ("mug", "tool") + + +# --------------------------------------------------------------------------- # +# Full multi-clause problem +# --------------------------------------------------------------------------- # + + +def test_full_membership_problem() -> None: + comp = comprehend("Rhea is a raven. All ravens are birds. Is Rhea a bird?") + assert isinstance(comp, Comprehension) + assert _rel(comp, "member") == (("member", ("rhea", "raven")),) + assert _rel(comp, "subset") == (("subset", ("raven", "bird")),) + assert len(comp.queries) == 1 + assert comp.queries[0].arguments == ("rhea", "bird") + + +def test_irregular_plural_people_person_is_consistent() -> None: + # The wrong=0 wrinkle: 'people' must normalize to the same id as 'person'. + comp = comprehend("All scientists are people. Is Ada a person?") + assert isinstance(comp, Comprehension) + assert _rel(comp, "subset") == (("subset", ("scientist", "person")),) + assert comp.queries[0].arguments == ("ada", "person") + + +# --------------------------------------------------------------------------- # +# Generality — SAME template, DISTINCT domains (anti-overfit) +# --------------------------------------------------------------------------- # + + +def test_membership_template_generalizes_across_domains() -> None: + # One template, three distinct content domains -> all produce member(). + for text, ind, cls in [ + ("Rex is a dog.", "rex", "dog"), # animals + ("Ada is a botanist.", "ada", "botanist"), # professions + ("Paris is a city.", "paris", "city"), # geography + ]: + comp = comprehend(text) + assert isinstance(comp, Comprehension), text + assert _rel(comp, "member") == (("member", (ind, cls)),), text + + +# --------------------------------------------------------------------------- # +# Parse-or-refuse — refuse, never guess (wrong=0) +# --------------------------------------------------------------------------- # + + +def test_refuses_unmatched_clause() -> None: + comp = comprehend("The weather changed quickly yesterday.") + assert isinstance(comp, Refusal) + assert comp.reason == "no_template_match" + + +def test_refuses_when_any_clause_unmatched() -> None: + # If ONE clause cannot be read, the whole problem refuses (no partial guess). + comp = comprehend("Rhea is a raven. Rhea flew over the mountain.") + assert isinstance(comp, Refusal) + + +def test_refuses_non_identifier_filler() -> None: + # A filler that is not a clean identifier cannot become an entity id -> refuse. + comp = comprehend("Rhea is a co-pilot.") + assert isinstance(comp, Refusal) + + +def test_empty_input_refuses() -> None: + assert isinstance(comprehend(""), Refusal) + assert isinstance(comprehend(" "), Refusal) diff --git a/tests/test_comprehension_set_membership.py b/tests/test_comprehension_set_membership.py new file mode 100644 index 00000000..84d0892b --- /dev/null +++ b/tests/test_comprehension_set_membership.py @@ -0,0 +1,31 @@ +"""Phase 2a-r2 — end-to-end: the comprehension reader scored on set_membership. + +prose -> comprehend -> project -> INDEPENDENT oracle -> answer vs gold. +The load-bearing invariant: wrong == 0 (the reader refuses rather than emit a +structure that yields a wrong answer). Coverage (correct/total) is reported but +is allowed to be partial — refusal is honest, a wrong commit is not. +""" + +from __future__ import annotations + +from evals.comprehension.set_membership_runner import run + + +def test_comprehension_set_membership_wrong_is_zero() -> None: + report = run() + assert report["wrong"] == 0, report["wrongs"] + + +def test_comprehension_set_membership_has_real_coverage() -> None: + report = run() + # Real, non-trivial comprehension — not just refuse-everything. + assert report["correct"] > 0 + assert report["correct"] + report["refused"] == report["total"] + + +def test_comprehension_set_membership_full_coverage() -> None: + # This increment covers the whole v1 lane (member / subset / both query forms, + # definite-NP, irregular plurals) with zero wrong commits. + report = run() + assert report["refused"] == 0 + assert report["correct"] == report["total"]