diff --git a/evals/capability_index/baseline.json b/evals/capability_index/baseline.json index b259f899..551e6b34 100644 --- a/evals/capability_index/baseline.json +++ b/evals/capability_index/baseline.json @@ -1,13 +1,13 @@ { - "capability_score": 0.814356, - "coverage_geomean": 0.814356, - "coverage_micro": 0.988266, + "capability_score": 0.917231, + "coverage_geomean": 0.917231, + "coverage_micro": 0.993481, "accuracy_micro": 1.0, "breadth": 6, - "min_domain_coverage": 0.5, + "min_domain_coverage": 0.833333, "wrong_total": 0, "assert_mode_valid": true, - "deterministic_digest": "0a98b9b41c23f5166d7e23bc194207ead53a89d5316dddc03194bd2739bb9753", + "deterministic_digest": "13d7db6c4fc8caed703164cd16c196ecb426a31fb5bcb9658fe630be760179ab", "domains": [ { "domain": "comprehension_set_membership", @@ -19,18 +19,18 @@ }, { "domain": "comprehension_syllogism", - "correct": 6, + "correct": 7, "wrong": 0, - "refused": 2, - "coverage": 0.75, + "refused": 1, + "coverage": 0.875, "accuracy": 1.0 }, { "domain": "comprehension_total_ordering", - "correct": 4, + "correct": 7, "wrong": 0, - "refused": 4, - "coverage": 0.5, + "refused": 1, + "coverage": 0.875, "accuracy": 1.0 }, { diff --git a/evals/comprehension/CANONICALIZATION.md b/evals/comprehension/CANONICALIZATION.md new file mode 100644 index 00000000..6a2fbd8b --- /dev/null +++ b/evals/comprehension/CANONICALIZATION.md @@ -0,0 +1,60 @@ +# Noun-phrase canonicalization contract (comprehension lanes) + +The comprehension reader (`generate/meaning_graph/reader.py`) and the staged gold +lanes (`evals/{set_membership,syllogism,total_ordering}/v1`) share **one** rule for +turning a surface noun phrase into a term/entity id. This contract is what makes +multi-word noun phrases `wrong=0`-safe: the reader and the gold canonicalize the +*same* way, so a committed answer can only match gold or refuse — it can never +silently mean something different. + +## The rule + +A noun-phrase slot (the tokens a template isolates between its function words) +becomes an id by: + +1. lowercasing every token; +2. for a **plural class slot** (the subject/predicate of `all/no/some … are …`), + singularizing the **head** (final) token — `objects → object`, `people → person`; + item and individual slots are **not** singularized; +3. joining the resulting tokens with `_`. + +| surface | role | canonical id | +|---|---|---| +| `metal objects` | class | `metal_object` | +| `soft objects` | class | `soft_object` | +| `North station` | item | `north_station` | +| `Level one` | item | `level_one` | +| `Red rank` | item | `red_rank` | +| `red car` | individual | `red_car` | +| `squares` | class | `square` | + +## Why join, not head-word-only + +An earlier (implicit) gold used head-word-only in places (`metal objects → metal`, +`North station → north`). That is **information-destroying** and an active +`wrong=0` hazard: `metal objects` and `metal tools` both collapse to `metal`, +creating a *false identity* the oracle would then reason over. Joining preserves +the distinction, so distinct phrases stay distinct ids. + +## Refusal cases (still parse-or-refuse) + +The reader refuses rather than guess when: + +- a multi-token slot contains a **reserved function word** (article, comparator, + quantifier, `is/are/than/with/not/the/from/to/order/in/of/on/at/by/and/or`) — + e.g. `Compare beta with beta in the same order` → the right slot + `beta in the same order` leaks `in/the/order`, so refuse (never + `beta_in_the_same_order`). A **single-token** slot is exempt: a literal item + named `A` is content even though `a` is also the article. +- a plural class head is not a recognizable plural (e.g. the adjectival predicate + `trained` in `All pilots are trained`) — cannot singularize → refuse. +- the two class NPs in `Are all ?` are adjacent with no separating + function word and either is multi-word — the boundary is unknown → refuse. + +## Changing this contract + +This contract is load-bearing for `wrong=0`. If it changes, the reader **and** +every affected gold case must change together in the same PR, and the +`tests/test_comprehension_wrong_zero_property.py` round-trip (which renders prose +from canonical structures and checks the reader reproduces the oracle's verdict) +must stay green. diff --git a/evals/syllogism/v1/cases.jsonl b/evals/syllogism/v1/cases.jsonl index f914f23d..77d6829e 100644 --- a/evals/syllogism/v1/cases.jsonl +++ b/evals/syllogism/v1/cases.jsonl @@ -5,4 +5,4 @@ {"id":"sy-v1-0005","seed":20260605,"text":"All cats are mammals. All dogs are mammals. Therefore all dogs are cats.","structure":{"terms":["dog","cat","mammal"],"domain_size":3,"premises":[{"form":"A","subject":"cat","predicate":"mammal"},{"form":"A","subject":"dog","predicate":"mammal"}]},"query":{"kind":"validity","conclusion":{"form":"A","subject":"dog","predicate":"cat"}},"gold":{"valid":false,"conclusion":null},"class":"invalid_undistributed_middle"} {"id":"sy-v1-0006","seed":20260605,"text":"All pilots are people. All pilots are trained. Therefore some trained people exist.","structure":{"terms":["pilot","person","trained"],"domain_size":3,"premises":[{"form":"A","subject":"pilot","predicate":"person"},{"form":"A","subject":"pilot","predicate":"trained"}]},"query":{"kind":"validity","conclusion":{"form":"I","subject":"trained","predicate":"person"}},"gold":{"valid":false,"conclusion":null},"class":"invalid_existential_import"} {"id":"sy-v1-0007","seed":20260605,"text":"Some musicians are teachers. All teachers are workers. Therefore some musicians are workers.","structure":{"terms":["musician","teacher","worker"],"domain_size":3,"premises":[{"form":"I","subject":"musician","predicate":"teacher"},{"form":"A","subject":"teacher","predicate":"worker"}]},"query":{"kind":"validity","conclusion":{"form":"I","subject":"musician","predicate":"worker"}},"gold":{"valid":true,"conclusion":{"form":"I","subject":"musician","predicate":"worker"}},"class":"datisi_valid"} -{"id":"sy-v1-0008","seed":20260605,"text":"No metal objects are soft objects. Some tools are metal objects. Therefore some tools are not soft objects.","structure":{"terms":["tool","metal","soft"],"domain_size":3,"premises":[{"form":"E","subject":"metal","predicate":"soft"},{"form":"I","subject":"tool","predicate":"metal"}]},"query":{"kind":"validity","conclusion":{"form":"O","subject":"tool","predicate":"soft"}},"gold":{"valid":true,"conclusion":{"form":"O","subject":"tool","predicate":"soft"}},"class":"ferio_valid"} +{"id":"sy-v1-0008","seed":20260605,"text":"No metal objects are soft objects. Some tools are metal objects. Therefore some tools are not soft objects.","structure":{"terms":["metal_object","soft_object","tool"],"domain_size":3,"premises":[{"form":"E","subject":"metal_object","predicate":"soft_object"},{"form":"I","subject":"tool","predicate":"metal_object"}]},"query":{"kind":"validity","conclusion":{"form":"O","subject":"tool","predicate":"soft_object"}},"gold":{"valid":true,"conclusion":{"form":"O","subject":"tool","predicate":"soft_object"}},"class":"ferio_valid"} diff --git a/evals/total_ordering/v1/cases.jsonl b/evals/total_ordering/v1/cases.jsonl index 6d21695f..e6b5df46 100644 --- a/evals/total_ordering/v1/cases.jsonl +++ b/evals/total_ordering/v1/cases.jsonl @@ -1,8 +1,8 @@ {"id":"to-v1-0001","seed":20260605,"text":"Bronze is below silver, and silver is below gold. Sort them from lowest to highest.","structure":{"items":["gold","bronze","silver"],"relations":[{"less":"bronze","greater":"silver"},{"less":"silver","greater":"gold"}]},"query":{"kind":"sort","order":"ascending"},"gold":["bronze","silver","gold"],"class":"sort_chain_three"} {"id":"to-v1-0002","seed":20260605,"text":"Oak is taller than birch. Pine is taller than oak. Which is the height order from shortest to tallest?","structure":{"items":["pine","birch","oak"],"relations":[{"less":"birch","greater":"oak"},{"less":"oak","greater":"pine"}]},"query":{"kind":"sort","order":"ascending"},"gold":["birch","oak","pine"],"class":"sort_chain_three"} {"id":"to-v1-0003","seed":20260605,"text":"A is earlier than B, B is earlier than C, and C is earlier than D. Compare A with D.","structure":{"items":["d","b","a","c"],"relations":[{"less":"a","greater":"b"},{"less":"b","greater":"c"},{"less":"c","greater":"d"}]},"query":{"kind":"compare","left":"a","right":"d"},"gold":"less","class":"compare_transitive_less"} -{"id":"to-v1-0004","seed":20260605,"text":"North station comes after central, and central comes after south. Compare north with south.","structure":{"items":["north","south","central"],"relations":[{"less":"south","greater":"central"},{"less":"central","greater":"north"}]},"query":{"kind":"compare","left":"north","right":"south"},"gold":"greater","class":"compare_transitive_greater"} -{"id":"to-v1-0005","seed":20260605,"text":"Red rank is below orange, orange below yellow, yellow below green. Sort descending.","structure":{"items":["orange","green","red","yellow"],"relations":[{"less":"red","greater":"orange"},{"less":"orange","greater":"yellow"},{"less":"yellow","greater":"green"}]},"query":{"kind":"sort","order":"descending"},"gold":["green","yellow","orange","red"],"class":"sort_descending_four"} +{"id":"to-v1-0004","seed":20260605,"text":"North station is after central, and central is after south. Compare north station with south.","structure":{"items":["central","north_station","south"],"relations":[{"less":"central","greater":"north_station"},{"less":"south","greater":"central"}]},"query":{"kind":"compare","left":"north_station","right":"south"},"gold":"greater","class":"compare_transitive_greater"} +{"id":"to-v1-0005","seed":20260605,"text":"Red rank is below orange, orange below yellow, yellow below green. Sort descending.","structure":{"items":["green","orange","red_rank","yellow"],"relations":[{"less":"red_rank","greater":"orange"},{"less":"orange","greater":"yellow"},{"less":"yellow","greater":"green"}]},"query":{"kind":"sort","order":"descending"},"gold":["green","yellow","orange","red_rank"],"class":"sort_descending_four"} {"id":"to-v1-0006","seed":20260605,"text":"Compare beta with beta in the same order.","structure":{"items":["alpha","beta","gamma"],"relations":[{"less":"alpha","greater":"beta"},{"less":"beta","greater":"gamma"}]},"query":{"kind":"compare","left":"beta","right":"beta"},"gold":"equal","class":"compare_equal"} {"id":"to-v1-0007","seed":20260605,"text":"Level one is below level two, level two below level three, level three below level four. Sort ascending.","structure":{"items":["level_four","level_one","level_three","level_two"],"relations":[{"less":"level_one","greater":"level_two"},{"less":"level_two","greater":"level_three"},{"less":"level_three","greater":"level_four"}]},"query":{"kind":"sort","order":"ascending"},"gold":["level_one","level_two","level_three","level_four"],"class":"sort_chain_four"} {"id":"to-v1-0008","seed":20260605,"text":"Mercury is closer than Venus, Venus closer than Earth, and Earth closer than Mars. Compare Mars with Venus.","structure":{"items":["earth","mars","mercury","venus"],"relations":[{"less":"mercury","greater":"venus"},{"less":"venus","greater":"earth"},{"less":"earth","greater":"mars"}]},"query":{"kind":"compare","left":"mars","right":"venus"},"gold":"greater","class":"compare_transitive_greater"} diff --git a/generate/meaning_graph/reader.py b/generate/meaning_graph/reader.py index ab9d167e..50bb96f1 100644 --- a/generate/meaning_graph/reader.py +++ b/generate/meaning_graph/reader.py @@ -35,11 +35,15 @@ animals, professions, geography, kin, metals, ranks): - ``sort [ascending|descending]`` / ``... order from to `` -> Query sort(order) -Multi-word noun phrases are REFUSED on purpose: the staged gold lanes canonicalize -multi-word NPs three contradictory ways ("North station"->"north", "Level one"-> -"level_one", "metal objects"->"metal"), so no single general rule is wrong=0-safe. -Until the gold lanes carry a canonicalization contract, the only honest reading of -a multi-word NP is refusal. +Multi-word noun phrases chunk by the CANONICALIZATION CONTRACT (see +``evals/comprehension/CANONICALIZATION.md``): a noun-phrase slot canonicalizes to +its tokens lowercased and joined with ``_`` ("North station"->"north_station", +"Level one"->"level_one"); a plural class slot singularizes its head first +("metal objects"->"metal_object"). Join is information-preserving on purpose — +head-word-only ("metal objects"->"metal") would collapse distinct phrases +("metal objects" vs "metal tools") into a false identity, itself a wrong=0 hazard. +A slot containing a reserved function word, or an adjacent-NP boundary that cannot +be located (the "are all ?" two-NP case), still REFUSES rather than guess. """ from __future__ import annotations @@ -98,6 +102,23 @@ _SORT_HIGH = frozenset( {"highest", "tallest", "largest", "greatest", "most", "latest", "top", "last", "high"} ) +# Function words that may NEVER appear inside a noun-phrase slot. A slot anchored +# by templates should hold only content tokens; if a reserved token leaks in, the +# clause is mis-parsed and we REFUSE rather than chunk junk (e.g. "beta in the +# same order" -> contains "the"/"order" -> refuse, not "beta_in_the_same_order"). +_RESERVED = ( + _ARTICLES + | _COMP_LESS + | _COMP_GREATER + | _SORT_LOW + | _SORT_HIGH + | { + "is", "are", "than", "with", "not", "all", "no", "some", "therefore", + "compare", "sort", "and", "or", "the", "from", "to", "order", + "exist", "exists", "them", "which", "in", "of", "on", "at", "by", + } +) + _SENTENCE_RE = re.compile(r"\s*([^.?!]+?)\s*([.?!])") @@ -137,12 +158,6 @@ class _Reject(Exception): self.refusal = Refusal(reason, detail) -def _identifier(word: str) -> str | None: - """Normalize a content token to a clean identifier, or None to refuse.""" - w = word.strip().lower() - return w if w.isidentifier() else None - - def _singularize(word: str) -> str | None: """Conservative plural -> singular. None when not confidently a plural.""" if word in _IRREGULAR_PLURALS: @@ -156,25 +171,37 @@ def _singularize(word: str) -> str | None: return None -def _one(words: list[str], detail: str) -> str: - """A single content token -> identifier, or REFUSE (multi-word / non-id).""" - if len(words) != 1: - raise _Reject("multiword_np", detail) - ident = _identifier(words[0]) - if ident is None: +def _chunk(words: list[str], detail: str) -> str: + """A noun-phrase slot -> a single canonical id by the canonicalization + contract: lowercase tokens joined with ``_`` (information-preserving — distinct + phrases never collapse). REFUSE if the slot is empty, holds a reserved function + word, or yields a non-identifier.""" + if not words: + raise _Reject("empty_np", detail) + toks = [w.strip().lower() for w in words] + # A reserved word is a parse-leak signal only INSIDE a multi-token slot. A + # single-token slot is the whole NP — its token is content even if it spells a + # function word (e.g. an item literally named "A", which is also the article). + if len(toks) > 1 and any(t in _RESERVED for t in toks): + raise _Reject("reserved_word_in_np", detail) + canonical = "_".join(toks) + if not canonical.isidentifier(): raise _Reject("non_identifier_filler", detail) - return ident + return canonical -def _one_class(words: list[str], detail: str) -> str: - """A single plural content token -> singular class id, or REFUSE.""" - word = _one(words, detail) - singular = _singularize(word) - if singular is None: +def _chunk_class(words: list[str], detail: str) -> str: + """A plural class noun-phrase slot -> singular canonical class id. The HEAD + (final) token is singularized, then the phrase is chunked: "metal objects" -> + "metal_object", "people" -> "person". REFUSE if the head is not a recognizable + plural (e.g. the adjectival predicate "trained").""" + if not words: + raise _Reject("empty_np", detail) + *modifiers, head = (w.strip().lower() for w in words) + singular_head = _singularize(head) + if singular_head is None: raise _Reject("unknown_morphology", detail) - if not singular.isidentifier(): - raise _Reject("non_identifier_filler", detail) - return singular + return _chunk([*modifiers, singular_head], detail) def _split_sentences(text: str) -> list[tuple[str, str, int, int]]: @@ -221,8 +248,8 @@ def _parse_categorical(toks: list[str], detail: str) -> tuple[str, str, str] | N predicate_words = predicate_words[1:] if not subject_words or not predicate_words: raise _Reject("incomplete_categorical", detail) - sub = _one_class(subject_words, detail) - sup = _one_class(predicate_words, detail) + sub = _chunk_class(subject_words, detail) + sup = _chunk_class(predicate_words, detail) if quant == "some" and negated: predicate = "some_not" elif negated: @@ -249,8 +276,8 @@ def _parse_comparative(toks: list[str], detail: str) -> tuple[str, str] | None: right = right[1:] if not left or not right: raise _Reject("incomplete_comparative", detail) - x = _one(left, detail) - y = _one(right, detail) + x = _chunk(left, detail) + y = _chunk(right, detail) if toks[comp_idx] in _COMP_LESS: return x, y # X < Y return y, x # X > Y -> less(Y, X) @@ -340,10 +367,11 @@ def _read_clause( # --- ordering queries (keyword-led; terminator-independent) ------------- # if toks[0] == "compare": - if len(toks) != 4 or toks[2] != "with": + if "with" not in toks[1:]: raise _Reject("unreadable_compare", clause) - left = _one([toks[1]], clause) - right = _one([toks[3]], clause) + with_idx = toks.index("with", 1) + left = _chunk(toks[1:with_idx], clause) + right = _chunk(toks[with_idx + 1:], clause) for item in (left, right): claim(item, "item", span) queries.append(Query("compare", (left, right), span)) @@ -370,19 +398,26 @@ def _read_clause( rest = toks[1:] if rest and rest[0] == "the": rest = rest[1:] - if len(rest) == 3 and rest[1] in _ARTICLES: - name = _one([rest[0]], clause) - cls = _one([rest[2]], clause) - claim(name, "individual", span) - claim(cls, "class", span) - queries.append(Query("member", (name, cls), span)) - return - raise _Reject("unreadable_member_query", clause) + art_idx = next((i for i, t in enumerate(rest) if t in _ARTICLES), None) + if art_idx is None or art_idx == 0 or art_idx == len(rest) - 1: + raise _Reject("unreadable_member_query", clause) + name = _chunk(rest[:art_idx], clause) + cls = _chunk(rest[art_idx + 1:], clause) + claim(name, "individual", span) + claim(cls, "class", span) + queries.append(Query("member", (name, cls), span)) + return # --- subset query: "are all ?" -------------------------------- # - if question and len(toks) == 4 and toks[0] == "are" and toks[1] == "all": - sub = _one_class([toks[2]], clause) - sup = _one_class([toks[3]], clause) + # The two class NPs are ADJACENT with no separating function word, so a + # multi-word split is ambiguous -> require exactly two single tokens, else + # refuse rather than guess the boundary. + if question and len(toks) >= 2 and toks[0] == "are" and toks[1] == "all": + body = toks[2:] + if len(body) != 2: + raise _Reject("ambiguous_subset_query", clause) + sub = _chunk_class([body[0]], clause) + sup = _chunk_class([body[1]], clause) claim(sub, "class", span) claim(sup, "class", span) queries.append(Query("subset", (sub, sup), span)) @@ -399,13 +434,16 @@ def _read_clause( # --- membership fact: "[the] is a|an " -------------------------- # body_toks = toks[1:] if toks[0] == "the" else toks - if len(body_toks) == 4 and body_toks[1] == "is" and body_toks[2] in _ARTICLES: - name = _one([body_toks[0]], clause) - cls = _one([body_toks[3]], clause) - claim(name, "individual", span) - claim(cls, "class", span) - relations.append(("member", (name, cls), span)) - return + if "is" in body_toks: + is_idx = body_toks.index("is") + after = body_toks[is_idx + 1:] + if is_idx > 0 and len(after) > 1 and after[0] in _ARTICLES: + name = _chunk(body_toks[:is_idx], clause) + cls = _chunk(after[1:], clause) + claim(name, "individual", span) + claim(cls, "class", span) + relations.append(("member", (name, cls), span)) + return # --- ordering fact: " [is] [than] " ----------------------- # comparative = _parse_comparative(toks, clause) diff --git a/tests/test_comprehension_reader.py b/tests/test_comprehension_reader.py index 30177050..9146e916 100644 --- a/tests/test_comprehension_reader.py +++ b/tests/test_comprehension_reader.py @@ -290,20 +290,50 @@ def test_comparative_template_generalizes_across_domains() -> None: # --------------------------------------------------------------------------- # -# Multi-word NP — REFUSE (the wrong=0 guard: no general canonicalization exists) +# Multi-word NP — CHUNK by the canonicalization contract (join tokens with "_") # --------------------------------------------------------------------------- # -def test_multiword_np_in_categorical_refuses() -> None: +def test_multiword_np_in_categorical_chunks() -> None: comp = comprehend("No metal objects are soft objects.") - assert isinstance(comp, Refusal) - assert comp.reason == "multiword_np" + assert isinstance(comp, Comprehension) + # plural class head singularized, then joined: "metal objects" -> "metal_object" + assert _rel(comp, "disjoint") == (("disjoint", ("metal_object", "soft_object")),) -def test_multiword_np_in_comparative_refuses() -> None: - comp = comprehend("North station comes after central.") - assert isinstance(comp, Refusal) - assert comp.reason == "multiword_np" +def test_multiword_np_in_comparative_chunks() -> None: + comp = comprehend("North station is below south.") + assert isinstance(comp, Comprehension) + assert _rel(comp, "less") == (("less", ("north_station", "south")),) + assert _entity_kind(comp, "north_station") == "item" + + +def test_multiword_item_in_compare_query_chunks() -> None: + comp = comprehend("Compare north station with south.") + assert isinstance(comp, Comprehension) + assert comp.queries[0].predicate == "compare" + assert comp.queries[0].arguments == ("north_station", "south") + + +def test_multiword_individual_in_membership_chunks() -> None: + comp = comprehend("The red car is a vehicle.") + assert isinstance(comp, Comprehension) + assert _rel(comp, "member") == (("member", ("red_car", "vehicle")),) + + +def test_join_is_information_preserving_distinct_nps_stay_distinct() -> None: + # WHY the contract JOINS instead of keeping the head word: distinct phrases must + # not collapse into a false identity ("metal objects" vs "metal tools" both -> + # "metal" would be a wrong=0 hazard). + comp = comprehend("All metal objects are heavy items. All metal tools are sharp items.") + assert isinstance(comp, Comprehension) + ids = {e.entity_id for e in comp.meaning_graph.entities} + assert {"metal_object", "metal_tool"} <= ids # NOT collapsed to "metal" + + +# --------------------------------------------------------------------------- # +# Parse-or-refuse — still refuse where no honest reading exists (wrong=0) +# --------------------------------------------------------------------------- # def test_adjectival_predicate_refuses_via_morphology() -> None: @@ -313,7 +343,17 @@ def test_adjectival_predicate_refuses_via_morphology() -> None: assert comp.reason == "unknown_morphology" -def test_trailing_tokens_in_compare_refuses() -> None: +def test_trailing_prepositional_phrase_in_compare_refuses() -> None: + # "...in the same order" leaks reserved words into the NP slot -> refuse, never + # chunk "beta_in_the_same_order". comp = comprehend("Compare beta with beta in the same order.") assert isinstance(comp, Refusal) - assert comp.reason == "unreadable_compare" + assert comp.reason == "reserved_word_in_np" + + +def test_ambiguous_two_np_subset_query_refuses() -> None: + # Two adjacent multi-word class NPs with no separating function word -> the + # boundary is unknown, so refuse rather than guess it. + comp = comprehend("Are all metal objects soft objects?") + assert isinstance(comp, Refusal) + assert comp.reason == "ambiguous_subset_query" diff --git a/tests/test_comprehension_syllogism.py b/tests/test_comprehension_syllogism.py index 0d684abd..a173df67 100644 --- a/tests/test_comprehension_syllogism.py +++ b/tests/test_comprehension_syllogism.py @@ -23,10 +23,12 @@ def test_comprehension_syllogism_has_real_coverage() -> None: def test_comprehension_syllogism_pinned_counts() -> None: - # Pins the lane: 6 read end-to-end (Barbara/Celarent/Darii/Ferio/Datisi + - # the invalid undistributed-middle, which the oracle correctly rejects); - # 2 refused — the existential-import conclusion and the multi-word-NP case. + # Pins the lane: 7 read end-to-end (Barbara/Celarent/Darii/Ferio/Datisi, the + # invalid undistributed-middle which the oracle correctly rejects, and the + # multi-word Ferio "metal objects"/"soft objects" now chunked by the + # canonicalization contract); 1 refused — the existential-import conclusion + # ("some trained people exist") which is out of the categorical grammar. report = run() - assert report["correct"] == 6 - assert report["refused"] == 2 + assert report["correct"] == 7 + assert report["refused"] == 1 assert report["total"] == 8 diff --git a/tests/test_comprehension_total_ordering.py b/tests/test_comprehension_total_ordering.py index b971ec59..c124219f 100644 --- a/tests/test_comprehension_total_ordering.py +++ b/tests/test_comprehension_total_ordering.py @@ -23,11 +23,11 @@ def test_comprehension_total_ordering_has_real_coverage() -> None: def test_comprehension_total_ordering_pinned_counts() -> None: - # Pins the lane: 4 read end-to-end (two sort chains + two transitive - # compares); 4 refused — three multi-word-NP cases and one compare with - # trailing tokens. No single multi-word-NP rule is wrong=0-safe here because - # the gold canonicalizes multi-word NPs three contradictory ways. + # Pins the lane: 7 read end-to-end — two sort chains, two transitive compares, + # and the three multi-word cases ("North station", "Red rank", "Level one…") + # now chunked by the canonicalization contract; 1 refused — the compare with a + # trailing prepositional phrase ("…in the same order"). report = run() - assert report["correct"] == 4 - assert report["refused"] == 4 + assert report["correct"] == 7 + assert report["refused"] == 1 assert report["total"] == 8 diff --git a/tests/test_comprehension_wrong_zero_property.py b/tests/test_comprehension_wrong_zero_property.py index a64e2eb7..0a6da290 100644 --- a/tests/test_comprehension_wrong_zero_property.py +++ b/tests/test_comprehension_wrong_zero_property.py @@ -186,3 +186,121 @@ def test_set_membership_reader_is_faithful_or_refuses() -> None: committed += 1 assert got == expected, (prose, got, expected) assert committed > 50 + + +# --------------------------------------------------------------------------- # +# Multi-word NP chunking — faithful under the canonicalization contract +# --------------------------------------------------------------------------- # + + +def _class_term(rng): + """Return (canonical_id, plural_surface) for a class NP, sometimes multi-word. + + "t1" -> ("t1", "t1s"); "t1 t2" -> ("t1_t2", "t1 t2s") (head pluralized). + """ + w1 = rng.choice(_TERMS) + if rng.random() < 0.5: + head = rng.choice([t for t in _TERMS if t != w1]) + return f"{w1}_{head}", f"{w1} {head}s" + return w1, f"{w1}s" + + +def _item_term(rng, pool): + """Return (canonical_id, surface) for an item NP, sometimes multi-word (no + pluralization for items): "t1" or "t1 t2" -> "t1_t2".""" + w1 = pool.pop() + if rng.random() < 0.5 and pool: + w2 = pool.pop() + return f"{w1}_{w2}", f"{w1} {w2}" + return w1, w1 + + +_MW_PREMISE = { + "A": lambda s, p: f"All {s} are {p}.", + "E": lambda s, p: f"No {s} are {p}.", + "I": lambda s, p: f"Some {s} are {p}.", + "O": lambda s, p: f"Some {s} are not {p}.", +} +_MW_CONCLUSION = { + "A": lambda s, p: f"Therefore all {s} are {p}.", + "E": lambda s, p: f"Therefore no {s} are {p}.", + "I": lambda s, p: f"Therefore some {s} are {p}.", + "O": lambda s, p: f"Therefore some {s} are not {p}.", +} + + +def test_multiword_syllogism_is_faithful_or_refuses() -> None: + rng = random.Random(31337) + committed = 0 + for _ in range(400): + meta = [] + seen = set() + while len(meta) < 3: + canon, plural = _class_term(rng) + if canon not in seen: + seen.add(canon) + meta.append((canon, plural)) + surf = dict(meta) + canons = [c for c, _ in meta] + (s1, p1), (s2, p2), (sc, pc) = ( + rng.sample(canons, 2), + rng.sample(canons, 2), + rng.sample(canons, 2), + ) + f1, f2, fc = rng.choice("AEIO"), rng.choice("AEIO"), rng.choice("AEIO") + prose = " ".join( + [_MW_PREMISE[f1](surf[s1], surf[p1]), _MW_PREMISE[f2](surf[s2], surf[p2]), + _MW_CONCLUSION[fc](surf[sc], surf[pc])] + ) + structure = { + "terms": sorted(canons), + "domain_size": 3, + "premises": [ + {"form": f1, "subject": s1, "predicate": p1}, + {"form": f2, "subject": s2, "predicate": p2}, + ], + } + query = {"kind": "validity", "conclusion": {"form": fc, "subject": sc, "predicate": pc}} + try: + expected = syl_oracle(structure, query) + except SylError: + expected = None + got = _committed(comprehend(prose), to_syllogism, syl_oracle, SylError) + if got is not None: + committed += 1 + assert got == expected, (prose, got, expected) + assert committed > 50 + + +def test_multiword_total_ordering_is_faithful_or_refuses() -> None: + rng = random.Random(2718) + committed = 0 + for _ in range(300): + pool = list(_TERMS) + rng.shuffle(pool) + n = rng.randint(2, 4) + chain, surf = [], {} + for _ in range(n): + canon, s = _item_term(rng, pool) + chain.append(canon) + surf[canon] = s + rels = [{"less": chain[i], "greater": chain[i + 1]} for i in range(n - 1)] + facts = ", and ".join(f"{surf[lo]} is below {surf[hi]}" for lo, hi in zip(chain, chain[1:])) + "." + if rng.random() < 0.5: + order = rng.choice(["ascending", "descending"]) + prose = f"{facts} Sort {order}." + query = {"kind": "sort", "order": order} + else: + x, y = rng.sample(chain, 2) + prose = f"{facts} Compare {surf[x]} with {surf[y]}." + query = {"kind": "compare", "left": x, "right": y} + structure = {"items": sorted(chain), "relations": rels} + try: + expected = ord_oracle(structure, query) + except OrdError: + expected = None + got = _committed(comprehend(prose), to_total_ordering, ord_oracle, OrdError) + if got is not None: + committed += 1 + assert got == expected, (prose, got, expected) + assert committed > 50