From df6332fcb55f065888209646acd20a59927ddb5f Mon Sep 17 00:00:00 2001 From: Shay Date: Sun, 26 Jul 2026 17:45:27 -0700 Subject: [PATCH] refactor(generate): one owner per linguistic fact (Phase 2A) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Collapses the duplicated closed English tables into generate/lexicon.py. Measured on main @ 0948f7cd: the same facts were encoded up to five times across the reading and writing paths, each copy written independently as a successive deduction band landed. The 2A/2B split is decided empirically per #129 — make the change, run the 11 pinned lanes, let byte-identity rule. Result: 11/11 byte-identical, so this entire unit is 2A and nothing spilled into the authorization-gated 2B. Collapsed to one object: - connectives 3 identical copies (member, verb, cond_member) - predicate display 2 identical 26-entry copies (semantic_templates, templates) - be-form inventory 5 copies, not the 2 §1.3 counted. The structural test found realizer_guard._BE_AUX and chat/runtime._BE_FORMS, which a COPULA-shaped grep could never see. Derived rather than duplicated, so they can no longer drift: - STRUCTURAL = CONNECTIVES | {therefore} - QUANTIFIER_TOKENS = QUANTIFIER_LEAD | QUANTIFIER_NON_LEAD - NEGATION_BEARING_WITH_NOT = NEGATION_BEARING | {not} - READER_IRREGULAR_SINGULARS = IRREGULAR_SINGULARS restricted to 8 keys §1.3 re-measured while doing this, and it overstated the disease (plan §1.9). Almost nothing contradicts: - the "3 divergent plural tables" are 1 pluralizer + 2 singularizers, i.e. inverse directions; comparing them as copies is a category error - reader's 8 singulars are a STRICT SUBSET of member's 29 with the difference in reader's favour empty, so its values are all correct and only its coverage is short - the "3 divergent quantifier sets" are 3 distinct facts; every/each lead a clause but take singular nouns, so their absence from PLURAL_QUANTIFIERS is correct - english's negation set is a subset of member's, and the difference is principled: v2-EN normalizes " not", v3-MEM refuses it Exactly one genuine contradiction exists in the whole inventory: discourse_planner maps is_defined_as -> "is" where the others map it to "is defined as". Preserved as a register choice, pinned by test. Found while reading, pinned as current behaviour for 2B to flip: reader._singularize does NOT refuse uncovered plurals despite a comment claiming it does — it falls through to a bare -s strip, so news -> new and species -> specy, exactly the corruptions member.py's table comment says its table exists to prevent. Two 2A exit criteria were unmeasurable as written and are replaced: - "Jaccard -> 1.00" cannot work. measure_grammar_seam.py scans source literals, so unifying a fact removes it from those files and Jaccard FELL, 0.083 -> 0.023 — the success direction reported by a metric shaped to read like failure. Replaced with an object-identity count: 3 distinct objects behind 8 names, from 8-behind-8. Value comparison cannot tell a shared object from two equal copies. - "one table per fact" presumed §1.3's inventory was right. The report now separates must-be-one-object (all UNIFIED) from related-but-distinct (6 relations, all HOLD). ADR-0258 §5 already decided morphology's destination is a ratified pack, triggered by a grc/he member band. No such band exists, so the promotion is correctly deferred and lexicon.py is a staging area, not a rival home — no new ADR. Destination measured for whoever picks it up: packs/en/morphology .jsonl has 9 records and zero irregular plurals, features.number has no consumer, and _pack_morph_roots_for reads a top-level "root" key that 0 of 31 records across all four packs define, so it returns {} every call. [Verification]: in-worktree on CPython 3.12.13, uv sync --locked — smoke 621 unchanged; deductive 383 (364 + 19 new); scripts/verify_lane_shas.py 11/11 byte-identical. --- chat/runtime.py | 4 +- core/cli_test.py | 1 + docs/plans/grammar-unification-2026-07-26.md | 94 ++++- generate/discourse_planner.py | 6 +- generate/lexicon.py | 305 ++++++++++++++++ generate/meaning_graph/reader.py | 12 +- generate/proof_chain/cond_member.py | 3 +- generate/proof_chain/english.py | 19 +- generate/proof_chain/member.py | 39 +- generate/proof_chain/verb.py | 3 +- generate/realizer_guard.py | 4 +- generate/semantic_templates.py | 30 +- generate/templates.py | 52 +-- scripts/measure_grammar_seam.py | 127 +++++-- tests/test_lexicon_single_source.py | 352 +++++++++++++++++++ 15 files changed, 907 insertions(+), 144 deletions(-) create mode 100644 generate/lexicon.py create mode 100644 tests/test_lexicon_single_source.py diff --git a/chat/runtime.py b/chat/runtime.py index 6bf2c283..f9d62ca2 100644 --- a/chat/runtime.py +++ b/chat/runtime.py @@ -8,6 +8,8 @@ import warnings from collections.abc import Sequence from typing import Any, List +from generate.lexicon import BE_FINITE + import numpy as np from algebra.versor import versor_condition @@ -181,7 +183,7 @@ _QUESTION_WORDS = frozenset({"what", "who", "how", "why", "when", "where", "whic # does not allocate a fresh set on every English turn. Aux-verbs that # precede the prompt's content noun ("is", "are", "was", "were") get # filtered out so the content-noun search lands on the actual subject. -_BE_FORMS: frozenset[str] = frozenset({"is", "are", "was", "were"}) +_BE_FORMS: frozenset[str] = BE_FINITE _TERMINALS = frozenset({".", "?", ";", "!"}) _UNKNOWN_DOMAIN_SURFACE = "I don't know — insufficient grounding for that yet." diff --git a/core/cli_test.py b/core/cli_test.py index d8f34cb2..bc88aba3 100644 --- a/core/cli_test.py +++ b/core/cli_test.py @@ -245,6 +245,7 @@ TEST_SUITES: dict[str, tuple[str, ...]] = { "tests/test_ratified_ledger_bridge.py", "tests/test_vocab_trigger_instrument.py", "tests/test_grammar_roundtrip.py", + "tests/test_lexicon_single_source.py", ), "full": ("tests/",), } diff --git a/docs/plans/grammar-unification-2026-07-26.md b/docs/plans/grammar-unification-2026-07-26.md index 1f9508d9..c76fc778 100644 --- a/docs/plans/grammar-unification-2026-07-26.md +++ b/docs/plans/grammar-unification-2026-07-26.md @@ -279,6 +279,61 @@ Round-trip therefore needs a **defined projection**, not identity. Choosing that projection is Phase 1's first design decision, and §6 records why this matters for direction. +### 1.9 §1.3 re-measured during Phase 2A — mostly subsets, not contradictions + +§1.3 was counted by name and by table size. Reading the actual values while +building `generate/lexicon.py` changed the picture materially, and in CORE's +favour: **almost nothing contradicts.** + +| §1.3 claim | measured | correction | +|---|---|---| +| irregular plurals: 3 copies, all diverge | 1 pluralizer + 2 singularizers | **direction confusion.** `templates` is singular→plural; `member`/`reader` are plural→singular. Comparing them as copies is a category error. | +| — | `reader` ⊂ `member`, difference empty | **no disagreement.** The reader's 8 values all agree; only its coverage is short. | +| quantifier tokens: 5 copies, 3-way divergence | 3 *distinct facts* | `QUANTIFIER_LEAD` (can lead a clause), `QUANTIFIER_TOKENS` (⊃ lead, adds pronouns), `PLURAL_QUANTIFIERS` (forces plural agreement). `every`/`each` lead but take singular nouns, so their absence from the third is **correct**. | +| connectives: 4 copies, `english` adds `therefore` | 3 identical + 1 derived | confirmed; the 4th is `english._STRUCTURAL`, exactly `CONNECTIVES ∪ {therefore}`. | +| negation-bearing: 2, diverge by `'not'` | `english` ⊂ `member` | confirmed, and the difference is **principled**: v2-EN normalizes ` not`, v3-MEM refuses all non-copular `not`. | +| copula forms: 2 copies | **5 copies** | undercounted. The structural test found `realizer_guard._BE_AUX` and `chat/runtime._BE_FORMS` — same four words, auxiliary role, invisible to a `COPULA`-shaped grep. | + +Exactly **one** genuine contradiction exists in the whole inventory: +`discourse_planner._PREDICATE_HUMANIZE` maps `is_defined_as` → `"is"` where the +other two map it to `"is defined as"`. Preserved as a deliberate register choice +(the short copula reads as prose mid-paragraph), pinned by test. + +⇒ The §1.3 framing "tables that should agree and do not" overstated the disease. +The accurate diagnosis is **coverage asymmetry plus three facts sharing one +name**, which is a better problem to have: subsets can be derived from their +superset with zero behaviour change, which is why Phase 2A came back 11/11 +byte-identical. + +**Worse than §1.7 stated, though.** `reader._singularize` does not refuse +uncovered plurals despite a comment claiming it does — it falls through to a +bare `-s` strip. So `news` → `new` and `species` → `specy`: precisely the +corruptions `member.py`'s table comment says its table exists to prevent. Same +fact, one file guarded, the other not. Pinned as current behaviour in +`tests/test_lexicon_single_source.py`; Phase 2B must flip it. + +### 1.10 The destination for morphology already exists and is disconnected + +ADR-0258 §5 decided that the number table "moves from module constants to a +ratified pack" **when a `grc_*`/`he_*` member band is built.** That trigger has +not fired (no such band exists in `generate/proof_chain/`), so the promotion is +correctly deferred — `generate/lexicon.py` is a staging area, not a competing +home, and no new ADR is warranted. + +The destination's measured state, so the promotion is not planned on a guess: + +- `packs/en/morphology.jsonl` = **9 records.** Seven are forms of *be*; two are + noun plurals (`word`, `beginning`) that are **regular** and exist to support + John 1:1. **Zero irregular English plurals exist as pack data.** +- **`features.number` has no consumer.** No code reads it. +- The only reader of `morphology.jsonl` is + `chat/pack_resolver.py::_pack_morph_roots_for`, which extracts a top-level + `root` key. **0 of 31 records across all four packs define `root`**, so it + returns `{}` on every call while its docstring claims it "enables root-level + depth (Hebrew triconsonantal, Greek stems)". A dead path, not a slow one. +- The pack is *richer* than the code on one point: it has `am` + (`en:be:present:1sg`), which all five Python be-form copies lack. + --- ## 2. Goal, non-goals, and the thesis check @@ -457,6 +512,33 @@ Deliver: hash, the tables were not actually identical — stop, treat it as a divergence requiring a decision, and move it to Phase 2B. +**RESULT — the split fell entirely on the 2A side.** `generate/lexicon.py` owns +the tables; **11/11 lane SHA pins came back byte-identical**, so by the #129 +rule every change in this unit is 2A and nothing spilled into 2B. Smoke 621 +unchanged, deductive 383 (364 + 19 new). + +Two exit criteria needed re-definition rather than just measuring, and both +re-definitions are recorded here because the original wording was unmeasurable: + +- **"Jaccard → 1.00"** cannot be the measure. `measure_grammar_seam.py` scans + *source literals*, so unifying a fact **removes** it from those files and + Jaccard **fell**, 0.083 → 0.023. That is the success direction, reported by a + metric shaped to read like failure. Replaced with an **object-identity** + count: distinct underlying objects behind the consumer names, which is what + "one source of truth" actually asserts. Comparing by value cannot distinguish + a shared object from two equal copies, so identity is the only honest test. + Measured: **3 distinct objects behind 8 names**, from 8-behind-8. +- **"one table per fact"** presumed the §1.3 fact inventory was right. §1.9 + shows it was not, so the report now separates `_SHOULD_AGREE` (must be one + object → all **UNIFIED**) from `_RELATED_BUT_DISTINCT` (six expected + relations, all **HOLD**: derived-plus-`therefore`, strict-superset, + distinct-fact, differ-by-`not`, subset, inverse-direction). + +**The structural test earned its place immediately**: it found two copies of the +be-form inventory (`realizer_guard._BE_AUX`, `chat/runtime._BE_FORMS`) that a +name-based grep missed because neither is named like a copula. §1.3's "2 copies" +was really 5. + ### Phase 2B — Serving-path tables and the categorical render defect — **authorization gate** *Touches:* `generate/meaning_graph/reader.py::_IRREGULAR_PLURALS`, @@ -467,9 +549,15 @@ Deliver: 1. Re-pluralization at categorical render time, fixing `all dog are mammal` → `all dogs are mammals` (§1.7 cause 1 — affects *all* nouns). -2. The reader's 8-entry plural table replaced by 2A's shared table, fixing the - 8 silently-wrong singulars and the 12-of-20 reader-vs-reader disagreement - (§1.7 cause 2). +2. The reader's 8-key view (`lexicon.READER_SINGULAR_KEYS`) widened to the full + 29-entry singularizer. **Corrected framing (§1.9):** the reader's 8 *values* + are already right — they agree with the full table entry for entry — so this + is a **coverage** fix, not a correctness fix, and there are no "8 silently + wrong singulars." The silent wrongness is in the *fallback*: + `reader._singularize` guesses via a bare `-s` strip instead of refusing, so + `wolves`→`wolve`, `news`→`new`, `species`→`specy`. Its own comment claims it + refuses; it does not. Those five are pinned as current behaviour in + `tests/test_lexicon_single_source.py` and this phase must flip them. 3. Updated lane SHA pins — **surgical single-line edits only**, never `--update` — with the old and new hash recorded per lane. diff --git a/generate/discourse_planner.py b/generate/discourse_planner.py index 47ed0d73..cd6e5936 100644 --- a/generate/discourse_planner.py +++ b/generate/discourse_planner.py @@ -53,6 +53,7 @@ import json from dataclasses import dataclass, field from enum import Enum, unique +from generate.lexicon import DISCOURSE_PREDICATE_DISPLAY from generate.graph_planner import Relation from generate.intent import ( CompoundIntent, @@ -767,10 +768,7 @@ def plan_compound_discourse( # * a fixed-template connective from the table below. # No synthesis, no LLM, no approximation. -_PREDICATE_HUMANIZE: dict[str, str] = { - "is_defined_as": "is", - "belongs_to": "belongs to", -} +_PREDICATE_HUMANIZE: dict[str, str] = DISCOURSE_PREDICATE_DISPLAY def _humanize_predicate(predicate: str) -> str: diff --git a/generate/lexicon.py b/generate/lexicon.py new file mode 100644 index 00000000..22f4df72 --- /dev/null +++ b/generate/lexicon.py @@ -0,0 +1,305 @@ +"""Single source of truth for CORE's closed English lexical tables. + +Before this module the same linguistic facts were encoded up to five times +across the reading and writing paths, each copy written independently as a +successive deduction band landed. Measured on ``main`` @ ``0948f7cd``: +35 word-tables on the reading path vs 9 on the writing path, sharing 43 of +518 distinct words (Jaccard 0.083). + +Every table here is **closed** — a finite enumeration, not a rule. Where two +call sites need different content, this module defines one literal and +*derives* the other, so a divergence is stated once and cannot drift. + +Naming convention: a name is the linguistic fact, not the consumer. When a +consumer needs a narrower view, the narrow name says whose view it is +(``STRUCTURAL``, ``READER_SINGULAR_KEYS``). + +**Deliberate non-goal:** this module does not decide anything. It holds +tables and derivations; the refusal/agreement logic stays with each band. +Consumers and their serving status are recorded in +``tests/test_lexicon_single_source.py``, which fails when a new consumer +appears. Phase 2A of ``docs/plans/grammar-unification-2026-07-26.md``. + +**This module is a staging area, not the final home.** ADR-0258 §5 already +decided where number morphology belongs: + + the number-linking table is CORE's first morphology table, and it is + exactly the artifact class the tri-language doctrine says must + eventually enter through pack ratification (ADR-0253). [...] When a + ``grc_*``/``he_*`` member band is built, the table moves from module + constants to a ratified pack; no premature parameterization now. + +That trigger has **not** fired — ``packs/{grc,he,el}/`` exist, but no +``grc``/``he`` member band does, so the promotion is correctly deferred and +this module does not pre-empt it. Consolidating five disagreeing copies into +one is a prerequisite for promotion either way: you cannot ratify a fact as +a pack row while the tree holds several answers to it. + +Measured state of the destination, for whoever picks the promotion up: +``packs/en/morphology.jsonl`` holds **9 records** (7 forms of *be*, plus +``word``/``beginning`` — both regular plurals, present to support John 1:1), +so **no irregular English plural exists as pack data**. Neither end of the +pipe is connected: ``features.number`` has no consumer anywhere, and the only +code that opens ``morphology.jsonl`` +(``chat/pack_resolver.py::_pack_morph_roots_for``) extracts a top-level +``root`` key that **0 of 31 records across all four packs** define, so it +returns ``{}`` every call. +""" + +from __future__ import annotations + +from typing import Final + +# -------------------------------------------------------------------------- +# Connectives +# -------------------------------------------------------------------------- + +#: Connective structure the member/verb/conditional bands do not compose. +#: Was three byte-identical copies: ``proof_chain/member.py::_CONNECTIVES``, +#: ``proof_chain/verb.py::_CONNECTIVES``, +#: ``proof_chain/cond_member.py::_CONNECTIVE_TOKENS``. +CONNECTIVES: Final[frozenset[str]] = frozenset({"if", "then", "or", "and", "either"}) + +#: The v2-EN band's structural-function-word set. Identical to +#: :data:`CONNECTIVES` plus ``therefore`` — that band consumes ``therefore`` +#: as the conclusion marker, so a leftover one signals an unconsumed parse. +#: Derived, not duplicated: the two sets can no longer drift apart. +STRUCTURAL: Final[frozenset[str]] = CONNECTIVES | {"therefore"} + +# -------------------------------------------------------------------------- +# Copulas and negation +# -------------------------------------------------------------------------- + +#: The finite forms of *be* that CORE recognizes, in canonical order. +#: +#: **One inventory, four roles.** This was four separate literals — a copula +#: set (``english.py``), a copula tuple (``member.py``), an auxiliary set for +#: negation lookahead (``realizer_guard.py::_BE_AUX``), and an auxiliary set +#: for prompt-anchor filtering (``chat/runtime.py::_BE_FORMS``). Copula and +#: auxiliary are different syntactic *roles*, but the token inventory is one +#: fact, and all four copies held the same four words. The role views below +#: are derived, so a fifth role cannot introduce a fifth answer. +#: +#: Known-incomplete: ``am``, ``be``, ``been``, ``being`` are absent from all +#: four original copies and remain absent here — widening the inventory +#: changes what every consumer recognizes, so it is not a 2A change. +#: ``packs/en/morphology.jsonl`` already carries ``am`` as +#: ``en:be:present:1sg``, which is one reason the eventual home for this fact +#: is the pack rather than this module. +BE_FINITE_FORMS: Final[tuple[str, ...]] = ("is", "are", "was", "were") + +#: Set view of :data:`BE_FINITE_FORMS` for membership tests. +BE_FINITE: Final[frozenset[str]] = frozenset(BE_FINITE_FORMS) + +#: Copular role view, ordered. ``proof_chain/member.py`` relies on ordered +#: iteration for dispatch, so this stays a tuple. +COPULA_FORMS: Final[tuple[str, ...]] = BE_FINITE_FORMS + +#: Copular role view, as a set. +COPULAS: Final[frozenset[str]] = BE_FINITE + +#: The "it is not the case that " sentential-negation prefix. Was two +#: byte-identical copies (``english.py``, ``member.py``). +SENTENTIAL_NOT: Final[tuple[str, ...]] = ("it", "is", "not", "the", "case", "that") + +#: Negation-bearing tokens a band cannot normalize away. Leaving one inside +#: an opaque atom would hide a negation from the engine, so the bands refuse. +NEGATION_BEARING: Final[frozenset[str]] = frozenset( + {"never", "cannot", "nor", "neither", "nothing", "none", "no"} +) + +#: :data:`NEGATION_BEARING` plus bare ``not``. +#: +#: **Recorded divergence (Phase 2A decision).** ``english.py`` (v2-EN) uses +#: the 7-token set and ``member.py`` (v3-MEM) uses this 8-token one. The +#: difference is not an oversight: v2-EN normalizes `` not`` into +#: ``~atom`` and so must let ``not`` through to that rule, while v3-MEM +#: normalizes only the copular slot and the sentential prefix and refuses +#: every other ``not``. +#: +#: Both bands serve, so **unifying them changes what CORE refuses** and is +#: deferred to Phase 2B under the authorization gate. Deriving the larger +#: set from the smaller removes the duplication now without changing any +#: behaviour, which is the whole point of the 2A/2B split. +NEGATION_BEARING_WITH_NOT: Final[frozenset[str]] = NEGATION_BEARING | {"not"} + +# -------------------------------------------------------------------------- +# Quantifiers — three distinct facts, previously conflated +# -------------------------------------------------------------------------- + +#: Quantifiers that can *lead* a clause, making it categorical +#: ("all whales are mammals"). Used to detect internal structure a band +#: must not flatten. +QUANTIFIER_LEAD: Final[frozenset[str]] = frozenset( + {"all", "every", "each", "no", "some", "none", "any", "most"} +) + +#: What ``member.py`` recognizes beyond :data:`QUANTIFIER_LEAD`. +#: +#: This is a **provenance grouping, not a linguistic category** — it mixes +#: determiners (``few``, ``many``, ``both``) with pronouns (``everyone``, +#: ``nothing``). It is named for what it is (the delta) rather than given a +#: category name it would not earn. Splitting it into real categories needs +#: a linguistic decision, not a refactor. +QUANTIFIER_NON_LEAD: Final[frozenset[str]] = frozenset( + { + "few", "many", "several", "both", "either", "neither", + "everyone", "everybody", "everything", + "someone", "somebody", "something", + "anyone", "anybody", "anything", + "nobody", "nothing", + } +) + +#: Every quantifier token ``member.py`` refuses to lower. Measured to be a +#: strict superset of :data:`QUANTIFIER_LEAD`, so it is derived rather than +#: re-listed: the two sets agreed on all 8 shared tokens and can no longer +#: disagree. +QUANTIFIER_TOKENS: Final[frozenset[str]] = QUANTIFIER_LEAD | QUANTIFIER_NON_LEAD + +#: Quantifiers that force **plural agreement** on the subject and verb. +#: +#: A genuinely different fact from :data:`QUANTIFIER_LEAD`, not a divergent +#: copy of it. ``every`` and ``each`` are quantifier-leads yet take a +#: *singular* noun by English rule ("each dog is"), so their absence here is +#: correct; ``few``/``many``/``several``/``various`` take plurals but cannot +#: lead a categorical reading. Overlap is 4 of 8 in each direction, which is +#: what made these look like a divergence in the §1.3 count. +PLURAL_QUANTIFIERS: Final[frozenset[str]] = frozenset( + {"all", "some", "many", "few", "most", "several", "various", "no"} +) + +# -------------------------------------------------------------------------- +# Predicate display +# -------------------------------------------------------------------------- + +#: Machine predicate id -> human display phrase. Was two byte-identical +#: 26-entry copies: ``semantic_templates.py::_PREDICATE_HUMANIZE`` (the +#: serving writer) and ``templates.py::_PREDICATE_DISPLAY`` (the eval-only +#: realizer). +PREDICATE_DISPLAY: Final[dict[str, str]] = { + "is_defined_as": "is defined as", + "is_caused_by": "is caused by", + "has_steps": "has the following steps", + "contrasts_with": "contrasts with", + "corrects": "corrects", + "recalls": "recalls", + "is_verified_as": "is verified as", + "addresses": "addresses", + "defines": "defines", + "means": "means", + "grounds": "grounds", + "supports": "supports", + "causes": "causes", + "reveals": "reveals", + "precedes": "precedes", + "follows": "follows", + "belongs_to": "belongs to", + "answers": "answers", + "is_grounded_in": "is grounded in", + "is_distinguished_from": "is distinguished from", + "implies": "implies", + "entails": "entails", + "requires": "requires", + "verifies": "verifies", + "evidences": "evidences", + "orders": "orders", +} + +#: The discourse planner's two-entry display table. +#: +#: **Recorded divergence (Phase 2A decision).** Two deliberate differences +#: from :data:`PREDICATE_DISPLAY`, both preserved: +#: +#: 1. ``is_defined_as`` renders as ``"is"``, not ``"is defined as"``. Inside +#: a flowing paragraph the short copula reads as prose; the long form +#: reads as a schema dump. This is a register choice. +#: 2. The table is **two entries, not 26**, on purpose. Every other +#: predicate falls through to ``predicate.replace("_", " ")``. Widening it +#: to the full table would silently change paragraph rendering for 24 +#: predicates, so scope is preserved exactly. +#: +#: ``belongs_to`` agreed with :data:`PREDICATE_DISPLAY` and is therefore +#: derived from it — the one entry that can drift no longer can. +DISCOURSE_PREDICATE_DISPLAY: Final[dict[str, str]] = { + "is_defined_as": "is", + "belongs_to": PREDICATE_DISPLAY["belongs_to"], +} + +# -------------------------------------------------------------------------- +# Number morphology — two inverse maps, not two divergent copies +# -------------------------------------------------------------------------- +# +# §1.3 of the plan counted "irregular plurals: 3 copies, all three diverge." +# Measured against source, that count conflates two directions: +# +# * ``templates.py`` singular -> plural (a PLURALIZER, 25 entries) +# * ``member.py`` plural -> singular (a SINGULARIZER, 29 entries) +# * ``reader.py`` plural -> singular (a SINGULARIZER, 8 entries) +# +# and the two singularizers do **not** contradict each other: reader's 8 +# entries are a *strict subset* of member's 29 (set difference in the +# reader's favour is empty). So there is no disagreement to arbitrate here — +# only a coverage asymmetry, plus one direction's knowledge being +# unavailable in the other direction. + +#: Singular -> plural. The pluralizer, from ``templates.py``. +IRREGULAR_PLURALS: Final[dict[str, str]] = { + "child": "children", "ox": "oxen", "foot": "feet", "tooth": "teeth", + "man": "men", "woman": "women", "person": "people", + "mouse": "mice", "louse": "lice", "goose": "geese", + # invariant + "sheep": "sheep", "fish": "fish", "deer": "deer", "moose": "moose", + "series": "series", "species": "species", + # latin/greek-origin domain vocabulary + "datum": "data", "criterion": "criteria", "phenomenon": "phenomena", + "analysis": "analyses", "axis": "axes", "basis": "bases", + "thesis": "theses", "hypothesis": "hypotheses", + "mitochondrion": "mitochondria", +} + +#: Plural -> singular, including invariants that map to themselves. The +#: singularizer, from ``member.py``, which is the widest number table CORE +#: has. Consulted BEFORE any suffix rule: if either token of a candidate +#: pair appears here, this table is the only authority for that pair — which +#: is what keeps ``species``/``specie`` and ``news``/``new`` unlinked. +IRREGULAR_SINGULARS: Final[dict[str, str]] = { + "men": "man", "women": "woman", "people": "person", + "children": "child", "mice": "mouse", "geese": "goose", + "feet": "foot", "teeth": "tooth", "oxen": "ox", + "wolves": "wolf", "knives": "knife", "lives": "life", + "leaves": "leaf", "halves": "half", "elves": "elf", + "loaves": "loaf", "thieves": "thief", "cacti": "cactus", + "fungi": "fungus", "dice": "die", + # invariants (plural == singular) + "sheep": "sheep", "fish": "fish", "deer": "deer", + "species": "species", "series": "series", "means": "means", + "offspring": "offspring", "aircraft": "aircraft", "news": "news", +} + +#: The 8 keys ``meaning_graph/reader.py`` currently covers. +#: +#: **Recorded divergence (Phase 2A decision).** The reader's values are all +#: *correct* — they agree with :data:`IRREGULAR_SINGULARS` entry for entry — +#: so the values are derived below and only the key list is local. What is +#: wrong is the **coverage**, and the consequence is not a refusal: +#: ``reader._singularize`` falls through to a bare ``-s`` strip, so an +#: uncovered plural is silently mis-singularized rather than declined +#: (``wolves`` -> ``wolve``, ``news`` -> ``new``, ``species`` -> ``specy``). +#: The reader's own comment claims it "REFUSES rather than guessing a wrong +#: singular (wrong=0)"; the code does not implement that. +#: +#: Widening this to the full table changes minted entity ids and therefore +#: served surfaces and trace hashes, so it is **Phase 2B** work behind the +#: authorization gate. Recorded here, with the gap named, rather than fixed +#: silently. +READER_SINGULAR_KEYS: Final[tuple[str, ...]] = ( + "people", "men", "women", "children", "feet", "teeth", "mice", "geese", +) + +#: The reader's view of :data:`IRREGULAR_SINGULARS`, restricted to +#: :data:`READER_SINGULAR_KEYS`. Derived, so the reader's 8 values cannot +#: drift from the 29-entry table they are a subset of. +READER_IRREGULAR_SINGULARS: Final[dict[str, str]] = { + key: IRREGULAR_SINGULARS[key] for key in READER_SINGULAR_KEYS +} diff --git a/generate/meaning_graph/reader.py b/generate/meaning_graph/reader.py index ddf8b264..ff3dc218 100644 --- a/generate/meaning_graph/reader.py +++ b/generate/meaning_graph/reader.py @@ -51,6 +51,7 @@ from __future__ import annotations import re from dataclasses import dataclass, field +from generate.lexicon import READER_IRREGULAR_SINGULARS from generate.meaning_graph.model import ( Entity, MeaningGraph, @@ -63,16 +64,7 @@ _ARTICLES = frozenset({"a", "an"}) # Common irregular plurals the corpus exercises. Conservative + closed; an # unrecognized plural REFUSES rather than guessing a wrong singular (wrong=0). -_IRREGULAR_PLURALS = { - "people": "person", - "men": "man", - "women": "woman", - "children": "child", - "feet": "foot", - "teeth": "tooth", - "mice": "mouse", - "geese": "goose", -} +_IRREGULAR_PLURALS = READER_IRREGULAR_SINGULARS # Categorical quantifier -> the MeaningGraph predicate it mints. The predicate # vocabulary is shared between facts and the "therefore" conclusion query, and is diff --git a/generate/proof_chain/cond_member.py b/generate/proof_chain/cond_member.py index 1e2ebb1a..f1523fd1 100644 --- a/generate/proof_chain/cond_member.py +++ b/generate/proof_chain/cond_member.py @@ -53,6 +53,7 @@ from dataclasses import dataclass # reused VERBATIM (one normalization discipline across every argument band — # deliberate private-name imports inside the proof_chain package, same # precedent as generate.proof_chain.member). +from generate.lexicon import CONNECTIVES from generate.proof_chain.english import _SENTENCE_RE, _split_on, _tokenize from generate.proof_chain.member import ( _A_LEADS, @@ -75,7 +76,7 @@ from generate.proof_chain.shape import ( #: Connective structure this band's grammar composes (v2-EN's inventory, #: minus "therefore" — that token is the commit-gate handled by the #: outer sentence loop, never part of a clause's own structure). -_CONNECTIVE_TOKENS = frozenset({"if", "then", "or", "and", "either"}) +_CONNECTIVE_TOKENS = CONNECTIVES #: Honesty caps — beyond these the argument is refused, never truncated. #: ``MAX_ATOMS`` counts minted (individual, class) pairs. diff --git a/generate/proof_chain/english.py b/generate/proof_chain/english.py index beb66105..c74750fe 100644 --- a/generate/proof_chain/english.py +++ b/generate/proof_chain/english.py @@ -49,6 +49,13 @@ from __future__ import annotations import re from dataclasses import dataclass +from generate.lexicon import ( + COPULAS, + NEGATION_BEARING, + QUANTIFIER_LEAD, + SENTENTIAL_NOT, + STRUCTURAL, +) from generate.proof_chain.shape import ( EN_ATOMIC, EN_CONDITIONAL_CHAIN, @@ -61,21 +68,23 @@ _SENTENCE_RE = re.compile(r"\s*([^.?!]+?)\s*([.?!])") #: Structural function words. One may never survive into an opaque atom — a #: leftover here means the clause has structure this grammar did not consume. -_STRUCTURAL = frozenset({"if", "then", "or", "and", "either", "therefore"}) +_STRUCTURAL = STRUCTURAL #: Quantifier-led clause = categorical shape ("all whales are mammals") — real #: internal structure the opaque band must NOT flatten (a valid syllogism read #: as three opaque atoms would yield a misleading "doesn't follow"). Refused #: here; the categorical band (v1b) is the honest home for these. -_QUANTIFIER_LEAD = frozenset({"all", "every", "each", "no", "some", "none", "any", "most"}) +_QUANTIFIER_LEAD = QUANTIFIER_LEAD #: Negation-bearing tokens inside a clause that this band cannot normalize. #: Leaving one inside an opaque atom would hide a negation from the engine #: ("it never rains" vs "it rains" would be unrelated atoms) — refuse instead. -_NEGATION_BEARING = frozenset({"never", "cannot", "nor", "neither", "nothing", "none", "no"}) +#: This band deliberately EXCLUDES bare ``not``, which the copular rule below +#: normalizes — see ``lexicon.NEGATION_BEARING_WITH_NOT`` for the divergence. +_NEGATION_BEARING = NEGATION_BEARING #: Copulas whose " not" form this band DOES normalize into ``~atom``. -_COPULAS = frozenset({"is", "are", "was", "were"}) +_COPULAS = COPULAS #: Contraction → expanded tokens (applied before parsing; deterministic). _CONTRACTIONS = { @@ -86,7 +95,7 @@ _CONTRACTIONS = { } #: The "it is not the case that " sentential-negation prefix. -_SENTENTIAL_NOT = ("it", "is", "not", "the", "case", "that") +_SENTENTIAL_NOT = SENTENTIAL_NOT #: Honesty caps — an argument beyond these is refused, not truncated. MAX_PREMISE_SENTENCES = 16 diff --git a/generate/proof_chain/member.py b/generate/proof_chain/member.py index a85666d0..2b827bfb 100644 --- a/generate/proof_chain/member.py +++ b/generate/proof_chain/member.py @@ -50,6 +50,14 @@ from dataclasses import dataclass # The v2-EN sentence splitter and tokenizer are reused VERBATIM (one # normalization discipline, no drift between the English-argument bands) — # a deliberate private-name import inside the proof_chain package. +from generate.lexicon import ( + CONNECTIVES, + COPULA_FORMS, + IRREGULAR_SINGULARS, + NEGATION_BEARING_WITH_NOT, + QUANTIFIER_TOKENS, + SENTENTIAL_NOT, +) from generate.proof_chain.english import _SENTENCE_RE, _tokenize from generate.proof_chain.shape import ( EN_MEMBER_ATOMIC, @@ -65,32 +73,25 @@ _E_LEAD = "no" #: Quantifier tokens (determiners AND pronouns) this band cannot lower — #: existential/plurality readings refuse rather than misread "everyone" or #: "some men" as an individual. A/E leads are dispatched before this check. -_QUANTIFIER_TOKENS = frozenset({ - "all", "every", "each", "no", "some", "none", "any", "most", "few", - "many", "several", "both", "either", "neither", - "everyone", "everybody", "everything", "someone", "somebody", - "something", "anyone", "anybody", "anything", "nobody", "nothing", -}) +_QUANTIFIER_TOKENS = QUANTIFIER_TOKENS #: Connective structure this band does not compose (a future band fuses the #: v2-EN connective grammar with these sentence readings — ADR-0258 §6.1). -_CONNECTIVES = frozenset({"if", "then", "or", "and", "either"}) +_CONNECTIVES = CONNECTIVES #: Negation-bearing tokens with no normalized reading here (only the copular #: ``is not`` slot and the sentential prefix are normalized). -_NEGATION_BEARING = frozenset({ - "not", "never", "cannot", "nor", "neither", "nothing", "none", "no", -}) +_NEGATION_BEARING = NEGATION_BEARING_WITH_NOT #: Relative-clause markers — internal structure a name/class run must not hide. _RELATIVE_MARKERS = frozenset({"that", "which", "who", "whom", "whose"}) #: All copular forms recognized for DISPATCH; only ``is``/``are`` are in-band #: (tense is a deliberate scope-out — ADR-0258 §6.3). -_COPULA_FORMS = ("is", "are", "was", "were") +_COPULA_FORMS = COPULA_FORMS #: The "it is not the case that " sentential-negation prefix. -_SENTENTIAL_NOT = ("it", "is", "not", "the", "case", "that") +_SENTENTIAL_NOT = SENTENTIAL_NOT #: Honesty caps — beyond these the argument is refused, never truncated. #: ``MAX_ATOMS`` counts minted (individual, class) pairs, so instantiation @@ -104,19 +105,7 @@ MAX_ATOMS = 24 #: only authority for that pair — which is what keeps ``species``/``specie`` #: and ``news``/``new`` unlinked. CORE's first morphology table; promoted to #: a ratified pack when the tri-language siblings land (ADR-0258 §5). -_IRREGULAR_PLURALS: dict[str, str] = { - "men": "man", "women": "woman", "people": "person", - "children": "child", "mice": "mouse", "geese": "goose", - "feet": "foot", "teeth": "tooth", "oxen": "ox", - "wolves": "wolf", "knives": "knife", "lives": "life", - "leaves": "leaf", "halves": "half", "elves": "elf", - "loaves": "loaf", "thieves": "thief", "cacti": "cactus", - "fungi": "fungus", "dice": "die", - # invariants (plural == singular) - "sheep": "sheep", "fish": "fish", "deer": "deer", - "species": "species", "series": "series", "means": "means", - "offspring": "offspring", "aircraft": "aircraft", "news": "news", -} +_IRREGULAR_PLURALS: dict[str, str] = IRREGULAR_SINGULARS _TABLE_TOKENS = frozenset(_IRREGULAR_PLURALS) | frozenset(_IRREGULAR_PLURALS.values()) _SIBILANT_ENDINGS = ("s", "x", "z", "ch", "sh") diff --git a/generate/proof_chain/verb.py b/generate/proof_chain/verb.py index 157294f1..3f6d06f9 100644 --- a/generate/proof_chain/verb.py +++ b/generate/proof_chain/verb.py @@ -66,6 +66,7 @@ from dataclasses import dataclass # sibilant set are reused VERBATIM — deliberate private-name imports inside # the proof_chain package, same precedent as generate.proof_chain.member and # generate.proof_chain.cond_member. +from generate.lexicon import CONNECTIVES from generate.proof_chain.english import _SENTENCE_RE, _tokenize from generate.proof_chain.member import ( _A_LEADS, @@ -91,7 +92,7 @@ from generate.proof_chain.shape import ( #: Connective structure this band does not compose (a future band fuses the #: v2-EN/v4-CM connective grammar with verb sentences — ADR-0260 §5). -_CONNECTIVES = frozenset({"if", "then", "or", "and", "either"}) +_CONNECTIVES = CONNECTIVES #: Auxiliary/determiner tokens that can never BE the verb or object of an #: in-band sentence. ``does`` is consumed only by the exact ``does not`` diff --git a/generate/realizer_guard.py b/generate/realizer_guard.py index 2af1d626..4508e59f 100644 --- a/generate/realizer_guard.py +++ b/generate/realizer_guard.py @@ -55,6 +55,8 @@ import re from dataclasses import dataclass from typing import Callable, Literal +from generate.lexicon import BE_FINITE + DISCLOSURE_SURFACE = "I do not have a reviewed articulation for that yet." """Bounded fallback surface used when the guard rejects a candidate. @@ -97,7 +99,7 @@ _ADVERB_FUNCTION_WORDS: frozenset[str] = frozenset({ _DO_AUX: frozenset[str] = frozenset({"do", "does", "did"}) -_BE_AUX: frozenset[str] = frozenset({"is", "are", "was", "were"}) +_BE_AUX: frozenset[str] = BE_FINITE @dataclass(frozen=True) diff --git a/generate/semantic_templates.py b/generate/semantic_templates.py index a3d5440d..8328a0ce 100644 --- a/generate/semantic_templates.py +++ b/generate/semantic_templates.py @@ -13,6 +13,7 @@ Design constraints: from __future__ import annotations +from generate.lexicon import PREDICATE_DISPLAY from generate.intent import IntentTag @@ -27,34 +28,7 @@ _INTENT_TEMPLATES: dict[IntentTag, str] = { IntentTag.UNKNOWN: "{subject} {predicate_h} {obj}", } -_PREDICATE_HUMANIZE: dict[str, str] = { - "is_defined_as": "is defined as", - "is_caused_by": "is caused by", - "has_steps": "has the following steps", - "contrasts_with": "contrasts with", - "corrects": "corrects", - "recalls": "recalls", - "is_verified_as": "is verified as", - "addresses": "addresses", - "defines": "defines", - "means": "means", - "grounds": "grounds", - "supports": "supports", - "causes": "causes", - "reveals": "reveals", - "precedes": "precedes", - "follows": "follows", - "belongs_to": "belongs to", - "answers": "answers", - "is_grounded_in": "is grounded in", - "is_distinguished_from": "is distinguished from", - "implies": "implies", - "entails": "entails", - "requires": "requires", - "verifies": "verifies", - "evidences": "evidences", - "orders": "orders", -} +_PREDICATE_HUMANIZE: dict[str, str] = PREDICATE_DISPLAY def humanize_predicate(predicate: str) -> str: diff --git a/generate/templates.py b/generate/templates.py index 944d93b9..55dfb046 100644 --- a/generate/templates.py +++ b/generate/templates.py @@ -12,6 +12,11 @@ consumes these as constraints rather than final output. from __future__ import annotations +from generate.lexicon import ( + IRREGULAR_PLURALS, + PLURAL_QUANTIFIERS, + PREDICATE_DISPLAY, +) from generate.articulation_legality import ( ArticulationLegality, validate_finite_predicate_legality, @@ -22,19 +27,7 @@ from generate.morphology import base_form, past_participle, past_tense, present_ # Noun pluralisation — used under quantifiers (all/some/many/few/most). # Closes english_fluency_ood gaps.md G2 (plural agreement). -_IRREGULAR_PLURALS: dict[str, str] = { - "child": "children", "ox": "oxen", "foot": "feet", "tooth": "teeth", - "man": "men", "woman": "women", "person": "people", - "mouse": "mice", "louse": "lice", "goose": "geese", - # invariant - "sheep": "sheep", "fish": "fish", "deer": "deer", "moose": "moose", - "series": "series", "species": "species", - # latin/greek-origin domain vocabulary - "datum": "data", "criterion": "criteria", "phenomenon": "phenomena", - "analysis": "analyses", "axis": "axes", "basis": "bases", - "thesis": "theses", "hypothesis": "hypotheses", - "mitochondrion": "mitochondria", -} +_IRREGULAR_PLURALS: dict[str, str] = IRREGULAR_PLURALS def pluralize(noun: str) -> str: @@ -57,9 +50,7 @@ def pluralize(noun: str) -> str: # Quantifiers that demand plural agreement on the subject + verb. # "the" / "a" stay singular; "every" / "each" are singular by English # rule even though semantically universal. -_PLURAL_QUANTIFIERS: frozenset[str] = frozenset({ - "all", "some", "many", "few", "most", "several", "various", "no", -}) +_PLURAL_QUANTIFIERS: frozenset[str] = PLURAL_QUANTIFIERS # Mass nouns — uncountable in English, so "all evidence", "some wisdom" # stay singular under quantifiers ("all evidences" is wrong). The @@ -87,34 +78,7 @@ def is_mass_noun(noun: str) -> bool: return noun.lower() in _MASS_NOUNS -_PREDICATE_DISPLAY: dict[str, str] = { - "is_defined_as": "is defined as", - "is_caused_by": "is caused by", - "has_steps": "has the following steps", - "contrasts_with": "contrasts with", - "corrects": "corrects", - "recalls": "recalls", - "is_verified_as": "is verified as", - "addresses": "addresses", - "defines": "defines", - "means": "means", - "grounds": "grounds", - "supports": "supports", - "causes": "causes", - "reveals": "reveals", - "precedes": "precedes", - "follows": "follows", - "belongs_to": "belongs to", - "answers": "answers", - "is_grounded_in": "is grounded in", - "is_distinguished_from": "is distinguished from", - "implies": "implies", - "entails": "entails", - "requires": "requires", - "verifies": "verifies", - "evidences": "evidences", - "orders": "orders", -} +_PREDICATE_DISPLAY: dict[str, str] = PREDICATE_DISPLAY def _humanize_predicate(predicate: str) -> str: diff --git a/scripts/measure_grammar_seam.py b/scripts/measure_grammar_seam.py index 2ffdc8c7..4fb7dc73 100644 --- a/scripts/measure_grammar_seam.py +++ b/scripts/measure_grammar_seam.py @@ -49,33 +49,75 @@ _WRITE_PATH_FILES = ( ) #: Groups of tables that encode the SAME linguistic fact and should agree. +#: Groups that are ONE fact and must therefore be ONE object. +#: +#: Corrected 2026-07-26 (plan §1.9). The original grouping asserted that the +#: three plural tables and the three quantifier sets were each one fact. They +#: are not: the plural tables are a pluralizer plus two singularizers (inverse +#: directions), and the quantifier sets are three distinct facts — "can lead a +#: clause", "is a quantifier token", "forces plural agreement". Reporting those +#: as DIVERGE was a correct answer to a wrong question, so they moved to +#: :data:`_RELATED_BUT_DISTINCT` below. _SHOULD_AGREE = { - "irregular plurals": ( - ("generate.proof_chain.member", "_IRREGULAR_PLURALS"), - ("generate.meaning_graph.reader", "_IRREGULAR_PLURALS"), - ("generate.templates", "_IRREGULAR_PLURALS"), - ), - "quantifier tokens": ( - ("generate.proof_chain.english", "_QUANTIFIER_LEAD"), - ("generate.proof_chain.member", "_QUANTIFIER_TOKENS"), - ("generate.templates", "_PLURAL_QUANTIFIERS"), - ), "connective tokens": ( - ("generate.proof_chain.english", "_STRUCTURAL"), ("generate.proof_chain.member", "_CONNECTIVES"), ("generate.proof_chain.verb", "_CONNECTIVES"), ("generate.proof_chain.cond_member", "_CONNECTIVE_TOKENS"), ), - "negation-bearing tokens": ( - ("generate.proof_chain.english", "_NEGATION_BEARING"), - ("generate.proof_chain.member", "_NEGATION_BEARING"), - ), "predicate display": ( ("generate.semantic_templates", "_PREDICATE_HUMANIZE"), ("generate.templates", "_PREDICATE_DISPLAY"), ), + "be-form inventory": ( + ("generate.proof_chain.english", "_COPULAS"), + ("generate.realizer_guard", "_BE_AUX"), + ("chat.runtime", "_BE_FORMS"), + ), } +#: Tables that are RELATED but genuinely distinct, with the relation that must +#: hold between them. Each is pinned by ``tests/test_lexicon_single_source.py``; +#: this table exists so the report states the relation instead of crying +#: divergence. ``relation`` is a predicate over the two loaded values. +_RELATED_BUT_DISTINCT = ( + ( + "STRUCTURAL = CONNECTIVES + therefore", + ("generate.proof_chain.english", "_STRUCTURAL"), + ("generate.proof_chain.member", "_CONNECTIVES"), + lambda a, b: frozenset(a) - frozenset(b) == {"therefore"}, + ), + ( + "QUANTIFIER_TOKENS strictly contains LEAD", + ("generate.proof_chain.member", "_QUANTIFIER_TOKENS"), + ("generate.proof_chain.english", "_QUANTIFIER_LEAD"), + lambda a, b: frozenset(b) < frozenset(a), + ), + ( + "PLURAL_QUANTIFIERS is a different fact (excludes every/each)", + ("generate.templates", "_PLURAL_QUANTIFIERS"), + ("generate.proof_chain.english", "_QUANTIFIER_LEAD"), + lambda a, b: {"every", "each"} <= frozenset(b) and not ({"every", "each"} & frozenset(a)), + ), + ( + "member negation = english negation + not", + ("generate.proof_chain.member", "_NEGATION_BEARING"), + ("generate.proof_chain.english", "_NEGATION_BEARING"), + lambda a, b: frozenset(a) - frozenset(b) == {"not"}, + ), + ( + "reader singulars are a subset of the full table", + ("generate.meaning_graph.reader", "_IRREGULAR_PLURALS"), + ("generate.proof_chain.member", "_IRREGULAR_PLURALS"), + lambda a, b: set(a.items()) < set(b.items()), + ), + ( + "pluralizer is the inverse direction of the singularizer", + ("generate.templates", "_IRREGULAR_PLURALS"), + ("generate.proof_chain.member", "_IRREGULAR_PLURALS"), + lambda a, b: all(b[p] == s for s, p in a.items() if s != p and p in b), + ), +) + #: Plural-subject agreement oracle for §1.4 — hand-written English, not derived #: from the code under test (deriving it from the code would make it agree by #: construction and measure nothing). @@ -141,8 +183,18 @@ def section_tables() -> None: print(f" writing path: {len(write_tables):3d} word-tables, {len(write_words):3d} distinct words") print(f" shared : {len(shared):3d} of {len(union)} union") print(f" JACCARD : {len(shared) / len(union):.3f}" if union else " JACCARD: n/a") + print( + " NOTE: these counts scan SOURCE LITERALS, so after Phase 2A they\n" + " measure how much table text still lives in these files, not\n" + " how many answers exist. A unified fact leaves the file as an\n" + " import and DROPS out of the count — Jaccard falling is the\n" + " expected direction, not a regression. Read the ownership\n" + " block below for the number the 2A exit criterion actually means." + ) - print("\n tables that encode the same fact:") + print("\n tables that encode the same fact (by OBJECT IDENTITY):") + owners_total = 0 + views_total = 0 for label, refs in _SHOULD_AGREE.items(): loaded = [] for module_name, attr in refs: @@ -151,13 +203,46 @@ def section_tables() -> None: except (ImportError, AttributeError): continue keys = frozenset(value) if not isinstance(value, tuple) else frozenset(value) - loaded.append((f"{module_name.split('.')[-1]}.{attr}", keys)) + loaded.append((f"{module_name.split('.')[-1]}.{attr}", keys, id(value))) if len(loaded) < 2: continue - all_equal = all(k == loaded[0][1] for _, k in loaded) - verdict = "IDENTICAL" if all_equal else "DIVERGE" - sizes = ", ".join(f"{n}={len(k)}" for n, k in loaded) - print(f" {verdict:9s} {label:26s} ({len(loaded)} copies: {sizes})") + # Distinct underlying OBJECTS, not distinct names. After unification a + # band attribute is the lexicon object itself, so N names backed by one + # object is one answer — which is what "unified" means. Comparing by + # value alone cannot tell a shared object from two equal copies. + distinct_objects = {oid for _, _, oid in loaded} + distinct_values = {keys for _, keys, _ in loaded} + owners_total += len(distinct_objects) + views_total += len(loaded) + if len(distinct_objects) == 1: + verdict = "UNIFIED" + elif len(distinct_values) == 1: + verdict = "EQUAL-COPY" # same content, still two objects + else: + verdict = "DIVERGE" + sizes = ", ".join(f"{n}={len(k)}" for n, k, _ in loaded) + print( + f" {verdict:10s} {label:26s} " + f"({len(distinct_objects)} owner(s) behind {len(loaded)} view(s): {sizes})" + ) + if views_total: + print( + f"\n OWNERSHIP : {owners_total} distinct objects behind {views_total} names" + f" (1 owner per fact is the 2A goal)" + ) + + print("\n related but DISTINCT facts — the relation that must hold:") + for label, ref_a, ref_b, relation in _RELATED_BUT_DISTINCT: + try: + a = getattr(importlib.import_module(ref_a[0]), ref_a[1]) + b = getattr(importlib.import_module(ref_b[0]), ref_b[1]) + except (ImportError, AttributeError): + continue + try: + ok = bool(relation(a, b)) + except Exception: # a broken relation is a red flag, not a crash + ok = False + print(f" {'HOLDS' if ok else 'BROKEN':10s} {label}") def section_agreement() -> None: diff --git a/tests/test_lexicon_single_source.py b/tests/test_lexicon_single_source.py new file mode 100644 index 00000000..08043929 --- /dev/null +++ b/tests/test_lexicon_single_source.py @@ -0,0 +1,352 @@ +"""Phase 2A pins: ``generate/lexicon.py`` is the only owner of its tables. + +Three kinds of test here, and the distinction matters: + +1. **Structural** — no module outside the lexicon may define a second copy of + an owned table. Goes red if a duplicate is reintroduced. +2. **Caller provenance** — the set of modules importing each owned symbol is + recorded. Goes red when a new consumer appears, forcing a decision rather + than a silent widening. +3. **Recorded decisions** — every divergence the lexicon preserves on purpose + is pinned here with its reason, so "unify these" becomes a test change + someone has to justify. + +A fourth group pins **current defective behaviour** that Phase 2B will fix. +Those tests are *supposed* to change; they exist so the fix cannot land +silently. + +Why provenance is recorded rather than computed: PR #129 measured the serving +import closure at 227 first-party modules containing **all 13** table-owning +modules, so a static import test returns "everything is serving" and +discriminates nothing. Import-reachable is not call-reachable. The empirical +arbiter for behaviour is the 11 pinned lane SHAs, not this file. +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +from generate import lexicon + +REPO_ROOT = Path(__file__).resolve().parents[1] + +#: Packages scanned for duplicate table literals. ``evals/`` is deliberately +#: excluded: ``evals/grammar_roundtrip/projection.py`` keeps its own copy of +#: the quantifier map so the lane never consumes the thing it measures, and +#: that copy is pinned by its own test. +SCANNED_PACKAGES = ("generate", "chat", "core", "teaching") + +#: Owned table -> the modules that may import it. Curated, not computed. +RECORDED_CONSUMERS: dict[str, frozenset[str]] = { + "CONNECTIVES": frozenset({ + "generate.proof_chain.member", + "generate.proof_chain.verb", + "generate.proof_chain.cond_member", + }), + "STRUCTURAL": frozenset({"generate.proof_chain.english"}), + "COPULAS": frozenset({"generate.proof_chain.english"}), + "COPULA_FORMS": frozenset({"generate.proof_chain.member"}), + # Auxiliary-role consumers of the same be-form inventory. Found by the + # duplicate scanner below, not by grepping for COPULA-ish names — which is + # precisely why the structural test exists. + "BE_FINITE": frozenset({"generate.realizer_guard", "chat.runtime"}), + "SENTENTIAL_NOT": frozenset({ + "generate.proof_chain.english", + "generate.proof_chain.member", + }), + "NEGATION_BEARING": frozenset({"generate.proof_chain.english"}), + "NEGATION_BEARING_WITH_NOT": frozenset({"generate.proof_chain.member"}), + "QUANTIFIER_LEAD": frozenset({"generate.proof_chain.english"}), + "QUANTIFIER_TOKENS": frozenset({"generate.proof_chain.member"}), + "PLURAL_QUANTIFIERS": frozenset({"generate.templates"}), + "PREDICATE_DISPLAY": frozenset({ + "generate.templates", + "generate.semantic_templates", + }), + "DISCOURSE_PREDICATE_DISPLAY": frozenset({"generate.discourse_planner"}), + "IRREGULAR_PLURALS": frozenset({"generate.templates"}), + "IRREGULAR_SINGULARS": frozenset({"generate.proof_chain.member"}), + "READER_IRREGULAR_SINGULARS": frozenset({"generate.meaning_graph.reader"}), +} + +#: The tables a duplicate literal would be a duplicate *of*. Frozen as +#: comparable values (dicts -> sorted item tuples, sets -> frozensets). +OWNED_TABLES: dict[str, object] = { + "CONNECTIVES": lexicon.CONNECTIVES, + "STRUCTURAL": lexicon.STRUCTURAL, + # COPULAS / COPULA_FORMS / BE_FINITE are all views of one inventory, so a + # single entry covers them; the report names the fact, not every alias. + "BE_FINITE_FORMS": lexicon.BE_FINITE, + "NEGATION_BEARING": lexicon.NEGATION_BEARING, + "NEGATION_BEARING_WITH_NOT": lexicon.NEGATION_BEARING_WITH_NOT, + "QUANTIFIER_LEAD": lexicon.QUANTIFIER_LEAD, + "QUANTIFIER_TOKENS": lexicon.QUANTIFIER_TOKENS, + "PLURAL_QUANTIFIERS": lexicon.PLURAL_QUANTIFIERS, + "PREDICATE_DISPLAY": tuple(sorted(lexicon.PREDICATE_DISPLAY.items())), + "IRREGULAR_PLURALS": tuple(sorted(lexicon.IRREGULAR_PLURALS.items())), + "IRREGULAR_SINGULARS": tuple(sorted(lexicon.IRREGULAR_SINGULARS.items())), +} + + +def _python_files() -> list[Path]: + out: list[Path] = [] + for pkg in SCANNED_PACKAGES: + root = REPO_ROOT / pkg + if root.is_dir(): + out.extend(p for p in root.rglob("*.py") if "__pycache__" not in p.parts) + return out + + +def _module_name(path: Path) -> str: + return ".".join(path.relative_to(REPO_ROOT).with_suffix("").parts) + + +def _frozen_literal(node: ast.AST) -> object | None: + """A comparable value for a set/dict/frozenset(...) literal, else None.""" + target = node + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "frozenset" + and len(node.args) == 1 + ): + target = node.args[0] + if not isinstance(target, (ast.Dict, ast.Set)): + return None + try: + value = ast.literal_eval(target) + except (ValueError, SyntaxError, TypeError): + return None + if isinstance(value, dict): + if not all(isinstance(k, str) for k in value): + return None + return tuple(sorted(value.items())) + if isinstance(value, (set, frozenset)): + return frozenset(value) + return None + + +# -------------------------------------------------------------------------- +# 1. Structural — one definition per fact +# -------------------------------------------------------------------------- + + +def test_no_module_outside_the_lexicon_defines_an_owned_table() -> None: + """A second copy of an owned table anywhere in the scanned packages is a + regression of the whole phase. Mutation check: paste any owned literal + back into its old home and this goes red.""" + lexicon_path = REPO_ROOT / "generate" / "lexicon.py" + # Keyed by (file, line) so one literal is reported once — a + # ``frozenset({...})`` call and its inner set are two AST nodes. + hits: dict[tuple[str, object], set[str]] = {} + for path in _python_files(): + if path == lexicon_path: + continue + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + frozen = _frozen_literal(node) + if frozen is None: + continue + for name, owned in OWNED_TABLES.items(): + if frozen == owned: + key = (str(path.relative_to(REPO_ROOT)), getattr(node, "lineno", "?")) + hits.setdefault(key, set()).add(name) + duplicates = [ + f"{file}:{line} redefines lexicon.{'/'.join(sorted(names))}" + for (file, line), names in sorted(hits.items()) + ] + assert not duplicates, "duplicate linguistic tables:\n " + "\n ".join(duplicates) + + +def test_the_duplicate_scanner_can_actually_find_a_duplicate(tmp_path: Path) -> None: + """The scanner above is worthless if it cannot detect the thing it forbids. + Feed it a known duplicate and require a hit.""" + planted = tmp_path / "planted.py" + planted.write_text( + 'X = frozenset({"if", "then", "or", "and", "either"})\n', encoding="utf-8" + ) + tree = ast.parse(planted.read_text(encoding="utf-8")) + hits = [ + name + for node in ast.walk(tree) + for name, owned in OWNED_TABLES.items() + if (frozen := _frozen_literal(node)) is not None and frozen == owned + ] + assert "CONNECTIVES" in hits + + +# -------------------------------------------------------------------------- +# 2. Caller provenance +# -------------------------------------------------------------------------- + + +def _actual_consumers() -> dict[str, set[str]]: + found: dict[str, set[str]] = {} + for path in _python_files(): + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.module == "generate.lexicon": + for alias in node.names: + found.setdefault(alias.name, set()).add(_module_name(path)) + return found + + +def test_every_lexicon_consumer_is_recorded() -> None: + """A new importer of an owned table must be added to RECORDED_CONSUMERS. + That is the point: widening a table's reach becomes a decision with a + diff, not a side effect.""" + actual = _actual_consumers() + unexpected = { + name: sorted(mods - RECORDED_CONSUMERS.get(name, frozenset())) + for name, mods in actual.items() + if mods - RECORDED_CONSUMERS.get(name, frozenset()) + } + assert not unexpected, f"unrecorded lexicon consumers: {unexpected}" + + +def test_every_recorded_consumer_still_imports_what_it_claims() -> None: + """The other direction — a stale record is also a defect, because it makes + the provenance table lie about who depends on what.""" + actual = _actual_consumers() + stale = { + name: sorted(mods - actual.get(name, set())) + for name, mods in RECORDED_CONSUMERS.items() + if mods - actual.get(name, set()) + } + assert not stale, f"recorded consumers that no longer import: {stale}" + + +# -------------------------------------------------------------------------- +# 3. Recorded decisions — derivations that can no longer drift +# -------------------------------------------------------------------------- + + +def test_structural_is_connectives_plus_therefore() -> None: + assert lexicon.STRUCTURAL == lexicon.CONNECTIVES | {"therefore"} + assert "therefore" not in lexicon.CONNECTIVES + + +def test_copulas_is_exactly_the_set_view_of_copula_forms() -> None: + assert lexicon.COPULAS == frozenset(lexicon.COPULA_FORMS) + assert lexicon.COPULA_FORMS == ("is", "are", "was", "were"), "order is load-bearing" + + +def test_the_two_negation_sets_differ_by_exactly_bare_not() -> None: + """v2-EN normalizes `` not`` and so must let ``not`` reach that + rule; v3-MEM refuses every non-copular ``not``. Both bands serve, so + unifying them changes what CORE refuses — Phase 2B, not 2A.""" + assert lexicon.NEGATION_BEARING_WITH_NOT - lexicon.NEGATION_BEARING == {"not"} + assert "not" not in lexicon.NEGATION_BEARING + + +def test_quantifier_lead_is_a_strict_subset_of_quantifier_tokens() -> None: + """Measured before the migration: member's 25 tokens were a strict + superset of english's 8, agreeing on all 8. Derivation preserves that.""" + assert lexicon.QUANTIFIER_LEAD < lexicon.QUANTIFIER_TOKENS + assert lexicon.QUANTIFIER_TOKENS - lexicon.QUANTIFIER_LEAD == lexicon.QUANTIFIER_NON_LEAD + + +def test_plural_quantifiers_excludes_the_singular_taking_leads() -> None: + """``every``/``each`` lead a categorical reading but take a SINGULAR noun + ("each dog is"), so their absence from PLURAL_QUANTIFIERS is correct, not + a coverage gap. This is why the two sets are different facts rather than + divergent copies.""" + for singular_taking in ("every", "each"): + assert singular_taking in lexicon.QUANTIFIER_LEAD + assert singular_taking not in lexicon.PLURAL_QUANTIFIERS + + +def test_discourse_display_diverges_from_the_shared_table_on_exactly_one_key() -> None: + """``is_defined_as`` -> "is" is a deliberate register choice (the short + copula reads as prose mid-paragraph). ``belongs_to`` is derived, so it + cannot drift.""" + shared, discourse = lexicon.PREDICATE_DISPLAY, lexicon.DISCOURSE_PREDICATE_DISPLAY + differing = {k for k in discourse if shared.get(k) != discourse[k]} + assert differing == {"is_defined_as"} + assert discourse["is_defined_as"] == "is" + assert discourse["belongs_to"] == shared["belongs_to"] + + +def test_discourse_display_scope_is_two_entries_on_purpose() -> None: + """Widening this to the full 26 would silently change paragraph rendering + for 24 predicates that currently fall through to + ``predicate.replace("_", " ")``. Scope is preserved exactly.""" + assert set(lexicon.DISCOURSE_PREDICATE_DISPLAY) == {"is_defined_as", "belongs_to"} + assert len(lexicon.PREDICATE_DISPLAY) == 26 + + +def test_the_two_number_tables_are_inverse_directions_not_copies() -> None: + """§1.3 counted "3 copies of irregular plurals, all diverging." Measured, + one is a PLURALIZER (singular -> plural) and the others SINGULARIZERS + (plural -> singular). Comparing them as copies is a category error.""" + assert lexicon.IRREGULAR_PLURALS["child"] == "children" + assert lexicon.IRREGULAR_SINGULARS["children"] == "child" + # Non-invariant entries present in both go opposite ways. + round_trips = [ + (sing, plur) + for sing, plur in lexicon.IRREGULAR_PLURALS.items() + if sing != plur and plur in lexicon.IRREGULAR_SINGULARS + ] + assert round_trips, "expected overlap between the two directions" + for sing, plur in round_trips: + assert lexicon.IRREGULAR_SINGULARS[plur] == sing, f"{sing}/{plur} not inverse" + + +def test_reader_number_table_is_a_derived_subset_that_cannot_drift() -> None: + """The reader's 8 values were measured to AGREE with the 29-entry table + entry for entry — the set difference in the reader's favour was empty. So + only its key list is local; the values are derived.""" + assert set(lexicon.READER_IRREGULAR_SINGULARS) == set(lexicon.READER_SINGULAR_KEYS) + assert len(lexicon.READER_SINGULAR_KEYS) == 8 + for key, value in lexicon.READER_IRREGULAR_SINGULARS.items(): + assert lexicon.IRREGULAR_SINGULARS[key] == value + assert set(lexicon.READER_IRREGULAR_SINGULARS) < set(lexicon.IRREGULAR_SINGULARS) + + +# -------------------------------------------------------------------------- +# 4. Defects Phase 2B must fix — pinned so the fix cannot land silently +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("plural", "current_wrong_singular"), + [ + ("wolves", "wolve"), + ("leaves", "leave"), + ("knives", "knive"), + ("news", "new"), + ("species", "specy"), + ], +) +def test_reader_silently_mis_singularizes_uncovered_plurals( + plural: str, current_wrong_singular: str +) -> None: + """CURRENT BEHAVIOUR, deliberately pinned as wrong. + + ``reader.py``'s comment claims an unrecognized plural "REFUSES rather + than guessing a wrong singular (wrong=0)". The code does not implement + that: ``_singularize`` falls through to a bare ``-s`` strip, so uncovered + plurals mint corrupted entity ids. ``news`` -> ``new`` and ``species`` -> + ``specy`` are exactly the corruptions ``member.py``'s table comment says + its table exists to prevent. + + Phase 2B replaces the reader's 8-key view with the full singularizer and + must flip these to ``wolf``/``leaf``/``knife``/``news``/``species``. + Changing minted ids moves trace hashes, which is why it is gated. + """ + from generate.meaning_graph.reader import _singularize + + assert _singularize(plural) == current_wrong_singular + + +def test_reader_covered_plurals_are_already_correct() -> None: + """The control for the test above: where the reader HAS an entry it is + right. So the defect is coverage, not wrong values — which is why 2A + could derive the values safely and leave coverage to 2B.""" + from generate.meaning_graph.reader import _singularize + + for plural, singular in lexicon.READER_IRREGULAR_SINGULARS.items(): + assert _singularize(plural) == singular