core/tests/test_lexicon_single_source.py

"""Phase 2A pins: ``generate/lexicon.py`` is the only owner of its tables.

Three kinds of test here, and the distinction matters:

1. **Structural** — no module outside the lexicon may define a second copy of
   an owned table. Goes red if a duplicate is reintroduced.
2. **Caller provenance** — the set of modules importing each owned symbol is
   recorded. Goes red when a new consumer appears, forcing a decision rather
   than a silent widening.
3. **Recorded decisions** — every divergence the lexicon preserves on purpose
   is pinned here with its reason, so "unify these" becomes a test change
   someone has to justify.

A fourth group pins **current defective behaviour** that Phase 2B will fix.
Those tests are *supposed* to change; they exist so the fix cannot land
silently.

Why provenance is recorded rather than computed: PR #129 measured the serving
import closure at 227 first-party modules containing **all 13** table-owning
modules, so a static import test returns "everything is serving" and
discriminates nothing. Import-reachable is not call-reachable. The empirical
arbiter for behaviour is the 11 pinned lane SHAs, not this file.
"""

from __future__ import annotations

import ast
from pathlib import Path

import pytest

from generate import lexicon

REPO_ROOT = Path(__file__).resolve().parents[1]

#: Packages scanned for duplicate table literals. ``evals/`` is deliberately
#: excluded: ``evals/grammar_roundtrip/projection.py`` keeps its own copy of
#: the quantifier map so the lane never consumes the thing it measures, and
#: that copy is pinned by its own test.
SCANNED_PACKAGES = ("generate", "chat", "core", "teaching")

#: Owned table -> the modules that may import it. Curated, not computed.
RECORDED_CONSUMERS: dict[str, frozenset[str]] = {
    "CONNECTIVES": frozenset({
        "generate.proof_chain.member",
        "generate.proof_chain.verb",
        "generate.proof_chain.cond_member",
    }),
    "STRUCTURAL": frozenset({"generate.proof_chain.english"}),
    "COPULAS": frozenset({"generate.proof_chain.english"}),
    "COPULA_FORMS": frozenset({"generate.proof_chain.member"}),
    # Auxiliary-role consumers of the same be-form inventory. Found by the
    # duplicate scanner below, not by grepping for COPULA-ish names — which is
    # precisely why the structural test exists.
    "BE_FINITE": frozenset({"generate.realizer_guard", "chat.runtime"}),
    "SENTENTIAL_NOT": frozenset({
        "generate.proof_chain.english",
        "generate.proof_chain.member",
    }),
    "NEGATION_BEARING": frozenset({"generate.proof_chain.english"}),
    "NEGATION_BEARING_WITH_NOT": frozenset({"generate.proof_chain.member"}),
    "QUANTIFIER_LEAD": frozenset({"generate.proof_chain.english"}),
    "QUANTIFIER_TOKENS": frozenset({"generate.proof_chain.member"}),
    "PLURAL_QUANTIFIERS": frozenset({"generate.templates"}),
    # Phase 4: the closed set of predicates whose object is a predicate nominal
    # and therefore agrees in number with the subject. Closed, not productive —
    # a rule would pluralize "grounded in evidence" into "evidences".
    "PREDICATIVE_NOMINAL": frozenset({"generate.templates"}),
    "PREDICATE_DISPLAY": frozenset({
        "generate.templates",
        "generate.semantic_templates",
    }),
    "DISCOURSE_PREDICATE_DISPLAY": frozenset({"generate.discourse_planner"}),
    # Phase 2B: generate.morphology became the single owner of the number
    # RULES, so it is now the consumer of the number tables. The reader no
    # longer imports a table of its own — it calls morphology.singularize —
    # which is why READER_IRREGULAR_SINGULARS/READER_SINGULAR_KEYS were deleted
    # rather than left behind documenting a coverage gap that no longer exists.
    "IRREGULAR_PLURALS": frozenset({"generate.morphology", "generate.templates"}),
    "IRREGULAR_SINGULARS": frozenset({
        "generate.morphology",
        "generate.proof_chain.member",
    }),
    "INVARIANT_NUMBER": frozenset({"generate.morphology"}),
    "MASS_NOUNS": frozenset({"generate.morphology"}),
    # Phase 3: the closed f/fe -> ves set, derived from the singularizer's own
    # ves-rows so the rule cannot claim a word the table does not know.
    "VES_PLURAL_SINGULARS": frozenset({"generate.morphology"}),
}

#: The tables a duplicate literal would be a duplicate *of*. Frozen as
#: comparable values (dicts -> sorted item tuples, sets -> frozensets).
OWNED_TABLES: dict[str, object] = {
    "CONNECTIVES": lexicon.CONNECTIVES,
    "STRUCTURAL": lexicon.STRUCTURAL,
    # COPULAS / COPULA_FORMS / BE_FINITE are all views of one inventory, so a
    # single entry covers them; the report names the fact, not every alias.
    "BE_FINITE_FORMS": lexicon.BE_FINITE,
    "NEGATION_BEARING": lexicon.NEGATION_BEARING,
    "NEGATION_BEARING_WITH_NOT": lexicon.NEGATION_BEARING_WITH_NOT,
    "QUANTIFIER_LEAD": lexicon.QUANTIFIER_LEAD,
    "QUANTIFIER_TOKENS": lexicon.QUANTIFIER_TOKENS,
    "PLURAL_QUANTIFIERS": lexicon.PLURAL_QUANTIFIERS,
    "PREDICATE_DISPLAY": tuple(sorted(lexicon.PREDICATE_DISPLAY.items())),
    "IRREGULAR_PLURALS": tuple(sorted(lexicon.IRREGULAR_PLURALS.items())),
    "IRREGULAR_SINGULARS": tuple(sorted(lexicon.IRREGULAR_SINGULARS.items())),
}


def _python_files() -> list[Path]:
    out: list[Path] = []
    for pkg in SCANNED_PACKAGES:
        root = REPO_ROOT / pkg
        if root.is_dir():
            out.extend(p for p in root.rglob("*.py") if "__pycache__" not in p.parts)
    return out


def _module_name(path: Path) -> str:
    return ".".join(path.relative_to(REPO_ROOT).with_suffix("").parts)


def _frozen_literal(node: ast.AST) -> object | None:
    """A comparable value for a set/dict/frozenset(...) literal, else None."""
    target = node
    if (
        isinstance(node, ast.Call)
        and isinstance(node.func, ast.Name)
        and node.func.id == "frozenset"
        and len(node.args) == 1
    ):
        target = node.args[0]
    if not isinstance(target, (ast.Dict, ast.Set)):
        return None
    try:
        value = ast.literal_eval(target)
    except (ValueError, SyntaxError, TypeError):
        return None
    if isinstance(value, dict):
        if not all(isinstance(k, str) for k in value):
            return None
        return tuple(sorted(value.items()))
    if isinstance(value, (set, frozenset)):
        return frozenset(value)
    return None


# --------------------------------------------------------------------------
# 1. Structural — one definition per fact
# --------------------------------------------------------------------------


def test_no_module_outside_the_lexicon_defines_an_owned_table() -> None:
    """A second copy of an owned table anywhere in the scanned packages is a
    regression of the whole phase. Mutation check: paste any owned literal
    back into its old home and this goes red."""
    lexicon_path = REPO_ROOT / "generate" / "lexicon.py"
    # Keyed by (file, line) so one literal is reported once — a
    # ``frozenset({...})`` call and its inner set are two AST nodes.
    hits: dict[tuple[str, object], set[str]] = {}
    for path in _python_files():
        if path == lexicon_path:
            continue
        tree = ast.parse(path.read_text(encoding="utf-8"))
        for node in ast.walk(tree):
            frozen = _frozen_literal(node)
            if frozen is None:
                continue
            for name, owned in OWNED_TABLES.items():
                if frozen == owned:
                    key = (str(path.relative_to(REPO_ROOT)), getattr(node, "lineno", "?"))
                    hits.setdefault(key, set()).add(name)
    duplicates = [
        f"{file}:{line} redefines lexicon.{'/'.join(sorted(names))}"
        for (file, line), names in sorted(hits.items())
    ]
    assert not duplicates, "duplicate linguistic tables:\n  " + "\n  ".join(duplicates)


def test_the_duplicate_scanner_can_actually_find_a_duplicate(tmp_path: Path) -> None:
    """The scanner above is worthless if it cannot detect the thing it forbids.
    Feed it a known duplicate and require a hit."""
    planted = tmp_path / "planted.py"
    planted.write_text(
        'X = frozenset({"if", "then", "or", "and", "either"})\n', encoding="utf-8"
    )
    tree = ast.parse(planted.read_text(encoding="utf-8"))
    hits = [
        name
        for node in ast.walk(tree)
        for name, owned in OWNED_TABLES.items()
        if (frozen := _frozen_literal(node)) is not None and frozen == owned
    ]
    assert "CONNECTIVES" in hits


# --------------------------------------------------------------------------
# 2. Caller provenance
# --------------------------------------------------------------------------


def _actual_consumers() -> dict[str, set[str]]:
    found: dict[str, set[str]] = {}
    for path in _python_files():
        tree = ast.parse(path.read_text(encoding="utf-8"))
        for node in ast.walk(tree):
            if isinstance(node, ast.ImportFrom) and node.module == "generate.lexicon":
                for alias in node.names:
                    found.setdefault(alias.name, set()).add(_module_name(path))
    return found


def test_every_lexicon_consumer_is_recorded() -> None:
    """A new importer of an owned table must be added to RECORDED_CONSUMERS.
    That is the point: widening a table's reach becomes a decision with a
    diff, not a side effect."""
    actual = _actual_consumers()
    unexpected = {
        name: sorted(mods - RECORDED_CONSUMERS.get(name, frozenset()))
        for name, mods in actual.items()
        if mods - RECORDED_CONSUMERS.get(name, frozenset())
    }
    assert not unexpected, f"unrecorded lexicon consumers: {unexpected}"


def test_every_recorded_consumer_still_imports_what_it_claims() -> None:
    """The other direction — a stale record is also a defect, because it makes
    the provenance table lie about who depends on what."""
    actual = _actual_consumers()
    stale = {
        name: sorted(mods - actual.get(name, set()))
        for name, mods in RECORDED_CONSUMERS.items()
        if mods - actual.get(name, set())
    }
    assert not stale, f"recorded consumers that no longer import: {stale}"


# --------------------------------------------------------------------------
# 3. Recorded decisions — derivations that can no longer drift
# --------------------------------------------------------------------------


def test_structural_is_connectives_plus_therefore() -> None:
    assert lexicon.STRUCTURAL == lexicon.CONNECTIVES | {"therefore"}
    assert "therefore" not in lexicon.CONNECTIVES


def test_copulas_is_exactly_the_set_view_of_copula_forms() -> None:
    assert lexicon.COPULAS == frozenset(lexicon.COPULA_FORMS)
    assert lexicon.COPULA_FORMS == ("is", "are", "was", "were"), "order is load-bearing"


def test_the_two_negation_sets_differ_by_exactly_bare_not() -> None:
    """v2-EN normalizes ``<copula> not`` and so must let ``not`` reach that
    rule; v3-MEM refuses every non-copular ``not``. Both bands serve, so
    unifying them changes what CORE refuses — Phase 2B, not 2A."""
    assert lexicon.NEGATION_BEARING_WITH_NOT - lexicon.NEGATION_BEARING == {"not"}
    assert "not" not in lexicon.NEGATION_BEARING


def test_quantifier_lead_is_a_strict_subset_of_quantifier_tokens() -> None:
    """Measured before the migration: member's 25 tokens were a strict
    superset of english's 8, agreeing on all 8. Derivation preserves that."""
    assert lexicon.QUANTIFIER_LEAD < lexicon.QUANTIFIER_TOKENS
    assert lexicon.QUANTIFIER_TOKENS - lexicon.QUANTIFIER_LEAD == lexicon.QUANTIFIER_NON_LEAD


def test_plural_quantifiers_excludes_the_singular_taking_leads() -> None:
    """``every``/``each`` lead a categorical reading but take a SINGULAR noun
    ("each dog is"), so their absence from PLURAL_QUANTIFIERS is correct, not
    a coverage gap. This is why the two sets are different facts rather than
    divergent copies."""
    for singular_taking in ("every", "each"):
        assert singular_taking in lexicon.QUANTIFIER_LEAD
        assert singular_taking not in lexicon.PLURAL_QUANTIFIERS


def test_discourse_display_diverges_from_the_shared_table_on_exactly_one_key() -> None:
    """``is_defined_as`` -> "is" is a deliberate register choice (the short
    copula reads as prose mid-paragraph). ``belongs_to`` is derived, so it
    cannot drift."""
    shared, discourse = lexicon.PREDICATE_DISPLAY, lexicon.DISCOURSE_PREDICATE_DISPLAY
    differing = {k for k in discourse if shared.get(k) != discourse[k]}
    assert differing == {"is_defined_as"}
    assert discourse["is_defined_as"] == "is"
    assert discourse["belongs_to"] == shared["belongs_to"]


def test_discourse_display_scope_is_two_entries_on_purpose() -> None:
    """Widening this to the full 26 would silently change paragraph rendering
    for 24 predicates that currently fall through to
    ``predicate.replace("_", " ")``. Scope is preserved exactly."""
    assert set(lexicon.DISCOURSE_PREDICATE_DISPLAY) == {"is_defined_as", "belongs_to"}
    assert len(lexicon.PREDICATE_DISPLAY) == 26


def test_the_two_number_tables_are_inverse_directions_not_copies() -> None:
    """§1.3 counted "3 copies of irregular plurals, all diverging." Measured,
    one is a PLURALIZER (singular -> plural) and the others SINGULARIZERS
    (plural -> singular). Comparing them as copies is a category error."""
    assert lexicon.IRREGULAR_PLURALS["child"] == "children"
    assert lexicon.IRREGULAR_SINGULARS["children"] == "child"
    # Non-invariant entries present in both go opposite ways.
    round_trips = [
        (sing, plur)
        for sing, plur in lexicon.IRREGULAR_PLURALS.items()
        if sing != plur and plur in lexicon.IRREGULAR_SINGULARS
    ]
    assert round_trips, "expected overlap between the two directions"
    for sing, plur in round_trips:
        assert lexicon.IRREGULAR_SINGULARS[plur] == sing, f"{sing}/{plur} not inverse"


def test_invariant_number_is_derived_from_the_singularizer() -> None:
    """Invariants must not be a second hand-written list.

    A hand-written list is exactly how the directions drifted: the pluralizer
    lacked ``aircraft``/``means``/``offspring`` and produced "aircrafts",
    "meanses", "offsprings" while the singularizer knew all three. Deriving
    from the ``key == value`` rows makes that class of gap unrepresentable."""
    expected = {p for p, s in lexicon.IRREGULAR_SINGULARS.items() if p == s}
    assert lexicon.INVARIANT_NUMBER == expected
    for word in ("sheep", "aircraft", "means", "offspring", "species", "series"):
        assert word in lexicon.INVARIANT_NUMBER


# --------------------------------------------------------------------------
# 4. Defects Phase 2B must fix — pinned so the fix cannot land silently
# --------------------------------------------------------------------------


@pytest.mark.parametrize(
    ("plural", "singular"),
    [
        # Phase 2A pinned these as WRONG (wolve / leave / knive). Phase 2B
        # flipped them by routing the reader through the shared 29-entry
        # singularizer.
        ("wolves", "wolf"),
        ("leaves", "leaf"),
        ("knives", "knife"),
        ("halves", "half"),
        ("thieves", "thief"),
        ("children", "child"),
        ("men", "man"),
        # Genuinely inflected irregulars the reader could not read before.
        ("cacti", "cactus"),
        ("fungi", "fungus"),
        ("oxen", "ox"),
    ],
)
def test_reader_singularizes_irregulars_correctly(plural: str, singular: str) -> None:
    """The defect Phase 2A pinned, now fixed.

    ``reader.py``'s comment always claimed an unrecognized plural "REFUSES
    rather than guessing a wrong singular (wrong=0)". Until 2B the code did
    not implement it — ``_singularize`` fell through to a bare ``-s`` strip
    and minted corrupted ids that reached served text ("all wolve are
    mammal"). The comment is now true of the code.
    """
    from generate.meaning_graph.reader import _singularize

    assert _singularize(plural) == singular


@pytest.mark.parametrize(
    "invariant", ["fish", "sheep", "deer", "species", "series", "news", "means"]
)
def test_reader_declines_number_invariant_forms(invariant: str) -> None:
    """Invariants are AMBIGUOUS in number and must be declined, not resolved.

    "fish are mammals" is plural; "a fish is a mammal" is singular; the token
    cannot tell you which. Two independent reasons this must decline:

    1. **Honesty** — resolving it is a guess about number, and 2A measured what
       guessing costs: the old ``-s`` strip turned ``news`` into ``new`` and
       ``species`` into ``specy``.
    2. **Soundness** — the serving composer tries the categorical band (v1b)
       first. Resolving an invariant makes v1b *accept* a sentence it cannot
       decide, stealing the case from a band that can. Measured: resolving
       ``fish`` made ds-ex-0012 ("No fish are mammals. Therefore some fish are
       mammals.") answer ``invalid`` instead of ``refuted`` — **wrong=1 on a
       ratified band.** Declining restores the fall-through.
    """
    from generate.meaning_graph.reader import _singularize

    assert _singularize(invariant) is None


@pytest.mark.parametrize("not_a_plural", ["child", "evidence", "wolf", "", "ss"])
def test_reader_declines_rather_than_guessing(not_a_plural: str) -> None:
    """The other half of wrong=0: a singular, a mass noun, or anything the
    closed rules do not confidently cover returns ``None`` so the caller can
    refuse instead of minting a corrupted id. Without this, ``child`` would
    become ``chil``."""
    from generate.meaning_graph.reader import _singularize

    assert _singularize(not_a_plural) is None


def test_regular_plurals_still_singularize() -> None:
    """Control: widening the irregular table must not break the regular rule
    that handles the overwhelming majority of real input."""
    from generate.meaning_graph.reader import _singularize

    assert _singularize("cars") == "car"
    assert _singularize("glasses") == "glass"
    assert _singularize("cities") == "city"


def test_the_two_number_directions_are_mutual_inverses() -> None:
    """Every irregular the singularizer knows, the pluralizer can produce.

    This is the law that makes read/write agreement possible at all: if CORE
    can read ``cacti`` but writes ``cactuses``, no round trip can close. Phase
    2B added the three missing inverses (cactus/fungus/die) and derived
    :data:`lexicon.INVARIANT_NUMBER` from the singularizer's own invariant
    rows, so ``aircraft``/``means``/``offspring`` stop becoming "aircrafts",
    "meanses", "offsprings".
    """
    from generate.morphology import pluralize

    broken = {
        singular: (pluralize(singular), plural)
        for plural, singular in lexicon.IRREGULAR_SINGULARS.items()
        if pluralize(singular) != plural
    }
    assert not broken, f"singular -> plural does not round-trip: {broken}"


def test_mass_nouns_and_compounds_inflect_correctly() -> None:
    """Two cases the categorical renderer depends on: mass nouns must not take
    a plural ("all evidence", not "all evidences"), and an English compound
    inflects on its HEAD ("guard dogs", not "guards dog")."""
    from generate.morphology import pluralize

    assert pluralize("evidence") == "evidence"
    assert pluralize("knowledge") == "knowledge"
    assert pluralize("guard_dog") == "guard_dogs"
    assert pluralize("guard dog") == "guard dogs"