From 9ba6abfa3e22b7531e85d7231c10c5d18dac1c96 Mon Sep 17 00:00:00 2001
From: Shay <assetoverflow@icloud.com>
Date: Wed, 13 May 2026 20:33:20 -0700
Subject: [PATCH] Ground unknown tokens in ingest

---
 ingest/gate.py                     | 197 ++++++++++++++++++++++++++++-
 tests/test_unknown_token_ingest.py |  59 +++++++++
 vocab/manifold.py                  |  49 +++++++
 3 files changed, 304 insertions(+), 1 deletion(-)
 create mode 100644 tests/test_unknown_token_ingest.py

diff --git a/ingest/gate.py b/ingest/gate.py
index 6b3e4a3d..40a21aa8 100644
--- a/ingest/gate.py
+++ b/ingest/gate.py
@@ -24,9 +24,204 @@ Contract:
   Output: FieldState with F satisfying versor_condition(F) < 1e-6
 """
 
+from dataclasses import dataclass
+
+import numpy as np
+
+from algebra.cl41 import geometric_product
 from algebra.versor import normalize_to_versor, versor_condition
 from algebra.holonomy import holonomy_encode
 from field.state import FieldState
+from language_packs.schema import MorphologyEntry
+from language_packs.compiler import _feature_rotor
+
+
+@dataclass(frozen=True, slots=True)
+class _GroundedUnknown:
+    token: str
+    root_used: str
+    versor: np.ndarray
+    operators_applied: tuple[str, ...]
+    condition: float
+
+
+def _compact_root(root: str) -> str:
+    return root.replace("-", "")
+
+
+def _known_edges(morphology_entries: tuple[MorphologyEntry, ...]) -> tuple[tuple[str, ...], tuple[str, ...]]:
+    prefixes = {
+        prefix
+        for morphology in morphology_entries
+        for prefix in morphology.prefix_chain
+        if prefix
+    }
+    suffixes = {
+        suffix
+        for morphology in morphology_entries
+        for suffix in morphology.suffix_chain
+        if suffix
+    }
+    return (
+        tuple(sorted(prefixes, key=len, reverse=True)),
+        tuple(sorted(suffixes, key=len, reverse=True)),
+    )
+
+
+def _root_surfaces(vocab, morphology_entries: tuple[MorphologyEntry, ...]) -> dict[str, str]:
+    roots: dict[str, str] = {}
+    for morphology in morphology_entries:
+        for candidate in (
+            morphology.surface,
+            morphology.lemma,
+            morphology.stem,
+            _compact_root(morphology.root) if morphology.root else None,
+        ):
+            if not candidate:
+                continue
+            try:
+                vocab.get_versor(candidate)
+            except KeyError:
+                continue
+            roots.setdefault(candidate, candidate)
+    return roots
+
+
+def _root_affinity(candidate: str, root: str) -> int:
+    common_prefix = 0
+    for left, right in zip(candidate, root):
+        if left != right:
+            break
+        common_prefix += 1
+    shared = len(set(candidate).intersection(root))
+    length_penalty = abs(len(candidate) - len(root))
+    return (common_prefix * 8) + (shared * 2) - length_penalty
+
+
+def _best_decomposition(
+    token: str,
+    vocab,
+    morphology_entries: tuple[MorphologyEntry, ...],
+) -> tuple[str, tuple[str, ...], tuple[str, ...]]:
+    prefixes, suffixes = _known_edges(morphology_entries)
+    roots = _root_surfaces(vocab, morphology_entries)
+    prefix_options = ("", *prefixes)
+    suffix_options = ("", *suffixes)
+
+    best: tuple[int, str, tuple[str, ...], tuple[str, ...]] | None = None
+    for prefix in prefix_options:
+        if prefix and not token.startswith(prefix):
+            continue
+        after_prefix = token[len(prefix):] if prefix else token
+        for suffix in suffix_options:
+            if suffix and not after_prefix.endswith(suffix):
+                continue
+            root_candidate = after_prefix[: -len(suffix)] if suffix else after_prefix
+            root_surface = roots.get(root_candidate)
+            if root_surface is None:
+                continue
+            score = len(root_candidate) * 8 + len(prefix) + len(suffix)
+            if prefix or suffix:
+                score += 64
+            if best is None or score > best[0]:
+                best = (
+                    score,
+                    root_surface,
+                    (prefix,) if prefix else (),
+                    (suffix,) if suffix else (),
+                )
+
+    if best is None:
+        for prefix in prefix_options:
+            if prefix and not token.startswith(prefix):
+                continue
+            after_prefix = token[len(prefix):] if prefix else token
+            for suffix in suffix_options:
+                if suffix and not after_prefix.endswith(suffix):
+                    continue
+                root_candidate = after_prefix[: -len(suffix)] if suffix else after_prefix
+                for known_root, root_surface in roots.items():
+                    affinity = _root_affinity(root_candidate, known_root)
+                    score = affinity + len(prefix) + len(suffix)
+                    if prefix or suffix:
+                        score += 32
+                    if best is None or score > best[0]:
+                        best = (
+                            score,
+                            root_surface,
+                            (prefix,) if prefix else (),
+                            (suffix,) if suffix else (),
+                        )
+
+    if best is None:
+        raise KeyError(f"Token '{token}' cannot be decomposed against mounted morphology.")
+    _, root_surface, applied_prefixes, applied_suffixes = best
+    return root_surface, applied_prefixes, applied_suffixes
+
+
+def _compose_delta(root_versor: np.ndarray, prefixes: tuple[str, ...], suffixes: tuple[str, ...]) -> tuple[np.ndarray, tuple[str, ...]]:
+    versor = np.asarray(root_versor, dtype=np.float32).copy()
+    operators: list[str] = []
+
+    for idx, prefix in enumerate(prefixes):
+        versor = geometric_product(
+            versor,
+            _feature_rotor(f"{idx}:{prefix.lower()}", "morph:prefix", 0.03 / (idx + 1)),
+        )
+        operators.append(f"prefix:{prefix}")
+
+    for idx, suffix in enumerate(suffixes):
+        versor = geometric_product(
+            versor,
+            _feature_rotor(f"{idx}:{suffix.lower()}", "morph:suffix", 0.02 / (idx + 1)),
+        )
+        operators.append(f"suffix:{suffix}")
+
+    return versor.astype(np.float32, copy=False), tuple(operators)
+
+
+def _ground_unknown_token(token: str, vocab) -> np.ndarray:
+    morphology_entries = (
+        vocab.morphology_entries()
+        if hasattr(vocab, "morphology_entries")
+        else ()
+    )
+    if not morphology_entries or not hasattr(vocab, "insert_transient"):
+        raise KeyError(f"Word '{token}' not in vocabulary.")
+
+    root_used, prefixes, suffixes = _best_decomposition(token, vocab, morphology_entries)
+    root_versor = vocab.get_versor(root_used)
+    versor, operators_applied = _compose_delta(root_versor, prefixes, suffixes)
+    condition = versor_condition(versor)
+    if condition > 1e-6:
+        raise RuntimeError(
+            f"Unknown-token construction for '{token}' produced non-versor: "
+            f"condition={condition:.2e}."
+        )
+
+    grounded = _GroundedUnknown(
+        token=token,
+        root_used=root_used,
+        versor=versor,
+        operators_applied=operators_applied,
+        condition=condition,
+    )
+    vocab.insert_transient(grounded.token, grounded.versor)
+    if hasattr(vocab, "record_unknown_token"):
+        vocab.record_unknown_token(
+            grounded.token,
+            grounded.root_used,
+            grounded.operators_applied,
+            grounded.condition,
+        )
+    return grounded.versor.copy()
+
+
+def _lookup_or_ground(token: str, vocab) -> np.ndarray:
+    try:
+        return vocab.get_versor(token)
+    except KeyError:
+        return _ground_unknown_token(token, vocab)
 
 
 def inject(tokens: list, vocab) -> FieldState:
@@ -39,7 +234,7 @@ def inject(tokens: list, vocab) -> FieldState:
     3. normalize_to_versor() — the single allowed gate normalization call
     4. Assert versor condition before returning
     """
-    word_versors = [vocab.get_versor(t) for t in tokens]
+    word_versors = [_lookup_or_ground(t, vocab) for t in tokens]
     H = holonomy_encode(word_versors)
     F = normalize_to_versor(H)
 
diff --git a/tests/test_unknown_token_ingest.py b/tests/test_unknown_token_ingest.py
new file mode 100644
index 00000000..7b1e3e09
--- /dev/null
+++ b/tests/test_unknown_token_ingest.py
@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import numpy as np
+
+from algebra.backend import cga_inner
+from algebra.versor import unitize_versor, versor_condition
+from ingest.gate import inject
+from language_packs.compiler import load_mounted_packs
+from session.context import SessionContext
+
+
+def _random_versor(seed: int) -> np.ndarray:
+    rng = np.random.default_rng(seed)
+    vec = rng.standard_normal(32).astype(np.float32)
+    return unitize_versor(vec)
+
+
+def test_unknown_token_is_grounded_as_valid_transient_versor() -> None:
+    vocab = load_mounted_packs(("he_logos_micro_v1",))
+    token = "דברית"
+
+    state = inject([token], vocab)
+    constructed = vocab.get_versor(token)
+    root = vocab.get_versor("דבר")
+    random = _random_versor(41)
+
+    assert versor_condition(constructed) < 1e-6
+    assert versor_condition(state.F) < 1e-6
+    assert vocab.is_transient(token)
+    token_idx = vocab.index_of(token)
+    assert vocab.nearest(constructed, exclude_indices=set(range(token_idx)))[0] == token
+    assert cga_inner(constructed, root) > cga_inner(constructed, random)
+
+    log = vocab.unknown_token_log
+    assert log == (
+        {
+            "token": token,
+            "root_used": "דבר",
+            "operators_applied": ("suffix:ית",),
+            "versor_condition_score": log[0]["versor_condition_score"],
+        },
+    )
+    assert log[0]["versor_condition_score"] < 1e-6
+
+
+def test_unknown_token_session_turn_evolves_field() -> None:
+    vocab = load_mounted_packs(("he_logos_micro_v1", "grc_logos_micro_v1"))
+    session = SessionContext(vocab=vocab)
+    token = "דברית"
+
+    first = session.ingest([token])
+    response = session.respond(max_tokens=3)
+    second = session.ingest([token])
+
+    assert vocab.is_transient(token)
+    assert versor_condition(first.F) < 1e-6
+    assert versor_condition(response.final_state.F) < 1e-6
+    assert versor_condition(second.F) < 1e-6
+    assert not np.array_equal(first.F, second.F)
diff --git a/vocab/manifold.py b/vocab/manifold.py
index e9d97123..28799251 100644
--- a/vocab/manifold.py
+++ b/vocab/manifold.py
@@ -38,6 +38,8 @@ class VocabManifold:
         self._words: list[str] = []
         self._versors: list[np.ndarray] = []  # each shape (32,), grade-normed to ±1
         self._morphology_by_word: dict[str, MorphologyEntry] = {}
+        self._transient_words: set[str] = set()
+        self._unknown_token_log: list[dict[str, object]] = []
 
     def add(self, word: str, versor: np.ndarray, morphology: MorphologyEntry | None = None) -> None:
         """
@@ -69,6 +71,53 @@ class VocabManifold:
         if morphology is not None:
             self._morphology_by_word[word] = morphology
 
+    def insert_transient(self, word: str, versor: np.ndarray) -> None:
+        """
+        Register a session-local ad hoc word-versor pair.
+
+        Transient entries live only in this manifold instance. They use the
+        same storage as compiled entries so nearest() and get_versor() remain
+        exact manifold operations, but no language pack persistence path ever
+        sees them.
+        """
+        if word in self._words and word not in self._transient_words:
+            raise ValueError(f"Word '{word}' already exists as a compiled manifold entry.")
+        if word in self._transient_words:
+            self.update(word, versor)
+            return
+        self.add(word, versor)
+        self._transient_words.add(word)
+
+    def is_transient(self, word: str) -> bool:
+        """Return True when word was inserted as a session-local transient."""
+        return word in self._transient_words
+
+    def morphology_entries(self) -> tuple[MorphologyEntry, ...]:
+        """Return morphology entries carried by compiled manifold surfaces."""
+        return tuple(self._morphology_by_word.values())
+
+    def record_unknown_token(
+        self,
+        token: str,
+        root_used: str,
+        operators_applied: tuple[str, ...],
+        versor_condition_score: float,
+    ) -> None:
+        """Append an audit record for gate-constructed unknown-token grounding."""
+        self._unknown_token_log.append(
+            {
+                "token": token,
+                "root_used": root_used,
+                "operators_applied": operators_applied,
+                "versor_condition_score": versor_condition_score,
+            }
+        )
+
+    @property
+    def unknown_token_log(self) -> tuple[dict[str, object], ...]:
+        """Session-local audit trail of ad hoc unknown-token constructions."""
+        return tuple(dict(entry) for entry in self._unknown_token_log)
+
     def update(self, word: str, versor: np.ndarray) -> None:
         """
         Replace the versor for an existing word in-place.