From 2a9285817df1521ee1673f1125319ded33a79783 Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 6 Jun 2026 05:46:01 -0700 Subject: [PATCH] fix: derive oov grounding from token content --- ...eterminism-morphology-affinity-collapse.md | 109 ++++++++++++++++++ ingest/gate.py | 96 +++++++++++++-- tests/test_oov_grounding_cache.py | 41 +++++++ tests/test_unknown_token_ingest.py | 12 +- 4 files changed, 241 insertions(+), 17 deletions(-) create mode 100644 docs/analysis/oov-grounding-determinism-morphology-affinity-collapse.md diff --git a/docs/analysis/oov-grounding-determinism-morphology-affinity-collapse.md b/docs/analysis/oov-grounding-determinism-morphology-affinity-collapse.md new file mode 100644 index 00000000..00bdb9f0 --- /dev/null +++ b/docs/analysis/oov-grounding-determinism-morphology-affinity-collapse.md @@ -0,0 +1,109 @@ +# OOV Grounding Determinism: Morphology-Affinity Collapse + +## Map + +Three candidate explanations were checked in the OOV grounding path: + +- Unseeded random synthesis: not found. `ingest/gate.py` did not call `random` or + `np.random` for unknown-token construction. +- Python hash / hash-seed ordering: not found on the active construction path. + The compiler feature rotors and the new fix use SHA-256, not `hash()`. +- Transient index or vocabulary process state: not the failing source. Transient + insertion happens after construction; it did not feed the failing field bytes. + +The exact failing source was `_best_decomposition()` in `ingest/gate.py`. When a +token had no exact prefix/root/suffix decomposition, the function fell back to a +root-affinity scan over mounted morphology. `_ground_unknown_token()` then used +that selected root as the whole OOV point whenever no prefix/suffix operators +were applied. The OOV token's byte content did not enter the versor at all in +that generic path. + +Observed RED repro: + +- `probe_ingest([""]).F.tobytes()` was already stable across fresh contexts. +- `probe_ingest(["xyzzy_unknown_token_12345"]).F.tobytes()` and + `probe_ingest(["zzq-no-morph-019"]).F.tobytes()` were byte-identical because + both were assigned the same affinity root, `ἀποκρίνομαι`, with no operators. + +## Build + +The fix lives only at the sanctioned ingest boundary, `ingest/gate.py`. + +- Exact morphology still uses the known root when the OOV token has a real + prefix or suffix decomposition. +- Empty-prefix/empty-suffix affinity fallback is no longer treated as structural + grounding. Generic OOV starts from the identity versor instead of inheriting an + arbitrary morphology root. +- Every OOV transient receives a token-byte delta: + `sha256("oov:token:v1" || token_utf8)` selects three small negative-bivector + Spin rotors and records a `token:sha256:` audit operator. +- Morphology prefix/suffix deltas in the OOV path now also use negative-bivector + Spin rotors rather than the compiler's feature rotors. + +No realization code and no vault code were touched. + +## Justify + +The corrected intrinsic space is the token's stable byte identity composed with +any real morphology decomposition. The previous fallback projected unknown +tokens into a borrowed root space; that collapsed unrelated symbols and made the +generic OOV point a function of mounted vocabulary shape rather than token +content. + +Closure is preserved by construction: each new delta is `cos(theta) + B +sin(theta)` where `B` is one of `(6, 7, 8, 10, 11, 13)`, the negative +bivectors in `Cl(4,1)`. These are Spin factors, so composing them with a closed +root or with the identity remains on the versor manifold. The OOV constructor +checks `versor_condition(versor) < 1e-6` and raises if construction violates the +contract; it does not normalize, unitize, grade-project, or repair the transient +after the fact. + +The field-level `inject()` holonomy boundary remains unchanged and continues to +own prompt-field closure. + +## Verification + +Initial RED: + +```bash +uv run python -m pytest tests/test_oov_grounding_cache.py::test_generic_oov_probe_is_byte_stable_across_contexts_and_restore -q +# FAILED: two distinct generic OOV tokens produced identical field bytes +``` + +Targeted GREEN after the fix: + +```bash +uv run python -m pytest tests/test_unknown_token_ingest.py tests/test_oov_grounding_cache.py -q +# 8 passed +``` + +Final verification: + +```bash +uv run python -m pytest tests/test_unknown_token_ingest.py tests/test_oov_grounding_cache.py -q +# 8 passed in 3.26s + +uv run python -m pytest tests/test_unknown_token_ingest.py tests/test_oov_grounding_cache.py tests/test_oov_pipeline.py tests/test_oov_surface.py tests/test_pack_grounded_unknown.py tests/test_partial_surface.py tests/test_cold_start_grounding_lane.py tests/test_language_pack_runtime.py tests/test_language_pack_cache.py tests/test_language_pack_load_safety.py -q +# 142 passed in 75.09s + +uv run python -m core.cli test --suite smoke -q +# 90 passed in 114.00s + +uv run python -m core.cli test --suite cognition -q +# 121 passed, 1 skipped in 55.84s + +uv run python -m core.cli eval cognition +# intent_accuracy: 100.0% +# term_capture_rate: 100.0% +# surface_groundedness: 100.0% +# versor_closure_rate: 100.0% + +uv run python -m core.cli eval cognition --json +# all 13 case records had intent_correct=true, surface_contains_pass=true, +# and versor_closure=true; this lane does not emit a separate wrong counter. +``` + +Wrinkle: the repository declares a `core` console script in `pyproject.toml`, but +`uv sync` reports that entry points are skipped because the project is not +packaged in this checkout. The exact `uv run core ...` form therefore fails to +spawn `core`; the verified equivalent is `uv run python -m core.cli ...`. diff --git a/ingest/gate.py b/ingest/gate.py index 5043384e..9f5d8d33 100644 --- a/ingest/gate.py +++ b/ingest/gate.py @@ -24,6 +24,7 @@ Contract: Output: FieldState with F satisfying versor_condition(F) < 1e-6 """ +import hashlib from dataclasses import dataclass import numpy as np @@ -35,7 +36,6 @@ from core.physics.valence import ValenceBundle from algebra.holonomy import holonomy_encode from field.state import FieldState from language_packs.schema import MorphologyEntry -from language_packs.compiler import _feature_rotor @dataclass(frozen=True, slots=True) @@ -57,6 +57,10 @@ class _MorphologyIndex: _MORPH_INDEX_CACHE: dict[int, _MorphologyIndex] = {} _DECOMPOSITION_CACHE: dict[tuple[int, str], tuple[str, tuple[str, ...], tuple[str, ...]]] = {} _DECOMPOSITION_CACHE_MAX = 4096 +_SPIN_BIVECTORS: tuple[int, ...] = (6, 7, 8, 10, 11, 13) +_OOV_TOKEN_DELTA_COUNT = 3 +_OOV_TOKEN_MIN_ANGLE = 0.004 +_OOV_TOKEN_ANGLE_SPAN = 0.012 def _compact_root(root: str) -> str: @@ -128,6 +132,42 @@ def _root_affinity(candidate: str, root: str) -> int: return (common_prefix * 8) + (shared * 2) - length_penalty +def _stable_digest(name: str, salt: str) -> bytes: + return hashlib.sha256( + salt.encode("utf-8") + b"\0" + name.encode("utf-8", "surrogatepass") + ).digest() + + +def _spin_feature_rotor(name: str, salt: str, weight: float) -> np.ndarray: + """Return a true Spin rotor over a negative bivector plane.""" + digest = _stable_digest(name, salt) + component = _SPIN_BIVECTORS[int.from_bytes(digest[:2], "big") % len(_SPIN_BIVECTORS)] + sign = 1.0 if digest[2] >= 128 else -1.0 + theta = sign * float(weight) + rotor = np.zeros(32, dtype=np.float64) + rotor[0] = np.cos(theta) + rotor[component] = np.sin(theta) + return rotor + + +def _token_spin_delta(token: str) -> tuple[tuple[np.ndarray, ...], tuple[str, ...]]: + digest = _stable_digest(token, "oov:token:v1") + rotors: list[np.ndarray] = [] + for idx in range(_OOV_TOKEN_DELTA_COUNT): + component = _SPIN_BIVECTORS[digest[idx] % len(_SPIN_BIVECTORS)] + sign = 1.0 if digest[_OOV_TOKEN_DELTA_COUNT + idx] >= 128 else -1.0 + unit = int.from_bytes( + digest[6 + (idx * 2): 8 + (idx * 2)], + "big", + ) / 65535.0 + theta = sign * (_OOV_TOKEN_MIN_ANGLE + unit * _OOV_TOKEN_ANGLE_SPAN) + rotor = np.zeros(32, dtype=np.float64) + rotor[0] = np.cos(theta) + rotor[component] = np.sin(theta) + rotors.append(rotor) + return tuple(rotors), (f"token:sha256:{digest.hex()[:16]}",) + + def _best_decomposition( token: str, vocab, @@ -202,24 +242,38 @@ def _best_decomposition( def _compose_delta(root_versor: np.ndarray, prefixes: tuple[str, ...], suffixes: tuple[str, ...]) -> tuple[np.ndarray, tuple[str, ...]]: - versor = np.asarray(root_versor, dtype=np.float32).copy() + versor = np.asarray(root_versor, dtype=np.float64).copy() operators: list[str] = [] for idx, prefix in enumerate(prefixes): versor = geometric_product( versor, - _feature_rotor(f"{idx}:{prefix.lower()}", "morph:prefix", 0.03 / (idx + 1)), + _spin_feature_rotor(f"{idx}:{prefix.lower()}", "morph:prefix", 0.03 / (idx + 1)), ) operators.append(f"prefix:{prefix}") for idx, suffix in enumerate(suffixes): versor = geometric_product( versor, - _feature_rotor(f"{idx}:{suffix.lower()}", "morph:suffix", 0.02 / (idx + 1)), + _spin_feature_rotor(f"{idx}:{suffix.lower()}", "morph:suffix", 0.02 / (idx + 1)), ) operators.append(f"suffix:{suffix}") - return versor.astype(np.float32, copy=False), tuple(operators) + return versor, tuple(operators) + + +def _compose_token_delta(versor: np.ndarray, token: str) -> tuple[np.ndarray, tuple[str, ...]]: + composed = np.asarray(versor, dtype=np.float64).copy() + token_rotors, operators = _token_spin_delta(token) + for rotor in token_rotors: + composed = geometric_product(composed, rotor) + return composed, operators + + +def _identity_versor() -> np.ndarray: + versor = np.zeros(32, dtype=np.float64) + versor[0] = 1.0 + return versor def _ground_unknown_token(token: str, vocab) -> np.ndarray: @@ -228,13 +282,37 @@ def _ground_unknown_token(token: str, vocab) -> np.ndarray: if hasattr(vocab, "morphology_entries") else () ) - if not morphology_entries or not hasattr(vocab, "insert_transient"): + if not hasattr(vocab, "insert_transient"): raise KeyError(f"Word '{token}' not in vocabulary.") - root_used, prefixes, suffixes = _best_decomposition(token, vocab, morphology_entries) - root_versor = vocab.get_versor(root_used) + root_used = "" + prefixes: tuple[str, ...] = () + suffixes: tuple[str, ...] = () + root_versor = _identity_versor() + if morphology_entries: + try: + candidate_root, candidate_prefixes, candidate_suffixes = _best_decomposition( + token, + vocab, + morphology_entries, + ) + except KeyError: + pass + else: + # Empty-prefix/empty-suffix fallback is only an affinity guess over + # mounted morphology. It is not a decomposition of the OOV token, so + # the generic token must be grounded from its own bytes instead of + # inheriting an arbitrary root point. + if candidate_prefixes or candidate_suffixes: + root_used = candidate_root + prefixes = candidate_prefixes + suffixes = candidate_suffixes + root_versor = vocab.get_versor(root_used) + versor, operators_applied = _compose_delta(root_versor, prefixes, suffixes) - versor = normalize_to_versor(versor) + versor, token_operators = _compose_token_delta(versor, token) + operators_applied = operators_applied + token_operators + versor = versor.astype(np.float32, copy=False) condition = versor_condition(versor) if condition > 1e-6: raise RuntimeError( diff --git a/tests/test_oov_grounding_cache.py b/tests/test_oov_grounding_cache.py index 69b98ec9..6be70626 100644 --- a/tests/test_oov_grounding_cache.py +++ b/tests/test_oov_grounding_cache.py @@ -5,9 +5,12 @@ import numpy as np import pytest from algebra.versor import versor_condition +from core.config import DEFAULT_CONFIG from ingest import gate from ingest.gate import inject from language_packs.compiler import load_mounted_packs +from persona.motor import PersonaMotor +from session.context import SessionContext def test_oov_grounding_repeated_token_is_deterministic() -> None: @@ -25,6 +28,44 @@ def test_oov_grounding_repeated_token_is_deterministic() -> None: np.testing.assert_allclose(state_a.F, state_b.F, atol=1e-6) +def test_generic_oov_probe_is_byte_stable_across_contexts_and_restore() -> None: + token = "" + collision_token_a = "xyzzy_unknown_token_12345" + collision_token_b = "zzq-no-morph-019" + persona = PersonaMotor.identity() + vocab_a = load_mounted_packs(DEFAULT_CONFIG.input_packs) + vocab_b = load_mounted_packs(DEFAULT_CONFIG.input_packs) + vocab_c = load_mounted_packs(DEFAULT_CONFIG.input_packs) + vocab_d = load_mounted_packs(DEFAULT_CONFIG.input_packs) + + ctx_a = SessionContext(vocab=vocab_a, persona=persona) + ctx_b = SessionContext(vocab=vocab_b, persona=persona) + ctx_c = SessionContext(vocab=vocab_c, persona=persona) + ctx_d = SessionContext(vocab=vocab_d, persona=persona) + + field_a = ctx_a.probe_ingest([token]).F + field_b = ctx_b.probe_ingest([token]).F + field_c = ctx_c.probe_ingest([collision_token_a]).F + field_d = ctx_d.probe_ingest([collision_token_b]).F + + assert field_a.tobytes() == field_b.tobytes() + assert field_c.tobytes() != field_d.tobytes() + assert versor_condition(field_a) < 1e-6 + assert versor_condition(field_b) < 1e-6 + assert versor_condition(field_c) < 1e-6 + assert versor_condition(field_d) < 1e-6 + + restored = SessionContext( + vocab=load_mounted_packs(DEFAULT_CONFIG.input_packs), + persona=persona, + ) + restored.restore(ctx_a.snapshot()) + restored_field = restored.probe_ingest([token]).F + + assert restored_field.tobytes() == field_a.tobytes() + assert versor_condition(restored_field) < 1e-6 + + def test_oov_cache_does_not_skip_unknown_token_audit() -> None: vocab = load_mounted_packs(("he_logos_micro_v1",)) diff --git a/tests/test_unknown_token_ingest.py b/tests/test_unknown_token_ingest.py index 7b1e3e09..90cf2782 100644 --- a/tests/test_unknown_token_ingest.py +++ b/tests/test_unknown_token_ingest.py @@ -32,14 +32,10 @@ def test_unknown_token_is_grounded_as_valid_transient_versor() -> None: assert cga_inner(constructed, root) > cga_inner(constructed, random) log = vocab.unknown_token_log - assert log == ( - { - "token": token, - "root_used": "דבר", - "operators_applied": ("suffix:ית",), - "versor_condition_score": log[0]["versor_condition_score"], - }, - ) + assert log[0]["token"] == token + assert log[0]["root_used"] == "דבר" + assert log[0]["operators_applied"][0] == "suffix:ית" + assert log[0]["operators_applied"][1].startswith("token:sha256:") assert log[0]["versor_condition_score"] < 1e-6