core-labs/core
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Add runtime output-language policy

Browse source 
Add RuntimeConfig with English default output policy, wire output language through runtime/frame selection/generation/CLI, preserve language metadata in mounted manifolds, and add runtime/CLI policy tests.
This commit is contained in:
Shay 2026-05-13 21:29:43 -07:00 committed by GitHub
parent 09c3664773
commit 30757ccc63
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 287 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
chat/runtime.py
View file

@ -7,6 +7,7 @@ from collections.abc import Sequence
import numpy as np
from algebra.versor import versor_condition
from core.config import DEFAULT_CONFIG, RuntimeConfig
from generate.dialogue import DialogueRole, classify_dialogue_blade, propose_dialogue
from generate.proposition import FrameRegistry, Proposition, propose
from generate.stream import generate
@ -15,7 +16,6 @@ from persona.motor import PersonaMotor
from session.context import SessionContext
_TOKEN_RE = re.compile(r"\w+", re.UNICODE)
_DEFAULT_PACKS = ("en_minimal_v1", "he_logos_micro_v1", "grc_logos_micro_v1")
_SEED_ALIASES = {
"logos": "λόγος",
"dabar": "דבר",
@ -33,16 +33,34 @@ class ChatResponse:
proposition: Proposition
dialogue_role: DialogueRole
versor_condition: float
output_language: str
frame_pack: str
walk_surface: str
class ChatRuntime:
def __init__(
self,
pack_id: str | Sequence[str] = _DEFAULT_PACKS,
pack_id: str | Sequence[str] | None = None,
*,
frame_pack: str | None = None,
config: RuntimeConfig = DEFAULT_CONFIG,
) -> None:
pack_ids = (pack_id,) if isinstance(pack_id, str) else tuple(pack_id)
if pack_id is not None or frame_pack is not None:
pack_ids = (pack_id,) if isinstance(pack_id, str) else tuple(pack_id or config.input_packs)
resolved_config = RuntimeConfig(
input_packs=pack_ids,
output_language=config.output_language,
frame_pack=frame_pack or config.frame_pack,
max_tokens=config.max_tokens,
allow_cross_language_recall=config.allow_cross_language_recall,
allow_cross_language_generation=config.allow_cross_language_generation,
)
else:
resolved_config = config
pack_ids = tuple(config.input_packs)
self.config = resolved_config
manifests = []
manifolds = []
entries = []
@ -56,7 +74,7 @@ class ChatRuntime:
self._manifests = tuple(manifests)
self._context = SessionContext(manifold, persona=PersonaMotor.identity())
self._frame_registry = FrameRegistry.from_pack(
frame_pack or self._default_frame_pack(pack_ids),
resolved_config.frame_pack,
self._context.vocab,
)
self._surface_by_fold = {e.surface.casefold(): e.surface for e in entries}
@ -69,14 +87,6 @@ class ChatRuntime:
def session(self) -> SessionContext:
return self._context
@staticmethod
def _default_frame_pack(pack_ids: tuple[str, ...]) -> str:
if any(pack_id.startswith("grc_") for pack_id in pack_ids):
return "grc"
if any(pack_id.startswith("he_") for pack_id in pack_ids):
return "he"
return "en"
def _tokenize(self, text: str) -> list[str]:
tokens: list[str] = []
for match in _TOKEN_RE.finditer(text):
@ -121,7 +131,7 @@ class ChatRuntime:
return None
return blade
def chat(self, text: str, max_tokens: int = 32) -> ChatResponse:
def chat(self, text: str, max_tokens: int | None = None) -> ChatResponse:
tokens = self._tokenize(text)
filtered = self._apply_oov_policy(tokens)
if not filtered:
@ -129,7 +139,13 @@ class ChatRuntime:
field_state = self._context.ingest(filtered)
reference_blade = self._dialogue_reference()
base_proposition = propose(field_state, None, self._context.vocab, self._frame_registry)
base_proposition = propose(
field_state,
None,
self._context.vocab,
self._frame_registry,
output_lang=self.config.output_language,
)
dialogue_role = classify_dialogue_blade(
base_proposition.relation,
reference_blade,
@ -140,6 +156,7 @@ class ChatRuntime:
self._context.vocab,
self._frame_registry,
reference_blade,
output_lang=self.config.output_language,
)
self._context.record_dialogue(proposition)
@ -147,8 +164,11 @@ class ChatRuntime:
field_state,
self._context.vocab,
self._context.persona,
max_tokens=max_tokens,
max_tokens=self.config.max_tokens if max_tokens is None else max_tokens,
vault=self._context.vault,
recall_top_k=3 if self.config.allow_cross_language_recall else 0,
output_lang=self.config.output_language,
allow_cross_language_generation=self.config.allow_cross_language_generation,
)
self._context.state = result.final_state
self._context.vault.store(
@ -157,15 +177,18 @@ class ChatRuntime:
)
self._context.turn += 1
guarded = self._syntactic_guard(result.tokens)
surface = " ".join(guarded)
walk_surface = " ".join(guarded)
return ChatResponse(
surface=surface,
surface=proposition.surface,
proposition=proposition,
dialogue_role=dialogue_role,
versor_condition=versor_condition(result.final_state.F),
output_language=self.config.output_language,
frame_pack=self.config.frame_pack,
walk_surface=walk_surface,
)
def respond(self, text: str, max_tokens: int = 32) -> str:
def respond(self, text: str, max_tokens: int | None = None) -> str:
try:
return self.chat(text, max_tokens=max_tokens).surface
except ValueError:

84
core/cli.py
View file

@ -20,7 +20,7 @@ if str(_REPO_ROOT) not in sys.path:
DESCRIPTION = "CORE versor engine command suite."
EPILOG = "Examples:\n core chat\n core trace \"word beginning truth\"\n core trace --pack en_minimal_v1 --json \"word beginning truth\"\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core test tests/test_alignment_graph.py -q"
EPILOG = "Examples:\n core chat\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core test tests/test_alignment_graph.py -q"
def _run(*args: str, check: bool = False) -> int:
@ -42,9 +42,47 @@ def _print_runtime_import_hint(exc: BaseException) -> NoReturn:
)
def _runtime_config_from_args(args: argparse.Namespace):
from core.config import DEFAULT_CONFIG, RuntimeConfig
output_language = args.output_language
frame_pack = args.frame_pack or output_language
input_packs = tuple(args.pack) if getattr(args, "pack", None) else DEFAULT_CONFIG.input_packs
return RuntimeConfig(
input_packs=input_packs,
output_language=output_language,
frame_pack=frame_pack,
max_tokens=args.max_tokens,
allow_cross_language_recall=not args.no_cross_language_recall,
allow_cross_language_generation=args.allow_cross_language_generation,
)
def cmd_chat(args: argparse.Namespace) -> int:
"""Launch the readline REPL backed by ChatRuntime."""
return _run(sys.executable, "-m", "chat", *args.args)
"""Launch a readline REPL backed by ChatRuntime."""
try:
from chat.runtime import ChatRuntime
except Exception as exc: # pragma: no cover - exercised by CLI in broken envs
_print_runtime_import_hint(exc)
runtime = ChatRuntime(config=_runtime_config_from_args(args))
while True:
try:
text = input("> ").strip()
except EOFError:
print()
break
if text in {"quit", "exit"}:
break
if not text:
continue
try:
response = runtime.chat(text)
except (KeyError, ValueError) as exc:
print(f"[{exc}]", file=sys.stderr)
continue
print(response.surface)
return 0
def cmd_test(args: argparse.Namespace) -> int:
@ -75,15 +113,13 @@ def cmd_check(args: argparse.Namespace) -> int:
return _run(sys.executable, "-m", "ruff", "check", *targets)
def _runtime_for_trace(pack: list[str] | None):
def _runtime_for_trace(args: argparse.Namespace):
try:
from chat.runtime import ChatRuntime
except Exception as exc: # pragma: no cover - exercised by CLI in broken envs
_print_runtime_import_hint(exc)
pack_arg: str | tuple[str, ...]
pack_arg = tuple(pack) if pack else ("en_minimal_v1", "he_logos_micro_v1", "grc_logos_micro_v1")
try:
return ChatRuntime(pack_arg)
return ChatRuntime(config=_runtime_config_from_args(args))
except Exception as exc:
_die(
"failed to initialize ChatRuntime. Check mounted language packs with "
@ -100,6 +136,9 @@ def _trace_payload(text: str, resp: Any, runtime: Any) -> dict[str, Any]:
payload: dict[str, Any] = {
"input": text,
"surface": resp.surface,
"walk_surface": resp.walk_surface,
"output_language": resp.output_language,
"frame_pack": resp.frame_pack,
"dialogue_role": str(resp.dialogue_role),
"versor_condition": float(resp.versor_condition),
"proposition": {
@ -119,6 +158,9 @@ def _trace_payload(text: str, resp: Any, runtime: Any) -> dict[str, Any]:
def _print_trace(payload: dict[str, Any]) -> None:
print(f"input : {payload['input']}")
print(f"surface : {payload['surface']}")
print(f"walk_surface : {payload['walk_surface']}")
print(f"output_language: {payload['output_language']}")
print(f"frame_pack : {payload['frame_pack']}")
print(f"dialogue_role : {payload['dialogue_role']}")
print(f"versor_cond : {payload['versor_condition']:.2e}")
proposition = payload["proposition"]
@ -143,7 +185,7 @@ def cmd_trace(args: argparse.Namespace) -> int:
if not text:
_die("trace requires input text. Try: core trace \"word beginning truth\"")
runtime = _runtime_for_trace(args.pack)
runtime = _runtime_for_trace(args)
try:
response = runtime.chat(text, max_tokens=args.max_tokens)
except Exception as exc:
@ -165,7 +207,7 @@ def cmd_oov(args: argparse.Namespace) -> int:
except Exception as exc: # pragma: no cover - exercised by CLI in broken envs
_print_runtime_import_hint(exc)
runtime = ChatRuntime(tuple(args.pack) if args.pack else ("en_minimal_v1", "he_logos_micro_v1", "grc_logos_micro_v1"))
runtime = ChatRuntime(config=_runtime_config_from_args(args))
vocab = runtime.session.vocab
try:
@ -244,6 +286,23 @@ def cmd_doctor(args: argparse.Namespace) -> int:
return 0 if ok else 1
def _add_runtime_policy_args(parser: argparse.ArgumentParser) -> None:
parser.add_argument("--pack", action="append", help="language pack to mount; repeat for multiple packs")
parser.add_argument("--output-language", default="en", help="target output language code; default: en")
parser.add_argument("--frame-pack", help="frame pack to use; defaults to output language")
parser.add_argument("--max-tokens", type=int, default=32, help="maximum generated tokens; default: 32")
parser.add_argument(
"--allow-cross-language-generation",
action="store_true",
help="allow generated walk tokens from any mounted language",
)
parser.add_argument(
"--no-cross-language-recall",
action="store_true",
help="disable vault recall during generation",
)
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog="core",
@ -255,7 +314,7 @@ def build_parser() -> argparse.ArgumentParser:
subparsers = parser.add_subparsers(dest="command", metavar="command")
chat = subparsers.add_parser("chat", help="start the interactive chat REPL")
chat.add_argument("args", nargs=argparse.REMAINDER, help="arguments forwarded to python -m chat")
_add_runtime_policy_args(chat)
chat.set_defaults(func=cmd_chat)
test = subparsers.add_parser("test", help="run pytest with sane defaults")
@ -271,14 +330,13 @@ def build_parser() -> argparse.ArgumentParser:
help="trace one chat turn with field telemetry",
description="trace one chat turn with field telemetry",
)
trace.add_argument("--pack", action="append", help="language pack to mount; repeat for multiple packs")
trace.add_argument("--max-tokens", type=int, default=32, help="maximum generated tokens; default: 32")
_add_runtime_policy_args(trace)
trace.add_argument("--json", action="store_true", help="emit machine-readable JSON")
trace.add_argument("text", nargs=argparse.REMAINDER, help="input text to trace")
trace.set_defaults(func=cmd_trace)
oov = subparsers.add_parser("oov", help="ground or inspect one token")
oov.add_argument("--pack", action="append", help="language pack to mount; repeat for multiple packs")
_add_runtime_policy_args(oov)
oov.add_argument("token", help="token to inspect or ground")
oov.set_defaults(func=cmd_oov)

16
core/config.py Normal file
View file

@ -0,0 +1,16 @@
from __future__ import annotations
from dataclasses import dataclass
@dataclass(frozen=True, slots=True)
class RuntimeConfig:
input_packs: tuple[str, ...] = ("en_minimal_v1", "he_logos_micro_v1", "grc_logos_micro_v1")
output_language: str = "en"
frame_pack: str = "en"
max_tokens: int = 32
allow_cross_language_recall: bool = True
allow_cross_language_generation: bool = False
DEFAULT_CONFIG = RuntimeConfig()

5
generate/dialogue.py
View file

@ -94,13 +94,14 @@ def propose_dialogue(
vocab,
frame_registry: FrameRegistry,
reference_blade: np.ndarray | None = None,
output_lang: str | None = None,
) -> Proposition:
"""
Generate a proposition through a dialogue-role constrained frame choice.
"""
base = propose(field_state, None, vocab, frame_registry)
base = propose(field_state, None, vocab, frame_registry, output_lang=output_lang)
role = classify_dialogue_blade(base.relation, reference_blade)
frame = frame_registry.select_dialogue(base.relation, role)
role_registry = FrameRegistry((frame,))
proposition = propose(field_state, vault, vocab, role_registry)
proposition = propose(field_state, vault, vocab, role_registry, output_lang=output_lang)
return proposition

35
generate/proposition.py
View file

@ -128,7 +128,25 @@ class FrameRegistry:
return len(self._frames)
def propose(field_state: FieldState, vault, vocab, frame_registry: FrameRegistry) -> Proposition:
def _candidate_indices_for_language(vocab, output_lang: str | None) -> np.ndarray | None:
if output_lang is None:
return None
indices_for_language = getattr(vocab, "indices_for_language", None)
if indices_for_language is None:
return None
indices = indices_for_language(output_lang)
if len(indices) == 0:
raise ValueError(f"No proposition candidates for output language {output_lang!r}.")
return indices
def propose(
field_state: FieldState,
vault,
vocab,
frame_registry: FrameRegistry,
output_lang: str | None = None,
) -> Proposition:
"""
Generate one structured proposition from the live field.
@ -141,17 +159,20 @@ def propose(field_state: FieldState, vault, vocab, frame_registry: FrameRegistry
prompt = _prompt_versor(field_state)
relation = outer_product(prompt, field_state.F)
frame = frame_registry.select(relation)
candidate_indices = _candidate_indices_for_language(vocab, output_lang)
subject_word, subject_idx = _nearest_content_word(
vocab,
prompt,
exclude_indices=frozenset(),
preferred_pos=frozenset({"noun", "pronoun"}),
candidate_indices=candidate_indices,
)
predicate_word, predicate_idx = _nearest_content_word(
vocab,
field_state.F,
exclude_indices=frozenset({subject_idx}),
candidate_indices=candidate_indices,
)
object_word: str | None = None
@ -162,6 +183,7 @@ def propose(field_state: FieldState, vault, vocab, frame_registry: FrameRegistry
relation,
exclude_indices=frozenset({subject_idx, predicate_idx}),
preferred_pos=frozenset({"noun", "pronoun"}),
candidate_indices=candidate_indices,
)
object_versor = vocab.get_versor_at(object_idx)
@ -273,6 +295,7 @@ def _nearest_content_word(
query: np.ndarray,
exclude_indices: frozenset[int],
preferred_pos: frozenset[str] = frozenset(),
candidate_indices: np.ndarray | None = None,
) -> tuple[str, int]:
stop_indices = {
vocab.index_of(surface)
@ -281,13 +304,13 @@ def _nearest_content_word(
}
blocked = set(exclude_indices) | stop_indices
if preferred_pos:
selected = _nearest_by_pos(vocab, query, blocked, preferred_pos)
selected = _nearest_by_pos(vocab, query, blocked, preferred_pos, candidate_indices)
if selected is not None:
return selected
try:
return vocab.nearest(query, exclude_indices=blocked)
return vocab.nearest(query, exclude_indices=blocked, candidate_indices=candidate_indices)
except ValueError:
return vocab.nearest(query, exclude_indices=set(exclude_indices))
return vocab.nearest(query, exclude_indices=set(exclude_indices), candidate_indices=candidate_indices)
def _nearest_by_pos(
@ -295,10 +318,12 @@ def _nearest_by_pos(
query: np.ndarray,
blocked: set[int],
preferred_pos: frozenset[str],
candidate_indices: np.ndarray | None = None,
) -> tuple[str, int] | None:
best_score = -np.inf
best: tuple[str, int] | None = None
for idx in range(len(vocab)):
candidates = range(len(vocab)) if candidate_indices is None else [int(idx) for idx in candidate_indices]
for idx in candidates:
if idx in blocked:
continue
word = vocab.get_word_at(idx)

30
generate/stream.py
View file

@ -18,6 +18,8 @@ F is always on the manifold. nearest() is exact.
from __future__ import annotations
from collections import deque
import numpy as np
from field.state import FieldState
from field.propagate import propagate_step
from algebra.rotor import word_transition_rotor
@ -48,6 +50,7 @@ def _nearest_next(
current_node: int,
recent_nodes: tuple[int, ...] = (),
stop_nodes: frozenset[int] = frozenset(),
candidate_indices: np.ndarray | None = None,
) -> tuple[str, int]:
"""
Select the nearest vocabulary point while avoiding short loops.
@ -59,7 +62,7 @@ def _nearest_next(
informative neighbors are available.
"""
if len(vocab) <= 1:
return vocab.nearest(F_voiced)
return vocab.nearest(F_voiced, candidate_indices=candidate_indices)
recent = set(recent_nodes)
stop = set(stop_nodes)
@ -71,10 +74,15 @@ def _nearest_next(
)
for extra in fallback_orders:
try:
return vocab.nearest(F_voiced, exclude_idx=current_node, exclude_indices=extra)
return vocab.nearest(
F_voiced,
exclude_idx=current_node,
exclude_indices=extra,
candidate_indices=candidate_indices,
)
except ValueError:
continue
return vocab.nearest(F_voiced, exclude_idx=current_node)
return vocab.nearest(F_voiced, exclude_idx=current_node, candidate_indices=candidate_indices)
def _voiced_state(state: FieldState, persona) -> FieldState:
@ -111,6 +119,18 @@ def _recall_state(state: FieldState, vault, top_k: int) -> FieldState:
return current
def _candidate_indices_for_language(vocab, output_lang: str | None) -> np.ndarray | None:
if output_lang is None:
return None
indices_for_language = getattr(vocab, "indices_for_language", None)
if indices_for_language is None:
return None
indices = indices_for_language(output_lang)
if len(indices) == 0:
raise ValueError(f"No generation candidates for output language {output_lang!r}.")
return indices
def generate(
state: FieldState,
vocab,
@ -119,6 +139,8 @@ def generate(
record_trajectory: bool = False,
vault=None,
recall_top_k: int = 3,
output_lang: str | None = None,
allow_cross_language_generation: bool = True,
) -> GenerationResult:
"""
Generate a token sequence from an initial FieldState.
@ -140,6 +162,7 @@ def generate(
trajectory = [] if record_trajectory else None
current = state
recent_nodes = deque([state.node], maxlen=_RECENT_WINDOW)
candidate_indices = None if allow_cross_language_generation else _candidate_indices_for_language(vocab, output_lang)
stop_nodes = frozenset(
vocab.index_of(token)
for token in _STOP_TOKENS
@ -154,6 +177,7 @@ def generate(
current.node,
recent_nodes=tuple(recent_nodes),
stop_nodes=stop_nodes,
candidate_indices=candidate_indices,
)
tokens.append(_articulate(vocab, word))

5
language_packs/compiler.py
View file

@ -268,7 +268,7 @@ def compile_entries_to_manifold(entries: list[LexicalEntry], morphology_registry
for entry in entries:
morphology = _resolved_morphology(entry, morphology_registry)
versor = _entry_to_coordinate(entry, morphology)
manifold.add(entry.surface, versor, morphology=morphology)
manifold.add(entry.surface, versor, morphology=morphology, language=entry.language)
entry_id_to_surface[entry.entry_id] = entry.surface
if morphology_registry is not None:
@ -394,12 +394,13 @@ def load_mounted_packs(pack_ids: tuple[str, ...] | list[str]) -> VocabManifold:
surface = manifold.get_word_at(idx)
if surface in seen:
continue
entry = entry_by_surface.get(surface)
mounted.add(
surface,
manifold.get_versor_at(idx),
morphology=manifold.morphology_for_word(surface),
language=None if entry is None else entry.language,
)
entry = entry_by_surface.get(surface)
if entry is not None and entry.semantic_domains:
primary_groups.setdefault(entry.semantic_domains[0].lower(), []).append(
(entry.language, surface)

6
tests/test_cli.py
View file

@ -21,6 +21,8 @@ def test_trace_help_exits_without_runtime_import(capsys: pytest.CaptureFixture[s
out = capsys.readouterr().out
assert "trace one chat turn" in out
assert "--pack" in out
assert "--output-language" in out
assert "--frame-pack" in out
assert "--json" in out
@ -51,6 +53,8 @@ def test_trace_formats_real_runtime_payload(capsys: pytest.CaptureFixture[str])
assert main(["trace", "--pack", "en_minimal_v1", "word", "beginning", "truth"]) == 0
out = capsys.readouterr().out
assert "input : word beginning truth" in out
assert "output_language: en" in out
assert "frame_pack : en" in out
assert "proposition" in out
assert "subject" in out
assert "predicate" in out
@ -60,6 +64,8 @@ def test_trace_json_formats_real_runtime_payload(capsys: pytest.CaptureFixture[s
assert main(["trace", "--pack", "en_minimal_v1", "--json", "word", "beginning", "truth"]) == 0
out = capsys.readouterr().out
assert '"input": "word beginning truth"' in out
assert '"output_language": "en"' in out
assert '"frame_pack": "en"' in out
assert '"proposition"' in out
assert '"subject"' in out
assert '"predicate"' in out

55
tests/test_runtime_config.py Normal file
View file

@ -0,0 +1,55 @@
from __future__ import annotations
import re
from chat.runtime import ChatRuntime
from core.config import RuntimeConfig
_GREEK_RE = re.compile(r"[\u0370-\u03ff\u1f00-\u1fff]")
_HEBREW_RE = re.compile(r"[\u0590-\u05ff]")
def _is_english_surface(text: str) -> bool:
return not _GREEK_RE.search(text) and not _HEBREW_RE.search(text)
def test_default_runtime_emits_english_surface() -> None:
runtime = ChatRuntime()
response = runtime.chat("word beginning truth")
assert response.output_language == "en"
assert response.frame_pack == "en"
assert _is_english_surface(response.surface)
def test_trilanguage_mounted_runtime_still_emits_english_surface_by_default() -> None:
runtime = ChatRuntime(
config=RuntimeConfig(
input_packs=("en_minimal_v1", "he_logos_micro_v1", "grc_logos_micro_v1"),
output_language="en",
frame_pack="en",
)
)
response = runtime.chat("word beginning truth")
assert _is_english_surface(response.surface)
assert response.proposition.frame_id.startswith("en:")
def test_greek_output_language_emits_greek_surface() -> None:
runtime = ChatRuntime(
config=RuntimeConfig(
input_packs=("en_minimal_v1", "he_logos_micro_v1", "grc_logos_micro_v1"),
output_language="grc",
frame_pack="grc",
)
)
response = runtime.chat("logos arche aletheia")
assert response.output_language == "grc"
assert response.frame_pack == "grc"
assert _GREEK_RE.search(response.surface)
assert response.proposition.frame_id.startswith(("grc:", "el:"))
def test_chat_response_surface_uses_proposition_when_relation_nonzero() -> None:
runtime = ChatRuntime(config=RuntimeConfig(output_language="en", frame_pack="en"))
response = runtime.chat("word beginning truth")
assert response.surface == response.proposition.surface

38
vocab/manifold.py
View file

@ -38,10 +38,17 @@ class VocabManifold:
self._words: list[str] = []
self._versors: list[np.ndarray] = [] # each shape (32,), grade-normed to ±1
self._morphology_by_word: dict[str, MorphologyEntry] = {}
self._language_by_word: dict[str, str] = {}
self._transient_words: set[str] = set()
self._unknown_token_log: list[dict[str, object]] = []
def add(self, word: str, versor: np.ndarray, morphology: MorphologyEntry | None = None) -> None:
def add(
self,
word: str,
versor: np.ndarray,
morphology: MorphologyEntry | None = None,
language: str | None = None,
) -> None:
"""
Register a word-versor pair.
@ -68,6 +75,9 @@ class VocabManifold:
)
self._words.append(word)
self._versors.append(v)
resolved_language = language or (morphology.language if morphology is not None else None)
if resolved_language:
self._language_by_word[word] = resolved_language
if morphology is not None:
self._morphology_by_word[word] = morphology
@ -175,11 +185,28 @@ class VocabManifold:
"""Return structured morphology for a stored surface, if the pack provided it."""
return self._morphology_by_word.get(word)
def language_for_word(self, word: str) -> str | None:
"""Return the language code for a stored surface, if known."""
morphology = self._morphology_by_word.get(word)
if morphology is not None:
return morphology.language
return self._language_by_word.get(word)
def indices_for_language(self, lang: str) -> np.ndarray:
"""Return manifold indices whose language matches lang."""
matches = [
idx
for idx, word in enumerate(self._words)
if self.language_for_word(word) == lang
]
return np.asarray(matches, dtype=np.int64)
def nearest(
self,
F: np.ndarray,
exclude_idx: int = -1,
exclude_indices: set[int] | frozenset[int] | None = None,
candidate_indices: np.ndarray | list[int] | tuple[int, ...] | None = None,
) -> tuple[str, int]:
"""
Find the word whose versor is closest to F by CGA inner product.
@ -192,12 +219,17 @@ class VocabManifold:
if exclude_idx >= 0:
blocked.add(exclude_idx)
if candidate_indices is None:
candidates = range(len(self._versors))
else:
candidates = [int(idx) for idx in candidate_indices]
best_score = -np.inf
best_idx = -1
for i, v in enumerate(self._versors):
for i in candidates:
if i in blocked:
continue
score = cga_inner(F, v)
score = cga_inner(F, self._versors[i])
if score > best_score:
best_score = score
best_idx = i