diff --git a/chat/__init__.py b/chat/__init__.py new file mode 100644 index 00000000..a400e87f --- /dev/null +++ b/chat/__init__.py @@ -0,0 +1,3 @@ +from .runtime import ChatRuntime + +__all__ = ["ChatRuntime"] diff --git a/chat/runtime.py b/chat/runtime.py new file mode 100644 index 00000000..cc603400 --- /dev/null +++ b/chat/runtime.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from language_packs import OOVPolicy, load_pack, load_pack_entries +from persona.motor import PersonaMotor +from field.state import FieldState +from session.context import SessionContext + + +class ChatRuntime: + def __init__(self, pack_id: str = "en_minimal_v1") -> None: + manifest, manifold = load_pack(pack_id) + self._manifest = manifest + self._context = SessionContext(manifold, persona=PersonaMotor.identity()) + self._index_by_surface = {w: i for i, w in enumerate(self._context.vocab._words)} + self._pos_by_surface = { + e.surface: (e.pos or e.part_of_speech or "X") for e in load_pack_entries(pack_id) + } + + def _apply_oov_policy(self, tokens: list[str]) -> list[str]: + kept: list[str] = [] + for token in tokens: + try: + self._context.vocab.get_versor(token) + kept.append(token) + except KeyError: + if self._manifest.oov_policy is OOVPolicy.FAIL_CLOSED: + raise + if self._manifest.oov_policy is OOVPolicy.PROPOSE_VOCAB_EXPANSION: + raise KeyError(f"OOV token requires vocab proposal: {token}") + return kept + + def _syntactic_guard(self, tokens: tuple[str, ...]) -> list[str]: + out: list[str] = [] + prev_pos: str | None = None + for token in tokens: + pos = self._pos_by_surface.get(token, "X") + if pos == prev_pos: + continue + out.append(token) + prev_pos = pos + return out + + def respond(self, text: str, max_tokens: int = 32) -> str: + tokens = [t.strip() for t in text.split() if t.strip()] + filtered = self._apply_oov_policy(tokens) + if not filtered: + return "" + self._context.ingest(filtered) + node_idx = self._index_by_surface.get(filtered[0], 0) + self._context.state = FieldState(F=self._context.state.F, node=node_idx, step=self._context.state.step) + result = self._context.respond(max_tokens=max_tokens) + guarded = self._syntactic_guard(result.tokens) + return " ".join(guarded) diff --git a/language_packs/__init__.py b/language_packs/__init__.py index 657a06c3..aeb39260 100644 --- a/language_packs/__init__.py +++ b/language_packs/__init__.py @@ -16,6 +16,12 @@ from .schema import ( MorphologyEntry, OOVPolicy, ) +from .compiler import ( + compile_entries_to_manifold, + compile_entries_to_modality_vocab, + load_pack, + load_pack_entries, +) __all__ = [ "AlignmentEdge", @@ -26,4 +32,8 @@ __all__ = [ "LexicalEntry", "MorphologyEntry", "OOVPolicy", + "compile_entries_to_manifold", + "compile_entries_to_modality_vocab", + "load_pack", + "load_pack_entries", ] diff --git a/language_packs/compiler.py b/language_packs/compiler.py new file mode 100644 index 00000000..d314bb0b --- /dev/null +++ b/language_packs/compiler.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import numpy as np + +from algebra.cl41 import geometric_product +from algebra.versor import unitize_versor +from language_packs.schema import LanguagePackManifest, LanguageRole, LexicalEntry, OOVPolicy +from sensorium.protocol import CL41_DIM, ModalityVocabulary +from vocab.manifold import VocabManifold + + +def _hash_to_blade(name: str, salt: str) -> int: + digest = hashlib.sha256(f"{salt}:{name}".encode("utf-8")).digest() + return int.from_bytes(digest[:2], "big") % CL41_DIM + + +def _hash_unit(name: str, salt: str) -> float: + digest = hashlib.sha256(f"{salt}:{name}".encode("utf-8")).digest() + return int.from_bytes(digest[:4], "big") / 2**32 + + +def _feature_rotor(name: str, salt: str, weight: float) -> np.ndarray: + # Grade-2 block indices 6..15. + idx = 6 + (_hash_to_blade(name, f"{salt}:biv") % 10) + theta = (0.2 + 0.8 * _hash_unit(name, f"{salt}:angle")) * weight + rotor = np.zeros(CL41_DIM, dtype=np.float32) + rotor[0] = np.cos(theta) + rotor[idx] = np.sin(theta) + return rotor + + +def _entry_to_coordinate(entry: LexicalEntry) -> np.ndarray: + vec = np.zeros(CL41_DIM, dtype=np.float32) + vec[0] = 1.0 + + pos = (entry.pos or entry.part_of_speech or "").lower() + for domain in entry.semantic_domains: + vec = geometric_product(vec, _feature_rotor(domain.lower(), "domain", 0.7)) + + if pos: + vec = geometric_product(vec, _feature_rotor(pos, "pos", 0.35)) + + for tag in entry.morphology_tags: + vec = geometric_product(vec, _feature_rotor(tag.lower(), "morph", 0.15)) + + vec = geometric_product(vec, _feature_rotor(entry.lemma.lower(), "lemma", 0.1)) + vec = geometric_product(vec, _feature_rotor(entry.surface.lower(), "surface", 0.05)) + return unitize_versor(vec) + + +def compile_entries_to_manifold(entries: list[LexicalEntry]) -> VocabManifold: + manifold = VocabManifold() + for entry in entries: + versor = _entry_to_coordinate(entry) + manifold.add(entry.surface, versor) + return manifold + + +def compile_entries_to_modality_vocab(entries: list[LexicalEntry]) -> ModalityVocabulary[str]: + vocab: ModalityVocabulary[str] = ModalityVocabulary() + for entry in entries: + point = _entry_to_coordinate(entry) + vocab.register_point(entry.surface, point) + return vocab + + +def _parse_entry(payload: dict) -> LexicalEntry: + return LexicalEntry( + entry_id=payload["entry_id"], + surface=payload["surface"], + lemma=payload.get("lemma", payload["surface"]), + language=payload["language"], + part_of_speech=payload.get("part_of_speech"), + pos=payload.get("pos"), + morphology_id=payload.get("morphology_id"), + morphology_tags=tuple(payload.get("morphology_tags", [])), + semantic_domains=tuple(payload.get("semantic_domains", [])), + manifold_point_checksum=payload.get("manifold_point_checksum"), + provenance_ids=tuple(payload.get("provenance_ids", [])), + ) + + +def load_pack(pack_id: str) -> tuple[LanguagePackManifest, VocabManifold]: + pack_dir = Path(__file__).parent / "data" / pack_id + manifest_path = pack_dir / "manifest.json" + lexicon_path = pack_dir / "lexicon.jsonl" + + manifest_payload = json.loads(manifest_path.read_text(encoding="utf-8")) + lexicon_bytes = lexicon_path.read_bytes() + checksum = hashlib.sha256(lexicon_bytes).hexdigest() + if checksum != manifest_payload["checksum"]: + raise ValueError(f"Checksum mismatch for {pack_id}: {checksum} != {manifest_payload['checksum']}") + + entries = load_pack_entries(pack_id) + + manifest = LanguagePackManifest( + pack_id=manifest_payload["pack_id"], + language=manifest_payload["language"], + role=LanguageRole(manifest_payload["role"]), + script=manifest_payload["script"], + normalization_policy=manifest_payload["normalization_policy"], + source_manifest=manifest_payload["source_manifest"], + determinism_class=manifest_payload["determinism_class"], + checksum=manifest_payload["checksum"], + version=manifest_payload.get("version", "1.0.0"), + gate_engaged=manifest_payload.get("gate_engaged", False), + oov_policy=OOVPolicy(manifest_payload.get("oov_policy", OOVPolicy.FAIL_CLOSED.value)), + ) + return manifest, compile_entries_to_manifold(entries) + + +def load_pack_entries(pack_id: str) -> list[LexicalEntry]: + pack_dir = Path(__file__).parent / "data" / pack_id + lexicon_path = pack_dir / "lexicon.jsonl" + entries: list[LexicalEntry] = [] + for line in lexicon_path.read_text(encoding="utf-8").splitlines(): + if line.strip(): + entries.append(_parse_entry(json.loads(line))) + return entries diff --git a/language_packs/data/en_minimal_v1/lexicon.jsonl b/language_packs/data/en_minimal_v1/lexicon.jsonl new file mode 100644 index 00000000..71268932 --- /dev/null +++ b/language_packs/data/en_minimal_v1/lexicon.jsonl @@ -0,0 +1,60 @@ +{"entry_id":"en-001","surface":"yes","lemma":"yes","language":"en","pos":"PART","semantic_domains":["affirmation","logic"],"morphology_tags":["particle"]} +{"entry_id":"en-002","surface":"no","lemma":"no","language":"en","pos":"PART","semantic_domains":["negation","logic"],"morphology_tags":["particle"]} +{"entry_id":"en-003","surface":"not","lemma":"not","language":"en","pos":"PART","semantic_domains":["negation","logic"],"morphology_tags":["adverbial"]} +{"entry_id":"en-004","surface":"if","lemma":"if","language":"en","pos":"SCONJ","semantic_domains":["condition","logic"],"morphology_tags":["conjunction"]} +{"entry_id":"en-005","surface":"then","lemma":"then","language":"en","pos":"ADV","semantic_domains":["sequence","logic"],"morphology_tags":["adverbial"]} +{"entry_id":"en-006","surface":"because","lemma":"because","language":"en","pos":"SCONJ","semantic_domains":["causality","logic"],"morphology_tags":["conjunction"]} +{"entry_id":"en-007","surface":"can","lemma":"can","language":"en","pos":"AUX","semantic_domains":["ability","modality"],"morphology_tags":["auxiliary"]} +{"entry_id":"en-008","surface":"cannot","lemma":"cannot","language":"en","pos":"AUX","semantic_domains":["negation","modality"],"morphology_tags":["auxiliary","negative"]} +{"entry_id":"en-009","surface":"I","lemma":"I","language":"en","pos":"PRON","semantic_domains":["self","agent"],"morphology_tags":["pronoun","first_person"]} +{"entry_id":"en-010","surface":"you","lemma":"you","language":"en","pos":"PRON","semantic_domains":["other","agent"],"morphology_tags":["pronoun","second_person"]} +{"entry_id":"en-011","surface":"it","lemma":"it","language":"en","pos":"PRON","semantic_domains":["object","reference"],"morphology_tags":["pronoun","third_person"]} +{"entry_id":"en-012","surface":"we","lemma":"we","language":"en","pos":"PRON","semantic_domains":["group","agent"],"morphology_tags":["pronoun","first_person_plural"]} +{"entry_id":"en-013","surface":"they","lemma":"they","language":"en","pos":"PRON","semantic_domains":["group","other"],"morphology_tags":["pronoun","third_person_plural"]} +{"entry_id":"en-014","surface":"am","lemma":"be","language":"en","pos":"AUX","semantic_domains":["being","identity"],"morphology_tags":["present","first_person"]} +{"entry_id":"en-015","surface":"is","lemma":"be","language":"en","pos":"AUX","semantic_domains":["being","identity"],"morphology_tags":["present","third_person_singular"]} +{"entry_id":"en-016","surface":"are","lemma":"be","language":"en","pos":"AUX","semantic_domains":["being","identity"],"morphology_tags":["present","plural"]} +{"entry_id":"en-017","surface":"know","lemma":"know","language":"en","pos":"VERB","semantic_domains":["knowledge","cognition"],"morphology_tags":["verb","present"]} +{"entry_id":"en-018","surface":"think","lemma":"think","language":"en","pos":"VERB","semantic_domains":["cognition","reason"],"morphology_tags":["verb","present"]} +{"entry_id":"en-019","surface":"mean","lemma":"mean","language":"en","pos":"VERB","semantic_domains":["semantics","reason"],"morphology_tags":["verb","present"]} +{"entry_id":"en-020","surface":"say","lemma":"say","language":"en","pos":"VERB","semantic_domains":["speech","communication"],"morphology_tags":["verb","present"]} +{"entry_id":"en-021","surface":"ask","lemma":"ask","language":"en","pos":"VERB","semantic_domains":["question","communication"],"morphology_tags":["verb","present"]} +{"entry_id":"en-022","surface":"answer","lemma":"answer","language":"en","pos":"NOUN","semantic_domains":["response","communication"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-023","surface":"question","lemma":"question","language":"en","pos":"NOUN","semantic_domains":["question","reason"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-024","surface":"word","lemma":"word","language":"en","pos":"NOUN","semantic_domains":["logos","language"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-025","surface":"truth","lemma":"truth","language":"en","pos":"NOUN","semantic_domains":["logos","epistemic"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-026","surface":"light","lemma":"light","language":"en","pos":"NOUN","semantic_domains":["logos","illumination"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-027","surface":"life","lemma":"life","language":"en","pos":"NOUN","semantic_domains":["logos","vitality"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-028","surface":"beginning","lemma":"beginning","language":"en","pos":"NOUN","semantic_domains":["logos","origin"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-029","surface":"create","lemma":"create","language":"en","pos":"VERB","semantic_domains":["logos","formation"],"morphology_tags":["verb","present"]} +{"entry_id":"en-030","surface":"spirit","lemma":"spirit","language":"en","pos":"NOUN","semantic_domains":["logos","breath"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-031","surface":"order","lemma":"order","language":"en","pos":"NOUN","semantic_domains":["logos","structure"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-032","surface":"reason","lemma":"reason","language":"en","pos":"NOUN","semantic_domains":["logos","cognition"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-033","surface":"name","lemma":"name","language":"en","pos":"NOUN","semantic_domains":["identity","language"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-034","surface":"time","lemma":"time","language":"en","pos":"NOUN","semantic_domains":["temporal","sequence"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-035","surface":"place","lemma":"place","language":"en","pos":"NOUN","semantic_domains":["spatial","location"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-036","surface":"thing","lemma":"thing","language":"en","pos":"NOUN","semantic_domains":["object","reference"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-037","surface":"begin","lemma":"begin","language":"en","pos":"VERB","semantic_domains":["origin","sequence"],"morphology_tags":["verb","present"]} +{"entry_id":"en-038","surface":"end","lemma":"end","language":"en","pos":"NOUN","semantic_domains":["boundary","sequence"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-039","surface":"good","lemma":"good","language":"en","pos":"ADJ","semantic_domains":["value","evaluation"],"morphology_tags":["adjective","positive"]} +{"entry_id":"en-040","surface":"evil","lemma":"evil","language":"en","pos":"ADJ","semantic_domains":["value","evaluation"],"morphology_tags":["adjective","negative"]} +{"entry_id":"en-041","surface":"love","lemma":"love","language":"en","pos":"NOUN","semantic_domains":["affect","relation"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-042","surface":"peace","lemma":"peace","language":"en","pos":"NOUN","semantic_domains":["affect","order"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-043","surface":"power","lemma":"power","language":"en","pos":"NOUN","semantic_domains":["ability","agency"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-044","surface":"will","lemma":"will","language":"en","pos":"NOUN","semantic_domains":["agency","modality"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-045","surface":"hear","lemma":"hear","language":"en","pos":"VERB","semantic_domains":["perception","audio"],"morphology_tags":["verb","present"]} +{"entry_id":"en-046","surface":"see","lemma":"see","language":"en","pos":"VERB","semantic_domains":["perception","vision"],"morphology_tags":["verb","present"]} +{"entry_id":"en-047","surface":"speak","lemma":"speak","language":"en","pos":"VERB","semantic_domains":["speech","communication"],"morphology_tags":["verb","present"]} +{"entry_id":"en-048","surface":"listen","lemma":"listen","language":"en","pos":"VERB","semantic_domains":["attention","audio"],"morphology_tags":["verb","present"]} +{"entry_id":"en-049","surface":"walk","lemma":"walk","language":"en","pos":"VERB","semantic_domains":["motion","body"],"morphology_tags":["verb","present"]} +{"entry_id":"en-050","surface":"path","lemma":"path","language":"en","pos":"NOUN","semantic_domains":["motion","sequence"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-051","surface":"field","lemma":"field","language":"en","pos":"NOUN","semantic_domains":["geometry","state"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-052","surface":"form","lemma":"form","language":"en","pos":"NOUN","semantic_domains":["geometry","structure"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-053","surface":"memory","lemma":"memory","language":"en","pos":"NOUN","semantic_domains":["storage","cognition"],"morphology_tags":["noun","singular"]} +{"entry_id":"en-054","surface":"recall","lemma":"recall","language":"en","pos":"VERB","semantic_domains":["memory","cognition"],"morphology_tags":["verb","present"]} +{"entry_id":"en-055","surface":"with","lemma":"with","language":"en","pos":"ADP","semantic_domains":["relation","association"],"morphology_tags":["preposition"]} +{"entry_id":"en-056","surface":"in","lemma":"in","language":"en","pos":"ADP","semantic_domains":["relation","location"],"morphology_tags":["preposition"]} +{"entry_id":"en-057","surface":"from","lemma":"from","language":"en","pos":"ADP","semantic_domains":["relation","origin"],"morphology_tags":["preposition"]} +{"entry_id":"en-058","surface":"to","lemma":"to","language":"en","pos":"PART","semantic_domains":["relation","direction"],"morphology_tags":["particle"]} +{"entry_id":"en-059","surface":"what","lemma":"what","language":"en","pos":"PRON","semantic_domains":["question","reference"],"morphology_tags":["pronoun","interrogative"]} +{"entry_id":"en-060","surface":"who","lemma":"who","language":"en","pos":"PRON","semantic_domains":["question","agent"],"morphology_tags":["pronoun","interrogative"]} diff --git a/language_packs/data/en_minimal_v1/manifest.json b/language_packs/data/en_minimal_v1/manifest.json new file mode 100644 index 00000000..8f84f5f0 --- /dev/null +++ b/language_packs/data/en_minimal_v1/manifest.json @@ -0,0 +1,13 @@ +{ + "pack_id": "en_minimal_v1", + "language": "en", + "role": "operational_base", + "script": "Latin", + "normalization_policy": "unitize_versor", + "source_manifest": "en_minimal_v1.lexicon.jsonl", + "determinism_class": "D0", + "checksum": "3b2804670b37fe51e2c93209a048df0f1fb1a426ea0ed3580a6fd08021b2bc70", + "version": "1.0.0", + "gate_engaged": true, + "oov_policy": "tagged_fallback" +} diff --git a/language_packs/data/grc_logos_micro_v1/lexicon.jsonl b/language_packs/data/grc_logos_micro_v1/lexicon.jsonl new file mode 100644 index 00000000..e31ebcd9 --- /dev/null +++ b/language_packs/data/grc_logos_micro_v1/lexicon.jsonl @@ -0,0 +1,7 @@ +{"entry_id":"grc-001","surface":"λόγος","lemma":"λόγος","language":"grc","pos":"NOUN","semantic_domains":["logos","word"],"morphology_tags":["noun","nominative","singular"]} +{"entry_id":"grc-002","surface":"ἀρχή","lemma":"ἀρχή","language":"grc","pos":"NOUN","semantic_domains":["logos","beginning"],"morphology_tags":["noun","nominative","singular"]} +{"entry_id":"grc-003","surface":"φῶς","lemma":"φῶς","language":"grc","pos":"NOUN","semantic_domains":["logos","light"],"morphology_tags":["noun","nominative","singular"]} +{"entry_id":"grc-004","surface":"ζωή","lemma":"ζωή","language":"grc","pos":"NOUN","semantic_domains":["logos","life"],"morphology_tags":["noun","nominative","singular"]} +{"entry_id":"grc-005","surface":"ἀλήθεια","lemma":"ἀλήθεια","language":"grc","pos":"NOUN","semantic_domains":["logos","truth"],"morphology_tags":["noun","nominative","singular"]} +{"entry_id":"grc-006","surface":"πνεῦμα","lemma":"πνεῦμα","language":"grc","pos":"NOUN","semantic_domains":["logos","spirit"],"morphology_tags":["noun","nominative","singular"]} +{"entry_id":"grc-007","surface":"κτίζω","lemma":"κτίζω","language":"grc","pos":"VERB","semantic_domains":["logos","create"],"morphology_tags":["verb","present","active"]} diff --git a/language_packs/data/grc_logos_micro_v1/manifest.json b/language_packs/data/grc_logos_micro_v1/manifest.json new file mode 100644 index 00000000..a96a5533 --- /dev/null +++ b/language_packs/data/grc_logos_micro_v1/manifest.json @@ -0,0 +1,13 @@ +{ + "pack_id": "grc_logos_micro_v1", + "language": "grc", + "role": "depth_relation", + "script": "Greek", + "normalization_policy": "unitize_versor", + "source_manifest": "grc_logos_micro_v1.lexicon.jsonl", + "determinism_class": "D0", + "checksum": "cf8930df2a4faf1b8d87916fb707d44fa3034685a7f14b56fcde1bef794547c8", + "version": "1.0.0", + "gate_engaged": false, + "oov_policy": "fail_closed" +} diff --git a/language_packs/data/he_logos_micro_v1/lexicon.jsonl b/language_packs/data/he_logos_micro_v1/lexicon.jsonl new file mode 100644 index 00000000..4a5e7a71 --- /dev/null +++ b/language_packs/data/he_logos_micro_v1/lexicon.jsonl @@ -0,0 +1,7 @@ +{"entry_id":"he-001","surface":"דבר","lemma":"דבר","language":"he","pos":"NOUN","semantic_domains":["logos","word"],"morphology_tags":["root:DBR","noun"]} +{"entry_id":"he-002","surface":"ראשית","lemma":"ראשית","language":"he","pos":"NOUN","semantic_domains":["logos","beginning"],"morphology_tags":["root:RASH","noun"]} +{"entry_id":"he-003","surface":"אור","lemma":"אור","language":"he","pos":"NOUN","semantic_domains":["logos","light"],"morphology_tags":["root:AWR","noun"]} +{"entry_id":"he-004","surface":"חיים","lemma":"חיים","language":"he","pos":"NOUN","semantic_domains":["logos","life"],"morphology_tags":["root:CHYY","noun"]} +{"entry_id":"he-005","surface":"אמת","lemma":"אמת","language":"he","pos":"NOUN","semantic_domains":["logos","truth"],"morphology_tags":["root:AMT","noun"]} +{"entry_id":"he-006","surface":"רוח","lemma":"רוח","language":"he","pos":"NOUN","semantic_domains":["logos","spirit"],"morphology_tags":["root:RWH","noun"]} +{"entry_id":"he-007","surface":"ברא","lemma":"ברא","language":"he","pos":"VERB","semantic_domains":["logos","create"],"morphology_tags":["root:BRA","verb","qal"]} diff --git a/language_packs/data/he_logos_micro_v1/manifest.json b/language_packs/data/he_logos_micro_v1/manifest.json new file mode 100644 index 00000000..c7d4874d --- /dev/null +++ b/language_packs/data/he_logos_micro_v1/manifest.json @@ -0,0 +1,13 @@ +{ + "pack_id": "he_logos_micro_v1", + "language": "he", + "role": "depth_root", + "script": "Hebrew", + "normalization_policy": "unitize_versor", + "source_manifest": "he_logos_micro_v1.lexicon.jsonl", + "determinism_class": "D0", + "checksum": "ef21e0cf2b15663603e64f94bdd42634acfde969280e556e915c1fbacc055921", + "version": "1.0.0", + "gate_engaged": false, + "oov_policy": "fail_closed" +} diff --git a/language_packs/schema.py b/language_packs/schema.py index 8631273c..4075b7d9 100644 --- a/language_packs/schema.py +++ b/language_packs/schema.py @@ -94,7 +94,9 @@ class LexicalEntry: lemma: str language: str part_of_speech: str | None = None + pos: str | None = None morphology_id: str | None = None + morphology_tags: tuple[str, ...] = field(default_factory=tuple) semantic_domains: tuple[str, ...] = field(default_factory=tuple) manifold_point_checksum: str | None = None provenance_ids: tuple[str, ...] = field(default_factory=tuple) diff --git a/probe/repl.py b/probe/repl.py new file mode 100644 index 00000000..d8d143f1 --- /dev/null +++ b/probe/repl.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from language_packs import load_pack + + +def field_walk(seed: str, steps: int = 4) -> list[str]: + _, vocab = load_pack("en_minimal_v1") + F = vocab.get_versor(seed) + walk = [seed] + idx = -1 + for _ in range(steps - 1): + word, idx = vocab.nearest(F, exclude_idx=idx) + walk.append(word) + F = vocab.get_versor(word) + return walk + + +def main() -> None: + while True: + text = input("> ").strip() + if text in {"quit", "exit"}: + break + try: + chain = field_walk(text) + print(f"[field walk: {' → '.join(chain)}]") + except KeyError: + print("[unknown token]") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 80501b6e..0b22f0db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,8 @@ requires-python = ">=3.11" dependencies = [ "numpy>=1.26", "mlx>=0.18; sys_platform == 'darwin'", + "pytest>=9.0.3", + "pytest-asyncio>=1.3.0", ] [project.optional-dependencies] diff --git a/tests/test_holonomy_resonance.py b/tests/test_holonomy_resonance.py new file mode 100644 index 00000000..542d10f5 --- /dev/null +++ b/tests/test_holonomy_resonance.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import numpy as np + +from algebra.cga import cga_inner +from algebra.holonomy import holonomy_encode, holonomy_similarity +from language_packs import load_pack + + +def _encode(manifold, tokens: list[str]) -> np.ndarray: + return holonomy_encode([manifold.get_versor(t) for t in tokens]) + + +def test_aligned_clauses_have_higher_similarity_than_unrelated(): + _, en = load_pack("en_minimal_v1") + _, he = load_pack("he_logos_micro_v1") + _, grc = load_pack("grc_logos_micro_v1") + + en_clause = _encode(en, ["word", "beginning", "with", "truth"]) + he_clause = _encode(he, ["דבר", "ראשית", "אמת"]) + grc_clause = _encode(grc, ["λόγος", "ἀρχή", "ἀλήθεια"]) + grc_unrelated = _encode(grc, ["ζωή", "πνεῦμα", "κτίζω"]) + + aligned = (holonomy_similarity(en_clause, he_clause) + holonomy_similarity(en_clause, grc_clause)) / 2.0 + unrelated = holonomy_similarity(en_clause, grc_unrelated) + assert aligned > unrelated + + +def test_triple_alignment_closer_than_other_triples(): + _, en = load_pack("en_minimal_v1") + _, he = load_pack("he_logos_micro_v1") + _, grc = load_pack("grc_logos_micro_v1") + + word_trip = [ + en.get_versor("word"), + he.get_versor("דבר"), + grc.get_versor("λόγος"), + ] + aligned_score = np.mean( + [ + cga_inner(en.get_versor("word"), he.get_versor("דבר")), + cga_inner(en.get_versor("word"), grc.get_versor("λόγος")), + cga_inner(he.get_versor("דבר"), grc.get_versor("λόγος")), + ] + ) + misaligned_score = np.mean( + [ + cga_inner(en.get_versor("word"), he.get_versor("ראשית")), + cga_inner(en.get_versor("word"), grc.get_versor("πνεῦμα")), + cga_inner(he.get_versor("דבר"), grc.get_versor("ἀρχή")), + ] + ) + assert aligned_score > misaligned_score + + +def test_word_order_permutation_changes_holonomy(): + _, en = load_pack("en_minimal_v1") + a = _encode(en, ["word", "truth", "light", "life"]) + b = _encode(en, ["life", "word", "truth", "light"]) + assert abs(holonomy_similarity(a, b) - holonomy_similarity(a, a)) > 1e-4 diff --git a/tests/test_language_pack_runtime.py b/tests/test_language_pack_runtime.py new file mode 100644 index 00000000..a620638b --- /dev/null +++ b/tests/test_language_pack_runtime.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from chat.runtime import ChatRuntime +from language_packs import load_pack + + +def test_load_pack_and_vocab_size(): + manifest, manifold = load_pack("en_minimal_v1") + assert manifest.pack_id == "en_minimal_v1" + assert len(manifold) == 60 + + +def test_light_neighborhood_not_negation(): + _, manifold = load_pack("en_minimal_v1") + light = manifold.get_versor("light") + word, _ = manifold.nearest(light, exclude_idx=25) + assert word not in {"no", "cannot"} + + +def test_chat_runtime_responds_and_varies(): + runtime = ChatRuntime("en_minimal_v1") + a = runtime.respond("what is truth", max_tokens=8) + b = runtime.respond("what is light", max_tokens=8) + assert a.strip() + assert b.strip() + assert a != b