diff --git a/language_packs/compiler.py b/language_packs/compiler.py index 8f898d40..806d867a 100644 --- a/language_packs/compiler.py +++ b/language_packs/compiler.py @@ -9,10 +9,17 @@ import numpy as np from algebra.cl41 import N_COMPONENTS, geometric_product from algebra.versor import unitize_versor -from language_packs.schema import LanguagePackManifest, LanguageRole, LexicalEntry, OOVPolicy +from language_packs.schema import ( + LanguagePackManifest, + LanguageRole, + LexicalEntry, + MorphologyEntry, + OOVPolicy, +) from vocab.manifold import VocabManifold if TYPE_CHECKING: + from morphology.registry import MorphologyRegistry from sensorium.protocol import ModalityVocabulary @@ -52,7 +59,82 @@ def _domain_features(domain: str) -> list[tuple[str, float]]: ] -def _entry_to_coordinate(entry: LexicalEntry) -> np.ndarray: +_INFLECTION_PRIORITY = ( + "pos", + "binyan", + "declension", + "tense", + "voice", + "mood", + "aspect", + "person", + "gender", + "number", + "case", + "state", +) + + +def _ordered_inflection_items(inflection: dict[str, str]) -> list[tuple[str, str]]: + priority = {key: idx for idx, key in enumerate(_INFLECTION_PRIORITY)} + return sorted( + inflection.items(), + key=lambda item: (priority.get(item[0], len(_INFLECTION_PRIORITY)), item[0]), + ) + + +def _compact_root(root: str) -> str: + return root.replace("-", "") + + +def _apply_morphology(vec: np.ndarray, morphology: MorphologyEntry) -> np.ndarray: + if morphology.root: + vec = geometric_product(vec, _feature_rotor(morphology.root.lower(), "morph:root", 0.08)) + vec = geometric_product( + vec, + _feature_rotor(f"root:{_compact_root(morphology.root).lower()}", "morph", 0.12), + ) + + for idx, prefix in enumerate(morphology.prefix_chain): + weight = 0.05 / (idx + 1) + vec = geometric_product( + vec, + _feature_rotor(f"{idx}:{prefix.lower()}", "morph:prefix", weight), + ) + + if morphology.stem: + vec = geometric_product(vec, _feature_rotor(morphology.stem.lower(), "morph:stem", 0.05)) + + for key, value in _ordered_inflection_items(dict(morphology.inflection)): + vec = geometric_product( + vec, + _feature_rotor(key.lower(), "morph:infl-role", 0.01), + ) + vec = geometric_product( + vec, + _feature_rotor(value.lower(), "morph", 0.035), + ) + + for idx, suffix in enumerate(morphology.suffix_chain): + weight = 0.04 / (idx + 1) + vec = geometric_product( + vec, + _feature_rotor(f"{idx}:{suffix.lower()}", "morph:suffix", weight), + ) + + return vec + + +def _apply_morphology_tags(vec: np.ndarray, tags: tuple[str, ...], weight: float) -> np.ndarray: + for tag in tags: + vec = geometric_product(vec, _feature_rotor(tag.lower(), "morph", weight)) + return vec + + +def _entry_to_coordinate( + entry: LexicalEntry, + morphology: MorphologyEntry | None = None, +) -> np.ndarray: vec = np.zeros(N_COMPONENTS, dtype=np.float32) vec[0] = 1.0 @@ -64,28 +146,54 @@ def _entry_to_coordinate(entry: LexicalEntry) -> np.ndarray: if pos: vec = geometric_product(vec, _feature_rotor(pos, "pos", 0.35)) - for tag in entry.morphology_tags: - vec = geometric_product(vec, _feature_rotor(tag.lower(), "morph", 0.15)) + if morphology is not None: + vec = _apply_morphology(vec, morphology) + else: + vec = _apply_morphology_tags(vec, entry.morphology_tags, 0.15) vec = geometric_product(vec, _feature_rotor(entry.lemma.lower(), "lemma", 0.1)) vec = geometric_product(vec, _feature_rotor(entry.surface.lower(), "surface", 0.05)) return unitize_versor(vec) -def compile_entries_to_manifold(entries: list[LexicalEntry]) -> VocabManifold: +def _uses_legacy_root_tags(entry: LexicalEntry) -> bool: + return any( + tag.startswith("root:") or tag.startswith("triliteral:") + for tag in entry.morphology_tags + ) + + +def _resolved_morphology( + entry: LexicalEntry, + morphology_registry: "MorphologyRegistry | None", +) -> MorphologyEntry | None: + if morphology_registry is None or not entry.morphology_id: + return None + if _uses_legacy_root_tags(entry): + return None + return morphology_registry.get(entry.morphology_id) + + +def compile_entries_to_manifold( + entries: list[LexicalEntry], + morphology_registry: "MorphologyRegistry | None" = None, +) -> VocabManifold: manifold = VocabManifold() for entry in entries: - versor = _entry_to_coordinate(entry) + versor = _entry_to_coordinate(entry, _resolved_morphology(entry, morphology_registry)) manifold.add(entry.surface, versor) return manifold -def compile_entries_to_modality_vocab(entries: list[LexicalEntry]) -> "ModalityVocabulary[str]": +def compile_entries_to_modality_vocab( + entries: list[LexicalEntry], + morphology_registry: "MorphologyRegistry | None" = None, +) -> "ModalityVocabulary[str]": from sensorium.protocol import ModalityVocabulary vocab: ModalityVocabulary[str] = ModalityVocabulary() for entry in entries: - point = _entry_to_coordinate(entry) + point = _entry_to_coordinate(entry, _resolved_morphology(entry, morphology_registry)) vocab.register_point(entry.surface, point) return vocab @@ -118,6 +226,11 @@ def load_pack(pack_id: str) -> tuple[LanguagePackManifest, VocabManifold]: raise ValueError(f"Checksum mismatch for {pack_id}: {checksum} != {manifest_payload['checksum']}") entries = load_pack_entries(pack_id) + morphology_registry = None + if any(entry.morphology_id for entry in entries): + from morphology.registry import load_morphology + + morphology_registry = load_morphology(pack_id) manifest = LanguagePackManifest( pack_id=manifest_payload["pack_id"], @@ -132,7 +245,7 @@ def load_pack(pack_id: str) -> tuple[LanguagePackManifest, VocabManifold]: gate_engaged=manifest_payload.get("gate_engaged", False), oov_policy=OOVPolicy(manifest_payload.get("oov_policy", OOVPolicy.FAIL_CLOSED.value)), ) - return manifest, compile_entries_to_manifold(entries) + return manifest, compile_entries_to_manifold(entries, morphology_registry=morphology_registry) def load_pack_entries(pack_id: str) -> list[LexicalEntry]: diff --git a/language_packs/data/he_logos_micro_v1/lexicon.jsonl b/language_packs/data/he_logos_micro_v1/lexicon.jsonl index 3b052c9c..746b3e1a 100644 --- a/language_packs/data/he_logos_micro_v1/lexicon.jsonl +++ b/language_packs/data/he_logos_micro_v1/lexicon.jsonl @@ -1,4 +1,5 @@ {"entry_id": "he-001", "surface": "\u05d3\u05d1\u05e8", "lemma": "\u05d3\u05d1\u05e8", "language": "he", "pos": "NOUN", "morphology_id": "he-morph-001", "semantic_domains": ["logos.utterance.word", "logos.core", "communication.articulation", "divine.revelation", "existence.utterance"], "morphology_tags": ["noun", "masculine", "singular", "root:\u05d3\u05d1\u05e8", "triliteral:D-B-R"]} +{"entry_id": "he-008", "surface": "\u05d3\u05d1\u05e8\u05d9\u05dd", "lemma": "\u05d3\u05d1\u05e8", "language": "he", "pos": "NOUN", "morphology_id": "he-morph-008", "semantic_domains": ["logos.utterance.word", "logos.core", "communication.articulation", "divine.revelation", "existence.utterance", "reference.plural"], "morphology_tags": ["noun", "masculine", "plural"]} {"entry_id": "he-002", "surface": "\u05e8\u05d0\u05e9\u05d9\u05ea", "lemma": "\u05e8\u05d0\u05e9\u05d9\u05ea", "language": "he", "pos": "NOUN", "morphology_id": "he-morph-002", "semantic_domains": ["logos.genesis.origin", "logos.core", "origin", "existence.ground", "temporal.absolute"], "morphology_tags": ["noun", "feminine", "singular", "root:\u05e8\u05d0\u05e9", "construct_state"]} {"entry_id": "he-003", "surface": "\u05d0\u05d5\u05e8", "lemma": "\u05d0\u05d5\u05e8", "language": "he", "pos": "NOUN", "morphology_id": "he-morph-003", "semantic_domains": ["logos.illumination.photon", "logos.core", "logos.illumination", "divine.revelation", "knowledge.clarity"], "morphology_tags": ["noun", "masculine", "singular", "root:\u05d0\u05d5\u05e8", "triliteral:A-W-R"]} {"entry_id": "he-004", "surface": "\u05d7\u05d9\u05d9\u05dd", "lemma": "\u05d7\u05d9\u05d9\u05dd", "language": "he", "pos": "NOUN", "morphology_id": "he-morph-004", "semantic_domains": ["logos.vitality.animate", "logos.core", "vitality", "existence.manifestation", "being.animate"], "morphology_tags": ["noun", "masculine", "plural", "root:\u05d7\u05d9\u05d4", "triliteral:H-Y-H"]} diff --git a/language_packs/data/he_logos_micro_v1/manifest.json b/language_packs/data/he_logos_micro_v1/manifest.json index 7a4546f7..e64b6af2 100644 --- a/language_packs/data/he_logos_micro_v1/manifest.json +++ b/language_packs/data/he_logos_micro_v1/manifest.json @@ -6,7 +6,7 @@ "normalization_policy": "unitize_versor", "source_manifest": "he_logos_micro_v1.lexicon.jsonl", "determinism_class": "D0", - "checksum": "6e2f30175937c56cae276007d9865555beb5dd0759925660ab259c54a67d7b1d", + "checksum": "a419d50f0bd9e326969e358d9f83ac7c56f0ed82cf1c3caba05cda7e06c5fbe0", "version": "1.0.0", "gate_engaged": true, "oov_policy": "fail_closed" diff --git a/language_packs/data/he_logos_micro_v1/morphology.jsonl b/language_packs/data/he_logos_micro_v1/morphology.jsonl index 213277bc..250e2257 100644 --- a/language_packs/data/he_logos_micro_v1/morphology.jsonl +++ b/language_packs/data/he_logos_micro_v1/morphology.jsonl @@ -1,4 +1,5 @@ {"morphology_id": "he-morph-001", "surface": "\u05d3\u05d1\u05e8", "lemma": "\u05d3\u05d1\u05e8", "language": "he", "root": "\u05d3-\u05d1-\u05e8", "prefix_chain": [], "stem": "\u05d3\u05d1\u05e8", "inflection": {"pos": "noun", "gender": "masculine", "number": "singular"}, "suffix_chain": []} +{"morphology_id": "he-morph-008", "surface": "\u05d3\u05d1\u05e8\u05d9\u05dd", "lemma": "\u05d3\u05d1\u05e8", "language": "he", "root": "\u05d3-\u05d1-\u05e8", "prefix_chain": [], "stem": "\u05d3\u05d1\u05e8", "inflection": {"pos": "noun", "gender": "masculine", "number": "plural"}, "suffix_chain": ["\u05d9\u05dd"]} {"morphology_id": "he-morph-002", "surface": "\u05e8\u05d0\u05e9\u05d9\u05ea", "lemma": "\u05e8\u05d0\u05e9\u05d9\u05ea", "language": "he", "root": "\u05e8-\u05d0-\u05e9", "prefix_chain": [], "stem": "\u05e8\u05d0\u05e9", "inflection": {"pos": "noun", "gender": "feminine", "number": "singular", "state": "construct"}, "suffix_chain": ["\u05d9\u05ea"]} {"morphology_id": "he-morph-003", "surface": "\u05d0\u05d5\u05e8", "lemma": "\u05d0\u05d5\u05e8", "language": "he", "root": "\u05d0-\u05d5-\u05e8", "prefix_chain": [], "stem": "\u05d0\u05d5\u05e8", "inflection": {"pos": "noun", "gender": "masculine", "number": "singular"}, "suffix_chain": []} {"morphology_id": "he-morph-004", "surface": "\u05d7\u05d9\u05d9\u05dd", "lemma": "\u05d7\u05d9\u05d9\u05dd", "language": "he", "root": "\u05d7-\u05d9-\u05d4", "prefix_chain": [], "stem": "\u05d7\u05d9", "inflection": {"pos": "noun", "gender": "masculine", "number": "plural"}, "suffix_chain": ["\u05d9\u05dd"]} diff --git a/tests/test_holonomy_resonance.py b/tests/test_holonomy_resonance.py index 2d95061b..c5506b27 100644 --- a/tests/test_holonomy_resonance.py +++ b/tests/test_holonomy_resonance.py @@ -5,6 +5,8 @@ import numpy as np from algebra.cga import cga_inner from algebra.holonomy import holonomy_encode, holonomy_similarity from language_packs import load_pack +from language_packs.compiler import compile_entries_to_manifold, load_pack_entries +from morphology.registry import load_morphology def _encode(manifold, tokens: list[str]) -> np.ndarray: @@ -58,3 +60,24 @@ def test_word_order_permutation_changes_holonomy(): a = _encode(en, ["word", "truth", "light", "life"]) b = _encode(en, ["life", "word", "truth", "light"]) assert abs(holonomy_similarity(a, b) - holonomy_similarity(a, a)) > 1e-4 + + +def test_same_root_hebrew_forms_land_closer_than_unrelated_noun(): + _, he = load_pack("he_logos_micro_v1") + + singular = he.get_versor("דבר") + plural = he.get_versor("דברים") + unrelated = he.get_versor("ראשית") + + assert cga_inner(singular, plural) > cga_inner(singular, unrelated) + + +def test_structured_morphology_improves_same_root_hebrew_resonance(): + entries = load_pack_entries("he_logos_micro_v1") + tag_only = compile_entries_to_manifold(entries) + structured = compile_entries_to_manifold(entries, load_morphology("he_logos_micro_v1")) + + tag_only_score = cga_inner(tag_only.get_versor("דבר"), tag_only.get_versor("דברים")) + structured_score = cga_inner(structured.get_versor("דבר"), structured.get_versor("דברים")) + + assert structured_score > tag_only_score diff --git a/tests/test_morphology_registry.py b/tests/test_morphology_registry.py index 06409889..ea160fd6 100644 --- a/tests/test_morphology_registry.py +++ b/tests/test_morphology_registry.py @@ -9,11 +9,16 @@ from morphology.registry import MorphologyRegistry, load_morphology def test_hebrew_morphology_registry_loads_triliteral_roots(): registry = load_morphology("he_logos_micro_v1") - assert len(registry) == 7 + assert len(registry) == 8 davar = registry.require("he-morph-001") assert davar.surface == "\u05d3\u05d1\u05e8" assert davar.root == "\u05d3-\u05d1-\u05e8" assert davar.inflection["gender"] == "masculine" + devarim = registry.require("he-morph-008") + assert devarim.surface == "\u05d3\u05d1\u05e8\u05d9\u05dd" + assert devarim.root == davar.root + assert devarim.suffix_chain == ("\u05d9\u05dd",) + assert devarim.inflection["number"] == "plural" def test_greek_morphology_registry_preserves_ordered_prefix_and_suffix():