core-labs/core
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Migrate Hebrew lexicon to structured morphology

Browse source
This commit is contained in:
Shay 2026-05-13 15:15:15 -07:00
parent 9e415a0bdf
commit 5c952a9f9d
3 changed files with 53 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
language_packs/compiler.py
View file

@ -87,12 +87,52 @@ def _compact_root(root: str) -> str:
return root.replace("-", "")
_HEBREW_ROOT_ROMANIZATION = {
"א": "A",
"ב": "B",
"ג": "G",
"ד": "D",
"ה": "H",
"ו": "W",
"ז": "Z",
"ח": "H",
"ט": "T",
"י": "Y",
"כ": "K",
"ך": "K",
"ל": "L",
"מ": "M",
"ם": "M",
"נ": "N",
"ן": "N",
"ס": "S",
"ע": "A",
"פ": "P",
"ף": "P",
"צ": "TS",
"ץ": "TS",
"ק": "Q",
"ר": "R",
"ש": "SH",
"ת": "T",
}
def _triliteral_root(root: str) -> str:
parts = [part for part in root.split("-") if part]
romanized = [_HEBREW_ROOT_ROMANIZATION.get(part, part.upper()) for part in parts]
return "-".join(romanized) if romanized else _compact_root(root).upper()
def _apply_morphology(vec: np.ndarray, morphology: MorphologyEntry) -> np.ndarray:
if morphology.root:
vec = geometric_product(vec, _feature_rotor(morphology.root.lower(), "morph:root", 0.08))
vec = geometric_product(
vec,
_feature_rotor(f"root:{_compact_root(morphology.root).lower()}", "morph", 0.12),
_feature_rotor(f"triliteral:{_triliteral_root(morphology.root).lower()}", "morph", 0.13),
)
vec = geometric_product(
vec,
_feature_rotor(f"root:{_compact_root(morphology.root).lower()}", "morph", 0.17),
)
for idx, prefix in enumerate(morphology.prefix_chain):
@ -103,16 +143,16 @@ def _apply_morphology(vec: np.ndarray, morphology: MorphologyEntry) -> np.ndarra
)
if morphology.stem:
vec = geometric_product(vec, _feature_rotor(morphology.stem.lower(), "morph:stem", 0.05))
vec = geometric_product(vec, _feature_rotor(morphology.stem.lower(), "morph:stem", 0.10))
for key, value in _ordered_inflection_items(dict(morphology.inflection)):
vec = geometric_product(
vec,
_feature_rotor(key.lower(), "morph:infl-role", 0.01),
_feature_rotor(key.lower(), "morph:infl-role", 0.02),
)
vec = geometric_product(
vec,
_feature_rotor(value.lower(), "morph", 0.035),
_feature_rotor(value.lower(), "morph", 0.05),
)
for idx, suffix in enumerate(morphology.suffix_chain):

14
language_packs/data/he_logos_micro_v1/lexicon.jsonl
View file

@ -1,8 +1,8 @@
{"entry_id": "he-001", "surface": "\u05d3\u05d1\u05e8", "lemma": "\u05d3\u05d1\u05e8", "language": "he", "pos": "NOUN", "morphology_id": "he-morph-001", "semantic_domains": ["logos.utterance.word", "logos.core", "communication.articulation", "divine.revelation", "existence.utterance"], "morphology_tags": ["noun", "masculine", "singular", "root:\u05d3\u05d1\u05e8", "triliteral:D-B-R"]}
{"entry_id": "he-001", "surface": "\u05d3\u05d1\u05e8", "lemma": "\u05d3\u05d1\u05e8", "language": "he", "pos": "NOUN", "morphology_id": "he-morph-001", "semantic_domains": ["logos.utterance.word", "logos.core", "communication.articulation", "divine.revelation", "existence.utterance"], "morphology_tags": ["noun", "masculine", "singular"]}
{"entry_id": "he-008", "surface": "\u05d3\u05d1\u05e8\u05d9\u05dd", "lemma": "\u05d3\u05d1\u05e8", "language": "he", "pos": "NOUN", "morphology_id": "he-morph-008", "semantic_domains": ["logos.utterance.word", "logos.core", "communication.articulation", "divine.revelation", "existence.utterance", "reference.plural"], "morphology_tags": ["noun", "masculine", "plural"]}
{"entry_id": "he-002", "surface": "\u05e8\u05d0\u05e9\u05d9\u05ea", "lemma": "\u05e8\u05d0\u05e9\u05d9\u05ea", "language": "he", "pos": "NOUN", "morphology_id": "he-morph-002", "semantic_domains": ["logos.genesis.origin", "logos.core", "origin", "existence.ground", "temporal.absolute"], "morphology_tags": ["noun", "feminine", "singular", "root:\u05e8\u05d0\u05e9", "construct_state"]}
{"entry_id": "he-003", "surface": "\u05d0\u05d5\u05e8", "lemma": "\u05d0\u05d5\u05e8", "language": "he", "pos": "NOUN", "morphology_id": "he-morph-003", "semantic_domains": ["logos.illumination.photon", "logos.core", "logos.illumination", "divine.revelation", "knowledge.clarity"], "morphology_tags": ["noun", "masculine", "singular", "root:\u05d0\u05d5\u05e8", "triliteral:A-W-R"]}
{"entry_id": "he-004", "surface": "\u05d7\u05d9\u05d9\u05dd", "lemma": "\u05d7\u05d9\u05d9\u05dd", "language": "he", "pos": "NOUN", "morphology_id": "he-morph-004", "semantic_domains": ["logos.vitality.animate", "logos.core", "vitality", "existence.manifestation", "being.animate"], "morphology_tags": ["noun", "masculine", "plural", "root:\u05d7\u05d9\u05d4", "triliteral:H-Y-H"]}
{"entry_id": "he-005", "surface": "\u05d0\u05de\u05ea", "lemma": "\u05d0\u05de\u05ea", "language": "he", "pos": "NOUN", "morphology_id": "he-morph-005", "semantic_domains": ["logos.aletheia.verity", "logos.core", "epistemic.ground", "knowledge.ground", "divine.revelation"], "morphology_tags": ["noun", "feminine", "singular", "root:\u05d0\u05de\u05df", "triliteral:A-M-N"]}
{"entry_id": "he-006", "surface": "\u05e8\u05d5\u05d7", "lemma": "\u05e8\u05d5\u05d7", "language": "he", "pos": "NOUN", "morphology_id": "he-morph-006", "semantic_domains": ["logos.pneuma.breath", "logos.core", "breath", "existence.animating", "logos.presence"], "morphology_tags": ["noun", "feminine", "singular", "root:\u05e8\u05d5\u05d7", "triliteral:R-W-H"]}
{"entry_id": "he-007", "surface": "\u05d1\u05e8\u05d0", "lemma": "\u05d1\u05e8\u05d0", "language": "he", "pos": "VERB", "morphology_id": "he-morph-007", "semantic_domains": ["logos.ktizo.formation", "logos.core", "formation", "logos.genesis", "divine.agency"], "morphology_tags": ["verb", "qal", "perfect", "third_person", "masculine", "singular", "root:\u05d1\u05e8\u05d0", "triliteral:B-R-A"]}
{"entry_id": "he-002", "surface": "\u05e8\u05d0\u05e9\u05d9\u05ea", "lemma": "\u05e8\u05d0\u05e9\u05d9\u05ea", "language": "he", "pos": "NOUN", "morphology_id": "he-morph-002", "semantic_domains": ["logos.genesis.origin", "logos.core", "origin", "existence.ground", "temporal.absolute"], "morphology_tags": ["noun", "feminine", "singular"]}
{"entry_id": "he-003", "surface": "\u05d0\u05d5\u05e8", "lemma": "\u05d0\u05d5\u05e8", "language": "he", "pos": "NOUN", "morphology_id": "he-morph-003", "semantic_domains": ["logos.illumination.photon", "logos.core", "logos.illumination", "divine.revelation", "knowledge.clarity"], "morphology_tags": ["noun", "masculine", "singular"]}
{"entry_id": "he-004", "surface": "\u05d7\u05d9\u05d9\u05dd", "lemma": "\u05d7\u05d9\u05d9\u05dd", "language": "he", "pos": "NOUN", "morphology_id": "he-morph-004", "semantic_domains": ["logos.vitality.animate", "logos.core", "vitality", "existence.manifestation", "being.animate"], "morphology_tags": ["noun", "masculine", "plural"]}
{"entry_id": "he-005", "surface": "\u05d0\u05de\u05ea", "lemma": "\u05d0\u05de\u05ea", "language": "he", "pos": "NOUN", "morphology_id": "he-morph-005", "semantic_domains": ["logos.aletheia.verity", "logos.core", "epistemic.ground", "knowledge.ground", "divine.revelation"], "morphology_tags": ["noun", "feminine", "singular"]}
{"entry_id": "he-006", "surface": "\u05e8\u05d5\u05d7", "lemma": "\u05e8\u05d5\u05d7", "language": "he", "pos": "NOUN", "morphology_id": "he-morph-006", "semantic_domains": ["logos.pneuma.breath", "logos.core", "breath", "existence.animating", "logos.presence"], "morphology_tags": ["noun", "feminine", "singular"]}
{"entry_id": "he-007", "surface": "\u05d1\u05e8\u05d0", "lemma": "\u05d1\u05e8\u05d0", "language": "he", "pos": "VERB", "morphology_id": "he-morph-007", "semantic_domains": ["logos.ktizo.formation", "logos.core", "formation", "logos.genesis", "divine.agency"], "morphology_tags": ["verb", "third_person", "masculine", "singular"]}

2
language_packs/data/he_logos_micro_v1/manifest.json
View file

@ -6,7 +6,7 @@
"normalization_policy": "unitize_versor",
"source_manifest": "he_logos_micro_v1.lexicon.jsonl",
"determinism_class": "D0",
"checksum": "a419d50f0bd9e326969e358d9f83ac7c56f0ed82cf1c3caba05cda7e06c5fbe0",
"checksum": "2740950b37671b0a2f7de117e3e8ff5dfde049abcea32a6a2a97cb5b199d7b61",
"version": "1.0.0",
"gate_engaged": true,
"oov_policy": "fail_closed"