From 390c2834f8a395095adba8b10ec3fa6602a1483f Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 18 May 2026 21:38:20 -0700 Subject: [PATCH] =?UTF-8?q?feat(packs):=20en=5Fcore=5Fspatial=5Fv1=20?= =?UTF-8?q?=E2=80=94=20spatial=20vocabulary=20(24=20lemmas)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Workstream 1 sixth pack. Closes the spatial-vocabulary gap. Prior packs had zero coverage of here/there, location nouns, or spatial prepositions. Pack composition (24 entries — 7 ADV / 8 ADP / 9 NOUN): spatial.deictic.* here there (2 ADV) spatial.direction.* forward backward left up down (5 ADV) spatial.relation.* near far above below inside outside between beyond (8 ADP) spatial.noun.* place location area region space end top bottom side (9 NOUN) ``right`` was deliberately omitted — en_core_attitude_v1 already owns it as evaluative.positive, and first-match-wins resolution preserves that claim. A regression test pins this invariant explicitly. Files: lexicon.jsonl / manifest.json + 12 contract tests. Verification: full lane 2204 passed / 2 skipped / 0 failed. Cognition eval byte-identical both splits. --- chat/pack_resolver.py | 1 + core/config.py | 1 + .../data/en_core_spatial_v1/lexicon.jsonl | 24 +++ .../data/en_core_spatial_v1/manifest.json | 13 ++ tests/test_en_core_spatial_v1_pack.py | 143 ++++++++++++++++++ 5 files changed, 182 insertions(+) create mode 100644 language_packs/data/en_core_spatial_v1/lexicon.jsonl create mode 100644 language_packs/data/en_core_spatial_v1/manifest.json create mode 100644 tests/test_en_core_spatial_v1_pack.py diff --git a/chat/pack_resolver.py b/chat/pack_resolver.py index 7e06600b..388ecfbf 100644 --- a/chat/pack_resolver.py +++ b/chat/pack_resolver.py @@ -47,6 +47,7 @@ DEFAULT_RESOLVABLE_PACK_IDS: tuple[str, ...] = ( "en_core_temporal_v1", "en_core_action_v1", "en_core_quantitative_v1", + "en_core_spatial_v1", "en_core_relations_v1", "en_core_relations_v2", ) diff --git a/core/config.py b/core/config.py index 9338c4be..33acb5cb 100644 --- a/core/config.py +++ b/core/config.py @@ -18,6 +18,7 @@ class RuntimeConfig: "en_core_temporal_v1", "en_core_action_v1", "en_core_quantitative_v1", + "en_core_spatial_v1", "en_core_relations_v1", "en_core_relations_v2", "he_logos_micro_v1", diff --git a/language_packs/data/en_core_spatial_v1/lexicon.jsonl b/language_packs/data/en_core_spatial_v1/lexicon.jsonl new file mode 100644 index 00000000..e09d8be6 --- /dev/null +++ b/language_packs/data/en_core_spatial_v1/lexicon.jsonl @@ -0,0 +1,24 @@ +{"entry_id":"en-core-spatial-001","surface":"here","lemma":"here","language":"en","pos":"ADV","semantic_domains":["spatial.deictic.proximal","spatial.deictic","cognition.location"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-002","surface":"there","lemma":"there","language":"en","pos":"ADV","semantic_domains":["spatial.deictic.distal","spatial.deictic","cognition.location"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-003","surface":"forward","lemma":"forward","language":"en","pos":"ADV","semantic_domains":["spatial.direction.forward","spatial.direction","cognition.direction"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-004","surface":"backward","lemma":"backward","language":"en","pos":"ADV","semantic_domains":["spatial.direction.backward","spatial.direction","cognition.direction"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-005","surface":"left","lemma":"left","language":"en","pos":"ADV","semantic_domains":["spatial.direction.lateral","spatial.direction","cognition.direction"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-006","surface":"up","lemma":"up","language":"en","pos":"ADV","semantic_domains":["spatial.direction.vertical_up","spatial.direction","cognition.direction"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-007","surface":"down","lemma":"down","language":"en","pos":"ADV","semantic_domains":["spatial.direction.vertical_down","spatial.direction","cognition.direction"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-008","surface":"near","lemma":"near","language":"en","pos":"ADP","semantic_domains":["spatial.relation.proximity","spatial.relation","cognition.location"],"morphology_tags":["adposition"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-009","surface":"far","lemma":"far","language":"en","pos":"ADP","semantic_domains":["spatial.relation.distance","spatial.relation","cognition.location"],"morphology_tags":["adposition"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-010","surface":"above","lemma":"above","language":"en","pos":"ADP","semantic_domains":["spatial.relation.vertical_above","spatial.relation","cognition.location"],"morphology_tags":["adposition"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-011","surface":"below","lemma":"below","language":"en","pos":"ADP","semantic_domains":["spatial.relation.vertical_below","spatial.relation","cognition.location"],"morphology_tags":["adposition"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-012","surface":"inside","lemma":"inside","language":"en","pos":"ADP","semantic_domains":["spatial.relation.containment_inside","spatial.relation","cognition.location"],"morphology_tags":["adposition"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-013","surface":"outside","lemma":"outside","language":"en","pos":"ADP","semantic_domains":["spatial.relation.containment_outside","spatial.relation","cognition.location"],"morphology_tags":["adposition"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-014","surface":"between","lemma":"between","language":"en","pos":"ADP","semantic_domains":["spatial.relation.between","spatial.relation","cognition.location"],"morphology_tags":["adposition"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-015","surface":"beyond","lemma":"beyond","language":"en","pos":"ADP","semantic_domains":["spatial.relation.distance","spatial.relation","cognition.location"],"morphology_tags":["adposition"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-016","surface":"place","lemma":"place","language":"en","pos":"NOUN","semantic_domains":["spatial.noun.location","spatial.noun","cognition.location"],"morphology_tags":["noun"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-017","surface":"location","lemma":"location","language":"en","pos":"NOUN","semantic_domains":["spatial.noun.location","spatial.noun","cognition.location"],"morphology_tags":["noun"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-018","surface":"area","lemma":"area","language":"en","pos":"NOUN","semantic_domains":["spatial.noun.region","spatial.noun","cognition.location"],"morphology_tags":["noun"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-019","surface":"region","lemma":"region","language":"en","pos":"NOUN","semantic_domains":["spatial.noun.region","spatial.noun","cognition.location"],"morphology_tags":["noun"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-020","surface":"space","lemma":"space","language":"en","pos":"NOUN","semantic_domains":["spatial.noun.location","spatial.noun","cognition.location"],"morphology_tags":["noun"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-021","surface":"end","lemma":"end","language":"en","pos":"NOUN","semantic_domains":["spatial.noun.boundary_end","spatial.noun","cognition.boundary"],"morphology_tags":["noun"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-022","surface":"top","lemma":"top","language":"en","pos":"NOUN","semantic_domains":["spatial.noun.boundary_top","spatial.noun","cognition.boundary"],"morphology_tags":["noun"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-023","surface":"bottom","lemma":"bottom","language":"en","pos":"NOUN","semantic_domains":["spatial.noun.boundary_bottom","spatial.noun","cognition.boundary"],"morphology_tags":["noun"],"provenance_ids":["seed:core_spatial_v1"]} +{"entry_id":"en-core-spatial-024","surface":"side","lemma":"side","language":"en","pos":"NOUN","semantic_domains":["spatial.noun.boundary_side","spatial.noun","cognition.boundary"],"morphology_tags":["noun"],"provenance_ids":["seed:core_spatial_v1"]} diff --git a/language_packs/data/en_core_spatial_v1/manifest.json b/language_packs/data/en_core_spatial_v1/manifest.json new file mode 100644 index 00000000..fe1f3d22 --- /dev/null +++ b/language_packs/data/en_core_spatial_v1/manifest.json @@ -0,0 +1,13 @@ +{ + "pack_id": "en_core_spatial_v1", + "language": "en", + "role": "operational_base", + "script": "Latin", + "normalization_policy": "unitize_versor", + "source_manifest": "en_core_spatial_v1.lexicon.jsonl", + "determinism_class": "D0", + "checksum": "f3ab77403f4f764f3828d0af6b7c6bcf54cef4137f73c577206cb522706ac14c", + "version": "1.0.0", + "gate_engaged": true, + "oov_policy": "tagged_fallback" +} diff --git a/tests/test_en_core_spatial_v1_pack.py b/tests/test_en_core_spatial_v1_pack.py new file mode 100644 index 00000000..994cad5e --- /dev/null +++ b/tests/test_en_core_spatial_v1_pack.py @@ -0,0 +1,143 @@ +"""``en_core_spatial_v1`` — spatial vocabulary pack tests. + +Spatial vocabulary had zero coverage in any prior pack — *here*, +*there*, *near*, *above*, *place*, *location* all fell through to +OOV. 24 entries across: + + - spatial.deictic.* — here, there + - spatial.direction.* — forward, backward, left, up, down (5 ADV) + - spatial.relation.* — near, far, above, below, inside, outside, + between, beyond (8 ADP) + - spatial.noun.* — place, location, area, region, space, end, + top, bottom, side (9 NOUN) + +POS mix (7 ADV / 8 ADP / 9 NOUN). The composer is POS-agnostic. + +Contracts pinned: checksum-verified load, primary-domain namespace +``spatial.*``, no collision with the 8 prior packs (notably ``right`` +is in en_core_attitude_v1 as evaluative.positive, so spatial-direction +``right`` was deliberately omitted), mounted by default, registered +after en_core_quantitative_v1, prior-pack resolution unchanged. +""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path + +from chat.pack_resolver import DEFAULT_RESOLVABLE_PACK_IDS, resolve_lemma +from core.config import RuntimeConfig +from language_packs.compiler import load_pack + + +PACK_ID = "en_core_spatial_v1" +_PACK_ROOT = Path(__file__).resolve().parent.parent / "language_packs" / "data" / PACK_ID + +EXPECTED_TOTAL = 24 +EXPECTED_POS_COUNTS = {"ADV": 7, "ADP": 8, "NOUN": 9} + +EXPECTED_LEMMAS: tuple[str, ...] = ( + "here", "there", "forward", "backward", "left", "up", "down", + "near", "far", "above", "below", "inside", "outside", "between", "beyond", + "place", "location", "area", "region", "space", "end", "top", "bottom", "side", +) + + +def _read_lexicon() -> list[dict]: + return [ + json.loads(line) + for line in (_PACK_ROOT / "lexicon.jsonl").read_text("utf-8").splitlines() + if line.strip() + ] + + +def test_pack_loads_with_matching_checksum() -> None: + manifest, manifold = load_pack(PACK_ID) + assert manifest.pack_id == PACK_ID + assert len(manifest.checksum) == 64 + assert len(manifold) == EXPECTED_TOTAL + + +def test_pos_distribution_matches_design() -> None: + pos_counts = Counter(e["pos"] for e in _read_lexicon()) + assert dict(pos_counts) == EXPECTED_POS_COUNTS + + +def test_all_expected_lemmas_present() -> None: + _, manifold = load_pack(PACK_ID) + assert {manifold.get_word_at(i) for i in range(len(manifold))} == set(EXPECTED_LEMMAS) + + +def test_every_entry_has_spatial_namespace_primary_domain() -> None: + for entry in _read_lexicon(): + assert entry["semantic_domains"][0].startswith("spatial."), entry + + +def test_no_collision_with_prior_packs() -> None: + prior: set[str] = set() + for pack in ( + "en_core_cognition_v1", "en_core_meta_v1", "en_core_attitude_v1", + "en_core_temporal_v1", "en_core_action_v1", "en_core_quantitative_v1", + "en_core_relations_v1", "en_core_relations_v2", + ): + for line in (_PACK_ROOT.parent / pack / "lexicon.jsonl").read_text("utf-8").splitlines(): + if line.strip(): + prior.add(json.loads(line)["lemma"].lower()) + for entry in _read_lexicon(): + assert entry["lemma"].lower() not in prior, entry + + +def test_provenance_is_seed_core_spatial_v1() -> None: + for entry in _read_lexicon(): + assert entry["provenance_ids"] == ["seed:core_spatial_v1"], entry + + +def test_entry_ids_contiguous_and_zero_padded() -> None: + entries = sorted(_read_lexicon(), key=lambda d: d["entry_id"]) + for i, entry in enumerate(entries, start=1): + assert entry["entry_id"] == f"en-core-spatial-{i:03d}", entry["entry_id"] + + +def test_pack_mounted_in_default_runtime_config() -> None: + assert PACK_ID in RuntimeConfig().input_packs + + +def test_pack_registered_after_prior_content_packs() -> None: + assert PACK_ID in DEFAULT_RESOLVABLE_PACK_IDS + for earlier in ( + "en_core_cognition_v1", "en_core_meta_v1", "en_core_attitude_v1", + "en_core_temporal_v1", "en_core_action_v1", "en_core_quantitative_v1", + ): + assert DEFAULT_RESOLVABLE_PACK_IDS.index(earlier) < DEFAULT_RESOLVABLE_PACK_IDS.index(PACK_ID) + + +def test_resolver_routes_spatial_lemmas_to_this_pack() -> None: + for lemma in EXPECTED_LEMMAS: + resolved = resolve_lemma(lemma) + assert resolved is not None and resolved[0] == PACK_ID, ( + f"{lemma!r} resolved to {resolved}" + ) + assert resolved[1][0].startswith("spatial.") + + +def test_right_remains_in_attitude_pack_not_spatial() -> None: + """``right`` was deliberately excluded from spatial-direction + because en_core_attitude_v1 already owns it as evaluative.positive. + Verify the first-match-wins resolver preserves attitude's claim.""" + resolved = resolve_lemma("right") + assert resolved is not None and resolved[0] == "en_core_attitude_v1" + assert resolved[1][0].startswith("attitude.evaluative.") + + +def test_prior_pack_lemma_resolution_unchanged() -> None: + for lemma, expected in ( + ("truth", "en_core_cognition_v1"), + ("doubt", "en_core_meta_v1"), + ("true", "en_core_attitude_v1"), + ("now", "en_core_temporal_v1"), + ("do", "en_core_action_v1"), + ("all", "en_core_quantitative_v1"), + ): + resolved = resolve_lemma(lemma) + assert resolved is not None and resolved[0] == expected