feat(packs): en_core_spatial_v1 — spatial vocabulary (24 lemmas)
Workstream 1 sixth pack. Closes the spatial-vocabulary gap. Prior
packs had zero coverage of here/there, location nouns, or spatial
prepositions.
Pack composition (24 entries — 7 ADV / 8 ADP / 9 NOUN):
spatial.deictic.* here there (2 ADV)
spatial.direction.* forward backward left up down (5 ADV)
spatial.relation.* near far above below inside outside
between beyond (8 ADP)
spatial.noun.* place location area region space
end top bottom side (9 NOUN)
``right`` was deliberately omitted — en_core_attitude_v1 already owns
it as evaluative.positive, and first-match-wins resolution preserves
that claim. A regression test pins this invariant explicitly.
Files: lexicon.jsonl / manifest.json + 12 contract tests.
Verification: full lane 2204 passed / 2 skipped / 0 failed.
Cognition eval byte-identical both splits.
This commit is contained in:
Shay
parent
891ffa8969
commit
390c2834f8
5 changed files with 182 additions and 0 deletions
|
|
@ -47,6 +47,7 @@ DEFAULT_RESOLVABLE_PACK_IDS: tuple[str, ...] = (
|
|||
"en_core_temporal_v1",
|
||||
"en_core_action_v1",
|
||||
"en_core_quantitative_v1",
|
||||
"en_core_spatial_v1",
|
||||
"en_core_relations_v1",
|
||||
"en_core_relations_v2",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ class RuntimeConfig:
|
|||
"en_core_temporal_v1",
|
||||
"en_core_action_v1",
|
||||
"en_core_quantitative_v1",
|
||||
"en_core_spatial_v1",
|
||||
"en_core_relations_v1",
|
||||
"en_core_relations_v2",
|
||||
"he_logos_micro_v1",
|
||||
|
|
|
|||
24
language_packs/data/en_core_spatial_v1/lexicon.jsonl
Normal file
24
language_packs/data/en_core_spatial_v1/lexicon.jsonl
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
{"entry_id":"en-core-spatial-001","surface":"here","lemma":"here","language":"en","pos":"ADV","semantic_domains":["spatial.deictic.proximal","spatial.deictic","cognition.location"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-002","surface":"there","lemma":"there","language":"en","pos":"ADV","semantic_domains":["spatial.deictic.distal","spatial.deictic","cognition.location"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-003","surface":"forward","lemma":"forward","language":"en","pos":"ADV","semantic_domains":["spatial.direction.forward","spatial.direction","cognition.direction"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-004","surface":"backward","lemma":"backward","language":"en","pos":"ADV","semantic_domains":["spatial.direction.backward","spatial.direction","cognition.direction"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-005","surface":"left","lemma":"left","language":"en","pos":"ADV","semantic_domains":["spatial.direction.lateral","spatial.direction","cognition.direction"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-006","surface":"up","lemma":"up","language":"en","pos":"ADV","semantic_domains":["spatial.direction.vertical_up","spatial.direction","cognition.direction"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-007","surface":"down","lemma":"down","language":"en","pos":"ADV","semantic_domains":["spatial.direction.vertical_down","spatial.direction","cognition.direction"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-008","surface":"near","lemma":"near","language":"en","pos":"ADP","semantic_domains":["spatial.relation.proximity","spatial.relation","cognition.location"],"morphology_tags":["adposition"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-009","surface":"far","lemma":"far","language":"en","pos":"ADP","semantic_domains":["spatial.relation.distance","spatial.relation","cognition.location"],"morphology_tags":["adposition"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-010","surface":"above","lemma":"above","language":"en","pos":"ADP","semantic_domains":["spatial.relation.vertical_above","spatial.relation","cognition.location"],"morphology_tags":["adposition"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-011","surface":"below","lemma":"below","language":"en","pos":"ADP","semantic_domains":["spatial.relation.vertical_below","spatial.relation","cognition.location"],"morphology_tags":["adposition"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-012","surface":"inside","lemma":"inside","language":"en","pos":"ADP","semantic_domains":["spatial.relation.containment_inside","spatial.relation","cognition.location"],"morphology_tags":["adposition"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-013","surface":"outside","lemma":"outside","language":"en","pos":"ADP","semantic_domains":["spatial.relation.containment_outside","spatial.relation","cognition.location"],"morphology_tags":["adposition"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-014","surface":"between","lemma":"between","language":"en","pos":"ADP","semantic_domains":["spatial.relation.between","spatial.relation","cognition.location"],"morphology_tags":["adposition"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-015","surface":"beyond","lemma":"beyond","language":"en","pos":"ADP","semantic_domains":["spatial.relation.distance","spatial.relation","cognition.location"],"morphology_tags":["adposition"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-016","surface":"place","lemma":"place","language":"en","pos":"NOUN","semantic_domains":["spatial.noun.location","spatial.noun","cognition.location"],"morphology_tags":["noun"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-017","surface":"location","lemma":"location","language":"en","pos":"NOUN","semantic_domains":["spatial.noun.location","spatial.noun","cognition.location"],"morphology_tags":["noun"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-018","surface":"area","lemma":"area","language":"en","pos":"NOUN","semantic_domains":["spatial.noun.region","spatial.noun","cognition.location"],"morphology_tags":["noun"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-019","surface":"region","lemma":"region","language":"en","pos":"NOUN","semantic_domains":["spatial.noun.region","spatial.noun","cognition.location"],"morphology_tags":["noun"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-020","surface":"space","lemma":"space","language":"en","pos":"NOUN","semantic_domains":["spatial.noun.location","spatial.noun","cognition.location"],"morphology_tags":["noun"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-021","surface":"end","lemma":"end","language":"en","pos":"NOUN","semantic_domains":["spatial.noun.boundary_end","spatial.noun","cognition.boundary"],"morphology_tags":["noun"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-022","surface":"top","lemma":"top","language":"en","pos":"NOUN","semantic_domains":["spatial.noun.boundary_top","spatial.noun","cognition.boundary"],"morphology_tags":["noun"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-023","surface":"bottom","lemma":"bottom","language":"en","pos":"NOUN","semantic_domains":["spatial.noun.boundary_bottom","spatial.noun","cognition.boundary"],"morphology_tags":["noun"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
{"entry_id":"en-core-spatial-024","surface":"side","lemma":"side","language":"en","pos":"NOUN","semantic_domains":["spatial.noun.boundary_side","spatial.noun","cognition.boundary"],"morphology_tags":["noun"],"provenance_ids":["seed:core_spatial_v1"]}
|
||||
13
language_packs/data/en_core_spatial_v1/manifest.json
Normal file
13
language_packs/data/en_core_spatial_v1/manifest.json
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"pack_id": "en_core_spatial_v1",
|
||||
"language": "en",
|
||||
"role": "operational_base",
|
||||
"script": "Latin",
|
||||
"normalization_policy": "unitize_versor",
|
||||
"source_manifest": "en_core_spatial_v1.lexicon.jsonl",
|
||||
"determinism_class": "D0",
|
||||
"checksum": "f3ab77403f4f764f3828d0af6b7c6bcf54cef4137f73c577206cb522706ac14c",
|
||||
"version": "1.0.0",
|
||||
"gate_engaged": true,
|
||||
"oov_policy": "tagged_fallback"
|
||||
}
|
||||
143
tests/test_en_core_spatial_v1_pack.py
Normal file
143
tests/test_en_core_spatial_v1_pack.py
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
"""``en_core_spatial_v1`` — spatial vocabulary pack tests.
|
||||
|
||||
Spatial vocabulary had zero coverage in any prior pack — *here*,
|
||||
*there*, *near*, *above*, *place*, *location* all fell through to
|
||||
OOV. 24 entries across:
|
||||
|
||||
- spatial.deictic.* — here, there
|
||||
- spatial.direction.* — forward, backward, left, up, down (5 ADV)
|
||||
- spatial.relation.* — near, far, above, below, inside, outside,
|
||||
between, beyond (8 ADP)
|
||||
- spatial.noun.* — place, location, area, region, space, end,
|
||||
top, bottom, side (9 NOUN)
|
||||
|
||||
POS mix (7 ADV / 8 ADP / 9 NOUN). The composer is POS-agnostic.
|
||||
|
||||
Contracts pinned: checksum-verified load, primary-domain namespace
|
||||
``spatial.*``, no collision with the 8 prior packs (notably ``right``
|
||||
is in en_core_attitude_v1 as evaluative.positive, so spatial-direction
|
||||
``right`` was deliberately omitted), mounted by default, registered
|
||||
after en_core_quantitative_v1, prior-pack resolution unchanged.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from chat.pack_resolver import DEFAULT_RESOLVABLE_PACK_IDS, resolve_lemma
|
||||
from core.config import RuntimeConfig
|
||||
from language_packs.compiler import load_pack
|
||||
|
||||
|
||||
PACK_ID = "en_core_spatial_v1"
|
||||
_PACK_ROOT = Path(__file__).resolve().parent.parent / "language_packs" / "data" / PACK_ID
|
||||
|
||||
EXPECTED_TOTAL = 24
|
||||
EXPECTED_POS_COUNTS = {"ADV": 7, "ADP": 8, "NOUN": 9}
|
||||
|
||||
EXPECTED_LEMMAS: tuple[str, ...] = (
|
||||
"here", "there", "forward", "backward", "left", "up", "down",
|
||||
"near", "far", "above", "below", "inside", "outside", "between", "beyond",
|
||||
"place", "location", "area", "region", "space", "end", "top", "bottom", "side",
|
||||
)
|
||||
|
||||
|
||||
def _read_lexicon() -> list[dict]:
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in (_PACK_ROOT / "lexicon.jsonl").read_text("utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
|
||||
|
||||
def test_pack_loads_with_matching_checksum() -> None:
|
||||
manifest, manifold = load_pack(PACK_ID)
|
||||
assert manifest.pack_id == PACK_ID
|
||||
assert len(manifest.checksum) == 64
|
||||
assert len(manifold) == EXPECTED_TOTAL
|
||||
|
||||
|
||||
def test_pos_distribution_matches_design() -> None:
|
||||
pos_counts = Counter(e["pos"] for e in _read_lexicon())
|
||||
assert dict(pos_counts) == EXPECTED_POS_COUNTS
|
||||
|
||||
|
||||
def test_all_expected_lemmas_present() -> None:
|
||||
_, manifold = load_pack(PACK_ID)
|
||||
assert {manifold.get_word_at(i) for i in range(len(manifold))} == set(EXPECTED_LEMMAS)
|
||||
|
||||
|
||||
def test_every_entry_has_spatial_namespace_primary_domain() -> None:
|
||||
for entry in _read_lexicon():
|
||||
assert entry["semantic_domains"][0].startswith("spatial."), entry
|
||||
|
||||
|
||||
def test_no_collision_with_prior_packs() -> None:
|
||||
prior: set[str] = set()
|
||||
for pack in (
|
||||
"en_core_cognition_v1", "en_core_meta_v1", "en_core_attitude_v1",
|
||||
"en_core_temporal_v1", "en_core_action_v1", "en_core_quantitative_v1",
|
||||
"en_core_relations_v1", "en_core_relations_v2",
|
||||
):
|
||||
for line in (_PACK_ROOT.parent / pack / "lexicon.jsonl").read_text("utf-8").splitlines():
|
||||
if line.strip():
|
||||
prior.add(json.loads(line)["lemma"].lower())
|
||||
for entry in _read_lexicon():
|
||||
assert entry["lemma"].lower() not in prior, entry
|
||||
|
||||
|
||||
def test_provenance_is_seed_core_spatial_v1() -> None:
|
||||
for entry in _read_lexicon():
|
||||
assert entry["provenance_ids"] == ["seed:core_spatial_v1"], entry
|
||||
|
||||
|
||||
def test_entry_ids_contiguous_and_zero_padded() -> None:
|
||||
entries = sorted(_read_lexicon(), key=lambda d: d["entry_id"])
|
||||
for i, entry in enumerate(entries, start=1):
|
||||
assert entry["entry_id"] == f"en-core-spatial-{i:03d}", entry["entry_id"]
|
||||
|
||||
|
||||
def test_pack_mounted_in_default_runtime_config() -> None:
|
||||
assert PACK_ID in RuntimeConfig().input_packs
|
||||
|
||||
|
||||
def test_pack_registered_after_prior_content_packs() -> None:
|
||||
assert PACK_ID in DEFAULT_RESOLVABLE_PACK_IDS
|
||||
for earlier in (
|
||||
"en_core_cognition_v1", "en_core_meta_v1", "en_core_attitude_v1",
|
||||
"en_core_temporal_v1", "en_core_action_v1", "en_core_quantitative_v1",
|
||||
):
|
||||
assert DEFAULT_RESOLVABLE_PACK_IDS.index(earlier) < DEFAULT_RESOLVABLE_PACK_IDS.index(PACK_ID)
|
||||
|
||||
|
||||
def test_resolver_routes_spatial_lemmas_to_this_pack() -> None:
|
||||
for lemma in EXPECTED_LEMMAS:
|
||||
resolved = resolve_lemma(lemma)
|
||||
assert resolved is not None and resolved[0] == PACK_ID, (
|
||||
f"{lemma!r} resolved to {resolved}"
|
||||
)
|
||||
assert resolved[1][0].startswith("spatial.")
|
||||
|
||||
|
||||
def test_right_remains_in_attitude_pack_not_spatial() -> None:
|
||||
"""``right`` was deliberately excluded from spatial-direction
|
||||
because en_core_attitude_v1 already owns it as evaluative.positive.
|
||||
Verify the first-match-wins resolver preserves attitude's claim."""
|
||||
resolved = resolve_lemma("right")
|
||||
assert resolved is not None and resolved[0] == "en_core_attitude_v1"
|
||||
assert resolved[1][0].startswith("attitude.evaluative.")
|
||||
|
||||
|
||||
def test_prior_pack_lemma_resolution_unchanged() -> None:
|
||||
for lemma, expected in (
|
||||
("truth", "en_core_cognition_v1"),
|
||||
("doubt", "en_core_meta_v1"),
|
||||
("true", "en_core_attitude_v1"),
|
||||
("now", "en_core_temporal_v1"),
|
||||
("do", "en_core_action_v1"),
|
||||
("all", "en_core_quantitative_v1"),
|
||||
):
|
||||
resolved = resolve_lemma(lemma)
|
||||
assert resolved is not None and resolved[0] == expected
|
||||
Loading…
Reference in a new issue