From 1c7408f7d086bc8c539f6843c0b1133752d2f71c Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 18 May 2026 19:59:20 -0700 Subject: [PATCH] =?UTF-8?q?feat(packs):=20en=5Fcore=5Ftemporal=5Fv1=20?= =?UTF-8?q?=E2=80=94=20temporal=20pack=20(28=20lemmas)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Workstream 1 third pack. Closes the temporal-vocabulary gap — prior to this pack zero time/sequence/aspect terms existed in any mounted English pack, so queries about *when*, *before*, *after*, *now*, *future*, *past* all fell through to OOV. Pack composition (28 entries, mixed POS — 12 ADV / 9 NOUN / 5 ADP / 1 SCONJ / 1 ADJ): temporal.deictic.* (10 ADV) now today tomorrow yesterday soon later recently eventually currently formerly temporal.relative.* (9 mix) before after during while until since ago prior henceforth temporal.noun.* (9 NOUN) moment period duration instant era future past present time The pack composer is POS-agnostic — surface composition uses the ratified ``semantic_domains`` list rather than the POS tag. Mixed-POS entries surface identically to noun/verb entries. Files: language_packs/data/en_core_temporal_v1/ lexicon.jsonl — 28 entries, SHA-256 checksum-sealed manifest.json — operational_base / D0 / checksum-verified chat/pack_resolver.py Appended to DEFAULT_RESOLVABLE_PACK_IDS after attitude. core/config.py Added to RuntimeConfig.input_packs default mount. tests/test_en_core_temporal_v1_pack.py 11 contract tests: checksum, POS-distribution invariant, primary- domain namespace, no-collision regression gate against all 5 prior packs, contiguous entry_ids, mounted-by-default, resolver-order invariant, routing correctness, and prior-pack resolution unchanged. Authoring: Three parallel subagents — deictic / relative / nouns. Strict exemplar + forbidden-lemma list against all 5 prior packs. Verification: Full lane: 2170 passed, 2 skipped, 0 failed (+11 new tests). Cognition eval byte-identical on both splits. Live composer probes confirm every temporal lemma surfaces deterministically from en_core_temporal_v1. --- chat/pack_resolver.py | 1 + core/config.py | 1 + .../data/en_core_temporal_v1/lexicon.jsonl | 28 +++ .../data/en_core_temporal_v1/manifest.json | 13 ++ tests/test_en_core_temporal_v1_pack.py | 162 ++++++++++++++++++ 5 files changed, 205 insertions(+) create mode 100644 language_packs/data/en_core_temporal_v1/lexicon.jsonl create mode 100644 language_packs/data/en_core_temporal_v1/manifest.json create mode 100644 tests/test_en_core_temporal_v1_pack.py diff --git a/chat/pack_resolver.py b/chat/pack_resolver.py index 0785e6e5..cc1d0a79 100644 --- a/chat/pack_resolver.py +++ b/chat/pack_resolver.py @@ -44,6 +44,7 @@ DEFAULT_RESOLVABLE_PACK_IDS: tuple[str, ...] = ( "en_core_cognition_v1", "en_core_meta_v1", "en_core_attitude_v1", + "en_core_temporal_v1", "en_core_relations_v1", "en_core_relations_v2", ) diff --git a/core/config.py b/core/config.py index 577c9524..60da1bc7 100644 --- a/core/config.py +++ b/core/config.py @@ -15,6 +15,7 @@ class RuntimeConfig: "en_core_cognition_v1", "en_core_meta_v1", "en_core_attitude_v1", + "en_core_temporal_v1", "en_core_relations_v1", "en_core_relations_v2", "he_logos_micro_v1", diff --git a/language_packs/data/en_core_temporal_v1/lexicon.jsonl b/language_packs/data/en_core_temporal_v1/lexicon.jsonl new file mode 100644 index 00000000..0b2690ee --- /dev/null +++ b/language_packs/data/en_core_temporal_v1/lexicon.jsonl @@ -0,0 +1,28 @@ +{"entry_id":"en-core-temporal-001","surface":"now","lemma":"now","language":"en","pos":"ADV","semantic_domains":["temporal.deictic.present","temporal.deictic","cognition.time"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-002","surface":"today","lemma":"today","language":"en","pos":"ADV","semantic_domains":["temporal.deictic.present","temporal.deictic","cognition.time"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-003","surface":"currently","lemma":"currently","language":"en","pos":"ADV","semantic_domains":["temporal.deictic.present","temporal.deictic","cognition.time"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-004","surface":"tomorrow","lemma":"tomorrow","language":"en","pos":"ADV","semantic_domains":["temporal.deictic.future","temporal.deictic","cognition.time"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-005","surface":"soon","lemma":"soon","language":"en","pos":"ADV","semantic_domains":["temporal.deictic.future","temporal.deictic","cognition.time"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-006","surface":"later","lemma":"later","language":"en","pos":"ADV","semantic_domains":["temporal.deictic.future","temporal.deictic","cognition.time"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-007","surface":"eventually","lemma":"eventually","language":"en","pos":"ADV","semantic_domains":["temporal.deictic.future","temporal.deictic","cognition.time"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-008","surface":"yesterday","lemma":"yesterday","language":"en","pos":"ADV","semantic_domains":["temporal.deictic.past","temporal.deictic","cognition.time"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-009","surface":"recently","lemma":"recently","language":"en","pos":"ADV","semantic_domains":["temporal.deictic.past","temporal.deictic","cognition.time"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-010","surface":"formerly","lemma":"formerly","language":"en","pos":"ADV","semantic_domains":["temporal.deictic.past","temporal.deictic","cognition.time"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-011","surface":"before","lemma":"before","language":"en","pos":"ADP","semantic_domains":["temporal.relative.anterior","temporal.relative","cognition.sequence"],"morphology_tags":["adposition"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-012","surface":"after","lemma":"after","language":"en","pos":"ADP","semantic_domains":["temporal.relative.posterior","temporal.relative","cognition.sequence"],"morphology_tags":["adposition"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-013","surface":"during","lemma":"during","language":"en","pos":"ADP","semantic_domains":["temporal.relative.contemporaneous","temporal.relative","cognition.sequence"],"morphology_tags":["adposition"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-014","surface":"while","lemma":"while","language":"en","pos":"SCONJ","semantic_domains":["temporal.relative.contemporaneous","temporal.relative","cognition.sequence"],"morphology_tags":["conjunction"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-015","surface":"until","lemma":"until","language":"en","pos":"ADP","semantic_domains":["temporal.relative.boundary_endpoint","temporal.relative","cognition.sequence"],"morphology_tags":["adposition"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-016","surface":"since","lemma":"since","language":"en","pos":"ADP","semantic_domains":["temporal.relative.boundary_origin","temporal.relative","cognition.sequence"],"morphology_tags":["adposition"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-017","surface":"ago","lemma":"ago","language":"en","pos":"ADV","semantic_domains":["temporal.relative.anterior","temporal.relative","cognition.sequence"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-018","surface":"prior","lemma":"prior","language":"en","pos":"ADJ","semantic_domains":["temporal.relative.anterior","temporal.relative","cognition.sequence"],"morphology_tags":["adjective"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-019","surface":"henceforth","lemma":"henceforth","language":"en","pos":"ADV","semantic_domains":["temporal.relative.posterior","temporal.relative","cognition.sequence"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-020","surface":"moment","lemma":"moment","language":"en","pos":"NOUN","semantic_domains":["temporal.noun.point","temporal.noun","cognition.time"],"morphology_tags":["noun"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-021","surface":"period","lemma":"period","language":"en","pos":"NOUN","semantic_domains":["temporal.noun.interval","temporal.noun","cognition.time"],"morphology_tags":["noun"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-022","surface":"duration","lemma":"duration","language":"en","pos":"NOUN","semantic_domains":["temporal.noun.interval","temporal.noun","cognition.time"],"morphology_tags":["noun"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-023","surface":"instant","lemma":"instant","language":"en","pos":"NOUN","semantic_domains":["temporal.noun.point","temporal.noun","cognition.time"],"morphology_tags":["noun"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-024","surface":"era","lemma":"era","language":"en","pos":"NOUN","semantic_domains":["temporal.noun.interval","temporal.noun","cognition.time"],"morphology_tags":["noun"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-025","surface":"future","lemma":"future","language":"en","pos":"NOUN","semantic_domains":["temporal.noun.tense","temporal.noun","cognition.time"],"morphology_tags":["noun"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-026","surface":"past","lemma":"past","language":"en","pos":"NOUN","semantic_domains":["temporal.noun.tense","temporal.noun","cognition.time"],"morphology_tags":["noun"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-027","surface":"present","lemma":"present","language":"en","pos":"NOUN","semantic_domains":["temporal.noun.tense","temporal.noun","cognition.time"],"morphology_tags":["noun"],"provenance_ids":["seed:core_temporal_v1"]} +{"entry_id":"en-core-temporal-028","surface":"time","lemma":"time","language":"en","pos":"NOUN","semantic_domains":["temporal.noun.dimension","temporal.noun","cognition.time"],"morphology_tags":["noun"],"provenance_ids":["seed:core_temporal_v1"]} diff --git a/language_packs/data/en_core_temporal_v1/manifest.json b/language_packs/data/en_core_temporal_v1/manifest.json new file mode 100644 index 00000000..67a2290e --- /dev/null +++ b/language_packs/data/en_core_temporal_v1/manifest.json @@ -0,0 +1,13 @@ +{ + "pack_id": "en_core_temporal_v1", + "language": "en", + "role": "operational_base", + "script": "Latin", + "normalization_policy": "unitize_versor", + "source_manifest": "en_core_temporal_v1.lexicon.jsonl", + "determinism_class": "D0", + "checksum": "cac13a815cbceb992647843e80bcc93b321fd5674d53d951ddd735036a18c705", + "version": "1.0.0", + "gate_engaged": true, + "oov_policy": "tagged_fallback" +} diff --git a/tests/test_en_core_temporal_v1_pack.py b/tests/test_en_core_temporal_v1_pack.py new file mode 100644 index 00000000..e62f6ab3 --- /dev/null +++ b/tests/test_en_core_temporal_v1_pack.py @@ -0,0 +1,162 @@ +"""``en_core_temporal_v1`` — temporal pack tests. + +The temporal pack is the first time-bearing pack in the runtime. Prior +to it, zero temporal vocabulary existed in any mounted English pack — +queries about *when*, *before*, *after*, *now*, *future*, *past* all +fell through to OOV. Pack contains 28 entries across three clusters: + + - temporal.deictic.* — now, today, soon, recently, ... (10 adverbs) + - temporal.relative.* — before, after, during, until, ago, prior, + henceforth, while, since (9 mixed POS) + - temporal.noun.* — moment, period, era, future, past, present, + duration, instant, time (9 nouns) + +Contracts pinned: + + - Checksum-verified load. + - 28 entries. + - Every entry's primary semantic_domain begins with ``temporal.``. + - Mixed POS distribution preserved (12 ADV / 9 NOUN / 5 ADP / 1 SCONJ + / 1 ADJ) — the resolver is POS-agnostic so this is exposed in the + underlying lexicon but does not affect resolution. + - No collision with any prior English pack. + - Mounted by default in ``RuntimeConfig.input_packs``. + - Registered in ``DEFAULT_RESOLVABLE_PACK_IDS`` after attitude pack; + cognition+meta+attitude resolution unchanged. +""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path + +from chat.pack_resolver import DEFAULT_RESOLVABLE_PACK_IDS, resolve_lemma +from core.config import RuntimeConfig +from language_packs.compiler import load_pack + + +PACK_ID = "en_core_temporal_v1" +_PACK_ROOT = Path(__file__).resolve().parent.parent / "language_packs" / "data" / PACK_ID + +EXPECTED_TOTAL = 28 +EXPECTED_POS_COUNTS = {"ADV": 12, "ADP": 5, "SCONJ": 1, "ADJ": 1, "NOUN": 9} + +EXPECTED_LEMMAS: tuple[str, ...] = ( + "now", "today", "tomorrow", "yesterday", "soon", "later", "recently", + "eventually", "currently", "formerly", + "before", "after", "during", "while", "until", "since", "ago", + "prior", "henceforth", + "moment", "period", "duration", "instant", "era", "future", "past", + "present", "time", +) + + +def _read_lexicon() -> list[dict]: + path = _PACK_ROOT / "lexicon.jsonl" + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def test_pack_loads_with_matching_checksum() -> None: + manifest, manifold = load_pack(PACK_ID) + assert manifest.pack_id == PACK_ID + assert len(manifest.checksum) == 64 + assert all(c in "0123456789abcdef" for c in manifest.checksum) + assert len(manifold) == EXPECTED_TOTAL + + +def test_pos_distribution_matches_design() -> None: + entries = _read_lexicon() + pos_counts = Counter(e["pos"] for e in entries) + assert dict(pos_counts) == EXPECTED_POS_COUNTS + + +def test_all_expected_lemmas_present() -> None: + _, manifold = load_pack(PACK_ID) + surfaces = {manifold.get_word_at(i) for i in range(len(manifold))} + assert surfaces == set(EXPECTED_LEMMAS) + + +def test_every_entry_has_temporal_namespace_primary_domain() -> None: + for entry in _read_lexicon(): + domains = entry["semantic_domains"] + assert isinstance(domains, list) and domains, entry + assert domains[0].startswith("temporal."), ( + f"entry {entry['entry_id']} primary domain {domains[0]!r} " + f"is not in the temporal.* namespace" + ) + + +def test_no_collision_with_prior_packs() -> None: + """Anti-leakage: no overlap with any earlier-mounted English pack.""" + prior_lemmas: set[str] = set() + pack_root = _PACK_ROOT.parent + for pack in ( + "en_core_cognition_v1", + "en_core_meta_v1", + "en_core_attitude_v1", + "en_core_relations_v1", + "en_core_relations_v2", + ): + for line in (pack_root / pack / "lexicon.jsonl").read_text("utf-8").splitlines(): + if line.strip(): + prior_lemmas.add(json.loads(line)["lemma"].lower()) + for entry in _read_lexicon(): + lemma = entry["lemma"].lower() + assert lemma not in prior_lemmas, ( + f"lemma {lemma!r} collides with a prior-mounted pack" + ) + + +def test_provenance_is_seed_core_temporal_v1() -> None: + for entry in _read_lexicon(): + assert entry["provenance_ids"] == ["seed:core_temporal_v1"], entry + + +def test_entry_ids_contiguous_and_zero_padded() -> None: + entries = sorted(_read_lexicon(), key=lambda d: d["entry_id"]) + for i, entry in enumerate(entries, start=1): + assert entry["entry_id"] == f"en-core-temporal-{i:03d}", entry["entry_id"] + + +def test_pack_mounted_in_default_runtime_config() -> None: + cfg = RuntimeConfig() + assert PACK_ID in cfg.input_packs + + +def test_pack_registered_after_prior_content_packs() -> None: + assert PACK_ID in DEFAULT_RESOLVABLE_PACK_IDS + for earlier in ("en_core_cognition_v1", "en_core_meta_v1", "en_core_attitude_v1"): + assert DEFAULT_RESOLVABLE_PACK_IDS.index(earlier) < DEFAULT_RESOLVABLE_PACK_IDS.index(PACK_ID) + + +def test_resolver_routes_temporal_lemmas_to_this_pack() -> None: + for lemma in EXPECTED_LEMMAS: + resolved = resolve_lemma(lemma) + assert resolved is not None, f"lemma {lemma!r} did not resolve" + pack_id, domains = resolved + assert pack_id == PACK_ID, ( + f"lemma {lemma!r} resolved to {pack_id} instead of {PACK_ID}" + ) + assert domains[0].startswith("temporal.") + + +def test_prior_pack_lemma_resolution_unchanged() -> None: + """Cognition / meta / attitude resolution unaffected by temporal pack.""" + for lemma, expected in ( + ("truth", "en_core_cognition_v1"), + ("knowledge", "en_core_cognition_v1"), + ("doubt", "en_core_meta_v1"), + ("fact", "en_core_meta_v1"), + ("true", "en_core_attitude_v1"), + ("certain", "en_core_attitude_v1"), + ): + resolved = resolve_lemma(lemma) + assert resolved is not None and resolved[0] == expected, ( + f"{lemma!r} resolved to {resolved[0] if resolved else None}, " + f"expected {expected}" + )