From a376a30bf8a845061db877aa85c031147a115ba4 Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 18 May 2026 19:40:19 -0700 Subject: [PATCH] =?UTF-8?q?feat(packs):=20en=5Fcore=5Fmeta=5Fv1=20?= =?UTF-8?q?=E2=80=94=20conversational=20substrate=20(73=20lemmas)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Workstream 1 (pack content scale-up) first load-bearing step. Adds a new ratified content pack covering the conversational vocabulary en_core_cognition_v1 deliberately omits — speech acts, mental states, perception, self-reference, and discourse-object nouns. These are the lemmas that show up in nearly every model response and that previously fell through to the OOV invitation surface. Pack composition (73 entries, 49 VERB + 24 NOUN): meta.speech_act.* (20 verbs) say tell speak reply claim state describe express name mention note observe declare assert deny confirm suggest propose articulate respond meta.mental_state.* (18 verbs) know believe think suppose assume expect hope want prefer doubt wonder guess recognize realize consider intend decide hold meta.perception.* (11 verbs) see hear feel sense perceive watch look listen find detect notice meta.self_reference.* (10 nouns) self mind view perspective position role agent model system speaker meta.discourse.* (14 nouns) response reply statement fact idea point argument proposal suggestion case instance example kind type Files: language_packs/data/en_core_meta_v1/ lexicon.jsonl — 73 entries, SHA-256 checksum-sealed manifest.json — operational_base / D0 / checksum-verified chat/pack_resolver.py Appended en_core_meta_v1 to DEFAULT_RESOLVABLE_PACK_IDS after en_core_cognition_v1 so cognition lemma resolution stays first- match-wins on any future collision (preserves cognition-lane byte-identity invariant). core/config.py Added en_core_meta_v1 to RuntimeConfig.input_packs default mount. tests/test_en_core_meta_v1_pack.py 11 contract tests: checksum-verified load, POS split, primary- domain namespace, no-collision-with-cognition-v1 regression gate, pack registration order, resolver routing, and cognition-lemma resolution unchanged. tests/test_procedure_surface.py Swapped two test fixtures from "claim" to "hypothesis". ``claim`` is now correctly pack-resident (meta.speech_act.claim) so the procedure composer's object-first selector picks it over the verb — the new behavior is semantically correct. ``hypothesis`` is genuinely OOV across all mounted packs and preserves the verb- fallback contract these tests pin. Authoring methodology: Four parallel subagents authored one cluster each from a strict exemplar + word list + forbidden-lemma list (every en_core_cognition_v1 lemma listed explicitly to prevent collision). Each subagent wrote only its cluster JSONL; the main pass assembled, validated, computed the SHA-256 over bytes-on-disk, and wrote the manifest. Verification: Full lane: 2148 passed, 2 skipped, 0 failed (+11 new tests). Cognition eval byte-identical on both splits: public 100 / 100 / 91.7 / 100 holdout 100 / 100 / 83.3 / 100 Live runtime probes: fresh ChatRuntime() for "What is X?" with X ∈ {fact, doubt, statement, model, self} all emit a pack-grounded sentence from en_core_meta_v1. OOV path still honest for genuinely-unknown terms (e.g. hypothesis). Scope note: This is one pack of ~70 lemmas, not "the model now articulates open-domain English." The architecturally-honest articulation story still requires more pack and teaching-chain content; this pack moves the conversational-substrate boundary forward by ~70 lemmas in one ratifiable, replay-stable step. --- chat/pack_resolver.py | 1 + core/config.py | 1 + .../data/en_core_meta_v1/lexicon.jsonl | 73 ++++++++ .../data/en_core_meta_v1/manifest.json | 13 ++ tests/test_en_core_meta_v1_pack.py | 167 ++++++++++++++++++ tests/test_procedure_surface.py | 21 ++- 6 files changed, 270 insertions(+), 6 deletions(-) create mode 100644 language_packs/data/en_core_meta_v1/lexicon.jsonl create mode 100644 language_packs/data/en_core_meta_v1/manifest.json create mode 100644 tests/test_en_core_meta_v1_pack.py diff --git a/chat/pack_resolver.py b/chat/pack_resolver.py index a5e2e62c..455220a9 100644 --- a/chat/pack_resolver.py +++ b/chat/pack_resolver.py @@ -42,6 +42,7 @@ from pathlib import Path # overlays and never carry vocabulary; they are deliberately excluded. DEFAULT_RESOLVABLE_PACK_IDS: tuple[str, ...] = ( "en_core_cognition_v1", + "en_core_meta_v1", "en_core_relations_v1", "en_core_relations_v2", ) diff --git a/core/config.py b/core/config.py index 64db59f7..2c10dfa4 100644 --- a/core/config.py +++ b/core/config.py @@ -13,6 +13,7 @@ class RuntimeConfig: input_packs: tuple[str, ...] = ( "en_minimal_v1", "en_core_cognition_v1", + "en_core_meta_v1", "en_core_relations_v1", "en_core_relations_v2", "he_logos_micro_v1", diff --git a/language_packs/data/en_core_meta_v1/lexicon.jsonl b/language_packs/data/en_core_meta_v1/lexicon.jsonl new file mode 100644 index 00000000..58f67fed --- /dev/null +++ b/language_packs/data/en_core_meta_v1/lexicon.jsonl @@ -0,0 +1,73 @@ +{"entry_id":"en-core-meta-001","surface":"say","lemma":"say","language":"en","pos":"VERB","semantic_domains":["meta.speech_act.assert","meta.speech_act","communication.articulation"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-002","surface":"tell","lemma":"tell","language":"en","pos":"VERB","semantic_domains":["meta.speech_act.assert","meta.speech_act","communication.articulation"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-003","surface":"speak","lemma":"speak","language":"en","pos":"VERB","semantic_domains":["meta.speech_act.speech","meta.speech_act","communication.articulation"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-004","surface":"reply","lemma":"reply","language":"en","pos":"VERB","semantic_domains":["meta.speech_act.response","meta.speech_act","communication.articulation"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-005","surface":"claim","lemma":"claim","language":"en","pos":"VERB","semantic_domains":["meta.speech_act.claim","meta.speech_act","communication.articulation"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-006","surface":"state","lemma":"state","language":"en","pos":"VERB","semantic_domains":["meta.speech_act.assert","meta.speech_act","communication.articulation"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-007","surface":"describe","lemma":"describe","language":"en","pos":"VERB","semantic_domains":["meta.speech_act.describe","meta.speech_act","communication.articulation"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-008","surface":"express","lemma":"express","language":"en","pos":"VERB","semantic_domains":["meta.speech_act.describe","meta.speech_act","communication.articulation"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-009","surface":"name","lemma":"name","language":"en","pos":"VERB","semantic_domains":["meta.speech_act.name","meta.speech_act","communication.articulation"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-010","surface":"mention","lemma":"mention","language":"en","pos":"VERB","semantic_domains":["meta.speech_act.name","meta.speech_act","communication.articulation"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-011","surface":"note","lemma":"note","language":"en","pos":"VERB","semantic_domains":["meta.speech_act.name","meta.speech_act","communication.articulation"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-012","surface":"observe","lemma":"observe","language":"en","pos":"VERB","semantic_domains":["meta.speech_act.describe","meta.speech_act","communication.articulation"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-013","surface":"declare","lemma":"declare","language":"en","pos":"VERB","semantic_domains":["meta.speech_act.assert","meta.speech_act","communication.articulation"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-014","surface":"assert","lemma":"assert","language":"en","pos":"VERB","semantic_domains":["meta.speech_act.assert","meta.speech_act","communication.articulation"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-015","surface":"deny","lemma":"deny","language":"en","pos":"VERB","semantic_domains":["meta.speech_act.denial","meta.speech_act","communication.articulation"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-016","surface":"confirm","lemma":"confirm","language":"en","pos":"VERB","semantic_domains":["meta.speech_act.assert","meta.speech_act","communication.articulation"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-017","surface":"suggest","lemma":"suggest","language":"en","pos":"VERB","semantic_domains":["meta.speech_act.directive","meta.speech_act","communication.directive"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-018","surface":"propose","lemma":"propose","language":"en","pos":"VERB","semantic_domains":["meta.speech_act.directive","meta.speech_act","communication.directive"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-019","surface":"articulate","lemma":"articulate","language":"en","pos":"VERB","semantic_domains":["meta.speech_act.describe","meta.speech_act","communication.articulation"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-020","surface":"respond","lemma":"respond","language":"en","pos":"VERB","semantic_domains":["meta.speech_act.response","meta.speech_act","communication.articulation"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-021","surface":"know","lemma":"know","language":"en","pos":"VERB","semantic_domains":["meta.mental_state.knowledge","meta.mental_state","cognition.epistemic"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-022","surface":"believe","lemma":"believe","language":"en","pos":"VERB","semantic_domains":["meta.mental_state.belief","meta.mental_state","cognition.attitude"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-023","surface":"think","lemma":"think","language":"en","pos":"VERB","semantic_domains":["meta.mental_state.belief","meta.mental_state","cognition.attitude"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-024","surface":"suppose","lemma":"suppose","language":"en","pos":"VERB","semantic_domains":["meta.mental_state.belief","meta.mental_state","cognition.attitude"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-025","surface":"assume","lemma":"assume","language":"en","pos":"VERB","semantic_domains":["meta.mental_state.belief","meta.mental_state","cognition.attitude"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-026","surface":"expect","lemma":"expect","language":"en","pos":"VERB","semantic_domains":["meta.mental_state.expectation","meta.mental_state","cognition.attitude"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-027","surface":"hope","lemma":"hope","language":"en","pos":"VERB","semantic_domains":["meta.mental_state.expectation","meta.mental_state","cognition.attitude"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-028","surface":"want","lemma":"want","language":"en","pos":"VERB","semantic_domains":["meta.mental_state.desire","meta.mental_state","cognition.attitude"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-029","surface":"prefer","lemma":"prefer","language":"en","pos":"VERB","semantic_domains":["meta.mental_state.desire","meta.mental_state","cognition.attitude"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-030","surface":"doubt","lemma":"doubt","language":"en","pos":"VERB","semantic_domains":["meta.mental_state.uncertainty","meta.mental_state","cognition.epistemic"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-031","surface":"wonder","lemma":"wonder","language":"en","pos":"VERB","semantic_domains":["meta.mental_state.uncertainty","meta.mental_state","cognition.epistemic"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-032","surface":"guess","lemma":"guess","language":"en","pos":"VERB","semantic_domains":["meta.mental_state.uncertainty","meta.mental_state","cognition.epistemic"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-033","surface":"recognize","lemma":"recognize","language":"en","pos":"VERB","semantic_domains":["meta.mental_state.knowledge","meta.mental_state","cognition.epistemic"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-034","surface":"realize","lemma":"realize","language":"en","pos":"VERB","semantic_domains":["meta.mental_state.knowledge","meta.mental_state","cognition.epistemic"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-035","surface":"consider","lemma":"consider","language":"en","pos":"VERB","semantic_domains":["meta.mental_state.deliberation","meta.mental_state","cognition.attitude"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-036","surface":"intend","lemma":"intend","language":"en","pos":"VERB","semantic_domains":["meta.mental_state.deliberation","meta.mental_state","cognition.attitude"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-037","surface":"decide","lemma":"decide","language":"en","pos":"VERB","semantic_domains":["meta.mental_state.deliberation","meta.mental_state","cognition.attitude"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-038","surface":"hold","lemma":"hold","language":"en","pos":"VERB","semantic_domains":["meta.mental_state.belief","meta.mental_state","cognition.attitude"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-039","surface":"see","lemma":"see","language":"en","pos":"VERB","semantic_domains":["meta.perception.visual","meta.perception","cognition.input"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-040","surface":"hear","lemma":"hear","language":"en","pos":"VERB","semantic_domains":["meta.perception.auditory","meta.perception","cognition.input"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-041","surface":"feel","lemma":"feel","language":"en","pos":"VERB","semantic_domains":["meta.perception.tactile_or_affective","meta.perception","cognition.input"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-042","surface":"sense","lemma":"sense","language":"en","pos":"VERB","semantic_domains":["meta.perception.general","meta.perception","cognition.input"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-043","surface":"perceive","lemma":"perceive","language":"en","pos":"VERB","semantic_domains":["meta.perception.general","meta.perception","cognition.input"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-044","surface":"watch","lemma":"watch","language":"en","pos":"VERB","semantic_domains":["meta.perception.visual","meta.perception","cognition.input"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-045","surface":"look","lemma":"look","language":"en","pos":"VERB","semantic_domains":["meta.perception.visual","meta.perception","cognition.input"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-046","surface":"listen","lemma":"listen","language":"en","pos":"VERB","semantic_domains":["meta.perception.auditory","meta.perception","cognition.input"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-047","surface":"find","lemma":"find","language":"en","pos":"VERB","semantic_domains":["meta.perception.discovery","meta.perception","cognition.input"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-048","surface":"detect","lemma":"detect","language":"en","pos":"VERB","semantic_domains":["meta.perception.discovery","meta.perception","cognition.input"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-049","surface":"notice","lemma":"notice","language":"en","pos":"VERB","semantic_domains":["meta.perception.discovery","meta.perception","cognition.input"],"morphology_tags":["verb"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-050","surface":"self","lemma":"self","language":"en","pos":"NOUN","semantic_domains":["meta.self_reference.self","meta.self_reference","cognition.identity"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-051","surface":"mind","lemma":"mind","language":"en","pos":"NOUN","semantic_domains":["meta.self_reference.self","meta.self_reference","cognition.identity"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-052","surface":"view","lemma":"view","language":"en","pos":"NOUN","semantic_domains":["meta.self_reference.viewpoint","meta.self_reference","cognition.perspective"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-053","surface":"perspective","lemma":"perspective","language":"en","pos":"NOUN","semantic_domains":["meta.self_reference.viewpoint","meta.self_reference","cognition.perspective"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-054","surface":"position","lemma":"position","language":"en","pos":"NOUN","semantic_domains":["meta.self_reference.viewpoint","meta.self_reference","cognition.perspective"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-055","surface":"role","lemma":"role","language":"en","pos":"NOUN","semantic_domains":["meta.self_reference.role","meta.self_reference","communication.participant"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-056","surface":"agent","lemma":"agent","language":"en","pos":"NOUN","semantic_domains":["meta.self_reference.role","meta.self_reference","communication.participant"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-057","surface":"speaker","lemma":"speaker","language":"en","pos":"NOUN","semantic_domains":["meta.self_reference.role","meta.self_reference","communication.participant"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-058","surface":"model","lemma":"model","language":"en","pos":"NOUN","semantic_domains":["meta.self_reference.system","meta.self_reference","cognition.system"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-059","surface":"system","lemma":"system","language":"en","pos":"NOUN","semantic_domains":["meta.self_reference.system","meta.self_reference","cognition.system"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-060","surface":"response","lemma":"response","language":"en","pos":"NOUN","semantic_domains":["meta.discourse.utterance","meta.discourse","communication.articulation"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-061","surface":"reply","lemma":"reply","language":"en","pos":"NOUN","semantic_domains":["meta.discourse.utterance","meta.discourse","communication.articulation"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-062","surface":"statement","lemma":"statement","language":"en","pos":"NOUN","semantic_domains":["meta.discourse.utterance","meta.discourse","communication.articulation"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-063","surface":"fact","lemma":"fact","language":"en","pos":"NOUN","semantic_domains":["meta.discourse.content","meta.discourse","cognition.content"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-064","surface":"idea","lemma":"idea","language":"en","pos":"NOUN","semantic_domains":["meta.discourse.content","meta.discourse","cognition.content"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-065","surface":"point","lemma":"point","language":"en","pos":"NOUN","semantic_domains":["meta.discourse.content","meta.discourse","cognition.content"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-066","surface":"argument","lemma":"argument","language":"en","pos":"NOUN","semantic_domains":["meta.discourse.argument","meta.discourse","communication.argument"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-067","surface":"proposal","lemma":"proposal","language":"en","pos":"NOUN","semantic_domains":["meta.discourse.argument","meta.discourse","communication.argument"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-068","surface":"suggestion","lemma":"suggestion","language":"en","pos":"NOUN","semantic_domains":["meta.discourse.argument","meta.discourse","communication.argument"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-069","surface":"case","lemma":"case","language":"en","pos":"NOUN","semantic_domains":["meta.discourse.instance","meta.discourse","cognition.instance"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-070","surface":"instance","lemma":"instance","language":"en","pos":"NOUN","semantic_domains":["meta.discourse.instance","meta.discourse","cognition.instance"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-071","surface":"example","lemma":"example","language":"en","pos":"NOUN","semantic_domains":["meta.discourse.instance","meta.discourse","cognition.instance"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-072","surface":"kind","lemma":"kind","language":"en","pos":"NOUN","semantic_domains":["meta.discourse.category","meta.discourse","cognition.category"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} +{"entry_id":"en-core-meta-073","surface":"type","lemma":"type","language":"en","pos":"NOUN","semantic_domains":["meta.discourse.category","meta.discourse","cognition.category"],"morphology_tags":["noun"],"provenance_ids":["seed:core_meta_v1"]} diff --git a/language_packs/data/en_core_meta_v1/manifest.json b/language_packs/data/en_core_meta_v1/manifest.json new file mode 100644 index 00000000..d2208b11 --- /dev/null +++ b/language_packs/data/en_core_meta_v1/manifest.json @@ -0,0 +1,13 @@ +{ + "pack_id": "en_core_meta_v1", + "language": "en", + "role": "operational_base", + "script": "Latin", + "normalization_policy": "unitize_versor", + "source_manifest": "en_core_meta_v1.lexicon.jsonl", + "determinism_class": "D0", + "checksum": "984e471ec9253b21263e74fc0a02e766db04442a88b3deb0e9a934b9856c44a8", + "version": "1.0.0", + "gate_engaged": true, + "oov_policy": "tagged_fallback" +} diff --git a/tests/test_en_core_meta_v1_pack.py b/tests/test_en_core_meta_v1_pack.py new file mode 100644 index 00000000..d5d889e6 --- /dev/null +++ b/tests/test_en_core_meta_v1_pack.py @@ -0,0 +1,167 @@ +"""``en_core_meta_v1`` — conversational substrate pack tests. + +The meta pack carries 73 lemmas across four semantic clusters that the +cognition pack deliberately omits: + + - meta.speech_act.* — say, tell, claim, deny, suggest, ... (20 verbs) + - meta.mental_state.* — know, believe, want, doubt, decide, ... (18 verbs) + - meta.perception.* — see, hear, feel, notice, find, ... (11 verbs) + - meta.self_reference.* — self, mind, view, role, model, ... (10 nouns) + - meta.discourse.* — fact, idea, statement, instance, ... (14 nouns) + +Contracts pinned here: + + - Checksum-verified load (bytes-on-disk match manifest digest). + - 73 entries total, broken down 49 VERB / 24 NOUN. + - Every lemma carries a semantic_domains list starting with ``meta.``. + - No collision with ``en_core_cognition_v1`` lemmas (forbidden list + enforced at authoring time — re-asserted here as a regression gate). + - Pack is mounted by default in ``RuntimeConfig.input_packs``. + - Pack is registered with ``DEFAULT_RESOLVABLE_PACK_IDS`` after + ``en_core_cognition_v1`` so any future lemma collision preserves + cognition's resolution (cognition lane byte-identity invariant). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from chat.pack_resolver import DEFAULT_RESOLVABLE_PACK_IDS, resolve_lemma +from core.config import RuntimeConfig +from language_packs.compiler import load_pack + + +PACK_ID = "en_core_meta_v1" +_PACK_ROOT = Path(__file__).resolve().parent.parent / "language_packs" / "data" / PACK_ID + +EXPECTED_TOTAL = 73 +EXPECTED_VERB = 49 +EXPECTED_NOUN = 24 + +EXPECTED_REPRESENTATIVE_LEMMAS: tuple[str, ...] = ( + "say", "tell", "deny", "suggest", "respond", + "know", "believe", "doubt", "decide", "hold", + "see", "hear", "notice", "find", + "self", "mind", "view", "model", "system", + "fact", "idea", "statement", "instance", "example", +) + +COGNITION_V1_FORBIDDEN: frozenset[str] = frozenset({ + "word", "truth", "light", "life", "beginning", "creation", + "knowledge", "wisdom", "spirit", "person", "question", "answer", + "reason", "cause", "memory", "correction", "meaning", "definition", + "comparison", "identity", "concept", "context", "define", "explain", + "compare", "infer", "remember", "correct", "verify", "ask", "mean", + "reveal", "relate", "distinguish", "teach", "learn", "recall", + "be", "have", "contrast", "precede", "follow", "belong", "ground", + "support", "why", "how", "because", "evidence", "inference", + "procedure", "verification", "distinction", "relation", "thought", + "understanding", "judgment", "principle", "order", "therefore", + "however", "then", "first", "metaphor", "simile", "analogy", + "narrative", "story", "voice", "style", "register", "tone", + "rhetoric", "figure", "symbol", "image", "discourse", "account", +}) + + +def _read_lexicon() -> list[dict]: + path = _PACK_ROOT / "lexicon.jsonl" + out: list[dict] = [] + for raw in path.read_text(encoding="utf-8").splitlines(): + if not raw.strip(): + continue + out.append(json.loads(raw)) + return out + + +def test_pack_loads_with_matching_checksum() -> None: + manifest, manifold = load_pack(PACK_ID) + assert manifest.pack_id == PACK_ID + assert len(manifest.checksum) == 64 + assert all(c in "0123456789abcdef" for c in manifest.checksum) + assert len(manifold) == EXPECTED_TOTAL + + +def test_lexicon_pos_split() -> None: + entries = _read_lexicon() + assert len(entries) == EXPECTED_TOTAL + verbs = [e for e in entries if e["pos"] == "VERB"] + nouns = [e for e in entries if e["pos"] == "NOUN"] + assert len(verbs) == EXPECTED_VERB + assert len(nouns) == EXPECTED_NOUN + + +def test_representative_lemmas_present() -> None: + _, manifold = load_pack(PACK_ID) + surfaces = {manifold.get_word_at(i) for i in range(len(manifold))} + for lemma in EXPECTED_REPRESENTATIVE_LEMMAS: + assert lemma in surfaces, f"expected lemma {lemma!r} missing from pack" + + +def test_every_entry_has_meta_namespace_primary_domain() -> None: + for entry in _read_lexicon(): + domains = entry["semantic_domains"] + assert isinstance(domains, list) and domains, entry + assert domains[0].startswith("meta."), ( + f"entry {entry['entry_id']} primary domain {domains[0]!r} " + f"is not in the meta.* namespace" + ) + + +def test_no_collision_with_cognition_v1() -> None: + """Anti-leakage: meta pack must not duplicate cognition_v1 lemmas. + + The author-time forbidden list is re-asserted here so a future edit + to the lexicon cannot quietly introduce a collision that would + silently break cognition's first-match-wins resolution. + """ + for entry in _read_lexicon(): + lemma = entry["lemma"].lower() + assert lemma not in COGNITION_V1_FORBIDDEN, ( + f"lemma {lemma!r} collides with en_core_cognition_v1" + ) + + +def test_provenance_is_seed_core_meta_v1() -> None: + for entry in _read_lexicon(): + assert entry["provenance_ids"] == ["seed:core_meta_v1"], entry + + +def test_entry_ids_contiguous_and_zero_padded() -> None: + entries = sorted(_read_lexicon(), key=lambda d: d["entry_id"]) + for i, entry in enumerate(entries, start=1): + assert entry["entry_id"] == f"en-core-meta-{i:03d}", entry["entry_id"] + + +def test_pack_mounted_in_default_runtime_config() -> None: + cfg = RuntimeConfig() + assert PACK_ID in cfg.input_packs + + +def test_pack_registered_in_default_resolvable_ids_after_cognition() -> None: + assert PACK_ID in DEFAULT_RESOLVABLE_PACK_IDS + cog_idx = DEFAULT_RESOLVABLE_PACK_IDS.index("en_core_cognition_v1") + meta_idx = DEFAULT_RESOLVABLE_PACK_IDS.index(PACK_ID) + assert cog_idx < meta_idx, ( + "cognition must precede meta in resolver order to preserve " + "the cognition-lane byte-identity invariant" + ) + + +def test_resolver_routes_meta_lemmas_to_this_pack() -> None: + """Every representative lemma resolves to en_core_meta_v1.""" + for lemma in EXPECTED_REPRESENTATIVE_LEMMAS: + resolved = resolve_lemma(lemma) + assert resolved is not None, f"lemma {lemma!r} did not resolve" + pack_id, domains = resolved + assert pack_id == PACK_ID, ( + f"lemma {lemma!r} resolved to {pack_id} instead of {PACK_ID}" + ) + assert domains[0].startswith("meta."), domains + + +def test_cognition_lemma_resolution_unchanged() -> None: + """First-match-wins keeps cognition lemmas resolving to cognition.""" + for lemma in ("truth", "knowledge", "memory", "evidence"): + resolved = resolve_lemma(lemma) + assert resolved is not None and resolved[0] == "en_core_cognition_v1" diff --git a/tests/test_procedure_surface.py b/tests/test_procedure_surface.py index a014158b..e3f951bc 100644 --- a/tests/test_procedure_surface.py +++ b/tests/test_procedure_surface.py @@ -55,7 +55,13 @@ def test_extract_returns_verb_when_only_pack_lemma() -> None: """When the verb is the only pack-resident lemma (object is OOV or filler), the verb is the topic by elimination — preserves coverage on procedure utterances with non-pack objects.""" - assert _extract_procedure_topic_lemma("verify a claim") == "verify" + # ``hypothesis`` is genuinely OOV across cognition/meta/relations. + # Pre-en_core_meta_v1 the canonical example was "verify a claim", + # but ``claim`` now resolves via the meta pack (meta.speech_act.claim), + # so the object-first selector would correctly pick ``claim`` over + # ``verify`` for that utterance. Use ``hypothesis`` to keep the + # verb-fallback contract isolated from pack-coverage changes. + assert _extract_procedure_topic_lemma("verify a hypothesis") == "verify" assert _extract_procedure_topic_lemma("correct an error") == "correct" assert _extract_procedure_topic_lemma("learn this") == "learn" @@ -163,11 +169,14 @@ def test_procedure_with_no_pack_lemma_routes_to_oov_invitation() -> None: assert "PackMutationProposal" in response.surface -def test_procedure_verify_a_claim_grounds() -> None: - """When the object is OOV (``claim`` isn't a pack lemma) but - the verb is pack-resident (``verify``), the composer surfaces - the verb — keeps surface_groundedness coverage.""" +def test_procedure_verify_an_oov_object_falls_back_to_verb() -> None: + """When the object is OOV but the verb is pack-resident, the + procedure composer surfaces the verb — keeps surface_groundedness + coverage on utterances whose object is not yet packed. + + ``hypothesis`` is genuinely OOV across cognition/meta/relations. + """ rt = ChatRuntime() - response = rt.chat("How do I verify a claim?") + response = rt.chat("How do I verify a hypothesis?") assert response.grounding_source == "pack" assert "verify" in response.surface.lower()