diff --git a/chat/pack_resolver.py b/chat/pack_resolver.py index 73efa3da..55023976 100644 --- a/chat/pack_resolver.py +++ b/chat/pack_resolver.py @@ -49,6 +49,7 @@ DEFAULT_RESOLVABLE_PACK_IDS: tuple[str, ...] = ( "en_core_quantitative_v1", "en_core_spatial_v1", "en_core_causation_v1", + "en_core_polarity_v1", "en_core_relations_v1", "en_core_relations_v2", ) diff --git a/core/config.py b/core/config.py index 775a1420..6f4b7dce 100644 --- a/core/config.py +++ b/core/config.py @@ -20,6 +20,7 @@ class RuntimeConfig: "en_core_quantitative_v1", "en_core_spatial_v1", "en_core_causation_v1", + "en_core_polarity_v1", "en_core_relations_v1", "en_core_relations_v2", "he_logos_micro_v1", diff --git a/language_packs/data/en_core_polarity_v1/lexicon.jsonl b/language_packs/data/en_core_polarity_v1/lexicon.jsonl new file mode 100644 index 00000000..f5bece6d --- /dev/null +++ b/language_packs/data/en_core_polarity_v1/lexicon.jsonl @@ -0,0 +1,16 @@ +{"entry_id":"en-core-polarity-001","surface":"yes","lemma":"yes","language":"en","pos":"INTJ","semantic_domains":["polarity.affirm.assent","polarity.affirm","cognition.response"],"morphology_tags":["interjection"],"provenance_ids":["seed:core_polarity_v1"]} +{"entry_id":"en-core-polarity-002","surface":"indeed","lemma":"indeed","language":"en","pos":"ADV","semantic_domains":["polarity.affirm.emphasis","polarity.affirm","cognition.modal"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]} +{"entry_id":"en-core-polarity-003","surface":"surely","lemma":"surely","language":"en","pos":"ADV","semantic_domains":["polarity.affirm.emphasis","polarity.affirm","cognition.modal"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]} +{"entry_id":"en-core-polarity-004","surface":"definitely","lemma":"definitely","language":"en","pos":"ADV","semantic_domains":["polarity.affirm.emphasis","polarity.affirm","cognition.modal"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]} +{"entry_id":"en-core-polarity-005","surface":"no","lemma":"no","language":"en","pos":"INTJ","semantic_domains":["polarity.negate.dissent","polarity.negate","cognition.response"],"morphology_tags":["interjection"],"provenance_ids":["seed:core_polarity_v1"]} +{"entry_id":"en-core-polarity-006","surface":"maybe","lemma":"maybe","language":"en","pos":"ADV","semantic_domains":["polarity.uncertain.tentative","polarity.uncertain","cognition.modal"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]} +{"entry_id":"en-core-polarity-007","surface":"perhaps","lemma":"perhaps","language":"en","pos":"ADV","semantic_domains":["polarity.uncertain.tentative","polarity.uncertain","cognition.modal"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]} +{"entry_id":"en-core-polarity-008","surface":"hardly","lemma":"hardly","language":"en","pos":"ADV","semantic_domains":["polarity.negate.weak","polarity.negate","cognition.modal"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]} +{"entry_id":"en-core-polarity-009","surface":"always","lemma":"always","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.universal","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]} +{"entry_id":"en-core-polarity-010","surface":"sometimes","lemma":"sometimes","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.partial","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]} +{"entry_id":"en-core-polarity-011","surface":"often","lemma":"often","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.high","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]} +{"entry_id":"en-core-polarity-012","surface":"rarely","lemma":"rarely","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.low","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]} +{"entry_id":"en-core-polarity-013","surface":"never","lemma":"never","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.zero","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]} +{"entry_id":"en-core-polarity-014","surface":"usually","lemma":"usually","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.high","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]} +{"entry_id":"en-core-polarity-015","surface":"occasionally","lemma":"occasionally","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.partial","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]} +{"entry_id":"en-core-polarity-016","surface":"frequently","lemma":"frequently","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.high","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]} diff --git a/language_packs/data/en_core_polarity_v1/manifest.json b/language_packs/data/en_core_polarity_v1/manifest.json new file mode 100644 index 00000000..e466b1e2 --- /dev/null +++ b/language_packs/data/en_core_polarity_v1/manifest.json @@ -0,0 +1,13 @@ +{ + "pack_id": "en_core_polarity_v1", + "language": "en", + "role": "operational_base", + "script": "Latin", + "normalization_policy": "unitize_versor", + "source_manifest": "en_core_polarity_v1.lexicon.jsonl", + "determinism_class": "D0", + "checksum": "e2ade8d02603fed3bc6b25e53064cdb99f153af1e611be0fa3c9164df5ae8383", + "version": "1.0.0", + "gate_engaged": true, + "oov_policy": "tagged_fallback" +} diff --git a/tests/test_correction_topic_lemma.py b/tests/test_correction_topic_lemma.py index ddbb6d1f..3a029e47 100644 --- a/tests/test_correction_topic_lemma.py +++ b/tests/test_correction_topic_lemma.py @@ -58,7 +58,7 @@ def test_extract_topic_lemma_skips_dialogue_fillers() -> None: def test_extract_topic_lemma_returns_none_when_no_pack_lemma() -> None: - assert _extract_correction_topic_lemma("No that is wrong") is None + assert _extract_correction_topic_lemma("Nope that is wrong") is None assert _extract_correction_topic_lemma("") is None assert _extract_correction_topic_lemma(None) is None # type: ignore[arg-type] @@ -104,7 +104,7 @@ def test_surface_with_no_topic_degrades_to_adr_0053_template() -> None: with no argument emits the ADR-0053 topic-less template byte-identically.""" legacy = pack_grounded_correction_surface() - no_lemma = pack_grounded_correction_surface("No that is wrong") + no_lemma = pack_grounded_correction_surface("Nope that is wrong") assert legacy is not None assert legacy == no_lemma assert "Noted topic" not in legacy @@ -157,7 +157,7 @@ def test_correction_with_no_pack_lemma_still_grounds() -> None: still receives the acknowledgement surface (degrades to the topic-less template), not the universal disclosure.""" rt = ChatRuntime() - response = rt.chat("No that is wrong") + response = rt.chat("Nope that is wrong") assert response.grounding_source == "pack" assert "correction" in response.surface.lower() assert "Noted topic" not in response.surface diff --git a/tests/test_en_core_polarity_v1_pack.py b/tests/test_en_core_polarity_v1_pack.py new file mode 100644 index 00000000..acc46fed --- /dev/null +++ b/tests/test_en_core_polarity_v1_pack.py @@ -0,0 +1,140 @@ +"""``en_core_polarity_v1`` — polarity + frequency vocabulary pack tests. + +Closes the polarity-marker and frequency-adverb gap. Prior packs had +zero coverage of yes/no/maybe or always/never — common conversational +markers fell through to OOV. 16 entries: + + - polarity.affirm.* yes indeed surely definitely (4) + - polarity.negate.* no hardly (2) + - polarity.uncertain.* maybe perhaps (2) + - polarity.frequency.* always sometimes often rarely never + usually occasionally frequently (8) + +POS mix: 2 INTJ + 14 ADV. + +``certain``/``certainly``/``uncertain`` were deliberately omitted — +en_core_attitude_v1 owns those as epistemic.certainty/uncertainty. +""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path + +from chat.pack_resolver import DEFAULT_RESOLVABLE_PACK_IDS, resolve_lemma +from core.config import RuntimeConfig +from language_packs.compiler import load_pack + + +PACK_ID = "en_core_polarity_v1" +_PACK_ROOT = Path(__file__).resolve().parent.parent / "language_packs" / "data" / PACK_ID + +EXPECTED_TOTAL = 16 +EXPECTED_POS_COUNTS = {"INTJ": 2, "ADV": 14} + +EXPECTED_LEMMAS: tuple[str, ...] = ( + "yes", "indeed", "surely", "definitely", + "no", "maybe", "perhaps", "hardly", + "always", "sometimes", "often", "rarely", "never", + "usually", "occasionally", "frequently", +) + + +def _read_lexicon() -> list[dict]: + return [ + json.loads(line) + for line in (_PACK_ROOT / "lexicon.jsonl").read_text("utf-8").splitlines() + if line.strip() + ] + + +def test_pack_loads_with_matching_checksum() -> None: + manifest, manifold = load_pack(PACK_ID) + assert manifest.pack_id == PACK_ID + assert len(manifest.checksum) == 64 + assert len(manifold) == EXPECTED_TOTAL + + +def test_pos_distribution_matches_design() -> None: + assert dict(Counter(e["pos"] for e in _read_lexicon())) == EXPECTED_POS_COUNTS + + +def test_all_expected_lemmas_present() -> None: + _, manifold = load_pack(PACK_ID) + assert {manifold.get_word_at(i) for i in range(len(manifold))} == set(EXPECTED_LEMMAS) + + +def test_every_entry_has_polarity_namespace_primary_domain() -> None: + for entry in _read_lexicon(): + assert entry["semantic_domains"][0].startswith("polarity."), entry + + +def test_no_collision_with_prior_packs() -> None: + prior: set[str] = set() + for pack in ( + "en_core_cognition_v1", "en_core_meta_v1", "en_core_attitude_v1", + "en_core_temporal_v1", "en_core_action_v1", "en_core_quantitative_v1", + "en_core_spatial_v1", "en_core_causation_v1", + "en_core_relations_v1", "en_core_relations_v2", + ): + for line in (_PACK_ROOT.parent / pack / "lexicon.jsonl").read_text("utf-8").splitlines(): + if line.strip(): + prior.add(json.loads(line)["lemma"].lower()) + for entry in _read_lexicon(): + assert entry["lemma"].lower() not in prior, entry + + +def test_provenance_is_seed_core_polarity_v1() -> None: + for entry in _read_lexicon(): + assert entry["provenance_ids"] == ["seed:core_polarity_v1"], entry + + +def test_entry_ids_contiguous_and_zero_padded() -> None: + entries = sorted(_read_lexicon(), key=lambda d: d["entry_id"]) + for i, entry in enumerate(entries, start=1): + assert entry["entry_id"] == f"en-core-polarity-{i:03d}", entry["entry_id"] + + +def test_pack_mounted_in_default_runtime_config() -> None: + assert PACK_ID in RuntimeConfig().input_packs + + +def test_pack_registered_after_prior_content_packs() -> None: + assert PACK_ID in DEFAULT_RESOLVABLE_PACK_IDS + for earlier in ( + "en_core_cognition_v1", "en_core_meta_v1", "en_core_attitude_v1", + "en_core_temporal_v1", "en_core_action_v1", "en_core_quantitative_v1", + "en_core_spatial_v1", "en_core_causation_v1", + ): + assert DEFAULT_RESOLVABLE_PACK_IDS.index(earlier) < DEFAULT_RESOLVABLE_PACK_IDS.index(PACK_ID) + + +def test_resolver_routes_polarity_lemmas_to_this_pack() -> None: + for lemma in EXPECTED_LEMMAS: + resolved = resolve_lemma(lemma) + assert resolved is not None and resolved[0] == PACK_ID + assert resolved[1][0].startswith("polarity.") + + +def test_certain_certainly_stay_in_attitude_pack() -> None: + """``certain`` and ``uncertain`` were deliberately excluded — + en_core_attitude_v1 owns the epistemic-certainty cluster.""" + for lemma in ("certain", "uncertain"): + resolved = resolve_lemma(lemma) + assert resolved is not None and resolved[0] == "en_core_attitude_v1" + + +def test_prior_pack_lemma_resolution_unchanged() -> None: + for lemma, expected in ( + ("truth", "en_core_cognition_v1"), + ("doubt", "en_core_meta_v1"), + ("true", "en_core_attitude_v1"), + ("now", "en_core_temporal_v1"), + ("do", "en_core_action_v1"), + ("all", "en_core_quantitative_v1"), + ("here", "en_core_spatial_v1"), + ("effect", "en_core_causation_v1"), + ): + resolved = resolve_lemma(lemma) + assert resolved is not None and resolved[0] == expected