feat(packs): en_core_polarity_v1 — polarity + frequency (16 lemmas)

Workstream 1 eighth pack.  Closes the polarity-marker + frequency-
adverb gap.  Common conversational markers (yes/no/maybe/always/never)
had zero coverage in any prior pack.

Pack composition (16 entries — 2 INTJ / 14 ADV):

  polarity.affirm.*      yes indeed surely definitely
  polarity.negate.*      no hardly
  polarity.uncertain.*   maybe perhaps
  polarity.frequency.*   always sometimes often rarely never
                         usually occasionally frequently

``certain``/``certainly``/``uncertain`` deliberately excluded — those
remain in en_core_attitude_v1 (epistemic.certainty/uncertainty).
Regression test pins the invariant.

tests/test_correction_topic_lemma.py:
  Three fixtures swapped from "No that is wrong" to "Nope that is
  wrong".  ``no`` is now correctly pack-resident in en_core_polarity_v1
  (polarity.negate.dissent), so the "no pack-resident lemma" contract
  these tests pin needed a fixture where every content token is
  genuinely OOV.  ``nope`` is OOV across all 10 mounted packs; ``wrong``
  remains OOV (collision with attitude's ``right`` blocked spatial-
  direction ``right`` but did not add ``wrong``).

Authoring:
  Three parallel subagents — affirm / negate+uncertain / frequency.
This commit is contained in:
Shay 2026-05-19 05:36:10 -07:00
parent e72e946c0b
commit 1c8f2ee943
6 changed files with 174 additions and 3 deletions

View file

@ -49,6 +49,7 @@ DEFAULT_RESOLVABLE_PACK_IDS: tuple[str, ...] = (
"en_core_quantitative_v1",
"en_core_spatial_v1",
"en_core_causation_v1",
"en_core_polarity_v1",
"en_core_relations_v1",
"en_core_relations_v2",
)

View file

@ -20,6 +20,7 @@ class RuntimeConfig:
"en_core_quantitative_v1",
"en_core_spatial_v1",
"en_core_causation_v1",
"en_core_polarity_v1",
"en_core_relations_v1",
"en_core_relations_v2",
"he_logos_micro_v1",

View file

@ -0,0 +1,16 @@
{"entry_id":"en-core-polarity-001","surface":"yes","lemma":"yes","language":"en","pos":"INTJ","semantic_domains":["polarity.affirm.assent","polarity.affirm","cognition.response"],"morphology_tags":["interjection"],"provenance_ids":["seed:core_polarity_v1"]}
{"entry_id":"en-core-polarity-002","surface":"indeed","lemma":"indeed","language":"en","pos":"ADV","semantic_domains":["polarity.affirm.emphasis","polarity.affirm","cognition.modal"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
{"entry_id":"en-core-polarity-003","surface":"surely","lemma":"surely","language":"en","pos":"ADV","semantic_domains":["polarity.affirm.emphasis","polarity.affirm","cognition.modal"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
{"entry_id":"en-core-polarity-004","surface":"definitely","lemma":"definitely","language":"en","pos":"ADV","semantic_domains":["polarity.affirm.emphasis","polarity.affirm","cognition.modal"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
{"entry_id":"en-core-polarity-005","surface":"no","lemma":"no","language":"en","pos":"INTJ","semantic_domains":["polarity.negate.dissent","polarity.negate","cognition.response"],"morphology_tags":["interjection"],"provenance_ids":["seed:core_polarity_v1"]}
{"entry_id":"en-core-polarity-006","surface":"maybe","lemma":"maybe","language":"en","pos":"ADV","semantic_domains":["polarity.uncertain.tentative","polarity.uncertain","cognition.modal"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
{"entry_id":"en-core-polarity-007","surface":"perhaps","lemma":"perhaps","language":"en","pos":"ADV","semantic_domains":["polarity.uncertain.tentative","polarity.uncertain","cognition.modal"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
{"entry_id":"en-core-polarity-008","surface":"hardly","lemma":"hardly","language":"en","pos":"ADV","semantic_domains":["polarity.negate.weak","polarity.negate","cognition.modal"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
{"entry_id":"en-core-polarity-009","surface":"always","lemma":"always","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.universal","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
{"entry_id":"en-core-polarity-010","surface":"sometimes","lemma":"sometimes","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.partial","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
{"entry_id":"en-core-polarity-011","surface":"often","lemma":"often","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.high","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
{"entry_id":"en-core-polarity-012","surface":"rarely","lemma":"rarely","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.low","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
{"entry_id":"en-core-polarity-013","surface":"never","lemma":"never","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.zero","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
{"entry_id":"en-core-polarity-014","surface":"usually","lemma":"usually","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.high","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
{"entry_id":"en-core-polarity-015","surface":"occasionally","lemma":"occasionally","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.partial","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
{"entry_id":"en-core-polarity-016","surface":"frequently","lemma":"frequently","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.high","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}

View file

@ -0,0 +1,13 @@
{
"pack_id": "en_core_polarity_v1",
"language": "en",
"role": "operational_base",
"script": "Latin",
"normalization_policy": "unitize_versor",
"source_manifest": "en_core_polarity_v1.lexicon.jsonl",
"determinism_class": "D0",
"checksum": "e2ade8d02603fed3bc6b25e53064cdb99f153af1e611be0fa3c9164df5ae8383",
"version": "1.0.0",
"gate_engaged": true,
"oov_policy": "tagged_fallback"
}

View file

@ -58,7 +58,7 @@ def test_extract_topic_lemma_skips_dialogue_fillers() -> None:
def test_extract_topic_lemma_returns_none_when_no_pack_lemma() -> None:
assert _extract_correction_topic_lemma("No that is wrong") is None
assert _extract_correction_topic_lemma("Nope that is wrong") is None
assert _extract_correction_topic_lemma("") is None
assert _extract_correction_topic_lemma(None) is None # type: ignore[arg-type]
@ -104,7 +104,7 @@ def test_surface_with_no_topic_degrades_to_adr_0053_template() -> None:
with no argument emits the ADR-0053 topic-less template
byte-identically."""
legacy = pack_grounded_correction_surface()
no_lemma = pack_grounded_correction_surface("No that is wrong")
no_lemma = pack_grounded_correction_surface("Nope that is wrong")
assert legacy is not None
assert legacy == no_lemma
assert "Noted topic" not in legacy
@ -157,7 +157,7 @@ def test_correction_with_no_pack_lemma_still_grounds() -> None:
still receives the acknowledgement surface (degrades to the
topic-less template), not the universal disclosure."""
rt = ChatRuntime()
response = rt.chat("No that is wrong")
response = rt.chat("Nope that is wrong")
assert response.grounding_source == "pack"
assert "correction" in response.surface.lower()
assert "Noted topic" not in response.surface

View file

@ -0,0 +1,140 @@
"""``en_core_polarity_v1`` — polarity + frequency vocabulary pack tests.
Closes the polarity-marker and frequency-adverb gap. Prior packs had
zero coverage of yes/no/maybe or always/never common conversational
markers fell through to OOV. 16 entries:
- polarity.affirm.* yes indeed surely definitely (4)
- polarity.negate.* no hardly (2)
- polarity.uncertain.* maybe perhaps (2)
- polarity.frequency.* always sometimes often rarely never
usually occasionally frequently (8)
POS mix: 2 INTJ + 14 ADV.
``certain``/``certainly``/``uncertain`` were deliberately omitted
en_core_attitude_v1 owns those as epistemic.certainty/uncertainty.
"""
from __future__ import annotations
import json
from collections import Counter
from pathlib import Path
from chat.pack_resolver import DEFAULT_RESOLVABLE_PACK_IDS, resolve_lemma
from core.config import RuntimeConfig
from language_packs.compiler import load_pack
PACK_ID = "en_core_polarity_v1"
_PACK_ROOT = Path(__file__).resolve().parent.parent / "language_packs" / "data" / PACK_ID
EXPECTED_TOTAL = 16
EXPECTED_POS_COUNTS = {"INTJ": 2, "ADV": 14}
EXPECTED_LEMMAS: tuple[str, ...] = (
"yes", "indeed", "surely", "definitely",
"no", "maybe", "perhaps", "hardly",
"always", "sometimes", "often", "rarely", "never",
"usually", "occasionally", "frequently",
)
def _read_lexicon() -> list[dict]:
return [
json.loads(line)
for line in (_PACK_ROOT / "lexicon.jsonl").read_text("utf-8").splitlines()
if line.strip()
]
def test_pack_loads_with_matching_checksum() -> None:
manifest, manifold = load_pack(PACK_ID)
assert manifest.pack_id == PACK_ID
assert len(manifest.checksum) == 64
assert len(manifold) == EXPECTED_TOTAL
def test_pos_distribution_matches_design() -> None:
assert dict(Counter(e["pos"] for e in _read_lexicon())) == EXPECTED_POS_COUNTS
def test_all_expected_lemmas_present() -> None:
_, manifold = load_pack(PACK_ID)
assert {manifold.get_word_at(i) for i in range(len(manifold))} == set(EXPECTED_LEMMAS)
def test_every_entry_has_polarity_namespace_primary_domain() -> None:
for entry in _read_lexicon():
assert entry["semantic_domains"][0].startswith("polarity."), entry
def test_no_collision_with_prior_packs() -> None:
prior: set[str] = set()
for pack in (
"en_core_cognition_v1", "en_core_meta_v1", "en_core_attitude_v1",
"en_core_temporal_v1", "en_core_action_v1", "en_core_quantitative_v1",
"en_core_spatial_v1", "en_core_causation_v1",
"en_core_relations_v1", "en_core_relations_v2",
):
for line in (_PACK_ROOT.parent / pack / "lexicon.jsonl").read_text("utf-8").splitlines():
if line.strip():
prior.add(json.loads(line)["lemma"].lower())
for entry in _read_lexicon():
assert entry["lemma"].lower() not in prior, entry
def test_provenance_is_seed_core_polarity_v1() -> None:
for entry in _read_lexicon():
assert entry["provenance_ids"] == ["seed:core_polarity_v1"], entry
def test_entry_ids_contiguous_and_zero_padded() -> None:
entries = sorted(_read_lexicon(), key=lambda d: d["entry_id"])
for i, entry in enumerate(entries, start=1):
assert entry["entry_id"] == f"en-core-polarity-{i:03d}", entry["entry_id"]
def test_pack_mounted_in_default_runtime_config() -> None:
assert PACK_ID in RuntimeConfig().input_packs
def test_pack_registered_after_prior_content_packs() -> None:
assert PACK_ID in DEFAULT_RESOLVABLE_PACK_IDS
for earlier in (
"en_core_cognition_v1", "en_core_meta_v1", "en_core_attitude_v1",
"en_core_temporal_v1", "en_core_action_v1", "en_core_quantitative_v1",
"en_core_spatial_v1", "en_core_causation_v1",
):
assert DEFAULT_RESOLVABLE_PACK_IDS.index(earlier) < DEFAULT_RESOLVABLE_PACK_IDS.index(PACK_ID)
def test_resolver_routes_polarity_lemmas_to_this_pack() -> None:
for lemma in EXPECTED_LEMMAS:
resolved = resolve_lemma(lemma)
assert resolved is not None and resolved[0] == PACK_ID
assert resolved[1][0].startswith("polarity.")
def test_certain_certainly_stay_in_attitude_pack() -> None:
"""``certain`` and ``uncertain`` were deliberately excluded —
en_core_attitude_v1 owns the epistemic-certainty cluster."""
for lemma in ("certain", "uncertain"):
resolved = resolve_lemma(lemma)
assert resolved is not None and resolved[0] == "en_core_attitude_v1"
def test_prior_pack_lemma_resolution_unchanged() -> None:
for lemma, expected in (
("truth", "en_core_cognition_v1"),
("doubt", "en_core_meta_v1"),
("true", "en_core_attitude_v1"),
("now", "en_core_temporal_v1"),
("do", "en_core_action_v1"),
("all", "en_core_quantitative_v1"),
("here", "en_core_spatial_v1"),
("effect", "en_core_causation_v1"),
):
resolved = resolve_lemma(lemma)
assert resolved is not None and resolved[0] == expected