feat(packs): en_core_polarity_v1 — polarity + frequency (16 lemmas)
Workstream 1 eighth pack. Closes the polarity-marker + frequency-
adverb gap. Common conversational markers (yes/no/maybe/always/never)
had zero coverage in any prior pack.
Pack composition (16 entries — 2 INTJ / 14 ADV):
polarity.affirm.* yes indeed surely definitely
polarity.negate.* no hardly
polarity.uncertain.* maybe perhaps
polarity.frequency.* always sometimes often rarely never
usually occasionally frequently
``certain``/``certainly``/``uncertain`` deliberately excluded — those
remain in en_core_attitude_v1 (epistemic.certainty/uncertainty).
Regression test pins the invariant.
tests/test_correction_topic_lemma.py:
Three fixtures swapped from "No that is wrong" to "Nope that is
wrong". ``no`` is now correctly pack-resident in en_core_polarity_v1
(polarity.negate.dissent), so the "no pack-resident lemma" contract
these tests pin needed a fixture where every content token is
genuinely OOV. ``nope`` is OOV across all 10 mounted packs; ``wrong``
remains OOV (collision with attitude's ``right`` blocked spatial-
direction ``right`` but did not add ``wrong``).
Authoring:
Three parallel subagents — affirm / negate+uncertain / frequency.
This commit is contained in:
parent
e72e946c0b
commit
1c8f2ee943
6 changed files with 174 additions and 3 deletions
|
|
@ -49,6 +49,7 @@ DEFAULT_RESOLVABLE_PACK_IDS: tuple[str, ...] = (
|
|||
"en_core_quantitative_v1",
|
||||
"en_core_spatial_v1",
|
||||
"en_core_causation_v1",
|
||||
"en_core_polarity_v1",
|
||||
"en_core_relations_v1",
|
||||
"en_core_relations_v2",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ class RuntimeConfig:
|
|||
"en_core_quantitative_v1",
|
||||
"en_core_spatial_v1",
|
||||
"en_core_causation_v1",
|
||||
"en_core_polarity_v1",
|
||||
"en_core_relations_v1",
|
||||
"en_core_relations_v2",
|
||||
"he_logos_micro_v1",
|
||||
|
|
|
|||
16
language_packs/data/en_core_polarity_v1/lexicon.jsonl
Normal file
16
language_packs/data/en_core_polarity_v1/lexicon.jsonl
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
{"entry_id":"en-core-polarity-001","surface":"yes","lemma":"yes","language":"en","pos":"INTJ","semantic_domains":["polarity.affirm.assent","polarity.affirm","cognition.response"],"morphology_tags":["interjection"],"provenance_ids":["seed:core_polarity_v1"]}
|
||||
{"entry_id":"en-core-polarity-002","surface":"indeed","lemma":"indeed","language":"en","pos":"ADV","semantic_domains":["polarity.affirm.emphasis","polarity.affirm","cognition.modal"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
|
||||
{"entry_id":"en-core-polarity-003","surface":"surely","lemma":"surely","language":"en","pos":"ADV","semantic_domains":["polarity.affirm.emphasis","polarity.affirm","cognition.modal"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
|
||||
{"entry_id":"en-core-polarity-004","surface":"definitely","lemma":"definitely","language":"en","pos":"ADV","semantic_domains":["polarity.affirm.emphasis","polarity.affirm","cognition.modal"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
|
||||
{"entry_id":"en-core-polarity-005","surface":"no","lemma":"no","language":"en","pos":"INTJ","semantic_domains":["polarity.negate.dissent","polarity.negate","cognition.response"],"morphology_tags":["interjection"],"provenance_ids":["seed:core_polarity_v1"]}
|
||||
{"entry_id":"en-core-polarity-006","surface":"maybe","lemma":"maybe","language":"en","pos":"ADV","semantic_domains":["polarity.uncertain.tentative","polarity.uncertain","cognition.modal"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
|
||||
{"entry_id":"en-core-polarity-007","surface":"perhaps","lemma":"perhaps","language":"en","pos":"ADV","semantic_domains":["polarity.uncertain.tentative","polarity.uncertain","cognition.modal"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
|
||||
{"entry_id":"en-core-polarity-008","surface":"hardly","lemma":"hardly","language":"en","pos":"ADV","semantic_domains":["polarity.negate.weak","polarity.negate","cognition.modal"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
|
||||
{"entry_id":"en-core-polarity-009","surface":"always","lemma":"always","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.universal","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
|
||||
{"entry_id":"en-core-polarity-010","surface":"sometimes","lemma":"sometimes","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.partial","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
|
||||
{"entry_id":"en-core-polarity-011","surface":"often","lemma":"often","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.high","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
|
||||
{"entry_id":"en-core-polarity-012","surface":"rarely","lemma":"rarely","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.low","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
|
||||
{"entry_id":"en-core-polarity-013","surface":"never","lemma":"never","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.zero","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
|
||||
{"entry_id":"en-core-polarity-014","surface":"usually","lemma":"usually","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.high","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
|
||||
{"entry_id":"en-core-polarity-015","surface":"occasionally","lemma":"occasionally","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.partial","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
|
||||
{"entry_id":"en-core-polarity-016","surface":"frequently","lemma":"frequently","language":"en","pos":"ADV","semantic_domains":["polarity.frequency.high","polarity.frequency","cognition.temporal_distribution"],"morphology_tags":["adverb"],"provenance_ids":["seed:core_polarity_v1"]}
|
||||
13
language_packs/data/en_core_polarity_v1/manifest.json
Normal file
13
language_packs/data/en_core_polarity_v1/manifest.json
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"pack_id": "en_core_polarity_v1",
|
||||
"language": "en",
|
||||
"role": "operational_base",
|
||||
"script": "Latin",
|
||||
"normalization_policy": "unitize_versor",
|
||||
"source_manifest": "en_core_polarity_v1.lexicon.jsonl",
|
||||
"determinism_class": "D0",
|
||||
"checksum": "e2ade8d02603fed3bc6b25e53064cdb99f153af1e611be0fa3c9164df5ae8383",
|
||||
"version": "1.0.0",
|
||||
"gate_engaged": true,
|
||||
"oov_policy": "tagged_fallback"
|
||||
}
|
||||
|
|
@ -58,7 +58,7 @@ def test_extract_topic_lemma_skips_dialogue_fillers() -> None:
|
|||
|
||||
|
||||
def test_extract_topic_lemma_returns_none_when_no_pack_lemma() -> None:
|
||||
assert _extract_correction_topic_lemma("No that is wrong") is None
|
||||
assert _extract_correction_topic_lemma("Nope that is wrong") is None
|
||||
assert _extract_correction_topic_lemma("") is None
|
||||
assert _extract_correction_topic_lemma(None) is None # type: ignore[arg-type]
|
||||
|
||||
|
|
@ -104,7 +104,7 @@ def test_surface_with_no_topic_degrades_to_adr_0053_template() -> None:
|
|||
with no argument emits the ADR-0053 topic-less template
|
||||
byte-identically."""
|
||||
legacy = pack_grounded_correction_surface()
|
||||
no_lemma = pack_grounded_correction_surface("No that is wrong")
|
||||
no_lemma = pack_grounded_correction_surface("Nope that is wrong")
|
||||
assert legacy is not None
|
||||
assert legacy == no_lemma
|
||||
assert "Noted topic" not in legacy
|
||||
|
|
@ -157,7 +157,7 @@ def test_correction_with_no_pack_lemma_still_grounds() -> None:
|
|||
still receives the acknowledgement surface (degrades to the
|
||||
topic-less template), not the universal disclosure."""
|
||||
rt = ChatRuntime()
|
||||
response = rt.chat("No that is wrong")
|
||||
response = rt.chat("Nope that is wrong")
|
||||
assert response.grounding_source == "pack"
|
||||
assert "correction" in response.surface.lower()
|
||||
assert "Noted topic" not in response.surface
|
||||
|
|
|
|||
140
tests/test_en_core_polarity_v1_pack.py
Normal file
140
tests/test_en_core_polarity_v1_pack.py
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
"""``en_core_polarity_v1`` — polarity + frequency vocabulary pack tests.
|
||||
|
||||
Closes the polarity-marker and frequency-adverb gap. Prior packs had
|
||||
zero coverage of yes/no/maybe or always/never — common conversational
|
||||
markers fell through to OOV. 16 entries:
|
||||
|
||||
- polarity.affirm.* yes indeed surely definitely (4)
|
||||
- polarity.negate.* no hardly (2)
|
||||
- polarity.uncertain.* maybe perhaps (2)
|
||||
- polarity.frequency.* always sometimes often rarely never
|
||||
usually occasionally frequently (8)
|
||||
|
||||
POS mix: 2 INTJ + 14 ADV.
|
||||
|
||||
``certain``/``certainly``/``uncertain`` were deliberately omitted —
|
||||
en_core_attitude_v1 owns those as epistemic.certainty/uncertainty.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from chat.pack_resolver import DEFAULT_RESOLVABLE_PACK_IDS, resolve_lemma
|
||||
from core.config import RuntimeConfig
|
||||
from language_packs.compiler import load_pack
|
||||
|
||||
|
||||
PACK_ID = "en_core_polarity_v1"
|
||||
_PACK_ROOT = Path(__file__).resolve().parent.parent / "language_packs" / "data" / PACK_ID
|
||||
|
||||
EXPECTED_TOTAL = 16
|
||||
EXPECTED_POS_COUNTS = {"INTJ": 2, "ADV": 14}
|
||||
|
||||
EXPECTED_LEMMAS: tuple[str, ...] = (
|
||||
"yes", "indeed", "surely", "definitely",
|
||||
"no", "maybe", "perhaps", "hardly",
|
||||
"always", "sometimes", "often", "rarely", "never",
|
||||
"usually", "occasionally", "frequently",
|
||||
)
|
||||
|
||||
|
||||
def _read_lexicon() -> list[dict]:
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in (_PACK_ROOT / "lexicon.jsonl").read_text("utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
|
||||
|
||||
def test_pack_loads_with_matching_checksum() -> None:
|
||||
manifest, manifold = load_pack(PACK_ID)
|
||||
assert manifest.pack_id == PACK_ID
|
||||
assert len(manifest.checksum) == 64
|
||||
assert len(manifold) == EXPECTED_TOTAL
|
||||
|
||||
|
||||
def test_pos_distribution_matches_design() -> None:
|
||||
assert dict(Counter(e["pos"] for e in _read_lexicon())) == EXPECTED_POS_COUNTS
|
||||
|
||||
|
||||
def test_all_expected_lemmas_present() -> None:
|
||||
_, manifold = load_pack(PACK_ID)
|
||||
assert {manifold.get_word_at(i) for i in range(len(manifold))} == set(EXPECTED_LEMMAS)
|
||||
|
||||
|
||||
def test_every_entry_has_polarity_namespace_primary_domain() -> None:
|
||||
for entry in _read_lexicon():
|
||||
assert entry["semantic_domains"][0].startswith("polarity."), entry
|
||||
|
||||
|
||||
def test_no_collision_with_prior_packs() -> None:
|
||||
prior: set[str] = set()
|
||||
for pack in (
|
||||
"en_core_cognition_v1", "en_core_meta_v1", "en_core_attitude_v1",
|
||||
"en_core_temporal_v1", "en_core_action_v1", "en_core_quantitative_v1",
|
||||
"en_core_spatial_v1", "en_core_causation_v1",
|
||||
"en_core_relations_v1", "en_core_relations_v2",
|
||||
):
|
||||
for line in (_PACK_ROOT.parent / pack / "lexicon.jsonl").read_text("utf-8").splitlines():
|
||||
if line.strip():
|
||||
prior.add(json.loads(line)["lemma"].lower())
|
||||
for entry in _read_lexicon():
|
||||
assert entry["lemma"].lower() not in prior, entry
|
||||
|
||||
|
||||
def test_provenance_is_seed_core_polarity_v1() -> None:
|
||||
for entry in _read_lexicon():
|
||||
assert entry["provenance_ids"] == ["seed:core_polarity_v1"], entry
|
||||
|
||||
|
||||
def test_entry_ids_contiguous_and_zero_padded() -> None:
|
||||
entries = sorted(_read_lexicon(), key=lambda d: d["entry_id"])
|
||||
for i, entry in enumerate(entries, start=1):
|
||||
assert entry["entry_id"] == f"en-core-polarity-{i:03d}", entry["entry_id"]
|
||||
|
||||
|
||||
def test_pack_mounted_in_default_runtime_config() -> None:
|
||||
assert PACK_ID in RuntimeConfig().input_packs
|
||||
|
||||
|
||||
def test_pack_registered_after_prior_content_packs() -> None:
|
||||
assert PACK_ID in DEFAULT_RESOLVABLE_PACK_IDS
|
||||
for earlier in (
|
||||
"en_core_cognition_v1", "en_core_meta_v1", "en_core_attitude_v1",
|
||||
"en_core_temporal_v1", "en_core_action_v1", "en_core_quantitative_v1",
|
||||
"en_core_spatial_v1", "en_core_causation_v1",
|
||||
):
|
||||
assert DEFAULT_RESOLVABLE_PACK_IDS.index(earlier) < DEFAULT_RESOLVABLE_PACK_IDS.index(PACK_ID)
|
||||
|
||||
|
||||
def test_resolver_routes_polarity_lemmas_to_this_pack() -> None:
|
||||
for lemma in EXPECTED_LEMMAS:
|
||||
resolved = resolve_lemma(lemma)
|
||||
assert resolved is not None and resolved[0] == PACK_ID
|
||||
assert resolved[1][0].startswith("polarity.")
|
||||
|
||||
|
||||
def test_certain_certainly_stay_in_attitude_pack() -> None:
|
||||
"""``certain`` and ``uncertain`` were deliberately excluded —
|
||||
en_core_attitude_v1 owns the epistemic-certainty cluster."""
|
||||
for lemma in ("certain", "uncertain"):
|
||||
resolved = resolve_lemma(lemma)
|
||||
assert resolved is not None and resolved[0] == "en_core_attitude_v1"
|
||||
|
||||
|
||||
def test_prior_pack_lemma_resolution_unchanged() -> None:
|
||||
for lemma, expected in (
|
||||
("truth", "en_core_cognition_v1"),
|
||||
("doubt", "en_core_meta_v1"),
|
||||
("true", "en_core_attitude_v1"),
|
||||
("now", "en_core_temporal_v1"),
|
||||
("do", "en_core_action_v1"),
|
||||
("all", "en_core_quantitative_v1"),
|
||||
("here", "en_core_spatial_v1"),
|
||||
("effect", "en_core_causation_v1"),
|
||||
):
|
||||
resolved = resolve_lemma(lemma)
|
||||
assert resolved is not None and resolved[0] == expected
|
||||
Loading…
Reference in a new issue