feat(packs): en_core_quantitative_v1 — quantifiers + numeric basics (24 lemmas)
Workstream 1 fifth pack. Closes the quantifier + basic-numeric gap.
Prior packs had zero coverage of universal / existential / comparative
quantifiers — queries about *all*, *some*, *many*, *more*, *most* all
fell through to OOV.
Pack composition (24 entries — mixed POS, 18 DET / 3 NUM / 2 ADJ / 1 NOUN):
quantitative.universal.* (6 DET) all every each both none neither
quantitative.existential.* (6 DET) some any several few many much
quantitative.comparative.* (6 DET) more less fewer most least enough
quantitative.numeric.* (3 NUM) one two three
quantitative.unit.* (3 mix) single (ADJ) half (NOUN) whole (ADJ)
The composer is POS-agnostic; surface composition uses
``semantic_domains`` rather than POS, so DET/NUM/ADJ/NOUN entries all
surface identically.
Files:
language_packs/data/en_core_quantitative_v1/
lexicon.jsonl — 24 entries, SHA-256 checksum-sealed
manifest.json — operational_base / D0
chat/pack_resolver.py
Appended to DEFAULT_RESOLVABLE_PACK_IDS after action.
core/config.py
Added to RuntimeConfig.input_packs default mount.
tests/test_en_core_quantitative_v1_pack.py
11 contract tests (load / POS-dist / namespace / no-collision /
contiguous-ids / mount / resolver-order / routing / invariance).
Authoring:
Three parallel subagents — universal+existential / comparative /
numeric. Strict exemplar + forbidden-lemma list against all 7
prior packs.
Verification:
Full lane: 2192 passed, 2 skipped, 0 failed.
Cognition eval byte-identical on both splits.
This commit is contained in:
Shay
parent
cb1eba72ae
commit
891ffa8969
5 changed files with 172 additions and 0 deletions
|
|
@ -46,6 +46,7 @@ DEFAULT_RESOLVABLE_PACK_IDS: tuple[str, ...] = (
|
|||
"en_core_attitude_v1",
|
||||
"en_core_temporal_v1",
|
||||
"en_core_action_v1",
|
||||
"en_core_quantitative_v1",
|
||||
"en_core_relations_v1",
|
||||
"en_core_relations_v2",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ class RuntimeConfig:
|
|||
"en_core_attitude_v1",
|
||||
"en_core_temporal_v1",
|
||||
"en_core_action_v1",
|
||||
"en_core_quantitative_v1",
|
||||
"en_core_relations_v1",
|
||||
"en_core_relations_v2",
|
||||
"he_logos_micro_v1",
|
||||
|
|
|
|||
24
language_packs/data/en_core_quantitative_v1/lexicon.jsonl
Normal file
24
language_packs/data/en_core_quantitative_v1/lexicon.jsonl
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
{"entry_id":"en-core-quant-001","surface":"all","lemma":"all","language":"en","pos":"DET","semantic_domains":["quantitative.universal.total","quantitative.universal","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-002","surface":"every","lemma":"every","language":"en","pos":"DET","semantic_domains":["quantitative.universal.total","quantitative.universal","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-003","surface":"each","lemma":"each","language":"en","pos":"DET","semantic_domains":["quantitative.universal.total","quantitative.universal","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-004","surface":"both","lemma":"both","language":"en","pos":"DET","semantic_domains":["quantitative.universal.pair","quantitative.universal","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-005","surface":"none","lemma":"none","language":"en","pos":"DET","semantic_domains":["quantitative.universal.empty","quantitative.universal","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-006","surface":"neither","lemma":"neither","language":"en","pos":"DET","semantic_domains":["quantitative.universal.empty","quantitative.universal","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-007","surface":"some","lemma":"some","language":"en","pos":"DET","semantic_domains":["quantitative.existential.indefinite","quantitative.existential","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-008","surface":"any","lemma":"any","language":"en","pos":"DET","semantic_domains":["quantitative.existential.indefinite","quantitative.existential","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-009","surface":"several","lemma":"several","language":"en","pos":"DET","semantic_domains":["quantitative.existential.plural_few","quantitative.existential","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-010","surface":"few","lemma":"few","language":"en","pos":"DET","semantic_domains":["quantitative.existential.plural_few","quantitative.existential","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-011","surface":"many","lemma":"many","language":"en","pos":"DET","semantic_domains":["quantitative.existential.plural_many","quantitative.existential","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-012","surface":"much","lemma":"much","language":"en","pos":"DET","semantic_domains":["quantitative.existential.plural_many","quantitative.existential","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-013","surface":"more","lemma":"more","language":"en","pos":"DET","semantic_domains":["quantitative.comparative.greater","quantitative.comparative","cognition.scale"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-014","surface":"less","lemma":"less","language":"en","pos":"DET","semantic_domains":["quantitative.comparative.lesser","quantitative.comparative","cognition.scale"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-015","surface":"fewer","lemma":"fewer","language":"en","pos":"DET","semantic_domains":["quantitative.comparative.lesser","quantitative.comparative","cognition.scale"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-016","surface":"most","lemma":"most","language":"en","pos":"DET","semantic_domains":["quantitative.comparative.greater","quantitative.comparative","cognition.scale"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-017","surface":"least","lemma":"least","language":"en","pos":"DET","semantic_domains":["quantitative.comparative.lesser","quantitative.comparative","cognition.scale"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-018","surface":"enough","lemma":"enough","language":"en","pos":"DET","semantic_domains":["quantitative.comparative.adequacy","quantitative.comparative","cognition.scale"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-019","surface":"one","lemma":"one","language":"en","pos":"NUM","semantic_domains":["quantitative.numeric.cardinal","quantitative.numeric","cognition.count"],"morphology_tags":["numeral"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-020","surface":"two","lemma":"two","language":"en","pos":"NUM","semantic_domains":["quantitative.numeric.cardinal","quantitative.numeric","cognition.count"],"morphology_tags":["numeral"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-021","surface":"three","lemma":"three","language":"en","pos":"NUM","semantic_domains":["quantitative.numeric.cardinal","quantitative.numeric","cognition.count"],"morphology_tags":["numeral"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-022","surface":"single","lemma":"single","language":"en","pos":"ADJ","semantic_domains":["quantitative.unit.single","quantitative.unit","cognition.count"],"morphology_tags":["adjective"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-023","surface":"half","lemma":"half","language":"en","pos":"NOUN","semantic_domains":["quantitative.unit.fraction","quantitative.unit","cognition.count"],"morphology_tags":["noun"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
{"entry_id":"en-core-quant-024","surface":"whole","lemma":"whole","language":"en","pos":"ADJ","semantic_domains":["quantitative.unit.complete","quantitative.unit","cognition.count"],"morphology_tags":["adjective"],"provenance_ids":["seed:core_quant_v1"]}
|
||||
13
language_packs/data/en_core_quantitative_v1/manifest.json
Normal file
13
language_packs/data/en_core_quantitative_v1/manifest.json
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"pack_id": "en_core_quantitative_v1",
|
||||
"language": "en",
|
||||
"role": "operational_base",
|
||||
"script": "Latin",
|
||||
"normalization_policy": "unitize_versor",
|
||||
"source_manifest": "en_core_quantitative_v1.lexicon.jsonl",
|
||||
"determinism_class": "D0",
|
||||
"checksum": "4cd9001879be2b8da44a852ce6763f784fd690f272181402286e395894f9ac7a",
|
||||
"version": "1.0.0",
|
||||
"gate_engaged": true,
|
||||
"oov_policy": "tagged_fallback"
|
||||
}
|
||||
133
tests/test_en_core_quantitative_v1_pack.py
Normal file
133
tests/test_en_core_quantitative_v1_pack.py
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
"""``en_core_quantitative_v1`` — quantifier + numeric basics pack tests.
|
||||
|
||||
Quantifiers and basic numerics had zero coverage in any prior pack —
|
||||
queries like "What does some mean?" or "What is a few?" all fell
|
||||
through to OOV. 24 entries across:
|
||||
|
||||
- quantitative.universal.* (6 DET) all every each both none neither
|
||||
- quantitative.existential.* (6 DET) some any several few many much
|
||||
- quantitative.comparative.* (6 DET) more less fewer most least enough
|
||||
- quantitative.numeric.* (3 NUM) one two three
|
||||
- quantitative.unit.* (3 mix) single (ADJ) half (NOUN) whole (ADJ)
|
||||
|
||||
Mixed POS distribution (18 DET / 3 NUM / 2 ADJ / 1 NOUN) — the resolver
|
||||
is POS-agnostic, surface composition uses semantic_domains.
|
||||
|
||||
Contracts pinned: checksum-verified load, primary-domain namespace
|
||||
``quantitative.*``, no collision with the 7 prior packs, mounted by
|
||||
default, registered after en_core_action_v1, prior-pack resolution
|
||||
unchanged.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from chat.pack_resolver import DEFAULT_RESOLVABLE_PACK_IDS, resolve_lemma
|
||||
from core.config import RuntimeConfig
|
||||
from language_packs.compiler import load_pack
|
||||
|
||||
|
||||
PACK_ID = "en_core_quantitative_v1"
|
||||
_PACK_ROOT = Path(__file__).resolve().parent.parent / "language_packs" / "data" / PACK_ID
|
||||
|
||||
EXPECTED_TOTAL = 24
|
||||
EXPECTED_POS_COUNTS = {"DET": 18, "NUM": 3, "ADJ": 2, "NOUN": 1}
|
||||
|
||||
EXPECTED_LEMMAS: tuple[str, ...] = (
|
||||
"all", "every", "each", "both", "none", "neither",
|
||||
"some", "any", "several", "few", "many", "much",
|
||||
"more", "less", "fewer", "most", "least", "enough",
|
||||
"one", "two", "three", "single", "half", "whole",
|
||||
)
|
||||
|
||||
|
||||
def _read_lexicon() -> list[dict]:
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in (_PACK_ROOT / "lexicon.jsonl").read_text("utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
|
||||
|
||||
def test_pack_loads_with_matching_checksum() -> None:
|
||||
manifest, manifold = load_pack(PACK_ID)
|
||||
assert manifest.pack_id == PACK_ID
|
||||
assert len(manifest.checksum) == 64
|
||||
assert len(manifold) == EXPECTED_TOTAL
|
||||
|
||||
|
||||
def test_pos_distribution_matches_design() -> None:
|
||||
pos_counts = Counter(e["pos"] for e in _read_lexicon())
|
||||
assert dict(pos_counts) == EXPECTED_POS_COUNTS
|
||||
|
||||
|
||||
def test_all_expected_lemmas_present() -> None:
|
||||
_, manifold = load_pack(PACK_ID)
|
||||
assert {manifold.get_word_at(i) for i in range(len(manifold))} == set(EXPECTED_LEMMAS)
|
||||
|
||||
|
||||
def test_every_entry_has_quantitative_namespace_primary_domain() -> None:
|
||||
for entry in _read_lexicon():
|
||||
assert entry["semantic_domains"][0].startswith("quantitative."), entry
|
||||
|
||||
|
||||
def test_no_collision_with_prior_packs() -> None:
|
||||
prior: set[str] = set()
|
||||
for pack in (
|
||||
"en_core_cognition_v1", "en_core_meta_v1", "en_core_attitude_v1",
|
||||
"en_core_temporal_v1", "en_core_action_v1",
|
||||
"en_core_relations_v1", "en_core_relations_v2",
|
||||
):
|
||||
for line in (_PACK_ROOT.parent / pack / "lexicon.jsonl").read_text("utf-8").splitlines():
|
||||
if line.strip():
|
||||
prior.add(json.loads(line)["lemma"].lower())
|
||||
for entry in _read_lexicon():
|
||||
assert entry["lemma"].lower() not in prior, entry
|
||||
|
||||
|
||||
def test_provenance_is_seed_core_quant_v1() -> None:
|
||||
for entry in _read_lexicon():
|
||||
assert entry["provenance_ids"] == ["seed:core_quant_v1"], entry
|
||||
|
||||
|
||||
def test_entry_ids_contiguous_and_zero_padded() -> None:
|
||||
entries = sorted(_read_lexicon(), key=lambda d: d["entry_id"])
|
||||
for i, entry in enumerate(entries, start=1):
|
||||
assert entry["entry_id"] == f"en-core-quant-{i:03d}", entry["entry_id"]
|
||||
|
||||
|
||||
def test_pack_mounted_in_default_runtime_config() -> None:
|
||||
assert PACK_ID in RuntimeConfig().input_packs
|
||||
|
||||
|
||||
def test_pack_registered_after_prior_content_packs() -> None:
|
||||
assert PACK_ID in DEFAULT_RESOLVABLE_PACK_IDS
|
||||
for earlier in (
|
||||
"en_core_cognition_v1", "en_core_meta_v1", "en_core_attitude_v1",
|
||||
"en_core_temporal_v1", "en_core_action_v1",
|
||||
):
|
||||
assert DEFAULT_RESOLVABLE_PACK_IDS.index(earlier) < DEFAULT_RESOLVABLE_PACK_IDS.index(PACK_ID)
|
||||
|
||||
|
||||
def test_resolver_routes_quantitative_lemmas_to_this_pack() -> None:
|
||||
for lemma in EXPECTED_LEMMAS:
|
||||
resolved = resolve_lemma(lemma)
|
||||
assert resolved is not None and resolved[0] == PACK_ID, (
|
||||
f"{lemma!r} resolved to {resolved}"
|
||||
)
|
||||
assert resolved[1][0].startswith("quantitative.")
|
||||
|
||||
|
||||
def test_prior_pack_lemma_resolution_unchanged() -> None:
|
||||
for lemma, expected in (
|
||||
("truth", "en_core_cognition_v1"),
|
||||
("doubt", "en_core_meta_v1"),
|
||||
("true", "en_core_attitude_v1"),
|
||||
("now", "en_core_temporal_v1"),
|
||||
("do", "en_core_action_v1"),
|
||||
):
|
||||
resolved = resolve_lemma(lemma)
|
||||
assert resolved is not None and resolved[0] == expected
|
||||
Loading…
Reference in a new issue