From 891ffa89692f918403e49576c0114260cefe5c20 Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 18 May 2026 21:06:54 -0700 Subject: [PATCH] =?UTF-8?q?feat(packs):=20en=5Fcore=5Fquantitative=5Fv1=20?= =?UTF-8?q?=E2=80=94=20quantifiers=20+=20numeric=20basics=20(24=20lemmas)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Workstream 1 fifth pack. Closes the quantifier + basic-numeric gap. Prior packs had zero coverage of universal / existential / comparative quantifiers — queries about *all*, *some*, *many*, *more*, *most* all fell through to OOV. Pack composition (24 entries — mixed POS, 18 DET / 3 NUM / 2 ADJ / 1 NOUN): quantitative.universal.* (6 DET) all every each both none neither quantitative.existential.* (6 DET) some any several few many much quantitative.comparative.* (6 DET) more less fewer most least enough quantitative.numeric.* (3 NUM) one two three quantitative.unit.* (3 mix) single (ADJ) half (NOUN) whole (ADJ) The composer is POS-agnostic; surface composition uses ``semantic_domains`` rather than POS, so DET/NUM/ADJ/NOUN entries all surface identically. Files: language_packs/data/en_core_quantitative_v1/ lexicon.jsonl — 24 entries, SHA-256 checksum-sealed manifest.json — operational_base / D0 chat/pack_resolver.py Appended to DEFAULT_RESOLVABLE_PACK_IDS after action. core/config.py Added to RuntimeConfig.input_packs default mount. tests/test_en_core_quantitative_v1_pack.py 11 contract tests (load / POS-dist / namespace / no-collision / contiguous-ids / mount / resolver-order / routing / invariance). Authoring: Three parallel subagents — universal+existential / comparative / numeric. Strict exemplar + forbidden-lemma list against all 7 prior packs. Verification: Full lane: 2192 passed, 2 skipped, 0 failed. Cognition eval byte-identical on both splits. --- chat/pack_resolver.py | 1 + core/config.py | 1 + .../en_core_quantitative_v1/lexicon.jsonl | 24 ++++ .../en_core_quantitative_v1/manifest.json | 13 ++ tests/test_en_core_quantitative_v1_pack.py | 133 ++++++++++++++++++ 5 files changed, 172 insertions(+) create mode 100644 language_packs/data/en_core_quantitative_v1/lexicon.jsonl create mode 100644 language_packs/data/en_core_quantitative_v1/manifest.json create mode 100644 tests/test_en_core_quantitative_v1_pack.py diff --git a/chat/pack_resolver.py b/chat/pack_resolver.py index 80e1e887..7e06600b 100644 --- a/chat/pack_resolver.py +++ b/chat/pack_resolver.py @@ -46,6 +46,7 @@ DEFAULT_RESOLVABLE_PACK_IDS: tuple[str, ...] = ( "en_core_attitude_v1", "en_core_temporal_v1", "en_core_action_v1", + "en_core_quantitative_v1", "en_core_relations_v1", "en_core_relations_v2", ) diff --git a/core/config.py b/core/config.py index 9a0e7534..9338c4be 100644 --- a/core/config.py +++ b/core/config.py @@ -17,6 +17,7 @@ class RuntimeConfig: "en_core_attitude_v1", "en_core_temporal_v1", "en_core_action_v1", + "en_core_quantitative_v1", "en_core_relations_v1", "en_core_relations_v2", "he_logos_micro_v1", diff --git a/language_packs/data/en_core_quantitative_v1/lexicon.jsonl b/language_packs/data/en_core_quantitative_v1/lexicon.jsonl new file mode 100644 index 00000000..95e947b9 --- /dev/null +++ b/language_packs/data/en_core_quantitative_v1/lexicon.jsonl @@ -0,0 +1,24 @@ +{"entry_id":"en-core-quant-001","surface":"all","lemma":"all","language":"en","pos":"DET","semantic_domains":["quantitative.universal.total","quantitative.universal","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-002","surface":"every","lemma":"every","language":"en","pos":"DET","semantic_domains":["quantitative.universal.total","quantitative.universal","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-003","surface":"each","lemma":"each","language":"en","pos":"DET","semantic_domains":["quantitative.universal.total","quantitative.universal","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-004","surface":"both","lemma":"both","language":"en","pos":"DET","semantic_domains":["quantitative.universal.pair","quantitative.universal","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-005","surface":"none","lemma":"none","language":"en","pos":"DET","semantic_domains":["quantitative.universal.empty","quantitative.universal","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-006","surface":"neither","lemma":"neither","language":"en","pos":"DET","semantic_domains":["quantitative.universal.empty","quantitative.universal","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-007","surface":"some","lemma":"some","language":"en","pos":"DET","semantic_domains":["quantitative.existential.indefinite","quantitative.existential","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-008","surface":"any","lemma":"any","language":"en","pos":"DET","semantic_domains":["quantitative.existential.indefinite","quantitative.existential","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-009","surface":"several","lemma":"several","language":"en","pos":"DET","semantic_domains":["quantitative.existential.plural_few","quantitative.existential","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-010","surface":"few","lemma":"few","language":"en","pos":"DET","semantic_domains":["quantitative.existential.plural_few","quantitative.existential","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-011","surface":"many","lemma":"many","language":"en","pos":"DET","semantic_domains":["quantitative.existential.plural_many","quantitative.existential","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-012","surface":"much","lemma":"much","language":"en","pos":"DET","semantic_domains":["quantitative.existential.plural_many","quantitative.existential","cognition.set"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-013","surface":"more","lemma":"more","language":"en","pos":"DET","semantic_domains":["quantitative.comparative.greater","quantitative.comparative","cognition.scale"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-014","surface":"less","lemma":"less","language":"en","pos":"DET","semantic_domains":["quantitative.comparative.lesser","quantitative.comparative","cognition.scale"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-015","surface":"fewer","lemma":"fewer","language":"en","pos":"DET","semantic_domains":["quantitative.comparative.lesser","quantitative.comparative","cognition.scale"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-016","surface":"most","lemma":"most","language":"en","pos":"DET","semantic_domains":["quantitative.comparative.greater","quantitative.comparative","cognition.scale"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-017","surface":"least","lemma":"least","language":"en","pos":"DET","semantic_domains":["quantitative.comparative.lesser","quantitative.comparative","cognition.scale"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-018","surface":"enough","lemma":"enough","language":"en","pos":"DET","semantic_domains":["quantitative.comparative.adequacy","quantitative.comparative","cognition.scale"],"morphology_tags":["determiner"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-019","surface":"one","lemma":"one","language":"en","pos":"NUM","semantic_domains":["quantitative.numeric.cardinal","quantitative.numeric","cognition.count"],"morphology_tags":["numeral"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-020","surface":"two","lemma":"two","language":"en","pos":"NUM","semantic_domains":["quantitative.numeric.cardinal","quantitative.numeric","cognition.count"],"morphology_tags":["numeral"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-021","surface":"three","lemma":"three","language":"en","pos":"NUM","semantic_domains":["quantitative.numeric.cardinal","quantitative.numeric","cognition.count"],"morphology_tags":["numeral"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-022","surface":"single","lemma":"single","language":"en","pos":"ADJ","semantic_domains":["quantitative.unit.single","quantitative.unit","cognition.count"],"morphology_tags":["adjective"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-023","surface":"half","lemma":"half","language":"en","pos":"NOUN","semantic_domains":["quantitative.unit.fraction","quantitative.unit","cognition.count"],"morphology_tags":["noun"],"provenance_ids":["seed:core_quant_v1"]} +{"entry_id":"en-core-quant-024","surface":"whole","lemma":"whole","language":"en","pos":"ADJ","semantic_domains":["quantitative.unit.complete","quantitative.unit","cognition.count"],"morphology_tags":["adjective"],"provenance_ids":["seed:core_quant_v1"]} diff --git a/language_packs/data/en_core_quantitative_v1/manifest.json b/language_packs/data/en_core_quantitative_v1/manifest.json new file mode 100644 index 00000000..af8a3e21 --- /dev/null +++ b/language_packs/data/en_core_quantitative_v1/manifest.json @@ -0,0 +1,13 @@ +{ + "pack_id": "en_core_quantitative_v1", + "language": "en", + "role": "operational_base", + "script": "Latin", + "normalization_policy": "unitize_versor", + "source_manifest": "en_core_quantitative_v1.lexicon.jsonl", + "determinism_class": "D0", + "checksum": "4cd9001879be2b8da44a852ce6763f784fd690f272181402286e395894f9ac7a", + "version": "1.0.0", + "gate_engaged": true, + "oov_policy": "tagged_fallback" +} diff --git a/tests/test_en_core_quantitative_v1_pack.py b/tests/test_en_core_quantitative_v1_pack.py new file mode 100644 index 00000000..6377d8d0 --- /dev/null +++ b/tests/test_en_core_quantitative_v1_pack.py @@ -0,0 +1,133 @@ +"""``en_core_quantitative_v1`` — quantifier + numeric basics pack tests. + +Quantifiers and basic numerics had zero coverage in any prior pack — +queries like "What does some mean?" or "What is a few?" all fell +through to OOV. 24 entries across: + + - quantitative.universal.* (6 DET) all every each both none neither + - quantitative.existential.* (6 DET) some any several few many much + - quantitative.comparative.* (6 DET) more less fewer most least enough + - quantitative.numeric.* (3 NUM) one two three + - quantitative.unit.* (3 mix) single (ADJ) half (NOUN) whole (ADJ) + +Mixed POS distribution (18 DET / 3 NUM / 2 ADJ / 1 NOUN) — the resolver +is POS-agnostic, surface composition uses semantic_domains. + +Contracts pinned: checksum-verified load, primary-domain namespace +``quantitative.*``, no collision with the 7 prior packs, mounted by +default, registered after en_core_action_v1, prior-pack resolution +unchanged. +""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path + +from chat.pack_resolver import DEFAULT_RESOLVABLE_PACK_IDS, resolve_lemma +from core.config import RuntimeConfig +from language_packs.compiler import load_pack + + +PACK_ID = "en_core_quantitative_v1" +_PACK_ROOT = Path(__file__).resolve().parent.parent / "language_packs" / "data" / PACK_ID + +EXPECTED_TOTAL = 24 +EXPECTED_POS_COUNTS = {"DET": 18, "NUM": 3, "ADJ": 2, "NOUN": 1} + +EXPECTED_LEMMAS: tuple[str, ...] = ( + "all", "every", "each", "both", "none", "neither", + "some", "any", "several", "few", "many", "much", + "more", "less", "fewer", "most", "least", "enough", + "one", "two", "three", "single", "half", "whole", +) + + +def _read_lexicon() -> list[dict]: + return [ + json.loads(line) + for line in (_PACK_ROOT / "lexicon.jsonl").read_text("utf-8").splitlines() + if line.strip() + ] + + +def test_pack_loads_with_matching_checksum() -> None: + manifest, manifold = load_pack(PACK_ID) + assert manifest.pack_id == PACK_ID + assert len(manifest.checksum) == 64 + assert len(manifold) == EXPECTED_TOTAL + + +def test_pos_distribution_matches_design() -> None: + pos_counts = Counter(e["pos"] for e in _read_lexicon()) + assert dict(pos_counts) == EXPECTED_POS_COUNTS + + +def test_all_expected_lemmas_present() -> None: + _, manifold = load_pack(PACK_ID) + assert {manifold.get_word_at(i) for i in range(len(manifold))} == set(EXPECTED_LEMMAS) + + +def test_every_entry_has_quantitative_namespace_primary_domain() -> None: + for entry in _read_lexicon(): + assert entry["semantic_domains"][0].startswith("quantitative."), entry + + +def test_no_collision_with_prior_packs() -> None: + prior: set[str] = set() + for pack in ( + "en_core_cognition_v1", "en_core_meta_v1", "en_core_attitude_v1", + "en_core_temporal_v1", "en_core_action_v1", + "en_core_relations_v1", "en_core_relations_v2", + ): + for line in (_PACK_ROOT.parent / pack / "lexicon.jsonl").read_text("utf-8").splitlines(): + if line.strip(): + prior.add(json.loads(line)["lemma"].lower()) + for entry in _read_lexicon(): + assert entry["lemma"].lower() not in prior, entry + + +def test_provenance_is_seed_core_quant_v1() -> None: + for entry in _read_lexicon(): + assert entry["provenance_ids"] == ["seed:core_quant_v1"], entry + + +def test_entry_ids_contiguous_and_zero_padded() -> None: + entries = sorted(_read_lexicon(), key=lambda d: d["entry_id"]) + for i, entry in enumerate(entries, start=1): + assert entry["entry_id"] == f"en-core-quant-{i:03d}", entry["entry_id"] + + +def test_pack_mounted_in_default_runtime_config() -> None: + assert PACK_ID in RuntimeConfig().input_packs + + +def test_pack_registered_after_prior_content_packs() -> None: + assert PACK_ID in DEFAULT_RESOLVABLE_PACK_IDS + for earlier in ( + "en_core_cognition_v1", "en_core_meta_v1", "en_core_attitude_v1", + "en_core_temporal_v1", "en_core_action_v1", + ): + assert DEFAULT_RESOLVABLE_PACK_IDS.index(earlier) < DEFAULT_RESOLVABLE_PACK_IDS.index(PACK_ID) + + +def test_resolver_routes_quantitative_lemmas_to_this_pack() -> None: + for lemma in EXPECTED_LEMMAS: + resolved = resolve_lemma(lemma) + assert resolved is not None and resolved[0] == PACK_ID, ( + f"{lemma!r} resolved to {resolved}" + ) + assert resolved[1][0].startswith("quantitative.") + + +def test_prior_pack_lemma_resolution_unchanged() -> None: + for lemma, expected in ( + ("truth", "en_core_cognition_v1"), + ("doubt", "en_core_meta_v1"), + ("true", "en_core_attitude_v1"), + ("now", "en_core_temporal_v1"), + ("do", "en_core_action_v1"), + ): + resolved = resolve_lemma(lemma) + assert resolved is not None and resolved[0] == expected