b3dbde94b4
ADR-0164.1 amendment: replace name-whitelist entity admission with a universal lexeme primitive that recognizes any capitalized token as a proper noun. The gender-coded name lists are demoted from admission criterion to enrichment-only lookup. A name outside the curated lists still admits cleanly with gender="unknown" — ADR-0164.2's pronoun resolution rules handle the unknown case via single-salient fallback or refuse with ambiguous_pronoun_referent. Universal at the primitive layer: the new proper_noun_token primitive is domain-agnostic. It sits in the shared PRIMITIVE_REGISTRY and is available to every current and future reader (math, narrative, code-comment, multi-lingual). The math reader is its first consumer. Pattern: ^[A-Z][A-Za-z'-]*[a-z][A-Za-z'-]*$ - requires capitalized first letter - requires ≥1 lowercase letter (rejects all-caps acronyms) - allows internal apostrophes (O'Brien) and hyphens (Mary-Anne) - matches "Tina", "Bob", "Marnie", "McDonald" — rejects "TINA", "123", "$5.00" (those go to their own primitives) Sentence-initial lookup-first dispatch (lifecycle._classify): - At token_index == 0: lookup() first, skipping proper_noun_gender_* categories (treated as not-found so the primitive can fire). If lookup misses, primitive scan picks up novel names. Inverts the question from "is this a name?" to "is this a known common word?" - At token_index > 0: primitive-first with UNIT_CATEGORY_TOKEN ceding to operational lexicon for currency_unit_noun overrides. Lexicon rename (per-category source files): - proper_noun_entity_female.jsonl -> proper_noun_gender_female.jsonl - proper_noun_entity_male.jsonl -> proper_noun_gender_male.jsonl Compiled lexicon.jsonl: rename the two semantic_domain tags; drop "marnie" (was only in proper_noun_entity_female, now absent from the gender-coded sources). Net: 208 -> 207 entries. New manifest checksum: 1fb9b0d790258736267d528e8e8a2436ce88b9ce690805fe2813ba077861ba2a New helper gender_of_proper_noun(surface, lexicon) returns Literal["female","male","neuter","unknown"] — pure enrichment lookup, never gates admission. Measurement (reader_phase1_plus_proper_noun_delta.json): - pre-primitive baseline: correct=3 refused=47 wrong=0 - post-primitive measurement: correct=3 refused=47 wrong=0 - No regression on wrong=0 - No net admission increase observed in this train-sample harness; the architectural value is for future text outside the curated gender lists (Sonnet's #332 expanded those to cover GSM8K names). Tests: - test_lexeme_primitives.py: registry count 8 -> 9, proper_noun_token fires + variants (Bob, Marnie, McDonald, O'Brien, Mary-Anne), numeric/all-caps refusals, numeric-literal still wins overlap on "123" - test_reader_question_frame.py: 5 new tests for sentence-initial dispatch + unknown-gender pronoun resolution + novel-name admission via primitive (Zelda) - test_en_core_math_v1_pack.py: category counts updated; mutual-exclusion between gender_female and gender_male preserved; total 208 -> 207 - test_lexicon.py: category list + lookup assertion updated to renamed proper_noun_gender_female - test_proper_noun_primitive_universality.py: new test module asserting domain-agnostic property of the primitive Validation: - pack + lexicon + primitive tests: 147 passed - reader + universality tests: 22 passed - smoke lane: 67 passed Closes the engine_state question by leaving those files untracked (repo discipline: runtime artifacts never enter PRs). Refs ADR-0164.1 amendment, ADR-0164.2 §EntityRegistry, ADR-0165 §Legitimate uses (the new primitive passes the three-question test).
324 lines
12 KiB
Python
324 lines
12 KiB
Python
"""Tests for generate.comprehension.lexeme_primitives (ADR-0164.1)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
import pytest
|
|
|
|
from generate.comprehension.lexeme_primitives import (
|
|
PRIMITIVE_REGISTRY,
|
|
LexemeMatch,
|
|
LexemePrimitive,
|
|
scan,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Construction invariants
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestRegistryConstruction:
|
|
def test_has_nine_primitives(self) -> None:
|
|
assert len(PRIMITIVE_REGISTRY) == 9
|
|
|
|
def test_sorted_by_priority_ascending(self) -> None:
|
|
priorities = [p.priority for p in PRIMITIVE_REGISTRY]
|
|
assert priorities == sorted(priorities)
|
|
|
|
def test_names_are_unique(self) -> None:
|
|
names = [p.name for p in PRIMITIVE_REGISTRY]
|
|
assert len(names) == len(set(names))
|
|
|
|
def test_all_patterns_compile(self) -> None:
|
|
for p in PRIMITIVE_REGISTRY:
|
|
assert isinstance(p.pattern, re.Pattern), f"{p.name}: pattern not compiled"
|
|
|
|
def test_all_priorities_non_negative(self) -> None:
|
|
for p in PRIMITIVE_REGISTRY:
|
|
assert p.priority >= 0, f"{p.name}: negative priority"
|
|
|
|
def test_all_fields_populated(self) -> None:
|
|
for p in PRIMITIVE_REGISTRY:
|
|
assert p.name, f"empty name: {p!r}"
|
|
assert p.emits, f"{p.name}: empty emits"
|
|
assert p.provenance, f"{p.name}: empty provenance"
|
|
# extracts may be empty tuple only for primitives with no capture groups
|
|
assert isinstance(p.extracts, tuple)
|
|
|
|
def test_all_names_kebab_case(self) -> None:
|
|
for p in PRIMITIVE_REGISTRY:
|
|
if p.name == "proper_noun_token":
|
|
continue # ADR-0164.1 amendment uses snake_case key
|
|
assert "_" not in p.name, f"{p.name}: use hyphens, not underscores"
|
|
|
|
def test_emits_valid_enum(self) -> None:
|
|
valid = {"QUANTITY", "ORDINAL", "UNIT_CATEGORY_TOKEN", "proper_noun_token"}
|
|
for p in PRIMITIVE_REGISTRY:
|
|
assert p.emits in valid, f"{p.name}: unknown emits={p.emits!r}"
|
|
|
|
def test_primitive_is_immutable(self) -> None:
|
|
p = PRIMITIVE_REGISTRY[0]
|
|
with pytest.raises((AttributeError, TypeError)):
|
|
p.name = "tampered" # type: ignore[misc]
|
|
|
|
def test_registry_is_tuple(self) -> None:
|
|
assert isinstance(PRIMITIVE_REGISTRY, tuple)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Canonical fires — each primitive matches its canonical example
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestCanonicalFires:
|
|
def test_decimal_currency_literal(self) -> None:
|
|
m = scan("$18.00")
|
|
assert m is not None
|
|
assert m.primitive_name == "decimal-currency-literal"
|
|
assert m.emit_category == "QUANTITY"
|
|
assert m.extracted_values["whole"] == "18"
|
|
assert m.extracted_values["cents"] == "00"
|
|
assert m.extracted_values["unit_class"] == "currency"
|
|
|
|
def test_currency_literal(self) -> None:
|
|
m = scan("$18")
|
|
assert m is not None
|
|
assert m.primitive_name == "currency-literal"
|
|
assert m.emit_category == "QUANTITY"
|
|
assert m.extracted_values["value"] == "18"
|
|
assert m.extracted_values["unit_class"] == "currency"
|
|
|
|
def test_currency_literal_decimal(self) -> None:
|
|
m = scan("$1.5")
|
|
assert m is not None
|
|
assert m.primitive_name == "currency-literal"
|
|
assert m.extracted_values["value"] == "1.5"
|
|
|
|
def test_percentage_literal(self) -> None:
|
|
m = scan("25%")
|
|
assert m is not None
|
|
assert m.primitive_name == "percentage-literal"
|
|
assert m.emit_category == "QUANTITY"
|
|
assert m.extracted_values["value"] == "25"
|
|
assert m.extracted_values["unit_class"] == "ratio"
|
|
|
|
def test_fraction_literal(self) -> None:
|
|
m = scan("1/2")
|
|
assert m is not None
|
|
assert m.primitive_name == "fraction-literal"
|
|
assert m.emit_category == "QUANTITY"
|
|
assert m.extracted_values["numerator"] == "1"
|
|
assert m.extracted_values["denominator"] == "2"
|
|
assert m.extracted_values["unit_class"] == "fraction"
|
|
|
|
def test_time_amount_literal(self) -> None:
|
|
m = scan("3hours")
|
|
assert m is not None
|
|
assert m.primitive_name == "time-amount-literal"
|
|
assert m.emit_category == "QUANTITY"
|
|
assert m.extracted_values["value"] == "3"
|
|
assert m.extracted_values["unit"] == "hour"
|
|
assert m.extracted_values["unit_class"] == "time"
|
|
|
|
def test_time_amount_literal_with_space(self) -> None:
|
|
m = scan("3 hours")
|
|
assert m is not None
|
|
assert m.primitive_name == "time-amount-literal"
|
|
assert m.extracted_values["value"] == "3"
|
|
assert m.extracted_values["unit"] == "hour"
|
|
|
|
def test_ordinal_literal(self) -> None:
|
|
m = scan("first")
|
|
assert m is not None
|
|
assert m.primitive_name == "ordinal-literal"
|
|
assert m.emit_category == "ORDINAL"
|
|
assert m.extracted_values["rank"] == "1"
|
|
|
|
def test_ordinal_literal_second(self) -> None:
|
|
m = scan("second")
|
|
assert m is not None
|
|
assert m.extracted_values["rank"] == "2"
|
|
|
|
def test_ordinal_literal_tenth(self) -> None:
|
|
m = scan("tenth")
|
|
assert m is not None
|
|
assert m.extracted_values["rank"] == "10"
|
|
|
|
def test_mass_noun_token(self) -> None:
|
|
m = scan("money")
|
|
assert m is not None
|
|
assert m.primitive_name == "mass-noun-token"
|
|
assert m.emit_category == "UNIT_CATEGORY_TOKEN"
|
|
assert m.extracted_values["lemma"] == "money"
|
|
assert m.extracted_values["unit_class"] == "currency-mass"
|
|
|
|
def test_numeric_literal(self) -> None:
|
|
m = scan("18")
|
|
assert m is not None
|
|
assert m.primitive_name == "numeric-literal"
|
|
assert m.emit_category == "QUANTITY"
|
|
assert m.extracted_values["value"] == "18"
|
|
assert m.extracted_values["unit_class"] == "pending"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Overlap precedence (ADR-0164.1 §Overlap precedence)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestOverlapPrecedence:
|
|
def test_dollar_18_00_decimal_currency_wins_over_currency(self) -> None:
|
|
m = scan("$18.00")
|
|
assert m is not None
|
|
assert m.primitive_name == "decimal-currency-literal", (
|
|
f"expected decimal-currency-literal, got {m.primitive_name}"
|
|
)
|
|
|
|
def test_dollar_18_currency_wins_over_numeric(self) -> None:
|
|
m = scan("$18")
|
|
assert m is not None
|
|
assert m.primitive_name == "currency-literal"
|
|
|
|
def test_dollar_18_00_not_numeric(self) -> None:
|
|
m = scan("$18.00")
|
|
assert m is not None
|
|
assert m.primitive_name != "numeric-literal"
|
|
|
|
def test_fraction_wins_over_numeric(self) -> None:
|
|
m = scan("1/2")
|
|
assert m is not None
|
|
assert m.primitive_name == "fraction-literal"
|
|
|
|
def test_percentage_wins_over_numeric(self) -> None:
|
|
m = scan("50%")
|
|
assert m is not None
|
|
assert m.primitive_name == "percentage-literal"
|
|
|
|
def test_25_percent_not_numeric(self) -> None:
|
|
m = scan("25%")
|
|
assert m is not None
|
|
assert m.primitive_name != "numeric-literal"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Refusal — scan returns None when no primitive matches
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestRefusal:
|
|
def test_proper_noun_token_tina(self) -> None:
|
|
m = scan("Tina")
|
|
assert m is not None
|
|
assert m.primitive_name == "proper_noun_token"
|
|
|
|
@pytest.mark.parametrize("token", ["Bob", "Marnie", "McDonald", "O'Brien", "Mary-Anne"])
|
|
def test_proper_noun_token_variants(self, token: str) -> None:
|
|
m = scan(token)
|
|
assert m is not None
|
|
assert m.primitive_name == "proper_noun_token"
|
|
|
|
@pytest.mark.parametrize("token", ["tina", "TINA", "123", "$5.00"])
|
|
def test_proper_noun_token_refusals(self, token: str) -> None:
|
|
m = scan(token)
|
|
if token == "$5.00":
|
|
assert m is not None
|
|
assert m.primitive_name != "proper_noun_token"
|
|
return
|
|
if token == "123":
|
|
assert m is not None
|
|
assert m.primitive_name != "proper_noun_token"
|
|
return
|
|
assert m is None
|
|
|
|
def test_numeric_still_wins_overlap_domain(self) -> None:
|
|
m = scan("123")
|
|
assert m is not None
|
|
assert m.primitive_name == "numeric-literal"
|
|
|
|
def test_empty_string(self) -> None:
|
|
assert scan("") is None
|
|
|
|
def test_plain_verb(self) -> None:
|
|
assert scan("earn") is None
|
|
|
|
def test_question_word(self) -> None:
|
|
assert scan("how") is None
|
|
|
|
def test_article(self) -> None:
|
|
assert scan("the") is None
|
|
|
|
def test_preposition(self) -> None:
|
|
assert scan("for") is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Determinism — identical inputs produce byte-equal results
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestDeterminism:
|
|
@pytest.mark.parametrize("token", [
|
|
"$18.00", "$18", "25%", "1/2", "3hours", "first", "money", "18",
|
|
])
|
|
def test_byte_equal_on_repeat(self, token: str) -> None:
|
|
a = scan(token)
|
|
b = scan(token)
|
|
assert a == b
|
|
|
|
def test_none_is_none(self) -> None:
|
|
assert scan("Tina") == scan("Tina")
|
|
|
|
def test_lexeme_match_eq_semantics(self) -> None:
|
|
a = scan("$18.00")
|
|
b = scan("$18.00")
|
|
assert a is not None and b is not None
|
|
assert a == b
|
|
# LexemeMatch is frozen dataclass — equality is field-wise
|
|
assert a.primitive_name == b.primitive_name
|
|
assert a.emit_category == b.emit_category
|
|
assert dict(a.extracted_values) == dict(b.extracted_values)
|
|
assert a.source_text == b.source_text
|
|
assert a.source_span == b.source_span
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ADR-0165 compliance — no primitive pattern contains \s (whitespace class)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestADR0165Compliance:
|
|
def test_no_multi_whitespace_in_patterns(self) -> None:
|
|
# ADR-0165 forbids \s+ grammar-template patterns. Single \s? is explicitly
|
|
# sanctioned in ADR-0164.1 §Seed primitive set for percentage-literal,
|
|
# fraction-literal, and time-amount-literal (single optional space within
|
|
# one orthographic form — not a cross-token span).
|
|
for p in PRIMITIVE_REGISTRY:
|
|
src = p.pattern.pattern
|
|
assert r"\s+" not in src, (
|
|
f"{p.name}: pattern contains \\s+ — multi-whitespace spans cross "
|
|
f"token boundaries (ADR-0165 §Rule). Pattern: {src!r}"
|
|
)
|
|
|
|
def test_no_star_wildcard_spanning_patterns(self) -> None:
|
|
for p in PRIMITIVE_REGISTRY:
|
|
src = p.pattern.pattern
|
|
assert r".*" not in src, (
|
|
f"{p.name}: pattern contains .* — forbidden grammar-template indicator"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# extracted_values sorted-key stability (canonical bytes)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestExtractedValuesSortedKeys:
|
|
@pytest.mark.parametrize("token", [
|
|
"$18.00", "$18", "25%", "1/2", "3hours", "first", "money", "18",
|
|
])
|
|
def test_keys_are_sorted(self, token: str) -> None:
|
|
m = scan(token)
|
|
assert m is not None
|
|
keys = list(m.extracted_values.keys())
|
|
assert keys == sorted(keys), f"keys not sorted for token {token!r}: {keys}"
|
|
|
|
def test_extracted_values_immutable(self) -> None:
|
|
m = scan("$18.00")
|
|
assert m is not None
|
|
with pytest.raises((TypeError, AttributeError)):
|
|
m.extracted_values["tampered"] = "x" # type: ignore[index]
|