feat(comprehension): lexeme primitive registry (ADR-0164.1) (#324)

Adds generate/comprehension/lexeme_primitives.py with the eight seed
primitives specified by ADR-0164.1:

  decimal-currency-literal (priority 10)
  currency-literal          (priority 20)
  percentage-literal        (priority 30)
  fraction-literal          (priority 40)
  time-amount-literal       (priority 50)
  ordinal-literal           (priority 60)
  mass-noun-token           (priority 70)
  numeric-literal           (priority 100)

LexemePrimitive and LexemeMatch are frozen/slots dataclasses. scan()
runs primitives in priority order and returns the first hit wrapped in
a MappingProxyType over sorted-key extracted_values for canonical-bytes
stability. All patterns use explicit space characters ([ ]?, [- ]?) not
\s so the ADR-0165 compliance invariant holds.

55 tests cover: construction invariants, canonical fires (each
primitive on its own example), overlap precedence ($18.00, 1/2, 50%),
refusal on Tina/empty/verbs, determinism, sorted-key stability, and
the ADR-0165 compliance smoke test.

generate/comprehension/lexeme_primitives.py

@@ -0,0 +1,197 @@
+"""ADR-0164.1 — Lexeme primitive registry.
+
+Eight seed primitives for the incremental comprehension reader's step-1
+lexical scan (ADR-0164 §Decision §3, ADR-0165 §Legitimate uses).
+
+Public API:
+  PRIMITIVE_REGISTRY  — immutable sorted tuple[LexemePrimitive, ...]
+  scan(token)         — first-hit match in priority order; None on miss
+"""
+
+from __future__ import annotations
+
+import re
+import types
+from dataclasses import dataclass
+from typing import Mapping
+
+# ---------------------------------------------------------------------------
+# Core types
+# ---------------------------------------------------------------------------
+
+_ORDINAL_RANKS: dict[str, str] = {
+    "first": "1",
+    "second": "2",
+    "third": "3",
+    "fourth": "4",
+    "fifth": "5",
+    "sixth": "6",
+    "seventh": "7",
+    "eighth": "8",
+    "ninth": "9",
+    "tenth": "10",
+}
+
+
+@dataclass(frozen=True, slots=True)
+class LexemePrimitive:
+    name: str
+    pattern: re.Pattern[str]
+    emits: str
+    extracts: tuple[str, ...]
+    priority: int
+    provenance: str
+
+
+@dataclass(frozen=True, slots=True)
+class LexemeMatch:
+    primitive_name: str
+    emit_category: str
+    extracted_values: Mapping[str, str]
+    source_text: str
+    source_span: tuple[int, int]
+
+
+# ---------------------------------------------------------------------------
+# Per-primitive constant fields (not captured by regex groups)
+# ---------------------------------------------------------------------------
+
+_PRIMITIVE_CONSTANTS: dict[str, dict[str, str]] = {
+    "decimal-currency-literal": {"unit_class": "currency"},
+    "currency-literal": {"unit_class": "currency"},
+    "percentage-literal": {"unit_class": "ratio"},
+    "fraction-literal": {"unit_class": "fraction"},
+    "time-amount-literal": {"unit_class": "time"},
+    "numeric-literal": {"unit_class": "pending"},
+    "ordinal-literal": {},
+    "mass-noun-token": {"unit_class": "currency-mass"},
+}
+
+
+def _make_registry() -> tuple[LexemePrimitive, ...]:
+    entries: list[LexemePrimitive] = [
+        LexemePrimitive(
+            name="decimal-currency-literal",
+            pattern=re.compile(r"\$(?P<whole>\d+)\.(?P<cents>\d{2})\b"),
+            emits="QUANTITY",
+            extracts=("whole", "cents"),
+            priority=10,
+            provenance="ADR-0164.1",
+        ),
+        LexemePrimitive(
+            name="currency-literal",
+            pattern=re.compile(r"\$(?P<value>\d+(?:\.\d+)?)\b"),
+            emits="QUANTITY",
+            extracts=("value",),
+            priority=20,
+            provenance="ADR-0164.1",
+        ),
+        LexemePrimitive(
+            name="percentage-literal",
+            pattern=re.compile(r"(?P<value>\d+(?:\.\d+)?)[ ]?%"),
+            emits="QUANTITY",
+            extracts=("value",),
+            priority=30,
+            provenance="ADR-0164.1",
+        ),
+        LexemePrimitive(
+            name="fraction-literal",
+            pattern=re.compile(r"(?P<numerator>\d+)[ ]?/[ ]?(?P<denominator>\d+)\b"),
+            emits="QUANTITY",
+            extracts=("numerator", "denominator"),
+            priority=40,
+            provenance="ADR-0164.1",
+        ),
+        LexemePrimitive(
+            name="time-amount-literal",
+            pattern=re.compile(
+                r"(?P<value>\d+)[- ]?"
+                r"(?P<unit>hour|minute|day|week|month|year|second)s?\b",
+                re.IGNORECASE,
+            ),
+            emits="QUANTITY",
+            extracts=("value", "unit"),
+            priority=50,
+            provenance="ADR-0164.1",
+        ),
+        LexemePrimitive(
+            name="ordinal-literal",
+            pattern=re.compile(
+                r"(?P<rank>first|second|third|fourth|fifth|"
+                r"sixth|seventh|eighth|ninth|tenth)\b",
+                re.IGNORECASE,
+            ),
+            emits="ORDINAL",
+            extracts=("rank",),
+            priority=60,
+            provenance="ADR-0164.1",
+        ),
+        LexemePrimitive(
+            name="mass-noun-token",
+            pattern=re.compile(
+                r"(?P<lemma>money|profit|interest|income|savings|cost|amount|total)\b",
+                re.IGNORECASE,
+            ),
+            emits="UNIT_CATEGORY_TOKEN",
+            extracts=("lemma",),
+            priority=70,
+            provenance="ADR-0164.1",
+        ),
+        LexemePrimitive(
+            name="numeric-literal",
+            pattern=re.compile(r"(?P<value>\d+(?:\.\d+)?)\b"),
+            emits="QUANTITY",
+            extracts=("value",),
+            priority=100,
+            provenance="ADR-0164.1",
+        ),
+    ]
+    return tuple(sorted(entries, key=lambda p: (p.priority, p.name)))
+
+
+PRIMITIVE_REGISTRY: tuple[LexemePrimitive, ...] = _make_registry()
+
+
+# ---------------------------------------------------------------------------
+# scan — hot path: priority-ordered first-hit match
+# ---------------------------------------------------------------------------
+
+def scan(token: str) -> LexemeMatch | None:
+    """Return the first matching LexemePrimitive result, or None.
+
+    Runs primitives in priority order (lower first). First non-empty match
+    wins. Does not advance past end-of-token; uses fullmatch semantics on
+    the trimmed token so cross-token patterns cannot fire.
+
+    Pure / deterministic / no I/O.
+    """
+    if not token:
+        return None
+
+    for primitive in PRIMITIVE_REGISTRY:
+        m = primitive.pattern.search(token)
+        if m is None:
+            continue
+
+        start, end = m.span()
+        groups = m.groupdict()
+
+        # For ordinal-literal, replace the word with its integer rank.
+        if primitive.name == "ordinal-literal" and "rank" in groups:
+            groups["rank"] = _ORDINAL_RANKS.get(groups["rank"].lower(), groups["rank"])
+
+        # Merge captured groups with per-primitive constants, sort keys.
+        merged = {**groups, **_PRIMITIVE_CONSTANTS.get(primitive.name, {})}
+        ev: Mapping[str, str] = types.MappingProxyType(
+            {k: merged[k] for k in sorted(merged)}
+        )
+
+        return LexemeMatch(
+            primitive_name=primitive.name,
+            emit_category=primitive.emits,
+            extracted_values=ev,
+            source_text=token[start:end],
+            source_span=(start, end),
+        )
+
+    return None

tests/test_lexeme_primitives.py

@@ -0,0 +1,296 @@
+"""Tests for generate.comprehension.lexeme_primitives (ADR-0164.1)."""
+
+from __future__ import annotations
+
+import re
+
+import pytest
+
+from generate.comprehension.lexeme_primitives import (
+    PRIMITIVE_REGISTRY,
+    LexemeMatch,
+    LexemePrimitive,
+    scan,
+)
+
+
+# ---------------------------------------------------------------------------
+# Construction invariants
+# ---------------------------------------------------------------------------
+
+class TestRegistryConstruction:
+    def test_has_eight_primitives(self) -> None:
+        assert len(PRIMITIVE_REGISTRY) == 8
+
+    def test_sorted_by_priority_ascending(self) -> None:
+        priorities = [p.priority for p in PRIMITIVE_REGISTRY]
+        assert priorities == sorted(priorities)
+
+    def test_names_are_unique(self) -> None:
+        names = [p.name for p in PRIMITIVE_REGISTRY]
+        assert len(names) == len(set(names))
+
+    def test_all_patterns_compile(self) -> None:
+        for p in PRIMITIVE_REGISTRY:
+            assert isinstance(p.pattern, re.Pattern), f"{p.name}: pattern not compiled"
+
+    def test_all_priorities_non_negative(self) -> None:
+        for p in PRIMITIVE_REGISTRY:
+            assert p.priority >= 0, f"{p.name}: negative priority"
+
+    def test_all_fields_populated(self) -> None:
+        for p in PRIMITIVE_REGISTRY:
+            assert p.name, f"empty name: {p!r}"
+            assert p.emits, f"{p.name}: empty emits"
+            assert p.provenance, f"{p.name}: empty provenance"
+            # extracts may be empty tuple only for primitives with no capture groups
+            assert isinstance(p.extracts, tuple)
+
+    def test_all_names_kebab_case(self) -> None:
+        for p in PRIMITIVE_REGISTRY:
+            assert "_" not in p.name, f"{p.name}: use hyphens, not underscores"
+
+    def test_emits_valid_enum(self) -> None:
+        valid = {"QUANTITY", "ORDINAL", "UNIT_CATEGORY_TOKEN"}
+        for p in PRIMITIVE_REGISTRY:
+            assert p.emits in valid, f"{p.name}: unknown emits={p.emits!r}"
+
+    def test_primitive_is_immutable(self) -> None:
+        p = PRIMITIVE_REGISTRY[0]
+        with pytest.raises((AttributeError, TypeError)):
+            p.name = "tampered"  # type: ignore[misc]
+
+    def test_registry_is_tuple(self) -> None:
+        assert isinstance(PRIMITIVE_REGISTRY, tuple)
+
+
+# ---------------------------------------------------------------------------
+# Canonical fires — each primitive matches its canonical example
+# ---------------------------------------------------------------------------
+
+class TestCanonicalFires:
+    def test_decimal_currency_literal(self) -> None:
+        m = scan("$18.00")
+        assert m is not None
+        assert m.primitive_name == "decimal-currency-literal"
+        assert m.emit_category == "QUANTITY"
+        assert m.extracted_values["whole"] == "18"
+        assert m.extracted_values["cents"] == "00"
+        assert m.extracted_values["unit_class"] == "currency"
+
+    def test_currency_literal(self) -> None:
+        m = scan("$18")
+        assert m is not None
+        assert m.primitive_name == "currency-literal"
+        assert m.emit_category == "QUANTITY"
+        assert m.extracted_values["value"] == "18"
+        assert m.extracted_values["unit_class"] == "currency"
+
+    def test_currency_literal_decimal(self) -> None:
+        m = scan("$1.5")
+        assert m is not None
+        assert m.primitive_name == "currency-literal"
+        assert m.extracted_values["value"] == "1.5"
+
+    def test_percentage_literal(self) -> None:
+        m = scan("25%")
+        assert m is not None
+        assert m.primitive_name == "percentage-literal"
+        assert m.emit_category == "QUANTITY"
+        assert m.extracted_values["value"] == "25"
+        assert m.extracted_values["unit_class"] == "ratio"
+
+    def test_fraction_literal(self) -> None:
+        m = scan("1/2")
+        assert m is not None
+        assert m.primitive_name == "fraction-literal"
+        assert m.emit_category == "QUANTITY"
+        assert m.extracted_values["numerator"] == "1"
+        assert m.extracted_values["denominator"] == "2"
+        assert m.extracted_values["unit_class"] == "fraction"
+
+    def test_time_amount_literal(self) -> None:
+        m = scan("3hours")
+        assert m is not None
+        assert m.primitive_name == "time-amount-literal"
+        assert m.emit_category == "QUANTITY"
+        assert m.extracted_values["value"] == "3"
+        assert m.extracted_values["unit"] == "hour"
+        assert m.extracted_values["unit_class"] == "time"
+
+    def test_time_amount_literal_with_space(self) -> None:
+        m = scan("3 hours")
+        assert m is not None
+        assert m.primitive_name == "time-amount-literal"
+        assert m.extracted_values["value"] == "3"
+        assert m.extracted_values["unit"] == "hour"
+
+    def test_ordinal_literal(self) -> None:
+        m = scan("first")
+        assert m is not None
+        assert m.primitive_name == "ordinal-literal"
+        assert m.emit_category == "ORDINAL"
+        assert m.extracted_values["rank"] == "1"
+
+    def test_ordinal_literal_second(self) -> None:
+        m = scan("second")
+        assert m is not None
+        assert m.extracted_values["rank"] == "2"
+
+    def test_ordinal_literal_tenth(self) -> None:
+        m = scan("tenth")
+        assert m is not None
+        assert m.extracted_values["rank"] == "10"
+
+    def test_mass_noun_token(self) -> None:
+        m = scan("money")
+        assert m is not None
+        assert m.primitive_name == "mass-noun-token"
+        assert m.emit_category == "UNIT_CATEGORY_TOKEN"
+        assert m.extracted_values["lemma"] == "money"
+        assert m.extracted_values["unit_class"] == "currency-mass"
+
+    def test_numeric_literal(self) -> None:
+        m = scan("18")
+        assert m is not None
+        assert m.primitive_name == "numeric-literal"
+        assert m.emit_category == "QUANTITY"
+        assert m.extracted_values["value"] == "18"
+        assert m.extracted_values["unit_class"] == "pending"
+
+
+# ---------------------------------------------------------------------------
+# Overlap precedence (ADR-0164.1 §Overlap precedence)
+# ---------------------------------------------------------------------------
+
+class TestOverlapPrecedence:
+    def test_dollar_18_00_decimal_currency_wins_over_currency(self) -> None:
+        m = scan("$18.00")
+        assert m is not None
+        assert m.primitive_name == "decimal-currency-literal", (
+            f"expected decimal-currency-literal, got {m.primitive_name}"
+        )
+
+    def test_dollar_18_currency_wins_over_numeric(self) -> None:
+        m = scan("$18")
+        assert m is not None
+        assert m.primitive_name == "currency-literal"
+
+    def test_dollar_18_00_not_numeric(self) -> None:
+        m = scan("$18.00")
+        assert m is not None
+        assert m.primitive_name != "numeric-literal"
+
+    def test_fraction_wins_over_numeric(self) -> None:
+        m = scan("1/2")
+        assert m is not None
+        assert m.primitive_name == "fraction-literal"
+
+    def test_percentage_wins_over_numeric(self) -> None:
+        m = scan("50%")
+        assert m is not None
+        assert m.primitive_name == "percentage-literal"
+
+    def test_25_percent_not_numeric(self) -> None:
+        m = scan("25%")
+        assert m is not None
+        assert m.primitive_name != "numeric-literal"
+
+
+# ---------------------------------------------------------------------------
+# Refusal — scan returns None when no primitive matches
+# ---------------------------------------------------------------------------
+
+class TestRefusal:
+    def test_proper_noun_entity(self) -> None:
+        assert scan("Tina") is None
+
+    def test_empty_string(self) -> None:
+        assert scan("") is None
+
+    def test_plain_verb(self) -> None:
+        assert scan("earn") is None
+
+    def test_question_word(self) -> None:
+        assert scan("how") is None
+
+    def test_article(self) -> None:
+        assert scan("the") is None
+
+    def test_preposition(self) -> None:
+        assert scan("for") is None
+
+
+# ---------------------------------------------------------------------------
+# Determinism — identical inputs produce byte-equal results
+# ---------------------------------------------------------------------------
+
+class TestDeterminism:
+    @pytest.mark.parametrize("token", [
+        "$18.00", "$18", "25%", "1/2", "3hours", "first", "money", "18",
+    ])
+    def test_byte_equal_on_repeat(self, token: str) -> None:
+        a = scan(token)
+        b = scan(token)
+        assert a == b
+
+    def test_none_is_none(self) -> None:
+        assert scan("Tina") == scan("Tina")
+
+    def test_lexeme_match_eq_semantics(self) -> None:
+        a = scan("$18.00")
+        b = scan("$18.00")
+        assert a is not None and b is not None
+        assert a == b
+        # LexemeMatch is frozen dataclass — equality is field-wise
+        assert a.primitive_name == b.primitive_name
+        assert a.emit_category == b.emit_category
+        assert dict(a.extracted_values) == dict(b.extracted_values)
+        assert a.source_text == b.source_text
+        assert a.source_span == b.source_span
+
+
+# ---------------------------------------------------------------------------
+# ADR-0165 compliance — no primitive pattern contains \s (whitespace class)
+# ---------------------------------------------------------------------------
+
+class TestADR0165Compliance:
+    def test_no_multi_whitespace_in_patterns(self) -> None:
+        # ADR-0165 forbids \s+ grammar-template patterns. Single \s? is explicitly
+        # sanctioned in ADR-0164.1 §Seed primitive set for percentage-literal,
+        # fraction-literal, and time-amount-literal (single optional space within
+        # one orthographic form — not a cross-token span).
+        for p in PRIMITIVE_REGISTRY:
+            src = p.pattern.pattern
+            assert r"\s+" not in src, (
+                f"{p.name}: pattern contains \\s+ — multi-whitespace spans cross "
+                f"token boundaries (ADR-0165 §Rule). Pattern: {src!r}"
+            )
+
+    def test_no_star_wildcard_spanning_patterns(self) -> None:
+        for p in PRIMITIVE_REGISTRY:
+            src = p.pattern.pattern
+            assert r".*" not in src, (
+                f"{p.name}: pattern contains .* — forbidden grammar-template indicator"
+            )
+
+
+# ---------------------------------------------------------------------------
+# extracted_values sorted-key stability (canonical bytes)
+# ---------------------------------------------------------------------------
+
+class TestExtractedValuesSortedKeys:
+    @pytest.mark.parametrize("token", [
+        "$18.00", "$18", "25%", "1/2", "3hours", "first", "money", "18",
+    ])
+    def test_keys_are_sorted(self, token: str) -> None:
+        m = scan(token)
+        assert m is not None
+        keys = list(m.extracted_values.keys())
+        assert keys == sorted(keys), f"keys not sorted for token {token!r}: {keys}"
+
+    def test_extracted_values_immutable(self) -> None:
+        m = scan("$18.00")
+        assert m is not None
+        with pytest.raises((TypeError, AttributeError)):
+            m.extracted_values["tampered"] = "x"  # type: ignore[index]