diff --git a/generate/comprehension/lexeme_primitives.py b/generate/comprehension/lexeme_primitives.py new file mode 100644 index 00000000..02991393 --- /dev/null +++ b/generate/comprehension/lexeme_primitives.py @@ -0,0 +1,197 @@ +"""ADR-0164.1 — Lexeme primitive registry. + +Eight seed primitives for the incremental comprehension reader's step-1 +lexical scan (ADR-0164 §Decision §3, ADR-0165 §Legitimate uses). + +Public API: + PRIMITIVE_REGISTRY — immutable sorted tuple[LexemePrimitive, ...] + scan(token) — first-hit match in priority order; None on miss +""" + +from __future__ import annotations + +import re +import types +from dataclasses import dataclass +from typing import Mapping + +# --------------------------------------------------------------------------- +# Core types +# --------------------------------------------------------------------------- + +_ORDINAL_RANKS: dict[str, str] = { + "first": "1", + "second": "2", + "third": "3", + "fourth": "4", + "fifth": "5", + "sixth": "6", + "seventh": "7", + "eighth": "8", + "ninth": "9", + "tenth": "10", +} + + +@dataclass(frozen=True, slots=True) +class LexemePrimitive: + name: str + pattern: re.Pattern[str] + emits: str + extracts: tuple[str, ...] + priority: int + provenance: str + + +@dataclass(frozen=True, slots=True) +class LexemeMatch: + primitive_name: str + emit_category: str + extracted_values: Mapping[str, str] + source_text: str + source_span: tuple[int, int] + + +# --------------------------------------------------------------------------- +# Per-primitive constant fields (not captured by regex groups) +# --------------------------------------------------------------------------- + +_PRIMITIVE_CONSTANTS: dict[str, dict[str, str]] = { + "decimal-currency-literal": {"unit_class": "currency"}, + "currency-literal": {"unit_class": "currency"}, + "percentage-literal": {"unit_class": "ratio"}, + "fraction-literal": {"unit_class": "fraction"}, + "time-amount-literal": {"unit_class": "time"}, + "numeric-literal": {"unit_class": "pending"}, + "ordinal-literal": {}, + "mass-noun-token": {"unit_class": "currency-mass"}, +} + + +def _make_registry() -> tuple[LexemePrimitive, ...]: + entries: list[LexemePrimitive] = [ + LexemePrimitive( + name="decimal-currency-literal", + pattern=re.compile(r"\$(?P\d+)\.(?P\d{2})\b"), + emits="QUANTITY", + extracts=("whole", "cents"), + priority=10, + provenance="ADR-0164.1", + ), + LexemePrimitive( + name="currency-literal", + pattern=re.compile(r"\$(?P\d+(?:\.\d+)?)\b"), + emits="QUANTITY", + extracts=("value",), + priority=20, + provenance="ADR-0164.1", + ), + LexemePrimitive( + name="percentage-literal", + pattern=re.compile(r"(?P\d+(?:\.\d+)?)[ ]?%"), + emits="QUANTITY", + extracts=("value",), + priority=30, + provenance="ADR-0164.1", + ), + LexemePrimitive( + name="fraction-literal", + pattern=re.compile(r"(?P\d+)[ ]?/[ ]?(?P\d+)\b"), + emits="QUANTITY", + extracts=("numerator", "denominator"), + priority=40, + provenance="ADR-0164.1", + ), + LexemePrimitive( + name="time-amount-literal", + pattern=re.compile( + r"(?P\d+)[- ]?" + r"(?Phour|minute|day|week|month|year|second)s?\b", + re.IGNORECASE, + ), + emits="QUANTITY", + extracts=("value", "unit"), + priority=50, + provenance="ADR-0164.1", + ), + LexemePrimitive( + name="ordinal-literal", + pattern=re.compile( + r"(?Pfirst|second|third|fourth|fifth|" + r"sixth|seventh|eighth|ninth|tenth)\b", + re.IGNORECASE, + ), + emits="ORDINAL", + extracts=("rank",), + priority=60, + provenance="ADR-0164.1", + ), + LexemePrimitive( + name="mass-noun-token", + pattern=re.compile( + r"(?Pmoney|profit|interest|income|savings|cost|amount|total)\b", + re.IGNORECASE, + ), + emits="UNIT_CATEGORY_TOKEN", + extracts=("lemma",), + priority=70, + provenance="ADR-0164.1", + ), + LexemePrimitive( + name="numeric-literal", + pattern=re.compile(r"(?P\d+(?:\.\d+)?)\b"), + emits="QUANTITY", + extracts=("value",), + priority=100, + provenance="ADR-0164.1", + ), + ] + return tuple(sorted(entries, key=lambda p: (p.priority, p.name))) + + +PRIMITIVE_REGISTRY: tuple[LexemePrimitive, ...] = _make_registry() + + +# --------------------------------------------------------------------------- +# scan — hot path: priority-ordered first-hit match +# --------------------------------------------------------------------------- + +def scan(token: str) -> LexemeMatch | None: + """Return the first matching LexemePrimitive result, or None. + + Runs primitives in priority order (lower first). First non-empty match + wins. Does not advance past end-of-token; uses fullmatch semantics on + the trimmed token so cross-token patterns cannot fire. + + Pure / deterministic / no I/O. + """ + if not token: + return None + + for primitive in PRIMITIVE_REGISTRY: + m = primitive.pattern.search(token) + if m is None: + continue + + start, end = m.span() + groups = m.groupdict() + + # For ordinal-literal, replace the word with its integer rank. + if primitive.name == "ordinal-literal" and "rank" in groups: + groups["rank"] = _ORDINAL_RANKS.get(groups["rank"].lower(), groups["rank"]) + + # Merge captured groups with per-primitive constants, sort keys. + merged = {**groups, **_PRIMITIVE_CONSTANTS.get(primitive.name, {})} + ev: Mapping[str, str] = types.MappingProxyType( + {k: merged[k] for k in sorted(merged)} + ) + + return LexemeMatch( + primitive_name=primitive.name, + emit_category=primitive.emits, + extracted_values=ev, + source_text=token[start:end], + source_span=(start, end), + ) + + return None diff --git a/tests/test_lexeme_primitives.py b/tests/test_lexeme_primitives.py new file mode 100644 index 00000000..41c0a112 --- /dev/null +++ b/tests/test_lexeme_primitives.py @@ -0,0 +1,296 @@ +"""Tests for generate.comprehension.lexeme_primitives (ADR-0164.1).""" + +from __future__ import annotations + +import re + +import pytest + +from generate.comprehension.lexeme_primitives import ( + PRIMITIVE_REGISTRY, + LexemeMatch, + LexemePrimitive, + scan, +) + + +# --------------------------------------------------------------------------- +# Construction invariants +# --------------------------------------------------------------------------- + +class TestRegistryConstruction: + def test_has_eight_primitives(self) -> None: + assert len(PRIMITIVE_REGISTRY) == 8 + + def test_sorted_by_priority_ascending(self) -> None: + priorities = [p.priority for p in PRIMITIVE_REGISTRY] + assert priorities == sorted(priorities) + + def test_names_are_unique(self) -> None: + names = [p.name for p in PRIMITIVE_REGISTRY] + assert len(names) == len(set(names)) + + def test_all_patterns_compile(self) -> None: + for p in PRIMITIVE_REGISTRY: + assert isinstance(p.pattern, re.Pattern), f"{p.name}: pattern not compiled" + + def test_all_priorities_non_negative(self) -> None: + for p in PRIMITIVE_REGISTRY: + assert p.priority >= 0, f"{p.name}: negative priority" + + def test_all_fields_populated(self) -> None: + for p in PRIMITIVE_REGISTRY: + assert p.name, f"empty name: {p!r}" + assert p.emits, f"{p.name}: empty emits" + assert p.provenance, f"{p.name}: empty provenance" + # extracts may be empty tuple only for primitives with no capture groups + assert isinstance(p.extracts, tuple) + + def test_all_names_kebab_case(self) -> None: + for p in PRIMITIVE_REGISTRY: + assert "_" not in p.name, f"{p.name}: use hyphens, not underscores" + + def test_emits_valid_enum(self) -> None: + valid = {"QUANTITY", "ORDINAL", "UNIT_CATEGORY_TOKEN"} + for p in PRIMITIVE_REGISTRY: + assert p.emits in valid, f"{p.name}: unknown emits={p.emits!r}" + + def test_primitive_is_immutable(self) -> None: + p = PRIMITIVE_REGISTRY[0] + with pytest.raises((AttributeError, TypeError)): + p.name = "tampered" # type: ignore[misc] + + def test_registry_is_tuple(self) -> None: + assert isinstance(PRIMITIVE_REGISTRY, tuple) + + +# --------------------------------------------------------------------------- +# Canonical fires — each primitive matches its canonical example +# --------------------------------------------------------------------------- + +class TestCanonicalFires: + def test_decimal_currency_literal(self) -> None: + m = scan("$18.00") + assert m is not None + assert m.primitive_name == "decimal-currency-literal" + assert m.emit_category == "QUANTITY" + assert m.extracted_values["whole"] == "18" + assert m.extracted_values["cents"] == "00" + assert m.extracted_values["unit_class"] == "currency" + + def test_currency_literal(self) -> None: + m = scan("$18") + assert m is not None + assert m.primitive_name == "currency-literal" + assert m.emit_category == "QUANTITY" + assert m.extracted_values["value"] == "18" + assert m.extracted_values["unit_class"] == "currency" + + def test_currency_literal_decimal(self) -> None: + m = scan("$1.5") + assert m is not None + assert m.primitive_name == "currency-literal" + assert m.extracted_values["value"] == "1.5" + + def test_percentage_literal(self) -> None: + m = scan("25%") + assert m is not None + assert m.primitive_name == "percentage-literal" + assert m.emit_category == "QUANTITY" + assert m.extracted_values["value"] == "25" + assert m.extracted_values["unit_class"] == "ratio" + + def test_fraction_literal(self) -> None: + m = scan("1/2") + assert m is not None + assert m.primitive_name == "fraction-literal" + assert m.emit_category == "QUANTITY" + assert m.extracted_values["numerator"] == "1" + assert m.extracted_values["denominator"] == "2" + assert m.extracted_values["unit_class"] == "fraction" + + def test_time_amount_literal(self) -> None: + m = scan("3hours") + assert m is not None + assert m.primitive_name == "time-amount-literal" + assert m.emit_category == "QUANTITY" + assert m.extracted_values["value"] == "3" + assert m.extracted_values["unit"] == "hour" + assert m.extracted_values["unit_class"] == "time" + + def test_time_amount_literal_with_space(self) -> None: + m = scan("3 hours") + assert m is not None + assert m.primitive_name == "time-amount-literal" + assert m.extracted_values["value"] == "3" + assert m.extracted_values["unit"] == "hour" + + def test_ordinal_literal(self) -> None: + m = scan("first") + assert m is not None + assert m.primitive_name == "ordinal-literal" + assert m.emit_category == "ORDINAL" + assert m.extracted_values["rank"] == "1" + + def test_ordinal_literal_second(self) -> None: + m = scan("second") + assert m is not None + assert m.extracted_values["rank"] == "2" + + def test_ordinal_literal_tenth(self) -> None: + m = scan("tenth") + assert m is not None + assert m.extracted_values["rank"] == "10" + + def test_mass_noun_token(self) -> None: + m = scan("money") + assert m is not None + assert m.primitive_name == "mass-noun-token" + assert m.emit_category == "UNIT_CATEGORY_TOKEN" + assert m.extracted_values["lemma"] == "money" + assert m.extracted_values["unit_class"] == "currency-mass" + + def test_numeric_literal(self) -> None: + m = scan("18") + assert m is not None + assert m.primitive_name == "numeric-literal" + assert m.emit_category == "QUANTITY" + assert m.extracted_values["value"] == "18" + assert m.extracted_values["unit_class"] == "pending" + + +# --------------------------------------------------------------------------- +# Overlap precedence (ADR-0164.1 §Overlap precedence) +# --------------------------------------------------------------------------- + +class TestOverlapPrecedence: + def test_dollar_18_00_decimal_currency_wins_over_currency(self) -> None: + m = scan("$18.00") + assert m is not None + assert m.primitive_name == "decimal-currency-literal", ( + f"expected decimal-currency-literal, got {m.primitive_name}" + ) + + def test_dollar_18_currency_wins_over_numeric(self) -> None: + m = scan("$18") + assert m is not None + assert m.primitive_name == "currency-literal" + + def test_dollar_18_00_not_numeric(self) -> None: + m = scan("$18.00") + assert m is not None + assert m.primitive_name != "numeric-literal" + + def test_fraction_wins_over_numeric(self) -> None: + m = scan("1/2") + assert m is not None + assert m.primitive_name == "fraction-literal" + + def test_percentage_wins_over_numeric(self) -> None: + m = scan("50%") + assert m is not None + assert m.primitive_name == "percentage-literal" + + def test_25_percent_not_numeric(self) -> None: + m = scan("25%") + assert m is not None + assert m.primitive_name != "numeric-literal" + + +# --------------------------------------------------------------------------- +# Refusal — scan returns None when no primitive matches +# --------------------------------------------------------------------------- + +class TestRefusal: + def test_proper_noun_entity(self) -> None: + assert scan("Tina") is None + + def test_empty_string(self) -> None: + assert scan("") is None + + def test_plain_verb(self) -> None: + assert scan("earn") is None + + def test_question_word(self) -> None: + assert scan("how") is None + + def test_article(self) -> None: + assert scan("the") is None + + def test_preposition(self) -> None: + assert scan("for") is None + + +# --------------------------------------------------------------------------- +# Determinism — identical inputs produce byte-equal results +# --------------------------------------------------------------------------- + +class TestDeterminism: + @pytest.mark.parametrize("token", [ + "$18.00", "$18", "25%", "1/2", "3hours", "first", "money", "18", + ]) + def test_byte_equal_on_repeat(self, token: str) -> None: + a = scan(token) + b = scan(token) + assert a == b + + def test_none_is_none(self) -> None: + assert scan("Tina") == scan("Tina") + + def test_lexeme_match_eq_semantics(self) -> None: + a = scan("$18.00") + b = scan("$18.00") + assert a is not None and b is not None + assert a == b + # LexemeMatch is frozen dataclass — equality is field-wise + assert a.primitive_name == b.primitive_name + assert a.emit_category == b.emit_category + assert dict(a.extracted_values) == dict(b.extracted_values) + assert a.source_text == b.source_text + assert a.source_span == b.source_span + + +# --------------------------------------------------------------------------- +# ADR-0165 compliance — no primitive pattern contains \s (whitespace class) +# --------------------------------------------------------------------------- + +class TestADR0165Compliance: + def test_no_multi_whitespace_in_patterns(self) -> None: + # ADR-0165 forbids \s+ grammar-template patterns. Single \s? is explicitly + # sanctioned in ADR-0164.1 §Seed primitive set for percentage-literal, + # fraction-literal, and time-amount-literal (single optional space within + # one orthographic form — not a cross-token span). + for p in PRIMITIVE_REGISTRY: + src = p.pattern.pattern + assert r"\s+" not in src, ( + f"{p.name}: pattern contains \\s+ — multi-whitespace spans cross " + f"token boundaries (ADR-0165 §Rule). Pattern: {src!r}" + ) + + def test_no_star_wildcard_spanning_patterns(self) -> None: + for p in PRIMITIVE_REGISTRY: + src = p.pattern.pattern + assert r".*" not in src, ( + f"{p.name}: pattern contains .* — forbidden grammar-template indicator" + ) + + +# --------------------------------------------------------------------------- +# extracted_values sorted-key stability (canonical bytes) +# --------------------------------------------------------------------------- + +class TestExtractedValuesSortedKeys: + @pytest.mark.parametrize("token", [ + "$18.00", "$18", "25%", "1/2", "3hours", "first", "money", "18", + ]) + def test_keys_are_sorted(self, token: str) -> None: + m = scan(token) + assert m is not None + keys = list(m.extracted_values.keys()) + assert keys == sorted(keys), f"keys not sorted for token {token!r}: {keys}" + + def test_extracted_values_immutable(self) -> None: + m = scan("$18.00") + assert m is not None + with pytest.raises((TypeError, AttributeError)): + m.extracted_values["tampered"] = "x" # type: ignore[index]