"""Primitives-pack loader (ADR-0084). Loads ``packs/primitives//{manifest.json,primitives.jsonl}`` with byte-checksum verification and strict schema parsing. Why a separate loader (not :mod:`language_packs.compiler`) ---------------------------------------------------------- Primitives are not lexicon entries — they have no surface, no morphology, no semantic-domain assignment, no manifold coordinate. They are a flat set of terminal symbols consulted by :func:`language_packs.definitions.verify_definitional_closure` as the ``floor`` argument. Treating them as a degenerate ``LexicalEntry`` would import irrelevant machinery and blur the substrate boundary the ADR is deliberately drawing. """ from __future__ import annotations import hashlib import json from dataclasses import dataclass from functools import lru_cache from pathlib import Path from typing import Iterable class PrimitivesPackError(RuntimeError): """Raised when a primitives pack is missing, malformed, or unverified. Inherits from :class:`RuntimeError` (not :class:`ValueError`) on the same principle as :class:`packs.safety.SafetyPackError` — a missing primitives pack is a fail-closed runtime condition for any system that opts into the definitional layer, not a recoverable input-validation error. """ DEFAULT_PRIMITIVES_PACK: str = "en_semantic_primitives_v1" _PACK_ROOT: Path = Path(__file__).resolve().parent _REQUIRED_MANIFEST_KEYS: frozenset[str] = frozenset( { "pack_id", "language", "kind", "definitional_layer", "version", "issued_at", "checksum", "primitive_count", "never_auto_mutable", "provenance", } ) _ALLOWED_PRIMITIVE_KEYS: frozenset[str] = frozenset( {"lemma", "category", "pos", "primitive_version", "provenance_ids"} ) _REQUIRED_PRIMITIVE_KEYS: frozenset[str] = frozenset( {"lemma", "category", "primitive_version", "provenance_ids"} ) @dataclass(frozen=True, slots=True) class Primitive: """One ratified primitive entry.""" lemma: str category: str pos: str primitive_version: int provenance_ids: tuple[str, ...] @dataclass(frozen=True, slots=True) class PrimitivesPack: """Loaded primitives pack — immutable terminal-symbol floor.""" pack_id: str language: str version: int issued_at: str primitives: tuple[Primitive, ...] checksum: str @property def lemmas(self) -> frozenset[str]: """Return the lower-cased frozen set of primitive lemmas.""" return frozenset(p.lemma.lower() for p in self.primitives) def _validate_pack_id(pack_id: object) -> str: if not isinstance(pack_id, str) or not pack_id: raise PrimitivesPackError(f"primitives pack_id must be a non-empty string, got {pack_id!r}") if ".." in pack_id or "/" in pack_id or "\\" in pack_id or pack_id.startswith("."): raise PrimitivesPackError(f"primitives pack_id contains forbidden path token(s): {pack_id!r}") for ch in pack_id: if not (ch.isascii() and (ch.isalnum() or ch in {"_", "-"})): raise PrimitivesPackError( f"primitives pack_id must be alphanumeric/_/-, got {pack_id!r}" ) return pack_id def _parse_primitive(payload: object, *, line_no: int, pack_id: str) -> Primitive: if not isinstance(payload, dict): raise PrimitivesPackError( f"{pack_id}/primitives.jsonl:{line_no}: entry must be a JSON object" ) unknown = set(payload.keys()) - _ALLOWED_PRIMITIVE_KEYS if unknown: raise PrimitivesPackError( f"{pack_id}/primitives.jsonl:{line_no}: unrecognised key(s): {sorted(unknown)}" ) missing = _REQUIRED_PRIMITIVE_KEYS - set(payload.keys()) if missing: raise PrimitivesPackError( f"{pack_id}/primitives.jsonl:{line_no}: missing required key(s): {sorted(missing)}" ) lemma = payload["lemma"] category = payload["category"] primitive_version = payload["primitive_version"] provenance_ids = payload["provenance_ids"] pos = payload.get("pos", "") if not isinstance(lemma, str) or not lemma.strip(): raise PrimitivesPackError( f"{pack_id}/primitives.jsonl:{line_no}: lemma must be a non-empty string" ) if not isinstance(category, str) or not category.strip(): raise PrimitivesPackError( f"{pack_id}/primitives.jsonl:{line_no}: category must be a non-empty string" ) if not isinstance(primitive_version, int) or isinstance(primitive_version, bool) or primitive_version < 1: raise PrimitivesPackError( f"{pack_id}/primitives.jsonl:{line_no}: primitive_version must be a positive int" ) if not isinstance(provenance_ids, list) or not provenance_ids or any( not isinstance(x, str) or not x for x in provenance_ids ): raise PrimitivesPackError( f"{pack_id}/primitives.jsonl:{line_no}: provenance_ids must be a non-empty list of strings" ) if not isinstance(pos, str): raise PrimitivesPackError( f"{pack_id}/primitives.jsonl:{line_no}: pos must be a string when present" ) return Primitive( lemma=lemma.strip(), category=category.strip(), pos=pos, primitive_version=primitive_version, provenance_ids=tuple(provenance_ids), ) @lru_cache(maxsize=8) def _load_primitives_pack_cached(pack_id: str) -> PrimitivesPack: pack_dir = _PACK_ROOT / pack_id manifest_path = pack_dir / "manifest.json" primitives_path = pack_dir / "primitives.jsonl" if not manifest_path.exists(): raise PrimitivesPackError(f"primitives pack manifest missing at {manifest_path}") if not primitives_path.exists(): raise PrimitivesPackError(f"primitives pack jsonl missing at {primitives_path}") try: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) except json.JSONDecodeError as exc: raise PrimitivesPackError(f"primitives pack manifest malformed: {exc}") from exc if not isinstance(manifest, dict): raise PrimitivesPackError("primitives pack manifest must be a JSON object") missing = _REQUIRED_MANIFEST_KEYS - set(manifest.keys()) if missing: raise PrimitivesPackError( f"primitives pack manifest missing required key(s): {sorted(missing)}" ) if manifest["kind"] != "primitives": raise PrimitivesPackError( f"primitives pack manifest kind must be 'primitives', got {manifest['kind']!r}" ) if manifest["definitional_layer"] is not True: raise PrimitivesPackError( "primitives pack manifest must have definitional_layer: true" ) if manifest["never_auto_mutable"] is not True: raise PrimitivesPackError( "primitives pack manifest must have never_auto_mutable: true" ) if manifest["pack_id"] != pack_id: raise PrimitivesPackError( f"primitives pack manifest pack_id {manifest['pack_id']!r} != directory {pack_id!r}" ) primitives_bytes = primitives_path.read_bytes() actual_checksum = hashlib.sha256(primitives_bytes).hexdigest() expected_checksum = manifest["checksum"] if not isinstance(expected_checksum, str) or actual_checksum != expected_checksum: raise PrimitivesPackError( f"primitives pack checksum mismatch for {pack_id}: " f"{actual_checksum} != {expected_checksum}" ) primitives: list[Primitive] = [] for line_no, raw_line in enumerate(primitives_bytes.decode("utf-8").splitlines(), start=1): line = raw_line.strip() if not line: continue try: payload = json.loads(line) except json.JSONDecodeError as exc: raise PrimitivesPackError( f"{pack_id}/primitives.jsonl:{line_no}: malformed JSON ({exc})" ) from exc primitives.append(_parse_primitive(payload, line_no=line_no, pack_id=pack_id)) declared_count = manifest["primitive_count"] if not isinstance(declared_count, int) or declared_count != len(primitives): raise PrimitivesPackError( f"primitives pack manifest declares primitive_count={declared_count} " f"but {len(primitives)} entries parsed from primitives.jsonl" ) # Lemma-uniqueness gate — a primitive must be the floor, so the # same lemma must not be ratified twice within one pack. seen: set[str] = set() for primitive in primitives: key = primitive.lemma.lower() if key in seen: raise PrimitivesPackError( f"primitives pack {pack_id} ratifies lemma {primitive.lemma!r} more than once" ) seen.add(key) version_raw = manifest["version"] if not isinstance(version_raw, int): raise PrimitivesPackError( f"primitives pack manifest version must be an int, got {version_raw!r}" ) issued_at_raw = manifest["issued_at"] if not isinstance(issued_at_raw, str) or not issued_at_raw: raise PrimitivesPackError("primitives pack manifest issued_at must be a non-empty string") language_raw = manifest["language"] if not isinstance(language_raw, str) or not language_raw: raise PrimitivesPackError("primitives pack manifest language must be a non-empty string") return PrimitivesPack( pack_id=pack_id, language=language_raw, version=version_raw, issued_at=issued_at_raw, primitives=tuple(primitives), checksum=actual_checksum, ) def load_primitives_pack(pack_id: str = DEFAULT_PRIMITIVES_PACK) -> PrimitivesPack: """Load and verify a primitives pack from disk. Fails closed on any error (missing files, checksum mismatch, schema violation). Result is cached — subsequent calls return the same immutable instance. """ return _load_primitives_pack_cached(_validate_pack_id(pack_id)) def clear_primitives_cache() -> None: """Drop the primitives-pack cache. Test-only escape hatch — ratified primitives packs are immutable in production and this cache is never invalidated outside tests. """ _load_primitives_pack_cached.cache_clear() def union_primitive_lemmas(pack_ids: Iterable[str]) -> frozenset[str]: """Return the lower-cased union of every primitive lemma across *pack_ids*.""" out: set[str] = set() for pack_id in pack_ids: out.update(load_primitives_pack(pack_id).lemmas) return frozenset(out)