From 4c447dea81fcea3d7e98fa2a63da8dddd0b7b6aa Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 13 May 2026 14:37:36 -0700 Subject: [PATCH] Add language_packs CLI: python -m language_packs compile/verify/list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three subcommands: compile — compile a pack, verify checksum, print manifold stats verify — checksum-only verification without full compilation list — list all available packs with role and entry count Checksum rule (AGENTS.md) is enforced by design: the CLI reads bytes from disk via read_bytes() as the single source of truth, never from in-memory strings. This makes the compiler an auditable D0 instrument — not just an importable function. Exit codes: 0 = success, 1 = checksum mismatch or pack not found. --- language_packs/__main__.py | 157 +++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 language_packs/__main__.py diff --git a/language_packs/__main__.py b/language_packs/__main__.py new file mode 100644 index 00000000..c6070a4f --- /dev/null +++ b/language_packs/__main__.py @@ -0,0 +1,157 @@ +""" +python -m language_packs [pack_id] + +Commands +-------- +compile Compile pack, verify checksum, print manifold stats. +verify Verify on-disk checksum against manifest only. +list List all packs in language_packs/data/ with metadata. + +Checksum contract +----------------- +The checksum is always computed from the raw bytes on disk: + hashlib.sha256(Path(lexicon_path).read_bytes()).hexdigest() +This is the AGENTS.md rule: hash what disk holds, not what Python strings hold. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path + +_DATA_DIR = Path(__file__).parent / "data" + + +def _checksum(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _available_packs() -> list[str]: + if not _DATA_DIR.exists(): + return [] + return sorted(p.name for p in _DATA_DIR.iterdir() if p.is_dir()) + + +def cmd_list(_args) -> int: + packs = _available_packs() + if not packs: + print("No packs found in language_packs/data/") + return 0 + print(f"{'pack_id':<30} {'role':<20} {'entries':>7} {'checksum_ok'}") + print("-" * 70) + for pack_id in packs: + pack_dir = _DATA_DIR / pack_id + manifest_path = pack_dir / "manifest.json" + lexicon_path = pack_dir / "lexicon.jsonl" + if not manifest_path.exists() or not lexicon_path.exists(): + print(f"{pack_id:<30} {'':<20}") + continue + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + actual = _checksum(lexicon_path) + ok = "\u2713" if actual == manifest.get("checksum", "") else "\u2717 MISMATCH" + entries = sum(1 for line in lexicon_path.read_text(encoding="utf-8").splitlines() if line.strip()) + role = manifest.get("role", "?") + print(f"{pack_id:<30} {role:<20} {entries:>7} {ok}") + return 0 + + +def cmd_verify(args) -> int: + pack_id = args.pack_id + pack_dir = _DATA_DIR / pack_id + manifest_path = pack_dir / "manifest.json" + lexicon_path = pack_dir / "lexicon.jsonl" + + if not manifest_path.exists(): + print(f"ERROR: manifest not found: {manifest_path}", file=sys.stderr) + return 1 + if not lexicon_path.exists(): + print(f"ERROR: lexicon not found: {lexicon_path}", file=sys.stderr) + return 1 + + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + actual = _checksum(lexicon_path) + expected = manifest.get("checksum", "") + + if actual == expected: + print(f"{pack_id}: checksum OK ({actual[:16]}...)") + return 0 + else: + print(f"ERROR: checksum MISMATCH for {pack_id}", file=sys.stderr) + print(f" expected: {expected}", file=sys.stderr) + print(f" actual: {actual}", file=sys.stderr) + print(f" Fix: update manifest.json checksum to: {actual}", file=sys.stderr) + return 1 + + +def cmd_compile(args) -> int: + # Verify first + rc = cmd_verify(args) + if rc != 0: + return rc + + pack_id = args.pack_id + from language_packs.compiler import load_pack, load_pack_entries + + try: + manifest, manifold = load_pack(pack_id) + entries = load_pack_entries(pack_id) + except Exception as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + n_entries = len(entries) + n_points = len(manifold) + print(f"{pack_id}: compiled {n_entries} entries -> {n_points} manifold points") + print(f" language: {manifest.language}") + print(f" role: {manifest.role.value}") + print(f" determinism: {manifest.determinism_class}") + print(f" oov_policy: {manifest.oov_policy.value}") + print(f" gate_engaged: {manifest.gate_engaged}") + print(f" version: {manifest.version}") + + # Spot-check: first and last surface should round-trip through manifold + first = entries[0].surface + last = entries[-1].surface + try: + manifold.get_versor(first) + manifold.get_versor(last) + print(f" spot-check: '{first}' and '{last}' resolve OK") + except KeyError as exc: + print(f" WARNING: spot-check failed: {exc}", file=sys.stderr) + + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser( + prog="python -m language_packs", + description="CORE language-pack compiler and verifier.", + ) + sub = parser.add_subparsers(dest="command") + + sub.add_parser("list", help="List all available packs.") + + p_verify = sub.add_parser("verify", help="Verify pack checksum.") + p_verify.add_argument("pack_id", help="Pack ID (directory name under language_packs/data/).") + + p_compile = sub.add_parser("compile", help="Compile pack and print manifold stats.") + p_compile.add_argument("pack_id", help="Pack ID (directory name under language_packs/data/).") + + args = parser.parse_args() + + if args.command == "list": + return cmd_list(args) + elif args.command == "verify": + return cmd_verify(args) + elif args.command == "compile": + return cmd_compile(args) + else: + parser.print_help() + return 0 + + +if __name__ == "__main__": + sys.exit(main())