diff --git a/docs/outreach/apple-uma-demo-runbook.md b/docs/outreach/apple-uma-demo-runbook.md new file mode 100644 index 00000000..6a7e92d0 --- /dev/null +++ b/docs/outreach/apple-uma-demo-runbook.md @@ -0,0 +1,143 @@ +# Apple UMA demo runbook + +This runbook turns the Apple UMA benchmark, Workbench report card, and claim-safe package builder into a repeatable recording/share workflow. + +## Goal + +Produce a shareable package for Apple Silicon engineering review that shows: + +- exact CGA recall benchmark evidence; +- Rust/native boundary status; +- MLX exact score-vector parity; +- explicit copy-in/copy-out boundaries; +- Workbench read-only report-card surface; +- explicit non-claims. + +The demo must not imply: + +- serving-path MLX integration; +- CoreML acceleration; +- ANE acceleration; +- Metal custom kernels; +- approximate recall or ANN search; +- zero-copy everywhere; +- generalized token-generation performance. + +## Local preparation + +Run from repo root on the Apple Silicon machine used for recording. + +```bash +cd /Users/kaizenpro/Projects/core +git switch main +git fetch origin --prune +git pull --ff-only origin main +``` + +Build the Rust extension if needed: + +```bash +PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 uv run python -m maturin develop --release --manifest-path core-rs/Cargo.toml +``` + +Confirm MLX is available: + +```bash +uv run python - <<'PY' +import mlx.core as mx +print(mx.default_device()) +PY +``` + +## Generate and package the report + +Preferred one-shot command: + +```bash +uv run python scripts/package_apple_uma_demo.py --refresh-report +``` + +This command: + +1. refreshes `evals/reports/apple_uma_mechanical_sympathy_latest.{json,md}` with `CORE_BACKEND=rust`; +2. reads the Workbench Apple UMA projection; +3. refuses to package stale/no-MLX or parity-failing evidence by default; +4. creates a package under `dist/apple-uma-demo//`. + +If you intentionally need to inspect stale state, use: + +```bash +uv run python scripts/package_apple_uma_demo.py --allow-stale +``` + +Do not use `--allow-stale` for the Apple-facing package. + +## Validate before recording + +Run focused backend and UI checks: + +```bash +uv run python -m pytest -q tests/test_apple_uma_demo_package.py tests/test_workbench_apple_uma_report.py tests/test_workbench_api.py +cd workbench-ui +pnpm vitest run src/app/apple-uma/AppleUmaReportRoute.test.tsx src/app/routeConformance.test.tsx +pnpm exec tsc -b +``` + +## Workbench recording flow + +Start the API server using the repository's normal Workbench launcher. + +Open the Workbench UI and navigate to: + +```text +/apple-uma +``` + +Record these beats: + +1. **Header / identity** — show benchmark name, version, source digest, read-only status. +2. **Backend status** — show Rust/native status and current backend truthfully. +3. **Track inventory** — show all required tracks present. +4. **MLX exact CGA recall** — show track present, executed, parity true, case count, timing rows. +5. **Copy boundaries** — show NumPy to MLX copy-in and MLX to NumPy copy-out. +6. **Non-claims** — show that the surface explicitly does not claim CoreML, ANE, ANN, serving integration, or zero-copy everywhere. + +Do not narrate beyond what the report proves. + +## Package contents + +The package directory contains: + +- `apple_uma_mechanical_sympathy_latest.json` +- `apple_uma_mechanical_sympathy_latest.md` +- `package_manifest.json` +- `README.md` +- `APPLE_SHARING_NOTE.md` + +Before sharing, inspect `package_manifest.json` and confirm: + +```text +allow_stale: false +mlx_summary.present: true +mlx_summary.skipped: false +mlx_summary.all_cases_parity_pass: true +mlx_summary.serving_authorized: false +``` + +## Suggested Apple-facing phrasing + +Use restrained language: + +> This is a deterministic CORE benchmark package for Apple Silicon UMA review. It isolates exact CGA recall, MLX score-vector parity, Rust/native boundary behavior, and copy-boundary truth tables. It is benchmark-only evidence: no CoreML, ANE, Metal, ANN, or serving-path acceleration claim is made. + +## Stop conditions + +Do not record/share the package if any of these are true: + +- MLX import fails; +- MLX exact CGA recall is absent or skipped; +- MLX parity is false; +- Workbench shows a stale-report warning; +- the package was generated with `--allow-stale`; +- the report claims serving authorization; +- copy boundaries are missing or vague. diff --git a/scripts/package_apple_uma_demo.py b/scripts/package_apple_uma_demo.py new file mode 100644 index 00000000..7b37fe2f --- /dev/null +++ b/scripts/package_apple_uma_demo.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +"""Build a shareable Apple UMA benchmark demo package. + +The package is generated locally from the persisted Apple UMA benchmark report. +It is intentionally claim-safe: by default it refuses to package stale reports +where the MLX exact recall track is absent, skipped, or parity-failing. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import shutil +import subprocess +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +import sys +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from workbench.apple_uma_report import read_apple_uma_report +DEFAULT_REPORT_JSON = REPO_ROOT / "evals" / "reports" / "apple_uma_mechanical_sympathy_latest.json" +DEFAULT_REPORT_MD = REPO_ROOT / "evals" / "reports" / "apple_uma_mechanical_sympathy_latest.md" +DEFAULT_OUT_ROOT = REPO_ROOT / "dist" / "apple-uma-demo" + + +class DemoPackageError(RuntimeError): + """Raised when the demo package would overclaim or lack required evidence.""" + + +@dataclass(frozen=True) +class PackagePaths: + package_dir: Path + report_json: Path + report_md: Path + manifest: Path + readme: Path + sharing_note: Path + + +def sha256_file(path: Path) -> str: + return "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() + + +def utc_stamp() -> str: + return datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + + +def run_report_refresh(*, core_backend: str) -> None: + env = os.environ.copy() + env["CORE_BACKEND"] = core_backend + command = [ + "uv", + "run", + "python", + "-m", + "benchmarks.apple_uma_mechanical_sympathy", + "--write-report", + ] + subprocess.run(command, cwd=REPO_ROOT, env=env, check=True) + + +def _mlx_summary(projection: dict[str, Any]) -> dict[str, Any]: + return projection["tracks"]["mlx_exact_cga_recall"] + + +def validate_demo_report(projection: dict[str, Any], *, allow_stale: bool) -> list[str]: + """Return warnings; raise when the report cannot be packaged safely.""" + + warnings: list[str] = [] + if projection.get("read_only") is not True: + raise DemoPackageError("Apple UMA report projection must be read-only") + + mlx = _mlx_summary(projection) + if not mlx.get("present"): + message = "MLX exact CGA recall track is absent from the report" + if not allow_stale: + raise DemoPackageError(message) + warnings.append(message) + if mlx.get("skipped"): + message = f"MLX exact CGA recall track is skipped: {mlx.get('reason') or 'no reason'}" + if not allow_stale: + raise DemoPackageError(message) + warnings.append(message) + if not mlx.get("all_cases_parity_pass"): + message = "MLX exact CGA recall parity did not pass for all cases" + if not allow_stale: + raise DemoPackageError(message) + warnings.append(message) + if int(mlx.get("case_count") or 0) <= 0: + message = "MLX exact CGA recall emitted no measured cases" + if not allow_stale: + raise DemoPackageError(message) + warnings.append(message) + if mlx.get("serving_authorized") is True: + raise DemoPackageError("MLX report unexpectedly claims serving authorization") + + non_claims = projection.get("non_claims") or [] + required_non_claim_fragments = [ + "MLX semantic", + "ANN", + "CoreML", + ] + missing = [ + fragment + for fragment in required_non_claim_fragments + if not any(fragment in str(item) for item in non_claims) + ] + if missing: + warnings.append( + "Non-claim list is missing expected fragments: " + ", ".join(missing) + ) + + return warnings + + +def _format_case_rows(mlx: dict[str, Any]) -> str: + cases = mlx.get("cases") or [] + if not cases: + return "- No MLX cases were emitted in this report." + rows = [] + for case in cases: + rows.append( + "- N={N}: p50={p50} ms, rows/sec={rows_per_sec}, parity={parity}".format( + N=case.get("N", "n/a"), + p50=case.get("p50_ms", "n/a"), + rows_per_sec=case.get("rows_per_sec", "n/a"), + parity=(case.get("parity") or {}).get("parity_pass"), + ) + ) + return "\n".join(rows) + + +def render_package_readme(projection: dict[str, Any], warnings: list[str]) -> str: + mlx = _mlx_summary(projection) + warning_block = "\n".join(f"- {warning}" for warning in warnings) or "- none" + return f"""# CORE Apple UMA demo package + +This package was generated from the persisted Apple UMA mechanical-sympathy benchmark report. + +## Report identity + +- Benchmark: {projection['benchmark_name']} +- Version: {projection['benchmark_version']} +- Source path: `{projection['source_path']}` +- Source digest: `{projection['source_digest']}` +- Read-only projection: {projection['read_only']} + +## MLX exact CGA recall + +- Track present: {mlx.get('present')} +- Track skipped: {mlx.get('skipped')} +- Case count: {mlx.get('case_count')} +- All cases parity pass: {mlx.get('all_cases_parity_pass')} +- Serving authorized: {mlx.get('serving_authorized')} + +{_format_case_rows(mlx)} + +## Copy boundaries + +The MLX exact recall path copies NumPy inputs into MLX arrays and copies MLX scores back to NumPy for canonical stable top-k ordering. The package does not claim zero-copy everywhere. + +## Warnings + +{warning_block} + +## Explicit non-claims + +""" + "\n".join(f"- {item}" for item in projection.get("non_claims", [])) + "\n" + + +def render_sharing_note(projection: dict[str, Any]) -> str: + mlx = _mlx_summary(projection) + return f"""# Suggested sharing note + +CORE now has a reproducible Apple Silicon UMA benchmark package that isolates exact CGA recall, Rust/native boundary behavior, MLX score-vector parity, copy boundaries, and Workbench report-card evidence. + +The current package reports MLX exact recall as: + +- present: {mlx.get('present')} +- skipped: {mlx.get('skipped')} +- cases: {mlx.get('case_count')} +- all cases parity pass: {mlx.get('all_cases_parity_pass')} + +Important boundaries: + +- This is benchmark-only evidence, not a serving-path integration. +- It does not claim CoreML or ANE acceleration. +- It does not claim approximate search or ANN recall. +- It does not claim zero-copy everywhere. +- It records copy-in and copy-out boundaries explicitly. +""" + + +def build_demo_package( + *, + report_json: Path, + report_md: Path, + out_root: Path, + stamp: str, + allow_stale: bool, +) -> PackagePaths: + if not report_json.exists(): + raise DemoPackageError(f"missing report JSON: {report_json}") + if not report_md.exists(): + raise DemoPackageError(f"missing report markdown: {report_md}") + + projection = read_apple_uma_report(report_json) + warnings = validate_demo_report(projection, allow_stale=allow_stale) + + package_dir = out_root / stamp + package_dir.mkdir(parents=True, exist_ok=True) + report_json_out = package_dir / "apple_uma_mechanical_sympathy_latest.json" + report_md_out = package_dir / "apple_uma_mechanical_sympathy_latest.md" + manifest_out = package_dir / "package_manifest.json" + readme_out = package_dir / "README.md" + sharing_note_out = package_dir / "APPLE_SHARING_NOTE.md" + + shutil.copy2(report_json, report_json_out) + shutil.copy2(report_md, report_md_out) + + manifest = { + "package_kind": "apple_uma_demo_package", + "generated_at_utc": datetime.now(UTC).isoformat(), + "report_id": projection["report_id"], + "benchmark_name": projection["benchmark_name"], + "benchmark_version": projection["benchmark_version"], + "source_report_digest": projection["source_digest"], + "packaged_report_json_digest": sha256_file(report_json_out), + "packaged_report_markdown_digest": sha256_file(report_md_out), + "allow_stale": allow_stale, + "warnings": warnings, + "mlx_summary": _mlx_summary(projection), + "platform": { + "system": platform.system(), + "machine": platform.machine(), + "processor": platform.processor(), + "python_version": platform.python_version(), + }, + "non_claims": projection.get("non_claims", []), + } + + manifest_out.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + readme_out.write_text(render_package_readme(projection, warnings), encoding="utf-8") + sharing_note_out.write_text(render_sharing_note(projection), encoding="utf-8") + + return PackagePaths( + package_dir=package_dir, + report_json=report_json_out, + report_md=report_md_out, + manifest=manifest_out, + readme=readme_out, + sharing_note=sharing_note_out, + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--report-json", type=Path, default=DEFAULT_REPORT_JSON) + parser.add_argument("--report-md", type=Path, default=DEFAULT_REPORT_MD) + parser.add_argument("--out-root", type=Path, default=DEFAULT_OUT_ROOT) + parser.add_argument("--stamp", default=utc_stamp()) + parser.add_argument("--allow-stale", action="store_true") + parser.add_argument("--refresh-report", action="store_true") + parser.add_argument("--core-backend", default="rust") + args = parser.parse_args(argv) + + if args.refresh_report: + run_report_refresh(core_backend=args.core_backend) + + paths = build_demo_package( + report_json=args.report_json, + report_md=args.report_md, + out_root=args.out_root, + stamp=args.stamp, + allow_stale=args.allow_stale, + ) + print(paths.package_dir) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_apple_uma_demo_package.py b/tests/test_apple_uma_demo_package.py new file mode 100644 index 00000000..66221ba0 --- /dev/null +++ b/tests/test_apple_uma_demo_package.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from scripts.package_apple_uma_demo import DemoPackageError, build_demo_package + + +def _base_report() -> dict: + return { + "benchmark_name": "CORE Apple Silicon UMA Mechanical Sympathy Benchmark", + "benchmark_version": "1.0.3", + "backend_status": { + "native_status": "rust_active", + "using_rust": True, + "requested_backend": "rust", + "core_rs_import_succeeded": True, + }, + "tracks": { + "cl41_scalar_ops": {"skipped": False}, + "exact_cga_recall": {"skipped": False}, + "diffusion_step": {"skipped": False}, + "frame_verdict_ttfv": {"skipped": False}, + "array_codec_replay": {"skipped": False}, + "mlx_exact_cga_recall": { + "skipped": False, + "benchmark_only": True, + "serving_authorized": False, + "semantic_backend": "python/rust canonical exact recall", + "score_computation": "MLX exact diagonal CGA score vector; no ANN or approximate search", + "top_k_ordering": "canonical NumPy stable ordering after score copy-out", + "copy_boundary": { + "input": "NumPy -> MLX array copy at benchmark boundary", + "output": "MLX score vector -> NumPy copy for stable top-k", + "zero_copy_input": "no", + }, + "mlx_status": { + "import_succeeded": True, + "default_device": "Device(gpu, 0)", + }, + "cases": [ + { + "N": 128, + "top_k": 5, + "rows_per_sec": 130088.678, + "copy_in_boundary": "NumPy contiguous float32 matrix/query copied into MLX arrays", + "copy_out_boundary": "MLX score vector copied to NumPy for canonical stable top-k ordering", + "timing": {"p50_ms": 0.977, "p95_ms": 1.2, "mean_ms": 1.0}, + "parity": { + "parity_pass": True, + "top_k_indices_match": True, + "scores_close": True, + "max_abs_score_delta": 0.0, + }, + } + ], + }, + }, + "claim_safety_audit": { + "safe_claims": ["Exact CGA recall via algebra.backend.vault_recall — no ANN."], + "rust_backend_notes": [], + "known_copy_paths": ["MLX score vector copied to NumPy for stable top-k."], + "known_zero_copy_input_paths": [], + "future_work": ["Metal kernel experiment requires separate ADR/parity lane."], + "unsafe_claims_not_made": [ + "No MLX semantic-backend claim.", + "No ANN/approximate-search benchmark.", + "No CoreML acceleration claim.", + ], + }, + "copy_zero_copy_truth_table": [ + { + "path": "benchmarks.apple_uma_mlx_exact_recall", + "input": "NumPy contiguous float32 matrix/query copied into MLX arrays", + "output": "MLX score vector copied to NumPy for canonical stable top-k", + "zero_copy_input": "no", + } + ], + } + + +def _write_report_pair(tmp_path: Path, report: dict) -> tuple[Path, Path]: + json_path = tmp_path / "apple_uma_mechanical_sympathy_latest.json" + md_path = tmp_path / "apple_uma_mechanical_sympathy_latest.md" + json_path.write_text(json.dumps(report, sort_keys=True) + "\n", encoding="utf-8") + md_path.write_text("# Apple UMA report\n\nMLX exact CGA recall present.\n", encoding="utf-8") + return json_path, md_path + + +def test_package_builder_refuses_stale_report_by_default(tmp_path: Path) -> None: + report = _base_report() + del report["tracks"]["mlx_exact_cga_recall"] + json_path, md_path = _write_report_pair(tmp_path, report) + + with pytest.raises(DemoPackageError, match="MLX exact CGA recall track is absent"): + build_demo_package( + report_json=json_path, + report_md=md_path, + out_root=tmp_path / "dist", + stamp="demo", + allow_stale=False, + ) + + +def test_package_builder_allows_stale_report_only_when_explicit(tmp_path: Path) -> None: + report = _base_report() + del report["tracks"]["mlx_exact_cga_recall"] + json_path, md_path = _write_report_pair(tmp_path, report) + + paths = build_demo_package( + report_json=json_path, + report_md=md_path, + out_root=tmp_path / "dist", + stamp="demo", + allow_stale=True, + ) + + manifest = json.loads(paths.manifest.read_text(encoding="utf-8")) + assert manifest["allow_stale"] is True + assert manifest["warnings"] + assert manifest["mlx_summary"]["present"] is False + assert paths.readme.read_text(encoding="utf-8").count("Track present: False") == 1 + + +def test_package_builder_emits_claim_safe_mlx_present_bundle(tmp_path: Path) -> None: + report = _base_report() + json_path, md_path = _write_report_pair(tmp_path, report) + + paths = build_demo_package( + report_json=json_path, + report_md=md_path, + out_root=tmp_path / "dist", + stamp="demo", + allow_stale=False, + ) + + assert paths.report_json.exists() + assert paths.report_md.exists() + assert paths.readme.exists() + assert paths.sharing_note.exists() + + manifest = json.loads(paths.manifest.read_text(encoding="utf-8")) + assert manifest["package_kind"] == "apple_uma_demo_package" + assert manifest["allow_stale"] is False + assert manifest["warnings"] == [] + assert manifest["mlx_summary"]["present"] is True + assert manifest["mlx_summary"]["all_cases_parity_pass"] is True + assert manifest["mlx_summary"]["serving_authorized"] is False + assert manifest["source_report_digest"].startswith("sha256:") + assert manifest["packaged_report_json_digest"].startswith("sha256:") + + readme = paths.readme.read_text(encoding="utf-8") + assert "All cases parity pass: True" in readme + assert "No MLX semantic-backend claim." in readme + assert "does not claim zero-copy everywhere" in readme + + sharing_note = paths.sharing_note.read_text(encoding="utf-8") + assert "benchmark-only evidence" in sharing_note + assert "does not claim CoreML" in sharing_note + assert "does not claim approximate search" in sharing_note + + +def test_package_builder_refuses_serving_authorization_claim(tmp_path: Path) -> None: + report = _base_report() + report["tracks"]["mlx_exact_cga_recall"]["serving_authorized"] = True + json_path, md_path = _write_report_pair(tmp_path, report) + + with pytest.raises(DemoPackageError, match="serving authorization"): + build_demo_package( + report_json=json_path, + report_md=md_path, + out_root=tmp_path / "dist", + stamp="demo", + allow_stale=False, + )