diff --git a/tests/test_workbench_api.py b/tests/test_workbench_api.py index b8ec8e1e..ec30767b 100644 --- a/tests/test_workbench_api.py +++ b/tests/test_workbench_api.py @@ -447,3 +447,13 @@ def test_trace_construction_route() -> None: assert data["status"] == "missing_evidence" assert data["diagnostic_only"] is True assert data["serving_allowed"] is False + +def test_apple_uma_report_route_returns_read_only_projection() -> None: + response = _request("GET", "/benchmarks/apple-uma/report") + assert response.status == 200 + data = response.payload["data"] + assert data["read_only"] is True + assert data["report_id"] == "apple_uma_mechanical_sympathy_latest" + assert "backend_status" in data + assert "mlx_exact_cga_recall" in data["tracks"] + assert any("No MLX semantic-backend" in item for item in data["non_claims"]) diff --git a/tests/test_workbench_apple_uma_report.py b/tests/test_workbench_apple_uma_report.py new file mode 100644 index 00000000..190bc807 --- /dev/null +++ b/tests/test_workbench_apple_uma_report.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from workbench.apple_uma_report import ( + AppleUmaReportMalformed, + project_apple_uma_report, + read_apple_uma_report, +) + + +def _base_report() -> dict: + return { + "benchmark_name": "CORE Apple Silicon UMA Mechanical Sympathy Benchmark", + "benchmark_version": "1.0.3", + "backend_status": { + "native_status": "rust_active", + "using_rust": True, + "requested_backend": "rust", + "core_rs_import_succeeded": True, + }, + "tracks": { + "cl41_scalar_ops": {"skipped": False}, + "exact_cga_recall": {"skipped": False}, + "diffusion_step": {"skipped": False}, + "frame_verdict_ttfv": {"skipped": False}, + "array_codec_replay": {"skipped": False}, + }, + "claim_safety_audit": { + "safe_claims": ["Exact CGA recall via algebra.backend.vault_recall — no ANN."], + "rust_backend_notes": [], + "known_copy_paths": [], + "known_zero_copy_input_paths": [], + "future_work": ["Metal kernel experiment requires separate ADR/parity lane."], + "unsafe_claims_not_made": [ + "No MLX semantic-backend claim.", + "No ANN/approximate-search benchmark.", + ], + }, + "copy_zero_copy_truth_table": [ + { + "path": "benchmarks.apple_uma_mlx_exact_recall", + "input": "NumPy contiguous float32 matrix/query copied into MLX arrays", + "output": "MLX score vector copied to NumPy for canonical stable top-k", + "zero_copy_input": "no", + } + ], + } + + +def _write_report(tmp_path: Path, report: dict) -> Path: + path = tmp_path / "apple_uma_mechanical_sympathy_latest.json" + path.write_text(json.dumps(report, sort_keys=True) + "\n", encoding="utf-8") + return path + + +def test_project_old_report_without_mlx_track_does_not_fabricate_success( + tmp_path: Path, +) -> None: + report = _base_report() + path = _write_report(tmp_path, report) + + projected = read_apple_uma_report(path) + + assert projected["read_only"] is True + assert projected["source_path"].endswith("apple_uma_mechanical_sympathy_latest.json") + assert projected["source_digest"].startswith("sha256:") + mlx = projected["tracks"]["mlx_exact_cga_recall"] + assert mlx["present"] is False + assert mlx["skipped"] is True + assert mlx["case_count"] == 0 + assert mlx["all_cases_parity_pass"] is False + assert "mlx_exact_cga_recall" in projected["tracks"]["missing_required"] + + +def test_project_mlx_present_report_summarizes_parity_and_boundaries( + tmp_path: Path, +) -> None: + report = _base_report() + report["tracks"]["mlx_exact_cga_recall"] = { + "skipped": False, + "benchmark_only": True, + "serving_authorized": False, + "semantic_backend": "python/rust canonical exact recall", + "score_computation": "MLX exact diagonal CGA score vector; no ANN or approximate search", + "top_k_ordering": "canonical NumPy stable ordering after score copy-out", + "copy_boundary": { + "input": "NumPy -> MLX array copy at benchmark boundary", + "output": "MLX score vector -> NumPy copy for stable top-k", + "zero_copy_input": "no", + }, + "mlx_status": {"import_succeeded": True, "default_device": "Device(gpu, 0)"}, + "cases": [ + { + "N": 128, + "top_k": 5, + "rows_per_sec": 123.4, + "copy_in_boundary": "NumPy contiguous float32 matrix/query copied into MLX arrays", + "copy_out_boundary": "MLX score vector copied to NumPy for canonical stable top-k ordering", + "timing": {"p50_ms": 1.0, "p95_ms": 1.5, "mean_ms": 1.2}, + "parity": { + "parity_pass": True, + "top_k_indices_match": True, + "scores_close": True, + "max_abs_score_delta": 0.0, + }, + } + ], + } + path = _write_report(tmp_path, report) + + projected = read_apple_uma_report(path) + + mlx = projected["tracks"]["mlx_exact_cga_recall"] + assert mlx["present"] is True + assert mlx["skipped"] is False + assert mlx["benchmark_only"] is True + assert mlx["serving_authorized"] is False + assert mlx["semantic_backend"] == "python/rust canonical exact recall" + assert mlx["case_count"] == 1 + assert mlx["all_cases_parity_pass"] is True + assert mlx["cases"][0]["parity"]["parity_pass"] is True + assert mlx["cases"][0]["copy_in_boundary"].startswith("NumPy") + assert mlx["cases"][0]["copy_out_boundary"].startswith("MLX") + assert projected["tracks"]["missing_required"] == [] + assert any("No MLX semantic-backend" in item for item in projected["non_claims"]) + assert projected["copy_boundaries"][0]["zero_copy_input"] == "no" + + +def test_project_mlx_skipped_report_preserves_reason(tmp_path: Path) -> None: + report = _base_report() + report["tracks"]["mlx_exact_cga_recall"] = { + "skipped": True, + "reason": "MLX unavailable: No module named 'mlx'", + "benchmark_only": True, + "serving_authorized": False, + "semantic_backend": "python/rust canonical exact recall", + "non_claims": ["No serving integration."], + } + path = _write_report(tmp_path, report) + + projected = read_apple_uma_report(path) + + mlx = projected["tracks"]["mlx_exact_cga_recall"] + assert mlx["present"] is True + assert mlx["skipped"] is True + assert mlx["reason"] == "MLX unavailable: No module named 'mlx'" + assert mlx["case_count"] == 0 + assert mlx["all_cases_parity_pass"] is False + assert mlx["serving_authorized"] is False + + +def test_malformed_report_fails_closed(tmp_path: Path) -> None: + path = _write_report(tmp_path, {"benchmark_name": "missing required fields"}) + + with pytest.raises(AppleUmaReportMalformed, match="missing required keys"): + read_apple_uma_report(path) + + +def test_project_rejects_non_object_report(tmp_path: Path) -> None: + path = tmp_path / "apple_uma_mechanical_sympathy_latest.json" + path.write_text("[]\n", encoding="utf-8") + + with pytest.raises(AppleUmaReportMalformed, match="must be an object"): + read_apple_uma_report(path) + + +def test_project_uses_supplied_source_path_for_digest(tmp_path: Path) -> None: + report = _base_report() + path = _write_report(tmp_path, report) + + projected_a = project_apple_uma_report(report, source_path=path) + projected_b = read_apple_uma_report(path) + + assert projected_a["source_digest"] == projected_b["source_digest"] + assert projected_a["source_path"] == projected_b["source_path"] diff --git a/workbench/api.py b/workbench/api.py index 7cd04a79..2e0e9bfb 100644 --- a/workbench/api.py +++ b/workbench/api.py @@ -25,6 +25,7 @@ from workbench.construction_endpoint import ( construction_evidence_response, construction_turn_id_from_path, ) +from workbench.apple_uma_report import read_apple_uma_report from workbench.tour import determinism_tour from workbench.field_evidence import ( field_evidence_from_journal_entry, @@ -326,6 +327,8 @@ class WorkbenchApi: return ApiResponse(400, error("bad_request", str(exc))) if method == "GET" and path == "/evals": return ApiResponse(200, ok({"lanes": readers.list_eval_lanes()})) + if method == "GET" and path == "/benchmarks/apple-uma/report": + return ApiResponse(200, ok(read_apple_uma_report())) if method == "GET" and path.startswith("/evals/"): lane = unquote(path.removeprefix("/evals/")) return ApiResponse(200, ok(readers.read_eval_lane(lane))) diff --git a/workbench/apple_uma_report.py b/workbench/apple_uma_report.py new file mode 100644 index 00000000..44df04d9 --- /dev/null +++ b/workbench/apple_uma_report.py @@ -0,0 +1,167 @@ +"""Read-only Apple UMA benchmark report projection for Workbench. + +This module projects the latest persisted Apple UMA benchmark JSON into a stable +UI-facing read model. It never runs benchmarks, imports MLX, imports Rust, or +mutates report artifacts. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_APPLE_UMA_REPORT = ( + REPO_ROOT / "evals" / "reports" / "apple_uma_mechanical_sympathy_latest.json" +) +REPORT_ID = "apple_uma_mechanical_sympathy_latest" +REQUIRED_TOP_LEVEL_KEYS = frozenset( + { + "benchmark_name", + "benchmark_version", + "backend_status", + "tracks", + "claim_safety_audit", + "copy_zero_copy_truth_table", + } +) + + +class AppleUmaReportMalformed(ValueError): + """Raised when a persisted Apple UMA report cannot be projected safely.""" + + +def _sha256_file(path: Path) -> str: + return "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() + + +def _relative(path: Path) -> str: + try: + return path.resolve().relative_to(REPO_ROOT.resolve()).as_posix() + except ValueError: + return path.resolve().as_posix() + + +def _require_mapping(value: object, label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise AppleUmaReportMalformed(f"{label} must be an object") + return value + + +def _require_list(value: object, label: str) -> list[Any]: + if not isinstance(value, list): + raise AppleUmaReportMalformed(f"{label} must be a list") + return value + + +def _timing_summary(case: dict[str, Any]) -> dict[str, Any]: + timing = _require_mapping(case.get("timing", {}), "MLX case timing") + return { + "N": case.get("N"), + "top_k": case.get("top_k"), + "p50_ms": timing.get("p50_ms"), + "p95_ms": timing.get("p95_ms"), + "mean_ms": timing.get("mean_ms"), + "rows_per_sec": case.get("rows_per_sec"), + "parity": _require_mapping(case.get("parity", {}), "MLX case parity"), + "copy_in_boundary": case.get("copy_in_boundary"), + "copy_out_boundary": case.get("copy_out_boundary"), + } + + +def _mlx_track_projection(track: object) -> dict[str, Any]: + if not isinstance(track, dict): + return { + "present": False, + "skipped": True, + "reason": "MLX exact CGA recall track absent from report", + "case_count": 0, + "all_cases_parity_pass": False, + "cases": [], + } + + skipped = bool(track.get("skipped", False)) + cases_raw = _require_list(track.get("cases", []), "MLX cases") if not skipped else [] + cases = [_timing_summary(_require_mapping(case, "MLX case")) for case in cases_raw] + parity_values = [bool(case["parity"].get("parity_pass", False)) for case in cases] + return { + "present": True, + "skipped": skipped, + "reason": track.get("reason"), + "benchmark_only": bool(track.get("benchmark_only", False)), + "serving_authorized": bool(track.get("serving_authorized", False)), + "semantic_backend": track.get("semantic_backend"), + "score_computation": track.get("score_computation"), + "top_k_ordering": track.get("top_k_ordering"), + "copy_boundary": track.get("copy_boundary"), + "mlx_status": track.get("mlx_status", {}), + "case_count": len(cases), + "all_cases_parity_pass": bool(cases) and all(parity_values), + "cases": cases, + } + + +def _track_inventory(tracks: dict[str, Any]) -> dict[str, Any]: + required = [ + "cl41_scalar_ops", + "exact_cga_recall", + "mlx_exact_cga_recall", + "diffusion_step", + "frame_verdict_ttfv", + "array_codec_replay", + ] + return { + "available": sorted(tracks.keys()), + "required": required, + "missing_required": [name for name in required if name not in tracks], + "mlx_exact_cga_recall": _mlx_track_projection(tracks.get("mlx_exact_cga_recall")), + } + + +def project_apple_uma_report(report: dict[str, Any], *, source_path: Path) -> dict[str, Any]: + missing = sorted(REQUIRED_TOP_LEVEL_KEYS - set(report.keys())) + if missing: + raise AppleUmaReportMalformed( + f"Apple UMA report missing required keys: {', '.join(missing)}" + ) + + tracks = _require_mapping(report.get("tracks"), "tracks") + claim_safety = _require_mapping(report.get("claim_safety_audit"), "claim_safety_audit") + truth_table = _require_list( + report.get("copy_zero_copy_truth_table"), "copy_zero_copy_truth_table" + ) + non_claims = _require_list( + claim_safety.get("unsafe_claims_not_made", []), "unsafe_claims_not_made" + ) + + return { + "read_only": True, + "report_id": REPORT_ID, + "source_path": _relative(source_path), + "source_digest": _sha256_file(source_path), + "benchmark_name": report["benchmark_name"], + "benchmark_version": report["benchmark_version"], + "metadata": report.get("_metadata", {}), + "backend_status": _require_mapping(report.get("backend_status"), "backend_status"), + "tracks": _track_inventory(tracks), + "copy_boundaries": truth_table, + "non_claims": non_claims, + "claim_safety": { + "safe_claims": claim_safety.get("safe_claims", []), + "rust_backend_notes": claim_safety.get("rust_backend_notes", []), + "known_copy_paths": claim_safety.get("known_copy_paths", []), + "known_zero_copy_input_paths": claim_safety.get( + "known_zero_copy_input_paths", [] + ), + "future_work": claim_safety.get("future_work", []), + }, + } + + +def read_apple_uma_report(path: Path | None = None) -> dict[str, Any]: + source_path = DEFAULT_APPLE_UMA_REPORT if path is None else path + raw = json.loads(source_path.read_text(encoding="utf-8")) + report = _require_mapping(raw, "Apple UMA report") + return project_apple_uma_report(report, source_path=source_path)