feat(workbench): add Apple UMA report read model (#911)
* feat(workbench): add Apple UMA report read model * test(workbench): cover Apple UMA report projection * feat(workbench): expose Apple UMA report route
This commit is contained in:
Shay
GitHub
parent
cd7579be32
commit
f8d882be48
4 changed files with 359 additions and 0 deletions
|
|
@ -447,3 +447,13 @@ def test_trace_construction_route() -> None:
|
|||
assert data["status"] == "missing_evidence"
|
||||
assert data["diagnostic_only"] is True
|
||||
assert data["serving_allowed"] is False
|
||||
|
||||
def test_apple_uma_report_route_returns_read_only_projection() -> None:
|
||||
response = _request("GET", "/benchmarks/apple-uma/report")
|
||||
assert response.status == 200
|
||||
data = response.payload["data"]
|
||||
assert data["read_only"] is True
|
||||
assert data["report_id"] == "apple_uma_mechanical_sympathy_latest"
|
||||
assert "backend_status" in data
|
||||
assert "mlx_exact_cga_recall" in data["tracks"]
|
||||
assert any("No MLX semantic-backend" in item for item in data["non_claims"])
|
||||
|
|
|
|||
179
tests/test_workbench_apple_uma_report.py
Normal file
179
tests/test_workbench_apple_uma_report.py
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from workbench.apple_uma_report import (
|
||||
AppleUmaReportMalformed,
|
||||
project_apple_uma_report,
|
||||
read_apple_uma_report,
|
||||
)
|
||||
|
||||
|
||||
def _base_report() -> dict:
|
||||
return {
|
||||
"benchmark_name": "CORE Apple Silicon UMA Mechanical Sympathy Benchmark",
|
||||
"benchmark_version": "1.0.3",
|
||||
"backend_status": {
|
||||
"native_status": "rust_active",
|
||||
"using_rust": True,
|
||||
"requested_backend": "rust",
|
||||
"core_rs_import_succeeded": True,
|
||||
},
|
||||
"tracks": {
|
||||
"cl41_scalar_ops": {"skipped": False},
|
||||
"exact_cga_recall": {"skipped": False},
|
||||
"diffusion_step": {"skipped": False},
|
||||
"frame_verdict_ttfv": {"skipped": False},
|
||||
"array_codec_replay": {"skipped": False},
|
||||
},
|
||||
"claim_safety_audit": {
|
||||
"safe_claims": ["Exact CGA recall via algebra.backend.vault_recall — no ANN."],
|
||||
"rust_backend_notes": [],
|
||||
"known_copy_paths": [],
|
||||
"known_zero_copy_input_paths": [],
|
||||
"future_work": ["Metal kernel experiment requires separate ADR/parity lane."],
|
||||
"unsafe_claims_not_made": [
|
||||
"No MLX semantic-backend claim.",
|
||||
"No ANN/approximate-search benchmark.",
|
||||
],
|
||||
},
|
||||
"copy_zero_copy_truth_table": [
|
||||
{
|
||||
"path": "benchmarks.apple_uma_mlx_exact_recall",
|
||||
"input": "NumPy contiguous float32 matrix/query copied into MLX arrays",
|
||||
"output": "MLX score vector copied to NumPy for canonical stable top-k",
|
||||
"zero_copy_input": "no",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _write_report(tmp_path: Path, report: dict) -> Path:
|
||||
path = tmp_path / "apple_uma_mechanical_sympathy_latest.json"
|
||||
path.write_text(json.dumps(report, sort_keys=True) + "\n", encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def test_project_old_report_without_mlx_track_does_not_fabricate_success(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
report = _base_report()
|
||||
path = _write_report(tmp_path, report)
|
||||
|
||||
projected = read_apple_uma_report(path)
|
||||
|
||||
assert projected["read_only"] is True
|
||||
assert projected["source_path"].endswith("apple_uma_mechanical_sympathy_latest.json")
|
||||
assert projected["source_digest"].startswith("sha256:")
|
||||
mlx = projected["tracks"]["mlx_exact_cga_recall"]
|
||||
assert mlx["present"] is False
|
||||
assert mlx["skipped"] is True
|
||||
assert mlx["case_count"] == 0
|
||||
assert mlx["all_cases_parity_pass"] is False
|
||||
assert "mlx_exact_cga_recall" in projected["tracks"]["missing_required"]
|
||||
|
||||
|
||||
def test_project_mlx_present_report_summarizes_parity_and_boundaries(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
report = _base_report()
|
||||
report["tracks"]["mlx_exact_cga_recall"] = {
|
||||
"skipped": False,
|
||||
"benchmark_only": True,
|
||||
"serving_authorized": False,
|
||||
"semantic_backend": "python/rust canonical exact recall",
|
||||
"score_computation": "MLX exact diagonal CGA score vector; no ANN or approximate search",
|
||||
"top_k_ordering": "canonical NumPy stable ordering after score copy-out",
|
||||
"copy_boundary": {
|
||||
"input": "NumPy -> MLX array copy at benchmark boundary",
|
||||
"output": "MLX score vector -> NumPy copy for stable top-k",
|
||||
"zero_copy_input": "no",
|
||||
},
|
||||
"mlx_status": {"import_succeeded": True, "default_device": "Device(gpu, 0)"},
|
||||
"cases": [
|
||||
{
|
||||
"N": 128,
|
||||
"top_k": 5,
|
||||
"rows_per_sec": 123.4,
|
||||
"copy_in_boundary": "NumPy contiguous float32 matrix/query copied into MLX arrays",
|
||||
"copy_out_boundary": "MLX score vector copied to NumPy for canonical stable top-k ordering",
|
||||
"timing": {"p50_ms": 1.0, "p95_ms": 1.5, "mean_ms": 1.2},
|
||||
"parity": {
|
||||
"parity_pass": True,
|
||||
"top_k_indices_match": True,
|
||||
"scores_close": True,
|
||||
"max_abs_score_delta": 0.0,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
path = _write_report(tmp_path, report)
|
||||
|
||||
projected = read_apple_uma_report(path)
|
||||
|
||||
mlx = projected["tracks"]["mlx_exact_cga_recall"]
|
||||
assert mlx["present"] is True
|
||||
assert mlx["skipped"] is False
|
||||
assert mlx["benchmark_only"] is True
|
||||
assert mlx["serving_authorized"] is False
|
||||
assert mlx["semantic_backend"] == "python/rust canonical exact recall"
|
||||
assert mlx["case_count"] == 1
|
||||
assert mlx["all_cases_parity_pass"] is True
|
||||
assert mlx["cases"][0]["parity"]["parity_pass"] is True
|
||||
assert mlx["cases"][0]["copy_in_boundary"].startswith("NumPy")
|
||||
assert mlx["cases"][0]["copy_out_boundary"].startswith("MLX")
|
||||
assert projected["tracks"]["missing_required"] == []
|
||||
assert any("No MLX semantic-backend" in item for item in projected["non_claims"])
|
||||
assert projected["copy_boundaries"][0]["zero_copy_input"] == "no"
|
||||
|
||||
|
||||
def test_project_mlx_skipped_report_preserves_reason(tmp_path: Path) -> None:
|
||||
report = _base_report()
|
||||
report["tracks"]["mlx_exact_cga_recall"] = {
|
||||
"skipped": True,
|
||||
"reason": "MLX unavailable: No module named 'mlx'",
|
||||
"benchmark_only": True,
|
||||
"serving_authorized": False,
|
||||
"semantic_backend": "python/rust canonical exact recall",
|
||||
"non_claims": ["No serving integration."],
|
||||
}
|
||||
path = _write_report(tmp_path, report)
|
||||
|
||||
projected = read_apple_uma_report(path)
|
||||
|
||||
mlx = projected["tracks"]["mlx_exact_cga_recall"]
|
||||
assert mlx["present"] is True
|
||||
assert mlx["skipped"] is True
|
||||
assert mlx["reason"] == "MLX unavailable: No module named 'mlx'"
|
||||
assert mlx["case_count"] == 0
|
||||
assert mlx["all_cases_parity_pass"] is False
|
||||
assert mlx["serving_authorized"] is False
|
||||
|
||||
|
||||
def test_malformed_report_fails_closed(tmp_path: Path) -> None:
|
||||
path = _write_report(tmp_path, {"benchmark_name": "missing required fields"})
|
||||
|
||||
with pytest.raises(AppleUmaReportMalformed, match="missing required keys"):
|
||||
read_apple_uma_report(path)
|
||||
|
||||
|
||||
def test_project_rejects_non_object_report(tmp_path: Path) -> None:
|
||||
path = tmp_path / "apple_uma_mechanical_sympathy_latest.json"
|
||||
path.write_text("[]\n", encoding="utf-8")
|
||||
|
||||
with pytest.raises(AppleUmaReportMalformed, match="must be an object"):
|
||||
read_apple_uma_report(path)
|
||||
|
||||
|
||||
def test_project_uses_supplied_source_path_for_digest(tmp_path: Path) -> None:
|
||||
report = _base_report()
|
||||
path = _write_report(tmp_path, report)
|
||||
|
||||
projected_a = project_apple_uma_report(report, source_path=path)
|
||||
projected_b = read_apple_uma_report(path)
|
||||
|
||||
assert projected_a["source_digest"] == projected_b["source_digest"]
|
||||
assert projected_a["source_path"] == projected_b["source_path"]
|
||||
|
|
@ -25,6 +25,7 @@ from workbench.construction_endpoint import (
|
|||
construction_evidence_response,
|
||||
construction_turn_id_from_path,
|
||||
)
|
||||
from workbench.apple_uma_report import read_apple_uma_report
|
||||
from workbench.tour import determinism_tour
|
||||
from workbench.field_evidence import (
|
||||
field_evidence_from_journal_entry,
|
||||
|
|
@ -326,6 +327,8 @@ class WorkbenchApi:
|
|||
return ApiResponse(400, error("bad_request", str(exc)))
|
||||
if method == "GET" and path == "/evals":
|
||||
return ApiResponse(200, ok({"lanes": readers.list_eval_lanes()}))
|
||||
if method == "GET" and path == "/benchmarks/apple-uma/report":
|
||||
return ApiResponse(200, ok(read_apple_uma_report()))
|
||||
if method == "GET" and path.startswith("/evals/"):
|
||||
lane = unquote(path.removeprefix("/evals/"))
|
||||
return ApiResponse(200, ok(readers.read_eval_lane(lane)))
|
||||
|
|
|
|||
167
workbench/apple_uma_report.py
Normal file
167
workbench/apple_uma_report.py
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
"""Read-only Apple UMA benchmark report projection for Workbench.
|
||||
|
||||
This module projects the latest persisted Apple UMA benchmark JSON into a stable
|
||||
UI-facing read model. It never runs benchmarks, imports MLX, imports Rust, or
|
||||
mutates report artifacts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_APPLE_UMA_REPORT = (
|
||||
REPO_ROOT / "evals" / "reports" / "apple_uma_mechanical_sympathy_latest.json"
|
||||
)
|
||||
REPORT_ID = "apple_uma_mechanical_sympathy_latest"
|
||||
REQUIRED_TOP_LEVEL_KEYS = frozenset(
|
||||
{
|
||||
"benchmark_name",
|
||||
"benchmark_version",
|
||||
"backend_status",
|
||||
"tracks",
|
||||
"claim_safety_audit",
|
||||
"copy_zero_copy_truth_table",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class AppleUmaReportMalformed(ValueError):
|
||||
"""Raised when a persisted Apple UMA report cannot be projected safely."""
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
return "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def _relative(path: Path) -> str:
|
||||
try:
|
||||
return path.resolve().relative_to(REPO_ROOT.resolve()).as_posix()
|
||||
except ValueError:
|
||||
return path.resolve().as_posix()
|
||||
|
||||
|
||||
def _require_mapping(value: object, label: str) -> dict[str, Any]:
|
||||
if not isinstance(value, dict):
|
||||
raise AppleUmaReportMalformed(f"{label} must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _require_list(value: object, label: str) -> list[Any]:
|
||||
if not isinstance(value, list):
|
||||
raise AppleUmaReportMalformed(f"{label} must be a list")
|
||||
return value
|
||||
|
||||
|
||||
def _timing_summary(case: dict[str, Any]) -> dict[str, Any]:
|
||||
timing = _require_mapping(case.get("timing", {}), "MLX case timing")
|
||||
return {
|
||||
"N": case.get("N"),
|
||||
"top_k": case.get("top_k"),
|
||||
"p50_ms": timing.get("p50_ms"),
|
||||
"p95_ms": timing.get("p95_ms"),
|
||||
"mean_ms": timing.get("mean_ms"),
|
||||
"rows_per_sec": case.get("rows_per_sec"),
|
||||
"parity": _require_mapping(case.get("parity", {}), "MLX case parity"),
|
||||
"copy_in_boundary": case.get("copy_in_boundary"),
|
||||
"copy_out_boundary": case.get("copy_out_boundary"),
|
||||
}
|
||||
|
||||
|
||||
def _mlx_track_projection(track: object) -> dict[str, Any]:
|
||||
if not isinstance(track, dict):
|
||||
return {
|
||||
"present": False,
|
||||
"skipped": True,
|
||||
"reason": "MLX exact CGA recall track absent from report",
|
||||
"case_count": 0,
|
||||
"all_cases_parity_pass": False,
|
||||
"cases": [],
|
||||
}
|
||||
|
||||
skipped = bool(track.get("skipped", False))
|
||||
cases_raw = _require_list(track.get("cases", []), "MLX cases") if not skipped else []
|
||||
cases = [_timing_summary(_require_mapping(case, "MLX case")) for case in cases_raw]
|
||||
parity_values = [bool(case["parity"].get("parity_pass", False)) for case in cases]
|
||||
return {
|
||||
"present": True,
|
||||
"skipped": skipped,
|
||||
"reason": track.get("reason"),
|
||||
"benchmark_only": bool(track.get("benchmark_only", False)),
|
||||
"serving_authorized": bool(track.get("serving_authorized", False)),
|
||||
"semantic_backend": track.get("semantic_backend"),
|
||||
"score_computation": track.get("score_computation"),
|
||||
"top_k_ordering": track.get("top_k_ordering"),
|
||||
"copy_boundary": track.get("copy_boundary"),
|
||||
"mlx_status": track.get("mlx_status", {}),
|
||||
"case_count": len(cases),
|
||||
"all_cases_parity_pass": bool(cases) and all(parity_values),
|
||||
"cases": cases,
|
||||
}
|
||||
|
||||
|
||||
def _track_inventory(tracks: dict[str, Any]) -> dict[str, Any]:
|
||||
required = [
|
||||
"cl41_scalar_ops",
|
||||
"exact_cga_recall",
|
||||
"mlx_exact_cga_recall",
|
||||
"diffusion_step",
|
||||
"frame_verdict_ttfv",
|
||||
"array_codec_replay",
|
||||
]
|
||||
return {
|
||||
"available": sorted(tracks.keys()),
|
||||
"required": required,
|
||||
"missing_required": [name for name in required if name not in tracks],
|
||||
"mlx_exact_cga_recall": _mlx_track_projection(tracks.get("mlx_exact_cga_recall")),
|
||||
}
|
||||
|
||||
|
||||
def project_apple_uma_report(report: dict[str, Any], *, source_path: Path) -> dict[str, Any]:
|
||||
missing = sorted(REQUIRED_TOP_LEVEL_KEYS - set(report.keys()))
|
||||
if missing:
|
||||
raise AppleUmaReportMalformed(
|
||||
f"Apple UMA report missing required keys: {', '.join(missing)}"
|
||||
)
|
||||
|
||||
tracks = _require_mapping(report.get("tracks"), "tracks")
|
||||
claim_safety = _require_mapping(report.get("claim_safety_audit"), "claim_safety_audit")
|
||||
truth_table = _require_list(
|
||||
report.get("copy_zero_copy_truth_table"), "copy_zero_copy_truth_table"
|
||||
)
|
||||
non_claims = _require_list(
|
||||
claim_safety.get("unsafe_claims_not_made", []), "unsafe_claims_not_made"
|
||||
)
|
||||
|
||||
return {
|
||||
"read_only": True,
|
||||
"report_id": REPORT_ID,
|
||||
"source_path": _relative(source_path),
|
||||
"source_digest": _sha256_file(source_path),
|
||||
"benchmark_name": report["benchmark_name"],
|
||||
"benchmark_version": report["benchmark_version"],
|
||||
"metadata": report.get("_metadata", {}),
|
||||
"backend_status": _require_mapping(report.get("backend_status"), "backend_status"),
|
||||
"tracks": _track_inventory(tracks),
|
||||
"copy_boundaries": truth_table,
|
||||
"non_claims": non_claims,
|
||||
"claim_safety": {
|
||||
"safe_claims": claim_safety.get("safe_claims", []),
|
||||
"rust_backend_notes": claim_safety.get("rust_backend_notes", []),
|
||||
"known_copy_paths": claim_safety.get("known_copy_paths", []),
|
||||
"known_zero_copy_input_paths": claim_safety.get(
|
||||
"known_zero_copy_input_paths", []
|
||||
),
|
||||
"future_work": claim_safety.get("future_work", []),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def read_apple_uma_report(path: Path | None = None) -> dict[str, Any]:
|
||||
source_path = DEFAULT_APPLE_UMA_REPORT if path is None else path
|
||||
raw = json.loads(source_path.read_text(encoding="utf-8"))
|
||||
report = _require_mapping(raw, "Apple UMA report")
|
||||
return project_apple_uma_report(report, source_path=source_path)
|
||||
Loading…
Reference in a new issue