From 29ca5524d0c0ed3b59d1a409445bd930ed10d509 Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 24 Jun 2026 15:33:42 -0700 Subject: [PATCH] feat(workbench-ui): add Apple UMA report card route (#912) * feat(workbench-ui): add Apple UMA report card route * feat(workbench-ui): register Apple UMA route * feat(workbench-ui): expose Apple UMA route path * feat(workbench-ui): wire Apple UMA report card into route registry * test(workbench-ui): fix UI and keyboard tests for Apple UMA route integration --- docs/workbench/UI-UX-GUIDE.md | 5 +- workbench-ui/src/app/App.tsx | 6 + workbench-ui/src/app/Shell.test.tsx | 5 +- .../apple-uma/AppleUmaReportRoute.test.tsx | 174 ++++++++++ .../src/app/apple-uma/AppleUmaReportRoute.tsx | 321 ++++++++++++++++++ .../src/app/routeConformance.test.tsx | 15 + workbench-ui/src/app/routes.ts | 13 + .../CommandPalette.keyboard.test.tsx | 8 +- 8 files changed, 540 insertions(+), 7 deletions(-) create mode 100644 workbench-ui/src/app/apple-uma/AppleUmaReportRoute.test.tsx create mode 100644 workbench-ui/src/app/apple-uma/AppleUmaReportRoute.tsx diff --git a/docs/workbench/UI-UX-GUIDE.md b/docs/workbench/UI-UX-GUIDE.md index 563d46c5..ea803073 100644 --- a/docs/workbench/UI-UX-GUIDE.md +++ b/docs/workbench/UI-UX-GUIDE.md @@ -60,7 +60,7 @@ successful proof. ## 4. Current Route Map The route registry in `workbench-ui/src/app/routes.ts` is the source of truth. -Current route count: 16. +Current route count: 17. | Section | Route | Path | Shortcut | Purpose | |---|---|---|---|---| @@ -79,6 +79,7 @@ Current route count: 16. | Discipline | Calibration | `/calibration` | Palette | Inspect practice-class reliability and license verdicts. | | Substrate | Packs | `/packs` | `⌘7` | Browse language/runtime pack metadata. | | Substrate | CORE-Logos | `/logos` | Palette | Inspect CORE-Logos pack identity and safety. | +| Substrate | Apple UMA | `/apple-uma` | Palette | Inspect the Apple Silicon mechanical-sympathy benchmark report. | | Settings | Settings | `/settings` | `⌘0` | Manage local UI preferences; engine config remains CLI-only. | Pinned route shortcuts cover Chat through Settings. All routes are searchable in @@ -101,6 +102,7 @@ the command palette. | Evals | Allowlisted eval lanes and wrong/correct/refused metrics are visible. | | Calibration | Practice classes show engine-owned Wilson floor and PROPOSE/SERVE license verdicts. | | Packs | Pack manifests, checksums, and determinism metadata are visible. | +| Apple UMA | Committed mechanical-sympathy benchmark reports, track execution/parity status, and copy boundaries are visible. | | Settings | UI preferences are local-only; density mode is consumed by shell/design tokens; runtime status is read-only. | ## 6. What Each Route Does Not Prove @@ -119,6 +121,7 @@ the command palette. | Evals | Does not run unsafe or sealed holdout lanes from the UI. | | Calibration | Does not mutate a license and does not claim serving wrong=0 from practice data. | | Packs | Does not apply pack mutation. | +| Apple UMA | Does not execute benchmarks, mutate reports, or authorize serving. | | Settings | Does not edit engine configuration. | ## 7. Evidence Subjects And Address Grammar diff --git a/workbench-ui/src/app/App.tsx b/workbench-ui/src/app/App.tsx index 48b8f6e8..2fa443ed 100644 --- a/workbench-ui/src/app/App.tsx +++ b/workbench-ui/src/app/App.tsx @@ -62,6 +62,11 @@ const PacksRoute = lazy(() => const LogosRoute = lazy(() => import("./logos/LogosRoute").then((module) => ({ default: module.LogosRoute })), ); +const AppleUmaReportRoute = lazy(() => + import("./apple-uma/AppleUmaReportRoute").then((module) => ({ + default: module.AppleUmaReportRoute, + })), +); const SettingsRoute = lazy(() => import("./settings/SettingsRoute").then((module) => ({ default: module.SettingsRoute, @@ -99,6 +104,7 @@ export const ROUTE_ELEMENTS: RouteElementMap = { calibration: lazyRoute(), packs: lazyRoute(), logos: lazyRoute(), + "apple-uma": lazyRoute(), settings: lazyRoute(), }; diff --git a/workbench-ui/src/app/Shell.test.tsx b/workbench-ui/src/app/Shell.test.tsx index 16552cdb..45091600 100644 --- a/workbench-ui/src/app/Shell.test.tsx +++ b/workbench-ui/src/app/Shell.test.tsx @@ -85,11 +85,11 @@ describe("Shell", () => { expect(document.querySelector('[data-density="compact"]')).toBeInTheDocument(); }); - it("LeftNav has exactly 16 items in section-grouped order", () => { + it("LeftNav has exactly 17 items in section-grouped order", () => { renderShell(); const nav = document.querySelector('[data-region="leftnav"]')!; const links = nav.querySelectorAll("a"); - expect(links).toHaveLength(16); + expect(links).toHaveLength(17); const labels = Array.from(links).map((l) => l.textContent); // Grouped by section (Converse → Cognition → Determinism → Evidence → // Discipline → Substrate → Settings), derived from the route registry. @@ -109,6 +109,7 @@ describe("Shell", () => { "Calibration", "Packs", "CORE-Logos", + "Apple UMA", "Settings", ]); }); diff --git a/workbench-ui/src/app/apple-uma/AppleUmaReportRoute.test.tsx b/workbench-ui/src/app/apple-uma/AppleUmaReportRoute.test.tsx new file mode 100644 index 00000000..4f7a4ea3 --- /dev/null +++ b/workbench-ui/src/app/apple-uma/AppleUmaReportRoute.test.tsx @@ -0,0 +1,174 @@ +import { render, screen } from "@testing-library/react"; +import { QueryClientProvider } from "@tanstack/react-query"; +import { MemoryRouter } from "react-router-dom"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createTestQueryClient } from "../../test/createTestQueryClient"; +import { + AppleUmaReportRoute, + APPLE_UMA_LOADING, + APPLE_UMA_ABSENCE_STATEMENT, + APPLE_UMA_ABSENCE_ACTION, +} from "./AppleUmaReportRoute"; + +const GENERATED_AT = "2026-06-14T00:00:00Z"; + +function stubFetch(data: unknown) { + vi.stubGlobal( + "fetch", + vi.fn(() => + Promise.resolve({ + json: async () => ({ ok: true, generated_at: GENERATED_AT, data }), + }), + ), + ); +} + +function stubFetchPending() { + vi.stubGlobal( + "fetch", + vi.fn(() => new Promise(() => {})), + ); +} + +function renderRoute() { + return render( + + + + + , + ); +} + +const STALE_REPORT = { + read_only: true, + report_id: "apple_uma_mechanical_sympathy_latest", + source_path: "reports/apple_uma.json", + source_digest: "sha256:1234567890abcdef", + benchmark_name: "Apple Silicon UMA Mechanical Sympathy", + benchmark_version: "1.0.0", + metadata: {}, + backend_status: { native_status: false, using_rust: true }, + tracks: { + available: ["rust"], + required: ["mlx_exact_cga_recall"], + missing_required: ["mlx_exact_cga_recall"], + mlx_exact_cga_recall: { + present: false, + skipped: true, + reason: "No MLX semantic-backend claim.", + serving_authorized: false, + case_count: 0, + all_cases_parity_pass: false, + cases: [], + }, + }, + copy_boundaries: [], + non_claims: ["No MLX semantic-backend claim."], + claim_safety: { + safe_claims: [], + rust_backend_notes: [], + known_copy_paths: [], + known_zero_copy_input_paths: [], + future_work: [], + }, +}; + +const MLX_PRESENT_REPORT = { + read_only: true, + report_id: "apple_uma_mechanical_sympathy_latest", + source_path: "reports/apple_uma.json", + source_digest: "sha256:1234567890abcdef", + benchmark_name: "Apple Silicon UMA Mechanical Sympathy", + benchmark_version: "1.0.0", + metadata: {}, + backend_status: { native_status: true, using_rust: true }, + tracks: { + available: ["rust", "mlx_exact_cga_recall"], + required: ["mlx_exact_cga_recall"], + missing_required: [], + mlx_exact_cga_recall: { + present: true, + skipped: false, + reason: null, + serving_authorized: true, + case_count: 1, + all_cases_parity_pass: true, + cases: [ + { + N: 128, + top_k: 5, + p50_ms: 0.977, + p95_ms: 1.2, + mean_ms: 1.0, + rows_per_sec: 1024, + parity: { parity_pass: true }, + copy_in_boundary: "copy-in zero borrow", + copy_out_boundary: "copy-out zero borrow", + }, + ], + }, + }, + copy_boundaries: [ + { + path: "test-path", + input: true, + output: true, + zero_copy_input: true, + }, + ], + non_claims: ["No MLX semantic-backend claim."], + claim_safety: { + safe_claims: [], + rust_backend_notes: [], + known_copy_paths: [], + known_zero_copy_input_paths: [], + future_work: [], + }, +}; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("AppleUmaReportRoute", () => { + it("loading state shows APPLE_UMA_LOADING and never 'Thinking'", async () => { + stubFetchPending(); + renderRoute(); + expect(await screen.findByText(APPLE_UMA_LOADING)).toBeInTheDocument(); + expect(screen.queryByText(/thinking/i)).not.toBeInTheDocument(); + }); + + it("malformed/empty API shape renders APPLE_UMA_ABSENCE_STATEMENT and APPLE_UMA_ABSENCE_ACTION", async () => { + stubFetch({}); + renderRoute(); + expect(await screen.findByText(APPLE_UMA_ABSENCE_STATEMENT)).toBeInTheDocument(); + expect(screen.getByText(APPLE_UMA_ABSENCE_ACTION)).toBeInTheDocument(); + }); + + it("stale committed-report shape renders expected labels", async () => { + stubFetch(STALE_REPORT); + renderRoute(); + + expect(await screen.findByText("Report artifact needs refresh")).toBeInTheDocument(); + expect(screen.getByText(/No MLX success is being inferred/)).toBeInTheDocument(); + expect(screen.getByText("track absent")).toBeInTheDocument(); + expect(screen.getByText("parity false")).toBeInTheDocument(); + expect(screen.getByText("serving authorized false")).toBeInTheDocument(); + expect(screen.getAllByText("No MLX semantic-backend claim.").length).toBeGreaterThan(0); + }); + + it("MLX-present shape renders expected metrics and copy boundaries", async () => { + stubFetch(MLX_PRESENT_REPORT); + renderRoute(); + + expect(await screen.findByText("track present")).toBeInTheDocument(); + expect(screen.getByText("executed")).toBeInTheDocument(); + expect(screen.getByText("parity true")).toBeInTheDocument(); + expect(screen.getByText("serving authorized true")).toBeInTheDocument(); + expect(screen.getByText("128")).toBeInTheDocument(); + expect(screen.getByText("0.977")).toBeInTheDocument(); + expect(screen.getByText("copy-in zero borrow")).toBeInTheDocument(); + expect(screen.getByText("copy-out zero borrow")).toBeInTheDocument(); + }); +}); diff --git a/workbench-ui/src/app/apple-uma/AppleUmaReportRoute.tsx b/workbench-ui/src/app/apple-uma/AppleUmaReportRoute.tsx new file mode 100644 index 00000000..4559afd7 --- /dev/null +++ b/workbench-ui/src/app/apple-uma/AppleUmaReportRoute.tsx @@ -0,0 +1,321 @@ +import { useQuery } from "@tanstack/react-query"; +import { apiFetch, WorkbenchApiError } from "../../api/client"; +import { EmptyState } from "../../design/components/states/EmptyState"; +import { ErrorState } from "../../design/components/states/ErrorState"; +import { LoadingState } from "../../design/components/states/LoadingState"; +import { Panel } from "../../design/components/Panel/Panel"; +import { type ReactNode } from "react"; + +export const APPLE_UMA_LOADING = "Loading Apple UMA report..."; +export const APPLE_UMA_ABSENCE_STATEMENT = "No Apple UMA report projection available."; +export const APPLE_UMA_ABSENCE_ACTION = + "CORE_BACKEND=rust uv run python -m benchmarks.apple_uma_mechanical_sympathy --write-report"; + +type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue }; + +interface AppleUmaMlxCase { + N: number | null; + top_k: number | null; + p50_ms: number | null; + p95_ms: number | null; + mean_ms: number | null; + rows_per_sec: number | null; + parity: Record; + copy_in_boundary: string | null; + copy_out_boundary: string | null; +} + +interface AppleUmaMlxTrack { + present: boolean; + skipped: boolean; + reason: string | null; + benchmark_only?: boolean; + serving_authorized?: boolean; + semantic_backend?: string | null; + score_computation?: string | null; + top_k_ordering?: string | null; + copy_boundary?: Record | null; + mlx_status?: Record; + case_count: number; + all_cases_parity_pass: boolean; + cases: AppleUmaMlxCase[]; +} + +interface AppleUmaReport { + read_only: boolean; + report_id: string; + source_path: string; + source_digest: string; + benchmark_name: string; + benchmark_version: string; + metadata: Record; + backend_status: Record; + tracks: { + available: string[]; + required: string[]; + missing_required: string[]; + mlx_exact_cga_recall: AppleUmaMlxTrack; + }; + copy_boundaries: Array>; + non_claims: string[]; + claim_safety: { + safe_claims: string[]; + rust_backend_notes: string[]; + known_copy_paths: string[]; + known_zero_copy_input_paths: string[]; + future_work: string[]; + }; +} + +function useAppleUmaReport() { + return useQuery({ + queryKey: ["api", "benchmarks", "apple-uma", "report"], + queryFn: () => apiFetch("/benchmarks/apple-uma/report"), + retry: false, + staleTime: 30_000, + refetchOnWindowFocus: false, + }); +} + +function isAppleUmaReport(value: unknown): value is AppleUmaReport { + if (!value || typeof value !== "object") return false; + const candidate = value as Partial; + return ( + candidate.read_only === true && + candidate.report_id === "apple_uma_mechanical_sympathy_latest" && + typeof candidate.benchmark_name === "string" && + typeof candidate.benchmark_version === "string" && + !!candidate.tracks && + typeof candidate.tracks === "object" + ); +} + +function boolLabel(value: unknown): string { + if (value === true) return "true"; + if (value === false) return "false"; + if (value === null || value === undefined || value === "") return "not declared"; + return String(value); +} + +function numericLabel(value: number | null | undefined): string { + if (typeof value !== "number" || !Number.isFinite(value)) return "n/a"; + return value.toLocaleString(undefined, { maximumFractionDigits: 3 }); +} + +function statusTone(kind: "good" | "warn" | "neutral") { + if (kind === "good") { + return "border-[var(--color-state-success-border)] bg-[var(--color-state-success-bg)] text-[var(--color-state-success-text)]"; + } + if (kind === "warn") { + return "border-[var(--color-state-warning-border)] bg-[var(--color-state-warning-bg)] text-[var(--color-state-warning-text)]"; + } + return "border-[var(--color-border-subtle)] bg-[var(--color-surface-inset)] text-[var(--color-text-secondary)]"; +} + +function StatusPill({ children, kind = "neutral" }: { children: ReactNode; kind?: "good" | "warn" | "neutral" }) { + return {children}; +} + +function MetricCard({ label, value, detail }: { label: string; value: string; detail?: string }) { + return ( +
+
{label}
+
{value}
+ {detail ?
{detail}
: null} +
+ ); +} + +function StaleReportNotice({ report }: { report: AppleUmaReport }) { + const missing = report.tracks.missing_required.join(", "); + return ( +
+

Report artifact needs refresh

+

+ The Workbench read-model is functioning, but the committed report is stale and lacks: {missing || "no required tracks"}. + No MLX success is being inferred from absent data. +

+ + {APPLE_UMA_ABSENCE_ACTION} + +
+ ); +} + +function BackendPanel({ report }: { report: AppleUmaReport }) { + const backend = report.backend_status; + return ( + +
+ + + + +
+ + ); +} + +function TrackInventoryPanel({ report }: { report: AppleUmaReport }) { + return ( + +
+
+

Available tracks

+
+ {report.tracks.available.map((track) => ( + {track} + ))} +
+
+
+

Missing required tracks

+
+ {report.tracks.missing_required.length === 0 ? ( + none + ) : ( + report.tracks.missing_required.map((track) => ( + {track} + )) + )} +
+
+
+ + ); +} + +function MlxPanel({ report }: { report: AppleUmaReport }) { + const mlx = report.tracks.mlx_exact_cga_recall; + const parityKind = mlx.all_cases_parity_pass ? "good" : "warn"; + return ( + +
+
+ {mlx.present ? "track present" : "track absent"} + {mlx.skipped ? "skipped" : "executed"} + parity {boolLabel(mlx.all_cases_parity_pass)} + serving authorized {boolLabel(mlx.serving_authorized)} +
+ {mlx.reason ?

{mlx.reason}

: null} +
+ + + +
+ {mlx.cases.length > 0 ? ( +
+ + + + + + + + + + + + {mlx.cases.map((entry) => ( + + + + + + + + ))} + +
Np50 msrows/secparitycopy boundary
{entry.N ?? "n/a"}{numericLabel(entry.p50_ms)}{numericLabel(entry.rows_per_sec)}{boolLabel(entry.parity.parity_pass)} +
{entry.copy_in_boundary ?? "copy-in not declared"}
+
{entry.copy_out_boundary ?? "copy-out not declared"}
+
+
+ ) : null} +
+ + ); +} + +function CopyBoundaryPanel({ report }: { report: AppleUmaReport }) { + return ( + +
+ {report.copy_boundaries.map((row, index) => ( +
+
{boolLabel(row.path)}
+
+ input: {boolLabel(row.input)} + output: {boolLabel(row.output)} + zero-copy input: {boolLabel(row.zero_copy_input)} +
+
+ ))} +
+ + ); +} + +function NonClaimsPanel({ report }: { report: AppleUmaReport }) { + return ( + +
    + {report.non_claims.map((claim) => ( +
  • {claim}
    • + ))} +
+ + ); +} + +export function AppleUmaReportRoute() { + const { data, isLoading, isError, error } = useAppleUmaReport(); + + if (isLoading) return ; + + if (isError) { + return ( + + ); + } + + if (!isAppleUmaReport(data)) { + return ( + + ); + } + + const stale = data.tracks.missing_required.includes("mlx_exact_cga_recall"); + + return ( +
+
+
+
+

Apple Silicon / UMA

+

{data.benchmark_name}

+

+ Read-only evidence surface for Python, Rust, and MLX benchmark tracks. This view never runs benchmarks and never upgrades absent evidence into success. +

+
+ read-only {boolLabel(data.read_only)} +
+
+ + {stale ? : null} + + + + + +
+ ); +} diff --git a/workbench-ui/src/app/routeConformance.test.tsx b/workbench-ui/src/app/routeConformance.test.tsx index a6839912..67ab4517 100644 --- a/workbench-ui/src/app/routeConformance.test.tsx +++ b/workbench-ui/src/app/routeConformance.test.tsx @@ -26,6 +26,12 @@ import { LogosRoute } from "./logos/LogosRoute"; import { VaultRoute, VAULT_ABSENCE_STATEMENT, VAULT_ABSENCE_ACTION } from "./vault/VaultRoute"; import { CalibrationRoute } from "./calibration/CalibrationRoute"; import { SettingsRoute } from "./settings/SettingsRoute"; +import { + AppleUmaReportRoute, + APPLE_UMA_LOADING, + APPLE_UMA_ABSENCE_STATEMENT, + APPLE_UMA_ABSENCE_ACTION, +} from "./apple-uma/AppleUmaReportRoute"; /** * ADR-0162 §6 route conformance — executable, not aspirational. @@ -243,6 +249,15 @@ const MOUNT_ROUTES: MountRouteSpec[] = [ "No calibration evidence yet. The per-class arena ledger is populated by the sealed practice lane (ADR-0175).", emptyCommand: "core eval math-contemplation", }, + { + name: "Apple UMA", + element: , + path: "/apple-uma", + initialEntry: "/apple-uma", + loadingLabel: APPLE_UMA_LOADING, + emptyStatement: APPLE_UMA_ABSENCE_STATEMENT, + emptyCommand: APPLE_UMA_ABSENCE_ACTION, + }, ]; describe.each(MOUNT_ROUTES)("route conformance: $name", (spec) => { diff --git a/workbench-ui/src/app/routes.ts b/workbench-ui/src/app/routes.ts index 4c72f098..1b78f6b6 100644 --- a/workbench-ui/src/app/routes.ts +++ b/workbench-ui/src/app/routes.ts @@ -261,6 +261,19 @@ export const WORKBENCH_ROUTES: readonly WorkbenchRoute[] = [ keyboardDigit: null, routeConformanceRequired: true, }, + { + id: "apple-uma", + path: "/apple-uma", + routePattern: "apple-uma", + label: "Apple UMA", + description: "Inspect the Apple Silicon mechanical-sympathy benchmark report.", + section: "Substrate", + leftNavVisible: true, + commandPaletteVisible: true, + landingRouteAllowed: true, + keyboardDigit: null, + routeConformanceRequired: true, + }, { id: "settings", path: "/settings", diff --git a/workbench-ui/src/design/components/primitives/CommandPalette.keyboard.test.tsx b/workbench-ui/src/design/components/primitives/CommandPalette.keyboard.test.tsx index f45d43e8..76ebe58a 100644 --- a/workbench-ui/src/design/components/primitives/CommandPalette.keyboard.test.tsx +++ b/workbench-ui/src/design/components/primitives/CommandPalette.keyboard.test.tsx @@ -52,11 +52,11 @@ describe("CommandPalette keyboard contract", () => { expect(screen.getByRole("button", { name: "Open Trace" })).toBeInTheDocument(); expect(screen.getByRole("button", { name: "Open Replay" })).toBeInTheDocument(); - // One Navigate command per palette-visible route (16), derived from the - // route registry — Demos, Calibration, Contemplation, Tour, CORE-Logos, and - // Lived Life are included (the prior hand-maintained list of 10 dropped them). + // One Navigate command per palette-visible route (17), derived from the + // route registry — Demos, Calibration, Contemplation, Tour, CORE-Logos, Lived + // Life, and Apple UMA are included (the prior hand-maintained list of 10 dropped them). const items = dialog.querySelectorAll('[role="option"]'); - expect(items.length).toBe(16); + expect(items.length).toBe(17); const lastIndex = items.length - 1; // Initially first item (index 0) is focused — check aria-selected