From 138f95a2b09fee4785c1bd50413b8f96d870dee3 Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 13 Jun 2026 01:00:16 -0700 Subject: [PATCH] =?UTF-8?q?feat(workbench):=20Calibration=20route=20?= =?UTF-8?q?=E2=80=94=20the=20gold-tether=20arena=20made=20visible=20(Wave?= =?UTF-8?q?=20M=20B2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consumes the B1 readers (#724). The route where you SEE 'the engine earns the right to guess' (ADR-0175). - list (failures-first, server-ordered): per class a reliability bar (Wilson floor vs the PROPOSE/SERVE θ markers — a class earns the right when its fill crosses the marker), correct/refused/wrong counts (wrong in the contradicted token), and a verdict pill (earned SERVE / earned PROPOSE / not yet licensed) - serving strip: the discipline's RESULT — live correct/refused/wrong per lane, wrong=0 in the verified token, with source DigestBadges - detail (Panel + TabBar): Counts / License math / Raw. The License-math tab shows the honest derivation (measured ≥ θ → licensed) and states the numbers come from core.reliability_gate, not the workbench - fail-closed: an empty/absent arena ledger (501 evidence_unavailable) renders the honest absence card pointing at the practice lane, not an error - nav entry (12th) + routeConformance row + Shell nav assertion updated Token-only (hexScan green); VirtualizedList + j/k; full vitest 398 green. --- workbench-ui/src/api/client.ts | 12 + workbench-ui/src/api/queries.ts | 24 ++ workbench-ui/src/app/App.tsx | 2 + workbench-ui/src/app/LeftNav.tsx | 1 + workbench-ui/src/app/Shell.test.tsx | 5 +- .../app/calibration/CalibrationRoute.test.tsx | 161 +++++++++ .../src/app/calibration/CalibrationRoute.tsx | 315 ++++++++++++++++++ .../src/app/routeConformance.test.tsx | 12 + 8 files changed, 530 insertions(+), 2 deletions(-) create mode 100644 workbench-ui/src/app/calibration/CalibrationRoute.test.tsx create mode 100644 workbench-ui/src/app/calibration/CalibrationRoute.tsx diff --git a/workbench-ui/src/api/client.ts b/workbench-ui/src/api/client.ts index 469fcb8f..00c562dc 100644 --- a/workbench-ui/src/api/client.ts +++ b/workbench-ui/src/api/client.ts @@ -19,6 +19,8 @@ import type { PackDetail, VaultSummary, VaultEntry, + CalibrationClass, + ServingMetrics, TurnJournalEntry, TurnJournalSummary, MathProposalSummary, @@ -260,3 +262,13 @@ export async function fetchVaultEntries(limit?: number, offset?: number): Promis ); return envelope.items; } + +export async function fetchCalibrationClasses(): Promise { + const envelope = await apiFetch>("/calibration/classes"); + return envelope.items; +} + +export async function fetchServingMetrics(): Promise { + const envelope = await apiFetch>("/serving/metrics"); + return envelope.items; +} diff --git a/workbench-ui/src/api/queries.ts b/workbench-ui/src/api/queries.ts index e6ebb42c..c3a32700 100644 --- a/workbench-ui/src/api/queries.ts +++ b/workbench-ui/src/api/queries.ts @@ -22,6 +22,8 @@ import { fetchPack, fetchVaultSummary, fetchVaultEntries, + fetchCalibrationClasses, + fetchServingMetrics, fetchTraceTurn, fetchTraceTurns, fetchMathProposals, @@ -47,6 +49,8 @@ import type { PackDetail, VaultSummary, VaultEntry, + CalibrationClass, + ServingMetrics, TurnJournalEntry, TurnJournalSummary, EvalLaneSummary, @@ -345,3 +349,23 @@ export function useVaultEntries(enabled: boolean, limit?: number, offset?: numbe refetchOnWindowFocus: false, }); } + +export function useCalibrationClasses() { + return useQuery({ + queryKey: ["api", "calibration", "classes"], + queryFn: () => fetchCalibrationClasses(), + retry: false, + staleTime: 30_000, + refetchOnWindowFocus: false, + }); +} + +export function useServingMetrics() { + return useQuery({ + queryKey: ["api", "serving", "metrics"], + queryFn: () => fetchServingMetrics(), + retry: false, + staleTime: 30_000, + refetchOnWindowFocus: false, + }); +} diff --git a/workbench-ui/src/app/App.tsx b/workbench-ui/src/app/App.tsx index a1d31f8b..77ef23cf 100644 --- a/workbench-ui/src/app/App.tsx +++ b/workbench-ui/src/app/App.tsx @@ -13,6 +13,7 @@ import { EvalsRoute } from "./evals/EvalsRoute"; import { RunsRoute } from "./runs/RunsRoute"; import { PacksRoute } from "./packs/PacksRoute"; import { VaultRoute } from "./vault/VaultRoute"; +import { CalibrationRoute } from "./calibration/CalibrationRoute"; import { SettingsRoute } from "./settings/SettingsRoute"; import { getWorkbenchPrefs } from "./workbenchPrefs"; @@ -32,6 +33,7 @@ export function App() { } /> } /> } /> + } /> } /> } /> diff --git a/workbench-ui/src/app/LeftNav.tsx b/workbench-ui/src/app/LeftNav.tsx index 57a46b52..3baa8c94 100644 --- a/workbench-ui/src/app/LeftNav.tsx +++ b/workbench-ui/src/app/LeftNav.tsx @@ -7,6 +7,7 @@ const NAV_ITEMS = [ { label: "Demos", to: "/demos" }, { label: "Proposals", to: "/proposals" }, { label: "Evals", to: "/evals" }, + { label: "Calibration", to: "/calibration" }, { label: "Runs", to: "/runs" }, { label: "Packs", to: "/packs" }, { label: "Vault", to: "/vault" }, diff --git a/workbench-ui/src/app/Shell.test.tsx b/workbench-ui/src/app/Shell.test.tsx index 2510dcf3..8b29718e 100644 --- a/workbench-ui/src/app/Shell.test.tsx +++ b/workbench-ui/src/app/Shell.test.tsx @@ -76,11 +76,11 @@ describe("Shell", () => { expect(document.querySelector('[data-region="statusfooter"]')).toBeInTheDocument(); }); - it("LeftNav has exactly 11 items in order", () => { + it("LeftNav has exactly 12 items in order", () => { renderShell(); const nav = document.querySelector('[data-region="leftnav"]')!; const links = nav.querySelectorAll("a"); - expect(links).toHaveLength(11); + expect(links).toHaveLength(12); const labels = Array.from(links).map((l) => l.textContent); expect(labels).toEqual([ "Chat", @@ -89,6 +89,7 @@ describe("Shell", () => { "Demos", "Proposals", "Evals", + "Calibration", "Runs", "Packs", "Vault", diff --git a/workbench-ui/src/app/calibration/CalibrationRoute.test.tsx b/workbench-ui/src/app/calibration/CalibrationRoute.test.tsx new file mode 100644 index 00000000..7ff864a9 --- /dev/null +++ b/workbench-ui/src/app/calibration/CalibrationRoute.test.tsx @@ -0,0 +1,161 @@ +import { QueryClientProvider } from "@tanstack/react-query"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { MemoryRouter } from "react-router-dom"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createTestQueryClient } from "../../test/createTestQueryClient"; +import type { CalibrationClass, ServingMetrics } from "../../types/api"; +import { CalibrationRoute } from "./CalibrationRoute"; + +const classes: CalibrationClass[] = [ + { + class_name: "unearned", + correct: 0, + wrong: 0, + refused: 9, + committed: 0, + reliability_floor: 0.0, + coverage: 0.0, + propose_required: 0.85, + propose_licensed: false, + serve_required: 0.99, + serve_licensed: false, + }, + { + class_name: "additive", + correct: 95, + wrong: 5, + refused: 50, + committed: 100, + reliability_floor: 0.860842, + coverage: 0.66, + propose_required: 0.85, + propose_licensed: true, + serve_required: 0.99, + serve_licensed: false, + }, +]; + +const metrics: ServingMetrics[] = [ + { + lane: "train_sample", + correct: 4, + refused: 46, + wrong: 0, + sample_count: 50, + source_path: "evals/gsm8k_math/train_sample/v1/report.json", + source_digest: "sha256:aaaaaaaaaaaaaaaa", + }, +]; + +function renderRoute() { + return render( + + + + + , + ); +} + +function okBody(data: unknown) { + return { ok: true, generated_at: "now", data }; +} + +function stub({ classesAvailable = true }: { classesAvailable?: boolean } = {}) { + vi.stubGlobal( + "fetch", + vi.fn((input: unknown) => { + const path = new URL(String(input)).pathname; + if (path === "/calibration/classes") { + return classesAvailable + ? Promise.resolve({ json: () => Promise.resolve(okBody({ items: classes })) }) + : Promise.resolve({ + json: () => + Promise.resolve({ + ok: false, + generated_at: "now", + error: { code: "evidence_unavailable", message: "no practice report" }, + }), + }); + } + if (path === "/serving/metrics") { + return Promise.resolve({ json: () => Promise.resolve(okBody({ items: metrics })) }); + } + return Promise.resolve({ + json: () => + Promise.resolve({ ok: false, generated_at: "now", error: { code: "not_found", message: path } }), + }); + }), + ); +} + +const offsetDescriptors = { + offsetHeight: Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetHeight"), + offsetWidth: Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetWidth"), +}; + +describe("CalibrationRoute", () => { + beforeEach(() => { + Object.defineProperty(HTMLElement.prototype, "offsetHeight", { configurable: true, get: () => 560 }); + Object.defineProperty(HTMLElement.prototype, "offsetWidth", { configurable: true, get: () => 520 }); + }); + + afterEach(() => { + if (offsetDescriptors.offsetHeight) { + Object.defineProperty(HTMLElement.prototype, "offsetHeight", offsetDescriptors.offsetHeight); + } + if (offsetDescriptors.offsetWidth) { + Object.defineProperty(HTMLElement.prototype, "offsetWidth", offsetDescriptors.offsetWidth); + } + vi.restoreAllMocks(); + localStorage.clear(); + }); + + it("renders the gold-tether classes with their earned verdicts", async () => { + stub(); + renderRoute(); + expect(await screen.findByText("additive")).toBeInTheDocument(); + expect(screen.getByText("unearned")).toBeInTheDocument(); + expect(screen.getByText("earned PROPOSE")).toBeInTheDocument(); + expect(screen.getByText("not yet licensed")).toBeInTheDocument(); + }); + + it("surfaces the live serving outcome with wrong=0", async () => { + stub(); + renderRoute(); + expect(await screen.findByText("train_sample")).toBeInTheDocument(); + // the discipline's result is present + expect(screen.getByText("Live serving outcome — the discipline's result")).toBeInTheDocument(); + expect(screen.getAllByText("0 wrong").length).toBeGreaterThan(0); + }); + + it("shows the honest license math (measured vs θ) for a selected class", async () => { + stub(); + const user = userEvent.setup(); + renderRoute(); + await user.click(await screen.findByText("additive")); + await user.click(screen.getByRole("tab", { name: "License math" })); + // PROPOSE licensed (>= 0.85), SERVE not (< 0.99) + expect(screen.getAllByText(/licensed/).length).toBeGreaterThan(0); + expect(screen.getByText(/core\.reliability_gate/)).toBeInTheDocument(); + }); + + it("fail-closed: an empty arena ledger renders the honest absence card, not an error", async () => { + stub({ classesAvailable: false }); + renderRoute(); + expect(await screen.findByText(/No calibration evidence yet\./)).toBeInTheDocument(); + expect(screen.queryByText("What failed")).not.toBeInTheDocument(); + }); + + it("moves the class list focus with j/k through the VirtualizedList spine", async () => { + stub(); + const user = userEvent.setup(); + renderRoute(); + const list = await screen.findByRole("listbox", { name: "Calibration classes" }); + list.focus(); + expect(screen.getAllByRole("option")[0]).toHaveAttribute("aria-selected", "true"); + await user.keyboard("j"); + expect(screen.getAllByRole("option")[1]).toHaveAttribute("aria-selected", "true"); + }); +}); diff --git a/workbench-ui/src/app/calibration/CalibrationRoute.tsx b/workbench-ui/src/app/calibration/CalibrationRoute.tsx new file mode 100644 index 00000000..73856d31 --- /dev/null +++ b/workbench-ui/src/app/calibration/CalibrationRoute.tsx @@ -0,0 +1,315 @@ +import { useMemo, useState } from "react"; +import { WorkbenchApiError } from "../../api/client"; +import { useCalibrationClasses, useServingMetrics } from "../../api/queries"; +import { DigestBadge } from "../../design/components/DigestBadge/DigestBadge"; +import { MetadataTable } from "../../design/components/MetadataTable/MetadataTable"; +import { Panel } from "../../design/components/Panel/Panel"; +import { SearchInput } from "../../design/components/SearchInput/SearchInput"; +import { SplitPane } from "../../design/components/SplitPane/SplitPane"; +import { StableJsonViewer } from "../../design/components/StableJsonViewer"; +import { TabBar, type Tab } from "../../design/components/TabBar/TabBar"; +import { VirtualizedList } from "../../design/components/VirtualizedList/VirtualizedList"; +import { EmptyState } from "../../design/components/states/EmptyState"; +import { ErrorState } from "../../design/components/states/ErrorState"; +import { LoadingState } from "../../design/components/states/LoadingState"; +import type { CalibrationClass, ServingMetrics } from "../../types/api"; + +const DETAIL_TABS: readonly Tab[] = [ + { id: "counts", label: "Counts" }, + { id: "license", label: "License math" }, + { id: "raw", label: "Raw" }, +]; + +// Sealed practice is opt-in evidence; absence is honest, not an error. +const FAIL_CLOSED_STATEMENT = + "No calibration evidence yet. The per-class arena ledger is populated by the sealed practice lane (ADR-0175)."; +const FAIL_CLOSED_ACTION = "core eval math-contemplation"; + +function errorMessage(error: unknown) { + return error instanceof WorkbenchApiError ? error.message : "Calibration request failed."; +} + +function pct(value: number): string { + return `${(value * 100).toFixed(1)}%`; +} + +function verdict(c: CalibrationClass): { label: string; tone: "serve" | "propose" | "none" } { + if (c.serve_licensed) return { label: "earned SERVE", tone: "serve" }; + if (c.propose_licensed) return { label: "earned PROPOSE", tone: "propose" }; + return { label: "not yet licensed", tone: "none" }; +} + +function VerdictPill({ c }: { c: CalibrationClass }) { + const v = verdict(c); + const cls = + v.tone === "serve" + ? "border-[var(--color-state-verified)] text-[var(--color-state-verified)]" + : v.tone === "propose" + ? "border-[var(--color-state-warning-text)] text-[var(--color-state-warning-text)]" + : "border-[var(--color-border-subtle)] text-[var(--color-text-muted)]"; + return ( + + {v.label} + + ); +} + +// reliability_floor as a fill, with the two θ thresholds marked. A class +// "earns the right to guess" when its fill crosses the marker. +function ReliabilityBar({ c }: { c: CalibrationClass }) { + return ( +
+
+
+
+
+ ); +} + +function ClassRow({ + c, + selected, + focused, + onSelect, +}: { + c: CalibrationClass; + selected: boolean; + focused: boolean; + onSelect: () => void; +}) { + return ( +
+ + {c.class_name} + + + + + + reliability {pct(c.reliability_floor)} · committed {c.committed} + + {c.correct} correct + {c.refused} refused + 0 + ? "text-[var(--color-state-contradicted)]" + : "text-[var(--color-text-muted)]" + } + > + {c.wrong} wrong + + +
+ ); +} + +function ServingStrip({ metrics }: { metrics: ServingMetrics[] }) { + return ( +
+

+ Live serving outcome — the discipline's result +

+
+ {metrics.map((m) => ( +
+ {m.lane} + {m.correct} correct + {m.refused} refused + 0 + ? "font-semibold text-[var(--color-state-contradicted)]" + : "font-semibold text-[var(--color-state-verified)]" + } + > + {m.wrong} wrong + +
+ ))} +
+
+ ); +} + +function LicenseMath({ c }: { c: CalibrationClass }) { + const row = (action: string, required: number, licensed: boolean) => ({ + key: action, + value: ( + + measured {pct(c.reliability_floor)} {licensed ? "≥" : "<"} θ {pct(required)} →{" "} + + {licensed ? "licensed" : "not licensed"} + + + ), + }); + return ( +
+

+ Reliability is the engine's one-sided Wilson conservative floor on commitment precision + (correct / committed), and is 0 below N_MIN=10 committed trials. A class is licensed for an + action when its measured reliability clears that action's θ. The workbench computes none of + this — it is read from core.reliability_gate. +

+ +
+ ); +} + +function ClassDetail({ c }: { c: CalibrationClass }) { + const [tab, setTab] = useState("counts"); + return ( + }> + + {tab === "counts" ? ( + + ) : null} + {tab === "license" ? : null} + {tab === "raw" ? : null} + + + ); +} + +export function CalibrationRoute() { + const [search, setSearch] = useState(""); + const [selected, setSelected] = useState(null); + + const classesQuery = useCalibrationClasses(); + const metricsQuery = useServingMetrics(); + + const classes = classesQuery.data ?? []; + const filtered = useMemo(() => { + const q = search.trim().toLowerCase(); + if (!q) return classes; + return classes.filter((c) => c.class_name.toLowerCase().includes(q)); + }, [search, classes]); + + const selectedClass = classes.find((c) => c.class_name === selected) ?? null; + + if (classesQuery.isLoading) { + return ; + } + + if (classesQuery.isError) { + if (classesQuery.error.code === "evidence_unavailable") { + return ; + } + return ( + + ); + } + + if (classes.length === 0) { + return ; + } + + return ( +
+ + +
+ {metricsQuery.data && metricsQuery.data.length > 0 ? ( + + ) : null} + + c.class_name} + height="calc(100vh - 20rem)" + initialRect={{ width: 520, height: 560 }} + items={filtered} + onActivate={(c) => setSelected(c.class_name)} + renderItem={(c, _index, focused) => ( + setSelected(c.class_name)} + /> + )} + /> +
+ + +
+ {selectedClass ? ( + + ) : ( + + )} + {metricsQuery.data && metricsQuery.data.length > 0 ? ( +
+ {metricsQuery.data.map((m) => ( + + {m.lane} source + + + ))} +
+ ) : null} +
+ +
+ ); +} diff --git a/workbench-ui/src/app/routeConformance.test.tsx b/workbench-ui/src/app/routeConformance.test.tsx index e6099f85..64e42492 100644 --- a/workbench-ui/src/app/routeConformance.test.tsx +++ b/workbench-ui/src/app/routeConformance.test.tsx @@ -16,6 +16,7 @@ import { DemoTheaterRoute } from "./demos/DemoTheaterRoute"; import { RunsRoute } from "./runs/RunsRoute"; import { PacksRoute } from "./packs/PacksRoute"; import { VaultRoute } from "./vault/VaultRoute"; +import { CalibrationRoute } from "./calibration/CalibrationRoute"; import { SettingsRoute } from "./settings/SettingsRoute"; /** @@ -194,6 +195,17 @@ const MOUNT_ROUTES: MountRouteSpec[] = [ "No persisted vault. Session memory is held in-process and discarded on exit; persistence is opt-in via RuntimeConfig.persist_session_state.", emptyCommand: "Set RuntimeConfig.persist_session_state = true", }, + { + name: "Calibration", + element: , + path: "/calibration", + initialEntry: "/calibration", + loadingLabel: "Loading calibration...", + // Fail-closed: an empty arena ledger is the honest primary state. + emptyStatement: + "No calibration evidence yet. The per-class arena ledger is populated by the sealed practice lane (ADR-0175).", + emptyCommand: "core eval math-contemplation", + }, ]; describe.each(MOUNT_ROUTES)("route conformance: $name", (spec) => {