From 00f50562093bfc3744d615a36d4503f1f34e48be Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 26 May 2026 20:39:30 -0700 Subject: [PATCH] =?UTF-8?q?feat(workbench/W-030):=20eval=20center=20(safe?= =?UTF-8?q?=20lanes=20only,=20ADR-0160=20=C2=A7Phase=205)=20(#327)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- workbench-ui/src/api/client.ts | 34 +- workbench-ui/src/api/queries.ts | 15 +- workbench-ui/src/app/App.tsx | 4 +- .../src/app/evals/EvalArtifactLink.tsx | 32 ++ .../src/app/evals/EvalFailureViewer.tsx | 106 +++++++ workbench-ui/src/app/evals/EvalLaneCard.tsx | 67 ++++ workbench-ui/src/app/evals/EvalMetricGrid.tsx | 98 ++++++ workbench-ui/src/app/evals/EvalRunButton.tsx | 104 ++++++ workbench-ui/src/app/evals/EvalsRoute.tsx | 188 +++++++++++ workbench-ui/src/app/evals/evals.test.tsx | 299 ++++++++++++++++++ .../src/routes/EvalsRoutePlaceholder.tsx | 10 - workbench-ui/src/types/api.ts | 10 +- 12 files changed, 951 insertions(+), 16 deletions(-) create mode 100644 workbench-ui/src/app/evals/EvalArtifactLink.tsx create mode 100644 workbench-ui/src/app/evals/EvalFailureViewer.tsx create mode 100644 workbench-ui/src/app/evals/EvalLaneCard.tsx create mode 100644 workbench-ui/src/app/evals/EvalMetricGrid.tsx create mode 100644 workbench-ui/src/app/evals/EvalRunButton.tsx create mode 100644 workbench-ui/src/app/evals/EvalsRoute.tsx create mode 100644 workbench-ui/src/app/evals/evals.test.tsx delete mode 100644 workbench-ui/src/routes/EvalsRoutePlaceholder.tsx diff --git a/workbench-ui/src/api/client.ts b/workbench-ui/src/api/client.ts index 13972447..1adb73f0 100644 --- a/workbench-ui/src/api/client.ts +++ b/workbench-ui/src/api/client.ts @@ -1,4 +1,4 @@ -import type { ApiResponse, ErrorCode } from "../types/api"; +import type { ApiResponse, ErrorCode, EvalLaneSummary, EvalRunRequest, EvalRunResult } from "../types/api"; export class WorkbenchApiError extends Error { constructor( @@ -27,3 +27,35 @@ export async function apiFetch(path: string, init?: RequestInit): Promise clearTimeout(timeout); } } + +export async function fetchEvalLanes(): Promise { + return apiFetch("/evals"); +} + +export async function runEvalLane(req: EvalRunRequest): Promise { + if (req.split === "holdout") { + const hasConfig = typeof window !== "undefined" && (window as any).sealedEvalConfig === true; + if (!hasConfig) { + throw new WorkbenchApiError( + "client_refused_sealed_holdout", + "Holdout runs require sealed-eval config — use CLI" + ); + } + } + + const lanes = await fetchEvalLanes(); + const lane = lanes.find((l) => l.lane === req.lane); + if (!lane) { + throw new WorkbenchApiError("not_found", `Eval lane not found: ${req.lane}`); + } + if (!lane.read_only) { + throw new WorkbenchApiError("client_refused_unsafe_lane", "API run disabled — use CLI"); + } + + return apiFetch("/evals/run", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(req), + }); +} + diff --git a/workbench-ui/src/api/queries.ts b/workbench-ui/src/api/queries.ts index a648790f..2777357a 100644 --- a/workbench-ui/src/api/queries.ts +++ b/workbench-ui/src/api/queries.ts @@ -4,7 +4,7 @@ import { useMutation, QueryClientProvider, } from "@tanstack/react-query"; -import { apiFetch } from "./client"; +import { apiFetch, fetchEvalLanes, runEvalLane } from "./client"; import type { WorkbenchApiError } from "./client"; import type { RuntimeStatus, @@ -15,6 +15,7 @@ import type { EvalLaneSummary, EvalRunResult, ChatTurnResult, + EvalRunRequest, } from "../types/api"; export { QueryClientProvider }; @@ -70,7 +71,9 @@ export function useProposal(id: string) { export function useEvalLanes() { return useQuery({ queryKey: ["api", "evals"], - queryFn: () => apiFetch("/evals"), + queryFn: fetchEvalLanes, + staleTime: 60_000, + refetchOnWindowFocus: false, }); } @@ -82,6 +85,13 @@ export function useEvalLane(name: string) { }); } +export function useEvalRun() { + return useMutation({ + mutationKey: ["eval-run"], + mutationFn: runEvalLane, + }); +} + export function useChatTurn() { return useMutation({ mutationKey: ["chat-turn"], @@ -93,3 +103,4 @@ export function useChatTurn() { }), }); } + diff --git a/workbench-ui/src/app/App.tsx b/workbench-ui/src/app/App.tsx index d0001324..92eaf828 100644 --- a/workbench-ui/src/app/App.tsx +++ b/workbench-ui/src/app/App.tsx @@ -7,7 +7,7 @@ import { ChatRoute } from "../routes/ChatRoute"; import { TraceRoutePlaceholder } from "../routes/TraceRoutePlaceholder"; import { ReplayRoutePlaceholder } from "../routes/ReplayRoutePlaceholder"; import { ProposalsRoutePlaceholder } from "../routes/ProposalsRoutePlaceholder"; -import { EvalsRoutePlaceholder } from "../routes/EvalsRoutePlaceholder"; +import { EvalsRoute } from "./evals/EvalsRoute"; import { RunsRoutePlaceholder } from "../routes/RunsRoutePlaceholder"; import { PacksRoutePlaceholder } from "../routes/PacksRoutePlaceholder"; import { VaultRoutePlaceholder } from "../routes/VaultRoutePlaceholder"; @@ -25,7 +25,7 @@ export function App() { } /> } /> } /> - } /> + } /> } /> } /> } /> diff --git a/workbench-ui/src/app/evals/EvalArtifactLink.tsx b/workbench-ui/src/app/evals/EvalArtifactLink.tsx new file mode 100644 index 00000000..c2e4f1d9 --- /dev/null +++ b/workbench-ui/src/app/evals/EvalArtifactLink.tsx @@ -0,0 +1,32 @@ +import { InfoBadge } from "../../design/components/badges/Badge"; + +export function EvalArtifactLink({ + lane, + sourceDigest, +}: { + lane: string; + sourceDigest: string; +}) { + const label = sourceDigest.slice(0, 12); + const location = lane.includes("contemplation") + ? "engine_state/" + : `evals/${lane}/results/`; + + return ( +
+ Source Digest: + + + Located at: {location} + +
+ ); +} diff --git a/workbench-ui/src/app/evals/EvalFailureViewer.tsx b/workbench-ui/src/app/evals/EvalFailureViewer.tsx new file mode 100644 index 00000000..ad6344b8 --- /dev/null +++ b/workbench-ui/src/app/evals/EvalFailureViewer.tsx @@ -0,0 +1,106 @@ +import { StableJsonViewer } from "../../design/components/StableJsonViewer/StableJsonViewer"; +import { EmptyState } from "../../design/components/states/EmptyState"; + +interface CaseItem { + case_id?: string; + id?: string; + passed?: boolean; + expected?: unknown; + actual?: unknown; + failure_reason?: string; + failure_reasons?: string[]; + [key: string]: unknown; +} + +function ensureJsonString(val: unknown): string { + if (val === undefined) return ""; + if (typeof val === "string") { + try { + JSON.parse(val); + return val; + } catch { + return JSON.stringify(val); + } + } + return JSON.stringify(val, null, 2); +} + +export function EvalFailureViewer({ + cases, + passed, + laneName, +}: { + cases: any[]; + passed: boolean | null; + laneName: string; +}) { + const typedCases = cases as CaseItem[]; + const failures = typedCases.filter((c) => c.passed === false); + + if (passed === true || failures.length === 0) { + return ( + + ); + } + + return ( +
+

+ Failures ({failures.length}) +

+
+ {failures.map((c, idx) => { + const caseId = c.case_id || c.id || `case-${idx}`; + const reason = c.failure_reason || (Array.isArray(c.failure_reasons) ? c.failure_reasons.join(", ") : "") || "No reason specified"; + + const hasExpectedActual = c.expected !== undefined || c.actual !== undefined; + + const expectedStr = c.expected !== undefined + ? ensureJsonString(c.expected) + : JSON.stringify(c, null, 2); + const actualStr = c.actual !== undefined + ? ensureJsonString(c.actual) + : ""; + + return ( +
+
+ + {caseId} + + + Failed + +
+ +
+
Reason
+
+ {reason} +
+
+ +
+
+ {hasExpectedActual ? "Expected vs Actual" : "Case Details"} +
+ {hasExpectedActual ? ( + + ) : ( + + )} +
+
+ ); + })} +
+
+ ); +} diff --git a/workbench-ui/src/app/evals/EvalLaneCard.tsx b/workbench-ui/src/app/evals/EvalLaneCard.tsx new file mode 100644 index 00000000..234d0ea4 --- /dev/null +++ b/workbench-ui/src/app/evals/EvalLaneCard.tsx @@ -0,0 +1,67 @@ +import type { EvalLaneSummary } from "../../types/api"; + +export function EvalLaneCard({ + lane, + isSelected, + onSelect, +}: { + lane: EvalLaneSummary; + isSelected: boolean; + onSelect: () => void; +}) { + const readOnlyStyle = lane.read_only + ? { + backgroundColor: "color-mix(in srgb, var(--color-state-verified) 18%, transparent)", + borderColor: "var(--color-state-verified)", + color: "var(--color-state-verified)", + } + : { + backgroundColor: "color-mix(in srgb, var(--color-state-undetermined) 18%, transparent)", + borderColor: "var(--color-state-undetermined)", + color: "var(--color-text-muted)", + }; + + return ( + + ); +} diff --git a/workbench-ui/src/app/evals/EvalMetricGrid.tsx b/workbench-ui/src/app/evals/EvalMetricGrid.tsx new file mode 100644 index 00000000..076f34d6 --- /dev/null +++ b/workbench-ui/src/app/evals/EvalMetricGrid.tsx @@ -0,0 +1,98 @@ +import { InfoBadge } from "../../design/components/badges/Badge"; + +export function EvalMetricGrid({ + metrics, +}: { + metrics: Record; +}) { + const sortedKeys = Object.keys(metrics).sort(); + + function formatValue(value: unknown): string { + if (typeof value === "boolean") { + return value ? "true" : "false"; + } + if (typeof value === "number") { + return String(value); + } + if (typeof value === "object" && value !== null) { + return JSON.stringify(value, null, 2); + } + return String(value ?? ""); + } + + function getUnit(key: string): string | undefined { + const k = key.toLowerCase(); + if (k.endsWith("_ms") || k.includes("ms") || k.includes("latency")) { + return "ms"; + } + if (k.endsWith("_rate") || k.endsWith("_pct") || k.includes("percentage")) { + return "%"; + } + if (k.endsWith("_sec") || k.includes("seconds")) { + return "s"; + } + return undefined; + } + + function renderPassFailBadge(key: string, value: unknown) { + const isBool = typeof value === "boolean"; + const k = key.toLowerCase(); + const isPassKey = k.includes("pass") || k.includes("success") || k === "passed"; + + if (isBool && isPassKey) { + if (value === true) { + return ( + + ); + } else { + return ( + + ); + } + } + return null; + } + + return ( +
+ {sortedKeys.map((key) => { + const val = metrics[key]; + const unit = getUnit(key); + const badge = renderPassFailBadge(key, val); + const formatted = formatValue(val); + const isObj = typeof val === "object" && val !== null; + + return ( +
+
+
+ {key.replaceAll("_", " ")} +
+
+ {formatted} + {unit && {unit}} +
+
+ {badge &&
{badge}
} +
+ ); + })} +
+ ); +} diff --git a/workbench-ui/src/app/evals/EvalRunButton.tsx b/workbench-ui/src/app/evals/EvalRunButton.tsx new file mode 100644 index 00000000..0aef547e --- /dev/null +++ b/workbench-ui/src/app/evals/EvalRunButton.tsx @@ -0,0 +1,104 @@ +import { useState, useEffect } from "react"; + import { useEvalRun } from "../../api/queries"; + import { Button } from "../../design/components/primitives/Button"; + import type { EvalLaneSummary, EvalRunResult } from "../../types/api"; + import type { WorkbenchApiError } from "../../api/client"; + + export function EvalRunButton({ + lane, + onRunStart, + onRunSuccess, + onRunError, + }: { + lane: EvalLaneSummary | null; + onRunStart: () => void; + onRunSuccess: (result: EvalRunResult) => void; + onRunError: (error: WorkbenchApiError) => void; + }) { + const evalRun = useEvalRun(); + const [version, setVersion] = useState(""); + const [split, setSplit] = useState<"dev" | "public" | "holdout">("public"); + + // Reset selection when lane changes + useEffect(() => { + if (lane) { + setVersion(lane.versions[0] || "v1"); + setSplit("public"); + } + }, [lane]); + + if (!lane || !lane.read_only) { + return null; + } + + const handleRun = () => { + onRunStart(); + evalRun.mutate( + { lane: lane.lane, version, split }, + { + onSuccess: (data) => { + onRunSuccess(data); + }, + onError: (err) => { + onRunError(err); + }, + } + ); + }; + + const isPending = evalRun.isPending; + + return ( +
+

Run Configuration

+
+
+ + +
+ +
+ + +
+ + +
+
+ ); + } diff --git a/workbench-ui/src/app/evals/EvalsRoute.tsx b/workbench-ui/src/app/evals/EvalsRoute.tsx new file mode 100644 index 00000000..13fc2bca --- /dev/null +++ b/workbench-ui/src/app/evals/EvalsRoute.tsx @@ -0,0 +1,188 @@ +import { useState } from "react"; +import { useSearchParams } from "react-router-dom"; +import { useEvalLanes } from "../../api/queries"; +import { EvalLaneCard } from "./EvalLaneCard"; +import { EvalRunButton } from "./EvalRunButton"; +import { EvalMetricGrid } from "./EvalMetricGrid"; +import { EvalFailureViewer } from "./EvalFailureViewer"; +import { EvalArtifactLink } from "./EvalArtifactLink"; +import { EmptyState } from "../../design/components/states/EmptyState"; +import { ErrorState } from "../../design/components/states/ErrorState"; +import { LoadingState } from "../../design/components/states/LoadingState"; +import type { EvalRunResult } from "../../types/api"; +import { WorkbenchApiError } from "../../api/client"; + +export function EvalsRoute() { + const { data: lanes, isLoading, isError, error } = useEvalLanes(); + const [searchParams, setSearchParams] = useSearchParams(); + const selectedLaneName = searchParams.get("lane") || ""; + + // Maintain per-lane run states (pending, result, error) + const [runStates, setRunStates] = useState< + Record< + string, + { + isPending: boolean; + result?: EvalRunResult; + error?: WorkbenchApiError; + } + > + >({}); + + if (isLoading) { + return ; + } + + if (isError) { + return ( + + ); + } + + const selectedLane = lanes?.find((l) => l.lane === selectedLaneName) || null; + const currentRunState = selectedLaneName ? runStates[selectedLaneName] : null; + + return ( +
+ {/* Left Pane: Lane List */} +
+

Eval Lanes

+
+ {lanes && lanes.length > 0 ? ( + lanes.map((lane) => ( + setSearchParams({ lane: lane.lane })} + /> + )) + ) : ( + + )} +
+
+ + {/* Right Pane: Results / Form */} +
+ {selectedLane ? ( + <> + {/* Header info */} +
+
+

+ {selectedLane.lane} +

+ {selectedLane.description && ( +

+ {selectedLane.description} +

+ )} +
+
+ + {/* Run Button configuration if read-only */} + {selectedLane.read_only ? ( + { + setRunStates((prev) => ({ + ...prev, + [selectedLane.lane]: { isPending: true }, + })); + }} + onRunSuccess={(result) => { + setRunStates((prev) => ({ + ...prev, + [selectedLane.lane]: { isPending: false, result }, + })); + }} + onRunError={(err) => { + setRunStates((prev) => ({ + ...prev, + [selectedLane.lane]: { isPending: false, error: err }, + })); + }} + /> + ) : ( +
+ ⚠️ API runs disabled for write-active lane. Use local CLI to execute this lane: + + core eval --lane {selectedLane.lane} + +
+ )} + + {/* Result display */} + {currentRunState?.isPending ? ( + + ) : currentRunState?.error ? ( + + ) : currentRunState?.result ? ( +
+
+ + Status: + + + {currentRunState.result.passed ? "Passed" : "Failed"} + + {currentRunState.result.source_digest && ( + + )} +
+ +
+

Metrics

+ +
+ + +
+ ) : ( + + )} + + ) : ( + + )} +
+
+ ); +} diff --git a/workbench-ui/src/app/evals/evals.test.tsx b/workbench-ui/src/app/evals/evals.test.tsx new file mode 100644 index 00000000..d1b155d0 --- /dev/null +++ b/workbench-ui/src/app/evals/evals.test.tsx @@ -0,0 +1,299 @@ +import { render, screen, fireEvent, waitFor } from "@testing-library/react"; +import { describe, expect, it, vi, beforeEach } from "vitest"; +import { MemoryRouter } from "react-router-dom"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { EvalLaneCard } from "./EvalLaneCard"; +import { EvalRunButton } from "./EvalRunButton"; +import { EvalMetricGrid } from "./EvalMetricGrid"; +import { EvalFailureViewer } from "./EvalFailureViewer"; +import { EvalsRoute } from "./EvalsRoute"; +import { runEvalLane, WorkbenchApiError } from "../../api/client"; +import type { EvalLaneSummary, EvalRunResult } from "../../types/api"; + +// Mock queries +vi.mock("../../api/queries", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + useEvalLanes: vi.fn(), + useEvalRun: vi.fn(), + }; +}); + +import { useEvalLanes, useEvalRun } from "../../api/queries"; + +const mockLanes: EvalLaneSummary[] = [ + { lane: "contemplation_quality", versions: ["v1", "v2"], read_only: true, description: "Contemplation checks" }, + { lane: "unsafe_lane", versions: ["v1"], read_only: false, description: "Unsafe checks" }, +]; + +const mockResult: EvalRunResult = { + lane: "contemplation_quality", + version: "v1", + split: "public", + passed: false, + metrics: { accuracy: 0.8, passed: 4, total: 5 }, + cases: [ + { case_id: "c1", passed: true }, + { case_id: "c2", passed: false, expected: "val1", actual: "val2", failure_reason: "Mismatch value" }, + ], + source_digest: "abcdef1234567890", +}; + +function makeClient() { + return new QueryClient({ defaultOptions: { queries: { retry: false } } }); +} + +describe("W-030 Component Tests", () => { + const fetchMock = vi.fn(); + + beforeEach(() => { + vi.resetAllMocks(); + fetchMock.mockImplementation((url: any) => { + const urlStr = typeof url === "string" ? url : String(url?.url || url || ""); + if (urlStr.endsWith("/evals")) { + return Promise.resolve({ + ok: true, + json: () => Promise.resolve({ ok: true, generated_at: "2026-05-26T00:00:00Z", data: mockLanes }), + }); + } + if (urlStr.endsWith("/evals/run")) { + return Promise.resolve({ + ok: true, + json: () => Promise.resolve({ ok: true, generated_at: "2026-05-26T00:00:00Z", data: mockResult }), + }); + } + return Promise.reject(new Error(`Unhandled URL: ${urlStr}`)); + }); + vi.stubGlobal("fetch", fetchMock); + if (typeof window !== "undefined") { + delete (window as any).sealedEvalConfig; + } + }); + + // 1. EvalLaneCard + describe("EvalLaneCard", () => { + it("renders read_only badge correctly for both values; clicking selects via callback", () => { + const selectSpy = vi.fn(); + + // Test read_only === true + const { rerender } = render( + + ); + expect(screen.getByText("Read-Only")).toBeInTheDocument(); + + // Click selects via callback + fireEvent.click(screen.getByTestId("eval-lane-card")); + expect(selectSpy).toHaveBeenCalledTimes(1); + + // Test read_only === false + rerender(); + expect(screen.getByText("CLI-Only")).toBeInTheDocument(); + expect(screen.getByTitle("API run disabled — use CLI")).toBeInTheDocument(); + }); + }); + + // 2. EvalRunButton + describe("EvalRunButton", () => { + it("hidden when lane.read_only === false; visible but holdout-disabled with correct hover-text otherwise", () => { + // Mock useEvalRun mutation + vi.mocked(useEvalRun).mockReturnValue({ + mutate: vi.fn(), + isPending: false, + isError: false, + isSuccess: false, + error: null, + } as any); + + const runStartSpy = vi.fn(); + const runSuccessSpy = vi.fn(); + const runErrorSpy = vi.fn(); + + // Non-read-only lane -> should be null + const { rerender } = render( + + ); + expect(screen.queryByTestId("eval-run-form")).not.toBeInTheDocument(); + + // Read-only lane -> should render, holdout disabled + rerender( + + ); + expect(screen.getByTestId("eval-run-form")).toBeInTheDocument(); + const holdoutOption = screen.getByRole("option", { name: "Holdout" }) as HTMLOptionElement; + expect(holdoutOption).toBeDisabled(); + expect(holdoutOption.getAttribute("title")).toBe("Holdout runs require sealed-eval config — use CLI"); + }); + }); + + // 3. EvalMetricGrid + describe("EvalMetricGrid", () => { + it("enforces deterministic key ordering across two identical runs", () => { + const metricsRun1 = { + total: 10, + passed: 9, + accuracy: 0.9, + latency_ms: 150, + }; + + const metricsRun2 = { + latency_ms: 150, + accuracy: 0.9, + passed: 9, + total: 10, + }; + + const { container: container1, unmount: unmount1 } = render(); + const cards1 = Array.from(container1.querySelectorAll('[data-testid="metric-card"]')).map(el => el.textContent); + unmount1(); + + const { container: container2 } = render(); + const cards2 = Array.from(container2.querySelectorAll('[data-testid="metric-card"]')).map(el => el.textContent); + + // Verify lexicographical order + expect(cards1).toEqual(cards2); + expect(cards1[0]).toContain("accuracy"); + expect(cards1[1]).toContain("latency ms"); + expect(cards1[2]).toContain("passed"); + expect(cards1[3]).toContain("total"); + }); + }); + + // 4. EvalFailureViewer + describe("EvalFailureViewer", () => { + it("renders calm-success EmptyState with 0 failures", () => { + render( + + ); + + expect(screen.queryByTestId("eval-failures")).not.toBeInTheDocument(); + expect(screen.getByText("All checks passed for eval lane contemplation_quality.")).toBeInTheDocument(); + }); + + it("renders failure cards with expected vs actual diff for failed cases", () => { + render( + + ); + + expect(screen.getByText("Failures (1)")).toBeInTheDocument(); + expect(screen.getByText("c2")).toBeInTheDocument(); + expect(screen.getByText("Wrong outcome")).toBeInTheDocument(); + expect(screen.getByTestId("json-diff")).toBeInTheDocument(); + }); + }); + + // 5. EvalsRoute + describe("EvalsRoute", () => { + it("full happy-path with selection, running and result viewing", async () => { + // Mock useEvalLanes query + vi.mocked(useEvalLanes).mockReturnValue({ + data: mockLanes, + isLoading: false, + isError: false, + error: null, + } as any); + + // Mock useEvalRun mutation + let mutateCallback: any; + vi.mocked(useEvalRun).mockReturnValue({ + mutate: vi.fn((req, options) => { + mutateCallback = options; + }), + isPending: false, + isError: false, + isSuccess: false, + error: null, + } as any); + + const client = makeClient(); + render( + + + + + + ); + + // Verify lane details are shown + expect(screen.getByText("contemplation_quality", { selector: "h2" })).toBeInTheDocument(); + expect(screen.getByTestId("run-button")).toBeInTheDocument(); + + // Trigger run + fireEvent.click(screen.getByTestId("run-button")); + + // Simulate success + expect(mutateCallback).toBeDefined(); + mutateCallback.onSuccess(mockResult); + + // Result should be visible + await waitFor(() => { + expect(screen.getByText("Status:")).toBeInTheDocument(); + }); + expect(screen.getByText("accuracy")).toBeInTheDocument(); + expect(screen.getByText("Failures (1)")).toBeInTheDocument(); + }); + + it("renders ErrorState when fetchEvalLanes fails", () => { + vi.mocked(useEvalLanes).mockReturnValue({ + data: null, + isLoading: false, + isError: true, + error: new Error("Network Error"), + } as any); + + const client = makeClient(); + render( + + + + + + ); + + expect(screen.getByText("Network Error")).toBeInTheDocument(); + expect(screen.getByText("Mutation status")).toBeInTheDocument(); + }); + + it("verifies direct runEvalLane client-side refusal logic", async () => { + // Refusal 1: lane.read_only === false + await expect( + runEvalLane({ lane: "unsafe_lane", split: "public" }) + ).rejects.toThrowError( + new WorkbenchApiError("client_refused_unsafe_lane", "API run disabled — use CLI") + ); + + // Refusal 2: split === "holdout" without sealed config flag + await expect( + runEvalLane({ lane: "contemplation_quality", split: "holdout" }) + ).rejects.toThrowError( + new WorkbenchApiError("client_refused_sealed_holdout", "Holdout runs require sealed-eval config — use CLI") + ); + + // Success: split === "holdout" with sealed config flag + if (typeof window !== "undefined") { + (window as any).sealedEvalConfig = true; + } + + // Expect it to call fetch successfully without throwing refusals + const res = await runEvalLane({ lane: "contemplation_quality", split: "holdout" }); + expect(res).toEqual(mockResult); + }); + }); +}); diff --git a/workbench-ui/src/routes/EvalsRoutePlaceholder.tsx b/workbench-ui/src/routes/EvalsRoutePlaceholder.tsx deleted file mode 100644 index 1ccc2476..00000000 --- a/workbench-ui/src/routes/EvalsRoutePlaceholder.tsx +++ /dev/null @@ -1,10 +0,0 @@ -import { EmptyState } from "../design/components/states/EmptyState"; - -export function EvalsRoutePlaceholder() { - return ( - - ); -} diff --git a/workbench-ui/src/types/api.ts b/workbench-ui/src/types/api.ts index 48d6a45a..b4862972 100644 --- a/workbench-ui/src/types/api.ts +++ b/workbench-ui/src/types/api.ts @@ -7,7 +7,9 @@ export type ErrorCode = | "unsupported" | "read_error" | "eval_failed" - | "runtime_unavailable"; + | "runtime_unavailable" + | "client_refused_unsafe_lane" + | "client_refused_sealed_holdout"; export type Backend = "numpy" | "mlx" | "rust" | "unknown"; export type MutationMode = "read_only" | "runtime_turn"; @@ -130,6 +132,12 @@ export interface EvalLaneSummary { description: string | null; } +export interface EvalRunRequest { + lane: string; + version?: string; + split?: "dev" | "public" | "holdout"; +} + export interface EvalRunResult { lane: string; version: string;