feat(workbench/W-030): eval center (safe lanes only, ADR-0160 §Phase 5) (#327)
This commit is contained in:
Shay
GitHub
parent
a0e9ca8535
commit
00f5056209
12 changed files with 951 additions and 16 deletions
|
|
@ -1,4 +1,4 @@
|
|||
import type { ApiResponse, ErrorCode } from "../types/api";
|
||||
import type { ApiResponse, ErrorCode, EvalLaneSummary, EvalRunRequest, EvalRunResult } from "../types/api";
|
||||
|
||||
export class WorkbenchApiError extends Error {
|
||||
constructor(
|
||||
|
|
@ -27,3 +27,35 @@ export async function apiFetch<T>(path: string, init?: RequestInit): Promise<T>
|
|||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchEvalLanes(): Promise<EvalLaneSummary[]> {
|
||||
return apiFetch<EvalLaneSummary[]>("/evals");
|
||||
}
|
||||
|
||||
export async function runEvalLane(req: EvalRunRequest): Promise<EvalRunResult> {
|
||||
if (req.split === "holdout") {
|
||||
const hasConfig = typeof window !== "undefined" && (window as any).sealedEvalConfig === true;
|
||||
if (!hasConfig) {
|
||||
throw new WorkbenchApiError(
|
||||
"client_refused_sealed_holdout",
|
||||
"Holdout runs require sealed-eval config — use CLI"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const lanes = await fetchEvalLanes();
|
||||
const lane = lanes.find((l) => l.lane === req.lane);
|
||||
if (!lane) {
|
||||
throw new WorkbenchApiError("not_found", `Eval lane not found: ${req.lane}`);
|
||||
}
|
||||
if (!lane.read_only) {
|
||||
throw new WorkbenchApiError("client_refused_unsafe_lane", "API run disabled — use CLI");
|
||||
}
|
||||
|
||||
return apiFetch<EvalRunResult>("/evals/run", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(req),
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import {
|
|||
useMutation,
|
||||
QueryClientProvider,
|
||||
} from "@tanstack/react-query";
|
||||
import { apiFetch } from "./client";
|
||||
import { apiFetch, fetchEvalLanes, runEvalLane } from "./client";
|
||||
import type { WorkbenchApiError } from "./client";
|
||||
import type {
|
||||
RuntimeStatus,
|
||||
|
|
@ -15,6 +15,7 @@ import type {
|
|||
EvalLaneSummary,
|
||||
EvalRunResult,
|
||||
ChatTurnResult,
|
||||
EvalRunRequest,
|
||||
} from "../types/api";
|
||||
|
||||
export { QueryClientProvider };
|
||||
|
|
@ -70,7 +71,9 @@ export function useProposal(id: string) {
|
|||
export function useEvalLanes() {
|
||||
return useQuery<EvalLaneSummary[]>({
|
||||
queryKey: ["api", "evals"],
|
||||
queryFn: () => apiFetch<EvalLaneSummary[]>("/evals"),
|
||||
queryFn: fetchEvalLanes,
|
||||
staleTime: 60_000,
|
||||
refetchOnWindowFocus: false,
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -82,6 +85,13 @@ export function useEvalLane(name: string) {
|
|||
});
|
||||
}
|
||||
|
||||
export function useEvalRun() {
|
||||
return useMutation<EvalRunResult, WorkbenchApiError, EvalRunRequest>({
|
||||
mutationKey: ["eval-run"],
|
||||
mutationFn: runEvalLane,
|
||||
});
|
||||
}
|
||||
|
||||
export function useChatTurn() {
|
||||
return useMutation<ChatTurnResult, WorkbenchApiError, { prompt: string }>({
|
||||
mutationKey: ["chat-turn"],
|
||||
|
|
@ -93,3 +103,4 @@ export function useChatTurn() {
|
|||
}),
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import { ChatRoute } from "../routes/ChatRoute";
|
|||
import { TraceRoutePlaceholder } from "../routes/TraceRoutePlaceholder";
|
||||
import { ReplayRoutePlaceholder } from "../routes/ReplayRoutePlaceholder";
|
||||
import { ProposalsRoutePlaceholder } from "../routes/ProposalsRoutePlaceholder";
|
||||
import { EvalsRoutePlaceholder } from "../routes/EvalsRoutePlaceholder";
|
||||
import { EvalsRoute } from "./evals/EvalsRoute";
|
||||
import { RunsRoutePlaceholder } from "../routes/RunsRoutePlaceholder";
|
||||
import { PacksRoutePlaceholder } from "../routes/PacksRoutePlaceholder";
|
||||
import { VaultRoutePlaceholder } from "../routes/VaultRoutePlaceholder";
|
||||
|
|
@ -25,7 +25,7 @@ export function App() {
|
|||
<Route path="trace" element={<TraceRoutePlaceholder />} />
|
||||
<Route path="replay" element={<ReplayRoutePlaceholder />} />
|
||||
<Route path="proposals" element={<ProposalsRoutePlaceholder />} />
|
||||
<Route path="evals" element={<EvalsRoutePlaceholder />} />
|
||||
<Route path="evals" element={<EvalsRoute />} />
|
||||
<Route path="runs" element={<RunsRoutePlaceholder />} />
|
||||
<Route path="packs" element={<PacksRoutePlaceholder />} />
|
||||
<Route path="vault" element={<VaultRoutePlaceholder />} />
|
||||
|
|
|
|||
32
workbench-ui/src/app/evals/EvalArtifactLink.tsx
Normal file
32
workbench-ui/src/app/evals/EvalArtifactLink.tsx
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
import { InfoBadge } from "../../design/components/badges/Badge";
|
||||
|
||||
export function EvalArtifactLink({
|
||||
lane,
|
||||
sourceDigest,
|
||||
}: {
|
||||
lane: string;
|
||||
sourceDigest: string;
|
||||
}) {
|
||||
const label = sourceDigest.slice(0, 12);
|
||||
const location = lane.includes("contemplation")
|
||||
? "engine_state/"
|
||||
: `evals/${lane}/results/`;
|
||||
|
||||
return (
|
||||
<div className="flex flex-wrap items-center gap-2 text-xs" data-testid="eval-artifact-link">
|
||||
<span className="text-[var(--color-text-secondary)] font-medium">Source Digest:</span>
|
||||
<InfoBadge
|
||||
label={label}
|
||||
colorToken="--color-text-mono"
|
||||
meaning="Deterministic source digest for this run. Click to copy."
|
||||
adr="ADR-0160 / ADR-0162"
|
||||
evidence="EvalRunResult.source_digest is present."
|
||||
mono
|
||||
onCopy={sourceDigest}
|
||||
/>
|
||||
<span className="text-[var(--color-text-muted)]">
|
||||
Located at: <code className="font-mono bg-[var(--color-surface-inset)] px-1.5 py-0.5 rounded border border-[var(--color-border-subtle)] text-[var(--color-text-primary)]">{location}</code>
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
106
workbench-ui/src/app/evals/EvalFailureViewer.tsx
Normal file
106
workbench-ui/src/app/evals/EvalFailureViewer.tsx
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
import { StableJsonViewer } from "../../design/components/StableJsonViewer/StableJsonViewer";
|
||||
import { EmptyState } from "../../design/components/states/EmptyState";
|
||||
|
||||
interface CaseItem {
|
||||
case_id?: string;
|
||||
id?: string;
|
||||
passed?: boolean;
|
||||
expected?: unknown;
|
||||
actual?: unknown;
|
||||
failure_reason?: string;
|
||||
failure_reasons?: string[];
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
function ensureJsonString(val: unknown): string {
|
||||
if (val === undefined) return "";
|
||||
if (typeof val === "string") {
|
||||
try {
|
||||
JSON.parse(val);
|
||||
return val;
|
||||
} catch {
|
||||
return JSON.stringify(val);
|
||||
}
|
||||
}
|
||||
return JSON.stringify(val, null, 2);
|
||||
}
|
||||
|
||||
export function EvalFailureViewer({
|
||||
cases,
|
||||
passed,
|
||||
laneName,
|
||||
}: {
|
||||
cases: any[];
|
||||
passed: boolean | null;
|
||||
laneName: string;
|
||||
}) {
|
||||
const typedCases = cases as CaseItem[];
|
||||
const failures = typedCases.filter((c) => c.passed === false);
|
||||
|
||||
if (passed === true || failures.length === 0) {
|
||||
return (
|
||||
<EmptyState
|
||||
statement={`All checks passed for eval lane ${laneName}.`}
|
||||
nextAction={{ kind: "cli", command: `core eval --lane ${laneName}` }}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="grid gap-4" data-testid="eval-failures">
|
||||
<h3 className="font-semibold text-lg text-[var(--color-state-contradicted)]">
|
||||
Failures ({failures.length})
|
||||
</h3>
|
||||
<div className="grid gap-4">
|
||||
{failures.map((c, idx) => {
|
||||
const caseId = c.case_id || c.id || `case-${idx}`;
|
||||
const reason = c.failure_reason || (Array.isArray(c.failure_reasons) ? c.failure_reasons.join(", ") : "") || "No reason specified";
|
||||
|
||||
const hasExpectedActual = c.expected !== undefined || c.actual !== undefined;
|
||||
|
||||
const expectedStr = c.expected !== undefined
|
||||
? ensureJsonString(c.expected)
|
||||
: JSON.stringify(c, null, 2);
|
||||
const actualStr = c.actual !== undefined
|
||||
? ensureJsonString(c.actual)
|
||||
: "";
|
||||
|
||||
return (
|
||||
<div
|
||||
key={caseId}
|
||||
className="rounded-lg border border-[var(--color-state-danger-border)] bg-[var(--color-surface-raised)] p-4 shadow-sm"
|
||||
data-testid="failure-card"
|
||||
>
|
||||
<div className="flex items-center justify-between gap-2 border-b border-[var(--color-border-subtle)] pb-2 mb-3">
|
||||
<span className="font-mono text-sm font-semibold text-[var(--color-text-primary)]">
|
||||
{caseId}
|
||||
</span>
|
||||
<span className="rounded bg-[var(--color-state-contradicted)]/10 px-2 py-0.5 text-xs font-semibold text-[var(--color-state-contradicted)]">
|
||||
Failed
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div className="mb-3">
|
||||
<div className="font-semibold text-xs text-[var(--color-text-secondary)] mb-1">Reason</div>
|
||||
<div className="text-sm text-[var(--color-text-primary)] font-mono bg-[var(--color-surface-inset)] p-2 rounded border border-[var(--color-border-subtle)]">
|
||||
{reason}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<div className="font-semibold text-xs text-[var(--color-text-secondary)] mb-1">
|
||||
{hasExpectedActual ? "Expected vs Actual" : "Case Details"}
|
||||
</div>
|
||||
{hasExpectedActual ? (
|
||||
<StableJsonViewer source={expectedStr} compareSource={actualStr} />
|
||||
) : (
|
||||
<StableJsonViewer source={expectedStr} />
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
67
workbench-ui/src/app/evals/EvalLaneCard.tsx
Normal file
67
workbench-ui/src/app/evals/EvalLaneCard.tsx
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
import type { EvalLaneSummary } from "../../types/api";
|
||||
|
||||
export function EvalLaneCard({
|
||||
lane,
|
||||
isSelected,
|
||||
onSelect,
|
||||
}: {
|
||||
lane: EvalLaneSummary;
|
||||
isSelected: boolean;
|
||||
onSelect: () => void;
|
||||
}) {
|
||||
const readOnlyStyle = lane.read_only
|
||||
? {
|
||||
backgroundColor: "color-mix(in srgb, var(--color-state-verified) 18%, transparent)",
|
||||
borderColor: "var(--color-state-verified)",
|
||||
color: "var(--color-state-verified)",
|
||||
}
|
||||
: {
|
||||
backgroundColor: "color-mix(in srgb, var(--color-state-undetermined) 18%, transparent)",
|
||||
borderColor: "var(--color-state-undetermined)",
|
||||
color: "var(--color-text-muted)",
|
||||
};
|
||||
|
||||
return (
|
||||
<button
|
||||
onClick={onSelect}
|
||||
className={`w-full text-left p-3 rounded-lg border transition-all duration-150 focus-visible:outline focus-visible:outline-2 focus-visible:outline-[var(--color-focus-ring)] ${
|
||||
isSelected
|
||||
? "border-[var(--color-border-strong)] bg-[var(--color-surface-raised)] shadow-[var(--shadow-panel)]"
|
||||
: "border-[var(--color-border-subtle)] bg-[var(--color-surface-base)] hover:bg-[var(--color-surface-raised)]"
|
||||
}`}
|
||||
type="button"
|
||||
data-testid="eval-lane-card"
|
||||
>
|
||||
<div className="flex items-start justify-between gap-2">
|
||||
<h4 className="font-mono text-sm font-semibold text-[var(--color-text-primary)] truncate">
|
||||
{lane.lane}
|
||||
</h4>
|
||||
<span
|
||||
style={readOnlyStyle}
|
||||
className="shrink-0 inline-flex items-center rounded border px-1.5 py-0.5 text-[10px] font-semibold transition-colors"
|
||||
title={lane.read_only ? "API runs enabled" : "API run disabled — use CLI"}
|
||||
data-testid="read-only-badge"
|
||||
>
|
||||
{lane.read_only ? "Read-Only" : "CLI-Only"}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{lane.description && (
|
||||
<p className="mt-1 text-xs text-[var(--color-text-secondary)] line-clamp-2">
|
||||
{lane.description}
|
||||
</p>
|
||||
)}
|
||||
|
||||
<div className="mt-2.5 flex flex-wrap gap-1" data-testid="version-badges">
|
||||
{lane.versions.map((version) => (
|
||||
<span
|
||||
key={version}
|
||||
className="inline-flex items-center rounded bg-[var(--color-state-neutral-bg)] border border-[var(--color-state-neutral-border)] px-1 py-0.5 font-mono text-[9px] text-[var(--color-state-neutral-text)]"
|
||||
>
|
||||
{version}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
</button>
|
||||
);
|
||||
}
|
||||
98
workbench-ui/src/app/evals/EvalMetricGrid.tsx
Normal file
98
workbench-ui/src/app/evals/EvalMetricGrid.tsx
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
import { InfoBadge } from "../../design/components/badges/Badge";
|
||||
|
||||
export function EvalMetricGrid({
|
||||
metrics,
|
||||
}: {
|
||||
metrics: Record<string, unknown>;
|
||||
}) {
|
||||
const sortedKeys = Object.keys(metrics).sort();
|
||||
|
||||
function formatValue(value: unknown): string {
|
||||
if (typeof value === "boolean") {
|
||||
return value ? "true" : "false";
|
||||
}
|
||||
if (typeof value === "number") {
|
||||
return String(value);
|
||||
}
|
||||
if (typeof value === "object" && value !== null) {
|
||||
return JSON.stringify(value, null, 2);
|
||||
}
|
||||
return String(value ?? "");
|
||||
}
|
||||
|
||||
function getUnit(key: string): string | undefined {
|
||||
const k = key.toLowerCase();
|
||||
if (k.endsWith("_ms") || k.includes("ms") || k.includes("latency")) {
|
||||
return "ms";
|
||||
}
|
||||
if (k.endsWith("_rate") || k.endsWith("_pct") || k.includes("percentage")) {
|
||||
return "%";
|
||||
}
|
||||
if (k.endsWith("_sec") || k.includes("seconds")) {
|
||||
return "s";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function renderPassFailBadge(key: string, value: unknown) {
|
||||
const isBool = typeof value === "boolean";
|
||||
const k = key.toLowerCase();
|
||||
const isPassKey = k.includes("pass") || k.includes("success") || k === "passed";
|
||||
|
||||
if (isBool && isPassKey) {
|
||||
if (value === true) {
|
||||
return (
|
||||
<InfoBadge
|
||||
label="Passed"
|
||||
colorToken="--color-state-verified"
|
||||
meaning="Metric passed the verification criteria."
|
||||
adr="ADR-0160 / ADR-0162"
|
||||
evidence="Metric value indicates success."
|
||||
/>
|
||||
);
|
||||
} else {
|
||||
return (
|
||||
<InfoBadge
|
||||
label="Failed"
|
||||
colorToken="--color-state-contradicted"
|
||||
meaning="Metric failed the verification criteria."
|
||||
adr="ADR-0160 / ADR-0162"
|
||||
evidence="Metric value indicates failure."
|
||||
/>
|
||||
);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="grid gap-3 sm:grid-cols-2 md:grid-cols-3" data-testid="eval-metric-grid">
|
||||
{sortedKeys.map((key) => {
|
||||
const val = metrics[key];
|
||||
const unit = getUnit(key);
|
||||
const badge = renderPassFailBadge(key, val);
|
||||
const formatted = formatValue(val);
|
||||
const isObj = typeof val === "object" && val !== null;
|
||||
|
||||
return (
|
||||
<div
|
||||
key={key}
|
||||
className="flex flex-col justify-between rounded-lg border border-[var(--color-border-subtle)] bg-[var(--color-surface-raised)] p-3 shadow-sm"
|
||||
data-testid="metric-card"
|
||||
>
|
||||
<div>
|
||||
<div className="text-[10px] font-semibold text-[var(--color-text-muted)] uppercase tracking-wider truncate" title={key}>
|
||||
{key.replaceAll("_", " ")}
|
||||
</div>
|
||||
<div className={`mt-1 font-mono text-sm font-semibold text-[var(--color-text-primary)] ${isObj ? "whitespace-pre overflow-x-auto text-[10px] bg-[var(--color-surface-inset)] p-1.5 rounded" : ""}`}>
|
||||
{formatted}
|
||||
{unit && <span className="ml-1 text-xs text-[var(--color-text-muted)] font-sans font-normal">{unit}</span>}
|
||||
</div>
|
||||
</div>
|
||||
{badge && <div className="mt-2 flex justify-start">{badge}</div>}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
104
workbench-ui/src/app/evals/EvalRunButton.tsx
Normal file
104
workbench-ui/src/app/evals/EvalRunButton.tsx
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
import { useState, useEffect } from "react";
|
||||
import { useEvalRun } from "../../api/queries";
|
||||
import { Button } from "../../design/components/primitives/Button";
|
||||
import type { EvalLaneSummary, EvalRunResult } from "../../types/api";
|
||||
import type { WorkbenchApiError } from "../../api/client";
|
||||
|
||||
export function EvalRunButton({
|
||||
lane,
|
||||
onRunStart,
|
||||
onRunSuccess,
|
||||
onRunError,
|
||||
}: {
|
||||
lane: EvalLaneSummary | null;
|
||||
onRunStart: () => void;
|
||||
onRunSuccess: (result: EvalRunResult) => void;
|
||||
onRunError: (error: WorkbenchApiError) => void;
|
||||
}) {
|
||||
const evalRun = useEvalRun();
|
||||
const [version, setVersion] = useState("");
|
||||
const [split, setSplit] = useState<"dev" | "public" | "holdout">("public");
|
||||
|
||||
// Reset selection when lane changes
|
||||
useEffect(() => {
|
||||
if (lane) {
|
||||
setVersion(lane.versions[0] || "v1");
|
||||
setSplit("public");
|
||||
}
|
||||
}, [lane]);
|
||||
|
||||
if (!lane || !lane.read_only) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const handleRun = () => {
|
||||
onRunStart();
|
||||
evalRun.mutate(
|
||||
{ lane: lane.lane, version, split },
|
||||
{
|
||||
onSuccess: (data) => {
|
||||
onRunSuccess(data);
|
||||
},
|
||||
onError: (err) => {
|
||||
onRunError(err);
|
||||
},
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
const isPending = evalRun.isPending;
|
||||
|
||||
return (
|
||||
<div className="rounded-lg border border-[var(--color-border-subtle)] bg-[var(--color-surface-raised)] p-4 shadow-sm" data-testid="eval-run-form">
|
||||
<h3 className="text-sm font-semibold text-[var(--color-text-primary)] mb-3">Run Configuration</h3>
|
||||
<div className="flex flex-wrap gap-4 items-end">
|
||||
<div className="flex flex-col gap-1.5">
|
||||
<label htmlFor="version-select" className="text-xs text-[var(--color-text-secondary)] font-medium">
|
||||
Version
|
||||
</label>
|
||||
<select
|
||||
id="version-select"
|
||||
value={version}
|
||||
onChange={(e) => setVersion(e.target.value)}
|
||||
disabled={isPending}
|
||||
className="bg-[var(--color-surface-base)] border border-[var(--color-border-subtle)] rounded px-2.5 py-1.5 text-xs text-[var(--color-text-primary)] focus:outline-none focus:ring-1 focus:ring-[var(--color-focus-ring)] disabled:opacity-50"
|
||||
>
|
||||
{lane.versions.map((v) => (
|
||||
<option key={v} value={v}>
|
||||
{v}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-1.5">
|
||||
<label htmlFor="split-select" className="text-xs text-[var(--color-text-secondary)] font-medium">
|
||||
Split
|
||||
</label>
|
||||
<select
|
||||
id="split-select"
|
||||
value={split}
|
||||
onChange={(e) => setSplit(e.target.value as any)}
|
||||
disabled={isPending}
|
||||
className="bg-[var(--color-surface-base)] border border-[var(--color-border-subtle)] rounded px-2.5 py-1.5 text-xs text-[var(--color-text-primary)] focus:outline-none focus:ring-1 focus:ring-[var(--color-focus-ring)] disabled:opacity-50"
|
||||
>
|
||||
<option value="public">Public</option>
|
||||
<option value="dev">Dev</option>
|
||||
<option value="holdout" title="Holdout runs require sealed-eval config — use CLI" disabled>
|
||||
Holdout
|
||||
</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<Button
|
||||
onClick={handleRun}
|
||||
disabled={isPending}
|
||||
className="h-8.5"
|
||||
data-testid="run-button"
|
||||
>
|
||||
{isPending ? "Running..." : "Run Eval"}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
188
workbench-ui/src/app/evals/EvalsRoute.tsx
Normal file
188
workbench-ui/src/app/evals/EvalsRoute.tsx
Normal file
|
|
@ -0,0 +1,188 @@
|
|||
import { useState } from "react";
|
||||
import { useSearchParams } from "react-router-dom";
|
||||
import { useEvalLanes } from "../../api/queries";
|
||||
import { EvalLaneCard } from "./EvalLaneCard";
|
||||
import { EvalRunButton } from "./EvalRunButton";
|
||||
import { EvalMetricGrid } from "./EvalMetricGrid";
|
||||
import { EvalFailureViewer } from "./EvalFailureViewer";
|
||||
import { EvalArtifactLink } from "./EvalArtifactLink";
|
||||
import { EmptyState } from "../../design/components/states/EmptyState";
|
||||
import { ErrorState } from "../../design/components/states/ErrorState";
|
||||
import { LoadingState } from "../../design/components/states/LoadingState";
|
||||
import type { EvalRunResult } from "../../types/api";
|
||||
import { WorkbenchApiError } from "../../api/client";
|
||||
|
||||
export function EvalsRoute() {
|
||||
const { data: lanes, isLoading, isError, error } = useEvalLanes();
|
||||
const [searchParams, setSearchParams] = useSearchParams();
|
||||
const selectedLaneName = searchParams.get("lane") || "";
|
||||
|
||||
// Maintain per-lane run states (pending, result, error)
|
||||
const [runStates, setRunStates] = useState<
|
||||
Record<
|
||||
string,
|
||||
{
|
||||
isPending: boolean;
|
||||
result?: EvalRunResult;
|
||||
error?: WorkbenchApiError;
|
||||
}
|
||||
>
|
||||
>({});
|
||||
|
||||
if (isLoading) {
|
||||
return <LoadingState label="Loading eval lanes..." />;
|
||||
}
|
||||
|
||||
if (isError) {
|
||||
return (
|
||||
<ErrorState
|
||||
whatFailed={error instanceof Error ? error.message : "Failed to load eval lanes."}
|
||||
mutationStatus="No corpus mutation occurred."
|
||||
reproducer="curl -X GET http://127.0.0.1:8765/evals"
|
||||
retrySafety="Retry: safe"
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
const selectedLane = lanes?.find((l) => l.lane === selectedLaneName) || null;
|
||||
const currentRunState = selectedLaneName ? runStates[selectedLaneName] : null;
|
||||
|
||||
return (
|
||||
<div className="grid h-full grid-cols-1 gap-4 md:grid-cols-[18rem_1fr]" data-testid="evals-route">
|
||||
{/* Left Pane: Lane List */}
|
||||
<div className="flex flex-col gap-3 border-r border-[var(--color-border-subtle)] pr-4 overflow-y-auto">
|
||||
<h2 className="text-md font-semibold text-[var(--color-text-primary)]">Eval Lanes</h2>
|
||||
<div className="flex flex-col gap-2">
|
||||
{lanes && lanes.length > 0 ? (
|
||||
lanes.map((lane) => (
|
||||
<EvalLaneCard
|
||||
key={lane.lane}
|
||||
lane={lane}
|
||||
isSelected={lane.lane === selectedLaneName}
|
||||
onSelect={() => setSearchParams({ lane: lane.lane })}
|
||||
/>
|
||||
))
|
||||
) : (
|
||||
<EmptyState
|
||||
statement="No eval lanes discovered."
|
||||
nextAction={{ kind: "cli", command: "core eval --list" }}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Right Pane: Results / Form */}
|
||||
<div className="flex flex-col gap-4 overflow-y-auto pl-2">
|
||||
{selectedLane ? (
|
||||
<>
|
||||
{/* Header info */}
|
||||
<div className="flex flex-wrap items-center justify-between gap-3 border-b border-[var(--color-border-subtle)] pb-3">
|
||||
<div>
|
||||
<h2 className="text-lg font-semibold text-[var(--color-text-primary)] font-mono">
|
||||
{selectedLane.lane}
|
||||
</h2>
|
||||
{selectedLane.description && (
|
||||
<p className="mt-1 text-xs text-[var(--color-text-secondary)]">
|
||||
{selectedLane.description}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Run Button configuration if read-only */}
|
||||
{selectedLane.read_only ? (
|
||||
<EvalRunButton
|
||||
lane={selectedLane}
|
||||
onRunStart={() => {
|
||||
setRunStates((prev) => ({
|
||||
...prev,
|
||||
[selectedLane.lane]: { isPending: true },
|
||||
}));
|
||||
}}
|
||||
onRunSuccess={(result) => {
|
||||
setRunStates((prev) => ({
|
||||
...prev,
|
||||
[selectedLane.lane]: { isPending: false, result },
|
||||
}));
|
||||
}}
|
||||
onRunError={(err) => {
|
||||
setRunStates((prev) => ({
|
||||
...prev,
|
||||
[selectedLane.lane]: { isPending: false, error: err },
|
||||
}));
|
||||
}}
|
||||
/>
|
||||
) : (
|
||||
<div className="text-xs text-[var(--color-text-muted)] bg-[var(--color-surface-inset)] p-3 rounded-lg border border-[var(--color-border-subtle)] font-medium">
|
||||
⚠️ API runs disabled for write-active lane. Use local CLI to execute this lane:
|
||||
<code className="block mt-1 font-mono text-[var(--color-text-primary)] bg-[var(--color-surface-base)] p-1.5 rounded">
|
||||
core eval --lane {selectedLane.lane}
|
||||
</code>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Result display */}
|
||||
{currentRunState?.isPending ? (
|
||||
<LoadingState label="Running eval lane..." />
|
||||
) : currentRunState?.error ? (
|
||||
<ErrorState
|
||||
whatFailed={currentRunState.error.message}
|
||||
mutationStatus="No corpus mutation occurred."
|
||||
reproducer={`core eval --lane ${selectedLane.lane}`}
|
||||
retrySafety="Retry: safe"
|
||||
/>
|
||||
) : currentRunState?.result ? (
|
||||
<div className="flex flex-col gap-4">
|
||||
<div className="flex flex-wrap items-center gap-3">
|
||||
<span className="text-xs text-[var(--color-text-secondary)] font-semibold">
|
||||
Status:
|
||||
</span>
|
||||
<span
|
||||
className={`rounded-md px-2 py-0.5 text-xs font-semibold ${
|
||||
currentRunState.result.passed
|
||||
? "bg-[var(--color-state-success-bg)] text-[var(--color-state-success-text)] border border-[var(--color-state-success-border)]"
|
||||
: "bg-[var(--color-state-danger-bg)] text-[var(--color-state-danger-text)] border border-[var(--color-state-danger-border)]"
|
||||
}`}
|
||||
>
|
||||
{currentRunState.result.passed ? "Passed" : "Failed"}
|
||||
</span>
|
||||
{currentRunState.result.source_digest && (
|
||||
<EvalArtifactLink
|
||||
lane={selectedLane.lane}
|
||||
sourceDigest={currentRunState.result.source_digest}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-2">
|
||||
<h3 className="text-sm font-semibold text-[var(--color-text-primary)]">Metrics</h3>
|
||||
<EvalMetricGrid metrics={currentRunState.result.metrics} />
|
||||
</div>
|
||||
|
||||
<EvalFailureViewer
|
||||
cases={currentRunState.result.cases}
|
||||
passed={currentRunState.result.passed}
|
||||
laneName={selectedLane.lane}
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
<EmptyState
|
||||
statement={
|
||||
selectedLane.read_only
|
||||
? `No run results for lane "${selectedLane.lane}" in this session. Trigger a run above.`
|
||||
: `Eval lane "${selectedLane.lane}" is CLI-only. No session results available.`
|
||||
}
|
||||
nextAction={{ kind: "cli", command: `core eval --lane ${selectedLane.lane}` }}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
) : (
|
||||
<EmptyState
|
||||
statement="Select an eval lane from the list to view results or run checks."
|
||||
nextAction={{ kind: "cli", command: "core eval --list" }}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
299
workbench-ui/src/app/evals/evals.test.tsx
Normal file
299
workbench-ui/src/app/evals/evals.test.tsx
Normal file
|
|
@ -0,0 +1,299 @@
|
|||
import { render, screen, fireEvent, waitFor } from "@testing-library/react";
|
||||
import { describe, expect, it, vi, beforeEach } from "vitest";
|
||||
import { MemoryRouter } from "react-router-dom";
|
||||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
import { EvalLaneCard } from "./EvalLaneCard";
|
||||
import { EvalRunButton } from "./EvalRunButton";
|
||||
import { EvalMetricGrid } from "./EvalMetricGrid";
|
||||
import { EvalFailureViewer } from "./EvalFailureViewer";
|
||||
import { EvalsRoute } from "./EvalsRoute";
|
||||
import { runEvalLane, WorkbenchApiError } from "../../api/client";
|
||||
import type { EvalLaneSummary, EvalRunResult } from "../../types/api";
|
||||
|
||||
// Mock queries
|
||||
vi.mock("../../api/queries", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("../../api/queries")>();
|
||||
return {
|
||||
...actual,
|
||||
useEvalLanes: vi.fn(),
|
||||
useEvalRun: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
import { useEvalLanes, useEvalRun } from "../../api/queries";
|
||||
|
||||
const mockLanes: EvalLaneSummary[] = [
|
||||
{ lane: "contemplation_quality", versions: ["v1", "v2"], read_only: true, description: "Contemplation checks" },
|
||||
{ lane: "unsafe_lane", versions: ["v1"], read_only: false, description: "Unsafe checks" },
|
||||
];
|
||||
|
||||
const mockResult: EvalRunResult = {
|
||||
lane: "contemplation_quality",
|
||||
version: "v1",
|
||||
split: "public",
|
||||
passed: false,
|
||||
metrics: { accuracy: 0.8, passed: 4, total: 5 },
|
||||
cases: [
|
||||
{ case_id: "c1", passed: true },
|
||||
{ case_id: "c2", passed: false, expected: "val1", actual: "val2", failure_reason: "Mismatch value" },
|
||||
],
|
||||
source_digest: "abcdef1234567890",
|
||||
};
|
||||
|
||||
function makeClient() {
|
||||
return new QueryClient({ defaultOptions: { queries: { retry: false } } });
|
||||
}
|
||||
|
||||
describe("W-030 Component Tests", () => {
|
||||
const fetchMock = vi.fn();
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
fetchMock.mockImplementation((url: any) => {
|
||||
const urlStr = typeof url === "string" ? url : String(url?.url || url || "");
|
||||
if (urlStr.endsWith("/evals")) {
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
json: () => Promise.resolve({ ok: true, generated_at: "2026-05-26T00:00:00Z", data: mockLanes }),
|
||||
});
|
||||
}
|
||||
if (urlStr.endsWith("/evals/run")) {
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
json: () => Promise.resolve({ ok: true, generated_at: "2026-05-26T00:00:00Z", data: mockResult }),
|
||||
});
|
||||
}
|
||||
return Promise.reject(new Error(`Unhandled URL: ${urlStr}`));
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
if (typeof window !== "undefined") {
|
||||
delete (window as any).sealedEvalConfig;
|
||||
}
|
||||
});
|
||||
|
||||
// 1. EvalLaneCard
|
||||
describe("EvalLaneCard", () => {
|
||||
it("renders read_only badge correctly for both values; clicking selects via callback", () => {
|
||||
const selectSpy = vi.fn();
|
||||
|
||||
// Test read_only === true
|
||||
const { rerender } = render(
|
||||
<EvalLaneCard lane={mockLanes[0]} isSelected={false} onSelect={selectSpy} />
|
||||
);
|
||||
expect(screen.getByText("Read-Only")).toBeInTheDocument();
|
||||
|
||||
// Click selects via callback
|
||||
fireEvent.click(screen.getByTestId("eval-lane-card"));
|
||||
expect(selectSpy).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Test read_only === false
|
||||
rerender(<EvalLaneCard lane={mockLanes[1]} isSelected={false} onSelect={selectSpy} />);
|
||||
expect(screen.getByText("CLI-Only")).toBeInTheDocument();
|
||||
expect(screen.getByTitle("API run disabled — use CLI")).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
// 2. EvalRunButton
|
||||
describe("EvalRunButton", () => {
|
||||
it("hidden when lane.read_only === false; visible but holdout-disabled with correct hover-text otherwise", () => {
|
||||
// Mock useEvalRun mutation
|
||||
vi.mocked(useEvalRun).mockReturnValue({
|
||||
mutate: vi.fn(),
|
||||
isPending: false,
|
||||
isError: false,
|
||||
isSuccess: false,
|
||||
error: null,
|
||||
} as any);
|
||||
|
||||
const runStartSpy = vi.fn();
|
||||
const runSuccessSpy = vi.fn();
|
||||
const runErrorSpy = vi.fn();
|
||||
|
||||
// Non-read-only lane -> should be null
|
||||
const { rerender } = render(
|
||||
<EvalRunButton
|
||||
lane={mockLanes[1]}
|
||||
onRunStart={runStartSpy}
|
||||
onRunSuccess={runSuccessSpy}
|
||||
onRunError={runErrorSpy}
|
||||
/>
|
||||
);
|
||||
expect(screen.queryByTestId("eval-run-form")).not.toBeInTheDocument();
|
||||
|
||||
// Read-only lane -> should render, holdout disabled
|
||||
rerender(
|
||||
<EvalRunButton
|
||||
lane={mockLanes[0]}
|
||||
onRunStart={runStartSpy}
|
||||
onRunSuccess={runSuccessSpy}
|
||||
onRunError={runErrorSpy}
|
||||
/>
|
||||
);
|
||||
expect(screen.getByTestId("eval-run-form")).toBeInTheDocument();
|
||||
const holdoutOption = screen.getByRole("option", { name: "Holdout" }) as HTMLOptionElement;
|
||||
expect(holdoutOption).toBeDisabled();
|
||||
expect(holdoutOption.getAttribute("title")).toBe("Holdout runs require sealed-eval config — use CLI");
|
||||
});
|
||||
});
|
||||
|
||||
// 3. EvalMetricGrid
|
||||
describe("EvalMetricGrid", () => {
|
||||
it("enforces deterministic key ordering across two identical runs", () => {
|
||||
const metricsRun1 = {
|
||||
total: 10,
|
||||
passed: 9,
|
||||
accuracy: 0.9,
|
||||
latency_ms: 150,
|
||||
};
|
||||
|
||||
const metricsRun2 = {
|
||||
latency_ms: 150,
|
||||
accuracy: 0.9,
|
||||
passed: 9,
|
||||
total: 10,
|
||||
};
|
||||
|
||||
const { container: container1, unmount: unmount1 } = render(<EvalMetricGrid metrics={metricsRun1} />);
|
||||
const cards1 = Array.from(container1.querySelectorAll('[data-testid="metric-card"]')).map(el => el.textContent);
|
||||
unmount1();
|
||||
|
||||
const { container: container2 } = render(<EvalMetricGrid metrics={metricsRun2} />);
|
||||
const cards2 = Array.from(container2.querySelectorAll('[data-testid="metric-card"]')).map(el => el.textContent);
|
||||
|
||||
// Verify lexicographical order
|
||||
expect(cards1).toEqual(cards2);
|
||||
expect(cards1[0]).toContain("accuracy");
|
||||
expect(cards1[1]).toContain("latency ms");
|
||||
expect(cards1[2]).toContain("passed");
|
||||
expect(cards1[3]).toContain("total");
|
||||
});
|
||||
});
|
||||
|
||||
// 4. EvalFailureViewer
|
||||
describe("EvalFailureViewer", () => {
|
||||
it("renders calm-success EmptyState with 0 failures", () => {
|
||||
render(
|
||||
<EvalFailureViewer cases={[{ case_id: "c1", passed: true }]} passed={true} laneName="contemplation_quality" />
|
||||
);
|
||||
|
||||
expect(screen.queryByTestId("eval-failures")).not.toBeInTheDocument();
|
||||
expect(screen.getByText("All checks passed for eval lane contemplation_quality.")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("renders failure cards with expected vs actual diff for failed cases", () => {
|
||||
render(
|
||||
<EvalFailureViewer
|
||||
cases={[
|
||||
{ case_id: "c1", passed: true },
|
||||
{ case_id: "c2", passed: false, expected: "val1", actual: "val2", failure_reason: "Wrong outcome" },
|
||||
]}
|
||||
passed={false}
|
||||
laneName="contemplation_quality"
|
||||
/>
|
||||
);
|
||||
|
||||
expect(screen.getByText("Failures (1)")).toBeInTheDocument();
|
||||
expect(screen.getByText("c2")).toBeInTheDocument();
|
||||
expect(screen.getByText("Wrong outcome")).toBeInTheDocument();
|
||||
expect(screen.getByTestId("json-diff")).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
// 5. EvalsRoute
|
||||
describe("EvalsRoute", () => {
|
||||
it("full happy-path with selection, running and result viewing", async () => {
|
||||
// Mock useEvalLanes query
|
||||
vi.mocked(useEvalLanes).mockReturnValue({
|
||||
data: mockLanes,
|
||||
isLoading: false,
|
||||
isError: false,
|
||||
error: null,
|
||||
} as any);
|
||||
|
||||
// Mock useEvalRun mutation
|
||||
let mutateCallback: any;
|
||||
vi.mocked(useEvalRun).mockReturnValue({
|
||||
mutate: vi.fn((req, options) => {
|
||||
mutateCallback = options;
|
||||
}),
|
||||
isPending: false,
|
||||
isError: false,
|
||||
isSuccess: false,
|
||||
error: null,
|
||||
} as any);
|
||||
|
||||
const client = makeClient();
|
||||
render(
|
||||
<QueryClientProvider client={client}>
|
||||
<MemoryRouter initialEntries={["/evals?lane=contemplation_quality"]}>
|
||||
<EvalsRoute />
|
||||
</MemoryRouter>
|
||||
</QueryClientProvider>
|
||||
);
|
||||
|
||||
// Verify lane details are shown
|
||||
expect(screen.getByText("contemplation_quality", { selector: "h2" })).toBeInTheDocument();
|
||||
expect(screen.getByTestId("run-button")).toBeInTheDocument();
|
||||
|
||||
// Trigger run
|
||||
fireEvent.click(screen.getByTestId("run-button"));
|
||||
|
||||
// Simulate success
|
||||
expect(mutateCallback).toBeDefined();
|
||||
mutateCallback.onSuccess(mockResult);
|
||||
|
||||
// Result should be visible
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText("Status:")).toBeInTheDocument();
|
||||
});
|
||||
expect(screen.getByText("accuracy")).toBeInTheDocument();
|
||||
expect(screen.getByText("Failures (1)")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("renders ErrorState when fetchEvalLanes fails", () => {
|
||||
vi.mocked(useEvalLanes).mockReturnValue({
|
||||
data: null,
|
||||
isLoading: false,
|
||||
isError: true,
|
||||
error: new Error("Network Error"),
|
||||
} as any);
|
||||
|
||||
const client = makeClient();
|
||||
render(
|
||||
<QueryClientProvider client={client}>
|
||||
<MemoryRouter initialEntries={["/evals"]}>
|
||||
<EvalsRoute />
|
||||
</MemoryRouter>
|
||||
</QueryClientProvider>
|
||||
);
|
||||
|
||||
expect(screen.getByText("Network Error")).toBeInTheDocument();
|
||||
expect(screen.getByText("Mutation status")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("verifies direct runEvalLane client-side refusal logic", async () => {
|
||||
// Refusal 1: lane.read_only === false
|
||||
await expect(
|
||||
runEvalLane({ lane: "unsafe_lane", split: "public" })
|
||||
).rejects.toThrowError(
|
||||
new WorkbenchApiError("client_refused_unsafe_lane", "API run disabled — use CLI")
|
||||
);
|
||||
|
||||
// Refusal 2: split === "holdout" without sealed config flag
|
||||
await expect(
|
||||
runEvalLane({ lane: "contemplation_quality", split: "holdout" })
|
||||
).rejects.toThrowError(
|
||||
new WorkbenchApiError("client_refused_sealed_holdout", "Holdout runs require sealed-eval config — use CLI")
|
||||
);
|
||||
|
||||
// Success: split === "holdout" with sealed config flag
|
||||
if (typeof window !== "undefined") {
|
||||
(window as any).sealedEvalConfig = true;
|
||||
}
|
||||
|
||||
// Expect it to call fetch successfully without throwing refusals
|
||||
const res = await runEvalLane({ lane: "contemplation_quality", split: "holdout" });
|
||||
expect(res).toEqual(mockResult);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
import { EmptyState } from "../design/components/states/EmptyState";
|
||||
|
||||
export function EvalsRoutePlaceholder() {
|
||||
return (
|
||||
<EmptyState
|
||||
statement="Evals — no data loaded yet."
|
||||
nextAction={{ kind: "cli", command: "core eval --list" }}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
|
@ -7,7 +7,9 @@ export type ErrorCode =
|
|||
| "unsupported"
|
||||
| "read_error"
|
||||
| "eval_failed"
|
||||
| "runtime_unavailable";
|
||||
| "runtime_unavailable"
|
||||
| "client_refused_unsafe_lane"
|
||||
| "client_refused_sealed_holdout";
|
||||
|
||||
export type Backend = "numpy" | "mlx" | "rust" | "unknown";
|
||||
export type MutationMode = "read_only" | "runtime_turn";
|
||||
|
|
@ -130,6 +132,12 @@ export interface EvalLaneSummary {
|
|||
description: string | null;
|
||||
}
|
||||
|
||||
export interface EvalRunRequest {
|
||||
lane: string;
|
||||
version?: string;
|
||||
split?: "dev" | "public" | "holdout";
|
||||
}
|
||||
|
||||
export interface EvalRunResult {
|
||||
lane: string;
|
||||
version: string;
|
||||
|
|
|
|||
Loading…
Reference in a new issue