feat(workbench/evals): polish states, numeric alignment, lane URL state (wave 1d)

This commit is contained in:
Shay 2026-05-28 08:11:06 -07:00
parent e954c660f7
commit 37b04d504b
5 changed files with 560 additions and 138 deletions

View file

@ -5,7 +5,7 @@ export function EvalMetricGrid({
}: { }: {
metrics: Record<string, unknown>; metrics: Record<string, unknown>;
}) { }) {
const sortedKeys = Object.keys(metrics).sort(); const allKeys = Object.keys(metrics);
function formatValue(value: unknown): string { function formatValue(value: unknown): string {
if (typeof value === "boolean") { if (typeof value === "boolean") {
@ -34,6 +34,20 @@ export function EvalMetricGrid({
return undefined; return undefined;
} }
function isLowerIsBetter(key: string): boolean {
const k = key.toLowerCase();
return (
k.includes("latency") ||
k.includes("duration") ||
k.includes("time") ||
k.includes("error") ||
k.includes("fail") ||
k.includes("fabrication") ||
k.includes("divergence") ||
k.includes("cost")
);
}
function renderPassFailBadge(key: string, value: unknown) { function renderPassFailBadge(key: string, value: unknown) {
const isBool = typeof value === "boolean"; const isBool = typeof value === "boolean";
const k = key.toLowerCase(); const k = key.toLowerCase();
@ -65,33 +79,159 @@ export function EvalMetricGrid({
return null; return null;
} }
// Find pairs
const pairedKeys = new Set<string>();
const pairs: Array<{
key: string;
actual: unknown;
target: unknown;
}> = [];
// 1. Group "passed" and "total"
if (allKeys.includes("passed") && allKeys.includes("total")) {
pairs.push({
key: "passed",
actual: metrics["passed"],
target: metrics["total"],
});
pairedKeys.add("passed");
pairedKeys.add("total");
}
// 2. Group "X" and "X_target" or "target_X"
allKeys.forEach((k) => {
if (pairedKeys.has(k)) return;
if (k.endsWith("_target")) {
const baseKey = k.slice(0, -7);
if (allKeys.includes(baseKey)) {
pairs.push({
key: baseKey,
actual: metrics[baseKey],
target: metrics[k],
});
pairedKeys.add(baseKey);
pairedKeys.add(k);
}
} else if (k.startsWith("target_")) {
const baseKey = k.slice(7);
if (allKeys.includes(baseKey)) {
pairs.push({
key: baseKey,
actual: metrics[baseKey],
target: metrics[k],
});
pairedKeys.add(baseKey);
pairedKeys.add(k);
}
}
});
// 3. Remaining singles
const singles: Array<{ key: string; value: unknown }> = [];
allKeys.forEach((k) => {
if (!pairedKeys.has(k)) {
singles.push({ key: k, value: metrics[k] });
}
});
interface DisplayMetric {
key: string;
isPair: boolean;
actual: unknown;
target?: unknown;
value?: unknown;
}
const list: DisplayMetric[] = [
...pairs.map((p) => ({ key: p.key, isPair: true, actual: p.actual, target: p.target })),
...singles.map((s) => ({ key: s.key, isPair: false, actual: s.value, value: s.value })),
];
// Sort lexicographically
list.sort((a, b) => a.key.localeCompare(b.key));
return ( return (
<div className="grid gap-3 sm:grid-cols-2 md:grid-cols-3" data-testid="eval-metric-grid"> <div className="grid gap-3 sm:grid-cols-2 md:grid-cols-3" data-testid="eval-metric-grid">
{sortedKeys.map((key) => { {list.map((item) => {
const val = metrics[key]; const key = item.key;
const unit = getUnit(key); const unit = getUnit(key);
const badge = renderPassFailBadge(key, val);
const formatted = formatValue(val);
const isObj = typeof val === "object" && val !== null;
return ( if (item.isPair) {
<div const actualVal = item.actual;
key={key} const targetVal = item.target;
className="flex flex-col justify-between rounded-lg border border-[var(--color-border-subtle)] bg-[var(--color-surface-raised)] p-3 shadow-sm" const badge = renderPassFailBadge(key, actualVal);
data-testid="metric-card"
> const actualNum = Number(actualVal);
<div> const targetNum = Number(targetVal);
<div className="text-[10px] font-semibold text-[var(--color-text-muted)] uppercase tracking-wider truncate" title={key}> const hasNumericValues = !isNaN(actualNum) && !isNaN(targetNum);
{key.replaceAll("_", " ")} const isFailing =
</div> hasNumericValues &&
<div className={`mt-1 font-mono text-sm font-semibold text-[var(--color-text-primary)] ${isObj ? "whitespace-pre overflow-x-auto text-[10px] bg-[var(--color-surface-inset)] p-1.5 rounded" : ""}`}> (isLowerIsBetter(key) ? actualNum > targetNum : actualNum < targetNum);
{formatted}
{unit && <span className="ml-1 text-xs text-[var(--color-text-muted)] font-sans font-normal">{unit}</span>} const formattedActual = formatValue(actualVal);
const formattedTarget = formatValue(targetVal);
return (
<div
key={key}
className="flex flex-col justify-between rounded-lg border border-[var(--color-border-subtle)] bg-[var(--color-surface-raised)] p-3 shadow-sm"
data-testid="metric-card"
>
<div>
<div className="text-[10px] font-semibold text-[var(--color-text-muted)] uppercase tracking-wider truncate" title={key}>
{key.replaceAll("_", " ")}
</div>
<div className="mt-1 font-mono text-sm font-semibold text-right tabular-nums">
<span
className={
isFailing
? "text-[var(--color-state-contradicted)] font-bold"
: "text-[var(--color-text-primary)]"
}
>
{formattedActual}
</span>
<span className="text-[var(--color-text-muted)] mx-1">/</span>
<span className="text-[var(--color-text-secondary)] font-normal">
{formattedTarget}
</span>
{unit && <span className="ml-1 text-xs text-[var(--color-text-muted)] font-sans font-normal">{unit}</span>}
</div>
</div> </div>
{badge && <div className="mt-2 flex justify-start">{badge}</div>}
</div> </div>
{badge && <div className="mt-2 flex justify-start">{badge}</div>} );
</div> } else {
); const val = item.value;
const badge = renderPassFailBadge(key, val);
const formatted = formatValue(val);
const isObj = typeof val === "object" && val !== null;
const isNum = typeof val === "number";
return (
<div
key={key}
className="flex flex-col justify-between rounded-lg border border-[var(--color-border-subtle)] bg-[var(--color-surface-raised)] p-3 shadow-sm"
data-testid="metric-card"
>
<div>
<div className="text-[10px] font-semibold text-[var(--color-text-muted)] uppercase tracking-wider truncate" title={key}>
{key.replaceAll("_", " ")}
</div>
<div
className={`mt-1 font-mono text-sm font-semibold text-[var(--color-text-primary)] ${
isNum ? "text-right tabular-nums" : ""
} ${isObj ? "whitespace-pre overflow-x-auto text-[10px] bg-[var(--color-surface-inset)] p-1.5 rounded" : ""}`}
>
{formatted}
{unit && <span className="ml-1 text-xs text-[var(--color-text-muted)] font-sans font-normal">{unit}</span>}
</div>
</div>
{badge && <div className="mt-2 flex justify-start">{badge}</div>}
</div>
);
}
})} })}
</div> </div>
); );

View file

@ -12,10 +12,33 @@ import { LoadingState } from "../../design/components/states/LoadingState";
import type { EvalRunResult } from "../../types/api"; import type { EvalRunResult } from "../../types/api";
import { WorkbenchApiError } from "../../api/client"; import { WorkbenchApiError } from "../../api/client";
import { useState, useEffect } from "react";
import { useSearchParams } from "react-router-dom";
import { useEvalLanes, useEvalLane } from "../../api/queries";
import { EvalLaneCard } from "./EvalLaneCard";
import { EvalRunButton } from "./EvalRunButton";
import { EvalMetricGrid } from "./EvalMetricGrid";
import { EvalFailureViewer } from "./EvalFailureViewer";
import { EvalArtifactLink } from "./EvalArtifactLink";
import { EmptyState } from "../../design/components/states/EmptyState";
import { ErrorState } from "../../design/components/states/ErrorState";
import { LoadingState } from "../../design/components/states/LoadingState";
import type { EvalRunResult } from "../../types/api";
import { WorkbenchApiError } from "../../api/client";
import { useQueryClient } from "@tanstack/react-query";
export function EvalsRoute() { export function EvalsRoute() {
const { data: lanes, isLoading, isError, error } = useEvalLanes(); const { data: lanes, isLoading, isError, error } = useEvalLanes();
const [searchParams, setSearchParams] = useSearchParams(); const [searchParams, setSearchParams] = useSearchParams();
const selectedLaneName = searchParams.get("lane") || ""; const selectedLaneName = searchParams.get("lane") || "";
const queryClient = useQueryClient();
const {
data: lastRunResult,
isLoading: isLaneLoading,
isError: isLaneError,
error: laneError,
} = useEvalLane(selectedLaneName);
// Maintain per-lane run states (pending, result, error) // Maintain per-lane run states (pending, result, error)
const [runStates, setRunStates] = useState< const [runStates, setRunStates] = useState<
@ -29,6 +52,13 @@ export function EvalsRoute() {
> >
>({}); >({});
// Auto-select the first lane on load if no selection is in the URL
useEffect(() => {
if (!selectedLaneName && lanes && lanes.length > 0) {
setSearchParams({ lane: lanes[0].lane }, { replace: true });
}
}, [selectedLaneName, lanes, setSearchParams]);
if (isLoading) { if (isLoading) {
return <LoadingState label="Loading eval lanes..." />; return <LoadingState label="Loading eval lanes..." />;
} }
@ -46,6 +76,178 @@ export function EvalsRoute() {
const selectedLane = lanes?.find((l) => l.lane === selectedLaneName) || null; const selectedLane = lanes?.find((l) => l.lane === selectedLaneName) || null;
const currentRunState = selectedLaneName ? runStates[selectedLaneName] : null; const currentRunState = selectedLaneName ? runStates[selectedLaneName] : null;
const isNotFoundError =
laneError instanceof WorkbenchApiError && laneError.code === "not_found";
function renderHeaderAndForm(lane: typeof selectedLane) {
if (!lane) return null;
return (
<>
{/* Header info */}
<div className="flex flex-wrap items-center justify-between gap-3 border-b border-[var(--color-border-subtle)] pb-3">
<div>
<h2 className="text-lg font-semibold text-[var(--color-text-primary)] font-mono">
{lane.lane}
</h2>
{lane.description && (
<p className="mt-1 text-xs text-[var(--color-text-secondary)]">
{lane.description}
</p>
)}
</div>
</div>
{/* Run Button configuration if read-only */}
{lane.read_only ? (
<EvalRunButton
lane={lane}
onRunStart={() => {
setRunStates((prev) => ({
...prev,
[lane.lane]: { isPending: true },
}));
}}
onRunSuccess={(result) => {
setRunStates((prev) => ({
...prev,
[lane.lane]: { isPending: false, result },
}));
queryClient.invalidateQueries({ queryKey: ["api", "eval", lane.lane] });
}}
onRunError={(err) => {
setRunStates((prev) => ({
...prev,
[lane.lane]: { isPending: false, error: err },
}));
}}
/>
) : (
<div className="text-xs text-[var(--color-text-muted)] bg-[var(--color-surface-inset)] p-3 rounded-lg border border-[var(--color-border-subtle)] font-medium">
API runs disabled for write-active lane. Use local CLI to execute this lane:
<code className="block mt-1 font-mono text-[var(--color-text-primary)] bg-[var(--color-surface-base)] p-1.5 rounded">
core eval --lane {lane.lane}
</code>
</div>
)}
</>
);
}
const renderRightPane = () => {
if (!selectedLaneName) {
return (
<EmptyState
statement="Select an eval lane from the list to view results or run checks."
nextAction={{ kind: "cli", command: "core eval --list" }}
/>
);
}
if (!selectedLane) {
return (
<EmptyState
statement={`Selected lane "${selectedLaneName}" not found.`}
nextAction={{ kind: "cli", command: "core eval --list" }}
/>
);
}
// 1. Loading state for fetching the lane's last run details
if (isLaneLoading && !lastRunResult) {
return <LoadingState label="Loading eval lane details..." />;
}
// 2. Error state for fetching the lane details (excluding not_found)
if (isLaneError && !isNotFoundError) {
return (
<ErrorState
whatFailed={laneError instanceof Error ? laneError.message : "Failed to load eval lane details."}
mutationStatus="No corpus mutation occurred."
reproducer={`core eval --lane ${selectedLaneName}`}
retrySafety="Retry: safe"
/>
);
}
// 3. Current run execution state
if (currentRunState?.isPending) {
return (
<div className="flex flex-col gap-4">
{renderHeaderAndForm(selectedLane)}
<LoadingState label="Running eval lane..." />
</div>
);
}
if (currentRunState?.error) {
return (
<div className="flex flex-col gap-4">
{renderHeaderAndForm(selectedLane)}
<ErrorState
whatFailed={currentRunState.error.message}
mutationStatus="No corpus mutation occurred."
reproducer={`core eval --lane ${selectedLane.lane}`}
retrySafety="Retry: safe"
/>
</div>
);
}
// 4. Success or Empty state
const result = currentRunState?.result || (lastRunResult as EvalRunResult | undefined);
if (result && result.metrics) {
return (
<div className="flex flex-col gap-4">
{renderHeaderAndForm(selectedLane)}
<div className="flex flex-wrap items-center gap-3">
<span className="text-xs text-[var(--color-text-secondary)] font-semibold">
Status:
</span>
<span
className={`rounded-md px-2 py-0.5 text-xs font-semibold ${
result.passed
? "bg-[var(--color-state-success-bg)] text-[var(--color-state-success-text)] border border-[var(--color-state-success-border)]"
: "bg-[var(--color-state-danger-bg)] text-[var(--color-state-danger-text)] border border-[var(--color-state-danger-border)]"
}`}
>
{result.passed ? "Passed" : "Failed"}
</span>
{result.source_digest && (
<EvalArtifactLink
lane={selectedLane.lane}
sourceDigest={result.source_digest}
/>
)}
</div>
<div className="flex flex-col gap-2">
<h3 className="text-sm font-semibold text-[var(--color-text-primary)]">Metrics</h3>
<EvalMetricGrid metrics={result.metrics} />
</div>
<EvalFailureViewer
cases={result.cases}
passed={result.passed}
laneName={selectedLane.lane}
/>
</div>
);
}
return (
<div className="flex flex-col gap-4">
{renderHeaderAndForm(selectedLane)}
<EmptyState
statement={
selectedLane.read_only
? `No run results for lane "${selectedLane.lane}" in this session. Trigger a run above.`
: `Eval lane "${selectedLane.lane}" is CLI-only. No session results available.`
}
nextAction={{ kind: "cli", command: `core eval --lane ${selectedLane.lane}` }}
/>
</div>
);
};
return ( return (
<div className="grid h-full grid-cols-1 gap-4 md:grid-cols-[18rem_1fr]" data-testid="evals-route"> <div className="grid h-full grid-cols-1 gap-4 md:grid-cols-[18rem_1fr]" data-testid="evals-route">
@ -73,115 +275,7 @@ export function EvalsRoute() {
{/* Right Pane: Results / Form */} {/* Right Pane: Results / Form */}
<div className="flex flex-col gap-4 overflow-y-auto pl-2"> <div className="flex flex-col gap-4 overflow-y-auto pl-2">
{selectedLane ? ( {renderRightPane()}
<>
{/* Header info */}
<div className="flex flex-wrap items-center justify-between gap-3 border-b border-[var(--color-border-subtle)] pb-3">
<div>
<h2 className="text-lg font-semibold text-[var(--color-text-primary)] font-mono">
{selectedLane.lane}
</h2>
{selectedLane.description && (
<p className="mt-1 text-xs text-[var(--color-text-secondary)]">
{selectedLane.description}
</p>
)}
</div>
</div>
{/* Run Button configuration if read-only */}
{selectedLane.read_only ? (
<EvalRunButton
lane={selectedLane}
onRunStart={() => {
setRunStates((prev) => ({
...prev,
[selectedLane.lane]: { isPending: true },
}));
}}
onRunSuccess={(result) => {
setRunStates((prev) => ({
...prev,
[selectedLane.lane]: { isPending: false, result },
}));
}}
onRunError={(err) => {
setRunStates((prev) => ({
...prev,
[selectedLane.lane]: { isPending: false, error: err },
}));
}}
/>
) : (
<div className="text-xs text-[var(--color-text-muted)] bg-[var(--color-surface-inset)] p-3 rounded-lg border border-[var(--color-border-subtle)] font-medium">
API runs disabled for write-active lane. Use local CLI to execute this lane:
<code className="block mt-1 font-mono text-[var(--color-text-primary)] bg-[var(--color-surface-base)] p-1.5 rounded">
core eval --lane {selectedLane.lane}
</code>
</div>
)}
{/* Result display */}
{currentRunState?.isPending ? (
<LoadingState label="Running eval lane..." />
) : currentRunState?.error ? (
<ErrorState
whatFailed={currentRunState.error.message}
mutationStatus="No corpus mutation occurred."
reproducer={`core eval --lane ${selectedLane.lane}`}
retrySafety="Retry: safe"
/>
) : currentRunState?.result ? (
<div className="flex flex-col gap-4">
<div className="flex flex-wrap items-center gap-3">
<span className="text-xs text-[var(--color-text-secondary)] font-semibold">
Status:
</span>
<span
className={`rounded-md px-2 py-0.5 text-xs font-semibold ${
currentRunState.result.passed
? "bg-[var(--color-state-success-bg)] text-[var(--color-state-success-text)] border border-[var(--color-state-success-border)]"
: "bg-[var(--color-state-danger-bg)] text-[var(--color-state-danger-text)] border border-[var(--color-state-danger-border)]"
}`}
>
{currentRunState.result.passed ? "Passed" : "Failed"}
</span>
{currentRunState.result.source_digest && (
<EvalArtifactLink
lane={selectedLane.lane}
sourceDigest={currentRunState.result.source_digest}
/>
)}
</div>
<div className="flex flex-col gap-2">
<h3 className="text-sm font-semibold text-[var(--color-text-primary)]">Metrics</h3>
<EvalMetricGrid metrics={currentRunState.result.metrics} />
</div>
<EvalFailureViewer
cases={currentRunState.result.cases}
passed={currentRunState.result.passed}
laneName={selectedLane.lane}
/>
</div>
) : (
<EmptyState
statement={
selectedLane.read_only
? `No run results for lane "${selectedLane.lane}" in this session. Trigger a run above.`
: `Eval lane "${selectedLane.lane}" is CLI-only. No session results available.`
}
nextAction={{ kind: "cli", command: `core eval --lane ${selectedLane.lane}` }}
/>
)}
</>
) : (
<EmptyState
statement="Select an eval lane from the list to view results or run checks."
nextAction={{ kind: "cli", command: "core eval --list" }}
/>
)}
</div> </div>
</div> </div>
); );

View file

@ -16,11 +16,13 @@ vi.mock("../../api/queries", async (importOriginal) => {
return { return {
...actual, ...actual,
useEvalLanes: vi.fn(), useEvalLanes: vi.fn(),
useEvalLane: vi.fn(),
useEvalRun: vi.fn(), useEvalRun: vi.fn(),
}; };
}); });
import { useEvalLanes, useEvalRun } from "../../api/queries"; import { useEvalLanes, useEvalLane, useEvalRun } from "../../api/queries";
import { CommandPalette } from "../../design/components/primitives/CommandPalette";
const mockLanes: EvalLaneSummary[] = [ const mockLanes: EvalLaneSummary[] = [
{ lane: "contemplation_quality", versions: ["v1", "v2"], read_only: true, description: "Contemplation checks" }, { lane: "contemplation_quality", versions: ["v1", "v2"], read_only: true, description: "Contemplation checks" },
@ -49,6 +51,27 @@ describe("W-030 Component Tests", () => {
beforeEach(() => { beforeEach(() => {
vi.resetAllMocks(); vi.resetAllMocks();
vi.mocked(useEvalLanes).mockReturnValue({
data: mockLanes,
isLoading: false,
isError: false,
error: null,
} as any);
vi.mocked(useEvalLane).mockReturnValue({
data: undefined,
isLoading: false,
isError: true,
error: new WorkbenchApiError("not_found", "No run history found"),
} as any);
vi.mocked(useEvalRun).mockReturnValue({
mutate: vi.fn(),
isPending: false,
isError: false,
isSuccess: false,
error: null,
} as any);
fetchMock.mockImplementation((url: any) => { fetchMock.mockImplementation((url: any) => {
const urlStr = typeof url === "string" ? url : String(url?.url || url || ""); const urlStr = typeof url === "string" ? url : String(url?.url || url || "");
if (urlStr.endsWith("/evals")) { if (urlStr.endsWith("/evals")) {
@ -162,10 +185,45 @@ describe("W-030 Component Tests", () => {
// Verify lexicographical order // Verify lexicographical order
expect(cards1).toEqual(cards2); expect(cards1).toEqual(cards2);
expect(cards1.length).toBe(3);
expect(cards1[0]).toContain("accuracy"); expect(cards1[0]).toContain("accuracy");
expect(cards1[1]).toContain("latency ms"); expect(cards1[1]).toContain("latency ms");
expect(cards1[2]).toContain("passed"); expect(cards1[2]).toContain("passed");
expect(cards1[3]).toContain("total"); expect(cards1[2]).toContain("9/10");
});
it("renders actual / target, emphasizes actual, and applies failure color token when target is not met", () => {
const metrics = {
passed: 4,
total: 5,
latency_ms: 150,
latency_ms_target: 100, // failing because actual > target
accuracy: 0.9,
accuracy_target: 0.95, // failing because actual < target
};
const { container } = render(<EvalMetricGrid metrics={metrics} />);
const cards = container.querySelectorAll('[data-testid="metric-card"]');
expect(cards.length).toBe(3);
const passedCard = Array.from(cards).find(c => c.textContent?.includes("passed"));
expect(passedCard).toBeDefined();
expect(passedCard?.querySelector(".text-\\[var\\(--color-state-contradicted\\)\\]")).toBeInTheDocument();
expect(passedCard?.textContent).toContain("4");
expect(passedCard?.textContent).toContain("5");
const latencyCard = Array.from(cards).find(c => c.textContent?.includes("latency ms"));
expect(latencyCard).toBeDefined();
expect(latencyCard?.querySelector(".text-\\[var\\(--color-state-contradicted\\)\\]")).toBeInTheDocument();
expect(latencyCard?.textContent).toContain("150");
expect(latencyCard?.textContent).toContain("100");
const accuracyCard = Array.from(cards).find(c => c.textContent?.includes("accuracy"));
expect(accuracyCard).toBeDefined();
expect(accuracyCard?.querySelector(".text-\\[var\\(--color-state-contradicted\\)\\]")).toBeInTheDocument();
expect(accuracyCard?.textContent).toContain("0.9");
expect(accuracyCard?.textContent).toContain("0.95");
}); });
}); });
@ -295,5 +353,115 @@ describe("W-030 Component Tests", () => {
const res = await runEvalLane({ lane: "contemplation_quality", split: "holdout" }); const res = await runEvalLane({ lane: "contemplation_quality", split: "holdout" });
expect(res).toEqual(mockResult); expect(res).toEqual(mockResult);
}); });
it("renders loading state when lane list is loading", () => {
vi.mocked(useEvalLanes).mockReturnValue({
data: null,
isLoading: true,
isError: false,
error: null,
} as any);
const client = makeClient();
render(
<QueryClientProvider client={client}>
<MemoryRouter initialEntries={["/evals"]}>
<EvalsRoute />
</MemoryRouter>
</QueryClientProvider>
);
expect(screen.getByText("Loading eval lanes...")).toBeInTheDocument();
});
it("renders empty state when there are no lanes", () => {
vi.mocked(useEvalLanes).mockReturnValue({
data: [],
isLoading: false,
isError: false,
error: null,
} as any);
const client = makeClient();
render(
<QueryClientProvider client={client}>
<MemoryRouter initialEntries={["/evals"]}>
<EvalsRoute />
</MemoryRouter>
</QueryClientProvider>
);
expect(screen.getByText("No eval lanes discovered.")).toBeInTheDocument();
});
it("renders loading and error states for the selected lane detail view", () => {
vi.mocked(useEvalLanes).mockReturnValue({
data: mockLanes,
isLoading: false,
isError: false,
error: null,
} as any);
// Mock useEvalLane to return loading
vi.mocked(useEvalLane).mockReturnValue({
data: undefined,
isLoading: true,
isError: false,
error: null,
} as any);
const client = makeClient();
const { rerender } = render(
<QueryClientProvider client={client}>
<MemoryRouter initialEntries={["/evals?lane=contemplation_quality"]}>
<EvalsRoute />
</MemoryRouter>
</QueryClientProvider>
);
expect(screen.getByText("Loading eval lane details...")).toBeInTheDocument();
// Mock useEvalLane to return error (other than not_found)
vi.mocked(useEvalLane).mockReturnValue({
data: undefined,
isLoading: false,
isError: true,
error: new WorkbenchApiError("read_error", "Disk read error"),
} as any);
rerender(
<QueryClientProvider client={client}>
<MemoryRouter initialEntries={["/evals?lane=contemplation_quality"]}>
<EvalsRoute />
</MemoryRouter>
</QueryClientProvider>
);
expect(screen.getByText("Disk read error")).toBeInTheDocument();
});
it("CommandPalette registers dynamic command entries driven by useEvalLanes", () => {
vi.mocked(useEvalLanes).mockReturnValue({
data: mockLanes,
isLoading: false,
isError: false,
error: null,
} as any);
const client = makeClient();
render(
<QueryClientProvider client={client}>
<MemoryRouter>
<CommandPalette open={true} onOpenChange={vi.fn()} />
</MemoryRouter>
</QueryClientProvider>
);
// Verify static commands
expect(screen.getByRole("button", { name: "Open Chat" })).toBeInTheDocument();
// Verify dynamic commands
expect(screen.getByRole("button", { name: "Open eval lane contemplation_quality" })).toBeInTheDocument();
expect(screen.getByRole("button", { name: "Open eval lane unsafe_lane" })).toBeInTheDocument();
});
}); });
}); });

View file

@ -1,10 +1,18 @@
import { render, screen } from "@testing-library/react"; import { render, screen } from "@testing-library/react";
import userEvent from "@testing-library/user-event"; import userEvent from "@testing-library/user-event";
import { describe, expect, it } from "vitest"; import { describe, expect, it, vi } from "vitest";
import { MemoryRouter } from "react-router-dom"; import { MemoryRouter } from "react-router-dom";
import { useState } from "react"; import { useState } from "react";
import { CommandPalette } from "./CommandPalette"; import { CommandPalette } from "./CommandPalette";
vi.mock("../../../api/queries", () => ({
useEvalLanes: () => ({
data: [],
isLoading: false,
isError: false,
}),
}));
function PaletteHarness({ initialOpen = false }: { initialOpen?: boolean }) { function PaletteHarness({ initialOpen = false }: { initialOpen?: boolean }) {
const [open, setOpen] = useState(initialOpen); const [open, setOpen] = useState(initialOpen);
return ( return (

View file

@ -2,13 +2,14 @@ import * as Dialog from "@radix-ui/react-dialog";
import { Search } from "lucide-react"; import { Search } from "lucide-react";
import { useRef, useState, useEffect, useCallback } from "react"; import { useRef, useState, useEffect, useCallback } from "react";
import { useNavigate, useInRouterContext } from "react-router-dom"; import { useNavigate, useInRouterContext } from "react-router-dom";
import { useEvalLanes } from "../../../api/queries";
interface Command { interface Command {
name: string; name: string;
path: string; path: string;
} }
const COMMANDS: Command[] = [ const STATIC_COMMANDS: Command[] = [
{ name: "Open Chat", path: "/chat" }, { name: "Open Chat", path: "/chat" },
{ name: "Open Proposals", path: "/proposals" }, { name: "Open Proposals", path: "/proposals" },
{ name: "Open Evals", path: "/evals" }, { name: "Open Evals", path: "/evals" },
@ -20,6 +21,15 @@ function RouterCommandPalette(props: {
onOpenChange: (open: boolean) => void; onOpenChange: (open: boolean) => void;
}) { }) {
const navigate = useNavigate(); const navigate = useNavigate();
const { data: lanes } = useEvalLanes();
const dynamicCommands = (lanes || []).map((lane) => ({
name: `Open eval lane ${lane.lane}`,
path: `/evals?lane=${lane.lane}`,
}));
const commands = [...STATIC_COMMANDS, ...dynamicCommands];
const activate = useCallback( const activate = useCallback(
(cmd: Command) => { (cmd: Command) => {
navigate(cmd.path); navigate(cmd.path);
@ -27,7 +37,7 @@ function RouterCommandPalette(props: {
}, },
[navigate, props], [navigate, props],
); );
return <CommandPaletteContent {...props} onActivate={activate} />; return <CommandPaletteContent {...props} commands={commands} onActivate={activate} />;
} }
// Fallback for design-system preview (no Router). // Fallback for design-system preview (no Router).
@ -41,23 +51,25 @@ function FallbackCommandPalette(props: {
}, },
[props], [props],
); );
return <CommandPaletteContent {...props} onActivate={activate} />; return <CommandPaletteContent {...props} commands={STATIC_COMMANDS} onActivate={activate} />;
} }
function CommandPaletteContent({ function CommandPaletteContent({
open, open,
onOpenChange, onOpenChange,
onActivate, onActivate,
commands,
}: { }: {
open: boolean; open: boolean;
onOpenChange: (open: boolean) => void; onOpenChange: (open: boolean) => void;
onActivate: (cmd: Command) => void; onActivate: (cmd: Command) => void;
commands: Command[];
}) { }) {
const [query, setQuery] = useState(""); const [query, setQuery] = useState("");
const [focusedIndex, setFocusedIndex] = useState(0); const [focusedIndex, setFocusedIndex] = useState(0);
const inputRef = useRef<HTMLInputElement>(null); const inputRef = useRef<HTMLInputElement>(null);
const filtered = COMMANDS.filter((cmd) => const filtered = commands.filter((cmd) =>
cmd.name.toLowerCase().includes(query.toLowerCase()), cmd.name.toLowerCase().includes(query.toLowerCase()),
); );