feat(workbench/evals): polish states, numeric alignment, lane URL state (wave 1d)
This commit is contained in:
Shay
parent
e954c660f7
commit
37b04d504b
5 changed files with 560 additions and 138 deletions
|
|
@ -5,7 +5,7 @@ export function EvalMetricGrid({
|
|||
}: {
|
||||
metrics: Record<string, unknown>;
|
||||
}) {
|
||||
const sortedKeys = Object.keys(metrics).sort();
|
||||
const allKeys = Object.keys(metrics);
|
||||
|
||||
function formatValue(value: unknown): string {
|
||||
if (typeof value === "boolean") {
|
||||
|
|
@ -34,6 +34,20 @@ export function EvalMetricGrid({
|
|||
return undefined;
|
||||
}
|
||||
|
||||
function isLowerIsBetter(key: string): boolean {
|
||||
const k = key.toLowerCase();
|
||||
return (
|
||||
k.includes("latency") ||
|
||||
k.includes("duration") ||
|
||||
k.includes("time") ||
|
||||
k.includes("error") ||
|
||||
k.includes("fail") ||
|
||||
k.includes("fabrication") ||
|
||||
k.includes("divergence") ||
|
||||
k.includes("cost")
|
||||
);
|
||||
}
|
||||
|
||||
function renderPassFailBadge(key: string, value: unknown) {
|
||||
const isBool = typeof value === "boolean";
|
||||
const k = key.toLowerCase();
|
||||
|
|
@ -65,33 +79,159 @@ export function EvalMetricGrid({
|
|||
return null;
|
||||
}
|
||||
|
||||
// Find pairs
|
||||
const pairedKeys = new Set<string>();
|
||||
const pairs: Array<{
|
||||
key: string;
|
||||
actual: unknown;
|
||||
target: unknown;
|
||||
}> = [];
|
||||
|
||||
// 1. Group "passed" and "total"
|
||||
if (allKeys.includes("passed") && allKeys.includes("total")) {
|
||||
pairs.push({
|
||||
key: "passed",
|
||||
actual: metrics["passed"],
|
||||
target: metrics["total"],
|
||||
});
|
||||
pairedKeys.add("passed");
|
||||
pairedKeys.add("total");
|
||||
}
|
||||
|
||||
// 2. Group "X" and "X_target" or "target_X"
|
||||
allKeys.forEach((k) => {
|
||||
if (pairedKeys.has(k)) return;
|
||||
|
||||
if (k.endsWith("_target")) {
|
||||
const baseKey = k.slice(0, -7);
|
||||
if (allKeys.includes(baseKey)) {
|
||||
pairs.push({
|
||||
key: baseKey,
|
||||
actual: metrics[baseKey],
|
||||
target: metrics[k],
|
||||
});
|
||||
pairedKeys.add(baseKey);
|
||||
pairedKeys.add(k);
|
||||
}
|
||||
} else if (k.startsWith("target_")) {
|
||||
const baseKey = k.slice(7);
|
||||
if (allKeys.includes(baseKey)) {
|
||||
pairs.push({
|
||||
key: baseKey,
|
||||
actual: metrics[baseKey],
|
||||
target: metrics[k],
|
||||
});
|
||||
pairedKeys.add(baseKey);
|
||||
pairedKeys.add(k);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// 3. Remaining singles
|
||||
const singles: Array<{ key: string; value: unknown }> = [];
|
||||
allKeys.forEach((k) => {
|
||||
if (!pairedKeys.has(k)) {
|
||||
singles.push({ key: k, value: metrics[k] });
|
||||
}
|
||||
});
|
||||
|
||||
interface DisplayMetric {
|
||||
key: string;
|
||||
isPair: boolean;
|
||||
actual: unknown;
|
||||
target?: unknown;
|
||||
value?: unknown;
|
||||
}
|
||||
|
||||
const list: DisplayMetric[] = [
|
||||
...pairs.map((p) => ({ key: p.key, isPair: true, actual: p.actual, target: p.target })),
|
||||
...singles.map((s) => ({ key: s.key, isPair: false, actual: s.value, value: s.value })),
|
||||
];
|
||||
|
||||
// Sort lexicographically
|
||||
list.sort((a, b) => a.key.localeCompare(b.key));
|
||||
|
||||
return (
|
||||
<div className="grid gap-3 sm:grid-cols-2 md:grid-cols-3" data-testid="eval-metric-grid">
|
||||
{sortedKeys.map((key) => {
|
||||
const val = metrics[key];
|
||||
{list.map((item) => {
|
||||
const key = item.key;
|
||||
const unit = getUnit(key);
|
||||
const badge = renderPassFailBadge(key, val);
|
||||
const formatted = formatValue(val);
|
||||
const isObj = typeof val === "object" && val !== null;
|
||||
|
||||
return (
|
||||
<div
|
||||
key={key}
|
||||
className="flex flex-col justify-between rounded-lg border border-[var(--color-border-subtle)] bg-[var(--color-surface-raised)] p-3 shadow-sm"
|
||||
data-testid="metric-card"
|
||||
>
|
||||
<div>
|
||||
<div className="text-[10px] font-semibold text-[var(--color-text-muted)] uppercase tracking-wider truncate" title={key}>
|
||||
{key.replaceAll("_", " ")}
|
||||
</div>
|
||||
<div className={`mt-1 font-mono text-sm font-semibold text-[var(--color-text-primary)] ${isObj ? "whitespace-pre overflow-x-auto text-[10px] bg-[var(--color-surface-inset)] p-1.5 rounded" : ""}`}>
|
||||
{formatted}
|
||||
{unit && <span className="ml-1 text-xs text-[var(--color-text-muted)] font-sans font-normal">{unit}</span>}
|
||||
if (item.isPair) {
|
||||
const actualVal = item.actual;
|
||||
const targetVal = item.target;
|
||||
const badge = renderPassFailBadge(key, actualVal);
|
||||
|
||||
const actualNum = Number(actualVal);
|
||||
const targetNum = Number(targetVal);
|
||||
const hasNumericValues = !isNaN(actualNum) && !isNaN(targetNum);
|
||||
const isFailing =
|
||||
hasNumericValues &&
|
||||
(isLowerIsBetter(key) ? actualNum > targetNum : actualNum < targetNum);
|
||||
|
||||
const formattedActual = formatValue(actualVal);
|
||||
const formattedTarget = formatValue(targetVal);
|
||||
|
||||
return (
|
||||
<div
|
||||
key={key}
|
||||
className="flex flex-col justify-between rounded-lg border border-[var(--color-border-subtle)] bg-[var(--color-surface-raised)] p-3 shadow-sm"
|
||||
data-testid="metric-card"
|
||||
>
|
||||
<div>
|
||||
<div className="text-[10px] font-semibold text-[var(--color-text-muted)] uppercase tracking-wider truncate" title={key}>
|
||||
{key.replaceAll("_", " ")}
|
||||
</div>
|
||||
<div className="mt-1 font-mono text-sm font-semibold text-right tabular-nums">
|
||||
<span
|
||||
className={
|
||||
isFailing
|
||||
? "text-[var(--color-state-contradicted)] font-bold"
|
||||
: "text-[var(--color-text-primary)]"
|
||||
}
|
||||
>
|
||||
{formattedActual}
|
||||
</span>
|
||||
<span className="text-[var(--color-text-muted)] mx-1">/</span>
|
||||
<span className="text-[var(--color-text-secondary)] font-normal">
|
||||
{formattedTarget}
|
||||
</span>
|
||||
{unit && <span className="ml-1 text-xs text-[var(--color-text-muted)] font-sans font-normal">{unit}</span>}
|
||||
</div>
|
||||
</div>
|
||||
{badge && <div className="mt-2 flex justify-start">{badge}</div>}
|
||||
</div>
|
||||
{badge && <div className="mt-2 flex justify-start">{badge}</div>}
|
||||
</div>
|
||||
);
|
||||
);
|
||||
} else {
|
||||
const val = item.value;
|
||||
const badge = renderPassFailBadge(key, val);
|
||||
const formatted = formatValue(val);
|
||||
const isObj = typeof val === "object" && val !== null;
|
||||
const isNum = typeof val === "number";
|
||||
|
||||
return (
|
||||
<div
|
||||
key={key}
|
||||
className="flex flex-col justify-between rounded-lg border border-[var(--color-border-subtle)] bg-[var(--color-surface-raised)] p-3 shadow-sm"
|
||||
data-testid="metric-card"
|
||||
>
|
||||
<div>
|
||||
<div className="text-[10px] font-semibold text-[var(--color-text-muted)] uppercase tracking-wider truncate" title={key}>
|
||||
{key.replaceAll("_", " ")}
|
||||
</div>
|
||||
<div
|
||||
className={`mt-1 font-mono text-sm font-semibold text-[var(--color-text-primary)] ${
|
||||
isNum ? "text-right tabular-nums" : ""
|
||||
} ${isObj ? "whitespace-pre overflow-x-auto text-[10px] bg-[var(--color-surface-inset)] p-1.5 rounded" : ""}`}
|
||||
>
|
||||
{formatted}
|
||||
{unit && <span className="ml-1 text-xs text-[var(--color-text-muted)] font-sans font-normal">{unit}</span>}
|
||||
</div>
|
||||
</div>
|
||||
{badge && <div className="mt-2 flex justify-start">{badge}</div>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -12,10 +12,33 @@ import { LoadingState } from "../../design/components/states/LoadingState";
|
|||
import type { EvalRunResult } from "../../types/api";
|
||||
import { WorkbenchApiError } from "../../api/client";
|
||||
|
||||
import { useState, useEffect } from "react";
|
||||
import { useSearchParams } from "react-router-dom";
|
||||
import { useEvalLanes, useEvalLane } from "../../api/queries";
|
||||
import { EvalLaneCard } from "./EvalLaneCard";
|
||||
import { EvalRunButton } from "./EvalRunButton";
|
||||
import { EvalMetricGrid } from "./EvalMetricGrid";
|
||||
import { EvalFailureViewer } from "./EvalFailureViewer";
|
||||
import { EvalArtifactLink } from "./EvalArtifactLink";
|
||||
import { EmptyState } from "../../design/components/states/EmptyState";
|
||||
import { ErrorState } from "../../design/components/states/ErrorState";
|
||||
import { LoadingState } from "../../design/components/states/LoadingState";
|
||||
import type { EvalRunResult } from "../../types/api";
|
||||
import { WorkbenchApiError } from "../../api/client";
|
||||
import { useQueryClient } from "@tanstack/react-query";
|
||||
|
||||
export function EvalsRoute() {
|
||||
const { data: lanes, isLoading, isError, error } = useEvalLanes();
|
||||
const [searchParams, setSearchParams] = useSearchParams();
|
||||
const selectedLaneName = searchParams.get("lane") || "";
|
||||
const queryClient = useQueryClient();
|
||||
|
||||
const {
|
||||
data: lastRunResult,
|
||||
isLoading: isLaneLoading,
|
||||
isError: isLaneError,
|
||||
error: laneError,
|
||||
} = useEvalLane(selectedLaneName);
|
||||
|
||||
// Maintain per-lane run states (pending, result, error)
|
||||
const [runStates, setRunStates] = useState<
|
||||
|
|
@ -29,6 +52,13 @@ export function EvalsRoute() {
|
|||
>
|
||||
>({});
|
||||
|
||||
// Auto-select the first lane on load if no selection is in the URL
|
||||
useEffect(() => {
|
||||
if (!selectedLaneName && lanes && lanes.length > 0) {
|
||||
setSearchParams({ lane: lanes[0].lane }, { replace: true });
|
||||
}
|
||||
}, [selectedLaneName, lanes, setSearchParams]);
|
||||
|
||||
if (isLoading) {
|
||||
return <LoadingState label="Loading eval lanes..." />;
|
||||
}
|
||||
|
|
@ -46,6 +76,178 @@ export function EvalsRoute() {
|
|||
|
||||
const selectedLane = lanes?.find((l) => l.lane === selectedLaneName) || null;
|
||||
const currentRunState = selectedLaneName ? runStates[selectedLaneName] : null;
|
||||
const isNotFoundError =
|
||||
laneError instanceof WorkbenchApiError && laneError.code === "not_found";
|
||||
|
||||
function renderHeaderAndForm(lane: typeof selectedLane) {
|
||||
if (!lane) return null;
|
||||
return (
|
||||
<>
|
||||
{/* Header info */}
|
||||
<div className="flex flex-wrap items-center justify-between gap-3 border-b border-[var(--color-border-subtle)] pb-3">
|
||||
<div>
|
||||
<h2 className="text-lg font-semibold text-[var(--color-text-primary)] font-mono">
|
||||
{lane.lane}
|
||||
</h2>
|
||||
{lane.description && (
|
||||
<p className="mt-1 text-xs text-[var(--color-text-secondary)]">
|
||||
{lane.description}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Run Button configuration if read-only */}
|
||||
{lane.read_only ? (
|
||||
<EvalRunButton
|
||||
lane={lane}
|
||||
onRunStart={() => {
|
||||
setRunStates((prev) => ({
|
||||
...prev,
|
||||
[lane.lane]: { isPending: true },
|
||||
}));
|
||||
}}
|
||||
onRunSuccess={(result) => {
|
||||
setRunStates((prev) => ({
|
||||
...prev,
|
||||
[lane.lane]: { isPending: false, result },
|
||||
}));
|
||||
queryClient.invalidateQueries({ queryKey: ["api", "eval", lane.lane] });
|
||||
}}
|
||||
onRunError={(err) => {
|
||||
setRunStates((prev) => ({
|
||||
...prev,
|
||||
[lane.lane]: { isPending: false, error: err },
|
||||
}));
|
||||
}}
|
||||
/>
|
||||
) : (
|
||||
<div className="text-xs text-[var(--color-text-muted)] bg-[var(--color-surface-inset)] p-3 rounded-lg border border-[var(--color-border-subtle)] font-medium">
|
||||
⚠️ API runs disabled for write-active lane. Use local CLI to execute this lane:
|
||||
<code className="block mt-1 font-mono text-[var(--color-text-primary)] bg-[var(--color-surface-base)] p-1.5 rounded">
|
||||
core eval --lane {lane.lane}
|
||||
</code>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
const renderRightPane = () => {
|
||||
if (!selectedLaneName) {
|
||||
return (
|
||||
<EmptyState
|
||||
statement="Select an eval lane from the list to view results or run checks."
|
||||
nextAction={{ kind: "cli", command: "core eval --list" }}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
if (!selectedLane) {
|
||||
return (
|
||||
<EmptyState
|
||||
statement={`Selected lane "${selectedLaneName}" not found.`}
|
||||
nextAction={{ kind: "cli", command: "core eval --list" }}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
// 1. Loading state for fetching the lane's last run details
|
||||
if (isLaneLoading && !lastRunResult) {
|
||||
return <LoadingState label="Loading eval lane details..." />;
|
||||
}
|
||||
|
||||
// 2. Error state for fetching the lane details (excluding not_found)
|
||||
if (isLaneError && !isNotFoundError) {
|
||||
return (
|
||||
<ErrorState
|
||||
whatFailed={laneError instanceof Error ? laneError.message : "Failed to load eval lane details."}
|
||||
mutationStatus="No corpus mutation occurred."
|
||||
reproducer={`core eval --lane ${selectedLaneName}`}
|
||||
retrySafety="Retry: safe"
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
// 3. Current run execution state
|
||||
if (currentRunState?.isPending) {
|
||||
return (
|
||||
<div className="flex flex-col gap-4">
|
||||
{renderHeaderAndForm(selectedLane)}
|
||||
<LoadingState label="Running eval lane..." />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (currentRunState?.error) {
|
||||
return (
|
||||
<div className="flex flex-col gap-4">
|
||||
{renderHeaderAndForm(selectedLane)}
|
||||
<ErrorState
|
||||
whatFailed={currentRunState.error.message}
|
||||
mutationStatus="No corpus mutation occurred."
|
||||
reproducer={`core eval --lane ${selectedLane.lane}`}
|
||||
retrySafety="Retry: safe"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// 4. Success or Empty state
|
||||
const result = currentRunState?.result || (lastRunResult as EvalRunResult | undefined);
|
||||
if (result && result.metrics) {
|
||||
return (
|
||||
<div className="flex flex-col gap-4">
|
||||
{renderHeaderAndForm(selectedLane)}
|
||||
<div className="flex flex-wrap items-center gap-3">
|
||||
<span className="text-xs text-[var(--color-text-secondary)] font-semibold">
|
||||
Status:
|
||||
</span>
|
||||
<span
|
||||
className={`rounded-md px-2 py-0.5 text-xs font-semibold ${
|
||||
result.passed
|
||||
? "bg-[var(--color-state-success-bg)] text-[var(--color-state-success-text)] border border-[var(--color-state-success-border)]"
|
||||
: "bg-[var(--color-state-danger-bg)] text-[var(--color-state-danger-text)] border border-[var(--color-state-danger-border)]"
|
||||
}`}
|
||||
>
|
||||
{result.passed ? "Passed" : "Failed"}
|
||||
</span>
|
||||
{result.source_digest && (
|
||||
<EvalArtifactLink
|
||||
lane={selectedLane.lane}
|
||||
sourceDigest={result.source_digest}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-2">
|
||||
<h3 className="text-sm font-semibold text-[var(--color-text-primary)]">Metrics</h3>
|
||||
<EvalMetricGrid metrics={result.metrics} />
|
||||
</div>
|
||||
|
||||
<EvalFailureViewer
|
||||
cases={result.cases}
|
||||
passed={result.passed}
|
||||
laneName={selectedLane.lane}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-4">
|
||||
{renderHeaderAndForm(selectedLane)}
|
||||
<EmptyState
|
||||
statement={
|
||||
selectedLane.read_only
|
||||
? `No run results for lane "${selectedLane.lane}" in this session. Trigger a run above.`
|
||||
: `Eval lane "${selectedLane.lane}" is CLI-only. No session results available.`
|
||||
}
|
||||
nextAction={{ kind: "cli", command: `core eval --lane ${selectedLane.lane}` }}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="grid h-full grid-cols-1 gap-4 md:grid-cols-[18rem_1fr]" data-testid="evals-route">
|
||||
|
|
@ -73,115 +275,7 @@ export function EvalsRoute() {
|
|||
|
||||
{/* Right Pane: Results / Form */}
|
||||
<div className="flex flex-col gap-4 overflow-y-auto pl-2">
|
||||
{selectedLane ? (
|
||||
<>
|
||||
{/* Header info */}
|
||||
<div className="flex flex-wrap items-center justify-between gap-3 border-b border-[var(--color-border-subtle)] pb-3">
|
||||
<div>
|
||||
<h2 className="text-lg font-semibold text-[var(--color-text-primary)] font-mono">
|
||||
{selectedLane.lane}
|
||||
</h2>
|
||||
{selectedLane.description && (
|
||||
<p className="mt-1 text-xs text-[var(--color-text-secondary)]">
|
||||
{selectedLane.description}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Run Button configuration if read-only */}
|
||||
{selectedLane.read_only ? (
|
||||
<EvalRunButton
|
||||
lane={selectedLane}
|
||||
onRunStart={() => {
|
||||
setRunStates((prev) => ({
|
||||
...prev,
|
||||
[selectedLane.lane]: { isPending: true },
|
||||
}));
|
||||
}}
|
||||
onRunSuccess={(result) => {
|
||||
setRunStates((prev) => ({
|
||||
...prev,
|
||||
[selectedLane.lane]: { isPending: false, result },
|
||||
}));
|
||||
}}
|
||||
onRunError={(err) => {
|
||||
setRunStates((prev) => ({
|
||||
...prev,
|
||||
[selectedLane.lane]: { isPending: false, error: err },
|
||||
}));
|
||||
}}
|
||||
/>
|
||||
) : (
|
||||
<div className="text-xs text-[var(--color-text-muted)] bg-[var(--color-surface-inset)] p-3 rounded-lg border border-[var(--color-border-subtle)] font-medium">
|
||||
⚠️ API runs disabled for write-active lane. Use local CLI to execute this lane:
|
||||
<code className="block mt-1 font-mono text-[var(--color-text-primary)] bg-[var(--color-surface-base)] p-1.5 rounded">
|
||||
core eval --lane {selectedLane.lane}
|
||||
</code>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Result display */}
|
||||
{currentRunState?.isPending ? (
|
||||
<LoadingState label="Running eval lane..." />
|
||||
) : currentRunState?.error ? (
|
||||
<ErrorState
|
||||
whatFailed={currentRunState.error.message}
|
||||
mutationStatus="No corpus mutation occurred."
|
||||
reproducer={`core eval --lane ${selectedLane.lane}`}
|
||||
retrySafety="Retry: safe"
|
||||
/>
|
||||
) : currentRunState?.result ? (
|
||||
<div className="flex flex-col gap-4">
|
||||
<div className="flex flex-wrap items-center gap-3">
|
||||
<span className="text-xs text-[var(--color-text-secondary)] font-semibold">
|
||||
Status:
|
||||
</span>
|
||||
<span
|
||||
className={`rounded-md px-2 py-0.5 text-xs font-semibold ${
|
||||
currentRunState.result.passed
|
||||
? "bg-[var(--color-state-success-bg)] text-[var(--color-state-success-text)] border border-[var(--color-state-success-border)]"
|
||||
: "bg-[var(--color-state-danger-bg)] text-[var(--color-state-danger-text)] border border-[var(--color-state-danger-border)]"
|
||||
}`}
|
||||
>
|
||||
{currentRunState.result.passed ? "Passed" : "Failed"}
|
||||
</span>
|
||||
{currentRunState.result.source_digest && (
|
||||
<EvalArtifactLink
|
||||
lane={selectedLane.lane}
|
||||
sourceDigest={currentRunState.result.source_digest}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-2">
|
||||
<h3 className="text-sm font-semibold text-[var(--color-text-primary)]">Metrics</h3>
|
||||
<EvalMetricGrid metrics={currentRunState.result.metrics} />
|
||||
</div>
|
||||
|
||||
<EvalFailureViewer
|
||||
cases={currentRunState.result.cases}
|
||||
passed={currentRunState.result.passed}
|
||||
laneName={selectedLane.lane}
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
<EmptyState
|
||||
statement={
|
||||
selectedLane.read_only
|
||||
? `No run results for lane "${selectedLane.lane}" in this session. Trigger a run above.`
|
||||
: `Eval lane "${selectedLane.lane}" is CLI-only. No session results available.`
|
||||
}
|
||||
nextAction={{ kind: "cli", command: `core eval --lane ${selectedLane.lane}` }}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
) : (
|
||||
<EmptyState
|
||||
statement="Select an eval lane from the list to view results or run checks."
|
||||
nextAction={{ kind: "cli", command: "core eval --list" }}
|
||||
/>
|
||||
)}
|
||||
{renderRightPane()}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -16,11 +16,13 @@ vi.mock("../../api/queries", async (importOriginal) => {
|
|||
return {
|
||||
...actual,
|
||||
useEvalLanes: vi.fn(),
|
||||
useEvalLane: vi.fn(),
|
||||
useEvalRun: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
import { useEvalLanes, useEvalRun } from "../../api/queries";
|
||||
import { useEvalLanes, useEvalLane, useEvalRun } from "../../api/queries";
|
||||
import { CommandPalette } from "../../design/components/primitives/CommandPalette";
|
||||
|
||||
const mockLanes: EvalLaneSummary[] = [
|
||||
{ lane: "contemplation_quality", versions: ["v1", "v2"], read_only: true, description: "Contemplation checks" },
|
||||
|
|
@ -49,6 +51,27 @@ describe("W-030 Component Tests", () => {
|
|||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
|
||||
vi.mocked(useEvalLanes).mockReturnValue({
|
||||
data: mockLanes,
|
||||
isLoading: false,
|
||||
isError: false,
|
||||
error: null,
|
||||
} as any);
|
||||
vi.mocked(useEvalLane).mockReturnValue({
|
||||
data: undefined,
|
||||
isLoading: false,
|
||||
isError: true,
|
||||
error: new WorkbenchApiError("not_found", "No run history found"),
|
||||
} as any);
|
||||
vi.mocked(useEvalRun).mockReturnValue({
|
||||
mutate: vi.fn(),
|
||||
isPending: false,
|
||||
isError: false,
|
||||
isSuccess: false,
|
||||
error: null,
|
||||
} as any);
|
||||
|
||||
fetchMock.mockImplementation((url: any) => {
|
||||
const urlStr = typeof url === "string" ? url : String(url?.url || url || "");
|
||||
if (urlStr.endsWith("/evals")) {
|
||||
|
|
@ -162,10 +185,45 @@ describe("W-030 Component Tests", () => {
|
|||
|
||||
// Verify lexicographical order
|
||||
expect(cards1).toEqual(cards2);
|
||||
expect(cards1.length).toBe(3);
|
||||
expect(cards1[0]).toContain("accuracy");
|
||||
expect(cards1[1]).toContain("latency ms");
|
||||
expect(cards1[2]).toContain("passed");
|
||||
expect(cards1[3]).toContain("total");
|
||||
expect(cards1[2]).toContain("9/10");
|
||||
});
|
||||
|
||||
it("renders actual / target, emphasizes actual, and applies failure color token when target is not met", () => {
|
||||
const metrics = {
|
||||
passed: 4,
|
||||
total: 5,
|
||||
latency_ms: 150,
|
||||
latency_ms_target: 100, // failing because actual > target
|
||||
accuracy: 0.9,
|
||||
accuracy_target: 0.95, // failing because actual < target
|
||||
};
|
||||
|
||||
const { container } = render(<EvalMetricGrid metrics={metrics} />);
|
||||
|
||||
const cards = container.querySelectorAll('[data-testid="metric-card"]');
|
||||
expect(cards.length).toBe(3);
|
||||
|
||||
const passedCard = Array.from(cards).find(c => c.textContent?.includes("passed"));
|
||||
expect(passedCard).toBeDefined();
|
||||
expect(passedCard?.querySelector(".text-\\[var\\(--color-state-contradicted\\)\\]")).toBeInTheDocument();
|
||||
expect(passedCard?.textContent).toContain("4");
|
||||
expect(passedCard?.textContent).toContain("5");
|
||||
|
||||
const latencyCard = Array.from(cards).find(c => c.textContent?.includes("latency ms"));
|
||||
expect(latencyCard).toBeDefined();
|
||||
expect(latencyCard?.querySelector(".text-\\[var\\(--color-state-contradicted\\)\\]")).toBeInTheDocument();
|
||||
expect(latencyCard?.textContent).toContain("150");
|
||||
expect(latencyCard?.textContent).toContain("100");
|
||||
|
||||
const accuracyCard = Array.from(cards).find(c => c.textContent?.includes("accuracy"));
|
||||
expect(accuracyCard).toBeDefined();
|
||||
expect(accuracyCard?.querySelector(".text-\\[var\\(--color-state-contradicted\\)\\]")).toBeInTheDocument();
|
||||
expect(accuracyCard?.textContent).toContain("0.9");
|
||||
expect(accuracyCard?.textContent).toContain("0.95");
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -295,5 +353,115 @@ describe("W-030 Component Tests", () => {
|
|||
const res = await runEvalLane({ lane: "contemplation_quality", split: "holdout" });
|
||||
expect(res).toEqual(mockResult);
|
||||
});
|
||||
|
||||
it("renders loading state when lane list is loading", () => {
|
||||
vi.mocked(useEvalLanes).mockReturnValue({
|
||||
data: null,
|
||||
isLoading: true,
|
||||
isError: false,
|
||||
error: null,
|
||||
} as any);
|
||||
|
||||
const client = makeClient();
|
||||
render(
|
||||
<QueryClientProvider client={client}>
|
||||
<MemoryRouter initialEntries={["/evals"]}>
|
||||
<EvalsRoute />
|
||||
</MemoryRouter>
|
||||
</QueryClientProvider>
|
||||
);
|
||||
|
||||
expect(screen.getByText("Loading eval lanes...")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("renders empty state when there are no lanes", () => {
|
||||
vi.mocked(useEvalLanes).mockReturnValue({
|
||||
data: [],
|
||||
isLoading: false,
|
||||
isError: false,
|
||||
error: null,
|
||||
} as any);
|
||||
|
||||
const client = makeClient();
|
||||
render(
|
||||
<QueryClientProvider client={client}>
|
||||
<MemoryRouter initialEntries={["/evals"]}>
|
||||
<EvalsRoute />
|
||||
</MemoryRouter>
|
||||
</QueryClientProvider>
|
||||
);
|
||||
|
||||
expect(screen.getByText("No eval lanes discovered.")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("renders loading and error states for the selected lane detail view", () => {
|
||||
vi.mocked(useEvalLanes).mockReturnValue({
|
||||
data: mockLanes,
|
||||
isLoading: false,
|
||||
isError: false,
|
||||
error: null,
|
||||
} as any);
|
||||
|
||||
// Mock useEvalLane to return loading
|
||||
vi.mocked(useEvalLane).mockReturnValue({
|
||||
data: undefined,
|
||||
isLoading: true,
|
||||
isError: false,
|
||||
error: null,
|
||||
} as any);
|
||||
|
||||
const client = makeClient();
|
||||
const { rerender } = render(
|
||||
<QueryClientProvider client={client}>
|
||||
<MemoryRouter initialEntries={["/evals?lane=contemplation_quality"]}>
|
||||
<EvalsRoute />
|
||||
</MemoryRouter>
|
||||
</QueryClientProvider>
|
||||
);
|
||||
|
||||
expect(screen.getByText("Loading eval lane details...")).toBeInTheDocument();
|
||||
|
||||
// Mock useEvalLane to return error (other than not_found)
|
||||
vi.mocked(useEvalLane).mockReturnValue({
|
||||
data: undefined,
|
||||
isLoading: false,
|
||||
isError: true,
|
||||
error: new WorkbenchApiError("read_error", "Disk read error"),
|
||||
} as any);
|
||||
|
||||
rerender(
|
||||
<QueryClientProvider client={client}>
|
||||
<MemoryRouter initialEntries={["/evals?lane=contemplation_quality"]}>
|
||||
<EvalsRoute />
|
||||
</MemoryRouter>
|
||||
</QueryClientProvider>
|
||||
);
|
||||
|
||||
expect(screen.getByText("Disk read error")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("CommandPalette registers dynamic command entries driven by useEvalLanes", () => {
|
||||
vi.mocked(useEvalLanes).mockReturnValue({
|
||||
data: mockLanes,
|
||||
isLoading: false,
|
||||
isError: false,
|
||||
error: null,
|
||||
} as any);
|
||||
|
||||
const client = makeClient();
|
||||
render(
|
||||
<QueryClientProvider client={client}>
|
||||
<MemoryRouter>
|
||||
<CommandPalette open={true} onOpenChange={vi.fn()} />
|
||||
</MemoryRouter>
|
||||
</QueryClientProvider>
|
||||
);
|
||||
|
||||
// Verify static commands
|
||||
expect(screen.getByRole("button", { name: "Open Chat" })).toBeInTheDocument();
|
||||
// Verify dynamic commands
|
||||
expect(screen.getByRole("button", { name: "Open eval lane contemplation_quality" })).toBeInTheDocument();
|
||||
expect(screen.getByRole("button", { name: "Open eval lane unsafe_lane" })).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,10 +1,18 @@
|
|||
import { render, screen } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { MemoryRouter } from "react-router-dom";
|
||||
import { useState } from "react";
|
||||
import { CommandPalette } from "./CommandPalette";
|
||||
|
||||
vi.mock("../../../api/queries", () => ({
|
||||
useEvalLanes: () => ({
|
||||
data: [],
|
||||
isLoading: false,
|
||||
isError: false,
|
||||
}),
|
||||
}));
|
||||
|
||||
function PaletteHarness({ initialOpen = false }: { initialOpen?: boolean }) {
|
||||
const [open, setOpen] = useState(initialOpen);
|
||||
return (
|
||||
|
|
|
|||
|
|
@ -2,13 +2,14 @@ import * as Dialog from "@radix-ui/react-dialog";
|
|||
import { Search } from "lucide-react";
|
||||
import { useRef, useState, useEffect, useCallback } from "react";
|
||||
import { useNavigate, useInRouterContext } from "react-router-dom";
|
||||
import { useEvalLanes } from "../../../api/queries";
|
||||
|
||||
interface Command {
|
||||
name: string;
|
||||
path: string;
|
||||
}
|
||||
|
||||
const COMMANDS: Command[] = [
|
||||
const STATIC_COMMANDS: Command[] = [
|
||||
{ name: "Open Chat", path: "/chat" },
|
||||
{ name: "Open Proposals", path: "/proposals" },
|
||||
{ name: "Open Evals", path: "/evals" },
|
||||
|
|
@ -20,6 +21,15 @@ function RouterCommandPalette(props: {
|
|||
onOpenChange: (open: boolean) => void;
|
||||
}) {
|
||||
const navigate = useNavigate();
|
||||
const { data: lanes } = useEvalLanes();
|
||||
|
||||
const dynamicCommands = (lanes || []).map((lane) => ({
|
||||
name: `Open eval lane ${lane.lane}`,
|
||||
path: `/evals?lane=${lane.lane}`,
|
||||
}));
|
||||
|
||||
const commands = [...STATIC_COMMANDS, ...dynamicCommands];
|
||||
|
||||
const activate = useCallback(
|
||||
(cmd: Command) => {
|
||||
navigate(cmd.path);
|
||||
|
|
@ -27,7 +37,7 @@ function RouterCommandPalette(props: {
|
|||
},
|
||||
[navigate, props],
|
||||
);
|
||||
return <CommandPaletteContent {...props} onActivate={activate} />;
|
||||
return <CommandPaletteContent {...props} commands={commands} onActivate={activate} />;
|
||||
}
|
||||
|
||||
// Fallback for design-system preview (no Router).
|
||||
|
|
@ -41,23 +51,25 @@ function FallbackCommandPalette(props: {
|
|||
},
|
||||
[props],
|
||||
);
|
||||
return <CommandPaletteContent {...props} onActivate={activate} />;
|
||||
return <CommandPaletteContent {...props} commands={STATIC_COMMANDS} onActivate={activate} />;
|
||||
}
|
||||
|
||||
function CommandPaletteContent({
|
||||
open,
|
||||
onOpenChange,
|
||||
onActivate,
|
||||
commands,
|
||||
}: {
|
||||
open: boolean;
|
||||
onOpenChange: (open: boolean) => void;
|
||||
onActivate: (cmd: Command) => void;
|
||||
commands: Command[];
|
||||
}) {
|
||||
const [query, setQuery] = useState("");
|
||||
const [focusedIndex, setFocusedIndex] = useState(0);
|
||||
const inputRef = useRef<HTMLInputElement>(null);
|
||||
|
||||
const filtered = COMMANDS.filter((cmd) =>
|
||||
const filtered = commands.filter((cmd) =>
|
||||
cmd.name.toLowerCase().includes(query.toLowerCase()),
|
||||
);
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue