feat(workbench/evals): polish states, numeric alignment, lane URL state (wave 1d)
This commit is contained in:
parent
e954c660f7
commit
37b04d504b
5 changed files with 560 additions and 138 deletions
|
|
@ -5,7 +5,7 @@ export function EvalMetricGrid({
|
||||||
}: {
|
}: {
|
||||||
metrics: Record<string, unknown>;
|
metrics: Record<string, unknown>;
|
||||||
}) {
|
}) {
|
||||||
const sortedKeys = Object.keys(metrics).sort();
|
const allKeys = Object.keys(metrics);
|
||||||
|
|
||||||
function formatValue(value: unknown): string {
|
function formatValue(value: unknown): string {
|
||||||
if (typeof value === "boolean") {
|
if (typeof value === "boolean") {
|
||||||
|
|
@ -34,6 +34,20 @@ export function EvalMetricGrid({
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isLowerIsBetter(key: string): boolean {
|
||||||
|
const k = key.toLowerCase();
|
||||||
|
return (
|
||||||
|
k.includes("latency") ||
|
||||||
|
k.includes("duration") ||
|
||||||
|
k.includes("time") ||
|
||||||
|
k.includes("error") ||
|
||||||
|
k.includes("fail") ||
|
||||||
|
k.includes("fabrication") ||
|
||||||
|
k.includes("divergence") ||
|
||||||
|
k.includes("cost")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function renderPassFailBadge(key: string, value: unknown) {
|
function renderPassFailBadge(key: string, value: unknown) {
|
||||||
const isBool = typeof value === "boolean";
|
const isBool = typeof value === "boolean";
|
||||||
const k = key.toLowerCase();
|
const k = key.toLowerCase();
|
||||||
|
|
@ -65,33 +79,159 @@ export function EvalMetricGrid({
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Find pairs
|
||||||
|
const pairedKeys = new Set<string>();
|
||||||
|
const pairs: Array<{
|
||||||
|
key: string;
|
||||||
|
actual: unknown;
|
||||||
|
target: unknown;
|
||||||
|
}> = [];
|
||||||
|
|
||||||
|
// 1. Group "passed" and "total"
|
||||||
|
if (allKeys.includes("passed") && allKeys.includes("total")) {
|
||||||
|
pairs.push({
|
||||||
|
key: "passed",
|
||||||
|
actual: metrics["passed"],
|
||||||
|
target: metrics["total"],
|
||||||
|
});
|
||||||
|
pairedKeys.add("passed");
|
||||||
|
pairedKeys.add("total");
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Group "X" and "X_target" or "target_X"
|
||||||
|
allKeys.forEach((k) => {
|
||||||
|
if (pairedKeys.has(k)) return;
|
||||||
|
|
||||||
|
if (k.endsWith("_target")) {
|
||||||
|
const baseKey = k.slice(0, -7);
|
||||||
|
if (allKeys.includes(baseKey)) {
|
||||||
|
pairs.push({
|
||||||
|
key: baseKey,
|
||||||
|
actual: metrics[baseKey],
|
||||||
|
target: metrics[k],
|
||||||
|
});
|
||||||
|
pairedKeys.add(baseKey);
|
||||||
|
pairedKeys.add(k);
|
||||||
|
}
|
||||||
|
} else if (k.startsWith("target_")) {
|
||||||
|
const baseKey = k.slice(7);
|
||||||
|
if (allKeys.includes(baseKey)) {
|
||||||
|
pairs.push({
|
||||||
|
key: baseKey,
|
||||||
|
actual: metrics[baseKey],
|
||||||
|
target: metrics[k],
|
||||||
|
});
|
||||||
|
pairedKeys.add(baseKey);
|
||||||
|
pairedKeys.add(k);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// 3. Remaining singles
|
||||||
|
const singles: Array<{ key: string; value: unknown }> = [];
|
||||||
|
allKeys.forEach((k) => {
|
||||||
|
if (!pairedKeys.has(k)) {
|
||||||
|
singles.push({ key: k, value: metrics[k] });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
interface DisplayMetric {
|
||||||
|
key: string;
|
||||||
|
isPair: boolean;
|
||||||
|
actual: unknown;
|
||||||
|
target?: unknown;
|
||||||
|
value?: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
const list: DisplayMetric[] = [
|
||||||
|
...pairs.map((p) => ({ key: p.key, isPair: true, actual: p.actual, target: p.target })),
|
||||||
|
...singles.map((s) => ({ key: s.key, isPair: false, actual: s.value, value: s.value })),
|
||||||
|
];
|
||||||
|
|
||||||
|
// Sort lexicographically
|
||||||
|
list.sort((a, b) => a.key.localeCompare(b.key));
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="grid gap-3 sm:grid-cols-2 md:grid-cols-3" data-testid="eval-metric-grid">
|
<div className="grid gap-3 sm:grid-cols-2 md:grid-cols-3" data-testid="eval-metric-grid">
|
||||||
{sortedKeys.map((key) => {
|
{list.map((item) => {
|
||||||
const val = metrics[key];
|
const key = item.key;
|
||||||
const unit = getUnit(key);
|
const unit = getUnit(key);
|
||||||
const badge = renderPassFailBadge(key, val);
|
|
||||||
const formatted = formatValue(val);
|
|
||||||
const isObj = typeof val === "object" && val !== null;
|
|
||||||
|
|
||||||
return (
|
if (item.isPair) {
|
||||||
<div
|
const actualVal = item.actual;
|
||||||
key={key}
|
const targetVal = item.target;
|
||||||
className="flex flex-col justify-between rounded-lg border border-[var(--color-border-subtle)] bg-[var(--color-surface-raised)] p-3 shadow-sm"
|
const badge = renderPassFailBadge(key, actualVal);
|
||||||
data-testid="metric-card"
|
|
||||||
>
|
const actualNum = Number(actualVal);
|
||||||
<div>
|
const targetNum = Number(targetVal);
|
||||||
<div className="text-[10px] font-semibold text-[var(--color-text-muted)] uppercase tracking-wider truncate" title={key}>
|
const hasNumericValues = !isNaN(actualNum) && !isNaN(targetNum);
|
||||||
{key.replaceAll("_", " ")}
|
const isFailing =
|
||||||
</div>
|
hasNumericValues &&
|
||||||
<div className={`mt-1 font-mono text-sm font-semibold text-[var(--color-text-primary)] ${isObj ? "whitespace-pre overflow-x-auto text-[10px] bg-[var(--color-surface-inset)] p-1.5 rounded" : ""}`}>
|
(isLowerIsBetter(key) ? actualNum > targetNum : actualNum < targetNum);
|
||||||
{formatted}
|
|
||||||
{unit && <span className="ml-1 text-xs text-[var(--color-text-muted)] font-sans font-normal">{unit}</span>}
|
const formattedActual = formatValue(actualVal);
|
||||||
|
const formattedTarget = formatValue(targetVal);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
key={key}
|
||||||
|
className="flex flex-col justify-between rounded-lg border border-[var(--color-border-subtle)] bg-[var(--color-surface-raised)] p-3 shadow-sm"
|
||||||
|
data-testid="metric-card"
|
||||||
|
>
|
||||||
|
<div>
|
||||||
|
<div className="text-[10px] font-semibold text-[var(--color-text-muted)] uppercase tracking-wider truncate" title={key}>
|
||||||
|
{key.replaceAll("_", " ")}
|
||||||
|
</div>
|
||||||
|
<div className="mt-1 font-mono text-sm font-semibold text-right tabular-nums">
|
||||||
|
<span
|
||||||
|
className={
|
||||||
|
isFailing
|
||||||
|
? "text-[var(--color-state-contradicted)] font-bold"
|
||||||
|
: "text-[var(--color-text-primary)]"
|
||||||
|
}
|
||||||
|
>
|
||||||
|
{formattedActual}
|
||||||
|
</span>
|
||||||
|
<span className="text-[var(--color-text-muted)] mx-1">/</span>
|
||||||
|
<span className="text-[var(--color-text-secondary)] font-normal">
|
||||||
|
{formattedTarget}
|
||||||
|
</span>
|
||||||
|
{unit && <span className="ml-1 text-xs text-[var(--color-text-muted)] font-sans font-normal">{unit}</span>}
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
{badge && <div className="mt-2 flex justify-start">{badge}</div>}
|
||||||
</div>
|
</div>
|
||||||
{badge && <div className="mt-2 flex justify-start">{badge}</div>}
|
);
|
||||||
</div>
|
} else {
|
||||||
);
|
const val = item.value;
|
||||||
|
const badge = renderPassFailBadge(key, val);
|
||||||
|
const formatted = formatValue(val);
|
||||||
|
const isObj = typeof val === "object" && val !== null;
|
||||||
|
const isNum = typeof val === "number";
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
key={key}
|
||||||
|
className="flex flex-col justify-between rounded-lg border border-[var(--color-border-subtle)] bg-[var(--color-surface-raised)] p-3 shadow-sm"
|
||||||
|
data-testid="metric-card"
|
||||||
|
>
|
||||||
|
<div>
|
||||||
|
<div className="text-[10px] font-semibold text-[var(--color-text-muted)] uppercase tracking-wider truncate" title={key}>
|
||||||
|
{key.replaceAll("_", " ")}
|
||||||
|
</div>
|
||||||
|
<div
|
||||||
|
className={`mt-1 font-mono text-sm font-semibold text-[var(--color-text-primary)] ${
|
||||||
|
isNum ? "text-right tabular-nums" : ""
|
||||||
|
} ${isObj ? "whitespace-pre overflow-x-auto text-[10px] bg-[var(--color-surface-inset)] p-1.5 rounded" : ""}`}
|
||||||
|
>
|
||||||
|
{formatted}
|
||||||
|
{unit && <span className="ml-1 text-xs text-[var(--color-text-muted)] font-sans font-normal">{unit}</span>}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{badge && <div className="mt-2 flex justify-start">{badge}</div>}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
})}
|
})}
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
|
|
||||||
|
|
@ -12,10 +12,33 @@ import { LoadingState } from "../../design/components/states/LoadingState";
|
||||||
import type { EvalRunResult } from "../../types/api";
|
import type { EvalRunResult } from "../../types/api";
|
||||||
import { WorkbenchApiError } from "../../api/client";
|
import { WorkbenchApiError } from "../../api/client";
|
||||||
|
|
||||||
|
import { useState, useEffect } from "react";
|
||||||
|
import { useSearchParams } from "react-router-dom";
|
||||||
|
import { useEvalLanes, useEvalLane } from "../../api/queries";
|
||||||
|
import { EvalLaneCard } from "./EvalLaneCard";
|
||||||
|
import { EvalRunButton } from "./EvalRunButton";
|
||||||
|
import { EvalMetricGrid } from "./EvalMetricGrid";
|
||||||
|
import { EvalFailureViewer } from "./EvalFailureViewer";
|
||||||
|
import { EvalArtifactLink } from "./EvalArtifactLink";
|
||||||
|
import { EmptyState } from "../../design/components/states/EmptyState";
|
||||||
|
import { ErrorState } from "../../design/components/states/ErrorState";
|
||||||
|
import { LoadingState } from "../../design/components/states/LoadingState";
|
||||||
|
import type { EvalRunResult } from "../../types/api";
|
||||||
|
import { WorkbenchApiError } from "../../api/client";
|
||||||
|
import { useQueryClient } from "@tanstack/react-query";
|
||||||
|
|
||||||
export function EvalsRoute() {
|
export function EvalsRoute() {
|
||||||
const { data: lanes, isLoading, isError, error } = useEvalLanes();
|
const { data: lanes, isLoading, isError, error } = useEvalLanes();
|
||||||
const [searchParams, setSearchParams] = useSearchParams();
|
const [searchParams, setSearchParams] = useSearchParams();
|
||||||
const selectedLaneName = searchParams.get("lane") || "";
|
const selectedLaneName = searchParams.get("lane") || "";
|
||||||
|
const queryClient = useQueryClient();
|
||||||
|
|
||||||
|
const {
|
||||||
|
data: lastRunResult,
|
||||||
|
isLoading: isLaneLoading,
|
||||||
|
isError: isLaneError,
|
||||||
|
error: laneError,
|
||||||
|
} = useEvalLane(selectedLaneName);
|
||||||
|
|
||||||
// Maintain per-lane run states (pending, result, error)
|
// Maintain per-lane run states (pending, result, error)
|
||||||
const [runStates, setRunStates] = useState<
|
const [runStates, setRunStates] = useState<
|
||||||
|
|
@ -29,6 +52,13 @@ export function EvalsRoute() {
|
||||||
>
|
>
|
||||||
>({});
|
>({});
|
||||||
|
|
||||||
|
// Auto-select the first lane on load if no selection is in the URL
|
||||||
|
useEffect(() => {
|
||||||
|
if (!selectedLaneName && lanes && lanes.length > 0) {
|
||||||
|
setSearchParams({ lane: lanes[0].lane }, { replace: true });
|
||||||
|
}
|
||||||
|
}, [selectedLaneName, lanes, setSearchParams]);
|
||||||
|
|
||||||
if (isLoading) {
|
if (isLoading) {
|
||||||
return <LoadingState label="Loading eval lanes..." />;
|
return <LoadingState label="Loading eval lanes..." />;
|
||||||
}
|
}
|
||||||
|
|
@ -46,6 +76,178 @@ export function EvalsRoute() {
|
||||||
|
|
||||||
const selectedLane = lanes?.find((l) => l.lane === selectedLaneName) || null;
|
const selectedLane = lanes?.find((l) => l.lane === selectedLaneName) || null;
|
||||||
const currentRunState = selectedLaneName ? runStates[selectedLaneName] : null;
|
const currentRunState = selectedLaneName ? runStates[selectedLaneName] : null;
|
||||||
|
const isNotFoundError =
|
||||||
|
laneError instanceof WorkbenchApiError && laneError.code === "not_found";
|
||||||
|
|
||||||
|
function renderHeaderAndForm(lane: typeof selectedLane) {
|
||||||
|
if (!lane) return null;
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
{/* Header info */}
|
||||||
|
<div className="flex flex-wrap items-center justify-between gap-3 border-b border-[var(--color-border-subtle)] pb-3">
|
||||||
|
<div>
|
||||||
|
<h2 className="text-lg font-semibold text-[var(--color-text-primary)] font-mono">
|
||||||
|
{lane.lane}
|
||||||
|
</h2>
|
||||||
|
{lane.description && (
|
||||||
|
<p className="mt-1 text-xs text-[var(--color-text-secondary)]">
|
||||||
|
{lane.description}
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Run Button configuration if read-only */}
|
||||||
|
{lane.read_only ? (
|
||||||
|
<EvalRunButton
|
||||||
|
lane={lane}
|
||||||
|
onRunStart={() => {
|
||||||
|
setRunStates((prev) => ({
|
||||||
|
...prev,
|
||||||
|
[lane.lane]: { isPending: true },
|
||||||
|
}));
|
||||||
|
}}
|
||||||
|
onRunSuccess={(result) => {
|
||||||
|
setRunStates((prev) => ({
|
||||||
|
...prev,
|
||||||
|
[lane.lane]: { isPending: false, result },
|
||||||
|
}));
|
||||||
|
queryClient.invalidateQueries({ queryKey: ["api", "eval", lane.lane] });
|
||||||
|
}}
|
||||||
|
onRunError={(err) => {
|
||||||
|
setRunStates((prev) => ({
|
||||||
|
...prev,
|
||||||
|
[lane.lane]: { isPending: false, error: err },
|
||||||
|
}));
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
) : (
|
||||||
|
<div className="text-xs text-[var(--color-text-muted)] bg-[var(--color-surface-inset)] p-3 rounded-lg border border-[var(--color-border-subtle)] font-medium">
|
||||||
|
⚠️ API runs disabled for write-active lane. Use local CLI to execute this lane:
|
||||||
|
<code className="block mt-1 font-mono text-[var(--color-text-primary)] bg-[var(--color-surface-base)] p-1.5 rounded">
|
||||||
|
core eval --lane {lane.lane}
|
||||||
|
</code>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const renderRightPane = () => {
|
||||||
|
if (!selectedLaneName) {
|
||||||
|
return (
|
||||||
|
<EmptyState
|
||||||
|
statement="Select an eval lane from the list to view results or run checks."
|
||||||
|
nextAction={{ kind: "cli", command: "core eval --list" }}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!selectedLane) {
|
||||||
|
return (
|
||||||
|
<EmptyState
|
||||||
|
statement={`Selected lane "${selectedLaneName}" not found.`}
|
||||||
|
nextAction={{ kind: "cli", command: "core eval --list" }}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1. Loading state for fetching the lane's last run details
|
||||||
|
if (isLaneLoading && !lastRunResult) {
|
||||||
|
return <LoadingState label="Loading eval lane details..." />;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Error state for fetching the lane details (excluding not_found)
|
||||||
|
if (isLaneError && !isNotFoundError) {
|
||||||
|
return (
|
||||||
|
<ErrorState
|
||||||
|
whatFailed={laneError instanceof Error ? laneError.message : "Failed to load eval lane details."}
|
||||||
|
mutationStatus="No corpus mutation occurred."
|
||||||
|
reproducer={`core eval --lane ${selectedLaneName}`}
|
||||||
|
retrySafety="Retry: safe"
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Current run execution state
|
||||||
|
if (currentRunState?.isPending) {
|
||||||
|
return (
|
||||||
|
<div className="flex flex-col gap-4">
|
||||||
|
{renderHeaderAndForm(selectedLane)}
|
||||||
|
<LoadingState label="Running eval lane..." />
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (currentRunState?.error) {
|
||||||
|
return (
|
||||||
|
<div className="flex flex-col gap-4">
|
||||||
|
{renderHeaderAndForm(selectedLane)}
|
||||||
|
<ErrorState
|
||||||
|
whatFailed={currentRunState.error.message}
|
||||||
|
mutationStatus="No corpus mutation occurred."
|
||||||
|
reproducer={`core eval --lane ${selectedLane.lane}`}
|
||||||
|
retrySafety="Retry: safe"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Success or Empty state
|
||||||
|
const result = currentRunState?.result || (lastRunResult as EvalRunResult | undefined);
|
||||||
|
if (result && result.metrics) {
|
||||||
|
return (
|
||||||
|
<div className="flex flex-col gap-4">
|
||||||
|
{renderHeaderAndForm(selectedLane)}
|
||||||
|
<div className="flex flex-wrap items-center gap-3">
|
||||||
|
<span className="text-xs text-[var(--color-text-secondary)] font-semibold">
|
||||||
|
Status:
|
||||||
|
</span>
|
||||||
|
<span
|
||||||
|
className={`rounded-md px-2 py-0.5 text-xs font-semibold ${
|
||||||
|
result.passed
|
||||||
|
? "bg-[var(--color-state-success-bg)] text-[var(--color-state-success-text)] border border-[var(--color-state-success-border)]"
|
||||||
|
: "bg-[var(--color-state-danger-bg)] text-[var(--color-state-danger-text)] border border-[var(--color-state-danger-border)]"
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
{result.passed ? "Passed" : "Failed"}
|
||||||
|
</span>
|
||||||
|
{result.source_digest && (
|
||||||
|
<EvalArtifactLink
|
||||||
|
lane={selectedLane.lane}
|
||||||
|
sourceDigest={result.source_digest}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex flex-col gap-2">
|
||||||
|
<h3 className="text-sm font-semibold text-[var(--color-text-primary)]">Metrics</h3>
|
||||||
|
<EvalMetricGrid metrics={result.metrics} />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<EvalFailureViewer
|
||||||
|
cases={result.cases}
|
||||||
|
passed={result.passed}
|
||||||
|
laneName={selectedLane.lane}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="flex flex-col gap-4">
|
||||||
|
{renderHeaderAndForm(selectedLane)}
|
||||||
|
<EmptyState
|
||||||
|
statement={
|
||||||
|
selectedLane.read_only
|
||||||
|
? `No run results for lane "${selectedLane.lane}" in this session. Trigger a run above.`
|
||||||
|
: `Eval lane "${selectedLane.lane}" is CLI-only. No session results available.`
|
||||||
|
}
|
||||||
|
nextAction={{ kind: "cli", command: `core eval --lane ${selectedLane.lane}` }}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="grid h-full grid-cols-1 gap-4 md:grid-cols-[18rem_1fr]" data-testid="evals-route">
|
<div className="grid h-full grid-cols-1 gap-4 md:grid-cols-[18rem_1fr]" data-testid="evals-route">
|
||||||
|
|
@ -73,115 +275,7 @@ export function EvalsRoute() {
|
||||||
|
|
||||||
{/* Right Pane: Results / Form */}
|
{/* Right Pane: Results / Form */}
|
||||||
<div className="flex flex-col gap-4 overflow-y-auto pl-2">
|
<div className="flex flex-col gap-4 overflow-y-auto pl-2">
|
||||||
{selectedLane ? (
|
{renderRightPane()}
|
||||||
<>
|
|
||||||
{/* Header info */}
|
|
||||||
<div className="flex flex-wrap items-center justify-between gap-3 border-b border-[var(--color-border-subtle)] pb-3">
|
|
||||||
<div>
|
|
||||||
<h2 className="text-lg font-semibold text-[var(--color-text-primary)] font-mono">
|
|
||||||
{selectedLane.lane}
|
|
||||||
</h2>
|
|
||||||
{selectedLane.description && (
|
|
||||||
<p className="mt-1 text-xs text-[var(--color-text-secondary)]">
|
|
||||||
{selectedLane.description}
|
|
||||||
</p>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{/* Run Button configuration if read-only */}
|
|
||||||
{selectedLane.read_only ? (
|
|
||||||
<EvalRunButton
|
|
||||||
lane={selectedLane}
|
|
||||||
onRunStart={() => {
|
|
||||||
setRunStates((prev) => ({
|
|
||||||
...prev,
|
|
||||||
[selectedLane.lane]: { isPending: true },
|
|
||||||
}));
|
|
||||||
}}
|
|
||||||
onRunSuccess={(result) => {
|
|
||||||
setRunStates((prev) => ({
|
|
||||||
...prev,
|
|
||||||
[selectedLane.lane]: { isPending: false, result },
|
|
||||||
}));
|
|
||||||
}}
|
|
||||||
onRunError={(err) => {
|
|
||||||
setRunStates((prev) => ({
|
|
||||||
...prev,
|
|
||||||
[selectedLane.lane]: { isPending: false, error: err },
|
|
||||||
}));
|
|
||||||
}}
|
|
||||||
/>
|
|
||||||
) : (
|
|
||||||
<div className="text-xs text-[var(--color-text-muted)] bg-[var(--color-surface-inset)] p-3 rounded-lg border border-[var(--color-border-subtle)] font-medium">
|
|
||||||
⚠️ API runs disabled for write-active lane. Use local CLI to execute this lane:
|
|
||||||
<code className="block mt-1 font-mono text-[var(--color-text-primary)] bg-[var(--color-surface-base)] p-1.5 rounded">
|
|
||||||
core eval --lane {selectedLane.lane}
|
|
||||||
</code>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{/* Result display */}
|
|
||||||
{currentRunState?.isPending ? (
|
|
||||||
<LoadingState label="Running eval lane..." />
|
|
||||||
) : currentRunState?.error ? (
|
|
||||||
<ErrorState
|
|
||||||
whatFailed={currentRunState.error.message}
|
|
||||||
mutationStatus="No corpus mutation occurred."
|
|
||||||
reproducer={`core eval --lane ${selectedLane.lane}`}
|
|
||||||
retrySafety="Retry: safe"
|
|
||||||
/>
|
|
||||||
) : currentRunState?.result ? (
|
|
||||||
<div className="flex flex-col gap-4">
|
|
||||||
<div className="flex flex-wrap items-center gap-3">
|
|
||||||
<span className="text-xs text-[var(--color-text-secondary)] font-semibold">
|
|
||||||
Status:
|
|
||||||
</span>
|
|
||||||
<span
|
|
||||||
className={`rounded-md px-2 py-0.5 text-xs font-semibold ${
|
|
||||||
currentRunState.result.passed
|
|
||||||
? "bg-[var(--color-state-success-bg)] text-[var(--color-state-success-text)] border border-[var(--color-state-success-border)]"
|
|
||||||
: "bg-[var(--color-state-danger-bg)] text-[var(--color-state-danger-text)] border border-[var(--color-state-danger-border)]"
|
|
||||||
}`}
|
|
||||||
>
|
|
||||||
{currentRunState.result.passed ? "Passed" : "Failed"}
|
|
||||||
</span>
|
|
||||||
{currentRunState.result.source_digest && (
|
|
||||||
<EvalArtifactLink
|
|
||||||
lane={selectedLane.lane}
|
|
||||||
sourceDigest={currentRunState.result.source_digest}
|
|
||||||
/>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div className="flex flex-col gap-2">
|
|
||||||
<h3 className="text-sm font-semibold text-[var(--color-text-primary)]">Metrics</h3>
|
|
||||||
<EvalMetricGrid metrics={currentRunState.result.metrics} />
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<EvalFailureViewer
|
|
||||||
cases={currentRunState.result.cases}
|
|
||||||
passed={currentRunState.result.passed}
|
|
||||||
laneName={selectedLane.lane}
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
) : (
|
|
||||||
<EmptyState
|
|
||||||
statement={
|
|
||||||
selectedLane.read_only
|
|
||||||
? `No run results for lane "${selectedLane.lane}" in this session. Trigger a run above.`
|
|
||||||
: `Eval lane "${selectedLane.lane}" is CLI-only. No session results available.`
|
|
||||||
}
|
|
||||||
nextAction={{ kind: "cli", command: `core eval --lane ${selectedLane.lane}` }}
|
|
||||||
/>
|
|
||||||
)}
|
|
||||||
</>
|
|
||||||
) : (
|
|
||||||
<EmptyState
|
|
||||||
statement="Select an eval lane from the list to view results or run checks."
|
|
||||||
nextAction={{ kind: "cli", command: "core eval --list" }}
|
|
||||||
/>
|
|
||||||
)}
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
|
|
||||||
|
|
@ -16,11 +16,13 @@ vi.mock("../../api/queries", async (importOriginal) => {
|
||||||
return {
|
return {
|
||||||
...actual,
|
...actual,
|
||||||
useEvalLanes: vi.fn(),
|
useEvalLanes: vi.fn(),
|
||||||
|
useEvalLane: vi.fn(),
|
||||||
useEvalRun: vi.fn(),
|
useEvalRun: vi.fn(),
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
import { useEvalLanes, useEvalRun } from "../../api/queries";
|
import { useEvalLanes, useEvalLane, useEvalRun } from "../../api/queries";
|
||||||
|
import { CommandPalette } from "../../design/components/primitives/CommandPalette";
|
||||||
|
|
||||||
const mockLanes: EvalLaneSummary[] = [
|
const mockLanes: EvalLaneSummary[] = [
|
||||||
{ lane: "contemplation_quality", versions: ["v1", "v2"], read_only: true, description: "Contemplation checks" },
|
{ lane: "contemplation_quality", versions: ["v1", "v2"], read_only: true, description: "Contemplation checks" },
|
||||||
|
|
@ -49,6 +51,27 @@ describe("W-030 Component Tests", () => {
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.resetAllMocks();
|
vi.resetAllMocks();
|
||||||
|
|
||||||
|
vi.mocked(useEvalLanes).mockReturnValue({
|
||||||
|
data: mockLanes,
|
||||||
|
isLoading: false,
|
||||||
|
isError: false,
|
||||||
|
error: null,
|
||||||
|
} as any);
|
||||||
|
vi.mocked(useEvalLane).mockReturnValue({
|
||||||
|
data: undefined,
|
||||||
|
isLoading: false,
|
||||||
|
isError: true,
|
||||||
|
error: new WorkbenchApiError("not_found", "No run history found"),
|
||||||
|
} as any);
|
||||||
|
vi.mocked(useEvalRun).mockReturnValue({
|
||||||
|
mutate: vi.fn(),
|
||||||
|
isPending: false,
|
||||||
|
isError: false,
|
||||||
|
isSuccess: false,
|
||||||
|
error: null,
|
||||||
|
} as any);
|
||||||
|
|
||||||
fetchMock.mockImplementation((url: any) => {
|
fetchMock.mockImplementation((url: any) => {
|
||||||
const urlStr = typeof url === "string" ? url : String(url?.url || url || "");
|
const urlStr = typeof url === "string" ? url : String(url?.url || url || "");
|
||||||
if (urlStr.endsWith("/evals")) {
|
if (urlStr.endsWith("/evals")) {
|
||||||
|
|
@ -162,10 +185,45 @@ describe("W-030 Component Tests", () => {
|
||||||
|
|
||||||
// Verify lexicographical order
|
// Verify lexicographical order
|
||||||
expect(cards1).toEqual(cards2);
|
expect(cards1).toEqual(cards2);
|
||||||
|
expect(cards1.length).toBe(3);
|
||||||
expect(cards1[0]).toContain("accuracy");
|
expect(cards1[0]).toContain("accuracy");
|
||||||
expect(cards1[1]).toContain("latency ms");
|
expect(cards1[1]).toContain("latency ms");
|
||||||
expect(cards1[2]).toContain("passed");
|
expect(cards1[2]).toContain("passed");
|
||||||
expect(cards1[3]).toContain("total");
|
expect(cards1[2]).toContain("9/10");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders actual / target, emphasizes actual, and applies failure color token when target is not met", () => {
|
||||||
|
const metrics = {
|
||||||
|
passed: 4,
|
||||||
|
total: 5,
|
||||||
|
latency_ms: 150,
|
||||||
|
latency_ms_target: 100, // failing because actual > target
|
||||||
|
accuracy: 0.9,
|
||||||
|
accuracy_target: 0.95, // failing because actual < target
|
||||||
|
};
|
||||||
|
|
||||||
|
const { container } = render(<EvalMetricGrid metrics={metrics} />);
|
||||||
|
|
||||||
|
const cards = container.querySelectorAll('[data-testid="metric-card"]');
|
||||||
|
expect(cards.length).toBe(3);
|
||||||
|
|
||||||
|
const passedCard = Array.from(cards).find(c => c.textContent?.includes("passed"));
|
||||||
|
expect(passedCard).toBeDefined();
|
||||||
|
expect(passedCard?.querySelector(".text-\\[var\\(--color-state-contradicted\\)\\]")).toBeInTheDocument();
|
||||||
|
expect(passedCard?.textContent).toContain("4");
|
||||||
|
expect(passedCard?.textContent).toContain("5");
|
||||||
|
|
||||||
|
const latencyCard = Array.from(cards).find(c => c.textContent?.includes("latency ms"));
|
||||||
|
expect(latencyCard).toBeDefined();
|
||||||
|
expect(latencyCard?.querySelector(".text-\\[var\\(--color-state-contradicted\\)\\]")).toBeInTheDocument();
|
||||||
|
expect(latencyCard?.textContent).toContain("150");
|
||||||
|
expect(latencyCard?.textContent).toContain("100");
|
||||||
|
|
||||||
|
const accuracyCard = Array.from(cards).find(c => c.textContent?.includes("accuracy"));
|
||||||
|
expect(accuracyCard).toBeDefined();
|
||||||
|
expect(accuracyCard?.querySelector(".text-\\[var\\(--color-state-contradicted\\)\\]")).toBeInTheDocument();
|
||||||
|
expect(accuracyCard?.textContent).toContain("0.9");
|
||||||
|
expect(accuracyCard?.textContent).toContain("0.95");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -295,5 +353,115 @@ describe("W-030 Component Tests", () => {
|
||||||
const res = await runEvalLane({ lane: "contemplation_quality", split: "holdout" });
|
const res = await runEvalLane({ lane: "contemplation_quality", split: "holdout" });
|
||||||
expect(res).toEqual(mockResult);
|
expect(res).toEqual(mockResult);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("renders loading state when lane list is loading", () => {
|
||||||
|
vi.mocked(useEvalLanes).mockReturnValue({
|
||||||
|
data: null,
|
||||||
|
isLoading: true,
|
||||||
|
isError: false,
|
||||||
|
error: null,
|
||||||
|
} as any);
|
||||||
|
|
||||||
|
const client = makeClient();
|
||||||
|
render(
|
||||||
|
<QueryClientProvider client={client}>
|
||||||
|
<MemoryRouter initialEntries={["/evals"]}>
|
||||||
|
<EvalsRoute />
|
||||||
|
</MemoryRouter>
|
||||||
|
</QueryClientProvider>
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(screen.getByText("Loading eval lanes...")).toBeInTheDocument();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders empty state when there are no lanes", () => {
|
||||||
|
vi.mocked(useEvalLanes).mockReturnValue({
|
||||||
|
data: [],
|
||||||
|
isLoading: false,
|
||||||
|
isError: false,
|
||||||
|
error: null,
|
||||||
|
} as any);
|
||||||
|
|
||||||
|
const client = makeClient();
|
||||||
|
render(
|
||||||
|
<QueryClientProvider client={client}>
|
||||||
|
<MemoryRouter initialEntries={["/evals"]}>
|
||||||
|
<EvalsRoute />
|
||||||
|
</MemoryRouter>
|
||||||
|
</QueryClientProvider>
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(screen.getByText("No eval lanes discovered.")).toBeInTheDocument();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders loading and error states for the selected lane detail view", () => {
|
||||||
|
vi.mocked(useEvalLanes).mockReturnValue({
|
||||||
|
data: mockLanes,
|
||||||
|
isLoading: false,
|
||||||
|
isError: false,
|
||||||
|
error: null,
|
||||||
|
} as any);
|
||||||
|
|
||||||
|
// Mock useEvalLane to return loading
|
||||||
|
vi.mocked(useEvalLane).mockReturnValue({
|
||||||
|
data: undefined,
|
||||||
|
isLoading: true,
|
||||||
|
isError: false,
|
||||||
|
error: null,
|
||||||
|
} as any);
|
||||||
|
|
||||||
|
const client = makeClient();
|
||||||
|
const { rerender } = render(
|
||||||
|
<QueryClientProvider client={client}>
|
||||||
|
<MemoryRouter initialEntries={["/evals?lane=contemplation_quality"]}>
|
||||||
|
<EvalsRoute />
|
||||||
|
</MemoryRouter>
|
||||||
|
</QueryClientProvider>
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(screen.getByText("Loading eval lane details...")).toBeInTheDocument();
|
||||||
|
|
||||||
|
// Mock useEvalLane to return error (other than not_found)
|
||||||
|
vi.mocked(useEvalLane).mockReturnValue({
|
||||||
|
data: undefined,
|
||||||
|
isLoading: false,
|
||||||
|
isError: true,
|
||||||
|
error: new WorkbenchApiError("read_error", "Disk read error"),
|
||||||
|
} as any);
|
||||||
|
|
||||||
|
rerender(
|
||||||
|
<QueryClientProvider client={client}>
|
||||||
|
<MemoryRouter initialEntries={["/evals?lane=contemplation_quality"]}>
|
||||||
|
<EvalsRoute />
|
||||||
|
</MemoryRouter>
|
||||||
|
</QueryClientProvider>
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(screen.getByText("Disk read error")).toBeInTheDocument();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("CommandPalette registers dynamic command entries driven by useEvalLanes", () => {
|
||||||
|
vi.mocked(useEvalLanes).mockReturnValue({
|
||||||
|
data: mockLanes,
|
||||||
|
isLoading: false,
|
||||||
|
isError: false,
|
||||||
|
error: null,
|
||||||
|
} as any);
|
||||||
|
|
||||||
|
const client = makeClient();
|
||||||
|
render(
|
||||||
|
<QueryClientProvider client={client}>
|
||||||
|
<MemoryRouter>
|
||||||
|
<CommandPalette open={true} onOpenChange={vi.fn()} />
|
||||||
|
</MemoryRouter>
|
||||||
|
</QueryClientProvider>
|
||||||
|
);
|
||||||
|
|
||||||
|
// Verify static commands
|
||||||
|
expect(screen.getByRole("button", { name: "Open Chat" })).toBeInTheDocument();
|
||||||
|
// Verify dynamic commands
|
||||||
|
expect(screen.getByRole("button", { name: "Open eval lane contemplation_quality" })).toBeInTheDocument();
|
||||||
|
expect(screen.getByRole("button", { name: "Open eval lane unsafe_lane" })).toBeInTheDocument();
|
||||||
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,18 @@
|
||||||
import { render, screen } from "@testing-library/react";
|
import { render, screen } from "@testing-library/react";
|
||||||
import userEvent from "@testing-library/user-event";
|
import userEvent from "@testing-library/user-event";
|
||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
import { MemoryRouter } from "react-router-dom";
|
import { MemoryRouter } from "react-router-dom";
|
||||||
import { useState } from "react";
|
import { useState } from "react";
|
||||||
import { CommandPalette } from "./CommandPalette";
|
import { CommandPalette } from "./CommandPalette";
|
||||||
|
|
||||||
|
vi.mock("../../../api/queries", () => ({
|
||||||
|
useEvalLanes: () => ({
|
||||||
|
data: [],
|
||||||
|
isLoading: false,
|
||||||
|
isError: false,
|
||||||
|
}),
|
||||||
|
}));
|
||||||
|
|
||||||
function PaletteHarness({ initialOpen = false }: { initialOpen?: boolean }) {
|
function PaletteHarness({ initialOpen = false }: { initialOpen?: boolean }) {
|
||||||
const [open, setOpen] = useState(initialOpen);
|
const [open, setOpen] = useState(initialOpen);
|
||||||
return (
|
return (
|
||||||
|
|
|
||||||
|
|
@ -2,13 +2,14 @@ import * as Dialog from "@radix-ui/react-dialog";
|
||||||
import { Search } from "lucide-react";
|
import { Search } from "lucide-react";
|
||||||
import { useRef, useState, useEffect, useCallback } from "react";
|
import { useRef, useState, useEffect, useCallback } from "react";
|
||||||
import { useNavigate, useInRouterContext } from "react-router-dom";
|
import { useNavigate, useInRouterContext } from "react-router-dom";
|
||||||
|
import { useEvalLanes } from "../../../api/queries";
|
||||||
|
|
||||||
interface Command {
|
interface Command {
|
||||||
name: string;
|
name: string;
|
||||||
path: string;
|
path: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
const COMMANDS: Command[] = [
|
const STATIC_COMMANDS: Command[] = [
|
||||||
{ name: "Open Chat", path: "/chat" },
|
{ name: "Open Chat", path: "/chat" },
|
||||||
{ name: "Open Proposals", path: "/proposals" },
|
{ name: "Open Proposals", path: "/proposals" },
|
||||||
{ name: "Open Evals", path: "/evals" },
|
{ name: "Open Evals", path: "/evals" },
|
||||||
|
|
@ -20,6 +21,15 @@ function RouterCommandPalette(props: {
|
||||||
onOpenChange: (open: boolean) => void;
|
onOpenChange: (open: boolean) => void;
|
||||||
}) {
|
}) {
|
||||||
const navigate = useNavigate();
|
const navigate = useNavigate();
|
||||||
|
const { data: lanes } = useEvalLanes();
|
||||||
|
|
||||||
|
const dynamicCommands = (lanes || []).map((lane) => ({
|
||||||
|
name: `Open eval lane ${lane.lane}`,
|
||||||
|
path: `/evals?lane=${lane.lane}`,
|
||||||
|
}));
|
||||||
|
|
||||||
|
const commands = [...STATIC_COMMANDS, ...dynamicCommands];
|
||||||
|
|
||||||
const activate = useCallback(
|
const activate = useCallback(
|
||||||
(cmd: Command) => {
|
(cmd: Command) => {
|
||||||
navigate(cmd.path);
|
navigate(cmd.path);
|
||||||
|
|
@ -27,7 +37,7 @@ function RouterCommandPalette(props: {
|
||||||
},
|
},
|
||||||
[navigate, props],
|
[navigate, props],
|
||||||
);
|
);
|
||||||
return <CommandPaletteContent {...props} onActivate={activate} />;
|
return <CommandPaletteContent {...props} commands={commands} onActivate={activate} />;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallback for design-system preview (no Router).
|
// Fallback for design-system preview (no Router).
|
||||||
|
|
@ -41,23 +51,25 @@ function FallbackCommandPalette(props: {
|
||||||
},
|
},
|
||||||
[props],
|
[props],
|
||||||
);
|
);
|
||||||
return <CommandPaletteContent {...props} onActivate={activate} />;
|
return <CommandPaletteContent {...props} commands={STATIC_COMMANDS} onActivate={activate} />;
|
||||||
}
|
}
|
||||||
|
|
||||||
function CommandPaletteContent({
|
function CommandPaletteContent({
|
||||||
open,
|
open,
|
||||||
onOpenChange,
|
onOpenChange,
|
||||||
onActivate,
|
onActivate,
|
||||||
|
commands,
|
||||||
}: {
|
}: {
|
||||||
open: boolean;
|
open: boolean;
|
||||||
onOpenChange: (open: boolean) => void;
|
onOpenChange: (open: boolean) => void;
|
||||||
onActivate: (cmd: Command) => void;
|
onActivate: (cmd: Command) => void;
|
||||||
|
commands: Command[];
|
||||||
}) {
|
}) {
|
||||||
const [query, setQuery] = useState("");
|
const [query, setQuery] = useState("");
|
||||||
const [focusedIndex, setFocusedIndex] = useState(0);
|
const [focusedIndex, setFocusedIndex] = useState(0);
|
||||||
const inputRef = useRef<HTMLInputElement>(null);
|
const inputRef = useRef<HTMLInputElement>(null);
|
||||||
|
|
||||||
const filtered = COMMANDS.filter((cmd) =>
|
const filtered = commands.filter((cmd) =>
|
||||||
cmd.name.toLowerCase().includes(query.toLowerCase()),
|
cmd.name.toLowerCase().includes(query.toLowerCase()),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue