diff --git a/docs/workbench/capability-mastery-implementation-plan.md b/docs/workbench/capability-mastery-implementation-plan.md new file mode 100644 index 00000000..61df341c --- /dev/null +++ b/docs/workbench/capability-mastery-implementation-plan.md @@ -0,0 +1,153 @@ +# Workbench Capability Mastery — Implementation Plan + +**Branch:** `feat/workbench-capability-mastery-reconcile` +**Reconciles:** Draft PR #821 (`feat/workbench-capability-mastery`) onto current `origin/main` after #824 +**Status:** Reconciled UI surfaces — honest documented milestones, no fake live data + +## 1. What #821 originally attempted + +PR #821 (`feat/workbench-capability-mastery`) added first Workbench surfaces on the Evals route: + +- `ExperienceFlywheelPanel` — practice-memory record inspection +- `CapabilityParadigmPanel` — derivation lift / Gate A* visibility +- Integration in `EvalsRoute.tsx` +- A draft `capability-mastery-implementation-plan.md` + +The draft branch also **deleted large sections** of `workbench-ui/src/api/queries.ts` and `workbench-ui/src/types/api.ts`, introduced stale API assumptions (`runtime.capability_health`, `GET /flywheel`), and left `ExperienceFlywheelPanel` as an incomplete stub. Workbench-ui CI failed at build. + +## 2. What changed since #821 was cut + +Capability sprints 9–12 landed on main: + +| PR | Sprint | Gates / organs | train_sample after | +|----|--------|----------------|-------------------:| +| #816 | Flywheel PR-1 | measurement-only adapter | (no serving change) | +| #817 | 6 | A2g duration_segment_total, A2h survey_rate_earnings | 14/36/0 | +| #818 | 7 | A2i round_trip_trip_duration, A2j giveaway_target_residual | 16/34/0 | +| #819 | 8 | A2k fraction_decrease, A2l percent_partition | 18/32/0 | +| #820 / #822 | 9 + hardening | A2m temporal_tariff, A2n affine_fraction_delta | 21/29/0 | +| #823 | 10 | A2o affine_comparative_inversion_total, A2p sequential_comparative_scale | 23/27/0 | +| #824 | 11 | A2q calendar_grounded_piecewise_daily_hours_total + ClusterContract | **24/26/0** | + +Lookback analyses through Sprint 11 are committed under `docs/analysis/gsm8k-capability-paradigm-sprint*-lookback-2026-06-17.md`. Sprint 12 lookback is **not on main** at reconciliation time. + +## 3. Current capability baseline + +**Documented serving state (train_sample after #824):** `24 correct / 26 refused / 0 wrong` + +This is shown in the UI as **documented** evidence from lookbacks — not as a live workbench API read. + +## 4. Gate ladder (A2e → A2q) + +| Gate | Organ | Sprint | +|------|-------|--------| +| A2e | goal_residual | Strike #814 | +| A2g | duration_segment_total | 6 (#817) | +| A2h | survey_rate_earnings | 6 (#817) | +| A2i | round_trip_trip_duration | 7 (#818) | +| A2j | giveaway_target_residual | 7 (#818) | +| A2k | fraction_decrease | 8 (#819) | +| A2l | percent_partition | 8 (#819) | +| A2m | temporal_tariff | 9 (#820) | +| A2n | affine_fraction_delta | 9 (#820) | +| A2o | affine_comparative_inversion_total | 10 (#823) | +| A2p | sequential_comparative_scale | 10 (#823) | +| A2q | calendar_grounded_piecewise_daily_hours_total | 11 (#824) | + +## 5. Experience Flywheel is measurement-only + +PR-1 (#816) adds `scripts/gsm8k_experience_flywheel.py` and `evals/gsm8k_math/train_sample/v1/experience.py`: + +- Read-only report / sealed-practice inputs +- Explicit `--out` path only +- No serving mutation, no `report.json` writes, no pack/corpus mutation +- No auto-promotion into teaching or serving + +## 6. ClusterContract (Sprint 11 / #824) + +First ClusterContract sprint ratified `calendar_grounded_piecewise_daily_hours_total` with: + +- `piecewise_daily_hours_total` (Gate A2q) +- `calendar_grounding.py` — `civil_month_day_count_table` with `calendar_table:{month}` provenance +- Included case: `gsm8k-train-sample-v1-0013` + +See `docs/analysis/gsm8k-capability-paradigm-sprint11-lookback-2026-06-17.md`. + +## 7. What is UI-live now + +| Surface | Live? | Source | +|---------|-------|--------| +| Eval lane list | Yes | `GET /evals` | +| Eval lane run (read-only lanes) | Yes | `POST /evals/run` | +| Wrong-zero ledger from session run | Yes | `EvalRunResult` in session | +| Experience Flywheel records | **No** | Honest empty + CLI guidance | +| Capability paradigm milestones | **Documented static** | Committed lookback markdown | +| train_sample 24/26/0 headline | **Documented** | Lookback #824, not live poll | + +## 8. Honest empty states + +- **Experience Flywheel:** states no workbench endpoint; points to `scripts/gsm8k_experience_flywheel.py --limit 50 --out /tmp/gsm8k-experience.json` +- **Capability Paradigm:** labeled "Documented" badge; sprint cards cite lookback doc paths +- **No lane selected:** capability section is the landing view; lane metrics require lane selection + run + +## 9. Backend endpoints still pending + +- `GET /experience-flywheel` (or equivalent) — read-only compacted record feed from explicit artifact path or repo-local cache +- `GET /capability-paradigm` — optional; could mirror lookback JSON if generated deterministically +- Any endpoint must be read-only, drift-gated, and typed in `workbench/schemas.py` before client wiring + +## 10. Explicitly not implemented + +- Mutation endpoints for practice memory, serving, or reports +- Fake / seeded flywheel records in the UI +- `runtime.capability_health` footer theater +- Wholesale copy of #821 `types/api.ts` / `queries.ts` deletions +- Changes to `evals/gsm8k_math/train_sample/v1/report.json` +- Changes to sealed practice artifacts +- Sprint 12 live cards (doc absent on main) + +## 11. Invariants preserved + +- ADR-0160 / ADR-0162 evidence-first, read-only Workbench posture +- No cognitive theater or invented mastery scores +- Route conformance unchanged (`EvalsRoute` empty/loading strings preserved for palette tests) +- `StatusFooter` unchanged on main (no fake capability_health) + +## 12. Files in this reconciliation + +| File | Action | +|------|--------| +| `workbench-ui/src/app/evals/capabilityMasteryData.ts` | New — documented milestones | +| `workbench-ui/src/app/evals/ExperienceFlywheelPanel.tsx` | New — honest empty state | +| `workbench-ui/src/app/evals/CapabilityParadigmPanel.tsx` | New — documented gate ladder | +| `workbench-ui/src/app/evals/CapabilityMasterySection.tsx` | New — composition | +| `workbench-ui/src/app/evals/EvalsRoute.tsx` | Integrate section | +| `workbench-ui/src/app/evals/capabilityMastery.test.tsx` | New tests | +| `docs/workbench/capability-mastery-implementation-plan.md` | This plan | + +**Not ported from #821:** `queries.ts` / `types/api.ts` mass deletions, `StatusFooter` capability_health stub. + +## 13. Next Workbench steps + +1. Add read-only `GET /experience-flywheel` backed by explicit artifact path config (no default repo writes) +2. Optional deterministic `capability-paradigm.json` generator from lookback tables for API parity +3. Wire `ExperienceFlywheelPanel` to real records when endpoint exists; keep empty state as primary until then +4. Sprint 12 lookback cards when doc lands on main +5. Close #821 in favor of this PR after review + +## 14. Verification + +```bash +cd workbench-ui +pnpm install --frozen-lockfile +pnpm exec tsc -b +pnpm build +pnpm test -- --run src/app/evals +pnpm test -- --run src/app +``` + +From repo root: + +```bash +uv run python -m core.cli test --suite smoke -q +``` \ No newline at end of file diff --git a/workbench-ui/src/app/evals/CapabilityMasterySection.tsx b/workbench-ui/src/app/evals/CapabilityMasterySection.tsx new file mode 100644 index 00000000..8f1b5c8a --- /dev/null +++ b/workbench-ui/src/app/evals/CapabilityMasterySection.tsx @@ -0,0 +1,16 @@ +import { CapabilityParadigmPanel } from "./CapabilityParadigmPanel"; +import { ExperienceFlywheelPanel } from "./ExperienceFlywheelPanel"; + +/** Capability mastery surfaces for the Evals route (reconciled from PR 821). */ +export function CapabilityMasterySection() { + return ( +
+ + +
+ ); +} \ No newline at end of file diff --git a/workbench-ui/src/app/evals/CapabilityParadigmPanel.tsx b/workbench-ui/src/app/evals/CapabilityParadigmPanel.tsx new file mode 100644 index 00000000..034d7154 --- /dev/null +++ b/workbench-ui/src/app/evals/CapabilityParadigmPanel.tsx @@ -0,0 +1,159 @@ +import { InfoBadge } from "../../design/components/badges/Badge"; +import { MetadataTable } from "../../design/components/MetadataTable/MetadataTable"; +import { Panel } from "../../design/components/Panel/Panel"; +import { TruncatedCell } from "../../design/components/TruncatedCell"; +import { + BLOCKED_FAMILIES, + CLUSTER_CONTRACT_SPRINT11, + DOCUMENTED_BASELINE_LABEL, + DOCUMENTED_TRAIN_SAMPLE_BASELINE, + GATE_LADDER_A2E_A2Q, +} from "./capabilityMasteryData"; + +function formatScore({ correct, refused, wrong }: { correct: number; refused: number; wrong: number }) { + return `${correct} / ${refused} / ${wrong}`; +} + +export function CapabilityParadigmPanel() { + const baseline = DOCUMENTED_TRAIN_SAMPLE_BASELINE; + + return ( + + } + > +
+
+

+ {DOCUMENTED_BASELINE_LABEL} +

+ + {formatScore(baseline)} + + ), + mono: true, + }, + { + key: "wrong invariant", + value: ( + + wrong = 0 (documented) + + ), + }, + { + key: "evidence through", + value: "PR 824 — Sprint 11 ClusterContract", + }, + ]} + /> +
+ +
+

+ Gate ladder (A2e → A2q) +

+
+ + + + + + + + + + + + + {GATE_LADDER_A2E_A2Q.map((row) => ( + + + + + + + + + ))} + +
GateOrganSprintLiftedScore afterLookback
{row.gate} + + {row.sprint} + {row.newlySolved.length > 0 ? row.newlySolved.join(", ") : "—"} + + {formatScore(row.scoreAfter)} + + +
+
+
+ +
+

+ ClusterContract — Sprint 11 (PR 824) +

+

+ First ClusterContract sprint: typed chain + explicit{" "} + calendar_table provenance for Gate A2q. +

+ +
+ +
+

+ Blocked families (documented) +

+
    + {BLOCKED_FAMILIES.map((blocked) => ( +
  • +
    {blocked.family}
    +
    + cases: {blocked.cases.join(", ")} +
    +
    {blocked.reason}
    +
    • + ))} +
+
+ +

+ Sprint 12 lookback doc is not on main yet. Re-run eval lanes or read lookback markdown in-repo + for replay evidence; this panel does not fabricate live serving counts. +

+
+ + ); +} \ No newline at end of file diff --git a/workbench-ui/src/app/evals/EvalsRoute.tsx b/workbench-ui/src/app/evals/EvalsRoute.tsx index 46663b89..31d94e8e 100644 --- a/workbench-ui/src/app/evals/EvalsRoute.tsx +++ b/workbench-ui/src/app/evals/EvalsRoute.tsx @@ -11,6 +11,7 @@ import { EvalMetricGrid } from "./EvalMetricGrid"; import { EvalFailureViewer } from "./EvalFailureViewer"; import { EvalArtifactLink } from "./EvalArtifactLink"; import { EvalWrongZeroLedger } from "./EvalWrongZeroLedger"; +import { CapabilityMasterySection } from "./CapabilityMasterySection"; import { EmptyState } from "../../design/components/states/EmptyState"; import { ErrorState } from "../../design/components/states/ErrorState"; import { LoadingState } from "../../design/components/states/LoadingState"; @@ -254,12 +255,17 @@ export function EvalsRoute() { nextAction={{ kind: "cli", command: `core eval --lane ${selectedLane.lane}` }} /> )} + + ) : ( - + <> + + + )} diff --git a/workbench-ui/src/app/evals/ExperienceFlywheelPanel.tsx b/workbench-ui/src/app/evals/ExperienceFlywheelPanel.tsx new file mode 100644 index 00000000..3947e299 --- /dev/null +++ b/workbench-ui/src/app/evals/ExperienceFlywheelPanel.tsx @@ -0,0 +1,66 @@ +import { InfoBadge } from "../../design/components/badges/Badge"; +import { MetadataTable } from "../../design/components/MetadataTable/MetadataTable"; +import { Panel } from "../../design/components/Panel/Panel"; +import { EmptyState } from "../../design/components/states/EmptyState"; +import { EXPERIENCE_FLYWHEEL_CLI, EXPERIENCE_RECORD_FIELDS } from "./capabilityMasteryData"; + +export const EXPERIENCE_FLYWHEEL_ABSENCE_STATEMENT = + "No Experience Flywheel records in this Workbench session. PR-1 (PR 816) is measurement-only — records are emitted by CLI to an explicit --out path; there is no GET /flywheel workbench endpoint yet."; + +/** + * Experience Flywheel — measurement-only sealed-practice memory (PR-1 / PR 816). + * Honest empty state until a read-only backend surface lands. + */ +export function ExperienceFlywheelPanel() { + return ( + + } + > +
+ + +
+

+ Compact record shape (when CLI emits --out) +

+ ({ + key: field.key, + value: field.value, + }))} + /> +
+ + +
+ + ); +} \ No newline at end of file diff --git a/workbench-ui/src/app/evals/capabilityMastery.test.tsx b/workbench-ui/src/app/evals/capabilityMastery.test.tsx new file mode 100644 index 00000000..7448bcb4 --- /dev/null +++ b/workbench-ui/src/app/evals/capabilityMastery.test.tsx @@ -0,0 +1,40 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { CapabilityParadigmPanel } from "./CapabilityParadigmPanel"; +import { + EXPERIENCE_FLYWHEEL_ABSENCE_STATEMENT, + ExperienceFlywheelPanel, +} from "./ExperienceFlywheelPanel"; +import { CapabilityMasterySection } from "./CapabilityMasterySection"; +import { DOCUMENTED_TRAIN_SAMPLE_BASELINE } from "./capabilityMasteryData"; + +describe("Capability mastery surfaces", () => { + it("ExperienceFlywheelPanel renders honest empty state and CLI guidance", () => { + render(); + expect(screen.getByTestId("experience-flywheel-panel")).toBeInTheDocument(); + expect(screen.getByText(EXPERIENCE_FLYWHEEL_ABSENCE_STATEMENT)).toBeInTheDocument(); + expect( + screen.getByText("scripts/gsm8k_experience_flywheel.py --limit 50 --out /tmp/gsm8k-experience.json"), + ).toBeInTheDocument(); + expect(screen.getByText("Measurement-only")).toBeInTheDocument(); + }); + + it("CapabilityParadigmPanel shows documented baseline and gate ladder through A2q", () => { + render(); + const { correct, refused, wrong } = DOCUMENTED_TRAIN_SAMPLE_BASELINE; + expect(screen.getAllByText(`${correct} / ${refused} / ${wrong}`).length).toBeGreaterThanOrEqual(1); + expect(screen.getByText("Documented")).toBeInTheDocument(); + expect(screen.getByTestId("gate-row-A2e")).toBeInTheDocument(); + expect(screen.getByTestId("gate-row-A2q")).toBeInTheDocument(); + expect(screen.getByTestId("cluster-contract-sprint11")).toBeInTheDocument(); + expect(screen.getByTestId("blocked-DCS")).toBeInTheDocument(); + expect(screen.getByTestId("blocked-multiplicative_aggregate")).toBeInTheDocument(); + }); + + it("CapabilityMasterySection composes both panels", () => { + render(); + expect(screen.getByTestId("capability-mastery-section")).toBeInTheDocument(); + expect(screen.getByText("Experience Flywheel")).toBeInTheDocument(); + expect(screen.getByText("Capability Paradigm")).toBeInTheDocument(); + }); +}); \ No newline at end of file diff --git a/workbench-ui/src/app/evals/capabilityMasteryData.ts b/workbench-ui/src/app/evals/capabilityMasteryData.ts new file mode 100644 index 00000000..1ee9e83c --- /dev/null +++ b/workbench-ui/src/app/evals/capabilityMasteryData.ts @@ -0,0 +1,190 @@ +/** + * Documented GSM8K capability milestones — sourced from committed lookback + * analyses on main through PR 824. These are NOT live API values; labels must + * stay honest (ADR-0160 / ADR-0162). + */ + +export type DocumentedScore = { + correct: number; + refused: number; + wrong: number; +}; + +export type GateMilestone = { + gate: string; + organ: string; + sprint: string; + pr: string; + newlySolved: string[]; + scoreAfter: DocumentedScore; + lookbackDoc: string; +}; + +export type BlockedFamily = { + family: string; + cases: string[]; + reason: string; +}; + +/** Serving baseline after PR 824 (Sprint 11), from lookback evidence. */ +export const DOCUMENTED_TRAIN_SAMPLE_BASELINE: DocumentedScore = { + correct: 24, + refused: 26, + wrong: 0, +}; + +export const DOCUMENTED_BASELINE_LABEL = + "Documented baseline (train_sample after PR 824 — not a live API read)"; + +export const GATE_LADDER_A2E_A2Q: GateMilestone[] = [ + { + gate: "A2e", + organ: "goal_residual", + sprint: "Strike batch 4 (PR 814)", + pr: "PR 814", + newlySolved: [], + scoreAfter: { correct: 10, refused: 40, wrong: 0 }, + lookbackDoc: "docs/analysis/gsm8k-capability-paradigm-sprint5-lookback-2026-06-17.md", + }, + { + gate: "A2g", + organ: "duration_segment_total", + sprint: "Sprint 6 (PR 817)", + pr: "PR 817", + newlySolved: ["0015"], + scoreAfter: { correct: 14, refused: 36, wrong: 0 }, + lookbackDoc: "docs/analysis/gsm8k-capability-paradigm-sprint6-lookback-2026-06-17.md", + }, + { + gate: "A2h", + organ: "survey_rate_earnings", + sprint: "Sprint 6 (PR 817)", + pr: "PR 817", + newlySolved: ["0045"], + scoreAfter: { correct: 14, refused: 36, wrong: 0 }, + lookbackDoc: "docs/analysis/gsm8k-capability-paradigm-sprint6-lookback-2026-06-17.md", + }, + { + gate: "A2i", + organ: "round_trip_trip_duration", + sprint: "Sprint 7 (PR 818)", + pr: "PR 818", + newlySolved: ["0030"], + scoreAfter: { correct: 16, refused: 34, wrong: 0 }, + lookbackDoc: "docs/analysis/gsm8k-capability-paradigm-sprint7-lookback-2026-06-17.md", + }, + { + gate: "A2j", + organ: "giveaway_target_residual", + sprint: "Sprint 7 (PR 818)", + pr: "PR 818", + newlySolved: ["0035"], + scoreAfter: { correct: 16, refused: 34, wrong: 0 }, + lookbackDoc: "docs/analysis/gsm8k-capability-paradigm-sprint7-lookback-2026-06-17.md", + }, + { + gate: "A2k", + organ: "fraction_decrease", + sprint: "Sprint 8 (PR 819)", + pr: "PR 819", + newlySolved: ["0005"], + scoreAfter: { correct: 18, refused: 32, wrong: 0 }, + lookbackDoc: "docs/analysis/gsm8k-capability-paradigm-sprint8-lookback-2026-06-17.md", + }, + { + gate: "A2l", + organ: "percent_partition", + sprint: "Sprint 8 (PR 819)", + pr: "PR 819", + newlySolved: ["0046"], + scoreAfter: { correct: 18, refused: 32, wrong: 0 }, + lookbackDoc: "docs/analysis/gsm8k-capability-paradigm-sprint8-lookback-2026-06-17.md", + }, + { + gate: "A2m", + organ: "temporal_tariff", + sprint: "Sprint 9 (PR 820/PR 822)", + pr: "PR 820", + newlySolved: ["0001"], + scoreAfter: { correct: 21, refused: 29, wrong: 0 }, + lookbackDoc: "docs/analysis/gsm8k-capability-paradigm-sprint9-lookback-2026-06-17.md", + }, + { + gate: "A2n", + organ: "affine_fraction_delta", + sprint: "Sprint 9 (PR 820/PR 822)", + pr: "PR 820", + newlySolved: ["0010", "0017"], + scoreAfter: { correct: 21, refused: 29, wrong: 0 }, + lookbackDoc: "docs/analysis/gsm8k-capability-paradigm-sprint9-lookback-2026-06-17.md", + }, + { + gate: "A2o", + organ: "affine_comparative_inversion_total", + sprint: "Sprint 10 (PR 823)", + pr: "PR 823", + newlySolved: ["0006"], + scoreAfter: { correct: 23, refused: 27, wrong: 0 }, + lookbackDoc: "docs/analysis/gsm8k-capability-paradigm-sprint10-lookback-2026-06-17.md", + }, + { + gate: "A2p", + organ: "sequential_comparative_scale", + sprint: "Sprint 10 (PR 823)", + pr: "PR 823", + newlySolved: ["0009"], + scoreAfter: { correct: 23, refused: 27, wrong: 0 }, + lookbackDoc: "docs/analysis/gsm8k-capability-paradigm-sprint10-lookback-2026-06-17.md", + }, + { + gate: "A2q", + organ: "calendar_grounded_piecewise_daily_hours_total", + sprint: "Sprint 11 (PR 824)", + pr: "PR 824", + newlySolved: ["0013"], + scoreAfter: { correct: 24, refused: 26, wrong: 0 }, + lookbackDoc: "docs/analysis/gsm8k-capability-paradigm-sprint11-lookback-2026-06-17.md", + }, +]; + +export const BLOCKED_FAMILIES: BlockedFamily[] = [ + { + family: "DCS / relation_hypothesis", + cases: ["0032", "0047"], + reason: "Sealed-wrong neighbors; divisive/count surfaces blocked until typed chain clears wrong-risk.", + }, + { + family: "currency_amount", + cases: ["0019", "0028"], + reason: "Currency grounding not ratified for serving; conservative refusal preserved.", + }, + { + family: "sealed_elimination", + cases: ["0011", "0026"], + reason: "Sealed practice elimination surfaces; serving admission blocked.", + }, + { + family: "multiplicative_aggregate (wholesale)", + cases: ["0006", "0013", "0025", "0047"], + reason: "Rejected broad MA promotion — 0025/0047 sealed-wrong share MA surface; end-to-end chain not safe.", + }, +]; + +export const CLUSTER_CONTRACT_SPRINT11 = { + familyId: "calendar_grounded_piecewise_daily_hours_total", + organs: ["piecewise_daily_hours_total (Gate A2q)", "calendar_grounding (civil_month_day_count_table)"], + includedCase: "gsm8k-train-sample-v1-0013", + provenance: "calendar_table:{month_name}", + lookbackDoc: "docs/analysis/gsm8k-capability-paradigm-sprint11-lookback-2026-06-17.md", +}; + +export const EXPERIENCE_FLYWHEEL_CLI = + "scripts/gsm8k_experience_flywheel.py --limit 50 --out /tmp/gsm8k-experience.json"; + +export const EXPERIENCE_RECORD_FIELDS: { key: string; value: string }[] = [ + { key: "record_id", value: "SHA-256 of load-bearing fields" }, + { key: "case_id / serving_status / sealed_status", value: "Per-case diagnostic posture" }, + { key: "candidate_family / first_missing_primitive", value: "Typed lift hypothesis" }, + { key: "hazard_tags / promotion_status", value: "Retention and promotion gates" }, + { key: "source_report_hash", value: "Read-only report digest — report.json never mutated" }, +]; \ No newline at end of file