From 0d203d5b55f5fec424e50c88a2f0a7022e0be08e Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 12 Jun 2026 06:44:25 -0700 Subject: [PATCH] feat(workbench): turn evidence journal + trace API Append-only JSONL journal records the exact ChatTurnResult envelope returned by /chat/turn with stable turn_id, trace_hash, all three surfaces, verdicts, and deterministic journal_digest. GET /trace/turns and GET /trace/{turn_id} serve journal evidence for the Trace route frontend. Read model only; no teaching, pack, or journal-owned engine_state mutation. --- .gitignore | 4 + docs/workbench/wave-1-evidence-spine.md | 391 ++++++++++++++++++++++++ tests/test_workbench_journal.py | 201 ++++++++++++ workbench/api.py | 42 ++- workbench/journal.py | 174 +++++++++++ workbench/schemas.py | 32 ++ workbench_data/README.md | 9 + 7 files changed, 848 insertions(+), 5 deletions(-) create mode 100644 docs/workbench/wave-1-evidence-spine.md create mode 100644 tests/test_workbench_journal.py create mode 100644 workbench/journal.py create mode 100644 workbench_data/README.md diff --git a/.gitignore b/.gitignore index 67403a8f..8996afa4 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,10 @@ __pycache__/ traces/ .formation_cache/ +workbench_data/* +!workbench_data/ +!workbench_data/README.md + core-rs/target/ core-rs/Cargo.lock diff --git a/docs/workbench/wave-1-evidence-spine.md b/docs/workbench/wave-1-evidence-spine.md new file mode 100644 index 00000000..2aed33dd --- /dev/null +++ b/docs/workbench/wave-1-evidence-spine.md @@ -0,0 +1,391 @@ +# Wave 1 — Evidence Spine + +Status: approved plan +Date: 2026-06-12 +Supersedes: implementation-plan.md phases W-032+ +Peer-reviewed by: Codex (architectural path selection), GPT5.5-Thinking (trace +honesty correction), Claude Opus 4.6 (original plan + synthesis) + +## Governing idea + +Evidence context is the intrinsic UI space. Every route is a projection of the +same evidence manifold. The workbench is not organized around routes (boxes on +screen); it is organized around the evidence chain: + +``` +operator intent + -> selected evidence subject + -> provenance + -> admissibility + -> replay + -> authority + -> allowed action +``` + +Chat shows the newest turn. Trace deepens it. Replay tests it. Proposals ask +whether it may alter reviewed memory. Packs/Vault/Audit reveal the substrate +that made it possible. Once the spine exists, remaining routes become inevitable +projections — not independent features. + +## Three governing principles (ADR-0160) + +1. **Audit-native, not analytics theater.** Every panel answers: what happened, + why was it allowed, what evidence exists, can it replay, who holds authority. +2. **Calm default, infinite depth.** Quiet surface by default; the deeper you + inspect, the more transparent it becomes. +3. **Replay before persuasion.** Show deterministic evidence before asking the + operator to trust anything. + +--- + +## Wave 1 deliverables + +Six pieces, one architectural idea. + +### 1A. Command Registry + Full Navigation + +- [ ] Replace hardcoded command list in `CommandPalette.tsx` with a + route-registered command registry +- [ ] Each route registers its commands on mount via a shared context/provider +- [ ] Fuzzy search across routes, recent resources (turns, proposals, artifacts), + and actions (run eval, copy hash) +- [ ] `Cmd+K` opens, type-ahead filters, arrow keys navigate, `Enter` executes, + `Esc` closes +- [ ] Recent items: last 10 visited resources (turns, proposals, artifacts) +- [ ] Test: palette finds and navigates to every route and registered command + +**Current state:** `CommandPalette.tsx` has only Chat/Proposals/Evals hardcoded. + +**Key files:** +- `workbench-ui/src/design/components/primitives/CommandPalette.tsx` +- New: `workbench-ui/src/app/commandRegistry.ts` (or context provider) + +--- + +### 1B. RightInspector as Evidence Drawer + +- [ ] Remove permanent `collapsed=true` from `Shell.tsx` +- [ ] Create shared evidence-subject context: `useEvidenceSubject()` hook + + provider in Shell +- [ ] Each route pushes its selection (turn, proposal, artifact, pack, eval + result) into the shared context +- [ ] Inspector renders the appropriate evidence projection for the selected + subject type +- [ ] Toggle via `Cmd+I` or "inspect" affordance on selectable items +- [ ] Stays open across route transitions (operator opened it deliberately) +- [ ] Collapsed by default on fresh load (calm default) +- [ ] Resizable width via drag handle +- [ ] Test: inspector opens, shows correct context for Chat selection, closes, + persists across route change + +**Current state:** `RightInspector.tsx` returns null. `Shell.tsx` hardcodes +`collapsed={true}`. + +**Key files:** +- `workbench-ui/src/app/RightInspector.tsx` +- `workbench-ui/src/app/Shell.tsx` +- New: `workbench-ui/src/app/evidenceContext.ts` + +--- + +### 1C. Minimal Evidence Primitives + +Build only the six components the spine needs. Defer everything else until a +route proves it requires the component. + +| Component | Purpose | Used by | +|---|---|---| +| `SplitPane` | Resizable horizontal/vertical split for list-detail | Trace, Proposals, Evals, Replay | +| `TabBar` | Accessible tab switching (see dependency note below) | Trace evidence sections, Inspector | +| `MetadataTable` | Key-value pair display for structured metadata | Trace, Proposals, Artifacts | +| `DigestBadge` | Copyable hash/digest with truncation + verify indicator | Trace, Replay, everywhere | +| `Timestamp` | Relative + absolute time, timezone-aware (PST/PDT) | All list views, Inspector | +| `SearchInput` | Filtered search with keyboard shortcut binding (`/`) | Command palette, list views | + +For each component: + +- [ ] Built with design tokens only (no raw hex/rgb) +- [ ] Motion via `--motion-duration-*` and `--motion-ease-*` tokens +- [ ] `prefers-reduced-motion` collapses to instant +- [ ] `:focus-visible` ring via `--color-focus-ring` +- [ ] Renders in PreviewPage (`/preview`) +- [ ] Unit test + +**TabBar dependency note:** `@radix-ui/react-tabs` is not currently in +`package.json` (only `react-dialog` and `react-popover` are). Either add the +dependency explicitly with lockfile update, or implement TabBar with native +ARIA tab semantics without Radix. Decide at implementation time. + +**Deferred primitives** (build when a route needs them): +- DataTable, TreeView, Timeline, CodeViewer, Drawer, Toast, SkeletonLoader, Kbd + +--- + +### 1D. Workbench Turn Evidence Journal + +**Critical correction (GPT5.5 review):** The workbench backend does NOT +currently attach a telemetry sink to chat turns. `WorkbenchApi()` constructs +with no sink, `_run_chat_turn()` creates a bare `ChatRuntime()`, and +`serialize_turn_event` redacts content by default. Raw runtime telemetry does +not contain the three surfaces needed for Trace. + +**Solution:** A Workbench Turn Evidence Journal — a local, append-only, +content-bearing record of the `ChatTurnResult` envelope already returned by +`/chat/turn`. + +This is a **read model**, not a cognitive runtime fork. It does not replace +runtime telemetry; it records the exact evidence the operator already saw. + +#### Backend + +- [x] New module: `workbench/journal.py` +- [x] `TurnJournal` class: append-only JSONL writer +- [x] Each `/chat/turn` response is journaled with: + - `turn_id` (stable, sequential) + - `timestamp` (ISO-8601 UTC) + - `trace_hash` (from ChatTurnResult) + - `prompt` (content-bearing — explicit in schema) + - `surface`, `articulation_surface`, `walk_surface` (all three, kept + separate per api-contract-v1.md line 231) + - `grounding_source`, `epistemic_state`, `normative_clearance` + - `verdicts` (identity, safety, ethics) + - `refusal_emitted`, `hedge_injected` + - `proposal_candidates` + - `turn_cost_ms` + - `journal_digest` (SHA-256 of the serialized entry) +- [x] Journal path: `workbench_data/turn_journal.jsonl` (under repo root, not + under `engine_state/` or `teaching/`) +- [x] Content-bearing warning: journal entries contain user prompts and engine + surfaces. Document this in `workbench_data/README.md` (not as a text + header in the JSONL file — every line must be valid JSON) +- [x] Path confinement: journal writes only to `workbench_data/` +- [x] No journal writes to `teaching/`, `packs/`, `language_packs/data/`, or + `engine_state/`. Note: existing chat turns DO write `engine_state/` + through the normal runtime checkpoint path governed by ADR-0146/0150 — + that is existing behavior, not journal behavior. + +#### New API endpoints + +- [x] `GET /trace/turns` — list journal entries (summary: turn_id, timestamp, + prompt excerpt, surface excerpt, trace_hash, grounding_source) +- [x] `GET /trace/{turn_id}` — full journal entry for a turn (replaces the + current 404 behavior) +- [x] Pagination: `?limit=50&offset=0` (default limit 50) +- [x] Unknown turn_id returns 404 (not synthetic data) + +#### Tests + +- [x] Append-only behavior: entries are never modified or deleted +- [x] Stable ordering: entries are sequential by turn_id +- [x] Prompt/content size limits respected (max 4096 chars prompt) +- [x] Path confinement: journal cannot write outside `workbench_data/` +- [x] No journal writes to `teaching/`, `packs/`, `language_packs/data/`; no + NEW writes to `engine_state/` beyond existing chat checkpoint behavior + (ADR-0146/0150) +- [x] Journal digest is deterministic for identical content +- [x] Round-trip: `/chat/turn` response -> journal -> `/trace/{turn_id}` -> + identical evidence fields + +#### Optional linkage to runtime telemetry + +If `ChatRuntime` is later configured with a `JsonlFileSink` +(`include_content=True`), the Trace route can cross-reference journal entries +with runtime telemetry events by `trace_hash`. This is additive — the journal +is the primary read model. + +#### Public interfaces and types + +Backend (Python): +- `workbench/journal.py` — `TurnJournalEntry` dataclass, `TurnJournalSummary` + dataclass, `TurnJournal` class (append, list, get) +- `workbench/schemas.py` — add `TurnJournalEntrySchema`, `TurnJournalSummarySchema` + for API serialization + +Frontend (TypeScript): +- `workbench-ui/src/types/api.ts` — add `TurnJournalEntry`, + `TurnJournalSummary` interfaces mirroring the Python shapes +- `workbench-ui/src/api/client.ts` — add `fetchTraceTurns(limit?, offset?)`, + `fetchTraceTurn(turnId)` +- `workbench-ui/src/api/queries.ts` — add `useTraceTurns()`, + `useTraceTurn(turnId)` React Query hooks +- `workbench-ui/src/app/evidenceContext.ts` — `EvidenceSubject` union type: + `{ kind: 'turn', data: TurnJournalEntry }` | `{ kind: 'proposal', ... }` | + `{ kind: 'artifact', ... }` | `{ kind: 'eval_result', ... }` | + `{ kind: 'none' }` + +All new API responses use the existing `{ ok, generated_at, data/error }` +envelope. + +--- + +### 1E. Trace Route + +- [ ] Replace `TraceRoutePlaceholder.tsx` with real Trace route +- [ ] Layout: `SplitPane` — turn timeline (left) + trace evidence panel (right) +- [ ] Turn timeline: list of journal entries with `DigestBadge` (trace_hash + thumbnail), `Timestamp`, prompt excerpt +- [ ] Trace evidence panel: `TabBar` with sections: + - **Surfaces** — all three, labeled explicitly (surface = user response, + walk_surface = telemetry evidence, articulation_surface = realizer + output). This IS the canonical proof of the api-contract-v1.md surface + separation contract. + - **Grounding** — source, epistemic state, normative clearance + - **Verdicts** — identity, safety, ethics verdicts with badge indicators + - **Metadata** — `MetadataTable` showing turn_cost_ms, checkpoint_emitted, + refusal/hedge status, proposal candidates + - **Raw** — collapsed-by-default full JSON viewer (StableJsonViewer) +- [ ] Selection pushes turn into evidence-subject context (RightInspector shows + same turn from inspector angle) +- [ ] `SearchInput` for filtering turns by prompt text or trace_hash prefix +- [ ] Empty state when journal is empty: "No turns recorded yet. Use Chat to + create evidence." +- [ ] Test: navigate to Trace, see real journal entries, select one, see evidence + panel, inspect raw JSON + +**Key design rules:** +- Versor condition (when available): green < 1e-6, red >= 1e-6 +- Walk surface labeled "telemetry/evidence" — never confused with user surface +- Trace hash always visible and copyable (replay before persuasion) +- Raw trace behind explicit expand (calm default, infinite depth) + +--- + +### 1F. Mutation Doctrine Reconciliation + +- [ ] Update `docs/workbench/implementation-plan.md` mutation section to match + reality +- [ ] Update `docs/workbench/acceptance-gates.md` to reflect admitted corridors +- [ ] Document the honest rule: + +**The mutation rule is not "no buttons ever." It is:** + +1. **Admitted corridor** — mutation only through an ADR-governed path + (math ratification via ADR-0172, chat turns via ADR-0146/0150). +2. **Explicit preconditions** — the UI shows what must be true before + mutation is allowed. +3. **Telemetry** — every mutation emits an auditable event. +4. **Replay evidence** — the operator can see replay-equivalence status + before acting. + +`RatificationCommandPanel.tsx` already implements this pattern for math +proposals. This is the template for future mutation surfaces, not an exception +to a "no mutation" rule. + +- [ ] Record what already exists: `ratify_math_proposal`, `reject`, `defer` + in `workbench/api.py` lines 112+; `RatificationCommandPanel.tsx` with + precondition gates +- [ ] No new mutation endpoints in Wave 1 beyond what exists + +--- + +## Wave 2 — Projections (after spine is live) + +Once the evidence spine exists, each remaining route becomes a projection. +These are parallelizable. + +### Packs Route + +- [ ] Backend: `GET /packs`, `GET /packs/{pack_id}` +- [ ] Frontend: pack list with verification badges, lexicon browser +- [ ] New primitive: `TreeView` (pack hierarchy) +- [ ] Pushes selected pack into evidence-subject context + +### Vault Route + +- [ ] Backend: `GET /vault/summary`, `GET /vault/entries`, + `GET /vault/entries/{entry_id}` +- [ ] Frontend: entry list with epistemic state badges, recall history +- [ ] Pushes selected entry into evidence-subject context + +### Audit Route + +- [ ] Backend: `GET /audit/events`, `GET /audit/events/{event_id}` +- [ ] Frontend: vertical event timeline, mutation boundary highlighting +- [ ] New primitive: `Timeline` +- [ ] Pushes selected event into evidence-subject context + +### Runs Route + +- [ ] Backend: `GET /runs`, `GET /runs/{session_id}` +- [ ] Frontend: session list with checkpoint badges, turn history +- [ ] Cross-links to Trace for any turn +- [ ] Pushes selected session into evidence-subject context + +### Settings Route + +- [ ] Frontend (mostly localStorage): inspector default, JSON depth, timestamp + format, API connection +- [ ] Runtime config display (read-only) +- [ ] No dangerous mutations — engine config changes require CLI + +--- + +## Wave 3 — Polish + Demo Theater + +### Existing module polish + +- [ ] Chat: multi-line composer, submission history, richer evidence strip +- [ ] Proposals: visual chain diagram, provenance links, metric deltas +- [ ] Eval Center: failure-first display, lane health overview, run progress +- [ ] Replay Theater: synchronized side-by-side diff, multi-artifact comparison + +### Demo Theater route + +- [ ] Backend: `GET /demos`, `POST /demos/{demo_id}/run`, + `GET /demos/{demo_id}/scenarios` +- [ ] Frontend: demo list, scenario results with evidence, "what this proves" / + "what this does not prove" honesty cards +- [ ] Evidence class badges: substrate-capability vs interface-contract +- [ ] "Proposer was wrong" scenarios visually highlighted + +--- + +## Keyboard map (global, shipped with Wave 1) + +| Shortcut | Action | +|---|---| +| `Cmd+K` | Command palette | +| `Cmd+I` | Toggle inspector | +| `Cmd+1`..`Cmd+0` | Navigate to route 1-10 | +| `j/k` or Up/Down | Navigate lists | +| `Enter` | Open selected item | +| `Esc` | Close drawer/palette/inspector | +| `/` | Focus search input | +| `?` | Show keyboard shortcut overlay | + +--- + +## Dependencies + +``` +Wave 1 (evidence spine) --> Wave 2 (projections, parallelizable) + --> Wave 3 (polish + demos) +``` + +Wave 1 must complete before Wave 2 work begins. Wave 2 routes are independent +of each other. Wave 3 can overlap with late Wave 2 work. + +--- + +## Explicit exclusions + +Per ADR-0160 doctrine and CLAUDE.md: + +- No multi-user auth, cloud deployment, or SaaS surface +- No animated "thinking" indicators or decorative motion +- No dashboard analytics walls +- No plugin/agent marketplace +- No corpus/pack mutation from the UI +- No mobile layout (engineering workstation only) +- No Deephaven, heavy JVM dependencies, or streaming database engines +- No framework upgrades (stays React 18 + stdlib HTTP) + +--- + +## Peer review record + +| Reviewer | Key contribution | +|---|---| +| Claude Opus 4.6 | Original 7-phase plan; synthesis into evidence spine after critique | +| Codex | Path selection: "evidence spine first" over routes-first; evidence chain as intrinsic UI manifold; mutation doctrine correction (admitted corridors, not "no buttons") | +| GPT5.5-Thinking | Trace honesty correction: workbench chat turns do not attach telemetry sink; `ChatTurnResult` content is not in runtime telemetry; solution = Workbench Turn Evidence Journal as honest read model | diff --git a/tests/test_workbench_journal.py b/tests/test_workbench_journal.py new file mode 100644 index 00000000..78b4004d --- /dev/null +++ b/tests/test_workbench_journal.py @@ -0,0 +1,201 @@ +from __future__ import annotations + +import json +from dataclasses import replace +from pathlib import Path + +import pytest + +from workbench import api as workbench_api +from workbench.api import MAX_CHAT_PROMPT_CHARS, WorkbenchApi +from workbench.journal import TurnJournal, TurnJournalEntry +from workbench.schemas import ChatTurnResult, TurnVerdict + + +def _chat_result(prompt: str = "What is truth?") -> ChatTurnResult: + return ChatTurnResult( + prompt=prompt, + surface="Truth is coherent structure.", + articulation_surface="Truth is coherent structure.", + walk_surface="truth -> coherence", + grounding_source="pack", + epistemic_state="decoded", + normative_clearance="cleared", + normative_detail="", + trace_hash="sha256:trace", + refusal_emitted=False, + hedge_injected=False, + mutation_mode="runtime_turn", + identity_verdict=TurnVerdict(outcome="cleared", runtime_detail=""), + safety_verdict=TurnVerdict(outcome="cleared", runtime_detail=""), + ethics_verdict=TurnVerdict(outcome="cleared", runtime_detail=""), + proposal_candidates=[], + turn_cost_ms=7, + checkpoint_emitted=False, + ) + + +def _entry(turn_id: int = 1, prompt: str = "What is truth?") -> TurnJournalEntry: + result = replace(_chat_result(prompt), turn_id=turn_id) + return TurnJournalEntry.from_chat_turn( + result, + turn_id=turn_id, + timestamp="2026-06-12T00:00:00+00:00", + ) + + +def _request(api: WorkbenchApi, method: str, path: str, body: dict | None = None): + raw = b"" if body is None else json.dumps(body).encode("utf-8") + return api.handle(method, path, raw) + + +def _snapshot(root: Path) -> dict[str, bytes]: + snap: dict[str, bytes] = {} + if not root.exists(): + return snap + for path in sorted(root.rglob("*")): + if path.is_file() and "__pycache__" not in path.relative_to(root).parts: + snap[path.relative_to(root).as_posix()] = path.read_bytes() + return snap + + +def test_journal_appends_without_modifying_existing_entries(tmp_path: Path) -> None: + journal = TurnJournal(tmp_path / "workbench_data") + + first = journal.append(_entry(1)) + first_line = journal.path.read_text(encoding="utf-8").splitlines()[0] + second = journal.append(_entry(2, "What is memory?")) + + lines = journal.path.read_text(encoding="utf-8").splitlines() + assert len(lines) == 2 + assert lines[0] == first_line + assert first.journal_digest + assert second.turn_id == 2 + + +def test_journal_ordering_and_pagination_are_sequential(tmp_path: Path) -> None: + journal = TurnJournal(tmp_path / "workbench_data") + for turn_id in range(1, 5): + journal.append(_entry(turn_id, f"prompt {turn_id}")) + + assert journal.next_turn_id() == 5 + page = journal.list_summaries(limit=2, offset=1) + assert [item.turn_id for item in page] == [2, 3] + + +def test_trace_turns_offset_beyond_journal_length_returns_empty_items( + tmp_path: Path, +) -> None: + journal = TurnJournal(tmp_path / "workbench_data") + journal.append(_entry(1)) + api = WorkbenchApi(journal=journal) + + response = _request(api, "GET", "/trace/turns?limit=50&offset=50") + + assert response.status == 200 + assert response.payload["data"]["items"] == [] + + +def test_empty_journal_returns_empty_items(tmp_path: Path) -> None: + api = WorkbenchApi(journal_dir=tmp_path / "workbench_data") + + response = _request(api, "GET", "/trace/turns") + + assert response.status == 200 + assert response.payload["data"]["items"] == [] + + +def test_prompt_size_limit_is_enforced_before_journaling(tmp_path: Path) -> None: + api = WorkbenchApi(journal_dir=tmp_path / "workbench_data") + prompt = "x" * (MAX_CHAT_PROMPT_CHARS + 1) + + response = _request(api, "POST", "/chat/turn", {"prompt": prompt}) + + assert response.status == 400 + assert not (tmp_path / "workbench_data" / "turn_journal.jsonl").exists() + + +def test_journal_rejects_paths_outside_workbench_data(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="workbench_data"): + TurnJournal(tmp_path / "not_workbench_data") + + +def test_journal_does_not_write_teaching_pack_or_engine_state_roots( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo_root = Path(__file__).resolve().parent.parent + guarded = { + "teaching": repo_root / "teaching", + "packs": repo_root / "packs", + "language_packs/data": repo_root / "language_packs" / "data", + "engine_state": repo_root / "engine_state", + } + before = {name: _snapshot(path) for name, path in guarded.items()} + + def fake_run(prompt: str) -> ChatTurnResult: + return _chat_result(prompt) + + monkeypatch.setattr(workbench_api, "_run_chat_turn", fake_run) + api = WorkbenchApi(journal_dir=tmp_path / "workbench_data") + + response = _request(api, "POST", "/chat/turn", {"prompt": "What is truth?"}) + + assert response.status == 200 + assert {name: _snapshot(path) for name, path in guarded.items()} == before + assert api._journal.path.exists() # noqa: SLF001 - verifies the configured boundary. + + +def test_journal_digest_is_deterministic_for_identical_content(tmp_path: Path) -> None: + journal = TurnJournal(tmp_path / "workbench_data") + first = journal.append(_entry(1)) + + other_journal = TurnJournal(tmp_path / "other" / "workbench_data") + second = other_journal.append(_entry(1)) + + assert first.journal_digest == second.journal_digest + + +def test_chat_turn_round_trips_through_trace_endpoint( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fake_run(prompt: str) -> ChatTurnResult: + return _chat_result(prompt) + + monkeypatch.setattr(workbench_api, "_run_chat_turn", fake_run) + api = WorkbenchApi(journal_dir=tmp_path / "workbench_data") + + chat = _request(api, "POST", "/chat/turn", {"prompt": "What is truth?"}) + turn_id = chat.payload["data"]["turn_id"] + trace = _request(api, "GET", f"/trace/{turn_id}") + + assert chat.status == 200 + assert trace.status == 200 + for field in [ + "turn_id", + "prompt", + "surface", + "articulation_surface", + "walk_surface", + "trace_hash", + "grounding_source", + "epistemic_state", + "normative_clearance", + "refusal_emitted", + "hedge_injected", + "proposal_candidates", + "turn_cost_ms", + "checkpoint_emitted", + ]: + assert trace.payload["data"][field] == chat.payload["data"][field] + + +def test_unknown_turn_returns_404(tmp_path: Path) -> None: + api = WorkbenchApi(journal_dir=tmp_path / "workbench_data") + + missing = _request(api, "GET", "/trace/999") + invalid = _request(api, "GET", "/trace/not-a-real-turn") + + assert missing.status == 404 + assert invalid.status == 404 diff --git a/workbench/api.py b/workbench/api.py index 9740761c..761ce0b6 100644 --- a/workbench/api.py +++ b/workbench/api.py @@ -6,6 +6,7 @@ import json import threading import time from dataclasses import dataclass +from pathlib import Path from typing import Any from urllib.parse import parse_qs, unquote, urlparse @@ -17,6 +18,7 @@ from core.epistemic_state import ( normative_detail_from_verdicts, ) from workbench import readers +from workbench.journal import DEFAULT_JOURNAL_DIR, TurnJournal, TurnJournalEntry from workbench.readers import ArtifactTooLargeError from workbench.schemas import ChatTurnResult, MathRatifyResult, ProposalRef, TurnVerdict, error, ok @@ -33,8 +35,17 @@ class ApiResponse: class WorkbenchApi: - def __init__(self, telemetry_sink: Any | None = None) -> None: + def __init__( + self, + telemetry_sink: Any | None = None, + *, + journal: TurnJournal | None = None, + journal_dir: Any | None = None, + ) -> None: self._telemetry_sink = telemetry_sink + self._journal = journal or TurnJournal( + DEFAULT_JOURNAL_DIR if journal_dir is None else Path(journal_dir) + ) def attach_telemetry_sink(self, sink: Any | None) -> None: self._telemetry_sink = sink @@ -145,8 +156,21 @@ class WorkbenchApi: return ApiResponse(200, ok(result)) if method == "POST" and path == "/chat/turn": return self._chat_turn(body) + if method == "GET" and path == "/trace/turns": + limit = int(query.get("limit", ["50"])[0]) + offset = int(query.get("offset", ["0"])[0]) + items = self._journal.list_summaries(limit=limit, offset=offset) + return ApiResponse(200, ok({"items": items})) if method == "GET" and path.startswith("/trace/"): - return ApiResponse(404, error("not_found", "trace storage is not wired in W-026")) + raw_turn_id = unquote(path.removeprefix("/trace/")) + try: + turn_id = int(raw_turn_id) + except ValueError: + return ApiResponse(404, error("not_found", f"trace turn not found: {raw_turn_id}")) + try: + return ApiResponse(200, ok(self._journal.get_entry(turn_id))) + except FileNotFoundError: + return ApiResponse(404, error("not_found", f"trace turn not found: {turn_id}")) if method == "GET" and path.startswith("/replay/"): return ApiResponse(501, error("unsupported", "route is deferred beyond W-026")) return ApiResponse(404, error("not_found", f"route not found: {method} {path}")) @@ -278,13 +302,21 @@ class WorkbenchApi: started = time.perf_counter() result = _run_chat_turn(prompt) elapsed_ms = max(0, int(round((time.perf_counter() - started) * 1000))) - return ApiResponse(200, ok(_with_turn_cost(result, elapsed_ms))) + turn_id = self._journal.next_turn_id() + result_with_cost = _with_turn_cost_and_id(result, elapsed_ms, turn_id) + entry = TurnJournalEntry.from_chat_turn(result_with_cost, turn_id=turn_id) + self._journal.append(entry) + return ApiResponse(200, ok(result_with_cost)) -def _with_turn_cost(result: ChatTurnResult, turn_cost_ms: int) -> ChatTurnResult: +def _with_turn_cost_and_id( + result: ChatTurnResult, + turn_cost_ms: int, + turn_id: int, +) -> ChatTurnResult: from dataclasses import replace - return replace(result, turn_cost_ms=turn_cost_ms) + return replace(result, turn_cost_ms=turn_cost_ms, turn_id=turn_id) def _coerce_grounding_source(value: object) -> str: diff --git a/workbench/journal.py b/workbench/journal.py new file mode 100644 index 00000000..2f97a805 --- /dev/null +++ b/workbench/journal.py @@ -0,0 +1,174 @@ +"""Append-only Workbench turn evidence journal.""" + +from __future__ import annotations + +import hashlib +import json +import threading +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Any + +from workbench.schemas import ChatTurnResult, to_data, utc_now + + +REPO_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_JOURNAL_DIR = REPO_ROOT / "workbench_data" +JOURNAL_FILENAME = "turn_journal.jsonl" +PROMPT_EXCERPT_CHARS = 120 +SURFACE_EXCERPT_CHARS = 120 + + +@dataclass(frozen=True, slots=True) +class TurnJournalSummary: + turn_id: int + timestamp: str + prompt_excerpt: str + surface_excerpt: str + trace_hash: str | None + grounding_source: str + + +@dataclass(frozen=True, slots=True) +class TurnJournalEntry: + turn_id: int + timestamp: str + trace_hash: str | None + prompt: str + surface: str + articulation_surface: str | None + walk_surface: str | None + grounding_source: str + epistemic_state: str + normative_clearance: str + verdicts: dict[str, Any] + refusal_emitted: bool + hedge_injected: bool + proposal_candidates: list[dict[str, Any]] + turn_cost_ms: int + checkpoint_emitted: bool + journal_digest: str = "" + + @classmethod + def from_chat_turn( + cls, + result: ChatTurnResult, + *, + turn_id: int, + timestamp: str | None = None, + ) -> "TurnJournalEntry": + return cls( + turn_id=turn_id, + timestamp=timestamp or utc_now(), + trace_hash=result.trace_hash, + prompt=result.prompt, + surface=result.surface, + articulation_surface=result.articulation_surface, + walk_surface=result.walk_surface, + grounding_source=result.grounding_source, + epistemic_state=result.epistemic_state, + normative_clearance=result.normative_clearance, + verdicts={ + "identity": to_data(result.identity_verdict), + "safety": to_data(result.safety_verdict), + "ethics": to_data(result.ethics_verdict), + }, + refusal_emitted=result.refusal_emitted, + hedge_injected=result.hedge_injected, + proposal_candidates=[ + candidate for candidate in to_data(result.proposal_candidates) + ], + turn_cost_ms=result.turn_cost_ms, + checkpoint_emitted=result.checkpoint_emitted, + ) + + def summary(self) -> TurnJournalSummary: + return TurnJournalSummary( + turn_id=self.turn_id, + timestamp=self.timestamp, + prompt_excerpt=self.prompt[:PROMPT_EXCERPT_CHARS], + surface_excerpt=self.surface[:SURFACE_EXCERPT_CHARS], + trace_hash=self.trace_hash, + grounding_source=self.grounding_source, + ) + + +class TurnJournal: + """Pure JSONL append/read model for Workbench chat evidence.""" + + def __init__(self, journal_dir: Path = DEFAULT_JOURNAL_DIR) -> None: + self._journal_dir = _validate_journal_dir(journal_dir) + self._path = self._journal_dir / JOURNAL_FILENAME + self._lock = threading.Lock() + + @property + def journal_dir(self) -> Path: + return self._journal_dir + + @property + def path(self) -> Path: + return self._path + + def next_turn_id(self) -> int: + entries = self._read_entries() + if not entries: + return 1 + return max(entry.turn_id for entry in entries) + 1 + + def append(self, entry: TurnJournalEntry) -> TurnJournalEntry: + with self._lock: + expected = self.next_turn_id() + if entry.turn_id != expected: + raise ValueError( + f"turn_id must be next sequential id {expected}, got {entry.turn_id}" + ) + sealed = replace(entry, journal_digest=_journal_digest(entry)) + self._journal_dir.mkdir(parents=True, exist_ok=True) + with self._path.open("a", encoding="utf-8") as fh: + fh.write(_canonical_json(to_data(sealed))) + fh.write("\n") + return sealed + + def list_summaries(self, *, limit: int = 50, offset: int = 0) -> list[TurnJournalSummary]: + if limit < 0: + raise ValueError("limit must be non-negative") + if offset < 0: + raise ValueError("offset must be non-negative") + entries = self._read_entries() + return [entry.summary() for entry in entries[offset : offset + limit]] + + def get_entry(self, turn_id: int) -> TurnJournalEntry: + for entry in self._read_entries(): + if entry.turn_id == turn_id: + return entry + raise FileNotFoundError(str(turn_id)) + + def _read_entries(self) -> list[TurnJournalEntry]: + if not self._path.exists(): + return [] + entries: list[TurnJournalEntry] = [] + with self._path.open("r", encoding="utf-8") as fh: + for line in fh: + if not line.strip(): + continue + payload = json.loads(line) + entries.append(TurnJournalEntry(**payload)) + return entries + + +def _validate_journal_dir(journal_dir: Path) -> Path: + resolved = journal_dir.resolve() + if resolved.name != "workbench_data": + raise ValueError("journal directory must be named workbench_data") + return resolved + + +def _canonical_json(payload: dict[str, Any]) -> str: + return json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def _journal_digest(entry: TurnJournalEntry) -> str: + payload = to_data(replace(entry, journal_digest="")) + payload.pop("journal_digest", None) + raw = _canonical_json(payload).encode("utf-8") + return "sha256:" + hashlib.sha256(raw).hexdigest() diff --git a/workbench/schemas.py b/workbench/schemas.py index c4231e59..48537640 100644 --- a/workbench/schemas.py +++ b/workbench/schemas.py @@ -113,6 +113,38 @@ class ChatTurnResult: proposal_candidates: list[ProposalRef] turn_cost_ms: int checkpoint_emitted: bool + turn_id: int | None = None + + +@dataclass(frozen=True, slots=True) +class TurnJournalSummarySchema: + turn_id: int + timestamp: str + prompt_excerpt: str + surface_excerpt: str + trace_hash: str | None + grounding_source: GroundingSource + + +@dataclass(frozen=True, slots=True) +class TurnJournalEntrySchema: + turn_id: int + timestamp: str + trace_hash: str | None + prompt: str + surface: str + articulation_surface: str | None + walk_surface: str | None + grounding_source: GroundingSource + epistemic_state: EpistemicStateValue + normative_clearance: NormativeClearanceValue + verdicts: dict[str, Any] + refusal_emitted: bool + hedge_injected: bool + proposal_candidates: list[dict[str, Any]] + turn_cost_ms: int + checkpoint_emitted: bool + journal_digest: str @dataclass(frozen=True, slots=True) diff --git a/workbench_data/README.md b/workbench_data/README.md new file mode 100644 index 00000000..80c6667b --- /dev/null +++ b/workbench_data/README.md @@ -0,0 +1,9 @@ +# Workbench Data + +`turn_journal.jsonl` is a local, content-bearing evidence journal for CORE +Workbench chat turns. Entries contain user prompts and engine surfaces already +returned to the local operator by `/chat/turn`. + +This directory is a Workbench read model only. It is not teaching memory, +runtime memory, semantic pack data, or a cognitive runtime fork. Generated JSONL +files are intentionally ignored by git.