feat(W-026): read-only workbench API (ADR-0160 Phase 1) (#292)
* feat(W-026): add read-only workbench API * fix(workbench): harden read-only API review gaps
This commit is contained in:
Shay
GitHub
parent
0909ef2782
commit
8a24ebe726
14 changed files with 1262 additions and 9 deletions
36
core/cli.py
36
core/cli.py
|
|
@ -23,7 +23,7 @@ _CORE_RS_DIR = _REPO_ROOT / "core-rs"
|
|||
_CORE_RS_MANIFEST = _CORE_RS_DIR / "Cargo.toml"
|
||||
|
||||
DESCRIPTION = "CORE versor engine command suite."
|
||||
EPILOG = "Examples:\n core chat\n core pulse \"What is truth?\"\n core pulse --no-glove --json \"Compare knowledge and wisdom\"\n core bench\n core bench --suite all\n core bench --suite all --json --report bench_all.json\n core bench --suite determinism --runs 50\n core bench --suite speedup --json\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core teaching audit\n core teaching audit --json\n core teaching gaps --top 10\n core teaching queue --threshold 3\n core teaching propose <candidate-jsonl-path>\n core teaching proposals --state pending\n core teaching review <proposal_id> --accept --review-date 2026-05-18\n core teaching supersede cause_light_reveals_truth --subject light --intent cause --connective grounds --object truth --review-date 2026-05-18\n core teaching supersessions\n core teaching supersessions --json\n core test --suite fast -q\n core test --suite pulse -q\n core test --suite proof -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core demo audit-tour\n core demo register-tour\n core demo anchor-lens-tour\n core demo orthogonality-tour\n core demo pack-measurements\n core demo long-context-comparison\n core demo anti-regression\n core demo learning-loop\n core demo learning-arc\n core demo articulation\n core demo conversation\n core demo conversation --no-stream\n core demo all\n core demo adr-0024-chain\n core eval --list\n core eval cognition\n core eval cognition --json --save\n core eval cognition --split dev --version v1\n core eval cognition --split holdout\n core eval contemplation_quality\n core eval contemplation_quality --json --save"
|
||||
EPILOG = "Examples:\n core chat\n core pulse \"What is truth?\"\n core pulse --no-glove --json \"Compare knowledge and wisdom\"\n core bench\n core bench --suite all\n core bench --suite all --json --report bench_all.json\n core bench --suite determinism --runs 50\n core bench --suite speedup --json\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core teaching audit\n core teaching audit --json\n core teaching gaps --top 10\n core teaching queue --threshold 3\n core teaching propose <candidate-jsonl-path>\n core teaching proposals --state pending\n core teaching review <proposal_id> --accept --review-date 2026-05-18\n core teaching supersede cause_light_reveals_truth --subject light --intent cause --connective grounds --object truth --review-date 2026-05-18\n core teaching supersessions\n core teaching supersessions --json\n core test --suite fast -q\n core test --suite pulse -q\n core test --suite proof -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core demo audit-tour\n core demo register-tour\n core demo anchor-lens-tour\n core demo orthogonality-tour\n core demo pack-measurements\n core demo long-context-comparison\n core demo anti-regression\n core demo learning-loop\n core demo learning-arc\n core demo articulation\n core demo conversation\n core demo conversation --no-stream\n core demo all\n core demo adr-0024-chain\n core eval --list\n core eval cognition\n core eval cognition --json --save\n core eval cognition --split dev --version v1\n core eval cognition --split holdout\n core eval contemplation_quality\n core eval contemplation_quality --json --save\n core workbench api\n core workbench api --port 9000\n core workbench api --host 0.0.0.0 --allow-nonlocal-bind"
|
||||
|
||||
_TEST_SUITES: dict[str, tuple[str, ...]] = {
|
||||
"fast": (
|
||||
|
|
@ -1795,6 +1795,18 @@ def cmd_eval(args: argparse.Namespace) -> int:
|
|||
return 0
|
||||
|
||||
|
||||
def cmd_workbench(args: argparse.Namespace) -> int:
|
||||
"""Run CORE Workbench local operator surfaces."""
|
||||
if args.workbench_command == "api":
|
||||
from workbench.server import main as workbench_api_main
|
||||
|
||||
argv = ["--host", args.host, "--port", str(args.port)]
|
||||
if args.allow_nonlocal_bind:
|
||||
argv.append("--allow-nonlocal-bind")
|
||||
return workbench_api_main(argv)
|
||||
_die("workbench requires a subcommand")
|
||||
|
||||
|
||||
def cmd_pulse(args: argparse.Namespace) -> int:
|
||||
"""Run a cognitive pulse and display recalled words + realized surface."""
|
||||
from scripts.run_pulse import run_pulse
|
||||
|
|
@ -3680,6 +3692,28 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
rust_test = rust_sub.add_parser("test", help="run cargo test --release for core-rs")
|
||||
rust_test.set_defaults(func=cmd_rust_test)
|
||||
|
||||
workbench = subparsers.add_parser(
|
||||
"workbench",
|
||||
help="run CORE Workbench local operator surfaces",
|
||||
)
|
||||
workbench_sub = workbench.add_subparsers(
|
||||
dest="workbench_command",
|
||||
metavar="workbench-command",
|
||||
required=True,
|
||||
)
|
||||
workbench_api = workbench_sub.add_parser(
|
||||
"api",
|
||||
help="start the W-026 read-only local Workbench API",
|
||||
)
|
||||
workbench_api.add_argument("--host", default="127.0.0.1")
|
||||
workbench_api.add_argument("--port", type=int, default=8765)
|
||||
workbench_api.add_argument(
|
||||
"--allow-nonlocal-bind",
|
||||
action="store_true",
|
||||
help="allow binding to a host other than 127.0.0.1 or localhost",
|
||||
)
|
||||
workbench_api.set_defaults(func=cmd_workbench)
|
||||
|
||||
pulse = subparsers.add_parser(
|
||||
"pulse",
|
||||
help="run a cognitive pulse from injection to realized surface",
|
||||
|
|
|
|||
|
|
@ -136,6 +136,66 @@ mutation capability.
|
|||
|
||||
---
|
||||
|
||||
# Current Status
|
||||
|
||||
The planning package is merged on `main` via `404e694`
|
||||
(`docs(workbench): CORE Workbench v1 planning architecture (ADR-0160)`).
|
||||
|
||||
The prototype branch `feat/w026-workbench-readonly-api` is superseded and must
|
||||
not be used as the implementation base. It mixed W-026 with frontend and trace
|
||||
work, added auth and web-framework dependencies before the local read-only
|
||||
boundary was proven, and included placeholder replay/trace behavior that could
|
||||
be mistaken for evidence.
|
||||
|
||||
The next accepted implementation starts clean from `main` with W-026 only:
|
||||
dataclass schemas, repo-root-constrained readers, a standard-library local HTTP
|
||||
API, and route/read-model tests. W-027 and later phases build on that boundary
|
||||
after it is accepted.
|
||||
|
||||
---
|
||||
|
||||
# W-026 Local Runbook
|
||||
|
||||
Start the read-only local API:
|
||||
|
||||
```bash
|
||||
core workbench api
|
||||
```
|
||||
|
||||
Verify liveness:
|
||||
|
||||
```bash
|
||||
curl http://127.0.0.1:8765/health
|
||||
```
|
||||
|
||||
Inspect each W-026 endpoint family:
|
||||
|
||||
```bash
|
||||
curl http://127.0.0.1:8765/runtime/status
|
||||
curl http://127.0.0.1:8765/artifacts
|
||||
curl http://127.0.0.1:8765/artifacts/evals/contemplation_quality/contract.md
|
||||
curl http://127.0.0.1:8765/proposals
|
||||
curl http://127.0.0.1:8765/evals
|
||||
curl http://127.0.0.1:8765/evals/contemplation_quality
|
||||
curl -X POST http://127.0.0.1:8765/evals/run \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"lane":"contemplation_quality","version":"v1","split":"public"}'
|
||||
```
|
||||
|
||||
Bind to a different local port:
|
||||
|
||||
```bash
|
||||
core workbench api --port 9000
|
||||
```
|
||||
|
||||
Non-local binds require an explicit operator flag:
|
||||
|
||||
```bash
|
||||
core workbench api --host 0.0.0.0 --allow-nonlocal-bind
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
# Initial Work Queue
|
||||
|
||||
| Work Item | Goal |
|
||||
|
|
|
|||
|
|
@ -23,18 +23,34 @@ The planning branch is acceptable when it includes:
|
|||
Required:
|
||||
|
||||
- local-only API defaults to `127.0.0.1`
|
||||
- non-local bind requires an explicit operator flag
|
||||
- typed response schemas
|
||||
- route tests
|
||||
- path traversal tests for artifact readers
|
||||
- proposal event-log read-model tests using `ProposalLog.current_state()`
|
||||
- unknown trace ids return `404`, not placeholder success payloads
|
||||
- no proposal accept/reject route
|
||||
- no corpus mutation route
|
||||
- no pack mutation route
|
||||
- no workflow dispatch route
|
||||
- no hidden background worker
|
||||
- no frontend commits
|
||||
- no auth surface
|
||||
- no FastAPI/uvicorn/pydantic dependency
|
||||
|
||||
Blockers:
|
||||
|
||||
- fake replay equality by comparing an artifact digest to itself
|
||||
- placeholder trace-as-success responses
|
||||
- parsing proposal events as proposal records
|
||||
- non-local bind without explicit operator opt-in
|
||||
- auth added before the local read-only boundary is accepted
|
||||
- frontend, visual intro, trace drawer, or chat UI included in W-026
|
||||
|
||||
Acceptance command candidates:
|
||||
|
||||
```bash
|
||||
uv run python -m pytest tests/test_workbench_schemas.py -q
|
||||
uv run python -m pytest tests/test_workbench_api.py -q
|
||||
uv run python -m pytest tests/test_workbench_readers.py -q
|
||||
```
|
||||
|
|
|
|||
|
|
@ -23,7 +23,10 @@ without widening the mutation surface.
|
|||
|
||||
## Response envelope
|
||||
|
||||
All responses SHOULD follow:
|
||||
All responses MUST include `generated_at` as an ISO-8601 UTC timestamp.
|
||||
`generated_at` is wall-clock UTC and not part of replay state. Clients hashing
|
||||
or caching responses must exclude this field.
|
||||
Successful responses:
|
||||
|
||||
```json
|
||||
{
|
||||
|
|
@ -38,6 +41,7 @@ Errors:
|
|||
```json
|
||||
{
|
||||
"ok": false,
|
||||
"generated_at": "2026-05-26T00:00:00Z",
|
||||
"error": {
|
||||
"code": "not_found",
|
||||
"message": "artifact not found"
|
||||
|
|
@ -45,6 +49,51 @@ Errors:
|
|||
}
|
||||
```
|
||||
|
||||
W-026 error codes:
|
||||
|
||||
- `bad_request`
|
||||
- `not_found`
|
||||
- `unsupported`
|
||||
- `read_error`
|
||||
- `eval_failed`
|
||||
- `runtime_unavailable`
|
||||
|
||||
`unauthorized` is reserved for a later auth ADR. W-026 remains unauthenticated
|
||||
and local-only by default.
|
||||
|
||||
Exception mapping:
|
||||
|
||||
| Condition | HTTP status | Error code |
|
||||
|---|---:|---|
|
||||
| malformed request, invalid path, unsafe traversal | 400 | `bad_request` |
|
||||
| well-formed missing artifact/proposal/lane/trace | 404 | `not_found` |
|
||||
| artifact exceeds the W-026 read size limit | 413 | `read_error` |
|
||||
| deferred W-027+ route | 501 | `unsupported` |
|
||||
| filesystem read failure | 500 | `read_error` |
|
||||
| unexpected runtime failure | 500 | `runtime_unavailable` |
|
||||
|
||||
## W-026 execution model
|
||||
|
||||
The W-026 API is a single-operator local service. It does not implement CORS
|
||||
preflight; browser clients must run on the same origin until W-027 defines the
|
||||
frontend serving model. Eval execution is serialized per server instance so
|
||||
two `/evals/run` requests cannot race over shared runtime checkpoint files.
|
||||
|
||||
### Side effects on `engine_state/`
|
||||
|
||||
W-026 is read-only with respect to teaching corpora and pack data. Two routes
|
||||
touch runtime checkpoint surfaces governed by existing ADRs:
|
||||
|
||||
- `GET /runtime/status` instantiates `EngineStateStore` and reads
|
||||
`engine_state/manifest.json` via `load_manifest`.
|
||||
- `POST /evals/run` with `lane="contemplation_quality"` transitively invokes
|
||||
the ADR-0159 replay baseline. That path can create normal `ChatRuntime`
|
||||
checkpoints under `engine_state/`, governed by ADR-0146 and ADR-0150.
|
||||
|
||||
These checkpoint writes are runtime artifacts, not teaching/corpus/pack
|
||||
mutation. They are intentionally excluded from the W-026 read-only snapshot
|
||||
invariant.
|
||||
|
||||
---
|
||||
|
||||
# Runtime
|
||||
|
|
@ -139,6 +188,12 @@ Purpose:
|
|||
|
||||
Inspect trace evidence for a turn.
|
||||
|
||||
W-026 status:
|
||||
|
||||
Trace storage is not implemented in W-026. Unknown turn ids MUST return
|
||||
`404 not_found`. The API must never return a synthetic empty trace as a
|
||||
successful response.
|
||||
|
||||
Response:
|
||||
|
||||
```json
|
||||
|
|
@ -170,6 +225,13 @@ Purpose:
|
|||
|
||||
Return replay comparison information.
|
||||
|
||||
Important:
|
||||
|
||||
The API must not claim `"equivalent": true` unless a real replay/compare path
|
||||
has run. Comparing an artifact digest to itself is not replay evidence. Until
|
||||
the replay theater is wired, this route should return `unsupported` or a
|
||||
non-equivalent comparison with explicit divergence evidence.
|
||||
|
||||
Response:
|
||||
|
||||
```json
|
||||
|
|
@ -178,9 +240,16 @@ Response:
|
|||
"data": {
|
||||
"artifact_id": "artifact-001",
|
||||
"original_hash": "sha256:...",
|
||||
"replay_hash": "sha256:...",
|
||||
"equivalent": true,
|
||||
"divergences": []
|
||||
"replay_hash": null,
|
||||
"equivalent": false,
|
||||
"divergences": [
|
||||
{
|
||||
"path": "$",
|
||||
"original": "artifact digest",
|
||||
"replay": null,
|
||||
"severity": "info"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
|
@ -195,6 +264,12 @@ Purpose:
|
|||
|
||||
List proposal metadata.
|
||||
|
||||
Implementation rule:
|
||||
|
||||
Proposal reads MUST derive their current view by replaying
|
||||
`teaching.proposals.ProposalLog.current_state()`. The append-only JSONL file is
|
||||
an event log, not a table of proposal records.
|
||||
|
||||
Response:
|
||||
|
||||
```json
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ boundaries while making the system legible to operators, engineers, and auditors
|
|||
|
||||
| Work item | Title | Scope | Mutation allowed? |
|
||||
|---|---|---|---|
|
||||
| W-026 | Read-only API contract | endpoint schemas + local server surface | No |
|
||||
| W-026 | Read-only API contract | endpoint schemas + stdlib local server surface only | No |
|
||||
| W-027 | Frontend shell | navigation, layout, design tokens, empty states | No |
|
||||
| W-028 | Chat + trace drawer | basic chat, turn metadata, trace drawer | Runtime turn only |
|
||||
| W-029 | Proposal queue | proposal-log read model, detail view, CLI-copy affordance | No |
|
||||
|
|
@ -39,6 +39,7 @@ Deliverables:
|
|||
- `workbench/api.py`
|
||||
- `workbench/schemas.py`
|
||||
- `workbench/readers.py`
|
||||
- `workbench/server.py`
|
||||
- route tests with temp fixtures
|
||||
- CLI command: `core workbench api --host 127.0.0.1 --port 8765`
|
||||
|
||||
|
|
@ -47,11 +48,22 @@ Initial endpoints:
|
|||
- `GET /health`
|
||||
- `GET /runtime/status`
|
||||
- `GET /artifacts`
|
||||
- `GET /artifacts/{artifact_id}`
|
||||
- `GET /proposals`
|
||||
- `GET /proposals/{proposal_id}`
|
||||
- `GET /evals`
|
||||
- `GET /evals/{lane}`
|
||||
- `POST /evals/run`
|
||||
|
||||
Strict rule: no proposal accept/reject endpoints.
|
||||
Strict rules:
|
||||
|
||||
- W-026 is API-only. No frontend, auth, visual intro, trace drawer, chat UI,
|
||||
live chat endpoint, or replay theater.
|
||||
- Use the Python standard-library HTTP server for W-026. Defer FastAPI or any
|
||||
other web framework unless a later ADR admits the dependency.
|
||||
- No proposal accept/reject endpoints.
|
||||
- No synthetic trace/replay evidence. Routes for later phases must return
|
||||
`unsupported` or `not_found` until a real evidence path exists.
|
||||
|
||||
### Phase 2 — Frontend shell
|
||||
|
||||
|
|
@ -153,12 +165,14 @@ Deliverables:
|
|||
|
||||
## API design constraints
|
||||
|
||||
- All schemas must be explicit dataclasses or pydantic models.
|
||||
- All schemas must be explicit dataclasses in W-026.
|
||||
- No raw arbitrary filesystem reads from user-provided paths.
|
||||
- Artifact readers must be rooted under known repo directories.
|
||||
- Every response should contain enough metadata to audit source path, digest,
|
||||
and timestamp when available.
|
||||
- Mutation endpoints are forbidden in v1.
|
||||
- Proposal reads must derive state from `ProposalLog.current_state()`, not by
|
||||
treating append-only JSONL events as proposal rows.
|
||||
|
||||
## Frontend constraints
|
||||
|
||||
|
|
@ -194,12 +208,24 @@ The v1 workbench is accepted when an operator can:
|
|||
|
||||
## Explicit deferrals
|
||||
|
||||
- auth
|
||||
- multi-user collaboration
|
||||
- cloud deployment
|
||||
- mobile UI
|
||||
- auth, unless a separate ADR admits it after the local read-only boundary is
|
||||
proven
|
||||
- proposal accept/reject buttons
|
||||
- workflow dispatch
|
||||
- packaged desktop app
|
||||
- public marketing site
|
||||
- arbitrary plugin/tool execution
|
||||
|
||||
## Prototype branch rejection criteria
|
||||
|
||||
Do not continue from a prototype branch that:
|
||||
|
||||
- mixes W-026 with W-027/W-028 frontend or trace/chat work
|
||||
- adds auth before the local read-only API boundary is accepted
|
||||
- adds FastAPI, uvicorn, pydantic, or another web framework without ADR review
|
||||
- claims replay equivalence by comparing an artifact digest to itself
|
||||
- returns placeholder trace data as a successful trace response
|
||||
- parses proposal log events as if they were proposal records
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ include = [
|
|||
"session*",
|
||||
"vault*",
|
||||
"vocab*",
|
||||
"workbench*",
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
|
|
|
|||
318
tests/test_workbench_api.py
Normal file
318
tests/test_workbench_api.py
Normal file
|
|
@ -0,0 +1,318 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import socket
|
||||
import threading
|
||||
from http.server import ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from workbench import readers
|
||||
from workbench.api import WorkbenchApi
|
||||
from workbench import server
|
||||
from workbench.server import WorkbenchRequestHandler
|
||||
|
||||
|
||||
def _request(method: str, path: str, body: dict | None = None):
|
||||
raw = b"" if body is None else json.dumps(body).encode("utf-8")
|
||||
return WorkbenchApi().handle(method, path, raw)
|
||||
|
||||
|
||||
def _without_generated_at(payload: dict) -> dict:
|
||||
out = dict(payload)
|
||||
out.pop("generated_at", None)
|
||||
return out
|
||||
|
||||
|
||||
def _snapshot(root: Path) -> dict[str, bytes]:
|
||||
snap: dict[str, bytes] = {}
|
||||
if not root.exists():
|
||||
return snap
|
||||
for path in sorted(root.rglob("*")):
|
||||
if not path.is_file():
|
||||
continue
|
||||
rel = path.relative_to(root)
|
||||
if "__pycache__" in rel.parts or rel.suffix in {".pyc", ".pyo"}:
|
||||
continue
|
||||
snap[rel.as_posix()] = path.read_bytes()
|
||||
return snap
|
||||
|
||||
|
||||
def _restore_snapshot(root: Path, snap: dict[str, bytes]) -> None:
|
||||
if root.exists():
|
||||
for path in sorted(root.rglob("*"), reverse=True):
|
||||
if path.is_file():
|
||||
rel = path.relative_to(root).as_posix()
|
||||
if rel not in snap:
|
||||
path.unlink()
|
||||
for path in sorted(root.rglob("*"), reverse=True):
|
||||
if path.is_dir():
|
||||
try:
|
||||
path.rmdir()
|
||||
except OSError:
|
||||
pass
|
||||
for rel, content in snap.items():
|
||||
path = root / rel
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(content)
|
||||
|
||||
|
||||
def test_health_route_returns_ok() -> None:
|
||||
response = _request("GET", "/health")
|
||||
|
||||
assert response.status == 200
|
||||
assert response.payload["ok"] is True
|
||||
assert response.payload["data"] == {"status": "ok"}
|
||||
assert response.payload["generated_at"]
|
||||
|
||||
|
||||
def test_generated_at_is_only_route_payload_nondeterminism() -> None:
|
||||
first = _request("GET", "/health").payload
|
||||
second = _request("GET", "/health").payload
|
||||
|
||||
assert _without_generated_at(first) == _without_generated_at(second)
|
||||
|
||||
|
||||
def test_runtime_status_is_read_only_shape() -> None:
|
||||
response = _request("GET", "/runtime/status")
|
||||
|
||||
assert response.status == 200
|
||||
data = response.payload["data"]
|
||||
assert data["mutation_mode"] == "read_only"
|
||||
assert "git_revision" in data
|
||||
assert "engine_state_present" in data
|
||||
|
||||
|
||||
def test_artifact_path_traversal_is_rejected() -> None:
|
||||
response = _request("GET", "/artifacts/../../pyproject.toml")
|
||||
|
||||
assert response.status == 400
|
||||
assert response.payload["ok"] is False
|
||||
assert response.payload["error"]["code"] == "bad_request"
|
||||
|
||||
|
||||
def test_well_formed_missing_artifact_returns_404() -> None:
|
||||
response = _request("GET", "/artifacts/evals/does-not-exist.json")
|
||||
|
||||
assert response.status == 404
|
||||
assert response.payload["error"]["code"] == "not_found"
|
||||
|
||||
|
||||
def test_artifacts_list_uses_envelope() -> None:
|
||||
response = _request("GET", "/artifacts?limit=5")
|
||||
|
||||
assert response.status == 200
|
||||
assert response.payload["ok"] is True
|
||||
assert isinstance(response.payload["data"]["items"], list)
|
||||
|
||||
|
||||
def test_known_artifact_detail_round_trip() -> None:
|
||||
response = _request("GET", "/artifacts/evals/contemplation_quality/contract.md")
|
||||
|
||||
assert response.status == 200
|
||||
assert response.payload["data"]["path"] == "evals/contemplation_quality/contract.md"
|
||||
assert response.payload["data"]["digest"].startswith("sha256:")
|
||||
|
||||
|
||||
def test_artifact_size_guard_returns_413(monkeypatch) -> None:
|
||||
monkeypatch.setattr(readers, "MAX_ARTIFACT_BYTES", 1)
|
||||
|
||||
response = _request("GET", "/artifacts/evals/contemplation_quality/contract.md")
|
||||
|
||||
assert response.status == 413
|
||||
assert response.payload["error"]["code"] == "read_error"
|
||||
|
||||
|
||||
def test_proposals_list_is_read_only_envelope() -> None:
|
||||
response = _request("GET", "/proposals")
|
||||
|
||||
assert response.status == 200
|
||||
assert response.payload["ok"] is True
|
||||
assert "items" in response.payload["data"]
|
||||
|
||||
|
||||
def test_missing_proposal_returns_404() -> None:
|
||||
response = _request("GET", "/proposals/not-a-real-proposal")
|
||||
|
||||
assert response.status == 404
|
||||
assert response.payload["ok"] is False
|
||||
assert response.payload["error"]["code"] == "not_found"
|
||||
|
||||
|
||||
def test_eval_lanes_route_returns_items() -> None:
|
||||
response = _request("GET", "/evals")
|
||||
|
||||
assert response.status == 200
|
||||
assert response.payload["ok"] is True
|
||||
assert isinstance(response.payload["data"]["lanes"], list)
|
||||
|
||||
|
||||
def test_eval_lane_detail_returns_read_only_flag() -> None:
|
||||
response = _request("GET", "/evals/contemplation_quality")
|
||||
|
||||
assert response.status == 200
|
||||
assert response.payload["data"]["lane"] == "contemplation_quality"
|
||||
assert response.payload["data"]["read_only"] is True
|
||||
|
||||
|
||||
def test_eval_run_rejects_unsafe_lane() -> None:
|
||||
response = _request(
|
||||
"POST",
|
||||
"/evals/run",
|
||||
{"lane": "cognition", "version": "v1", "split": "public"},
|
||||
)
|
||||
|
||||
assert response.status == 400
|
||||
assert response.payload["ok"] is False
|
||||
assert response.payload["error"]["code"] == "bad_request"
|
||||
assert "not workbench-safe" in response.payload["error"]["message"]
|
||||
|
||||
|
||||
def test_eval_run_rejects_holdout() -> None:
|
||||
response = _request(
|
||||
"POST",
|
||||
"/evals/run",
|
||||
{"lane": "contemplation_quality", "version": "v1", "split": "holdout"},
|
||||
)
|
||||
|
||||
assert response.status == 400
|
||||
assert response.payload["ok"] is False
|
||||
assert response.payload["error"]["code"] == "bad_request"
|
||||
|
||||
|
||||
def test_eval_run_rejects_unknown_version_without_path_leak() -> None:
|
||||
response = _request(
|
||||
"POST",
|
||||
"/evals/run",
|
||||
{"lane": "contemplation_quality", "version": "v999", "split": "public"},
|
||||
)
|
||||
|
||||
assert response.status == 400
|
||||
assert response.payload["error"]["code"] == "bad_request"
|
||||
assert "unsupported eval version" in response.payload["error"]["message"]
|
||||
assert "cases.jsonl" not in response.payload["error"]["message"]
|
||||
|
||||
|
||||
def test_unknown_trace_returns_404_not_placeholder_success() -> None:
|
||||
response = _request("GET", "/trace/not-a-real-turn")
|
||||
|
||||
assert response.status == 404
|
||||
assert response.payload["ok"] is False
|
||||
assert response.payload["error"]["code"] == "not_found"
|
||||
|
||||
|
||||
def test_chat_and_replay_are_explicitly_unsupported_in_w026() -> None:
|
||||
chat = _request("POST", "/chat/turn", {"prompt": "What is truth?"})
|
||||
replay = _request("GET", "/replay/evals/cognition/results/example.json")
|
||||
|
||||
assert chat.status == 501
|
||||
assert chat.payload["error"]["code"] == "unsupported"
|
||||
assert replay.status == 501
|
||||
assert replay.payload["error"]["code"] == "unsupported"
|
||||
|
||||
|
||||
def test_unexpected_dispatch_error_stays_json(monkeypatch) -> None:
|
||||
def fail_runtime_status():
|
||||
raise TypeError("boom")
|
||||
|
||||
monkeypatch.setattr(readers, "runtime_status", fail_runtime_status)
|
||||
|
||||
response = _request("GET", "/runtime/status")
|
||||
|
||||
assert response.status == 500
|
||||
assert response.payload["ok"] is False
|
||||
assert response.payload["generated_at"]
|
||||
assert response.payload["error"]["code"] == "runtime_unavailable"
|
||||
|
||||
|
||||
def test_full_w026_route_table_preserves_teaching_and_pack_bytes() -> None:
|
||||
"""W-026 read-only invariant.
|
||||
|
||||
The Workbench API must not mutate teaching corpora or pack data. Normal
|
||||
runtime checkpoints under engine_state/ are governed by ADR-0146/0150/0159
|
||||
and are restored after this test to keep the worktree clean.
|
||||
"""
|
||||
repo_root = Path(__file__).resolve().parent.parent
|
||||
guarded = {
|
||||
"teaching": repo_root / "teaching",
|
||||
"packs": repo_root / "packs",
|
||||
"language_packs/data": repo_root / "language_packs" / "data",
|
||||
}
|
||||
before = {name: _snapshot(path) for name, path in guarded.items()}
|
||||
engine_state = repo_root / "engine_state"
|
||||
engine_state_before = _snapshot(engine_state)
|
||||
|
||||
try:
|
||||
calls = [
|
||||
("GET", "/health", None),
|
||||
("GET", "/runtime/status", None),
|
||||
("GET", "/artifacts?limit=5", None),
|
||||
("GET", "/artifacts/evals/contemplation_quality/contract.md", None),
|
||||
("GET", "/proposals", None),
|
||||
("GET", "/evals", None),
|
||||
("GET", "/evals/contemplation_quality", None),
|
||||
(
|
||||
"POST",
|
||||
"/evals/run",
|
||||
{"lane": "contemplation_quality", "version": "v1", "split": "public"},
|
||||
),
|
||||
]
|
||||
responses = [_request(method, path, body) for method, path, body in calls]
|
||||
finally:
|
||||
_restore_snapshot(engine_state, engine_state_before)
|
||||
|
||||
assert all(response.status == 200 for response in responses)
|
||||
after = {name: _snapshot(path) for name, path in guarded.items()}
|
||||
assert before == after
|
||||
|
||||
|
||||
def test_server_rejects_nonlocal_bind_without_explicit_flag(monkeypatch) -> None:
|
||||
called = False
|
||||
|
||||
def fake_serve(*, host: str, port: int) -> None:
|
||||
nonlocal called
|
||||
called = True
|
||||
|
||||
monkeypatch.setattr(server, "serve", fake_serve)
|
||||
|
||||
with pytest.raises(SystemExit):
|
||||
server.main(["--host", "0.0.0.0"])
|
||||
|
||||
assert called is False
|
||||
|
||||
|
||||
def test_server_allows_nonlocal_bind_with_explicit_flag(monkeypatch) -> None:
|
||||
seen: dict[str, object] = {}
|
||||
|
||||
def fake_serve(*, host: str, port: int) -> None:
|
||||
seen["host"] = host
|
||||
seen["port"] = port
|
||||
|
||||
monkeypatch.setattr(server, "serve", fake_serve)
|
||||
|
||||
assert server.main(["--host", "0.0.0.0", "--port", "9000", "--allow-nonlocal-bind"]) is None
|
||||
assert seen == {"host": "0.0.0.0", "port": 9000}
|
||||
|
||||
|
||||
def test_invalid_content_length_does_not_crash_server() -> None:
|
||||
httpd = ThreadingHTTPServer(("127.0.0.1", 0), WorkbenchRequestHandler)
|
||||
thread = threading.Thread(target=httpd.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
host, port = httpd.server_address
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=5) as sock:
|
||||
sock.sendall(
|
||||
b"POST /evals/run HTTP/1.1\r\n"
|
||||
b"Host: 127.0.0.1\r\n"
|
||||
b"Content-Length: -1\r\n"
|
||||
b"Connection: close\r\n\r\n"
|
||||
)
|
||||
data = sock.recv(4096)
|
||||
finally:
|
||||
httpd.shutdown()
|
||||
httpd.server_close()
|
||||
thread.join(timeout=5)
|
||||
|
||||
assert b"HTTP/1.0 400" in data or b"HTTP/1.1 400" in data
|
||||
assert b"application/json" in data
|
||||
96
tests/test_workbench_readers.py
Normal file
96
tests/test_workbench_readers.py
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from workbench.readers import list_artifacts, list_proposals, read_artifact, read_proposal
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"artifact_id",
|
||||
[
|
||||
"../../pyproject.toml",
|
||||
"../engine_state/manifest.json",
|
||||
"/etc/passwd",
|
||||
],
|
||||
)
|
||||
def test_read_artifact_rejects_path_traversal(artifact_id: str) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
read_artifact(artifact_id)
|
||||
|
||||
|
||||
def test_read_artifact_missing_file_raises_not_found() -> None:
|
||||
with pytest.raises(FileNotFoundError):
|
||||
read_artifact("evals/does-not-exist.json")
|
||||
|
||||
|
||||
def test_read_known_allowed_artifact_when_present() -> None:
|
||||
detail = read_artifact("evals/contemplation_quality/contract.md")
|
||||
|
||||
assert detail.path == "evals/contemplation_quality/contract.md"
|
||||
assert detail.digest and detail.digest.startswith("sha256:")
|
||||
assert detail.content_type == "text"
|
||||
|
||||
|
||||
def test_list_artifacts_hashes_without_reading_whole_files(monkeypatch) -> None:
|
||||
def fail_read_bytes(self: Path) -> bytes:
|
||||
raise AssertionError(f"list_artifacts should not call read_bytes: {self}")
|
||||
|
||||
monkeypatch.setattr(Path, "read_bytes", fail_read_bytes)
|
||||
|
||||
items = list_artifacts(limit=3)
|
||||
|
||||
assert len(items) <= 3
|
||||
assert all(item.digest and item.digest.startswith("sha256:") for item in items)
|
||||
|
||||
|
||||
def test_proposals_use_append_only_event_log_current_state(tmp_path: Path) -> None:
|
||||
log_path = tmp_path / "proposals.jsonl"
|
||||
proposal_id = "proposal-001"
|
||||
proposal = {
|
||||
"proposal_id": proposal_id,
|
||||
"source_candidate_id": "candidate-001",
|
||||
"proposed_chain": {
|
||||
"subject": "alpha",
|
||||
"intent": "cause",
|
||||
"connective": "causes",
|
||||
"object": "beta",
|
||||
},
|
||||
"polarity": "affirms",
|
||||
"claim_domain": "descriptive",
|
||||
"evidence": [],
|
||||
"source": {
|
||||
"kind": "contemplation",
|
||||
"source_id": "candidate-001",
|
||||
"emitted_at_revision": "abc123",
|
||||
},
|
||||
"review_state": "pending",
|
||||
"operator_note": "",
|
||||
"replay_evidence": None,
|
||||
"provenance": None,
|
||||
}
|
||||
replay = {
|
||||
"baseline": {"accuracy": 1.0},
|
||||
"candidate": {"accuracy": 1.0},
|
||||
"regressed_metrics": [],
|
||||
"replay_equivalent": True,
|
||||
}
|
||||
events = [
|
||||
{"event": "created", "proposal": proposal},
|
||||
{"event": "replay", "proposal_id": proposal_id, "replay_evidence": replay},
|
||||
]
|
||||
log_path.write_text(
|
||||
"\n".join(json.dumps(event, sort_keys=True) for event in events) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
summaries = list_proposals(log_path=log_path)
|
||||
detail = read_proposal(proposal_id, log_path=log_path)
|
||||
|
||||
assert [p.proposal_id for p in summaries] == [proposal_id]
|
||||
assert summaries[0].source_kind == "contemplation"
|
||||
assert summaries[0].replay_equivalent is True
|
||||
assert detail.proposed_chain == proposal["proposed_chain"]
|
||||
assert detail.replay_evidence == replay
|
||||
27
tests/test_workbench_schemas.py
Normal file
27
tests/test_workbench_schemas.py
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from workbench.schemas import RuntimeStatus, error, ok
|
||||
|
||||
|
||||
def test_ok_envelope_contains_generated_at_and_data() -> None:
|
||||
payload = ok(RuntimeStatus(
|
||||
backend="numpy",
|
||||
git_revision="abc123",
|
||||
engine_state_present=False,
|
||||
checkpoint_revision="unknown",
|
||||
revision_warning=False,
|
||||
active_session_id=None,
|
||||
))
|
||||
|
||||
assert payload["ok"] is True
|
||||
assert isinstance(payload["generated_at"], str)
|
||||
assert payload["data"]["backend"] == "numpy"
|
||||
assert payload["data"]["mutation_mode"] == "read_only"
|
||||
|
||||
|
||||
def test_error_envelope_contains_generated_at_and_code() -> None:
|
||||
payload = error("not_found", "missing")
|
||||
|
||||
assert payload["ok"] is False
|
||||
assert isinstance(payload["generated_at"], str)
|
||||
assert payload["error"] == {"code": "not_found", "message": "missing"}
|
||||
3
workbench/__init__.py
Normal file
3
workbench/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
"""CORE Workbench local operator API."""
|
||||
|
||||
__all__ = []
|
||||
91
workbench/api.py
Normal file
91
workbench/api.py
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
"""Small stdlib route layer for CORE Workbench W-026."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
|
||||
from workbench import readers
|
||||
from workbench.readers import ArtifactTooLargeError
|
||||
from workbench.schemas import error, ok
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ApiResponse:
|
||||
status: int
|
||||
payload: dict[str, Any]
|
||||
|
||||
|
||||
class WorkbenchApi:
|
||||
def handle(self, method: str, raw_path: str, body: bytes = b"") -> ApiResponse:
|
||||
parsed = urlparse(raw_path)
|
||||
path = parsed.path.rstrip("/") or "/"
|
||||
query = parse_qs(parsed.query)
|
||||
try:
|
||||
return self._dispatch(method.upper(), path, query, body)
|
||||
except json.JSONDecodeError as exc:
|
||||
return ApiResponse(400, error("bad_request", "invalid JSON body", detail=str(exc)))
|
||||
except ValueError as exc:
|
||||
return ApiResponse(400, error("bad_request", str(exc)))
|
||||
except FileNotFoundError as exc:
|
||||
missing = str(exc) or "resource"
|
||||
return ApiResponse(404, error("not_found", f"not found: {missing}"))
|
||||
except ArtifactTooLargeError as exc:
|
||||
return ApiResponse(413, error("read_error", str(exc)))
|
||||
except OSError as exc:
|
||||
return ApiResponse(500, error("read_error", str(exc)))
|
||||
except Exception as exc: # noqa: BLE001 - API contract requires JSON errors.
|
||||
return ApiResponse(500, error("runtime_unavailable", f"internal error: {exc}"))
|
||||
|
||||
def _dispatch(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
query: dict[str, list[str]],
|
||||
body: bytes,
|
||||
) -> ApiResponse:
|
||||
if method == "GET" and path == "/health":
|
||||
return ApiResponse(200, ok({"status": "ok"}))
|
||||
if method == "GET" and path == "/runtime/status":
|
||||
return ApiResponse(200, ok(readers.runtime_status()))
|
||||
if method == "GET" and path == "/artifacts":
|
||||
limit = int(query.get("limit", ["100"])[0])
|
||||
return ApiResponse(200, ok({"items": readers.list_artifacts(limit=limit)}))
|
||||
if method == "GET" and path.startswith("/artifacts/"):
|
||||
artifact_id = unquote(path.removeprefix("/artifacts/"))
|
||||
return ApiResponse(200, ok(readers.read_artifact(artifact_id)))
|
||||
if method == "GET" and path == "/proposals":
|
||||
return ApiResponse(200, ok({"items": readers.list_proposals()}))
|
||||
if method == "GET" and path.startswith("/proposals/"):
|
||||
proposal_id = unquote(path.removeprefix("/proposals/"))
|
||||
return ApiResponse(200, ok(readers.read_proposal(proposal_id)))
|
||||
if method == "GET" and path == "/evals":
|
||||
return ApiResponse(200, ok({"lanes": readers.list_eval_lanes()}))
|
||||
if method == "GET" and path.startswith("/evals/"):
|
||||
lane = unquote(path.removeprefix("/evals/"))
|
||||
return ApiResponse(200, ok(readers.read_eval_lane(lane)))
|
||||
if method == "POST" and path == "/evals/run":
|
||||
request = json.loads(body.decode("utf-8") or "{}")
|
||||
if not isinstance(request, dict):
|
||||
return ApiResponse(400, error("bad_request", "eval request must be an object"))
|
||||
try:
|
||||
result = readers.run_safe_eval_lane(
|
||||
str(request.get("lane") or ""),
|
||||
version=str(request.get("version") or "v1"),
|
||||
split=str(request.get("split") or "public"),
|
||||
)
|
||||
except FileNotFoundError as exc:
|
||||
return ApiResponse(404, error("not_found", str(exc)))
|
||||
except ValueError as exc:
|
||||
return ApiResponse(400, error("bad_request", str(exc)))
|
||||
return ApiResponse(200, ok(result))
|
||||
if method == "GET" and path.startswith("/trace/"):
|
||||
return ApiResponse(404, error("not_found", "trace storage is not wired in W-026"))
|
||||
if (
|
||||
(method == "POST" and path == "/chat/turn")
|
||||
or (method == "GET" and path.startswith("/replay/"))
|
||||
):
|
||||
return ApiResponse(501, error("unsupported", "route is deferred beyond W-026"))
|
||||
return ApiResponse(404, error("not_found", f"route not found: {method} {path}"))
|
||||
300
workbench/readers.py
Normal file
300
workbench/readers.py
Normal file
|
|
@ -0,0 +1,300 @@
|
|||
"""Read-only readers for the CORE Workbench W-026 API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, get_args
|
||||
|
||||
from engine_state import EngineStateStore, get_git_revision
|
||||
from evals.framework import discover_lanes, get_lane, run_lane
|
||||
from teaching.proposals import DEFAULT_PROPOSAL_LOG_PATH, ProposalLog, ReviewState
|
||||
from workbench.schemas import (
|
||||
ArtifactDetail,
|
||||
ArtifactRef,
|
||||
EvalLaneSummary,
|
||||
EvalRunResult,
|
||||
ProposalDetail,
|
||||
ProposalSummary,
|
||||
RuntimeStatus,
|
||||
)
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
SAFE_EVAL_LANES = frozenset({"contemplation_quality"})
|
||||
MAX_ARTIFACT_BYTES = 16 * 1024 * 1024
|
||||
READ_CHUNK_BYTES = 64 * 1024
|
||||
_EVAL_RUN_LOCK = threading.Lock()
|
||||
_REVIEW_STATES = frozenset(get_args(ReviewState))
|
||||
ALLOWED_ARTIFACT_ROOTS = (
|
||||
REPO_ROOT / "engine_state",
|
||||
REPO_ROOT / "teaching" / "proposals",
|
||||
REPO_ROOT / "evals",
|
||||
REPO_ROOT / "contemplation" / "runs",
|
||||
)
|
||||
|
||||
|
||||
class ArtifactTooLargeError(OSError):
|
||||
"""Raised when an artifact is too large for direct Workbench reads."""
|
||||
|
||||
|
||||
def _sha256_bytes(content: bytes) -> str:
|
||||
return "sha256:" + hashlib.sha256(content).hexdigest()
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
hasher = hashlib.sha256()
|
||||
with path.open("rb") as fh:
|
||||
for chunk in iter(lambda: fh.read(READ_CHUNK_BYTES), b""):
|
||||
hasher.update(chunk)
|
||||
return "sha256:" + hasher.hexdigest()
|
||||
|
||||
|
||||
def _relative(path: Path) -> str:
|
||||
return path.resolve().relative_to(REPO_ROOT.resolve()).as_posix()
|
||||
|
||||
|
||||
def _validate_artifact_id(artifact_id: str) -> PurePosixPath:
|
||||
if not artifact_id or artifact_id.startswith("/"):
|
||||
raise ValueError("artifact id must be a repo-relative path")
|
||||
rel = PurePosixPath(artifact_id)
|
||||
if any(part in ("", ".", "..") for part in rel.parts):
|
||||
raise ValueError("artifact id must not contain path traversal")
|
||||
return rel
|
||||
|
||||
|
||||
def _is_allowed(path: Path) -> bool:
|
||||
resolved = path.resolve()
|
||||
for root in ALLOWED_ARTIFACT_ROOTS:
|
||||
root_resolved = root.resolve()
|
||||
if resolved == root_resolved or root_resolved in resolved.parents:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _resolve_artifact(artifact_id: str) -> Path:
|
||||
rel = _validate_artifact_id(artifact_id)
|
||||
candidate = (REPO_ROOT / rel.as_posix()).resolve()
|
||||
if not _is_allowed(candidate):
|
||||
raise ValueError("artifact path is outside allowed roots")
|
||||
return candidate
|
||||
|
||||
|
||||
def _artifact_kind(path: Path) -> str:
|
||||
rel = _relative(path)
|
||||
if rel == "engine_state/manifest.json":
|
||||
return "engine_state_manifest"
|
||||
if rel.startswith("teaching/proposals/"):
|
||||
return "proposal"
|
||||
if rel.startswith("evals/") and "/results/" in rel:
|
||||
return "eval_result"
|
||||
if rel.startswith("contemplation/runs/"):
|
||||
return "contemplation_report"
|
||||
if path.suffix == ".jsonl":
|
||||
return "telemetry"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def runtime_status() -> RuntimeStatus:
|
||||
store = EngineStateStore()
|
||||
manifest = store.load_manifest() or {}
|
||||
current_revision = get_git_revision()
|
||||
checkpoint_revision = str(manifest.get("written_at_revision") or "unknown")
|
||||
backend_raw = os.environ.get("CORE_BACKEND", "numpy")
|
||||
backend = backend_raw if backend_raw in {"numpy", "mlx", "rust"} else "unknown"
|
||||
return RuntimeStatus(
|
||||
backend=backend, # type: ignore[arg-type]
|
||||
git_revision=current_revision,
|
||||
engine_state_present=store.exists(),
|
||||
checkpoint_revision=checkpoint_revision,
|
||||
revision_warning=(
|
||||
checkpoint_revision not in {"", "unknown"}
|
||||
and current_revision not in {"", "unknown"}
|
||||
and checkpoint_revision != current_revision
|
||||
),
|
||||
active_session_id=None,
|
||||
)
|
||||
|
||||
|
||||
def list_artifacts(*, limit: int = 100) -> list[ArtifactRef]:
|
||||
items: list[ArtifactRef] = []
|
||||
for root in ALLOWED_ARTIFACT_ROOTS:
|
||||
if not root.exists():
|
||||
continue
|
||||
for path in sorted(root.rglob("*")):
|
||||
if len(items) >= limit:
|
||||
return items
|
||||
if not path.is_file() or path.suffix not in {".json", ".jsonl", ".md", ".txt"}:
|
||||
continue
|
||||
if not _is_allowed(path):
|
||||
continue
|
||||
try:
|
||||
digest = _sha256_file(path)
|
||||
except OSError:
|
||||
continue
|
||||
rel = _relative(path)
|
||||
items.append(
|
||||
ArtifactRef(
|
||||
artifact_id=rel,
|
||||
kind=_artifact_kind(path), # type: ignore[arg-type]
|
||||
path=rel,
|
||||
digest=digest,
|
||||
created_at=None,
|
||||
)
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
def read_artifact(artifact_id: str) -> ArtifactDetail:
|
||||
path = _resolve_artifact(artifact_id)
|
||||
if not path.exists() or not path.is_file():
|
||||
raise FileNotFoundError(artifact_id)
|
||||
if path.stat().st_size > MAX_ARTIFACT_BYTES:
|
||||
raise ArtifactTooLargeError(
|
||||
f"artifact exceeds {MAX_ARTIFACT_BYTES} byte read limit: {artifact_id}"
|
||||
)
|
||||
raw = path.read_bytes()
|
||||
text = raw.decode("utf-8")
|
||||
content_type = "text"
|
||||
content: Any = text
|
||||
if path.suffix == ".json":
|
||||
content_type = "json"
|
||||
content = json.loads(text)
|
||||
elif path.suffix == ".jsonl":
|
||||
content_type = "jsonl"
|
||||
rows: list[Any] = []
|
||||
for line in text.splitlines():
|
||||
if line.strip():
|
||||
rows.append(json.loads(line))
|
||||
content = rows
|
||||
rel = _relative(path)
|
||||
return ArtifactDetail(
|
||||
artifact_id=rel,
|
||||
kind=_artifact_kind(path), # type: ignore[arg-type]
|
||||
path=rel,
|
||||
digest=_sha256_bytes(raw),
|
||||
created_at=None,
|
||||
content_type=content_type, # type: ignore[arg-type]
|
||||
content=content,
|
||||
)
|
||||
|
||||
|
||||
def _state_value(value: Any) -> str:
|
||||
text = str(value or "unknown")
|
||||
return text if text in _REVIEW_STATES else "unknown"
|
||||
|
||||
|
||||
def _source_kind(source: Any) -> str:
|
||||
if isinstance(source, dict):
|
||||
return str(source.get("kind") or "unknown")
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _replay_equivalent(replay: Any) -> bool | None:
|
||||
if isinstance(replay, dict) and isinstance(replay.get("replay_equivalent"), bool):
|
||||
return bool(replay["replay_equivalent"])
|
||||
return None
|
||||
|
||||
|
||||
def _proposal_summary(proposal_id: str, record: dict[str, Any]) -> ProposalSummary:
|
||||
return ProposalSummary(
|
||||
proposal_id=proposal_id,
|
||||
state=_state_value(record.get("state")), # type: ignore[arg-type]
|
||||
source_kind=_source_kind(record.get("source")),
|
||||
replay_equivalent=_replay_equivalent(record.get("replay_evidence")),
|
||||
created_at=None,
|
||||
downstream_effect="observed" if record.get("accepted_chain_id") else "unknown",
|
||||
)
|
||||
|
||||
|
||||
def list_proposals(*, log_path: Path | None = None) -> list[ProposalSummary]:
|
||||
log = ProposalLog(path=log_path or DEFAULT_PROPOSAL_LOG_PATH)
|
||||
state = log.current_state()
|
||||
return [
|
||||
_proposal_summary(proposal_id, state[proposal_id])
|
||||
for proposal_id in sorted(state)
|
||||
]
|
||||
|
||||
|
||||
def read_proposal(proposal_id: str, *, log_path: Path | None = None) -> ProposalDetail:
|
||||
log = ProposalLog(path=log_path or DEFAULT_PROPOSAL_LOG_PATH)
|
||||
record = log.find(proposal_id)
|
||||
if record is None:
|
||||
raise FileNotFoundError(proposal_id)
|
||||
proposal = record.get("proposal") if isinstance(record.get("proposal"), dict) else {}
|
||||
summary = _proposal_summary(proposal_id, record)
|
||||
review_state = summary.state
|
||||
return ProposalDetail(
|
||||
proposal_id=summary.proposal_id,
|
||||
state=review_state,
|
||||
source_kind=summary.source_kind,
|
||||
replay_equivalent=summary.replay_equivalent,
|
||||
created_at=summary.created_at,
|
||||
downstream_effect=summary.downstream_effect,
|
||||
proposed_chain=proposal.get("proposed_chain"),
|
||||
replay_evidence=record.get("replay_evidence"),
|
||||
source=record.get("source"),
|
||||
evidence=proposal.get("evidence") if isinstance(proposal.get("evidence"), list) else [],
|
||||
artifact_refs=[],
|
||||
suggested_cli=(
|
||||
f"core teaching review {proposal_id} --accept --review-date YYYY-MM-DD"
|
||||
if review_state == "pending"
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def list_eval_lanes() -> list[EvalLaneSummary]:
|
||||
return [
|
||||
EvalLaneSummary(
|
||||
lane=lane.name,
|
||||
versions=list(lane.versions),
|
||||
read_only=lane.name in SAFE_EVAL_LANES,
|
||||
description=None,
|
||||
)
|
||||
for lane in discover_lanes()
|
||||
]
|
||||
|
||||
|
||||
def read_eval_lane(lane_name: str) -> EvalLaneSummary:
|
||||
lane = get_lane(lane_name)
|
||||
return EvalLaneSummary(
|
||||
lane=lane.name,
|
||||
versions=list(lane.versions),
|
||||
read_only=lane.name in SAFE_EVAL_LANES,
|
||||
description=None,
|
||||
)
|
||||
|
||||
|
||||
def run_safe_eval_lane(
|
||||
lane_name: str,
|
||||
*,
|
||||
version: str = "v1",
|
||||
split: str = "public",
|
||||
) -> EvalRunResult:
|
||||
if lane_name not in SAFE_EVAL_LANES:
|
||||
raise ValueError(f"eval lane is not workbench-safe/read-only: {lane_name}")
|
||||
if split == "holdout":
|
||||
raise ValueError("holdout execution is disabled in Workbench v1")
|
||||
lane = get_lane(lane_name)
|
||||
if version not in lane.versions:
|
||||
raise ValueError(f"unsupported eval version for {lane_name}: {version}")
|
||||
with _EVAL_RUN_LOCK:
|
||||
result = run_lane(lane, version=version, split=split, workers=1)
|
||||
passed_raw = result.metrics.get("passed") if isinstance(result.metrics, dict) else None
|
||||
passed = bool(passed_raw) if isinstance(passed_raw, bool) else None
|
||||
return EvalRunResult(
|
||||
lane=result.lane,
|
||||
version=result.version,
|
||||
split=result.split,
|
||||
passed=passed,
|
||||
metrics=result.metrics,
|
||||
cases=result.case_details,
|
||||
source_digest=(
|
||||
str(result.metrics["source_digest"])
|
||||
if isinstance(result.metrics, dict) and "source_digest" in result.metrics
|
||||
else None
|
||||
),
|
||||
)
|
||||
134
workbench/schemas.py
Normal file
134
workbench/schemas.py
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
"""Typed UI-facing schemas for CORE Workbench v1."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Literal
|
||||
|
||||
|
||||
ErrorCode = Literal[
|
||||
"bad_request",
|
||||
"not_found",
|
||||
"unsupported",
|
||||
"read_error",
|
||||
"eval_failed",
|
||||
"runtime_unavailable",
|
||||
]
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def to_data(value: Any) -> Any:
|
||||
if hasattr(value, "as_dict") and callable(value.as_dict):
|
||||
return value.as_dict()
|
||||
if hasattr(value, "__dataclass_fields__"):
|
||||
return asdict(value)
|
||||
if isinstance(value, dict):
|
||||
return {str(k): to_data(v) for k, v in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [to_data(v) for v in value]
|
||||
return value
|
||||
|
||||
|
||||
def ok(data: Any) -> dict[str, Any]:
|
||||
return {"ok": True, "generated_at": utc_now(), "data": to_data(data)}
|
||||
|
||||
|
||||
def error(code: ErrorCode, message: str, *, detail: Any | None = None) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {"code": code, "message": message}
|
||||
if detail is not None:
|
||||
payload["detail"] = to_data(detail)
|
||||
return {"ok": False, "generated_at": utc_now(), "error": payload}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RuntimeStatus:
|
||||
backend: Literal["numpy", "mlx", "rust", "unknown"]
|
||||
git_revision: str
|
||||
engine_state_present: bool
|
||||
checkpoint_revision: str
|
||||
revision_warning: bool
|
||||
active_session_id: str | None
|
||||
mutation_mode: Literal["read_only", "runtime_turn"] = "read_only"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ArtifactRef:
|
||||
artifact_id: str
|
||||
kind: Literal[
|
||||
"trace",
|
||||
"eval_result",
|
||||
"proposal",
|
||||
"contemplation_report",
|
||||
"telemetry",
|
||||
"engine_state_manifest",
|
||||
"unknown",
|
||||
]
|
||||
path: str
|
||||
digest: str | None
|
||||
created_at: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ArtifactDetail(ArtifactRef):
|
||||
content_type: Literal["json", "jsonl", "text", "unknown"]
|
||||
content: Any
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProposalSummary:
|
||||
proposal_id: str
|
||||
state: Literal["pending", "accepted", "rejected", "withdrawn", "unknown"]
|
||||
source_kind: str
|
||||
replay_equivalent: bool | None
|
||||
created_at: str | None
|
||||
downstream_effect: Literal["unknown", "none", "observed"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProposalDetail(ProposalSummary):
|
||||
proposed_chain: Any
|
||||
replay_evidence: Any
|
||||
source: Any
|
||||
evidence: list[Any] = field(default_factory=list)
|
||||
artifact_refs: list[ArtifactRef] = field(default_factory=list)
|
||||
suggested_cli: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class EvalLaneSummary:
|
||||
lane: str
|
||||
versions: list[str]
|
||||
read_only: bool
|
||||
description: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class EvalRunResult:
|
||||
lane: str
|
||||
version: str
|
||||
split: str
|
||||
passed: bool | None
|
||||
metrics: dict[str, Any]
|
||||
cases: list[Any]
|
||||
source_digest: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReplayDivergence:
|
||||
path: str
|
||||
original: Any
|
||||
replay: Any
|
||||
severity: Literal["info", "warning", "failure"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReplayComparison:
|
||||
artifact_id: str
|
||||
original_hash: str | None
|
||||
replay_hash: str | None
|
||||
equivalent: bool
|
||||
divergences: list[ReplayDivergence] = field(default_factory=list)
|
||||
72
workbench/server.py
Normal file
72
workbench/server.py
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
"""Stdlib HTTP server entrypoint for CORE Workbench W-026."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
from workbench.api import WorkbenchApi
|
||||
|
||||
|
||||
class WorkbenchRequestHandler(BaseHTTPRequestHandler):
|
||||
api = WorkbenchApi()
|
||||
|
||||
def do_GET(self) -> None: # noqa: N802 - stdlib handler API
|
||||
self._handle()
|
||||
|
||||
def do_POST(self) -> None: # noqa: N802 - stdlib handler API
|
||||
self._handle()
|
||||
|
||||
def log_message(self, format: str, *args: object) -> None:
|
||||
if os.environ.get("CORE_WORKBENCH_QUIET") == "1":
|
||||
return
|
||||
sys.stderr.write(
|
||||
"%s - - [%s] %s\n"
|
||||
% (self.address_string(), self.log_date_time_string(), format % args)
|
||||
)
|
||||
|
||||
def _handle(self) -> None:
|
||||
try:
|
||||
length = max(0, int(self.headers.get("Content-Length") or "0"))
|
||||
except ValueError:
|
||||
length = 0
|
||||
body = self.rfile.read(length) if length else b""
|
||||
response = self.api.handle(self.command, self.path, body)
|
||||
payload = json.dumps(response.payload, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
self.send_response(response.status)
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
|
||||
def serve(*, host: str = "127.0.0.1", port: int = 8765) -> None:
|
||||
server = ThreadingHTTPServer((host, port), WorkbenchRequestHandler)
|
||||
print(f"CORE Workbench API listening on http://{host}:{port}")
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
print()
|
||||
raise SystemExit(0)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="CORE Workbench local API")
|
||||
parser.add_argument("--host", default="127.0.0.1")
|
||||
parser.add_argument("--port", type=int, default=8765)
|
||||
parser.add_argument(
|
||||
"--allow-nonlocal-bind",
|
||||
action="store_true",
|
||||
help="allow binding to a host other than 127.0.0.1 or localhost",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
if args.host not in {"127.0.0.1", "localhost"} and not args.allow_nonlocal_bind:
|
||||
parser.error("non-local bind requires --allow-nonlocal-bind")
|
||||
serve(host=args.host, port=args.port)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
Reference in a new issue