From 1fe56e9b6fe1d4919b7a0f5cad036b6025421c9c Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 13 Jun 2026 00:38:16 -0700 Subject: [PATCH] =?UTF-8?q?feat(workbench):=20calibration=20+=20serving-me?= =?UTF-8?q?trics=20readers=20=E2=80=94=20the=20gold-tether=20loop,=20visib?= =?UTF-8?q?le=20(Wave=20M=20B1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First Wave M / Phase B piece (GATING): read-only backend that makes the calibrated-learning / serving-discipline loop inspectable — 'the engine earns the right to guess', ADR-0175. The workbench computes NONE of these numbers: - GET /calibration/classes — per-class gold-tether view from the persisted practice arena ledger (evals/gsm8k_math/practice/v1/report.json per_class). Each class's reliability_floor is the engine's own one-sided Wilson conservative_floor (via ClassTally.reliability); PROPOSE (θ=0.85) / SERVE (θ=0.99) license verdicts come from core.reliability_gate.license_for. Failures-first ordering. A test proves the reader's floor equals a direct conservative_floor() call — no re-implementation. - GET /serving/metrics — the live correct/refused/wrong counts read unchanged from the committed train_sample + holdout_dev report.json (currently 4/46/0 and 5/495/0 — wrong=0). Never re-runs a lane. Honest current state: the committed practice ledger's three classes (additive/divisive/multiplicative) are all below N_MIN=10, so none has earned a license yet — the reader shows exactly that, no fake green light. - workbench/calibration.py: pure readers; imports core.reliability_gate; EvidenceUnavailableError -> 501 (fail-closed) when the artifact is absent. - schemas + TS mirrors (CalibrationClass, ServingMetrics); both snapshots regenerated (deterministic); both drift gates pass. - trust boundary: read-only over committed artifacts + engine-owned derivation; no execution, no mutation, no license ever changed. Verified: 30 Python tests (incl. the no-reimplementation proof + fail-closed), 390 vitest, both schema drift gates, snapshots deterministic. --- tests/test_workbench_calibration.py | 100 ++++++++++++++++++++++ workbench-ui/api-schema-snapshot.json | 26 ++++++ workbench-ui/schema-snapshot.json | 29 ++++++- workbench-ui/src/types/api.ts | 27 ++++++ workbench/api.py | 6 +- workbench/calibration.py | 119 ++++++++++++++++++++++++++ workbench/schemas.py | 36 ++++++++ 7 files changed, 340 insertions(+), 3 deletions(-) create mode 100644 tests/test_workbench_calibration.py create mode 100644 workbench/calibration.py diff --git a/tests/test_workbench_calibration.py b/tests/test_workbench_calibration.py new file mode 100644 index 00000000..0c3bbdc6 --- /dev/null +++ b/tests/test_workbench_calibration.py @@ -0,0 +1,100 @@ +"""Wave M Phase B — calibration / serving-discipline readers (ADR-0175). + +The load-bearing obligation: the workbench re-implements none of the engine's +calibration math. These tests prove the reader's numbers come from +``core.reliability_gate`` (``conservative_floor`` / ``license_for``), and that +the serving counts are read from the committed reports unchanged. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from core.reliability_gate import conservative_floor +from workbench import calibration +from workbench.api import WorkbenchApi +from workbench.readers import EvidenceUnavailableError + + +def _write_practice_report(tmp_path: Path, per_class: dict) -> Path: + path = tmp_path / "report.json" + path.write_text( + json.dumps({"adr": "0175", "regime": "practice", "per_class": per_class}), + encoding="utf-8", + ) + return path + + +def test_serving_metrics_read_committed_counts_unchanged() -> None: + metrics = {m.lane: m for m in calibration.read_serving_metrics()} + assert "train_sample" in metrics + # The live invariant: the committed serving lane commits zero wrong answers. + assert metrics["train_sample"].wrong == 0 + assert metrics["train_sample"].correct >= 0 + assert metrics["train_sample"].source_digest.startswith("sha256:") + + +def test_calibration_classes_over_committed_report_are_honest() -> None: + # The committed practice report's classes are all below N_MIN today, so + # none has earned a license — the reader must show exactly that, not fake + # a green light. + rows = calibration.read_calibration_classes() + assert rows, "expected the committed per_class ledger to yield rows" + for row in rows: + if row.committed < 10: # N_MIN + assert row.reliability_floor == 0.0 + assert row.propose_licensed is False + assert row.serve_licensed is False + + +def test_reader_uses_the_engine_math_not_its_own(tmp_path) -> None: + # A class that has earned PROPOSE (0.86 >= 0.85) but not SERVE (< 0.99). + report = _write_practice_report( + tmp_path, + { + "additive": {"correct": 95, "wrong": 5, "refused": 50}, + "novice": {"correct": 0, "wrong": 0, "refused": 4}, + }, + ) + rows = {r.class_name: r for r in calibration.read_calibration_classes(report)} + + earned = rows["additive"] + # The reader's reliability is the engine's own Wilson floor, to the digit. + assert earned.reliability_floor == round(conservative_floor(95, 100), 9) + assert earned.committed == 100 + assert earned.propose_required == 0.85 and earned.propose_licensed is True + assert earned.serve_required == 0.99 and earned.serve_licensed is False + + novice = rows["novice"] + assert novice.reliability_floor == 0.0 # below N_MIN + assert novice.propose_licensed is False + + +def test_calibration_classes_are_failures_first(tmp_path) -> None: + report = _write_practice_report( + tmp_path, + { + "earned": {"correct": 95, "wrong": 5, "refused": 0}, + "unearned": {"correct": 0, "wrong": 0, "refused": 9}, + }, + ) + rows = calibration.read_calibration_classes(report) + # Un-licensed / lowest-reliability comes first. + assert rows[0].class_name == "unearned" + assert rows[-1].class_name == "earned" + + +def test_endpoints_return_items() -> None: + api = WorkbenchApi() + r1 = api.handle("GET", "/calibration/classes", b"") + assert r1.status == 200 and isinstance(r1.payload["data"]["items"], list) + r2 = api.handle("GET", "/serving/metrics", b"") + assert r2.status == 200 and isinstance(r2.payload["data"]["items"], list) + + +def test_missing_practice_report_is_evidence_unavailable(tmp_path) -> None: + with pytest.raises(EvidenceUnavailableError): + calibration.read_calibration_classes(tmp_path / "nope.json") diff --git a/workbench-ui/api-schema-snapshot.json b/workbench-ui/api-schema-snapshot.json index d4d93995..ec8e0c6d 100644 --- a/workbench-ui/api-schema-snapshot.json +++ b/workbench-ui/api-schema-snapshot.json @@ -322,6 +322,32 @@ "metadata": "dict[str, Any]", "versor_digest": "str | None" } + }, + "CalibrationClass": { + "fields": { + "class_name": "str", + "correct": "int", + "wrong": "int", + "refused": "int", + "committed": "int", + "reliability_floor": "float", + "coverage": "float", + "propose_required": "float", + "propose_licensed": "bool", + "serve_required": "float", + "serve_licensed": "bool" + } + }, + "ServingMetrics": { + "fields": { + "lane": "str", + "correct": "int", + "refused": "int", + "wrong": "int", + "sample_count": "int", + "source_path": "str", + "source_digest": "str" + } } } } diff --git a/workbench-ui/schema-snapshot.json b/workbench-ui/schema-snapshot.json index 6b023353..ed992d37 100644 --- a/workbench-ui/schema-snapshot.json +++ b/workbench-ui/schema-snapshot.json @@ -22,6 +22,19 @@ "payload_digest", "payload" ], + "CalibrationClass": [ + "class_name", + "correct", + "wrong", + "refused", + "committed", + "reliability_floor", + "coverage", + "propose_required", + "propose_licensed", + "serve_required", + "serve_licensed" + ], "ChatTurnResult": [ "prompt", "surface", @@ -186,7 +199,8 @@ "trace_hash", "timestamp", "trace_path", - "surface_excerpt" + "surface_excerpt", + "trace_integrity" ], "RuntimeStatus": [ "backend", @@ -197,6 +211,15 @@ "active_session_id", "mutation_mode" ], + "ServingMetrics": [ + "lane", + "correct", + "refused", + "wrong", + "sample_count", + "source_path", + "source_digest" + ], "TurnJournalEntrySchema": [ "turn_id", "timestamp", @@ -214,6 +237,7 @@ "proposal_candidates", "turn_cost_ms", "checkpoint_emitted", + "trace_integrity", "journal_digest" ], "TurnJournalSummarySchema": [ @@ -222,7 +246,8 @@ "prompt_excerpt", "surface_excerpt", "trace_hash", - "grounding_source" + "grounding_source", + "trace_integrity" ], "TurnReplayComparison": [ "turn_id", diff --git a/workbench-ui/src/types/api.ts b/workbench-ui/src/types/api.ts index e5be1afb..58bef4d4 100644 --- a/workbench-ui/src/types/api.ts +++ b/workbench-ui/src/types/api.ts @@ -338,6 +338,33 @@ export interface VaultEntry { versor_digest: string | null; } +// Wave M Phase B — calibrated-learning / serving-discipline read views. +// reliability_floor + the license verdicts are computed by the engine +// (core.reliability_gate), never the workbench. +export interface CalibrationClass { + class_name: string; + correct: number; + wrong: number; + refused: number; + committed: number; + reliability_floor: number; + coverage: number; + propose_required: number; + propose_licensed: boolean; + serve_required: number; + serve_licensed: boolean; +} + +export interface ServingMetrics { + lane: string; + correct: number; + refused: number; + wrong: number; + sample_count: number; + source_path: string; + source_digest: string; +} + // API envelope types export interface ApiOk { ok: true; diff --git a/workbench/api.py b/workbench/api.py index 43f28a9c..e027b9bc 100644 --- a/workbench/api.py +++ b/workbench/api.py @@ -18,7 +18,7 @@ from core.epistemic_state import ( epistemic_state_for_grounding_source, normative_detail_from_verdicts, ) -from workbench import readers +from workbench import calibration, readers from workbench.journal import DEFAULT_JOURNAL_DIR, TurnJournal, TurnJournalEntry from workbench.readers import ArtifactTooLargeError, EvidenceUnavailableError from workbench.replay import replay_turn @@ -205,6 +205,10 @@ class WorkbenchApi: ) ), ) + if method == "GET" and path == "/calibration/classes": + return ApiResponse(200, ok({"items": calibration.read_calibration_classes()})) + if method == "GET" and path == "/serving/metrics": + return ApiResponse(200, ok({"items": calibration.read_serving_metrics()})) if method == "GET" and path == "/vault/summary": return ApiResponse(200, ok(readers.read_vault_summary())) if method == "GET" and path == "/vault/entries": diff --git a/workbench/calibration.py b/workbench/calibration.py new file mode 100644 index 00000000..9c6f287e --- /dev/null +++ b/workbench/calibration.py @@ -0,0 +1,119 @@ +"""Read-only views over the calibrated-learning / serving-discipline loop. + +ADR-0175. This is where "the engine earns the right to guess" becomes +inspectable. The workbench computes nothing the engine owns: + +- per-class **reliability** is the engine's own one-sided Wilson + ``conservative_floor`` (via ``ClassTally.reliability``), and the + **license** verdicts come from ``core.reliability_gate.license_for`` — + never re-implemented here; +- **serving counts** are read from the committed ``report.json`` artifacts; + no lane is ever re-run. + +Trust boundary: read-only over committed artifacts + engine-owned +derivation. No execution, no mutation, no license is ever changed. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Mapping, Sequence + +from core.reliability_gate import Action, Ceilings, ClassTally, license_for +from workbench.readers import ( + REPO_ROOT, + EvidenceUnavailableError, + _display_path, + _read_json_object, + _sha256_file, +) +from workbench.schemas import CalibrationClass, ServingMetrics + +# The persisted per-class arena ledger (sealed practice, ADR-0175). +PRACTICE_REPORT = REPO_ROOT / "evals" / "gsm8k_math" / "practice" / "v1" / "report.json" + +# Committed serving lanes — their counts are the live wrong=0 evidence. +SERVING_LANES: tuple[tuple[str, Path], ...] = ( + ("train_sample", REPO_ROOT / "evals" / "gsm8k_math" / "train_sample" / "v1" / "report.json"), + ("holdout_dev", REPO_ROOT / "evals" / "gsm8k_math" / "holdout_dev" / "v1" / "report.json"), +) + +_CEILINGS = Ceilings() + + +def _calibration_class(class_name: str, counts: Mapping[str, Any]) -> CalibrationClass: + tally = ClassTally( + class_name, + correct=int(counts.get("correct", 0)), + wrong=int(counts.get("wrong", 0)), + refused=int(counts.get("refused", 0)), + ) + propose = license_for(tally, Action.PROPOSE, _CEILINGS) + serve = license_for(tally, Action.SERVE, _CEILINGS) + return CalibrationClass( + class_name=class_name, + correct=tally.correct, + wrong=tally.wrong, + refused=tally.refused, + committed=tally.committed, + reliability_floor=round(tally.reliability, 9), + coverage=round(tally.coverage, 9), + propose_required=propose.required, + propose_licensed=propose.licensed, + serve_required=serve.required, + serve_licensed=serve.licensed, + ) + + +def read_calibration_classes(report_path: Path = PRACTICE_REPORT) -> list[CalibrationClass]: + """Per-class gold-tether view: what each class has earned, by the real gate.""" + if not report_path.exists(): + raise EvidenceUnavailableError( + "calibration evidence unavailable: practice report.json is absent " + "(run the sealed practice lane to populate the arena ledger)" + ) + report = _read_json_object(report_path) + per_class = report.get("per_class") + if not isinstance(per_class, dict): + raise EvidenceUnavailableError( + "calibration evidence unavailable: report has no per_class ledger" + ) + rows = [ + _calibration_class(name, counts) + for name, counts in per_class.items() + if isinstance(counts, dict) + ] + # Failures-first: un-licensed / lowest-reliability at the top; stable by name. + rows.sort( + key=lambda r: (r.serve_licensed, r.propose_licensed, r.reliability_floor, r.class_name) + ) + return rows + + +def read_serving_metrics(lanes: Sequence[tuple[str, Path]] = SERVING_LANES) -> list[ServingMetrics]: + """The live serving counts (correct / refused / wrong) from committed reports.""" + out: list[ServingMetrics] = [] + for lane, path in lanes: + if not path.exists(): + continue + report = _read_json_object(path) + counts = report.get("counts") or {} + correct = int(counts.get("correct", 0)) + refused = int(counts.get("refused", 0)) + wrong = int(counts.get("wrong", 0)) + out.append( + ServingMetrics( + lane=lane, + correct=correct, + refused=refused, + wrong=wrong, + sample_count=int(report.get("sample_count", correct + refused + wrong)), + source_path=_display_path(path), + source_digest=_sha256_file(path), + ) + ) + if not out: + raise EvidenceUnavailableError( + "serving metrics unavailable: no committed report.json found" + ) + return out diff --git a/workbench/schemas.py b/workbench/schemas.py index 6fe1b3b1..37807189 100644 --- a/workbench/schemas.py +++ b/workbench/schemas.py @@ -448,3 +448,39 @@ class VaultEntry: epistemic_state: str metadata: dict[str, Any] versor_digest: str | None + + +# --------------------------------------------------------------------------- +# Wave M Phase B — calibrated-learning / serving-discipline read views. +# The workbench computes none of these numbers: reliability_floor and the +# license verdicts come from core.reliability_gate's own conservative_floor / +# license_for; serving counts come from committed eval report.json artifacts. +# Read-only — no lane is re-run, no license is changed. +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class CalibrationClass: + class_name: str + correct: int + wrong: int + refused: int + committed: int + # One-sided Wilson conservative floor (0.0 below N_MIN committed trials). + reliability_floor: float + coverage: float + propose_required: float # θ for PROPOSE (0.85) + propose_licensed: bool + serve_required: float # θ for SERVE (0.99) + serve_licensed: bool + + +@dataclass(frozen=True, slots=True) +class ServingMetrics: + lane: str + correct: int + refused: int + wrong: int + sample_count: int + source_path: str + source_digest: str