feat(workbench): add UI catch-up evidence scaffolding (#891)

* feat(workbench): add construction evidence read model scaffolding

* feat(workbench-ui): add construction evidence TS types

* feat(workbench): add construction evidence journal projector

* test(workbench): cover construction evidence read model

* feat(workbench-ui): add construction evidence view helpers

* test(workbench-ui): cover construction evidence view helpers

* feat(workbench): add generalization audit evidence scaffolding

* test(workbench): cover generalization audit evidence view

* feat(workbench-ui): add generalization evidence TS types

* feat(workbench-ui): add generalization evidence view helpers

* test(workbench-ui): cover generalization evidence helpers

* feat(workbench): add proposal artifact authority scaffolding

* test(workbench): cover proposal artifact authority rules

* feat(workbench-ui): add proposal artifact TS types

* feat(workbench-ui): add proposal artifact view helpers

* test(workbench-ui): cover proposal artifact view helpers

* feat(workbench): add demo narrative evidence scaffolding

* test(workbench): cover demo narrative scaffolding

* feat(workbench-ui): add demo narrative TS types

* feat(workbench-ui): add demo narrative view helpers

* test(workbench-ui): cover demo narrative view helpers

* fix(evals): resolve discovery_candidates.jsonl via EngineStateStore
This commit is contained in:
Shay 2026-06-23 10:09:45 -07:00 committed by GitHub
parent 172afc2d9e
commit 2c8258fe1a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 1638 additions and 2 deletions

View file

@ -177,7 +177,8 @@ def _scene1_cold_session(
rt = ChatRuntime(config=cfg, engine_state_path=engine_state_dir)
response = rt.chat(_DEMO_PROMPT)
candidates_file = engine_state_dir / "discovery_candidates.jsonl"
from engine_state import EngineStateStore
candidates_file = EngineStateStore(engine_state_dir)._resolve_dir() / "discovery_candidates.jsonl"
candidates_persisted = (
len(candidates_file.read_text(encoding="utf-8").splitlines())
if candidates_file.exists()
@ -213,7 +214,8 @@ def _scene2_checkpoint_enrichment(
"not by the operator. Sub-questions enumerate candidate "
"chains the engine identified through corpus decomposition.",
)
candidates_file = engine_state_dir / "discovery_candidates.jsonl"
from engine_state import EngineStateStore
candidates_file = EngineStateStore(engine_state_dir)._resolve_dir() / "discovery_candidates.jsonl"
if not candidates_file.exists():
raise RuntimeError("engine state has no discovery_candidates.jsonl — S1 did not persist")
lines = [l for l in candidates_file.read_text(encoding="utf-8").splitlines() if l.strip()]

View file

@ -0,0 +1,98 @@
from __future__ import annotations
from dataclasses import dataclass
from workbench.construction_evidence import (
CONSTRUCTION_EVIDENCE_ABSENT,
ConstructionEvidence,
SourceSpanView,
construction_evidence_from_journal_entry,
missing_construction_evidence,
span_is_exact,
)
from workbench.schemas import to_data
@dataclass(frozen=True, slots=True)
class _Entry:
turn_id: int
def test_missing_construction_evidence_is_diagnostic_and_non_serving() -> None:
evidence = missing_construction_evidence(7, "not persisted")
assert evidence.schema_version == "construction_evidence_v1"
assert evidence.turn_id == 7
assert evidence.status == "missing_evidence"
assert evidence.missing_reason == "not persisted"
assert evidence.problem_text is None
assert evidence.proposals == []
assert evidence.mentions == []
assert evidence.bindings == []
assert evidence.bound_relations == []
assert evidence.contract_assessments == []
assert evidence.diagnostic_only is True
assert evidence.serving_allowed is False
def test_construction_evidence_from_legacy_journal_entry_fails_closed() -> None:
evidence = construction_evidence_from_journal_entry(_Entry(turn_id=3))
assert evidence.status == "missing_evidence"
assert evidence.turn_id == 3
assert evidence.missing_reason == CONSTRUCTION_EVIDENCE_ABSENT
assert evidence.diagnostic_only is True
assert evidence.serving_allowed is False
def test_construction_evidence_to_data_serializes_dataclasses() -> None:
payload = to_data(missing_construction_evidence(1, "absent"))
assert payload == {
"schema_version": "construction_evidence_v1",
"turn_id": 1,
"status": "missing_evidence",
"missing_reason": "absent",
"problem_text": None,
"proposals": [],
"mentions": [],
"bindings": [],
"bound_relations": [],
"contract_assessments": [],
"diagnostic_only": True,
"serving_allowed": False,
}
def test_span_is_exact_accepts_exact_slice() -> None:
text = "Lena has 3 red marbles."
assert span_is_exact(text, SourceSpanView(start=9, end=10, text="3")) is True
def test_span_is_exact_rejects_repaired_or_shifted_slice() -> None:
text = "Lena has 3 red marbles."
assert span_is_exact(text, SourceSpanView(start=9, end=10, text="three")) is False
assert span_is_exact(text, SourceSpanView(start=8, end=9, text="3")) is False
assert span_is_exact(text, SourceSpanView(start=-1, end=1, text="L")) is False
assert span_is_exact(text, SourceSpanView(start=9, end=200, text="3")) is False
assert span_is_exact(text, SourceSpanView(start=9, end=9, text="")) is False
def test_existing_construction_evidence_instance_round_trips() -> None:
existing = ConstructionEvidence(
schema_version="construction_evidence_v1",
turn_id=11,
status="recorded",
missing_reason=None,
problem_text="x",
diagnostic_only=True,
serving_allowed=False,
)
@dataclass(frozen=True, slots=True)
class EntryWithPayload:
turn_id: int
construction_evidence: ConstructionEvidence
assert construction_evidence_from_journal_entry(EntryWithPayload(11, existing)) is existing

View file

@ -0,0 +1,132 @@
from __future__ import annotations
from workbench.demo_narrative import (
DemoEvidenceLink,
DemoNarrative,
DemoNarrativeStep,
demo_partner_blurb,
validate_demo_narrative,
)
from workbench.schemas import to_data
def test_demo_narrative_requires_proof_and_non_proof_claims() -> None:
narrative = DemoNarrative(
narrative_id="bad",
title="Bad demo",
summary="missing honesty cards",
steps=[
DemoNarrativeStep(
step_id="s1",
order=1,
kind="evidence",
title="Trace",
claim="trace exists",
what_this_proves="",
what_this_does_not_prove="",
)
],
)
assert validate_demo_narrative(narrative) == [
"step s1 is missing what_this_proves",
"step s1 is missing what_this_does_not_prove",
]
def test_demo_narrative_requires_route_evidence_links() -> None:
narrative = DemoNarrative(
narrative_id="bad-route",
title="Bad route",
summary="bad evidence link",
steps=[
DemoNarrativeStep(
step_id="s1",
order=1,
kind="evidence",
title="Trace",
claim="trace exists",
what_this_proves="a trace route can be opened",
what_this_does_not_prove="that the answer is correct",
evidence_links=[DemoEvidenceLink(label="bad", route="trace/1")],
)
],
)
assert validate_demo_narrative(narrative) == [
"step s1 has non-route evidence link: trace/1"
]
def test_demo_narrative_detects_duplicate_steps_and_order() -> None:
narrative = DemoNarrative(
narrative_id="bad-order",
title="Bad order",
summary="duplicate ids and unsorted order",
steps=[
DemoNarrativeStep(
step_id="s1",
order=2,
kind="evidence",
title="Trace B",
claim="claim",
what_this_proves="proof",
what_this_does_not_prove="non-proof",
),
DemoNarrativeStep(
step_id="s1",
order=1,
kind="audit",
title="Audit A",
claim="claim",
what_this_proves="proof",
what_this_does_not_prove="non-proof",
),
],
)
assert validate_demo_narrative(narrative) == [
"duplicate step_id: s1",
"steps must be sorted by order",
]
def test_valid_demo_narrative_serializes() -> None:
narrative = DemoNarrative(
narrative_id="deterministic-turn",
title="Deterministic Turn Evidence",
summary="Chat to Trace to Replay",
steps=[
DemoNarrativeStep(
step_id="trace",
order=1,
kind="evidence",
title="Open Trace",
claim="The journaled turn has trace evidence.",
what_this_proves="The selected turn has recorded evidence.",
what_this_does_not_prove="It does not prove answer correctness by itself.",
evidence_links=[
DemoEvidenceLink(
label="Trace",
route="/trace/1",
artifact_id="turn:1",
digest="sha256:abc",
reproducer="curl /trace/1",
)
],
)
],
)
assert validate_demo_narrative(narrative) == []
payload = to_data(narrative)
assert payload["narrative_id"] == "deterministic-turn"
assert payload["steps"][0]["evidence_links"][0]["route"] == "/trace/1"
def test_partner_blurb_states_core_demo_thesis() -> None:
blurb = demo_partner_blurb()
assert "frontier model can propose" in blurb
assert "CORE can govern" in blurb
assert "Workbench can prove" in blurb

View file

@ -0,0 +1,91 @@
from __future__ import annotations
import pytest
from workbench.generalization_evidence import (
contains_forbidden_raw_item_keys,
generalization_audit_banner,
generalization_report_view_from_payload,
)
from workbench.schemas import to_data
def test_generalization_report_view_is_aggregate_only() -> None:
view = generalization_report_view_from_payload(
{
"policy_version": "generalization_benchmark_policy_v1",
"dataset": "gsm1k",
"split": "local_audit",
"n_items": 10,
"correct": 4,
"wrong": 0,
"refused": 6,
"unsupported": 0,
"candidate_attempts": 3,
"binding_failures": 2,
"replay_refusals": 1,
"sealed_trace_dispositions": {"sealed": 7, "refused": 3},
"dominant_residual_kinds": {"missing_binding": 5},
"reason_codes": ["ok", "binding_missing"],
},
source_path="evals/generalization/reports/gsm1k.json",
source_digest="sha256:abc",
report_kind="committed_pin",
)
assert view.policy_version == "generalization_benchmark_policy_v1"
assert view.dataset == "gsm1k"
assert view.split == "local_audit"
assert view.correct == 4
assert view.wrong == 0
assert view.refused == 6
assert view.audit_only is True
assert view.raw_items_exposed is False
assert view.report_kind == "committed_pin"
assert view.sealed_trace_dispositions == [("refused", 3), ("sealed", 7)]
assert view.dominant_residual_kinds == [("missing_binding", 5)]
def test_generalization_report_rejects_raw_item_fields() -> None:
with pytest.raises(ValueError, match="raw item"):
generalization_report_view_from_payload(
{
"dataset": "bad",
"split": "sealed",
"n_items": 1,
"items": [{"question": "secret", "answer": "secret"}],
}
)
def test_contains_forbidden_raw_item_keys_recurses() -> None:
assert contains_forbidden_raw_item_keys({"nested": {"gold_answer": "42"}}) is True
assert contains_forbidden_raw_item_keys({"aggregate": {"correct": 1}}) is False
def test_generalization_report_serializes_as_safe_dataclass() -> None:
payload = to_data(
generalization_report_view_from_payload(
{
"dataset": "asdiv",
"split": "public",
"n_items": 0,
}
)
)
assert payload["dataset"] == "asdiv"
assert payload["split"] == "public"
assert payload["audit_only"] is True
assert payload["raw_items_exposed"] is False
assert "items" not in payload
assert "question" not in payload
assert "answer" not in payload
def test_generalization_audit_banner_states_governance_boundary() -> None:
banner = generalization_audit_banner()
assert "Audit-only" in banner
assert "No raw sealed items" in banner
assert "not direct mutation targets" in banner

View file

@ -0,0 +1,88 @@
from __future__ import annotations
import pytest
from workbench.proposal_artifact import (
capability_disclosure,
proposal_artifact_from_minimal,
ratification_affordance_allowed,
)
from workbench.schemas import to_data
def test_inspect_only_artifact_disallows_ratification_affordance() -> None:
artifact = proposal_artifact_from_minimal(
proposal_id="p1",
subject_kind="construction",
subject_id="turn:1",
display_name="Construction proposal preview",
source_kind="construction_evidence",
)
assert artifact.capability_level == "inspect_only"
assert ratification_affordance_allowed(artifact) is False
assert "Inspect-only" in capability_disclosure(artifact)
def test_proposal_only_artifact_disallows_ratification_affordance() -> None:
artifact = proposal_artifact_from_minimal(
proposal_id="p2",
subject_kind="logos_pack",
subject_id="logos:demo",
display_name="CORE-Logos draft",
source_kind="logos_pack",
capability_level="proposal_only",
)
assert ratification_affordance_allowed(artifact) is False
assert "Proposal-only" in capability_disclosure(artifact)
def test_ratification_enabled_requires_handler_route_for_affordance() -> None:
artifact = proposal_artifact_from_minimal(
proposal_id="p3",
subject_kind="math",
subject_id="math:p3",
display_name="Math proposal",
source_kind="math",
capability_level="ratification_enabled",
handler_route="/math-proposals/p3/ratify",
)
assert ratification_affordance_allowed(artifact) is True
assert "admitted handler" in capability_disclosure(artifact)
def test_handler_route_without_ratification_capability_is_rejected() -> None:
with pytest.raises(ValueError, match="handler_route"):
proposal_artifact_from_minimal(
proposal_id="p4",
subject_kind="construction",
subject_id="turn:4",
display_name="Invalid preview",
source_kind="construction_evidence",
capability_level="inspect_only",
handler_route="/invalid",
)
def test_proposal_artifact_serializes_as_dataclass_payload() -> None:
payload = to_data(
proposal_artifact_from_minimal(
proposal_id="p5",
subject_kind="cognition",
subject_id="candidate:p5",
display_name="Cognition proposal",
source_kind="teaching_proposal_log",
state="pending",
)
)
assert payload["proposal_id"] == "p5"
assert payload["subject"] == {
"kind": "cognition",
"subject_id": "candidate:p5",
"display_name": "Cognition proposal",
}
assert payload["capability_level"] == "inspect_only"
assert payload["handler_route"] is None

View file

@ -0,0 +1,92 @@
import { describe, expect, it } from "vitest";
import type { DemoNarrative } from "../../types/demoNarrative";
import {
DEMO_PARTNER_BLURB,
demoNarrativeValidationWarnings,
demoStepHonestyPair,
} from "./demoNarrativeView";
const narrative: DemoNarrative = {
narrative_id: "deterministic-turn",
title: "Deterministic Turn Evidence",
summary: "Chat to Trace to Replay",
steps: [
{
step_id: "trace",
order: 1,
kind: "evidence",
title: "Open Trace",
claim: "The journaled turn has trace evidence.",
what_this_proves: "The selected turn has recorded evidence.",
what_this_does_not_prove: "It does not prove answer correctness by itself.",
evidence_links: [
{
label: "Trace",
route: "/trace/1",
artifact_id: "turn:1",
digest: "sha256:abc",
reproducer: "curl /trace/1",
},
],
failure_mode: null,
},
],
};
describe("demo narrative view helpers", () => {
it("states the partner demo thesis", () => {
expect(DEMO_PARTNER_BLURB).toContain("frontier model can propose");
expect(DEMO_PARTNER_BLURB).toContain("CORE can govern");
expect(DEMO_PARTNER_BLURB).toContain("Workbench can prove");
});
it("accepts valid narratives", () => {
expect(demoNarrativeValidationWarnings(narrative)).toEqual([]);
});
it("requires both proof and non-proof claims", () => {
expect(
demoNarrativeValidationWarnings({
...narrative,
steps: [
{
...narrative.steps[0],
what_this_proves: "",
what_this_does_not_prove: "",
},
],
}),
).toEqual([
"step trace is missing what_this_proves",
"step trace is missing what_this_does_not_prove",
]);
});
it("requires route-shaped evidence links", () => {
expect(
demoNarrativeValidationWarnings({
...narrative,
steps: [
{
...narrative.steps[0],
evidence_links: [
{
label: "bad",
route: "trace/1",
artifact_id: null,
digest: null,
reproducer: null,
},
],
},
],
}),
).toEqual(["step trace has non-route evidence link: trace/1"]);
});
it("renders honesty pair", () => {
expect(demoStepHonestyPair(narrative.steps[0])).toBe(
"Proves: The selected turn has recorded evidence. / Does not prove: It does not prove answer correctness by itself.",
);
});
});

View file

@ -0,0 +1,38 @@
import type { DemoNarrative } from "../../types/demoNarrative";
export const DEMO_PARTNER_BLURB =
"A frontier model can propose. CORE can govern. Workbench can prove what happened, what was refused, what replayed, and what remains only a proposal.";
export function demoNarrativeValidationWarnings(narrative: DemoNarrative): string[] {
const warnings: string[] = [];
const seen = new Set<string>();
const orders = narrative.steps.map((step) => step.order);
for (const step of narrative.steps) {
if (seen.has(step.step_id)) {
warnings.push(`duplicate step_id: ${step.step_id}`);
}
seen.add(step.step_id);
if (step.what_this_proves.trim().length === 0) {
warnings.push(`step ${step.step_id} is missing what_this_proves`);
}
if (step.what_this_does_not_prove.trim().length === 0) {
warnings.push(`step ${step.step_id} is missing what_this_does_not_prove`);
}
for (const link of step.evidence_links) {
if (!link.route.startsWith("/")) {
warnings.push(`step ${step.step_id} has non-route evidence link: ${link.route}`);
}
}
}
const sorted = [...orders].sort((a, b) => a - b);
if (orders.some((order, index) => order !== sorted[index])) {
warnings.push("steps must be sorted by order");
}
return warnings;
}
export function demoStepHonestyPair(step: DemoNarrative["steps"][number]): string {
return `Proves: ${step.what_this_proves} / Does not prove: ${step.what_this_does_not_prove}`;
}

View file

@ -0,0 +1,74 @@
import { describe, expect, it } from "vitest";
import type { GeneralizationAuditReportView } from "../../types/generalizationEvidence";
import {
GENERALIZATION_AUDIT_BANNER,
generalizationGovernanceWarnings,
generalizationOutcomeSummary,
reportKindLabel,
} from "./generalizationEvidenceView";
const baseReport: GeneralizationAuditReportView = {
policy_version: "generalization_benchmark_policy_v1",
dataset: "gsm1k",
split: "local_audit",
n_items: 10,
correct: 4,
wrong: 0,
refused: 6,
unsupported: 0,
candidate_attempts: 3,
binding_failures: 2,
replay_refusals: 1,
sealed_trace_dispositions: [["sealed", 7]],
dominant_residual_kinds: [["missing_binding", 5]],
reason_codes: ["ok"],
source_path: "evals/generalization/reports/gsm1k.json",
source_digest: "sha256:abc",
report_kind: "committed_pin",
audit_only: true,
raw_items_exposed: false,
};
describe("generalization evidence view helpers", () => {
it("states the benchmark governance boundary", () => {
expect(GENERALIZATION_AUDIT_BANNER).toContain("Audit-only");
expect(GENERALIZATION_AUDIT_BANNER).toContain("No raw sealed items");
expect(GENERALIZATION_AUDIT_BANNER).toContain("not direct mutation targets");
});
it("labels report authority", () => {
expect(reportKindLabel(baseReport)).toBe("Committed report pin");
expect(reportKindLabel({ ...baseReport, report_kind: "ephemeral_local" })).toBe(
"Ephemeral local output",
);
expect(reportKindLabel({ ...baseReport, report_kind: "rebaseline_candidate" })).toBe(
"Governed rebaseline candidate",
);
expect(reportKindLabel({ ...baseReport, report_kind: "unknown" })).toBe(
"Unknown report authority",
);
});
it("summarizes outcomes without hiding wrong count", () => {
expect(generalizationOutcomeSummary(baseReport)).toBe(
"4 correct / 0 wrong / 6 refused / 0 unsupported",
);
expect(generalizationOutcomeSummary({ ...baseReport, wrong: 2 })).toContain("2 wrong");
});
it("warns when report should not be treated as canonical", () => {
expect(generalizationGovernanceWarnings(baseReport)).toEqual([]);
expect(
generalizationGovernanceWarnings({
...baseReport,
report_kind: "ephemeral_local",
audit_only: false,
raw_items_exposed: true,
}),
).toEqual([
"Report is not marked audit-only.",
"Report claims raw items are exposed; do not render item payloads.",
"Report is not a committed pin; do not present as canonical baseline.",
]);
});
});

View file

@ -0,0 +1,36 @@
import type { GeneralizationAuditReportView } from "../../types/generalizationEvidence";
export const GENERALIZATION_AUDIT_BANNER =
"Audit-only. No raw sealed items are exposed here. Benchmark failures are diagnosis signals, not direct mutation targets.";
export function reportKindLabel(report: GeneralizationAuditReportView): string {
switch (report.report_kind) {
case "committed_pin":
return "Committed report pin";
case "ephemeral_local":
return "Ephemeral local output";
case "rebaseline_candidate":
return "Governed rebaseline candidate";
case "unknown":
default:
return "Unknown report authority";
}
}
export function generalizationOutcomeSummary(report: GeneralizationAuditReportView): string {
return `${report.correct} correct / ${report.wrong} wrong / ${report.refused} refused / ${report.unsupported} unsupported`;
}
export function generalizationGovernanceWarnings(report: GeneralizationAuditReportView): string[] {
const warnings: string[] = [];
if (!report.audit_only) {
warnings.push("Report is not marked audit-only.");
}
if (report.raw_items_exposed) {
warnings.push("Report claims raw items are exposed; do not render item payloads.");
}
if (report.report_kind !== "committed_pin") {
warnings.push("Report is not a committed pin; do not present as canonical baseline.");
}
return warnings;
}

View file

@ -0,0 +1,83 @@
import { describe, expect, it } from "vitest";
import type { ProposalArtifact } from "../../types/proposalArtifact";
import {
proposalArtifactWarnings,
proposalCapabilityLabel,
ratificationControlsAllowed,
} from "./proposalArtifactView";
const inspectOnly: ProposalArtifact = {
proposal_id: "p1",
subject: { kind: "construction", subject_id: "turn:1", display_name: "Construction" },
state: "unknown",
capability_level: "inspect_only",
source_kind: "construction_evidence",
proposed_change: null,
reasoning_trace: null,
evidence_pointers: [],
validation: null,
replay_evidence: null,
safety_report: null,
affected_artifacts: [],
handler_route: null,
suggested_cli: null,
audit_refs: [],
ui_disclosure: "inspect only",
};
describe("proposal artifact view helpers", () => {
it("disallows ratification controls for inspect-only artifacts", () => {
expect(ratificationControlsAllowed(inspectOnly)).toBe(false);
expect(proposalCapabilityLabel(inspectOnly)).toBe(
"Inspect-only: no mutation affordance allowed",
);
});
it("disallows ratification controls for proposal-only artifacts", () => {
const artifact = { ...inspectOnly, capability_level: "proposal_only" as const };
expect(ratificationControlsAllowed(artifact)).toBe(false);
expect(proposalCapabilityLabel(artifact)).toBe(
"Proposal-only: review/export/copy allowed; apply disabled",
);
});
it("allows ratification controls only with handler route", () => {
const artifact = {
...inspectOnly,
capability_level: "ratification_enabled" as const,
handler_route: "/math-proposals/p1/ratify",
};
expect(ratificationControlsAllowed(artifact)).toBe(true);
expect(proposalCapabilityLabel(artifact)).toBe(
"Ratification enabled by admitted handler",
);
});
it("warns on impossible handler/capability combinations", () => {
expect(
proposalArtifactWarnings({
...inspectOnly,
handler_route: "/invalid",
}),
).toEqual(["Handler route must not be present without ratification authority."]);
expect(
proposalArtifactWarnings({
...inspectOnly,
capability_level: "ratification_enabled",
}),
).toEqual(["Ratification authority declared without handler route."]);
});
it("surfaces validation and safety blockers", () => {
expect(
proposalArtifactWarnings({
...inspectOnly,
validation: { status: "blocked", blockers: ["missing evidence"], warnings: [] },
safety_report: { status: "failed", disclosures: ["handler not admitted"] },
}),
).toEqual(["missing evidence", "handler not admitted"]);
});
});

View file

@ -0,0 +1,36 @@
import type { ProposalArtifact } from "../../types/proposalArtifact";
export function ratificationControlsAllowed(artifact: ProposalArtifact): boolean {
return artifact.capability_level === "ratification_enabled" && artifact.handler_route !== null;
}
export function proposalCapabilityLabel(artifact: ProposalArtifact): string {
switch (artifact.capability_level) {
case "ratification_enabled":
return artifact.handler_route === null
? "Ratification disabled: no handler route"
: "Ratification enabled by admitted handler";
case "proposal_only":
return "Proposal-only: review/export/copy allowed; apply disabled";
case "inspect_only":
default:
return "Inspect-only: no mutation affordance allowed";
}
}
export function proposalArtifactWarnings(artifact: ProposalArtifact): string[] {
const warnings: string[] = [];
if (artifact.capability_level !== "ratification_enabled" && artifact.handler_route !== null) {
warnings.push("Handler route must not be present without ratification authority.");
}
if (artifact.capability_level === "ratification_enabled" && artifact.handler_route === null) {
warnings.push("Ratification authority declared without handler route.");
}
if (artifact.validation?.status === "blocked") {
warnings.push(...artifact.validation.blockers);
}
if (artifact.safety_report?.status === "failed") {
warnings.push(...artifact.safety_report.disclosures);
}
return warnings;
}

View file

@ -0,0 +1,82 @@
import { describe, expect, it } from "vitest";
import type { ConstructionEvidence, ContractAssessmentView } from "../../types/constructionEvidence";
import {
CONSTRUCTION_AUTHORITY_DISCLOSURES,
assessmentBlockerSummary,
assessmentDisposition,
constructionEvidenceEmptyMessage,
sourceSpanIsExact,
sourceSpanLabel,
} from "./constructionEvidenceView";
const missingEvidence: ConstructionEvidence = {
schema_version: "construction_evidence_v1",
turn_id: 1,
status: "missing_evidence",
missing_reason: "construction evidence was not persisted for this turn",
problem_text: null,
proposals: [],
mentions: [],
bindings: [],
bound_relations: [],
contract_assessments: [],
diagnostic_only: true,
serving_allowed: false,
};
const blockedAssessment: ContractAssessmentView = {
candidate_organ: "quantity_entity_binding_candidate.v1",
family_id: "binding.quantity_entity",
missing_bindings: ["entity"],
unresolved_hazards: ["ambiguous_quantity"],
runnable: false,
explanation: "missing entity binding",
evidence_spans: [],
};
describe("construction evidence view helpers", () => {
it("keeps load-bearing authority disclosures explicit", () => {
expect(CONSTRUCTION_AUTHORITY_DISCLOSURES).toEqual([
"Proposal != Runnable",
"Contract Determines",
"Diagnostic Only",
"Serving Disallowed",
"Exact Span Required",
]);
});
it("renders missing evidence as honest absence", () => {
expect(constructionEvidenceEmptyMessage(missingEvidence)).toBe(
"construction evidence was not persisted for this turn",
);
});
it("labels source spans without normalization", () => {
expect(sourceSpanLabel({ start: 9, end: 10, text: "3" })).toBe("9:10 3");
});
it("checks exact source spans against problem text", () => {
const text = "Lena has 3 red marbles.";
expect(sourceSpanIsExact(text, { start: 9, end: 10, text: "3" })).toBe(true);
expect(sourceSpanIsExact(text, { start: 9, end: 10, text: "three" })).toBe(false);
expect(sourceSpanIsExact(null, { start: 9, end: 10, text: "3" })).toBe(false);
});
it("classifies assessments as blocked when bindings or hazards remain", () => {
expect(assessmentDisposition(blockedAssessment)).toBe("blocked");
expect(assessmentBlockerSummary(blockedAssessment)).toBe("entity, ambiguous_quantity");
});
it("classifies assessments as runnable only with no blockers", () => {
const runnable: ContractAssessmentView = {
...blockedAssessment,
missing_bindings: [],
unresolved_hazards: [],
runnable: true,
};
expect(assessmentDisposition(runnable)).toBe("runnable");
expect(assessmentBlockerSummary(runnable)).toBe("No blockers recorded.");
});
});

View file

@ -0,0 +1,56 @@
import type {
ConstructionEvidence,
ContractAssessmentView,
SourceSpanView,
} from "../../types/constructionEvidence";
export const CONSTRUCTION_AUTHORITY_DISCLOSURES = [
"Proposal != Runnable",
"Contract Determines",
"Diagnostic Only",
"Serving Disallowed",
"Exact Span Required",
] as const;
export function constructionEvidenceEmptyMessage(evidence: ConstructionEvidence): string | null {
if (evidence.status === "missing_evidence") {
return evidence.missing_reason ?? "No construction evidence recorded for this turn.";
}
const hasEvidence =
evidence.proposals.length > 0 ||
evidence.mentions.length > 0 ||
evidence.bindings.length > 0 ||
evidence.bound_relations.length > 0 ||
evidence.contract_assessments.length > 0;
return hasEvidence ? null : "No construction evidence recorded for this turn.";
}
export function sourceSpanLabel(span: SourceSpanView): string {
return `${span.start}:${span.end} ${span.text}`;
}
export function sourceSpanIsExact(problemText: string | null, span: SourceSpanView): boolean {
if (problemText === null) {
return false;
}
return (
span.start >= 0 &&
span.end >= span.start &&
span.end <= problemText.length &&
span.text.length > 0 &&
problemText.slice(span.start, span.end) === span.text
);
}
export function assessmentDisposition(assessment: ContractAssessmentView): "runnable" | "blocked" {
return assessment.runnable &&
assessment.missing_bindings.length === 0 &&
assessment.unresolved_hazards.length === 0
? "runnable"
: "blocked";
}
export function assessmentBlockerSummary(assessment: ContractAssessmentView): string {
const blockers = [...assessment.missing_bindings, ...assessment.unresolved_hazards];
return blockers.length > 0 ? blockers.join(", ") : "No blockers recorded.";
}

View file

@ -0,0 +1,76 @@
export type ConstructionEvidenceStatus = "recorded" | "missing_evidence";
export interface SourceSpanView {
start: number;
end: number;
text: string;
}
export interface RoleObligationView {
role: string;
required: boolean;
description: string;
}
export interface ConstructionProposalView {
family_id: string;
relation_type: string;
candidate_organ: string;
status: "proposed";
evidence_spans: SourceSpanView[];
role_obligations: RoleObligationView[];
diagnostic_only: boolean;
serving_allowed: boolean;
}
export interface MentionView {
mention_id: string;
kind: string;
surface: string;
span: SourceSpanView;
fact_id: string | null;
}
export interface MentionBindingView {
binding_type: string;
source_mention_id: string;
target_mention_id: string;
evidence_spans: SourceSpanView[];
}
export interface BoundRelationRoleView {
role: string;
target_id: string;
evidence_spans: SourceSpanView[];
}
export interface BoundRelationView {
relation_type: string;
roles: BoundRelationRoleView[];
evidence_spans: SourceSpanView[];
}
export interface ContractAssessmentView {
candidate_organ: string;
family_id: string | null;
missing_bindings: string[];
unresolved_hazards: string[];
runnable: boolean;
explanation: string;
evidence_spans: SourceSpanView[];
}
export interface ConstructionEvidence {
schema_version: "construction_evidence_v1";
turn_id: number;
status: ConstructionEvidenceStatus;
missing_reason: string | null;
problem_text: string | null;
proposals: ConstructionProposalView[];
mentions: MentionView[];
bindings: MentionBindingView[];
bound_relations: BoundRelationView[];
contract_assessments: ContractAssessmentView[];
diagnostic_only: boolean;
serving_allowed: boolean;
}

View file

@ -0,0 +1,28 @@
export type DemoStepKind = "intro" | "evidence" | "replay" | "proposal" | "audit" | "payoff";
export interface DemoEvidenceLink {
label: string;
route: string;
artifact_id: string | null;
digest: string | null;
reproducer: string | null;
}
export interface DemoNarrativeStep {
step_id: string;
order: number;
kind: DemoStepKind;
title: string;
claim: string;
what_this_proves: string;
what_this_does_not_prove: string;
evidence_links: DemoEvidenceLink[];
failure_mode: string | null;
}
export interface DemoNarrative {
narrative_id: string;
title: string;
summary: string;
steps: DemoNarrativeStep[];
}

View file

@ -0,0 +1,46 @@
export type ChecksumStatus = "verified" | "missing" | "mismatch" | "unknown";
export type GeneralizationReportKind =
| "committed_pin"
| "ephemeral_local"
| "rebaseline_candidate"
| "unknown";
export interface GeneralizationManifestSummary {
dataset: string;
manifest_path: string;
split_names: string[];
license: string | null;
checksum_status: ChecksumStatus;
sealed_splits: string[];
policy_version: string;
}
export interface GeneralizationCacheStatus {
dataset: string;
cache_path: string;
present: boolean;
verified: boolean;
reason: string | null;
}
export interface GeneralizationAuditReportView {
policy_version: string;
dataset: string;
split: string;
n_items: number;
correct: number;
wrong: number;
refused: number;
unsupported: number;
candidate_attempts: number;
binding_failures: number;
replay_refusals: number;
sealed_trace_dispositions: [string, number][];
dominant_residual_kinds: [string, number][];
reason_codes: string[];
source_path: string | null;
source_digest: string | null;
report_kind: GeneralizationReportKind;
audit_only: boolean;
raw_items_exposed: boolean;
}

View file

@ -0,0 +1,55 @@
export type ProposalCapabilityLevel =
| "inspect_only"
| "proposal_only"
| "ratification_enabled";
export type ProposalArtifactState =
| "pending"
| "accepted"
| "rejected"
| "withdrawn"
| "deferred"
| "unknown";
export interface ProposalSubject {
kind: string;
subject_id: string;
display_name: string;
}
export interface EvidencePointer {
kind: string;
ref: string;
label: string;
digest: string | null;
}
export interface ProposalValidationReport {
status: "valid" | "blocked" | "unknown";
blockers: string[];
warnings: string[];
}
export interface ProposalSafetyReport {
status: "clear" | "warning" | "failed" | "unknown";
disclosures: string[];
}
export interface ProposalArtifact {
proposal_id: string;
subject: ProposalSubject;
state: ProposalArtifactState;
capability_level: ProposalCapabilityLevel;
source_kind: string;
proposed_change: unknown;
reasoning_trace: unknown;
evidence_pointers: EvidencePointer[];
validation: ProposalValidationReport | null;
replay_evidence: unknown;
safety_report: ProposalSafetyReport | null;
affected_artifacts: EvidencePointer[];
handler_route: string | null;
suggested_cli: string | null;
audit_refs: EvidencePointer[];
ui_disclosure: string;
}

View file

@ -0,0 +1,185 @@
"""Read-only construction evidence projection helpers for Workbench.
This module is intentionally inert: it defines the Workbench-facing construction
read model and the honest missing-evidence constructor for legacy turns. It does
not parse problem text, execute candidate operators, run replay, mutate journals,
or grant serving authority.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Literal
PipelineEvidenceStatus = Literal["recorded", "missing_evidence"]
CONSTRUCTION_EVIDENCE_ABSENT = "construction evidence was not persisted for this turn"
@dataclass(frozen=True, slots=True)
class SourceSpanView:
"""Exact source span projected to the Workbench.
The caller owns validation against the source string. This record must not be
normalized or repaired by the UI.
"""
start: int
end: int
text: str
@dataclass(frozen=True, slots=True)
class RoleObligationView:
role: str
required: bool
description: str
@dataclass(frozen=True, slots=True)
class ConstructionProposalView:
"""Diagnostic construction proposal.
A proposal is a hypothesis only. Runnable/refused authority belongs to the
corresponding ContractAssessmentView.
"""
family_id: str
relation_type: str
candidate_organ: str
status: Literal["proposed"]
evidence_spans: list[SourceSpanView] = field(default_factory=list)
role_obligations: list[RoleObligationView] = field(default_factory=list)
diagnostic_only: bool = True
serving_allowed: bool = False
@dataclass(frozen=True, slots=True)
class MentionView:
mention_id: str
kind: str
surface: str
span: SourceSpanView
fact_id: str | None = None
@dataclass(frozen=True, slots=True)
class MentionBindingView:
binding_type: str
source_mention_id: str
target_mention_id: str
evidence_spans: list[SourceSpanView] = field(default_factory=list)
@dataclass(frozen=True, slots=True)
class BoundRelationRoleView:
role: str
target_id: str
evidence_spans: list[SourceSpanView] = field(default_factory=list)
@dataclass(frozen=True, slots=True)
class BoundRelationView:
relation_type: str
roles: list[BoundRelationRoleView] = field(default_factory=list)
evidence_spans: list[SourceSpanView] = field(default_factory=list)
@dataclass(frozen=True, slots=True)
class ContractAssessmentView:
"""Read-only projection of generate.problem_frame_contracts.ContractAssessment."""
candidate_organ: str
family_id: str | None
missing_bindings: list[str] = field(default_factory=list)
unresolved_hazards: list[str] = field(default_factory=list)
runnable: bool = False
explanation: str = ""
evidence_spans: list[SourceSpanView] = field(default_factory=list)
@dataclass(frozen=True, slots=True)
class ConstructionEvidence:
"""Workbench-facing construction evidence for one turn.
This record is a projection, not an executor. `diagnostic_only=True` and
`serving_allowed=False` are load-bearing disclosure fields.
"""
schema_version: Literal["construction_evidence_v1"]
turn_id: int
status: PipelineEvidenceStatus
missing_reason: str | None
problem_text: str | None
proposals: list[ConstructionProposalView] = field(default_factory=list)
mentions: list[MentionView] = field(default_factory=list)
bindings: list[MentionBindingView] = field(default_factory=list)
bound_relations: list[BoundRelationView] = field(default_factory=list)
contract_assessments: list[ContractAssessmentView] = field(default_factory=list)
diagnostic_only: bool = True
serving_allowed: bool = False
def missing_construction_evidence(turn_id: int, reason: str) -> ConstructionEvidence:
"""Return the honest absence state for legacy turns.
Absence is not an error and not a failed proof. It means the selected turn has
no persisted construction evidence to project.
"""
return ConstructionEvidence(
schema_version="construction_evidence_v1",
turn_id=turn_id,
status="missing_evidence",
missing_reason=reason,
problem_text=None,
proposals=[],
mentions=[],
bindings=[],
bound_relations=[],
contract_assessments=[],
diagnostic_only=True,
serving_allowed=False,
)
def construction_evidence_from_journal_entry(entry: Any) -> ConstructionEvidence:
"""Project persisted construction evidence from a journal entry if present.
Current turn journals do not yet persist a construction evidence payload. This
helper is deliberately fail-closed and returns a typed missing-evidence record
rather than reconstructing a ProblemFrame from prose. When a later PR starts
persisting `construction_evidence`, this function is the narrow projection
seam to extend.
"""
turn_id = int(getattr(entry, "turn_id"))
payload = getattr(entry, "construction_evidence", None)
if payload is None:
return missing_construction_evidence(turn_id, CONSTRUCTION_EVIDENCE_ABSENT)
if isinstance(payload, ConstructionEvidence):
return payload
if not isinstance(payload, dict):
return missing_construction_evidence(
turn_id,
"construction evidence payload has unsupported shape",
)
# Future-compatible but conservative: do not attempt lossy coercion. The first
# persistence PR should replace this with explicit field-by-field projection and
# exact-span validation tests.
return missing_construction_evidence(
turn_id,
"construction evidence payload projection is not yet admitted",
)
def span_is_exact(problem_text: str, span: SourceSpanView) -> bool:
"""Return whether a span exactly matches its source text slice."""
return (
0 <= span.start <= span.end <= len(problem_text)
and bool(span.text)
and problem_text[span.start : span.end] == span.text
)

View file

@ -0,0 +1,72 @@
"""Demo narrative scaffolding for Workbench proof theater.
A demo narrative is an authored path over real evidence routes. It must state both
what a step proves and what it does not prove, so demo polish cannot outrun
substrate evidence.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Literal
DemoStepKind = Literal["intro", "evidence", "replay", "proposal", "audit", "payoff"]
@dataclass(frozen=True, slots=True)
class DemoEvidenceLink:
label: str
route: str
artifact_id: str | None = None
digest: str | None = None
reproducer: str | None = None
@dataclass(frozen=True, slots=True)
class DemoNarrativeStep:
step_id: str
order: int
kind: DemoStepKind
title: str
claim: str
what_this_proves: str
what_this_does_not_prove: str
evidence_links: list[DemoEvidenceLink] = field(default_factory=list)
failure_mode: str | None = None
@dataclass(frozen=True, slots=True)
class DemoNarrative:
narrative_id: str
title: str
summary: str
steps: list[DemoNarrativeStep] = field(default_factory=list)
def validate_demo_narrative(narrative: DemoNarrative) -> list[str]:
"""Return validation blockers for a Workbench demo narrative."""
blockers: list[str] = []
seen: set[str] = set()
for step in narrative.steps:
if step.step_id in seen:
blockers.append(f"duplicate step_id: {step.step_id}")
seen.add(step.step_id)
if not step.what_this_proves.strip():
blockers.append(f"step {step.step_id} is missing what_this_proves")
if not step.what_this_does_not_prove.strip():
blockers.append(f"step {step.step_id} is missing what_this_does_not_prove")
for link in step.evidence_links:
if not link.route.startswith("/"):
blockers.append(f"step {step.step_id} has non-route evidence link: {link.route}")
orders = [step.order for step in narrative.steps]
if orders != sorted(orders):
blockers.append("steps must be sorted by order")
return blockers
def demo_partner_blurb() -> str:
return (
"A frontier model can propose. CORE can govern. Workbench can prove what "
"happened, what was refused, what replayed, and what remains only a proposal."
)

View file

@ -0,0 +1,144 @@
"""Read-only generalization audit evidence projections for Workbench.
Generalization benchmark data is audit/test-only. This module deliberately models
aggregate report metadata only. It must not expose raw prompt/question/answer
content, sealed item payloads, or direct patch suggestions.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Literal
ChecksumStatus = Literal["verified", "missing", "mismatch", "unknown"]
ReportKind = Literal["committed_pin", "ephemeral_local", "rebaseline_candidate", "unknown"]
FORBIDDEN_RAW_ITEM_KEYS = frozenset(
{
"prompt",
"question",
"answer",
"gold_answer",
"raw_item",
"raw_items",
"items",
"examples",
"sealed_items",
}
)
@dataclass(frozen=True, slots=True)
class GeneralizationManifestSummary:
dataset: str
manifest_path: str
split_names: list[str]
license: str | None
checksum_status: ChecksumStatus
sealed_splits: list[str]
policy_version: str
@dataclass(frozen=True, slots=True)
class GeneralizationCacheStatus:
dataset: str
cache_path: str
present: bool
verified: bool
reason: str | None
@dataclass(frozen=True, slots=True)
class GeneralizationAuditReportView:
policy_version: str
dataset: str
split: str
n_items: int
correct: int
wrong: int
refused: int
unsupported: int
candidate_attempts: int
binding_failures: int
replay_refusals: int
sealed_trace_dispositions: list[tuple[str, int]] = field(default_factory=list)
dominant_residual_kinds: list[tuple[str, int]] = field(default_factory=list)
reason_codes: list[str] = field(default_factory=list)
source_path: str | None = None
source_digest: str | None = None
report_kind: ReportKind = "unknown"
audit_only: bool = True
raw_items_exposed: bool = False
def contains_forbidden_raw_item_keys(payload: Any) -> bool:
"""Return True if a payload contains raw/sealed item keys anywhere."""
if isinstance(payload, dict):
for key, value in payload.items():
if str(key) in FORBIDDEN_RAW_ITEM_KEYS:
return True
if contains_forbidden_raw_item_keys(value):
return True
elif isinstance(payload, list):
return any(contains_forbidden_raw_item_keys(item) for item in payload)
return False
def _count_pairs(value: Any) -> list[tuple[str, int]]:
if not isinstance(value, dict):
return []
pairs: list[tuple[str, int]] = []
for key, count in sorted(value.items(), key=lambda item: str(item[0])):
try:
pairs.append((str(key), int(count)))
except (TypeError, ValueError):
continue
return pairs
def generalization_report_view_from_payload(
payload: dict[str, Any],
*,
source_path: str | None = None,
source_digest: str | None = None,
report_kind: ReportKind = "unknown",
) -> GeneralizationAuditReportView:
"""Project an aggregate audit report into a Workbench-safe view.
This function is intentionally aggregate-only. It rejects payloads that carry
obvious raw/sealed item fields so the Workbench cannot accidentally render
benchmark examples or answers.
"""
if contains_forbidden_raw_item_keys(payload):
raise ValueError("generalization report payload exposes raw item fields")
return GeneralizationAuditReportView(
policy_version=str(payload.get("policy_version") or "unknown"),
dataset=str(payload.get("dataset") or "unknown"),
split=str(payload.get("split") or "unknown"),
n_items=int(payload.get("n_items") or 0),
correct=int(payload.get("correct") or 0),
wrong=int(payload.get("wrong") or 0),
refused=int(payload.get("refused") or 0),
unsupported=int(payload.get("unsupported") or 0),
candidate_attempts=int(payload.get("candidate_attempts") or 0),
binding_failures=int(payload.get("binding_failures") or 0),
replay_refusals=int(payload.get("replay_refusals") or 0),
sealed_trace_dispositions=_count_pairs(payload.get("sealed_trace_dispositions")),
dominant_residual_kinds=_count_pairs(payload.get("dominant_residual_kinds")),
reason_codes=[str(code) for code in payload.get("reason_codes") or []],
source_path=source_path,
source_digest=source_digest,
report_kind=report_kind,
audit_only=True,
raw_items_exposed=False,
)
def generalization_audit_banner() -> str:
return (
"Audit-only. No raw sealed items are exposed here. Benchmark failures are "
"diagnosis signals, not direct mutation targets."
)

View file

@ -0,0 +1,122 @@
"""ProposalArtifact authority scaffolding for Workbench.
This module defines a shared proposal-review envelope without admitting any new
ratification handler. It is safe, inert UI/read-model substrate: capability level
controls what the Workbench may display as an affordance.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Literal
ProposalCapabilityLevel = Literal[
"inspect_only",
"proposal_only",
"ratification_enabled",
]
ProposalArtifactState = Literal[
"pending",
"accepted",
"rejected",
"withdrawn",
"deferred",
"unknown",
]
@dataclass(frozen=True, slots=True)
class ProposalSubject:
kind: str
subject_id: str
display_name: str
@dataclass(frozen=True, slots=True)
class EvidencePointer:
kind: str
ref: str
label: str
digest: str | None = None
@dataclass(frozen=True, slots=True)
class ProposalValidationReport:
status: Literal["valid", "blocked", "unknown"]
blockers: list[str] = field(default_factory=list)
warnings: list[str] = field(default_factory=list)
@dataclass(frozen=True, slots=True)
class ProposalSafetyReport:
status: Literal["clear", "warning", "failed", "unknown"]
disclosures: list[str] = field(default_factory=list)
@dataclass(frozen=True, slots=True)
class ProposalArtifact:
proposal_id: str
subject: ProposalSubject
state: ProposalArtifactState
capability_level: ProposalCapabilityLevel
source_kind: str
proposed_change: Any = None
reasoning_trace: Any = None
evidence_pointers: list[EvidencePointer] = field(default_factory=list)
validation: ProposalValidationReport | None = None
replay_evidence: Any = None
safety_report: ProposalSafetyReport | None = None
affected_artifacts: list[EvidencePointer] = field(default_factory=list)
handler_route: str | None = None
suggested_cli: str | None = None
audit_refs: list[EvidencePointer] = field(default_factory=list)
ui_disclosure: str = "Proposal artifact is inspect-only unless an admitted handler exists."
def ratification_affordance_allowed(artifact: ProposalArtifact) -> bool:
"""Return whether UI may show ratify/reject/defer controls."""
return artifact.capability_level == "ratification_enabled" and bool(
artifact.handler_route
)
def capability_disclosure(artifact: ProposalArtifact) -> str:
if artifact.capability_level == "ratification_enabled":
if artifact.handler_route:
return "Ratification controls may render through the admitted handler route."
return "Ratification is disabled because no handler route is present."
if artifact.capability_level == "proposal_only":
return "Proposal-only artifact: review/export/copy are allowed; apply is not."
return "Inspect-only artifact: no apply, ratify, promote, or mutation affordance is allowed."
def proposal_artifact_from_minimal(
*,
proposal_id: str,
subject_kind: str,
subject_id: str,
display_name: str,
source_kind: str,
capability_level: ProposalCapabilityLevel = "inspect_only",
state: ProposalArtifactState = "unknown",
handler_route: str | None = None,
) -> ProposalArtifact:
"""Build a minimal shared proposal review envelope."""
artifact = ProposalArtifact(
proposal_id=proposal_id,
subject=ProposalSubject(
kind=subject_kind,
subject_id=subject_id,
display_name=display_name,
),
state=state,
capability_level=capability_level,
source_kind=source_kind,
handler_route=handler_route,
)
if capability_level != "ratification_enabled" and handler_route is not None:
raise ValueError("handler_route requires ratification_enabled capability")
return artifact