diff --git a/evals/learning_arc/run_demo.py b/evals/learning_arc/run_demo.py index fcd134e6..934431d8 100644 --- a/evals/learning_arc/run_demo.py +++ b/evals/learning_arc/run_demo.py @@ -177,7 +177,8 @@ def _scene1_cold_session( rt = ChatRuntime(config=cfg, engine_state_path=engine_state_dir) response = rt.chat(_DEMO_PROMPT) - candidates_file = engine_state_dir / "discovery_candidates.jsonl" + from engine_state import EngineStateStore + candidates_file = EngineStateStore(engine_state_dir)._resolve_dir() / "discovery_candidates.jsonl" candidates_persisted = ( len(candidates_file.read_text(encoding="utf-8").splitlines()) if candidates_file.exists() @@ -213,7 +214,8 @@ def _scene2_checkpoint_enrichment( "not by the operator. Sub-questions enumerate candidate " "chains the engine identified through corpus decomposition.", ) - candidates_file = engine_state_dir / "discovery_candidates.jsonl" + from engine_state import EngineStateStore + candidates_file = EngineStateStore(engine_state_dir)._resolve_dir() / "discovery_candidates.jsonl" if not candidates_file.exists(): raise RuntimeError("engine state has no discovery_candidates.jsonl — S1 did not persist") lines = [l for l in candidates_file.read_text(encoding="utf-8").splitlines() if l.strip()] diff --git a/tests/test_workbench_construction_evidence.py b/tests/test_workbench_construction_evidence.py new file mode 100644 index 00000000..c093ae2e --- /dev/null +++ b/tests/test_workbench_construction_evidence.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from workbench.construction_evidence import ( + CONSTRUCTION_EVIDENCE_ABSENT, + ConstructionEvidence, + SourceSpanView, + construction_evidence_from_journal_entry, + missing_construction_evidence, + span_is_exact, +) +from workbench.schemas import to_data + + +@dataclass(frozen=True, slots=True) +class _Entry: + turn_id: int + + +def test_missing_construction_evidence_is_diagnostic_and_non_serving() -> None: + evidence = missing_construction_evidence(7, "not persisted") + + assert evidence.schema_version == "construction_evidence_v1" + assert evidence.turn_id == 7 + assert evidence.status == "missing_evidence" + assert evidence.missing_reason == "not persisted" + assert evidence.problem_text is None + assert evidence.proposals == [] + assert evidence.mentions == [] + assert evidence.bindings == [] + assert evidence.bound_relations == [] + assert evidence.contract_assessments == [] + assert evidence.diagnostic_only is True + assert evidence.serving_allowed is False + + +def test_construction_evidence_from_legacy_journal_entry_fails_closed() -> None: + evidence = construction_evidence_from_journal_entry(_Entry(turn_id=3)) + + assert evidence.status == "missing_evidence" + assert evidence.turn_id == 3 + assert evidence.missing_reason == CONSTRUCTION_EVIDENCE_ABSENT + assert evidence.diagnostic_only is True + assert evidence.serving_allowed is False + + +def test_construction_evidence_to_data_serializes_dataclasses() -> None: + payload = to_data(missing_construction_evidence(1, "absent")) + + assert payload == { + "schema_version": "construction_evidence_v1", + "turn_id": 1, + "status": "missing_evidence", + "missing_reason": "absent", + "problem_text": None, + "proposals": [], + "mentions": [], + "bindings": [], + "bound_relations": [], + "contract_assessments": [], + "diagnostic_only": True, + "serving_allowed": False, + } + + +def test_span_is_exact_accepts_exact_slice() -> None: + text = "Lena has 3 red marbles." + assert span_is_exact(text, SourceSpanView(start=9, end=10, text="3")) is True + + +def test_span_is_exact_rejects_repaired_or_shifted_slice() -> None: + text = "Lena has 3 red marbles." + + assert span_is_exact(text, SourceSpanView(start=9, end=10, text="three")) is False + assert span_is_exact(text, SourceSpanView(start=8, end=9, text="3")) is False + assert span_is_exact(text, SourceSpanView(start=-1, end=1, text="L")) is False + assert span_is_exact(text, SourceSpanView(start=9, end=200, text="3")) is False + assert span_is_exact(text, SourceSpanView(start=9, end=9, text="")) is False + + +def test_existing_construction_evidence_instance_round_trips() -> None: + existing = ConstructionEvidence( + schema_version="construction_evidence_v1", + turn_id=11, + status="recorded", + missing_reason=None, + problem_text="x", + diagnostic_only=True, + serving_allowed=False, + ) + + @dataclass(frozen=True, slots=True) + class EntryWithPayload: + turn_id: int + construction_evidence: ConstructionEvidence + + assert construction_evidence_from_journal_entry(EntryWithPayload(11, existing)) is existing diff --git a/tests/test_workbench_demo_narrative.py b/tests/test_workbench_demo_narrative.py new file mode 100644 index 00000000..3a8036cb --- /dev/null +++ b/tests/test_workbench_demo_narrative.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +from workbench.demo_narrative import ( + DemoEvidenceLink, + DemoNarrative, + DemoNarrativeStep, + demo_partner_blurb, + validate_demo_narrative, +) +from workbench.schemas import to_data + + +def test_demo_narrative_requires_proof_and_non_proof_claims() -> None: + narrative = DemoNarrative( + narrative_id="bad", + title="Bad demo", + summary="missing honesty cards", + steps=[ + DemoNarrativeStep( + step_id="s1", + order=1, + kind="evidence", + title="Trace", + claim="trace exists", + what_this_proves="", + what_this_does_not_prove="", + ) + ], + ) + + assert validate_demo_narrative(narrative) == [ + "step s1 is missing what_this_proves", + "step s1 is missing what_this_does_not_prove", + ] + + +def test_demo_narrative_requires_route_evidence_links() -> None: + narrative = DemoNarrative( + narrative_id="bad-route", + title="Bad route", + summary="bad evidence link", + steps=[ + DemoNarrativeStep( + step_id="s1", + order=1, + kind="evidence", + title="Trace", + claim="trace exists", + what_this_proves="a trace route can be opened", + what_this_does_not_prove="that the answer is correct", + evidence_links=[DemoEvidenceLink(label="bad", route="trace/1")], + ) + ], + ) + + assert validate_demo_narrative(narrative) == [ + "step s1 has non-route evidence link: trace/1" + ] + + +def test_demo_narrative_detects_duplicate_steps_and_order() -> None: + narrative = DemoNarrative( + narrative_id="bad-order", + title="Bad order", + summary="duplicate ids and unsorted order", + steps=[ + DemoNarrativeStep( + step_id="s1", + order=2, + kind="evidence", + title="Trace B", + claim="claim", + what_this_proves="proof", + what_this_does_not_prove="non-proof", + ), + DemoNarrativeStep( + step_id="s1", + order=1, + kind="audit", + title="Audit A", + claim="claim", + what_this_proves="proof", + what_this_does_not_prove="non-proof", + ), + ], + ) + + assert validate_demo_narrative(narrative) == [ + "duplicate step_id: s1", + "steps must be sorted by order", + ] + + +def test_valid_demo_narrative_serializes() -> None: + narrative = DemoNarrative( + narrative_id="deterministic-turn", + title="Deterministic Turn Evidence", + summary="Chat to Trace to Replay", + steps=[ + DemoNarrativeStep( + step_id="trace", + order=1, + kind="evidence", + title="Open Trace", + claim="The journaled turn has trace evidence.", + what_this_proves="The selected turn has recorded evidence.", + what_this_does_not_prove="It does not prove answer correctness by itself.", + evidence_links=[ + DemoEvidenceLink( + label="Trace", + route="/trace/1", + artifact_id="turn:1", + digest="sha256:abc", + reproducer="curl /trace/1", + ) + ], + ) + ], + ) + + assert validate_demo_narrative(narrative) == [] + payload = to_data(narrative) + assert payload["narrative_id"] == "deterministic-turn" + assert payload["steps"][0]["evidence_links"][0]["route"] == "/trace/1" + + +def test_partner_blurb_states_core_demo_thesis() -> None: + blurb = demo_partner_blurb() + + assert "frontier model can propose" in blurb + assert "CORE can govern" in blurb + assert "Workbench can prove" in blurb diff --git a/tests/test_workbench_generalization_evidence.py b/tests/test_workbench_generalization_evidence.py new file mode 100644 index 00000000..b4fd46d1 --- /dev/null +++ b/tests/test_workbench_generalization_evidence.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import pytest + +from workbench.generalization_evidence import ( + contains_forbidden_raw_item_keys, + generalization_audit_banner, + generalization_report_view_from_payload, +) +from workbench.schemas import to_data + + +def test_generalization_report_view_is_aggregate_only() -> None: + view = generalization_report_view_from_payload( + { + "policy_version": "generalization_benchmark_policy_v1", + "dataset": "gsm1k", + "split": "local_audit", + "n_items": 10, + "correct": 4, + "wrong": 0, + "refused": 6, + "unsupported": 0, + "candidate_attempts": 3, + "binding_failures": 2, + "replay_refusals": 1, + "sealed_trace_dispositions": {"sealed": 7, "refused": 3}, + "dominant_residual_kinds": {"missing_binding": 5}, + "reason_codes": ["ok", "binding_missing"], + }, + source_path="evals/generalization/reports/gsm1k.json", + source_digest="sha256:abc", + report_kind="committed_pin", + ) + + assert view.policy_version == "generalization_benchmark_policy_v1" + assert view.dataset == "gsm1k" + assert view.split == "local_audit" + assert view.correct == 4 + assert view.wrong == 0 + assert view.refused == 6 + assert view.audit_only is True + assert view.raw_items_exposed is False + assert view.report_kind == "committed_pin" + assert view.sealed_trace_dispositions == [("refused", 3), ("sealed", 7)] + assert view.dominant_residual_kinds == [("missing_binding", 5)] + + +def test_generalization_report_rejects_raw_item_fields() -> None: + with pytest.raises(ValueError, match="raw item"): + generalization_report_view_from_payload( + { + "dataset": "bad", + "split": "sealed", + "n_items": 1, + "items": [{"question": "secret", "answer": "secret"}], + } + ) + + +def test_contains_forbidden_raw_item_keys_recurses() -> None: + assert contains_forbidden_raw_item_keys({"nested": {"gold_answer": "42"}}) is True + assert contains_forbidden_raw_item_keys({"aggregate": {"correct": 1}}) is False + + +def test_generalization_report_serializes_as_safe_dataclass() -> None: + payload = to_data( + generalization_report_view_from_payload( + { + "dataset": "asdiv", + "split": "public", + "n_items": 0, + } + ) + ) + + assert payload["dataset"] == "asdiv" + assert payload["split"] == "public" + assert payload["audit_only"] is True + assert payload["raw_items_exposed"] is False + assert "items" not in payload + assert "question" not in payload + assert "answer" not in payload + + +def test_generalization_audit_banner_states_governance_boundary() -> None: + banner = generalization_audit_banner() + + assert "Audit-only" in banner + assert "No raw sealed items" in banner + assert "not direct mutation targets" in banner diff --git a/tests/test_workbench_proposal_artifact.py b/tests/test_workbench_proposal_artifact.py new file mode 100644 index 00000000..8f7ec6a0 --- /dev/null +++ b/tests/test_workbench_proposal_artifact.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import pytest + +from workbench.proposal_artifact import ( + capability_disclosure, + proposal_artifact_from_minimal, + ratification_affordance_allowed, +) +from workbench.schemas import to_data + + +def test_inspect_only_artifact_disallows_ratification_affordance() -> None: + artifact = proposal_artifact_from_minimal( + proposal_id="p1", + subject_kind="construction", + subject_id="turn:1", + display_name="Construction proposal preview", + source_kind="construction_evidence", + ) + + assert artifact.capability_level == "inspect_only" + assert ratification_affordance_allowed(artifact) is False + assert "Inspect-only" in capability_disclosure(artifact) + + +def test_proposal_only_artifact_disallows_ratification_affordance() -> None: + artifact = proposal_artifact_from_minimal( + proposal_id="p2", + subject_kind="logos_pack", + subject_id="logos:demo", + display_name="CORE-Logos draft", + source_kind="logos_pack", + capability_level="proposal_only", + ) + + assert ratification_affordance_allowed(artifact) is False + assert "Proposal-only" in capability_disclosure(artifact) + + +def test_ratification_enabled_requires_handler_route_for_affordance() -> None: + artifact = proposal_artifact_from_minimal( + proposal_id="p3", + subject_kind="math", + subject_id="math:p3", + display_name="Math proposal", + source_kind="math", + capability_level="ratification_enabled", + handler_route="/math-proposals/p3/ratify", + ) + + assert ratification_affordance_allowed(artifact) is True + assert "admitted handler" in capability_disclosure(artifact) + + +def test_handler_route_without_ratification_capability_is_rejected() -> None: + with pytest.raises(ValueError, match="handler_route"): + proposal_artifact_from_minimal( + proposal_id="p4", + subject_kind="construction", + subject_id="turn:4", + display_name="Invalid preview", + source_kind="construction_evidence", + capability_level="inspect_only", + handler_route="/invalid", + ) + + +def test_proposal_artifact_serializes_as_dataclass_payload() -> None: + payload = to_data( + proposal_artifact_from_minimal( + proposal_id="p5", + subject_kind="cognition", + subject_id="candidate:p5", + display_name="Cognition proposal", + source_kind="teaching_proposal_log", + state="pending", + ) + ) + + assert payload["proposal_id"] == "p5" + assert payload["subject"] == { + "kind": "cognition", + "subject_id": "candidate:p5", + "display_name": "Cognition proposal", + } + assert payload["capability_level"] == "inspect_only" + assert payload["handler_route"] is None diff --git a/workbench-ui/src/app/demos/demoNarrativeView.test.ts b/workbench-ui/src/app/demos/demoNarrativeView.test.ts new file mode 100644 index 00000000..7a8ecbac --- /dev/null +++ b/workbench-ui/src/app/demos/demoNarrativeView.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from "vitest"; +import type { DemoNarrative } from "../../types/demoNarrative"; +import { + DEMO_PARTNER_BLURB, + demoNarrativeValidationWarnings, + demoStepHonestyPair, +} from "./demoNarrativeView"; + +const narrative: DemoNarrative = { + narrative_id: "deterministic-turn", + title: "Deterministic Turn Evidence", + summary: "Chat to Trace to Replay", + steps: [ + { + step_id: "trace", + order: 1, + kind: "evidence", + title: "Open Trace", + claim: "The journaled turn has trace evidence.", + what_this_proves: "The selected turn has recorded evidence.", + what_this_does_not_prove: "It does not prove answer correctness by itself.", + evidence_links: [ + { + label: "Trace", + route: "/trace/1", + artifact_id: "turn:1", + digest: "sha256:abc", + reproducer: "curl /trace/1", + }, + ], + failure_mode: null, + }, + ], +}; + +describe("demo narrative view helpers", () => { + it("states the partner demo thesis", () => { + expect(DEMO_PARTNER_BLURB).toContain("frontier model can propose"); + expect(DEMO_PARTNER_BLURB).toContain("CORE can govern"); + expect(DEMO_PARTNER_BLURB).toContain("Workbench can prove"); + }); + + it("accepts valid narratives", () => { + expect(demoNarrativeValidationWarnings(narrative)).toEqual([]); + }); + + it("requires both proof and non-proof claims", () => { + expect( + demoNarrativeValidationWarnings({ + ...narrative, + steps: [ + { + ...narrative.steps[0], + what_this_proves: "", + what_this_does_not_prove: "", + }, + ], + }), + ).toEqual([ + "step trace is missing what_this_proves", + "step trace is missing what_this_does_not_prove", + ]); + }); + + it("requires route-shaped evidence links", () => { + expect( + demoNarrativeValidationWarnings({ + ...narrative, + steps: [ + { + ...narrative.steps[0], + evidence_links: [ + { + label: "bad", + route: "trace/1", + artifact_id: null, + digest: null, + reproducer: null, + }, + ], + }, + ], + }), + ).toEqual(["step trace has non-route evidence link: trace/1"]); + }); + + it("renders honesty pair", () => { + expect(demoStepHonestyPair(narrative.steps[0])).toBe( + "Proves: The selected turn has recorded evidence. / Does not prove: It does not prove answer correctness by itself.", + ); + }); +}); diff --git a/workbench-ui/src/app/demos/demoNarrativeView.ts b/workbench-ui/src/app/demos/demoNarrativeView.ts new file mode 100644 index 00000000..d0284c65 --- /dev/null +++ b/workbench-ui/src/app/demos/demoNarrativeView.ts @@ -0,0 +1,38 @@ +import type { DemoNarrative } from "../../types/demoNarrative"; + +export const DEMO_PARTNER_BLURB = + "A frontier model can propose. CORE can govern. Workbench can prove what happened, what was refused, what replayed, and what remains only a proposal."; + +export function demoNarrativeValidationWarnings(narrative: DemoNarrative): string[] { + const warnings: string[] = []; + const seen = new Set(); + const orders = narrative.steps.map((step) => step.order); + + for (const step of narrative.steps) { + if (seen.has(step.step_id)) { + warnings.push(`duplicate step_id: ${step.step_id}`); + } + seen.add(step.step_id); + if (step.what_this_proves.trim().length === 0) { + warnings.push(`step ${step.step_id} is missing what_this_proves`); + } + if (step.what_this_does_not_prove.trim().length === 0) { + warnings.push(`step ${step.step_id} is missing what_this_does_not_prove`); + } + for (const link of step.evidence_links) { + if (!link.route.startsWith("/")) { + warnings.push(`step ${step.step_id} has non-route evidence link: ${link.route}`); + } + } + } + + const sorted = [...orders].sort((a, b) => a - b); + if (orders.some((order, index) => order !== sorted[index])) { + warnings.push("steps must be sorted by order"); + } + return warnings; +} + +export function demoStepHonestyPair(step: DemoNarrative["steps"][number]): string { + return `Proves: ${step.what_this_proves} / Does not prove: ${step.what_this_does_not_prove}`; +} diff --git a/workbench-ui/src/app/evals/generalizationEvidenceView.test.ts b/workbench-ui/src/app/evals/generalizationEvidenceView.test.ts new file mode 100644 index 00000000..4fe1d355 --- /dev/null +++ b/workbench-ui/src/app/evals/generalizationEvidenceView.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; +import type { GeneralizationAuditReportView } from "../../types/generalizationEvidence"; +import { + GENERALIZATION_AUDIT_BANNER, + generalizationGovernanceWarnings, + generalizationOutcomeSummary, + reportKindLabel, +} from "./generalizationEvidenceView"; + +const baseReport: GeneralizationAuditReportView = { + policy_version: "generalization_benchmark_policy_v1", + dataset: "gsm1k", + split: "local_audit", + n_items: 10, + correct: 4, + wrong: 0, + refused: 6, + unsupported: 0, + candidate_attempts: 3, + binding_failures: 2, + replay_refusals: 1, + sealed_trace_dispositions: [["sealed", 7]], + dominant_residual_kinds: [["missing_binding", 5]], + reason_codes: ["ok"], + source_path: "evals/generalization/reports/gsm1k.json", + source_digest: "sha256:abc", + report_kind: "committed_pin", + audit_only: true, + raw_items_exposed: false, +}; + +describe("generalization evidence view helpers", () => { + it("states the benchmark governance boundary", () => { + expect(GENERALIZATION_AUDIT_BANNER).toContain("Audit-only"); + expect(GENERALIZATION_AUDIT_BANNER).toContain("No raw sealed items"); + expect(GENERALIZATION_AUDIT_BANNER).toContain("not direct mutation targets"); + }); + + it("labels report authority", () => { + expect(reportKindLabel(baseReport)).toBe("Committed report pin"); + expect(reportKindLabel({ ...baseReport, report_kind: "ephemeral_local" })).toBe( + "Ephemeral local output", + ); + expect(reportKindLabel({ ...baseReport, report_kind: "rebaseline_candidate" })).toBe( + "Governed rebaseline candidate", + ); + expect(reportKindLabel({ ...baseReport, report_kind: "unknown" })).toBe( + "Unknown report authority", + ); + }); + + it("summarizes outcomes without hiding wrong count", () => { + expect(generalizationOutcomeSummary(baseReport)).toBe( + "4 correct / 0 wrong / 6 refused / 0 unsupported", + ); + expect(generalizationOutcomeSummary({ ...baseReport, wrong: 2 })).toContain("2 wrong"); + }); + + it("warns when report should not be treated as canonical", () => { + expect(generalizationGovernanceWarnings(baseReport)).toEqual([]); + expect( + generalizationGovernanceWarnings({ + ...baseReport, + report_kind: "ephemeral_local", + audit_only: false, + raw_items_exposed: true, + }), + ).toEqual([ + "Report is not marked audit-only.", + "Report claims raw items are exposed; do not render item payloads.", + "Report is not a committed pin; do not present as canonical baseline.", + ]); + }); +}); diff --git a/workbench-ui/src/app/evals/generalizationEvidenceView.ts b/workbench-ui/src/app/evals/generalizationEvidenceView.ts new file mode 100644 index 00000000..c5147125 --- /dev/null +++ b/workbench-ui/src/app/evals/generalizationEvidenceView.ts @@ -0,0 +1,36 @@ +import type { GeneralizationAuditReportView } from "../../types/generalizationEvidence"; + +export const GENERALIZATION_AUDIT_BANNER = + "Audit-only. No raw sealed items are exposed here. Benchmark failures are diagnosis signals, not direct mutation targets."; + +export function reportKindLabel(report: GeneralizationAuditReportView): string { + switch (report.report_kind) { + case "committed_pin": + return "Committed report pin"; + case "ephemeral_local": + return "Ephemeral local output"; + case "rebaseline_candidate": + return "Governed rebaseline candidate"; + case "unknown": + default: + return "Unknown report authority"; + } +} + +export function generalizationOutcomeSummary(report: GeneralizationAuditReportView): string { + return `${report.correct} correct / ${report.wrong} wrong / ${report.refused} refused / ${report.unsupported} unsupported`; +} + +export function generalizationGovernanceWarnings(report: GeneralizationAuditReportView): string[] { + const warnings: string[] = []; + if (!report.audit_only) { + warnings.push("Report is not marked audit-only."); + } + if (report.raw_items_exposed) { + warnings.push("Report claims raw items are exposed; do not render item payloads."); + } + if (report.report_kind !== "committed_pin") { + warnings.push("Report is not a committed pin; do not present as canonical baseline."); + } + return warnings; +} diff --git a/workbench-ui/src/app/proposal-artifacts/proposalArtifactView.test.ts b/workbench-ui/src/app/proposal-artifacts/proposalArtifactView.test.ts new file mode 100644 index 00000000..ce2623b7 --- /dev/null +++ b/workbench-ui/src/app/proposal-artifacts/proposalArtifactView.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from "vitest"; +import type { ProposalArtifact } from "../../types/proposalArtifact"; +import { + proposalArtifactWarnings, + proposalCapabilityLabel, + ratificationControlsAllowed, +} from "./proposalArtifactView"; + +const inspectOnly: ProposalArtifact = { + proposal_id: "p1", + subject: { kind: "construction", subject_id: "turn:1", display_name: "Construction" }, + state: "unknown", + capability_level: "inspect_only", + source_kind: "construction_evidence", + proposed_change: null, + reasoning_trace: null, + evidence_pointers: [], + validation: null, + replay_evidence: null, + safety_report: null, + affected_artifacts: [], + handler_route: null, + suggested_cli: null, + audit_refs: [], + ui_disclosure: "inspect only", +}; + +describe("proposal artifact view helpers", () => { + it("disallows ratification controls for inspect-only artifacts", () => { + expect(ratificationControlsAllowed(inspectOnly)).toBe(false); + expect(proposalCapabilityLabel(inspectOnly)).toBe( + "Inspect-only: no mutation affordance allowed", + ); + }); + + it("disallows ratification controls for proposal-only artifacts", () => { + const artifact = { ...inspectOnly, capability_level: "proposal_only" as const }; + + expect(ratificationControlsAllowed(artifact)).toBe(false); + expect(proposalCapabilityLabel(artifact)).toBe( + "Proposal-only: review/export/copy allowed; apply disabled", + ); + }); + + it("allows ratification controls only with handler route", () => { + const artifact = { + ...inspectOnly, + capability_level: "ratification_enabled" as const, + handler_route: "/math-proposals/p1/ratify", + }; + + expect(ratificationControlsAllowed(artifact)).toBe(true); + expect(proposalCapabilityLabel(artifact)).toBe( + "Ratification enabled by admitted handler", + ); + }); + + it("warns on impossible handler/capability combinations", () => { + expect( + proposalArtifactWarnings({ + ...inspectOnly, + handler_route: "/invalid", + }), + ).toEqual(["Handler route must not be present without ratification authority."]); + + expect( + proposalArtifactWarnings({ + ...inspectOnly, + capability_level: "ratification_enabled", + }), + ).toEqual(["Ratification authority declared without handler route."]); + }); + + it("surfaces validation and safety blockers", () => { + expect( + proposalArtifactWarnings({ + ...inspectOnly, + validation: { status: "blocked", blockers: ["missing evidence"], warnings: [] }, + safety_report: { status: "failed", disclosures: ["handler not admitted"] }, + }), + ).toEqual(["missing evidence", "handler not admitted"]); + }); +}); diff --git a/workbench-ui/src/app/proposal-artifacts/proposalArtifactView.ts b/workbench-ui/src/app/proposal-artifacts/proposalArtifactView.ts new file mode 100644 index 00000000..31a679b3 --- /dev/null +++ b/workbench-ui/src/app/proposal-artifacts/proposalArtifactView.ts @@ -0,0 +1,36 @@ +import type { ProposalArtifact } from "../../types/proposalArtifact"; + +export function ratificationControlsAllowed(artifact: ProposalArtifact): boolean { + return artifact.capability_level === "ratification_enabled" && artifact.handler_route !== null; +} + +export function proposalCapabilityLabel(artifact: ProposalArtifact): string { + switch (artifact.capability_level) { + case "ratification_enabled": + return artifact.handler_route === null + ? "Ratification disabled: no handler route" + : "Ratification enabled by admitted handler"; + case "proposal_only": + return "Proposal-only: review/export/copy allowed; apply disabled"; + case "inspect_only": + default: + return "Inspect-only: no mutation affordance allowed"; + } +} + +export function proposalArtifactWarnings(artifact: ProposalArtifact): string[] { + const warnings: string[] = []; + if (artifact.capability_level !== "ratification_enabled" && artifact.handler_route !== null) { + warnings.push("Handler route must not be present without ratification authority."); + } + if (artifact.capability_level === "ratification_enabled" && artifact.handler_route === null) { + warnings.push("Ratification authority declared without handler route."); + } + if (artifact.validation?.status === "blocked") { + warnings.push(...artifact.validation.blockers); + } + if (artifact.safety_report?.status === "failed") { + warnings.push(...artifact.safety_report.disclosures); + } + return warnings; +} diff --git a/workbench-ui/src/app/trace/constructionEvidenceView.test.ts b/workbench-ui/src/app/trace/constructionEvidenceView.test.ts new file mode 100644 index 00000000..0face1f7 --- /dev/null +++ b/workbench-ui/src/app/trace/constructionEvidenceView.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from "vitest"; +import type { ConstructionEvidence, ContractAssessmentView } from "../../types/constructionEvidence"; +import { + CONSTRUCTION_AUTHORITY_DISCLOSURES, + assessmentBlockerSummary, + assessmentDisposition, + constructionEvidenceEmptyMessage, + sourceSpanIsExact, + sourceSpanLabel, +} from "./constructionEvidenceView"; + +const missingEvidence: ConstructionEvidence = { + schema_version: "construction_evidence_v1", + turn_id: 1, + status: "missing_evidence", + missing_reason: "construction evidence was not persisted for this turn", + problem_text: null, + proposals: [], + mentions: [], + bindings: [], + bound_relations: [], + contract_assessments: [], + diagnostic_only: true, + serving_allowed: false, +}; + +const blockedAssessment: ContractAssessmentView = { + candidate_organ: "quantity_entity_binding_candidate.v1", + family_id: "binding.quantity_entity", + missing_bindings: ["entity"], + unresolved_hazards: ["ambiguous_quantity"], + runnable: false, + explanation: "missing entity binding", + evidence_spans: [], +}; + +describe("construction evidence view helpers", () => { + it("keeps load-bearing authority disclosures explicit", () => { + expect(CONSTRUCTION_AUTHORITY_DISCLOSURES).toEqual([ + "Proposal != Runnable", + "Contract Determines", + "Diagnostic Only", + "Serving Disallowed", + "Exact Span Required", + ]); + }); + + it("renders missing evidence as honest absence", () => { + expect(constructionEvidenceEmptyMessage(missingEvidence)).toBe( + "construction evidence was not persisted for this turn", + ); + }); + + it("labels source spans without normalization", () => { + expect(sourceSpanLabel({ start: 9, end: 10, text: "3" })).toBe("9:10 3"); + }); + + it("checks exact source spans against problem text", () => { + const text = "Lena has 3 red marbles."; + + expect(sourceSpanIsExact(text, { start: 9, end: 10, text: "3" })).toBe(true); + expect(sourceSpanIsExact(text, { start: 9, end: 10, text: "three" })).toBe(false); + expect(sourceSpanIsExact(null, { start: 9, end: 10, text: "3" })).toBe(false); + }); + + it("classifies assessments as blocked when bindings or hazards remain", () => { + expect(assessmentDisposition(blockedAssessment)).toBe("blocked"); + expect(assessmentBlockerSummary(blockedAssessment)).toBe("entity, ambiguous_quantity"); + }); + + it("classifies assessments as runnable only with no blockers", () => { + const runnable: ContractAssessmentView = { + ...blockedAssessment, + missing_bindings: [], + unresolved_hazards: [], + runnable: true, + }; + + expect(assessmentDisposition(runnable)).toBe("runnable"); + expect(assessmentBlockerSummary(runnable)).toBe("No blockers recorded."); + }); +}); diff --git a/workbench-ui/src/app/trace/constructionEvidenceView.ts b/workbench-ui/src/app/trace/constructionEvidenceView.ts new file mode 100644 index 00000000..6b8ec97e --- /dev/null +++ b/workbench-ui/src/app/trace/constructionEvidenceView.ts @@ -0,0 +1,56 @@ +import type { + ConstructionEvidence, + ContractAssessmentView, + SourceSpanView, +} from "../../types/constructionEvidence"; + +export const CONSTRUCTION_AUTHORITY_DISCLOSURES = [ + "Proposal != Runnable", + "Contract Determines", + "Diagnostic Only", + "Serving Disallowed", + "Exact Span Required", +] as const; + +export function constructionEvidenceEmptyMessage(evidence: ConstructionEvidence): string | null { + if (evidence.status === "missing_evidence") { + return evidence.missing_reason ?? "No construction evidence recorded for this turn."; + } + const hasEvidence = + evidence.proposals.length > 0 || + evidence.mentions.length > 0 || + evidence.bindings.length > 0 || + evidence.bound_relations.length > 0 || + evidence.contract_assessments.length > 0; + return hasEvidence ? null : "No construction evidence recorded for this turn."; +} + +export function sourceSpanLabel(span: SourceSpanView): string { + return `${span.start}:${span.end} ${span.text}`; +} + +export function sourceSpanIsExact(problemText: string | null, span: SourceSpanView): boolean { + if (problemText === null) { + return false; + } + return ( + span.start >= 0 && + span.end >= span.start && + span.end <= problemText.length && + span.text.length > 0 && + problemText.slice(span.start, span.end) === span.text + ); +} + +export function assessmentDisposition(assessment: ContractAssessmentView): "runnable" | "blocked" { + return assessment.runnable && + assessment.missing_bindings.length === 0 && + assessment.unresolved_hazards.length === 0 + ? "runnable" + : "blocked"; +} + +export function assessmentBlockerSummary(assessment: ContractAssessmentView): string { + const blockers = [...assessment.missing_bindings, ...assessment.unresolved_hazards]; + return blockers.length > 0 ? blockers.join(", ") : "No blockers recorded."; +} diff --git a/workbench-ui/src/types/constructionEvidence.ts b/workbench-ui/src/types/constructionEvidence.ts new file mode 100644 index 00000000..7e4a0c84 --- /dev/null +++ b/workbench-ui/src/types/constructionEvidence.ts @@ -0,0 +1,76 @@ +export type ConstructionEvidenceStatus = "recorded" | "missing_evidence"; + +export interface SourceSpanView { + start: number; + end: number; + text: string; +} + +export interface RoleObligationView { + role: string; + required: boolean; + description: string; +} + +export interface ConstructionProposalView { + family_id: string; + relation_type: string; + candidate_organ: string; + status: "proposed"; + evidence_spans: SourceSpanView[]; + role_obligations: RoleObligationView[]; + diagnostic_only: boolean; + serving_allowed: boolean; +} + +export interface MentionView { + mention_id: string; + kind: string; + surface: string; + span: SourceSpanView; + fact_id: string | null; +} + +export interface MentionBindingView { + binding_type: string; + source_mention_id: string; + target_mention_id: string; + evidence_spans: SourceSpanView[]; +} + +export interface BoundRelationRoleView { + role: string; + target_id: string; + evidence_spans: SourceSpanView[]; +} + +export interface BoundRelationView { + relation_type: string; + roles: BoundRelationRoleView[]; + evidence_spans: SourceSpanView[]; +} + +export interface ContractAssessmentView { + candidate_organ: string; + family_id: string | null; + missing_bindings: string[]; + unresolved_hazards: string[]; + runnable: boolean; + explanation: string; + evidence_spans: SourceSpanView[]; +} + +export interface ConstructionEvidence { + schema_version: "construction_evidence_v1"; + turn_id: number; + status: ConstructionEvidenceStatus; + missing_reason: string | null; + problem_text: string | null; + proposals: ConstructionProposalView[]; + mentions: MentionView[]; + bindings: MentionBindingView[]; + bound_relations: BoundRelationView[]; + contract_assessments: ContractAssessmentView[]; + diagnostic_only: boolean; + serving_allowed: boolean; +} diff --git a/workbench-ui/src/types/demoNarrative.ts b/workbench-ui/src/types/demoNarrative.ts new file mode 100644 index 00000000..1623f2da --- /dev/null +++ b/workbench-ui/src/types/demoNarrative.ts @@ -0,0 +1,28 @@ +export type DemoStepKind = "intro" | "evidence" | "replay" | "proposal" | "audit" | "payoff"; + +export interface DemoEvidenceLink { + label: string; + route: string; + artifact_id: string | null; + digest: string | null; + reproducer: string | null; +} + +export interface DemoNarrativeStep { + step_id: string; + order: number; + kind: DemoStepKind; + title: string; + claim: string; + what_this_proves: string; + what_this_does_not_prove: string; + evidence_links: DemoEvidenceLink[]; + failure_mode: string | null; +} + +export interface DemoNarrative { + narrative_id: string; + title: string; + summary: string; + steps: DemoNarrativeStep[]; +} diff --git a/workbench-ui/src/types/generalizationEvidence.ts b/workbench-ui/src/types/generalizationEvidence.ts new file mode 100644 index 00000000..83672228 --- /dev/null +++ b/workbench-ui/src/types/generalizationEvidence.ts @@ -0,0 +1,46 @@ +export type ChecksumStatus = "verified" | "missing" | "mismatch" | "unknown"; +export type GeneralizationReportKind = + | "committed_pin" + | "ephemeral_local" + | "rebaseline_candidate" + | "unknown"; + +export interface GeneralizationManifestSummary { + dataset: string; + manifest_path: string; + split_names: string[]; + license: string | null; + checksum_status: ChecksumStatus; + sealed_splits: string[]; + policy_version: string; +} + +export interface GeneralizationCacheStatus { + dataset: string; + cache_path: string; + present: boolean; + verified: boolean; + reason: string | null; +} + +export interface GeneralizationAuditReportView { + policy_version: string; + dataset: string; + split: string; + n_items: number; + correct: number; + wrong: number; + refused: number; + unsupported: number; + candidate_attempts: number; + binding_failures: number; + replay_refusals: number; + sealed_trace_dispositions: [string, number][]; + dominant_residual_kinds: [string, number][]; + reason_codes: string[]; + source_path: string | null; + source_digest: string | null; + report_kind: GeneralizationReportKind; + audit_only: boolean; + raw_items_exposed: boolean; +} diff --git a/workbench-ui/src/types/proposalArtifact.ts b/workbench-ui/src/types/proposalArtifact.ts new file mode 100644 index 00000000..864a450f --- /dev/null +++ b/workbench-ui/src/types/proposalArtifact.ts @@ -0,0 +1,55 @@ +export type ProposalCapabilityLevel = + | "inspect_only" + | "proposal_only" + | "ratification_enabled"; + +export type ProposalArtifactState = + | "pending" + | "accepted" + | "rejected" + | "withdrawn" + | "deferred" + | "unknown"; + +export interface ProposalSubject { + kind: string; + subject_id: string; + display_name: string; +} + +export interface EvidencePointer { + kind: string; + ref: string; + label: string; + digest: string | null; +} + +export interface ProposalValidationReport { + status: "valid" | "blocked" | "unknown"; + blockers: string[]; + warnings: string[]; +} + +export interface ProposalSafetyReport { + status: "clear" | "warning" | "failed" | "unknown"; + disclosures: string[]; +} + +export interface ProposalArtifact { + proposal_id: string; + subject: ProposalSubject; + state: ProposalArtifactState; + capability_level: ProposalCapabilityLevel; + source_kind: string; + proposed_change: unknown; + reasoning_trace: unknown; + evidence_pointers: EvidencePointer[]; + validation: ProposalValidationReport | null; + replay_evidence: unknown; + safety_report: ProposalSafetyReport | null; + affected_artifacts: EvidencePointer[]; + handler_route: string | null; + suggested_cli: string | null; + audit_refs: EvidencePointer[]; + ui_disclosure: string; +} diff --git a/workbench/construction_evidence.py b/workbench/construction_evidence.py new file mode 100644 index 00000000..4eaa6823 --- /dev/null +++ b/workbench/construction_evidence.py @@ -0,0 +1,185 @@ +"""Read-only construction evidence projection helpers for Workbench. + +This module is intentionally inert: it defines the Workbench-facing construction +read model and the honest missing-evidence constructor for legacy turns. It does +not parse problem text, execute candidate operators, run replay, mutate journals, +or grant serving authority. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Literal + +PipelineEvidenceStatus = Literal["recorded", "missing_evidence"] +CONSTRUCTION_EVIDENCE_ABSENT = "construction evidence was not persisted for this turn" + + +@dataclass(frozen=True, slots=True) +class SourceSpanView: + """Exact source span projected to the Workbench. + + The caller owns validation against the source string. This record must not be + normalized or repaired by the UI. + """ + + start: int + end: int + text: str + + +@dataclass(frozen=True, slots=True) +class RoleObligationView: + role: str + required: bool + description: str + + +@dataclass(frozen=True, slots=True) +class ConstructionProposalView: + """Diagnostic construction proposal. + + A proposal is a hypothesis only. Runnable/refused authority belongs to the + corresponding ContractAssessmentView. + """ + + family_id: str + relation_type: str + candidate_organ: str + status: Literal["proposed"] + evidence_spans: list[SourceSpanView] = field(default_factory=list) + role_obligations: list[RoleObligationView] = field(default_factory=list) + diagnostic_only: bool = True + serving_allowed: bool = False + + +@dataclass(frozen=True, slots=True) +class MentionView: + mention_id: str + kind: str + surface: str + span: SourceSpanView + fact_id: str | None = None + + +@dataclass(frozen=True, slots=True) +class MentionBindingView: + binding_type: str + source_mention_id: str + target_mention_id: str + evidence_spans: list[SourceSpanView] = field(default_factory=list) + + +@dataclass(frozen=True, slots=True) +class BoundRelationRoleView: + role: str + target_id: str + evidence_spans: list[SourceSpanView] = field(default_factory=list) + + +@dataclass(frozen=True, slots=True) +class BoundRelationView: + relation_type: str + roles: list[BoundRelationRoleView] = field(default_factory=list) + evidence_spans: list[SourceSpanView] = field(default_factory=list) + + +@dataclass(frozen=True, slots=True) +class ContractAssessmentView: + """Read-only projection of generate.problem_frame_contracts.ContractAssessment.""" + + candidate_organ: str + family_id: str | None + missing_bindings: list[str] = field(default_factory=list) + unresolved_hazards: list[str] = field(default_factory=list) + runnable: bool = False + explanation: str = "" + evidence_spans: list[SourceSpanView] = field(default_factory=list) + + +@dataclass(frozen=True, slots=True) +class ConstructionEvidence: + """Workbench-facing construction evidence for one turn. + + This record is a projection, not an executor. `diagnostic_only=True` and + `serving_allowed=False` are load-bearing disclosure fields. + """ + + schema_version: Literal["construction_evidence_v1"] + turn_id: int + status: PipelineEvidenceStatus + missing_reason: str | None + problem_text: str | None + proposals: list[ConstructionProposalView] = field(default_factory=list) + mentions: list[MentionView] = field(default_factory=list) + bindings: list[MentionBindingView] = field(default_factory=list) + bound_relations: list[BoundRelationView] = field(default_factory=list) + contract_assessments: list[ContractAssessmentView] = field(default_factory=list) + diagnostic_only: bool = True + serving_allowed: bool = False + + +def missing_construction_evidence(turn_id: int, reason: str) -> ConstructionEvidence: + """Return the honest absence state for legacy turns. + + Absence is not an error and not a failed proof. It means the selected turn has + no persisted construction evidence to project. + """ + + return ConstructionEvidence( + schema_version="construction_evidence_v1", + turn_id=turn_id, + status="missing_evidence", + missing_reason=reason, + problem_text=None, + proposals=[], + mentions=[], + bindings=[], + bound_relations=[], + contract_assessments=[], + diagnostic_only=True, + serving_allowed=False, + ) + + +def construction_evidence_from_journal_entry(entry: Any) -> ConstructionEvidence: + """Project persisted construction evidence from a journal entry if present. + + Current turn journals do not yet persist a construction evidence payload. This + helper is deliberately fail-closed and returns a typed missing-evidence record + rather than reconstructing a ProblemFrame from prose. When a later PR starts + persisting `construction_evidence`, this function is the narrow projection + seam to extend. + """ + + turn_id = int(getattr(entry, "turn_id")) + payload = getattr(entry, "construction_evidence", None) + if payload is None: + return missing_construction_evidence(turn_id, CONSTRUCTION_EVIDENCE_ABSENT) + + if isinstance(payload, ConstructionEvidence): + return payload + + if not isinstance(payload, dict): + return missing_construction_evidence( + turn_id, + "construction evidence payload has unsupported shape", + ) + + # Future-compatible but conservative: do not attempt lossy coercion. The first + # persistence PR should replace this with explicit field-by-field projection and + # exact-span validation tests. + return missing_construction_evidence( + turn_id, + "construction evidence payload projection is not yet admitted", + ) + + +def span_is_exact(problem_text: str, span: SourceSpanView) -> bool: + """Return whether a span exactly matches its source text slice.""" + + return ( + 0 <= span.start <= span.end <= len(problem_text) + and bool(span.text) + and problem_text[span.start : span.end] == span.text + ) diff --git a/workbench/demo_narrative.py b/workbench/demo_narrative.py new file mode 100644 index 00000000..6f280d5c --- /dev/null +++ b/workbench/demo_narrative.py @@ -0,0 +1,72 @@ +"""Demo narrative scaffolding for Workbench proof theater. + +A demo narrative is an authored path over real evidence routes. It must state both +what a step proves and what it does not prove, so demo polish cannot outrun +substrate evidence. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Literal + +DemoStepKind = Literal["intro", "evidence", "replay", "proposal", "audit", "payoff"] + + +@dataclass(frozen=True, slots=True) +class DemoEvidenceLink: + label: str + route: str + artifact_id: str | None = None + digest: str | None = None + reproducer: str | None = None + + +@dataclass(frozen=True, slots=True) +class DemoNarrativeStep: + step_id: str + order: int + kind: DemoStepKind + title: str + claim: str + what_this_proves: str + what_this_does_not_prove: str + evidence_links: list[DemoEvidenceLink] = field(default_factory=list) + failure_mode: str | None = None + + +@dataclass(frozen=True, slots=True) +class DemoNarrative: + narrative_id: str + title: str + summary: str + steps: list[DemoNarrativeStep] = field(default_factory=list) + + +def validate_demo_narrative(narrative: DemoNarrative) -> list[str]: + """Return validation blockers for a Workbench demo narrative.""" + + blockers: list[str] = [] + seen: set[str] = set() + for step in narrative.steps: + if step.step_id in seen: + blockers.append(f"duplicate step_id: {step.step_id}") + seen.add(step.step_id) + if not step.what_this_proves.strip(): + blockers.append(f"step {step.step_id} is missing what_this_proves") + if not step.what_this_does_not_prove.strip(): + blockers.append(f"step {step.step_id} is missing what_this_does_not_prove") + for link in step.evidence_links: + if not link.route.startswith("/"): + blockers.append(f"step {step.step_id} has non-route evidence link: {link.route}") + orders = [step.order for step in narrative.steps] + if orders != sorted(orders): + blockers.append("steps must be sorted by order") + return blockers + + +def demo_partner_blurb() -> str: + return ( + "A frontier model can propose. CORE can govern. Workbench can prove what " + "happened, what was refused, what replayed, and what remains only a proposal." + ) diff --git a/workbench/generalization_evidence.py b/workbench/generalization_evidence.py new file mode 100644 index 00000000..8de197a2 --- /dev/null +++ b/workbench/generalization_evidence.py @@ -0,0 +1,144 @@ +"""Read-only generalization audit evidence projections for Workbench. + +Generalization benchmark data is audit/test-only. This module deliberately models +aggregate report metadata only. It must not expose raw prompt/question/answer +content, sealed item payloads, or direct patch suggestions. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Literal + +ChecksumStatus = Literal["verified", "missing", "mismatch", "unknown"] +ReportKind = Literal["committed_pin", "ephemeral_local", "rebaseline_candidate", "unknown"] + +FORBIDDEN_RAW_ITEM_KEYS = frozenset( + { + "prompt", + "question", + "answer", + "gold_answer", + "raw_item", + "raw_items", + "items", + "examples", + "sealed_items", + } +) + + +@dataclass(frozen=True, slots=True) +class GeneralizationManifestSummary: + dataset: str + manifest_path: str + split_names: list[str] + license: str | None + checksum_status: ChecksumStatus + sealed_splits: list[str] + policy_version: str + + +@dataclass(frozen=True, slots=True) +class GeneralizationCacheStatus: + dataset: str + cache_path: str + present: bool + verified: bool + reason: str | None + + +@dataclass(frozen=True, slots=True) +class GeneralizationAuditReportView: + policy_version: str + dataset: str + split: str + n_items: int + correct: int + wrong: int + refused: int + unsupported: int + candidate_attempts: int + binding_failures: int + replay_refusals: int + sealed_trace_dispositions: list[tuple[str, int]] = field(default_factory=list) + dominant_residual_kinds: list[tuple[str, int]] = field(default_factory=list) + reason_codes: list[str] = field(default_factory=list) + source_path: str | None = None + source_digest: str | None = None + report_kind: ReportKind = "unknown" + audit_only: bool = True + raw_items_exposed: bool = False + + +def contains_forbidden_raw_item_keys(payload: Any) -> bool: + """Return True if a payload contains raw/sealed item keys anywhere.""" + + if isinstance(payload, dict): + for key, value in payload.items(): + if str(key) in FORBIDDEN_RAW_ITEM_KEYS: + return True + if contains_forbidden_raw_item_keys(value): + return True + elif isinstance(payload, list): + return any(contains_forbidden_raw_item_keys(item) for item in payload) + return False + + +def _count_pairs(value: Any) -> list[tuple[str, int]]: + if not isinstance(value, dict): + return [] + pairs: list[tuple[str, int]] = [] + for key, count in sorted(value.items(), key=lambda item: str(item[0])): + try: + pairs.append((str(key), int(count))) + except (TypeError, ValueError): + continue + return pairs + + +def generalization_report_view_from_payload( + payload: dict[str, Any], + *, + source_path: str | None = None, + source_digest: str | None = None, + report_kind: ReportKind = "unknown", +) -> GeneralizationAuditReportView: + """Project an aggregate audit report into a Workbench-safe view. + + This function is intentionally aggregate-only. It rejects payloads that carry + obvious raw/sealed item fields so the Workbench cannot accidentally render + benchmark examples or answers. + """ + + if contains_forbidden_raw_item_keys(payload): + raise ValueError("generalization report payload exposes raw item fields") + + return GeneralizationAuditReportView( + policy_version=str(payload.get("policy_version") or "unknown"), + dataset=str(payload.get("dataset") or "unknown"), + split=str(payload.get("split") or "unknown"), + n_items=int(payload.get("n_items") or 0), + correct=int(payload.get("correct") or 0), + wrong=int(payload.get("wrong") or 0), + refused=int(payload.get("refused") or 0), + unsupported=int(payload.get("unsupported") or 0), + candidate_attempts=int(payload.get("candidate_attempts") or 0), + binding_failures=int(payload.get("binding_failures") or 0), + replay_refusals=int(payload.get("replay_refusals") or 0), + sealed_trace_dispositions=_count_pairs(payload.get("sealed_trace_dispositions")), + dominant_residual_kinds=_count_pairs(payload.get("dominant_residual_kinds")), + reason_codes=[str(code) for code in payload.get("reason_codes") or []], + source_path=source_path, + source_digest=source_digest, + report_kind=report_kind, + audit_only=True, + raw_items_exposed=False, + ) + + +def generalization_audit_banner() -> str: + return ( + "Audit-only. No raw sealed items are exposed here. Benchmark failures are " + "diagnosis signals, not direct mutation targets." + ) diff --git a/workbench/proposal_artifact.py b/workbench/proposal_artifact.py new file mode 100644 index 00000000..72771c6f --- /dev/null +++ b/workbench/proposal_artifact.py @@ -0,0 +1,122 @@ +"""ProposalArtifact authority scaffolding for Workbench. + +This module defines a shared proposal-review envelope without admitting any new +ratification handler. It is safe, inert UI/read-model substrate: capability level +controls what the Workbench may display as an affordance. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Literal + +ProposalCapabilityLevel = Literal[ + "inspect_only", + "proposal_only", + "ratification_enabled", +] + +ProposalArtifactState = Literal[ + "pending", + "accepted", + "rejected", + "withdrawn", + "deferred", + "unknown", +] + + +@dataclass(frozen=True, slots=True) +class ProposalSubject: + kind: str + subject_id: str + display_name: str + + +@dataclass(frozen=True, slots=True) +class EvidencePointer: + kind: str + ref: str + label: str + digest: str | None = None + + +@dataclass(frozen=True, slots=True) +class ProposalValidationReport: + status: Literal["valid", "blocked", "unknown"] + blockers: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + + +@dataclass(frozen=True, slots=True) +class ProposalSafetyReport: + status: Literal["clear", "warning", "failed", "unknown"] + disclosures: list[str] = field(default_factory=list) + + +@dataclass(frozen=True, slots=True) +class ProposalArtifact: + proposal_id: str + subject: ProposalSubject + state: ProposalArtifactState + capability_level: ProposalCapabilityLevel + source_kind: str + proposed_change: Any = None + reasoning_trace: Any = None + evidence_pointers: list[EvidencePointer] = field(default_factory=list) + validation: ProposalValidationReport | None = None + replay_evidence: Any = None + safety_report: ProposalSafetyReport | None = None + affected_artifacts: list[EvidencePointer] = field(default_factory=list) + handler_route: str | None = None + suggested_cli: str | None = None + audit_refs: list[EvidencePointer] = field(default_factory=list) + ui_disclosure: str = "Proposal artifact is inspect-only unless an admitted handler exists." + + +def ratification_affordance_allowed(artifact: ProposalArtifact) -> bool: + """Return whether UI may show ratify/reject/defer controls.""" + + return artifact.capability_level == "ratification_enabled" and bool( + artifact.handler_route + ) + + +def capability_disclosure(artifact: ProposalArtifact) -> str: + if artifact.capability_level == "ratification_enabled": + if artifact.handler_route: + return "Ratification controls may render through the admitted handler route." + return "Ratification is disabled because no handler route is present." + if artifact.capability_level == "proposal_only": + return "Proposal-only artifact: review/export/copy are allowed; apply is not." + return "Inspect-only artifact: no apply, ratify, promote, or mutation affordance is allowed." + + +def proposal_artifact_from_minimal( + *, + proposal_id: str, + subject_kind: str, + subject_id: str, + display_name: str, + source_kind: str, + capability_level: ProposalCapabilityLevel = "inspect_only", + state: ProposalArtifactState = "unknown", + handler_route: str | None = None, +) -> ProposalArtifact: + """Build a minimal shared proposal review envelope.""" + + artifact = ProposalArtifact( + proposal_id=proposal_id, + subject=ProposalSubject( + kind=subject_kind, + subject_id=subject_id, + display_name=display_name, + ), + state=state, + capability_level=capability_level, + source_kind=source_kind, + handler_route=handler_route, + ) + if capability_level != "ratification_enabled" and handler_route is not None: + raise ValueError("handler_route requires ratification_enabled capability") + return artifact