feat(workbench): add UI catch-up evidence scaffolding (#891)
* feat(workbench): add construction evidence read model scaffolding * feat(workbench-ui): add construction evidence TS types * feat(workbench): add construction evidence journal projector * test(workbench): cover construction evidence read model * feat(workbench-ui): add construction evidence view helpers * test(workbench-ui): cover construction evidence view helpers * feat(workbench): add generalization audit evidence scaffolding * test(workbench): cover generalization audit evidence view * feat(workbench-ui): add generalization evidence TS types * feat(workbench-ui): add generalization evidence view helpers * test(workbench-ui): cover generalization evidence helpers * feat(workbench): add proposal artifact authority scaffolding * test(workbench): cover proposal artifact authority rules * feat(workbench-ui): add proposal artifact TS types * feat(workbench-ui): add proposal artifact view helpers * test(workbench-ui): cover proposal artifact view helpers * feat(workbench): add demo narrative evidence scaffolding * test(workbench): cover demo narrative scaffolding * feat(workbench-ui): add demo narrative TS types * feat(workbench-ui): add demo narrative view helpers * test(workbench-ui): cover demo narrative view helpers * fix(evals): resolve discovery_candidates.jsonl via EngineStateStore
This commit is contained in:
parent
172afc2d9e
commit
2c8258fe1a
21 changed files with 1638 additions and 2 deletions
|
|
@ -177,7 +177,8 @@ def _scene1_cold_session(
|
|||
rt = ChatRuntime(config=cfg, engine_state_path=engine_state_dir)
|
||||
response = rt.chat(_DEMO_PROMPT)
|
||||
|
||||
candidates_file = engine_state_dir / "discovery_candidates.jsonl"
|
||||
from engine_state import EngineStateStore
|
||||
candidates_file = EngineStateStore(engine_state_dir)._resolve_dir() / "discovery_candidates.jsonl"
|
||||
candidates_persisted = (
|
||||
len(candidates_file.read_text(encoding="utf-8").splitlines())
|
||||
if candidates_file.exists()
|
||||
|
|
@ -213,7 +214,8 @@ def _scene2_checkpoint_enrichment(
|
|||
"not by the operator. Sub-questions enumerate candidate "
|
||||
"chains the engine identified through corpus decomposition.",
|
||||
)
|
||||
candidates_file = engine_state_dir / "discovery_candidates.jsonl"
|
||||
from engine_state import EngineStateStore
|
||||
candidates_file = EngineStateStore(engine_state_dir)._resolve_dir() / "discovery_candidates.jsonl"
|
||||
if not candidates_file.exists():
|
||||
raise RuntimeError("engine state has no discovery_candidates.jsonl — S1 did not persist")
|
||||
lines = [l for l in candidates_file.read_text(encoding="utf-8").splitlines() if l.strip()]
|
||||
|
|
|
|||
98
tests/test_workbench_construction_evidence.py
Normal file
98
tests/test_workbench_construction_evidence.py
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from workbench.construction_evidence import (
|
||||
CONSTRUCTION_EVIDENCE_ABSENT,
|
||||
ConstructionEvidence,
|
||||
SourceSpanView,
|
||||
construction_evidence_from_journal_entry,
|
||||
missing_construction_evidence,
|
||||
span_is_exact,
|
||||
)
|
||||
from workbench.schemas import to_data
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Entry:
|
||||
turn_id: int
|
||||
|
||||
|
||||
def test_missing_construction_evidence_is_diagnostic_and_non_serving() -> None:
|
||||
evidence = missing_construction_evidence(7, "not persisted")
|
||||
|
||||
assert evidence.schema_version == "construction_evidence_v1"
|
||||
assert evidence.turn_id == 7
|
||||
assert evidence.status == "missing_evidence"
|
||||
assert evidence.missing_reason == "not persisted"
|
||||
assert evidence.problem_text is None
|
||||
assert evidence.proposals == []
|
||||
assert evidence.mentions == []
|
||||
assert evidence.bindings == []
|
||||
assert evidence.bound_relations == []
|
||||
assert evidence.contract_assessments == []
|
||||
assert evidence.diagnostic_only is True
|
||||
assert evidence.serving_allowed is False
|
||||
|
||||
|
||||
def test_construction_evidence_from_legacy_journal_entry_fails_closed() -> None:
|
||||
evidence = construction_evidence_from_journal_entry(_Entry(turn_id=3))
|
||||
|
||||
assert evidence.status == "missing_evidence"
|
||||
assert evidence.turn_id == 3
|
||||
assert evidence.missing_reason == CONSTRUCTION_EVIDENCE_ABSENT
|
||||
assert evidence.diagnostic_only is True
|
||||
assert evidence.serving_allowed is False
|
||||
|
||||
|
||||
def test_construction_evidence_to_data_serializes_dataclasses() -> None:
|
||||
payload = to_data(missing_construction_evidence(1, "absent"))
|
||||
|
||||
assert payload == {
|
||||
"schema_version": "construction_evidence_v1",
|
||||
"turn_id": 1,
|
||||
"status": "missing_evidence",
|
||||
"missing_reason": "absent",
|
||||
"problem_text": None,
|
||||
"proposals": [],
|
||||
"mentions": [],
|
||||
"bindings": [],
|
||||
"bound_relations": [],
|
||||
"contract_assessments": [],
|
||||
"diagnostic_only": True,
|
||||
"serving_allowed": False,
|
||||
}
|
||||
|
||||
|
||||
def test_span_is_exact_accepts_exact_slice() -> None:
|
||||
text = "Lena has 3 red marbles."
|
||||
assert span_is_exact(text, SourceSpanView(start=9, end=10, text="3")) is True
|
||||
|
||||
|
||||
def test_span_is_exact_rejects_repaired_or_shifted_slice() -> None:
|
||||
text = "Lena has 3 red marbles."
|
||||
|
||||
assert span_is_exact(text, SourceSpanView(start=9, end=10, text="three")) is False
|
||||
assert span_is_exact(text, SourceSpanView(start=8, end=9, text="3")) is False
|
||||
assert span_is_exact(text, SourceSpanView(start=-1, end=1, text="L")) is False
|
||||
assert span_is_exact(text, SourceSpanView(start=9, end=200, text="3")) is False
|
||||
assert span_is_exact(text, SourceSpanView(start=9, end=9, text="")) is False
|
||||
|
||||
|
||||
def test_existing_construction_evidence_instance_round_trips() -> None:
|
||||
existing = ConstructionEvidence(
|
||||
schema_version="construction_evidence_v1",
|
||||
turn_id=11,
|
||||
status="recorded",
|
||||
missing_reason=None,
|
||||
problem_text="x",
|
||||
diagnostic_only=True,
|
||||
serving_allowed=False,
|
||||
)
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class EntryWithPayload:
|
||||
turn_id: int
|
||||
construction_evidence: ConstructionEvidence
|
||||
|
||||
assert construction_evidence_from_journal_entry(EntryWithPayload(11, existing)) is existing
|
||||
132
tests/test_workbench_demo_narrative.py
Normal file
132
tests/test_workbench_demo_narrative.py
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from workbench.demo_narrative import (
|
||||
DemoEvidenceLink,
|
||||
DemoNarrative,
|
||||
DemoNarrativeStep,
|
||||
demo_partner_blurb,
|
||||
validate_demo_narrative,
|
||||
)
|
||||
from workbench.schemas import to_data
|
||||
|
||||
|
||||
def test_demo_narrative_requires_proof_and_non_proof_claims() -> None:
|
||||
narrative = DemoNarrative(
|
||||
narrative_id="bad",
|
||||
title="Bad demo",
|
||||
summary="missing honesty cards",
|
||||
steps=[
|
||||
DemoNarrativeStep(
|
||||
step_id="s1",
|
||||
order=1,
|
||||
kind="evidence",
|
||||
title="Trace",
|
||||
claim="trace exists",
|
||||
what_this_proves="",
|
||||
what_this_does_not_prove="",
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert validate_demo_narrative(narrative) == [
|
||||
"step s1 is missing what_this_proves",
|
||||
"step s1 is missing what_this_does_not_prove",
|
||||
]
|
||||
|
||||
|
||||
def test_demo_narrative_requires_route_evidence_links() -> None:
|
||||
narrative = DemoNarrative(
|
||||
narrative_id="bad-route",
|
||||
title="Bad route",
|
||||
summary="bad evidence link",
|
||||
steps=[
|
||||
DemoNarrativeStep(
|
||||
step_id="s1",
|
||||
order=1,
|
||||
kind="evidence",
|
||||
title="Trace",
|
||||
claim="trace exists",
|
||||
what_this_proves="a trace route can be opened",
|
||||
what_this_does_not_prove="that the answer is correct",
|
||||
evidence_links=[DemoEvidenceLink(label="bad", route="trace/1")],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert validate_demo_narrative(narrative) == [
|
||||
"step s1 has non-route evidence link: trace/1"
|
||||
]
|
||||
|
||||
|
||||
def test_demo_narrative_detects_duplicate_steps_and_order() -> None:
|
||||
narrative = DemoNarrative(
|
||||
narrative_id="bad-order",
|
||||
title="Bad order",
|
||||
summary="duplicate ids and unsorted order",
|
||||
steps=[
|
||||
DemoNarrativeStep(
|
||||
step_id="s1",
|
||||
order=2,
|
||||
kind="evidence",
|
||||
title="Trace B",
|
||||
claim="claim",
|
||||
what_this_proves="proof",
|
||||
what_this_does_not_prove="non-proof",
|
||||
),
|
||||
DemoNarrativeStep(
|
||||
step_id="s1",
|
||||
order=1,
|
||||
kind="audit",
|
||||
title="Audit A",
|
||||
claim="claim",
|
||||
what_this_proves="proof",
|
||||
what_this_does_not_prove="non-proof",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
assert validate_demo_narrative(narrative) == [
|
||||
"duplicate step_id: s1",
|
||||
"steps must be sorted by order",
|
||||
]
|
||||
|
||||
|
||||
def test_valid_demo_narrative_serializes() -> None:
|
||||
narrative = DemoNarrative(
|
||||
narrative_id="deterministic-turn",
|
||||
title="Deterministic Turn Evidence",
|
||||
summary="Chat to Trace to Replay",
|
||||
steps=[
|
||||
DemoNarrativeStep(
|
||||
step_id="trace",
|
||||
order=1,
|
||||
kind="evidence",
|
||||
title="Open Trace",
|
||||
claim="The journaled turn has trace evidence.",
|
||||
what_this_proves="The selected turn has recorded evidence.",
|
||||
what_this_does_not_prove="It does not prove answer correctness by itself.",
|
||||
evidence_links=[
|
||||
DemoEvidenceLink(
|
||||
label="Trace",
|
||||
route="/trace/1",
|
||||
artifact_id="turn:1",
|
||||
digest="sha256:abc",
|
||||
reproducer="curl /trace/1",
|
||||
)
|
||||
],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert validate_demo_narrative(narrative) == []
|
||||
payload = to_data(narrative)
|
||||
assert payload["narrative_id"] == "deterministic-turn"
|
||||
assert payload["steps"][0]["evidence_links"][0]["route"] == "/trace/1"
|
||||
|
||||
|
||||
def test_partner_blurb_states_core_demo_thesis() -> None:
|
||||
blurb = demo_partner_blurb()
|
||||
|
||||
assert "frontier model can propose" in blurb
|
||||
assert "CORE can govern" in blurb
|
||||
assert "Workbench can prove" in blurb
|
||||
91
tests/test_workbench_generalization_evidence.py
Normal file
91
tests/test_workbench_generalization_evidence.py
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from workbench.generalization_evidence import (
|
||||
contains_forbidden_raw_item_keys,
|
||||
generalization_audit_banner,
|
||||
generalization_report_view_from_payload,
|
||||
)
|
||||
from workbench.schemas import to_data
|
||||
|
||||
|
||||
def test_generalization_report_view_is_aggregate_only() -> None:
|
||||
view = generalization_report_view_from_payload(
|
||||
{
|
||||
"policy_version": "generalization_benchmark_policy_v1",
|
||||
"dataset": "gsm1k",
|
||||
"split": "local_audit",
|
||||
"n_items": 10,
|
||||
"correct": 4,
|
||||
"wrong": 0,
|
||||
"refused": 6,
|
||||
"unsupported": 0,
|
||||
"candidate_attempts": 3,
|
||||
"binding_failures": 2,
|
||||
"replay_refusals": 1,
|
||||
"sealed_trace_dispositions": {"sealed": 7, "refused": 3},
|
||||
"dominant_residual_kinds": {"missing_binding": 5},
|
||||
"reason_codes": ["ok", "binding_missing"],
|
||||
},
|
||||
source_path="evals/generalization/reports/gsm1k.json",
|
||||
source_digest="sha256:abc",
|
||||
report_kind="committed_pin",
|
||||
)
|
||||
|
||||
assert view.policy_version == "generalization_benchmark_policy_v1"
|
||||
assert view.dataset == "gsm1k"
|
||||
assert view.split == "local_audit"
|
||||
assert view.correct == 4
|
||||
assert view.wrong == 0
|
||||
assert view.refused == 6
|
||||
assert view.audit_only is True
|
||||
assert view.raw_items_exposed is False
|
||||
assert view.report_kind == "committed_pin"
|
||||
assert view.sealed_trace_dispositions == [("refused", 3), ("sealed", 7)]
|
||||
assert view.dominant_residual_kinds == [("missing_binding", 5)]
|
||||
|
||||
|
||||
def test_generalization_report_rejects_raw_item_fields() -> None:
|
||||
with pytest.raises(ValueError, match="raw item"):
|
||||
generalization_report_view_from_payload(
|
||||
{
|
||||
"dataset": "bad",
|
||||
"split": "sealed",
|
||||
"n_items": 1,
|
||||
"items": [{"question": "secret", "answer": "secret"}],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_contains_forbidden_raw_item_keys_recurses() -> None:
|
||||
assert contains_forbidden_raw_item_keys({"nested": {"gold_answer": "42"}}) is True
|
||||
assert contains_forbidden_raw_item_keys({"aggregate": {"correct": 1}}) is False
|
||||
|
||||
|
||||
def test_generalization_report_serializes_as_safe_dataclass() -> None:
|
||||
payload = to_data(
|
||||
generalization_report_view_from_payload(
|
||||
{
|
||||
"dataset": "asdiv",
|
||||
"split": "public",
|
||||
"n_items": 0,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
assert payload["dataset"] == "asdiv"
|
||||
assert payload["split"] == "public"
|
||||
assert payload["audit_only"] is True
|
||||
assert payload["raw_items_exposed"] is False
|
||||
assert "items" not in payload
|
||||
assert "question" not in payload
|
||||
assert "answer" not in payload
|
||||
|
||||
|
||||
def test_generalization_audit_banner_states_governance_boundary() -> None:
|
||||
banner = generalization_audit_banner()
|
||||
|
||||
assert "Audit-only" in banner
|
||||
assert "No raw sealed items" in banner
|
||||
assert "not direct mutation targets" in banner
|
||||
88
tests/test_workbench_proposal_artifact.py
Normal file
88
tests/test_workbench_proposal_artifact.py
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from workbench.proposal_artifact import (
|
||||
capability_disclosure,
|
||||
proposal_artifact_from_minimal,
|
||||
ratification_affordance_allowed,
|
||||
)
|
||||
from workbench.schemas import to_data
|
||||
|
||||
|
||||
def test_inspect_only_artifact_disallows_ratification_affordance() -> None:
|
||||
artifact = proposal_artifact_from_minimal(
|
||||
proposal_id="p1",
|
||||
subject_kind="construction",
|
||||
subject_id="turn:1",
|
||||
display_name="Construction proposal preview",
|
||||
source_kind="construction_evidence",
|
||||
)
|
||||
|
||||
assert artifact.capability_level == "inspect_only"
|
||||
assert ratification_affordance_allowed(artifact) is False
|
||||
assert "Inspect-only" in capability_disclosure(artifact)
|
||||
|
||||
|
||||
def test_proposal_only_artifact_disallows_ratification_affordance() -> None:
|
||||
artifact = proposal_artifact_from_minimal(
|
||||
proposal_id="p2",
|
||||
subject_kind="logos_pack",
|
||||
subject_id="logos:demo",
|
||||
display_name="CORE-Logos draft",
|
||||
source_kind="logos_pack",
|
||||
capability_level="proposal_only",
|
||||
)
|
||||
|
||||
assert ratification_affordance_allowed(artifact) is False
|
||||
assert "Proposal-only" in capability_disclosure(artifact)
|
||||
|
||||
|
||||
def test_ratification_enabled_requires_handler_route_for_affordance() -> None:
|
||||
artifact = proposal_artifact_from_minimal(
|
||||
proposal_id="p3",
|
||||
subject_kind="math",
|
||||
subject_id="math:p3",
|
||||
display_name="Math proposal",
|
||||
source_kind="math",
|
||||
capability_level="ratification_enabled",
|
||||
handler_route="/math-proposals/p3/ratify",
|
||||
)
|
||||
|
||||
assert ratification_affordance_allowed(artifact) is True
|
||||
assert "admitted handler" in capability_disclosure(artifact)
|
||||
|
||||
|
||||
def test_handler_route_without_ratification_capability_is_rejected() -> None:
|
||||
with pytest.raises(ValueError, match="handler_route"):
|
||||
proposal_artifact_from_minimal(
|
||||
proposal_id="p4",
|
||||
subject_kind="construction",
|
||||
subject_id="turn:4",
|
||||
display_name="Invalid preview",
|
||||
source_kind="construction_evidence",
|
||||
capability_level="inspect_only",
|
||||
handler_route="/invalid",
|
||||
)
|
||||
|
||||
|
||||
def test_proposal_artifact_serializes_as_dataclass_payload() -> None:
|
||||
payload = to_data(
|
||||
proposal_artifact_from_minimal(
|
||||
proposal_id="p5",
|
||||
subject_kind="cognition",
|
||||
subject_id="candidate:p5",
|
||||
display_name="Cognition proposal",
|
||||
source_kind="teaching_proposal_log",
|
||||
state="pending",
|
||||
)
|
||||
)
|
||||
|
||||
assert payload["proposal_id"] == "p5"
|
||||
assert payload["subject"] == {
|
||||
"kind": "cognition",
|
||||
"subject_id": "candidate:p5",
|
||||
"display_name": "Cognition proposal",
|
||||
}
|
||||
assert payload["capability_level"] == "inspect_only"
|
||||
assert payload["handler_route"] is None
|
||||
92
workbench-ui/src/app/demos/demoNarrativeView.test.ts
Normal file
92
workbench-ui/src/app/demos/demoNarrativeView.test.ts
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import type { DemoNarrative } from "../../types/demoNarrative";
|
||||
import {
|
||||
DEMO_PARTNER_BLURB,
|
||||
demoNarrativeValidationWarnings,
|
||||
demoStepHonestyPair,
|
||||
} from "./demoNarrativeView";
|
||||
|
||||
const narrative: DemoNarrative = {
|
||||
narrative_id: "deterministic-turn",
|
||||
title: "Deterministic Turn Evidence",
|
||||
summary: "Chat to Trace to Replay",
|
||||
steps: [
|
||||
{
|
||||
step_id: "trace",
|
||||
order: 1,
|
||||
kind: "evidence",
|
||||
title: "Open Trace",
|
||||
claim: "The journaled turn has trace evidence.",
|
||||
what_this_proves: "The selected turn has recorded evidence.",
|
||||
what_this_does_not_prove: "It does not prove answer correctness by itself.",
|
||||
evidence_links: [
|
||||
{
|
||||
label: "Trace",
|
||||
route: "/trace/1",
|
||||
artifact_id: "turn:1",
|
||||
digest: "sha256:abc",
|
||||
reproducer: "curl /trace/1",
|
||||
},
|
||||
],
|
||||
failure_mode: null,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
describe("demo narrative view helpers", () => {
|
||||
it("states the partner demo thesis", () => {
|
||||
expect(DEMO_PARTNER_BLURB).toContain("frontier model can propose");
|
||||
expect(DEMO_PARTNER_BLURB).toContain("CORE can govern");
|
||||
expect(DEMO_PARTNER_BLURB).toContain("Workbench can prove");
|
||||
});
|
||||
|
||||
it("accepts valid narratives", () => {
|
||||
expect(demoNarrativeValidationWarnings(narrative)).toEqual([]);
|
||||
});
|
||||
|
||||
it("requires both proof and non-proof claims", () => {
|
||||
expect(
|
||||
demoNarrativeValidationWarnings({
|
||||
...narrative,
|
||||
steps: [
|
||||
{
|
||||
...narrative.steps[0],
|
||||
what_this_proves: "",
|
||||
what_this_does_not_prove: "",
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toEqual([
|
||||
"step trace is missing what_this_proves",
|
||||
"step trace is missing what_this_does_not_prove",
|
||||
]);
|
||||
});
|
||||
|
||||
it("requires route-shaped evidence links", () => {
|
||||
expect(
|
||||
demoNarrativeValidationWarnings({
|
||||
...narrative,
|
||||
steps: [
|
||||
{
|
||||
...narrative.steps[0],
|
||||
evidence_links: [
|
||||
{
|
||||
label: "bad",
|
||||
route: "trace/1",
|
||||
artifact_id: null,
|
||||
digest: null,
|
||||
reproducer: null,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toEqual(["step trace has non-route evidence link: trace/1"]);
|
||||
});
|
||||
|
||||
it("renders honesty pair", () => {
|
||||
expect(demoStepHonestyPair(narrative.steps[0])).toBe(
|
||||
"Proves: The selected turn has recorded evidence. / Does not prove: It does not prove answer correctness by itself.",
|
||||
);
|
||||
});
|
||||
});
|
||||
38
workbench-ui/src/app/demos/demoNarrativeView.ts
Normal file
38
workbench-ui/src/app/demos/demoNarrativeView.ts
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
import type { DemoNarrative } from "../../types/demoNarrative";
|
||||
|
||||
export const DEMO_PARTNER_BLURB =
|
||||
"A frontier model can propose. CORE can govern. Workbench can prove what happened, what was refused, what replayed, and what remains only a proposal.";
|
||||
|
||||
export function demoNarrativeValidationWarnings(narrative: DemoNarrative): string[] {
|
||||
const warnings: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
const orders = narrative.steps.map((step) => step.order);
|
||||
|
||||
for (const step of narrative.steps) {
|
||||
if (seen.has(step.step_id)) {
|
||||
warnings.push(`duplicate step_id: ${step.step_id}`);
|
||||
}
|
||||
seen.add(step.step_id);
|
||||
if (step.what_this_proves.trim().length === 0) {
|
||||
warnings.push(`step ${step.step_id} is missing what_this_proves`);
|
||||
}
|
||||
if (step.what_this_does_not_prove.trim().length === 0) {
|
||||
warnings.push(`step ${step.step_id} is missing what_this_does_not_prove`);
|
||||
}
|
||||
for (const link of step.evidence_links) {
|
||||
if (!link.route.startsWith("/")) {
|
||||
warnings.push(`step ${step.step_id} has non-route evidence link: ${link.route}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const sorted = [...orders].sort((a, b) => a - b);
|
||||
if (orders.some((order, index) => order !== sorted[index])) {
|
||||
warnings.push("steps must be sorted by order");
|
||||
}
|
||||
return warnings;
|
||||
}
|
||||
|
||||
export function demoStepHonestyPair(step: DemoNarrative["steps"][number]): string {
|
||||
return `Proves: ${step.what_this_proves} / Does not prove: ${step.what_this_does_not_prove}`;
|
||||
}
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import type { GeneralizationAuditReportView } from "../../types/generalizationEvidence";
|
||||
import {
|
||||
GENERALIZATION_AUDIT_BANNER,
|
||||
generalizationGovernanceWarnings,
|
||||
generalizationOutcomeSummary,
|
||||
reportKindLabel,
|
||||
} from "./generalizationEvidenceView";
|
||||
|
||||
const baseReport: GeneralizationAuditReportView = {
|
||||
policy_version: "generalization_benchmark_policy_v1",
|
||||
dataset: "gsm1k",
|
||||
split: "local_audit",
|
||||
n_items: 10,
|
||||
correct: 4,
|
||||
wrong: 0,
|
||||
refused: 6,
|
||||
unsupported: 0,
|
||||
candidate_attempts: 3,
|
||||
binding_failures: 2,
|
||||
replay_refusals: 1,
|
||||
sealed_trace_dispositions: [["sealed", 7]],
|
||||
dominant_residual_kinds: [["missing_binding", 5]],
|
||||
reason_codes: ["ok"],
|
||||
source_path: "evals/generalization/reports/gsm1k.json",
|
||||
source_digest: "sha256:abc",
|
||||
report_kind: "committed_pin",
|
||||
audit_only: true,
|
||||
raw_items_exposed: false,
|
||||
};
|
||||
|
||||
describe("generalization evidence view helpers", () => {
|
||||
it("states the benchmark governance boundary", () => {
|
||||
expect(GENERALIZATION_AUDIT_BANNER).toContain("Audit-only");
|
||||
expect(GENERALIZATION_AUDIT_BANNER).toContain("No raw sealed items");
|
||||
expect(GENERALIZATION_AUDIT_BANNER).toContain("not direct mutation targets");
|
||||
});
|
||||
|
||||
it("labels report authority", () => {
|
||||
expect(reportKindLabel(baseReport)).toBe("Committed report pin");
|
||||
expect(reportKindLabel({ ...baseReport, report_kind: "ephemeral_local" })).toBe(
|
||||
"Ephemeral local output",
|
||||
);
|
||||
expect(reportKindLabel({ ...baseReport, report_kind: "rebaseline_candidate" })).toBe(
|
||||
"Governed rebaseline candidate",
|
||||
);
|
||||
expect(reportKindLabel({ ...baseReport, report_kind: "unknown" })).toBe(
|
||||
"Unknown report authority",
|
||||
);
|
||||
});
|
||||
|
||||
it("summarizes outcomes without hiding wrong count", () => {
|
||||
expect(generalizationOutcomeSummary(baseReport)).toBe(
|
||||
"4 correct / 0 wrong / 6 refused / 0 unsupported",
|
||||
);
|
||||
expect(generalizationOutcomeSummary({ ...baseReport, wrong: 2 })).toContain("2 wrong");
|
||||
});
|
||||
|
||||
it("warns when report should not be treated as canonical", () => {
|
||||
expect(generalizationGovernanceWarnings(baseReport)).toEqual([]);
|
||||
expect(
|
||||
generalizationGovernanceWarnings({
|
||||
...baseReport,
|
||||
report_kind: "ephemeral_local",
|
||||
audit_only: false,
|
||||
raw_items_exposed: true,
|
||||
}),
|
||||
).toEqual([
|
||||
"Report is not marked audit-only.",
|
||||
"Report claims raw items are exposed; do not render item payloads.",
|
||||
"Report is not a committed pin; do not present as canonical baseline.",
|
||||
]);
|
||||
});
|
||||
});
|
||||
36
workbench-ui/src/app/evals/generalizationEvidenceView.ts
Normal file
36
workbench-ui/src/app/evals/generalizationEvidenceView.ts
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
import type { GeneralizationAuditReportView } from "../../types/generalizationEvidence";
|
||||
|
||||
export const GENERALIZATION_AUDIT_BANNER =
|
||||
"Audit-only. No raw sealed items are exposed here. Benchmark failures are diagnosis signals, not direct mutation targets.";
|
||||
|
||||
export function reportKindLabel(report: GeneralizationAuditReportView): string {
|
||||
switch (report.report_kind) {
|
||||
case "committed_pin":
|
||||
return "Committed report pin";
|
||||
case "ephemeral_local":
|
||||
return "Ephemeral local output";
|
||||
case "rebaseline_candidate":
|
||||
return "Governed rebaseline candidate";
|
||||
case "unknown":
|
||||
default:
|
||||
return "Unknown report authority";
|
||||
}
|
||||
}
|
||||
|
||||
export function generalizationOutcomeSummary(report: GeneralizationAuditReportView): string {
|
||||
return `${report.correct} correct / ${report.wrong} wrong / ${report.refused} refused / ${report.unsupported} unsupported`;
|
||||
}
|
||||
|
||||
export function generalizationGovernanceWarnings(report: GeneralizationAuditReportView): string[] {
|
||||
const warnings: string[] = [];
|
||||
if (!report.audit_only) {
|
||||
warnings.push("Report is not marked audit-only.");
|
||||
}
|
||||
if (report.raw_items_exposed) {
|
||||
warnings.push("Report claims raw items are exposed; do not render item payloads.");
|
||||
}
|
||||
if (report.report_kind !== "committed_pin") {
|
||||
warnings.push("Report is not a committed pin; do not present as canonical baseline.");
|
||||
}
|
||||
return warnings;
|
||||
}
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import type { ProposalArtifact } from "../../types/proposalArtifact";
|
||||
import {
|
||||
proposalArtifactWarnings,
|
||||
proposalCapabilityLabel,
|
||||
ratificationControlsAllowed,
|
||||
} from "./proposalArtifactView";
|
||||
|
||||
const inspectOnly: ProposalArtifact = {
|
||||
proposal_id: "p1",
|
||||
subject: { kind: "construction", subject_id: "turn:1", display_name: "Construction" },
|
||||
state: "unknown",
|
||||
capability_level: "inspect_only",
|
||||
source_kind: "construction_evidence",
|
||||
proposed_change: null,
|
||||
reasoning_trace: null,
|
||||
evidence_pointers: [],
|
||||
validation: null,
|
||||
replay_evidence: null,
|
||||
safety_report: null,
|
||||
affected_artifacts: [],
|
||||
handler_route: null,
|
||||
suggested_cli: null,
|
||||
audit_refs: [],
|
||||
ui_disclosure: "inspect only",
|
||||
};
|
||||
|
||||
describe("proposal artifact view helpers", () => {
|
||||
it("disallows ratification controls for inspect-only artifacts", () => {
|
||||
expect(ratificationControlsAllowed(inspectOnly)).toBe(false);
|
||||
expect(proposalCapabilityLabel(inspectOnly)).toBe(
|
||||
"Inspect-only: no mutation affordance allowed",
|
||||
);
|
||||
});
|
||||
|
||||
it("disallows ratification controls for proposal-only artifacts", () => {
|
||||
const artifact = { ...inspectOnly, capability_level: "proposal_only" as const };
|
||||
|
||||
expect(ratificationControlsAllowed(artifact)).toBe(false);
|
||||
expect(proposalCapabilityLabel(artifact)).toBe(
|
||||
"Proposal-only: review/export/copy allowed; apply disabled",
|
||||
);
|
||||
});
|
||||
|
||||
it("allows ratification controls only with handler route", () => {
|
||||
const artifact = {
|
||||
...inspectOnly,
|
||||
capability_level: "ratification_enabled" as const,
|
||||
handler_route: "/math-proposals/p1/ratify",
|
||||
};
|
||||
|
||||
expect(ratificationControlsAllowed(artifact)).toBe(true);
|
||||
expect(proposalCapabilityLabel(artifact)).toBe(
|
||||
"Ratification enabled by admitted handler",
|
||||
);
|
||||
});
|
||||
|
||||
it("warns on impossible handler/capability combinations", () => {
|
||||
expect(
|
||||
proposalArtifactWarnings({
|
||||
...inspectOnly,
|
||||
handler_route: "/invalid",
|
||||
}),
|
||||
).toEqual(["Handler route must not be present without ratification authority."]);
|
||||
|
||||
expect(
|
||||
proposalArtifactWarnings({
|
||||
...inspectOnly,
|
||||
capability_level: "ratification_enabled",
|
||||
}),
|
||||
).toEqual(["Ratification authority declared without handler route."]);
|
||||
});
|
||||
|
||||
it("surfaces validation and safety blockers", () => {
|
||||
expect(
|
||||
proposalArtifactWarnings({
|
||||
...inspectOnly,
|
||||
validation: { status: "blocked", blockers: ["missing evidence"], warnings: [] },
|
||||
safety_report: { status: "failed", disclosures: ["handler not admitted"] },
|
||||
}),
|
||||
).toEqual(["missing evidence", "handler not admitted"]);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
import type { ProposalArtifact } from "../../types/proposalArtifact";
|
||||
|
||||
export function ratificationControlsAllowed(artifact: ProposalArtifact): boolean {
|
||||
return artifact.capability_level === "ratification_enabled" && artifact.handler_route !== null;
|
||||
}
|
||||
|
||||
export function proposalCapabilityLabel(artifact: ProposalArtifact): string {
|
||||
switch (artifact.capability_level) {
|
||||
case "ratification_enabled":
|
||||
return artifact.handler_route === null
|
||||
? "Ratification disabled: no handler route"
|
||||
: "Ratification enabled by admitted handler";
|
||||
case "proposal_only":
|
||||
return "Proposal-only: review/export/copy allowed; apply disabled";
|
||||
case "inspect_only":
|
||||
default:
|
||||
return "Inspect-only: no mutation affordance allowed";
|
||||
}
|
||||
}
|
||||
|
||||
export function proposalArtifactWarnings(artifact: ProposalArtifact): string[] {
|
||||
const warnings: string[] = [];
|
||||
if (artifact.capability_level !== "ratification_enabled" && artifact.handler_route !== null) {
|
||||
warnings.push("Handler route must not be present without ratification authority.");
|
||||
}
|
||||
if (artifact.capability_level === "ratification_enabled" && artifact.handler_route === null) {
|
||||
warnings.push("Ratification authority declared without handler route.");
|
||||
}
|
||||
if (artifact.validation?.status === "blocked") {
|
||||
warnings.push(...artifact.validation.blockers);
|
||||
}
|
||||
if (artifact.safety_report?.status === "failed") {
|
||||
warnings.push(...artifact.safety_report.disclosures);
|
||||
}
|
||||
return warnings;
|
||||
}
|
||||
82
workbench-ui/src/app/trace/constructionEvidenceView.test.ts
Normal file
82
workbench-ui/src/app/trace/constructionEvidenceView.test.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import type { ConstructionEvidence, ContractAssessmentView } from "../../types/constructionEvidence";
|
||||
import {
|
||||
CONSTRUCTION_AUTHORITY_DISCLOSURES,
|
||||
assessmentBlockerSummary,
|
||||
assessmentDisposition,
|
||||
constructionEvidenceEmptyMessage,
|
||||
sourceSpanIsExact,
|
||||
sourceSpanLabel,
|
||||
} from "./constructionEvidenceView";
|
||||
|
||||
const missingEvidence: ConstructionEvidence = {
|
||||
schema_version: "construction_evidence_v1",
|
||||
turn_id: 1,
|
||||
status: "missing_evidence",
|
||||
missing_reason: "construction evidence was not persisted for this turn",
|
||||
problem_text: null,
|
||||
proposals: [],
|
||||
mentions: [],
|
||||
bindings: [],
|
||||
bound_relations: [],
|
||||
contract_assessments: [],
|
||||
diagnostic_only: true,
|
||||
serving_allowed: false,
|
||||
};
|
||||
|
||||
const blockedAssessment: ContractAssessmentView = {
|
||||
candidate_organ: "quantity_entity_binding_candidate.v1",
|
||||
family_id: "binding.quantity_entity",
|
||||
missing_bindings: ["entity"],
|
||||
unresolved_hazards: ["ambiguous_quantity"],
|
||||
runnable: false,
|
||||
explanation: "missing entity binding",
|
||||
evidence_spans: [],
|
||||
};
|
||||
|
||||
describe("construction evidence view helpers", () => {
|
||||
it("keeps load-bearing authority disclosures explicit", () => {
|
||||
expect(CONSTRUCTION_AUTHORITY_DISCLOSURES).toEqual([
|
||||
"Proposal != Runnable",
|
||||
"Contract Determines",
|
||||
"Diagnostic Only",
|
||||
"Serving Disallowed",
|
||||
"Exact Span Required",
|
||||
]);
|
||||
});
|
||||
|
||||
it("renders missing evidence as honest absence", () => {
|
||||
expect(constructionEvidenceEmptyMessage(missingEvidence)).toBe(
|
||||
"construction evidence was not persisted for this turn",
|
||||
);
|
||||
});
|
||||
|
||||
it("labels source spans without normalization", () => {
|
||||
expect(sourceSpanLabel({ start: 9, end: 10, text: "3" })).toBe("9:10 3");
|
||||
});
|
||||
|
||||
it("checks exact source spans against problem text", () => {
|
||||
const text = "Lena has 3 red marbles.";
|
||||
|
||||
expect(sourceSpanIsExact(text, { start: 9, end: 10, text: "3" })).toBe(true);
|
||||
expect(sourceSpanIsExact(text, { start: 9, end: 10, text: "three" })).toBe(false);
|
||||
expect(sourceSpanIsExact(null, { start: 9, end: 10, text: "3" })).toBe(false);
|
||||
});
|
||||
|
||||
it("classifies assessments as blocked when bindings or hazards remain", () => {
|
||||
expect(assessmentDisposition(blockedAssessment)).toBe("blocked");
|
||||
expect(assessmentBlockerSummary(blockedAssessment)).toBe("entity, ambiguous_quantity");
|
||||
});
|
||||
|
||||
it("classifies assessments as runnable only with no blockers", () => {
|
||||
const runnable: ContractAssessmentView = {
|
||||
...blockedAssessment,
|
||||
missing_bindings: [],
|
||||
unresolved_hazards: [],
|
||||
runnable: true,
|
||||
};
|
||||
|
||||
expect(assessmentDisposition(runnable)).toBe("runnable");
|
||||
expect(assessmentBlockerSummary(runnable)).toBe("No blockers recorded.");
|
||||
});
|
||||
});
|
||||
56
workbench-ui/src/app/trace/constructionEvidenceView.ts
Normal file
56
workbench-ui/src/app/trace/constructionEvidenceView.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
import type {
|
||||
ConstructionEvidence,
|
||||
ContractAssessmentView,
|
||||
SourceSpanView,
|
||||
} from "../../types/constructionEvidence";
|
||||
|
||||
export const CONSTRUCTION_AUTHORITY_DISCLOSURES = [
|
||||
"Proposal != Runnable",
|
||||
"Contract Determines",
|
||||
"Diagnostic Only",
|
||||
"Serving Disallowed",
|
||||
"Exact Span Required",
|
||||
] as const;
|
||||
|
||||
export function constructionEvidenceEmptyMessage(evidence: ConstructionEvidence): string | null {
|
||||
if (evidence.status === "missing_evidence") {
|
||||
return evidence.missing_reason ?? "No construction evidence recorded for this turn.";
|
||||
}
|
||||
const hasEvidence =
|
||||
evidence.proposals.length > 0 ||
|
||||
evidence.mentions.length > 0 ||
|
||||
evidence.bindings.length > 0 ||
|
||||
evidence.bound_relations.length > 0 ||
|
||||
evidence.contract_assessments.length > 0;
|
||||
return hasEvidence ? null : "No construction evidence recorded for this turn.";
|
||||
}
|
||||
|
||||
export function sourceSpanLabel(span: SourceSpanView): string {
|
||||
return `${span.start}:${span.end} ${span.text}`;
|
||||
}
|
||||
|
||||
export function sourceSpanIsExact(problemText: string | null, span: SourceSpanView): boolean {
|
||||
if (problemText === null) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
span.start >= 0 &&
|
||||
span.end >= span.start &&
|
||||
span.end <= problemText.length &&
|
||||
span.text.length > 0 &&
|
||||
problemText.slice(span.start, span.end) === span.text
|
||||
);
|
||||
}
|
||||
|
||||
export function assessmentDisposition(assessment: ContractAssessmentView): "runnable" | "blocked" {
|
||||
return assessment.runnable &&
|
||||
assessment.missing_bindings.length === 0 &&
|
||||
assessment.unresolved_hazards.length === 0
|
||||
? "runnable"
|
||||
: "blocked";
|
||||
}
|
||||
|
||||
export function assessmentBlockerSummary(assessment: ContractAssessmentView): string {
|
||||
const blockers = [...assessment.missing_bindings, ...assessment.unresolved_hazards];
|
||||
return blockers.length > 0 ? blockers.join(", ") : "No blockers recorded.";
|
||||
}
|
||||
76
workbench-ui/src/types/constructionEvidence.ts
Normal file
76
workbench-ui/src/types/constructionEvidence.ts
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
export type ConstructionEvidenceStatus = "recorded" | "missing_evidence";
|
||||
|
||||
export interface SourceSpanView {
|
||||
start: number;
|
||||
end: number;
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface RoleObligationView {
|
||||
role: string;
|
||||
required: boolean;
|
||||
description: string;
|
||||
}
|
||||
|
||||
export interface ConstructionProposalView {
|
||||
family_id: string;
|
||||
relation_type: string;
|
||||
candidate_organ: string;
|
||||
status: "proposed";
|
||||
evidence_spans: SourceSpanView[];
|
||||
role_obligations: RoleObligationView[];
|
||||
diagnostic_only: boolean;
|
||||
serving_allowed: boolean;
|
||||
}
|
||||
|
||||
export interface MentionView {
|
||||
mention_id: string;
|
||||
kind: string;
|
||||
surface: string;
|
||||
span: SourceSpanView;
|
||||
fact_id: string | null;
|
||||
}
|
||||
|
||||
export interface MentionBindingView {
|
||||
binding_type: string;
|
||||
source_mention_id: string;
|
||||
target_mention_id: string;
|
||||
evidence_spans: SourceSpanView[];
|
||||
}
|
||||
|
||||
export interface BoundRelationRoleView {
|
||||
role: string;
|
||||
target_id: string;
|
||||
evidence_spans: SourceSpanView[];
|
||||
}
|
||||
|
||||
export interface BoundRelationView {
|
||||
relation_type: string;
|
||||
roles: BoundRelationRoleView[];
|
||||
evidence_spans: SourceSpanView[];
|
||||
}
|
||||
|
||||
export interface ContractAssessmentView {
|
||||
candidate_organ: string;
|
||||
family_id: string | null;
|
||||
missing_bindings: string[];
|
||||
unresolved_hazards: string[];
|
||||
runnable: boolean;
|
||||
explanation: string;
|
||||
evidence_spans: SourceSpanView[];
|
||||
}
|
||||
|
||||
export interface ConstructionEvidence {
|
||||
schema_version: "construction_evidence_v1";
|
||||
turn_id: number;
|
||||
status: ConstructionEvidenceStatus;
|
||||
missing_reason: string | null;
|
||||
problem_text: string | null;
|
||||
proposals: ConstructionProposalView[];
|
||||
mentions: MentionView[];
|
||||
bindings: MentionBindingView[];
|
||||
bound_relations: BoundRelationView[];
|
||||
contract_assessments: ContractAssessmentView[];
|
||||
diagnostic_only: boolean;
|
||||
serving_allowed: boolean;
|
||||
}
|
||||
28
workbench-ui/src/types/demoNarrative.ts
Normal file
28
workbench-ui/src/types/demoNarrative.ts
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
export type DemoStepKind = "intro" | "evidence" | "replay" | "proposal" | "audit" | "payoff";
|
||||
|
||||
export interface DemoEvidenceLink {
|
||||
label: string;
|
||||
route: string;
|
||||
artifact_id: string | null;
|
||||
digest: string | null;
|
||||
reproducer: string | null;
|
||||
}
|
||||
|
||||
export interface DemoNarrativeStep {
|
||||
step_id: string;
|
||||
order: number;
|
||||
kind: DemoStepKind;
|
||||
title: string;
|
||||
claim: string;
|
||||
what_this_proves: string;
|
||||
what_this_does_not_prove: string;
|
||||
evidence_links: DemoEvidenceLink[];
|
||||
failure_mode: string | null;
|
||||
}
|
||||
|
||||
export interface DemoNarrative {
|
||||
narrative_id: string;
|
||||
title: string;
|
||||
summary: string;
|
||||
steps: DemoNarrativeStep[];
|
||||
}
|
||||
46
workbench-ui/src/types/generalizationEvidence.ts
Normal file
46
workbench-ui/src/types/generalizationEvidence.ts
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
export type ChecksumStatus = "verified" | "missing" | "mismatch" | "unknown";
|
||||
export type GeneralizationReportKind =
|
||||
| "committed_pin"
|
||||
| "ephemeral_local"
|
||||
| "rebaseline_candidate"
|
||||
| "unknown";
|
||||
|
||||
export interface GeneralizationManifestSummary {
|
||||
dataset: string;
|
||||
manifest_path: string;
|
||||
split_names: string[];
|
||||
license: string | null;
|
||||
checksum_status: ChecksumStatus;
|
||||
sealed_splits: string[];
|
||||
policy_version: string;
|
||||
}
|
||||
|
||||
export interface GeneralizationCacheStatus {
|
||||
dataset: string;
|
||||
cache_path: string;
|
||||
present: boolean;
|
||||
verified: boolean;
|
||||
reason: string | null;
|
||||
}
|
||||
|
||||
export interface GeneralizationAuditReportView {
|
||||
policy_version: string;
|
||||
dataset: string;
|
||||
split: string;
|
||||
n_items: number;
|
||||
correct: number;
|
||||
wrong: number;
|
||||
refused: number;
|
||||
unsupported: number;
|
||||
candidate_attempts: number;
|
||||
binding_failures: number;
|
||||
replay_refusals: number;
|
||||
sealed_trace_dispositions: [string, number][];
|
||||
dominant_residual_kinds: [string, number][];
|
||||
reason_codes: string[];
|
||||
source_path: string | null;
|
||||
source_digest: string | null;
|
||||
report_kind: GeneralizationReportKind;
|
||||
audit_only: boolean;
|
||||
raw_items_exposed: boolean;
|
||||
}
|
||||
55
workbench-ui/src/types/proposalArtifact.ts
Normal file
55
workbench-ui/src/types/proposalArtifact.ts
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
export type ProposalCapabilityLevel =
|
||||
| "inspect_only"
|
||||
| "proposal_only"
|
||||
| "ratification_enabled";
|
||||
|
||||
export type ProposalArtifactState =
|
||||
| "pending"
|
||||
| "accepted"
|
||||
| "rejected"
|
||||
| "withdrawn"
|
||||
| "deferred"
|
||||
| "unknown";
|
||||
|
||||
export interface ProposalSubject {
|
||||
kind: string;
|
||||
subject_id: string;
|
||||
display_name: string;
|
||||
}
|
||||
|
||||
export interface EvidencePointer {
|
||||
kind: string;
|
||||
ref: string;
|
||||
label: string;
|
||||
digest: string | null;
|
||||
}
|
||||
|
||||
export interface ProposalValidationReport {
|
||||
status: "valid" | "blocked" | "unknown";
|
||||
blockers: string[];
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
export interface ProposalSafetyReport {
|
||||
status: "clear" | "warning" | "failed" | "unknown";
|
||||
disclosures: string[];
|
||||
}
|
||||
|
||||
export interface ProposalArtifact {
|
||||
proposal_id: string;
|
||||
subject: ProposalSubject;
|
||||
state: ProposalArtifactState;
|
||||
capability_level: ProposalCapabilityLevel;
|
||||
source_kind: string;
|
||||
proposed_change: unknown;
|
||||
reasoning_trace: unknown;
|
||||
evidence_pointers: EvidencePointer[];
|
||||
validation: ProposalValidationReport | null;
|
||||
replay_evidence: unknown;
|
||||
safety_report: ProposalSafetyReport | null;
|
||||
affected_artifacts: EvidencePointer[];
|
||||
handler_route: string | null;
|
||||
suggested_cli: string | null;
|
||||
audit_refs: EvidencePointer[];
|
||||
ui_disclosure: string;
|
||||
}
|
||||
185
workbench/construction_evidence.py
Normal file
185
workbench/construction_evidence.py
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
"""Read-only construction evidence projection helpers for Workbench.
|
||||
|
||||
This module is intentionally inert: it defines the Workbench-facing construction
|
||||
read model and the honest missing-evidence constructor for legacy turns. It does
|
||||
not parse problem text, execute candidate operators, run replay, mutate journals,
|
||||
or grant serving authority.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Literal
|
||||
|
||||
PipelineEvidenceStatus = Literal["recorded", "missing_evidence"]
|
||||
CONSTRUCTION_EVIDENCE_ABSENT = "construction evidence was not persisted for this turn"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SourceSpanView:
|
||||
"""Exact source span projected to the Workbench.
|
||||
|
||||
The caller owns validation against the source string. This record must not be
|
||||
normalized or repaired by the UI.
|
||||
"""
|
||||
|
||||
start: int
|
||||
end: int
|
||||
text: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RoleObligationView:
|
||||
role: str
|
||||
required: bool
|
||||
description: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ConstructionProposalView:
|
||||
"""Diagnostic construction proposal.
|
||||
|
||||
A proposal is a hypothesis only. Runnable/refused authority belongs to the
|
||||
corresponding ContractAssessmentView.
|
||||
"""
|
||||
|
||||
family_id: str
|
||||
relation_type: str
|
||||
candidate_organ: str
|
||||
status: Literal["proposed"]
|
||||
evidence_spans: list[SourceSpanView] = field(default_factory=list)
|
||||
role_obligations: list[RoleObligationView] = field(default_factory=list)
|
||||
diagnostic_only: bool = True
|
||||
serving_allowed: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MentionView:
|
||||
mention_id: str
|
||||
kind: str
|
||||
surface: str
|
||||
span: SourceSpanView
|
||||
fact_id: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MentionBindingView:
|
||||
binding_type: str
|
||||
source_mention_id: str
|
||||
target_mention_id: str
|
||||
evidence_spans: list[SourceSpanView] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BoundRelationRoleView:
|
||||
role: str
|
||||
target_id: str
|
||||
evidence_spans: list[SourceSpanView] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BoundRelationView:
|
||||
relation_type: str
|
||||
roles: list[BoundRelationRoleView] = field(default_factory=list)
|
||||
evidence_spans: list[SourceSpanView] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ContractAssessmentView:
|
||||
"""Read-only projection of generate.problem_frame_contracts.ContractAssessment."""
|
||||
|
||||
candidate_organ: str
|
||||
family_id: str | None
|
||||
missing_bindings: list[str] = field(default_factory=list)
|
||||
unresolved_hazards: list[str] = field(default_factory=list)
|
||||
runnable: bool = False
|
||||
explanation: str = ""
|
||||
evidence_spans: list[SourceSpanView] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ConstructionEvidence:
|
||||
"""Workbench-facing construction evidence for one turn.
|
||||
|
||||
This record is a projection, not an executor. `diagnostic_only=True` and
|
||||
`serving_allowed=False` are load-bearing disclosure fields.
|
||||
"""
|
||||
|
||||
schema_version: Literal["construction_evidence_v1"]
|
||||
turn_id: int
|
||||
status: PipelineEvidenceStatus
|
||||
missing_reason: str | None
|
||||
problem_text: str | None
|
||||
proposals: list[ConstructionProposalView] = field(default_factory=list)
|
||||
mentions: list[MentionView] = field(default_factory=list)
|
||||
bindings: list[MentionBindingView] = field(default_factory=list)
|
||||
bound_relations: list[BoundRelationView] = field(default_factory=list)
|
||||
contract_assessments: list[ContractAssessmentView] = field(default_factory=list)
|
||||
diagnostic_only: bool = True
|
||||
serving_allowed: bool = False
|
||||
|
||||
|
||||
def missing_construction_evidence(turn_id: int, reason: str) -> ConstructionEvidence:
|
||||
"""Return the honest absence state for legacy turns.
|
||||
|
||||
Absence is not an error and not a failed proof. It means the selected turn has
|
||||
no persisted construction evidence to project.
|
||||
"""
|
||||
|
||||
return ConstructionEvidence(
|
||||
schema_version="construction_evidence_v1",
|
||||
turn_id=turn_id,
|
||||
status="missing_evidence",
|
||||
missing_reason=reason,
|
||||
problem_text=None,
|
||||
proposals=[],
|
||||
mentions=[],
|
||||
bindings=[],
|
||||
bound_relations=[],
|
||||
contract_assessments=[],
|
||||
diagnostic_only=True,
|
||||
serving_allowed=False,
|
||||
)
|
||||
|
||||
|
||||
def construction_evidence_from_journal_entry(entry: Any) -> ConstructionEvidence:
|
||||
"""Project persisted construction evidence from a journal entry if present.
|
||||
|
||||
Current turn journals do not yet persist a construction evidence payload. This
|
||||
helper is deliberately fail-closed and returns a typed missing-evidence record
|
||||
rather than reconstructing a ProblemFrame from prose. When a later PR starts
|
||||
persisting `construction_evidence`, this function is the narrow projection
|
||||
seam to extend.
|
||||
"""
|
||||
|
||||
turn_id = int(getattr(entry, "turn_id"))
|
||||
payload = getattr(entry, "construction_evidence", None)
|
||||
if payload is None:
|
||||
return missing_construction_evidence(turn_id, CONSTRUCTION_EVIDENCE_ABSENT)
|
||||
|
||||
if isinstance(payload, ConstructionEvidence):
|
||||
return payload
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
return missing_construction_evidence(
|
||||
turn_id,
|
||||
"construction evidence payload has unsupported shape",
|
||||
)
|
||||
|
||||
# Future-compatible but conservative: do not attempt lossy coercion. The first
|
||||
# persistence PR should replace this with explicit field-by-field projection and
|
||||
# exact-span validation tests.
|
||||
return missing_construction_evidence(
|
||||
turn_id,
|
||||
"construction evidence payload projection is not yet admitted",
|
||||
)
|
||||
|
||||
|
||||
def span_is_exact(problem_text: str, span: SourceSpanView) -> bool:
|
||||
"""Return whether a span exactly matches its source text slice."""
|
||||
|
||||
return (
|
||||
0 <= span.start <= span.end <= len(problem_text)
|
||||
and bool(span.text)
|
||||
and problem_text[span.start : span.end] == span.text
|
||||
)
|
||||
72
workbench/demo_narrative.py
Normal file
72
workbench/demo_narrative.py
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
"""Demo narrative scaffolding for Workbench proof theater.
|
||||
|
||||
A demo narrative is an authored path over real evidence routes. It must state both
|
||||
what a step proves and what it does not prove, so demo polish cannot outrun
|
||||
substrate evidence.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Literal
|
||||
|
||||
DemoStepKind = Literal["intro", "evidence", "replay", "proposal", "audit", "payoff"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DemoEvidenceLink:
|
||||
label: str
|
||||
route: str
|
||||
artifact_id: str | None = None
|
||||
digest: str | None = None
|
||||
reproducer: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DemoNarrativeStep:
|
||||
step_id: str
|
||||
order: int
|
||||
kind: DemoStepKind
|
||||
title: str
|
||||
claim: str
|
||||
what_this_proves: str
|
||||
what_this_does_not_prove: str
|
||||
evidence_links: list[DemoEvidenceLink] = field(default_factory=list)
|
||||
failure_mode: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DemoNarrative:
|
||||
narrative_id: str
|
||||
title: str
|
||||
summary: str
|
||||
steps: list[DemoNarrativeStep] = field(default_factory=list)
|
||||
|
||||
|
||||
def validate_demo_narrative(narrative: DemoNarrative) -> list[str]:
|
||||
"""Return validation blockers for a Workbench demo narrative."""
|
||||
|
||||
blockers: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for step in narrative.steps:
|
||||
if step.step_id in seen:
|
||||
blockers.append(f"duplicate step_id: {step.step_id}")
|
||||
seen.add(step.step_id)
|
||||
if not step.what_this_proves.strip():
|
||||
blockers.append(f"step {step.step_id} is missing what_this_proves")
|
||||
if not step.what_this_does_not_prove.strip():
|
||||
blockers.append(f"step {step.step_id} is missing what_this_does_not_prove")
|
||||
for link in step.evidence_links:
|
||||
if not link.route.startswith("/"):
|
||||
blockers.append(f"step {step.step_id} has non-route evidence link: {link.route}")
|
||||
orders = [step.order for step in narrative.steps]
|
||||
if orders != sorted(orders):
|
||||
blockers.append("steps must be sorted by order")
|
||||
return blockers
|
||||
|
||||
|
||||
def demo_partner_blurb() -> str:
|
||||
return (
|
||||
"A frontier model can propose. CORE can govern. Workbench can prove what "
|
||||
"happened, what was refused, what replayed, and what remains only a proposal."
|
||||
)
|
||||
144
workbench/generalization_evidence.py
Normal file
144
workbench/generalization_evidence.py
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
"""Read-only generalization audit evidence projections for Workbench.
|
||||
|
||||
Generalization benchmark data is audit/test-only. This module deliberately models
|
||||
aggregate report metadata only. It must not expose raw prompt/question/answer
|
||||
content, sealed item payloads, or direct patch suggestions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Literal
|
||||
|
||||
ChecksumStatus = Literal["verified", "missing", "mismatch", "unknown"]
|
||||
ReportKind = Literal["committed_pin", "ephemeral_local", "rebaseline_candidate", "unknown"]
|
||||
|
||||
FORBIDDEN_RAW_ITEM_KEYS = frozenset(
|
||||
{
|
||||
"prompt",
|
||||
"question",
|
||||
"answer",
|
||||
"gold_answer",
|
||||
"raw_item",
|
||||
"raw_items",
|
||||
"items",
|
||||
"examples",
|
||||
"sealed_items",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GeneralizationManifestSummary:
|
||||
dataset: str
|
||||
manifest_path: str
|
||||
split_names: list[str]
|
||||
license: str | None
|
||||
checksum_status: ChecksumStatus
|
||||
sealed_splits: list[str]
|
||||
policy_version: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GeneralizationCacheStatus:
|
||||
dataset: str
|
||||
cache_path: str
|
||||
present: bool
|
||||
verified: bool
|
||||
reason: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GeneralizationAuditReportView:
|
||||
policy_version: str
|
||||
dataset: str
|
||||
split: str
|
||||
n_items: int
|
||||
correct: int
|
||||
wrong: int
|
||||
refused: int
|
||||
unsupported: int
|
||||
candidate_attempts: int
|
||||
binding_failures: int
|
||||
replay_refusals: int
|
||||
sealed_trace_dispositions: list[tuple[str, int]] = field(default_factory=list)
|
||||
dominant_residual_kinds: list[tuple[str, int]] = field(default_factory=list)
|
||||
reason_codes: list[str] = field(default_factory=list)
|
||||
source_path: str | None = None
|
||||
source_digest: str | None = None
|
||||
report_kind: ReportKind = "unknown"
|
||||
audit_only: bool = True
|
||||
raw_items_exposed: bool = False
|
||||
|
||||
|
||||
def contains_forbidden_raw_item_keys(payload: Any) -> bool:
|
||||
"""Return True if a payload contains raw/sealed item keys anywhere."""
|
||||
|
||||
if isinstance(payload, dict):
|
||||
for key, value in payload.items():
|
||||
if str(key) in FORBIDDEN_RAW_ITEM_KEYS:
|
||||
return True
|
||||
if contains_forbidden_raw_item_keys(value):
|
||||
return True
|
||||
elif isinstance(payload, list):
|
||||
return any(contains_forbidden_raw_item_keys(item) for item in payload)
|
||||
return False
|
||||
|
||||
|
||||
def _count_pairs(value: Any) -> list[tuple[str, int]]:
|
||||
if not isinstance(value, dict):
|
||||
return []
|
||||
pairs: list[tuple[str, int]] = []
|
||||
for key, count in sorted(value.items(), key=lambda item: str(item[0])):
|
||||
try:
|
||||
pairs.append((str(key), int(count)))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return pairs
|
||||
|
||||
|
||||
def generalization_report_view_from_payload(
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
source_path: str | None = None,
|
||||
source_digest: str | None = None,
|
||||
report_kind: ReportKind = "unknown",
|
||||
) -> GeneralizationAuditReportView:
|
||||
"""Project an aggregate audit report into a Workbench-safe view.
|
||||
|
||||
This function is intentionally aggregate-only. It rejects payloads that carry
|
||||
obvious raw/sealed item fields so the Workbench cannot accidentally render
|
||||
benchmark examples or answers.
|
||||
"""
|
||||
|
||||
if contains_forbidden_raw_item_keys(payload):
|
||||
raise ValueError("generalization report payload exposes raw item fields")
|
||||
|
||||
return GeneralizationAuditReportView(
|
||||
policy_version=str(payload.get("policy_version") or "unknown"),
|
||||
dataset=str(payload.get("dataset") or "unknown"),
|
||||
split=str(payload.get("split") or "unknown"),
|
||||
n_items=int(payload.get("n_items") or 0),
|
||||
correct=int(payload.get("correct") or 0),
|
||||
wrong=int(payload.get("wrong") or 0),
|
||||
refused=int(payload.get("refused") or 0),
|
||||
unsupported=int(payload.get("unsupported") or 0),
|
||||
candidate_attempts=int(payload.get("candidate_attempts") or 0),
|
||||
binding_failures=int(payload.get("binding_failures") or 0),
|
||||
replay_refusals=int(payload.get("replay_refusals") or 0),
|
||||
sealed_trace_dispositions=_count_pairs(payload.get("sealed_trace_dispositions")),
|
||||
dominant_residual_kinds=_count_pairs(payload.get("dominant_residual_kinds")),
|
||||
reason_codes=[str(code) for code in payload.get("reason_codes") or []],
|
||||
source_path=source_path,
|
||||
source_digest=source_digest,
|
||||
report_kind=report_kind,
|
||||
audit_only=True,
|
||||
raw_items_exposed=False,
|
||||
)
|
||||
|
||||
|
||||
def generalization_audit_banner() -> str:
|
||||
return (
|
||||
"Audit-only. No raw sealed items are exposed here. Benchmark failures are "
|
||||
"diagnosis signals, not direct mutation targets."
|
||||
)
|
||||
122
workbench/proposal_artifact.py
Normal file
122
workbench/proposal_artifact.py
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
"""ProposalArtifact authority scaffolding for Workbench.
|
||||
|
||||
This module defines a shared proposal-review envelope without admitting any new
|
||||
ratification handler. It is safe, inert UI/read-model substrate: capability level
|
||||
controls what the Workbench may display as an affordance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Literal
|
||||
|
||||
ProposalCapabilityLevel = Literal[
|
||||
"inspect_only",
|
||||
"proposal_only",
|
||||
"ratification_enabled",
|
||||
]
|
||||
|
||||
ProposalArtifactState = Literal[
|
||||
"pending",
|
||||
"accepted",
|
||||
"rejected",
|
||||
"withdrawn",
|
||||
"deferred",
|
||||
"unknown",
|
||||
]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProposalSubject:
|
||||
kind: str
|
||||
subject_id: str
|
||||
display_name: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class EvidencePointer:
|
||||
kind: str
|
||||
ref: str
|
||||
label: str
|
||||
digest: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProposalValidationReport:
|
||||
status: Literal["valid", "blocked", "unknown"]
|
||||
blockers: list[str] = field(default_factory=list)
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProposalSafetyReport:
|
||||
status: Literal["clear", "warning", "failed", "unknown"]
|
||||
disclosures: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProposalArtifact:
|
||||
proposal_id: str
|
||||
subject: ProposalSubject
|
||||
state: ProposalArtifactState
|
||||
capability_level: ProposalCapabilityLevel
|
||||
source_kind: str
|
||||
proposed_change: Any = None
|
||||
reasoning_trace: Any = None
|
||||
evidence_pointers: list[EvidencePointer] = field(default_factory=list)
|
||||
validation: ProposalValidationReport | None = None
|
||||
replay_evidence: Any = None
|
||||
safety_report: ProposalSafetyReport | None = None
|
||||
affected_artifacts: list[EvidencePointer] = field(default_factory=list)
|
||||
handler_route: str | None = None
|
||||
suggested_cli: str | None = None
|
||||
audit_refs: list[EvidencePointer] = field(default_factory=list)
|
||||
ui_disclosure: str = "Proposal artifact is inspect-only unless an admitted handler exists."
|
||||
|
||||
|
||||
def ratification_affordance_allowed(artifact: ProposalArtifact) -> bool:
|
||||
"""Return whether UI may show ratify/reject/defer controls."""
|
||||
|
||||
return artifact.capability_level == "ratification_enabled" and bool(
|
||||
artifact.handler_route
|
||||
)
|
||||
|
||||
|
||||
def capability_disclosure(artifact: ProposalArtifact) -> str:
|
||||
if artifact.capability_level == "ratification_enabled":
|
||||
if artifact.handler_route:
|
||||
return "Ratification controls may render through the admitted handler route."
|
||||
return "Ratification is disabled because no handler route is present."
|
||||
if artifact.capability_level == "proposal_only":
|
||||
return "Proposal-only artifact: review/export/copy are allowed; apply is not."
|
||||
return "Inspect-only artifact: no apply, ratify, promote, or mutation affordance is allowed."
|
||||
|
||||
|
||||
def proposal_artifact_from_minimal(
|
||||
*,
|
||||
proposal_id: str,
|
||||
subject_kind: str,
|
||||
subject_id: str,
|
||||
display_name: str,
|
||||
source_kind: str,
|
||||
capability_level: ProposalCapabilityLevel = "inspect_only",
|
||||
state: ProposalArtifactState = "unknown",
|
||||
handler_route: str | None = None,
|
||||
) -> ProposalArtifact:
|
||||
"""Build a minimal shared proposal review envelope."""
|
||||
|
||||
artifact = ProposalArtifact(
|
||||
proposal_id=proposal_id,
|
||||
subject=ProposalSubject(
|
||||
kind=subject_kind,
|
||||
subject_id=subject_id,
|
||||
display_name=display_name,
|
||||
),
|
||||
state=state,
|
||||
capability_level=capability_level,
|
||||
source_kind=source_kind,
|
||||
handler_route=handler_route,
|
||||
)
|
||||
if capability_level != "ratification_enabled" and handler_route is not None:
|
||||
raise ValueError("handler_route requires ratification_enabled capability")
|
||||
return artifact
|
||||
Loading…
Reference in a new issue