294cfc3576
Ships `core demo audit-tour` as the first investor-facing
walkthrough of the ADR-0027→0041 pack-layer architecture. Four
scenes, each making one falsifiable claim no transformer-LLM
wrapper can reproduce:
S1. Identity is geometric, not prompt-veneer.
Three identity packs load three structurally distinct
manifolds (ADR-0027). Distinct alignment thresholds +
distinct hedge phrases from JSON pack files, not prompts.
S2. Safety is the universal floor.
Runtime-checkable safety violation produces a deterministic
typed refusal string (ADR-0036). walk_surface preserved
for audit. Byte-identical across runs.
S3. Ethics commitments choose their remediation.
Per-commitment opt-in (ADR-0037 / ADR-0038): pure-helper
evidence (should_inject_hedge + inject_hedge worked
example) against a synthetic violation. Default pack
returns False; deployment pack (with acknowledge_uncertainty
in hedge_commitments) returns True. Pack JSON drives the
policy tier.
S4. Deterministic replay across runtime instances.
Two fresh ChatRuntime instances, same input, same packs.
Byte-identical JSONL audit lines (ADR-0040).
Load-bearing evidence over surface inspection: the draft compared
response.surface across packs. Cold-start hits stub path; pack
differences don't manifest at the surface by design. Shipped
version pulls evidence from structural surfaces (manifold fields,
opt-in lists, pure helpers) — what actually distinguishes the
packs. No fake claims.
Scene 3 uses synthetic verdict (not chat()) because ADR-0038
specifies stub path skips hedge by design. Main-path end-to-end
is asserted in tests/test_hedge_injection.py and referenced in
the tour's evidence comment.
Test gate: tests/test_audit_tour.py asserts
result["all_claims_supported"] is True. Any scene flipping to
False fails the test and catches the regression.
CLI integration:
core demo audit-tour # narration to stdout
core demo audit-tour --json # structured report, no narration
Files:
- evals/audit_tour/__init__.py + run_tour.py (new) — 4-scene tour
- core/cli.py — audit-tour target on demo subcommand;
_AUDIT_TOUR_PREAMBLE; --json suppresses narration
- tests/test_audit_tour.py (new) — 8 tests gating all four claims
- docs/decisions/ADR-0042-audit-tour-demo.md (new) — decision record
- docs/decisions/README.md — ADR index now lists ADR-0027..0042
+ Pack-Layer chain section describing the three-tier composition,
remediation tiers, and verification surface
- docs/PROGRESS.md — adds core demo audit-tour to verify cheatsheet
- README.md — adds core demo audit-tour to commands cheatsheet
Verification:
- Combined pack-layer + telemetry + tour suite: 220 green
(was 212 after ADR-0041; +8)
- CLI suites unchanged: smoke 67, runtime 19, cognition 121
- core eval cognition: intent 100%, versor_closure 100% (baseline)
- Manual: core demo audit-tour and --json both correct;
all_claims_supported = true
93 lines
3.6 KiB
Python
93 lines
3.6 KiB
Python
"""Tests for the audit tour demo.
|
|
|
|
The tour ships as `core demo audit-tour` and is the primary
|
|
investor-facing artifact for the pack-layer architecture story.
|
|
These tests ensure the four claim flags stay green and that the
|
|
JSON mode emits a stable structured report.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import json
|
|
from contextlib import redirect_stdout
|
|
|
|
from evals.audit_tour.run_tour import run_tour
|
|
|
|
|
|
class TestRunTourStructuredOutput:
|
|
def test_all_claims_supported(self) -> None:
|
|
"""The headline gate — every scene's load-bearing claim must
|
|
pass. Any scene flipping to False represents a regression of
|
|
the pack-layer architecture story."""
|
|
with redirect_stdout(io.StringIO()):
|
|
result = run_tour(emit_json=True)
|
|
assert result["all_claims_supported"] is True
|
|
|
|
def test_scene_1_distinct_alignment_thresholds(self) -> None:
|
|
with redirect_stdout(io.StringIO()):
|
|
result = run_tour(emit_json=True)
|
|
s1 = result["scene_1_identity_geometric"]
|
|
# Three packs ship with three distinct thresholds.
|
|
assert s1["distinct_alignment_thresholds"] == 3
|
|
assert s1["distinct_hedge_phrases"] >= 2
|
|
|
|
def test_scene_2_typed_refusal(self) -> None:
|
|
with redirect_stdout(io.StringIO()):
|
|
result = run_tour(emit_json=True)
|
|
s2 = result["scene_2_safety_typed_refusal"]
|
|
assert s2["refusal_emitted"] is True
|
|
assert s2["refused_surface"].startswith("I cannot proceed")
|
|
# walk_surface is preserved unchanged.
|
|
assert s2["walk_surface"] != s2["refused_surface"]
|
|
|
|
def test_scene_3_pack_drives_remediation(self) -> None:
|
|
with redirect_stdout(io.StringIO()):
|
|
result = run_tour(emit_json=True)
|
|
s3 = result["scene_3_ethics_hedge_opt_in"]
|
|
assert s3["default_fires"] is False
|
|
assert s3["deployment_fires"] is True
|
|
assert s3["deployment_pack_hedge_commitments"] == ["acknowledge_uncertainty"]
|
|
# Hedged surface starts with the manifold's hedge phrase.
|
|
assert s3["hedged_surface"].startswith(s3["hedge_prefix"])
|
|
|
|
def test_scene_4_byte_identical_replay(self) -> None:
|
|
with redirect_stdout(io.StringIO()):
|
|
result = run_tour(emit_json=True)
|
|
s4 = result["scene_4_deterministic_replay"]
|
|
assert s4["byte_identical"] is True
|
|
# The two short-hash previews must match too (sanity).
|
|
assert s4["line_1_sha_preview"] == s4["line_2_sha_preview"]
|
|
|
|
|
|
class TestNarrationMode:
|
|
def test_narration_prints_when_emit_json_false(self) -> None:
|
|
buf = io.StringIO()
|
|
with redirect_stdout(buf):
|
|
run_tour(emit_json=False)
|
|
output = buf.getvalue()
|
|
assert "CORE Audit Tour" in output
|
|
assert "Scene 1" in output
|
|
assert "Scene 2" in output
|
|
assert "Scene 3" in output
|
|
assert "Scene 4" in output
|
|
assert "Summary" in output
|
|
|
|
def test_emit_json_suppresses_narration(self) -> None:
|
|
buf = io.StringIO()
|
|
with redirect_stdout(buf):
|
|
run_tour(emit_json=True)
|
|
# Nothing should have been printed — caller is responsible
|
|
# for serialising the returned dict.
|
|
assert buf.getvalue() == ""
|
|
|
|
|
|
class TestStructuredReportSerialisable:
|
|
def test_result_is_json_serialisable(self) -> None:
|
|
with redirect_stdout(io.StringIO()):
|
|
result = run_tour(emit_json=True)
|
|
# Round-trip through json to ensure no non-serialisable types
|
|
# leak into the report.
|
|
encoded = json.dumps(result, default=str, sort_keys=True)
|
|
decoded = json.loads(encoded)
|
|
assert decoded["all_claims_supported"] is True
|