dc4b565b5a
Four-scene investor/operator-facing walkthrough proving the discourse-
planner spine is load-bearing. Each scene runs the same prompt under
flag-off (BRIEF baseline) and flag-on (RuntimeConfig.discourse_planner)
and pins a falsifiable lift assertion.
S1. EXPLAIN — Explain truth.
Flag-on: pack→teaching upgrade + 2 chain
continuation sentences over baseline.
S2. COMPOUND — What is truth, and why does it matter?
Flag-on: 9 grounded sentences across two sub-
plans; flag-off routes to OOV.
S3. WALKTHROUGH — Walk me through recall.
Flag-on emits the CLOSURE chain hop
'Recall reveals memory.'; flag-off
does not.
S4. Determinism — N=3 reruns × 3 prompts, unique(surface)=1.
Read-only against live packs + active corpus. Demo is test-gated
(7 tests, all green) and ships a stable JSON contract for downstream
consumers.
Wired into CLI as `core demo articulation [--json]` alongside the
existing trilogy (audit-tour / anti-regression / learning-loop).
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
97 lines
3.6 KiB
Python
97 lines
3.6 KiB
Python
"""Articulation demo — pins the load-bearing claim per scene.
|
|
|
|
The headline claim: ``RuntimeConfig.discourse_planner=True`` produces
|
|
deterministic, grounded, multi-sentence articulation across EXPLAIN,
|
|
COMPOUND, and WALKTHROUGH prompt shapes; the same prompts under the
|
|
flag-off baseline collapse to single-anchor or OOV surfaces.
|
|
|
|
If any assertion below fails, the demo's headline claim no longer
|
|
holds.
|
|
|
|
Performance: ``run_demo()`` instantiates ~13 ``ChatRuntime`` objects
|
|
(3 scenes x 2 flags + 3 prompts x 3 reruns for determinism). Module-
|
|
scoped fixture caches one run across every test in this file.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from evals.articulation.run_demo import run_demo
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def demo_report() -> dict:
|
|
return run_demo(emit_json=True)
|
|
|
|
|
|
def test_demo_all_claims_supported(demo_report: dict) -> None:
|
|
assert demo_report["all_claims_supported"] is True
|
|
assert len(demo_report["scenes"]) == 4
|
|
|
|
|
|
def test_s1_explain_lifts_to_multi_sentence_teaching(demo_report: dict) -> None:
|
|
s1 = demo_report["scenes"][0]
|
|
assert s1["scene"] == "S1_explain"
|
|
assert s1["detail"]["claim_supported"] is True
|
|
on = s1["detail"]["flag_on"]
|
|
off = s1["detail"]["flag_off"]
|
|
assert on["grounding_source"] == "teaching"
|
|
assert off["grounding_source"] == "pack"
|
|
assert on["sentence_count"] >= off["sentence_count"] + 2
|
|
assert on["sentence_count"] >= 3
|
|
assert "truth" in on["surface"].lower()
|
|
|
|
|
|
def test_s2_compound_lifts_oov_to_grounded(demo_report: dict) -> None:
|
|
s2 = demo_report["scenes"][1]
|
|
assert s2["scene"] == "S2_compound"
|
|
assert s2["detail"]["claim_supported"] is True
|
|
on = s2["detail"]["flag_on"]
|
|
off = s2["detail"]["flag_off"]
|
|
assert on["grounding_source"] in {"pack", "teaching"}
|
|
assert off["grounding_source"] in {"oov", "none"}
|
|
assert on["sentence_count"] >= 4
|
|
assert "haven't learned" in off["surface"].lower()
|
|
assert "truth" in on["surface"].lower()
|
|
|
|
|
|
def test_s3_walkthrough_emits_chain_closure(demo_report: dict) -> None:
|
|
s3 = demo_report["scenes"][2]
|
|
assert s3["scene"] == "S3_walkthrough"
|
|
assert s3["detail"]["claim_supported"] is True
|
|
on = s3["detail"]["flag_on"]
|
|
off = s3["detail"]["flag_off"]
|
|
assert on["grounding_source"] == "teaching"
|
|
# The CLOSURE chain hop appears only flag-on.
|
|
assert "reveals memory" in on["surface"].lower()
|
|
assert "reveals memory" not in off["surface"].lower()
|
|
|
|
|
|
def test_s4_determinism_byte_identical_across_reruns(demo_report: dict) -> None:
|
|
s4 = demo_report["scenes"][3]
|
|
assert s4["scene"] == "S4_determinism"
|
|
assert s4["detail"]["all_identical"] is True
|
|
assert s4["detail"]["reruns_per_prompt"] == 3
|
|
per_prompt = s4["detail"]["per_prompt"]
|
|
assert len(per_prompt) == 3
|
|
for entry in per_prompt:
|
|
assert entry["unique_surfaces"] == 1
|
|
assert entry["identical"] is True
|
|
|
|
|
|
def test_demo_does_not_mutate_active_teaching_corpus() -> None:
|
|
"""Demo is read-only — re-running it twice must not change corpus bytes."""
|
|
from chat import teaching_grounding as _tg
|
|
|
|
before = _tg._CORPUS_PATH.read_bytes() if _tg._CORPUS_PATH.exists() else b""
|
|
run_demo(emit_json=True)
|
|
after = _tg._CORPUS_PATH.read_bytes() if _tg._CORPUS_PATH.exists() else b""
|
|
assert before == after
|
|
|
|
|
|
def test_demo_json_shape_is_stable(demo_report: dict) -> None:
|
|
"""Stable JSON contract for downstream consumers."""
|
|
assert set(demo_report.keys()) == {"scenes", "all_claims_supported"}
|
|
for scene in demo_report["scenes"]:
|
|
assert set(scene.keys()) == {"scene", "claim", "detail"}
|