diff --git a/core/cli.py b/core/cli.py index b9cec656..ba673fcc 100644 --- a/core/cli.py +++ b/core/cli.py @@ -23,7 +23,7 @@ _CORE_RS_DIR = _REPO_ROOT / "core-rs" _CORE_RS_MANIFEST = _CORE_RS_DIR / "Cargo.toml" DESCRIPTION = "CORE versor engine command suite." -EPILOG = "Examples:\n core chat\n core pulse \"What is truth?\"\n core pulse --no-glove --json \"Compare knowledge and wisdom\"\n core bench\n core bench --suite all\n core bench --suite all --json --report bench_all.json\n core bench --suite determinism --runs 50\n core bench --suite speedup --json\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core teaching audit\n core teaching audit --json\n core teaching gaps --top 10\n core teaching queue --threshold 3\n core teaching propose \n core teaching proposals --state pending\n core teaching review --accept --review-date 2026-05-18\n core teaching supersede cause_light_reveals_truth --subject light --intent cause --connective grounds --object truth --review-date 2026-05-18\n core teaching supersessions\n core teaching supersessions --json\n core test --suite fast -q\n core test --suite pulse -q\n core test --suite proof -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core demo audit-tour\n core demo pack-measurements\n core demo long-context-comparison\n core demo anti-regression\n core demo learning-loop\n core demo articulation\n core demo all\n core demo adr-0024-chain\n core eval --list\n core eval cognition\n core eval cognition --json --save\n core eval cognition --split dev --version v1\n core eval cognition --split holdout" +EPILOG = "Examples:\n core chat\n core pulse \"What is truth?\"\n core pulse --no-glove --json \"Compare knowledge and wisdom\"\n core bench\n core bench --suite all\n core bench --suite all --json --report bench_all.json\n core bench --suite determinism --runs 50\n core bench --suite speedup --json\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core teaching audit\n core teaching audit --json\n core teaching gaps --top 10\n core teaching queue --threshold 3\n core teaching propose \n core teaching proposals --state pending\n core teaching review --accept --review-date 2026-05-18\n core teaching supersede cause_light_reveals_truth --subject light --intent cause --connective grounds --object truth --review-date 2026-05-18\n core teaching supersessions\n core teaching supersessions --json\n core test --suite fast -q\n core test --suite pulse -q\n core test --suite proof -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core demo audit-tour\n core demo pack-measurements\n core demo long-context-comparison\n core demo anti-regression\n core demo learning-loop\n core demo articulation\n core demo conversation\n core demo conversation --no-stream\n core demo all\n core demo adr-0024-chain\n core eval --list\n core eval cognition\n core eval cognition --json --save\n core eval cognition --split dev --version v1\n core eval cognition --split holdout" _TEST_SUITES: dict[str, tuple[str, ...]] = { "fast": ( @@ -2001,6 +2001,17 @@ def cmd_demo(args: argparse.Namespace) -> int: print(json.dumps(report, indent=2, sort_keys=True)) return 0 + if target == "conversation": + from evals.conversation.run_demo import run_demo as run_conversation_demo + + # Stream by default; --no-stream disables per-character/per-word + # delays for CI / tests / fast capture. + stream = not getattr(args, "no_stream", False) + report = run_conversation_demo(emit_json=args.json, stream=stream) + if args.json: + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + if target == "long-context-comparison": from evals.long_context_cost.comparison_runner import ( run_comparison, @@ -2817,6 +2828,7 @@ def build_parser() -> argparse.ArgumentParser: "anti-regression", "learning-loop", "articulation", + "conversation", "all", "list-results", ], @@ -2838,10 +2850,21 @@ def build_parser() -> argparse.ArgumentParser: "propose → accept → same-prompt-now-grounded walkthrough. " "articulation: discourse-planner spine — EXPLAIN / COMPOUND / " "WALKTHROUGH multi-sentence articulation + determinism gate. " + "conversation: layperson-facing chat transcript with live " + "word-by-word streaming and plain-English captions. " "list-results: index every JSON report in the results directory." ), ) demo.add_argument("--json", action="store_true", help="emit machine-readable JSON") + demo.add_argument( + "--no-stream", + dest="no_stream", + action="store_true", + help=( + "for `conversation` target: disable per-character/per-word " + "streaming delays (used by CI / tests / fast capture)" + ), + ) demo.set_defaults(func=cmd_demo) eval_cmd = subparsers.add_parser("eval", help="run eval lanes") diff --git a/evals/conversation/__init__.py b/evals/conversation/__init__.py new file mode 100644 index 00000000..6fb29e80 --- /dev/null +++ b/evals/conversation/__init__.py @@ -0,0 +1,4 @@ +"""Conversation demo — layperson-facing chat transcript. + +See ``run_demo`` for the four-scene live walkthrough. +""" diff --git a/evals/conversation/run_demo.py b/evals/conversation/run_demo.py new file mode 100644 index 00000000..89c31a24 --- /dev/null +++ b/evals/conversation/run_demo.py @@ -0,0 +1,376 @@ +"""Conversation demo — layperson-facing chat transcript. + +Four scenes that show CORE actually being used, framed as a chat +transcript with plain-English notes between turns. No metric tables, +no flag jargon — just ``You: …`` / ``CORE: …`` and a short caption +after each turn that explains what just happened. + +Scenes: + + 1. Pack lookup — "What is truth?" + Shows the system answering from its + lexicon, deterministically. + + 2. Teaching-chain — "Walk me through recall." + Shows CORE chaining reviewed facts to + produce a multi-sentence answer. + + 3. Compound prompt — "What is truth, and why does it matter?" + Shows CORE handling both clauses, + composing two sub-answers in order. + + 4. Cold turn → learn — "Why does narrative exist?" + Shows CORE saying "I haven't learned + this yet", an operator teaching it, then + the same prompt answered. The full + learning loop in plain English. + +Stream mode (default) emits the response word-by-word with a small +inter-word delay so the layperson sees the answer "arriving live". +This is presentation only — the underlying surface is byte-identical +to the non-streamed version, because CORE's articulation path is +deterministic. + +``--no-stream`` disables the delay (CI / tests / fast capture). +``--json`` emits a structured report and suppresses all chat output. +""" + +from __future__ import annotations + +import sys +import textwrap +import time +from dataclasses import dataclass +from typing import Any + +from chat.runtime import ChatRuntime +from core.config import RuntimeConfig + + +# --------------------------------------------------------------------------- +# Streaming presentation +# --------------------------------------------------------------------------- + + +_WORD_DELAY_SECONDS: float = 0.04 # ~25 words/second; conversational pace +_CARET_DELAY_SECONDS: float = 0.012 # per-char delay for the "typed" prompt + + +def _stream_write(text: str, delay: float = _CARET_DELAY_SECONDS) -> None: + """Write text to stdout with a per-character delay.""" + for ch in text: + sys.stdout.write(ch) + sys.stdout.flush() + if delay > 0: + time.sleep(delay) + + +def _stream_words(text: str, *, prefix: str = " ", width: int = 60, + delay: float = _WORD_DELAY_SECONDS) -> None: + """Emit ``text`` word-by-word, wrapped to ``width`` after ``prefix``. + + The caller is expected to have already written the first-line + label (e.g. ``" CORE: "``), so no prefix is written on the very + first line — only on wrapped continuation lines. + """ + line = "" # tracks rendered width on current line; caller wrote the label + first_line = True + for word in text.split(): + if first_line: + sep = "" if not line else " " + candidate_width = len(line) + len(sep) + len(word) + else: + sep = "" if not line else " " + candidate_width = len(line) + len(sep) + len(word) + if candidate_width > width and line: + sys.stdout.write("\n") + sys.stdout.write(prefix) + line = "" + first_line = False + sep = "" + sys.stdout.write(sep + word) + sys.stdout.flush() + line = line + sep + word + if delay > 0: + time.sleep(delay) + sys.stdout.write("\n") + sys.stdout.flush() + + +def _stream_note(text: str, *, prefix: str = " ← ", width: int = 56) -> None: + """Emit a plain-English caption after a CORE turn.""" + wrapped = textwrap.fill( + text, + width=width, + initial_indent=prefix, + subsequent_indent=" ", + ) + sys.stdout.write("\n") + for line in wrapped.splitlines(): + sys.stdout.write(line + "\n") + sys.stdout.flush() + time.sleep(_WORD_DELAY_SECONDS) + + +def _scene_header(num: int, title: str) -> None: + sys.stdout.write("\n") + sys.stdout.write("─" * 64 + "\n") + sys.stdout.write(f" Scene {num} — {title}\n") + sys.stdout.write("─" * 64 + "\n\n") + sys.stdout.flush() + + +def _emit_turn(prompt: str, response_text: str, note: str, *, stream: bool) -> None: + """Render one You/CORE turn with a caption. + + ``stream=True`` adds per-character / per-word delays (live feel). + ``stream=False`` prints the same layout instantly (CI / tests / + fast capture). + """ + if stream: + sys.stdout.write(" You: ") + _stream_write(prompt, _CARET_DELAY_SECONDS) + sys.stdout.write("\n\n") + sys.stdout.write(" CORE: ") + sys.stdout.flush() + time.sleep(0.25) # tiny "thinking" pause + _stream_words(response_text, prefix=" ", width=58) + _stream_note(note) + else: + sys.stdout.write(f" You: {prompt}\n\n") + wrapped_response = textwrap.fill( + response_text, width=58, + initial_indent=" ", subsequent_indent=" ", + ) + sys.stdout.write(f" CORE: {wrapped_response.lstrip()}\n\n") + wrapped_note = textwrap.fill( + note, width=56, + initial_indent=" ← ", subsequent_indent=" ", + ) + sys.stdout.write(f"{wrapped_note}\n") + sys.stdout.flush() + + +# --------------------------------------------------------------------------- +# Report shapes +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class TurnRecord: + scene: str + prompt: str + surface: str + grounding_source: str + note: str + + def as_dict(self) -> dict[str, Any]: + return { + "scene": self.scene, + "prompt": self.prompt, + "surface": self.surface, + "grounding_source": self.grounding_source, + "note": self.note, + } + + +@dataclass(frozen=True, slots=True) +class ConversationReport: + turns: tuple[TurnRecord, ...] + learning_loop_closed: bool + active_corpus_byte_identical: bool + + def as_dict(self) -> dict[str, Any]: + return { + "turns": [t.as_dict() for t in self.turns], + "learning_loop_closed": self.learning_loop_closed, + "active_corpus_byte_identical": self.active_corpus_byte_identical, + } + + +# --------------------------------------------------------------------------- +# CORE wrappers +# --------------------------------------------------------------------------- + + +def _ask(prompt: str, *, planner: bool = True) -> tuple[str, str]: + rt = ChatRuntime(config=RuntimeConfig(discourse_planner=planner)) + response = rt.chat(prompt) + return response.surface, response.grounding_source + + +# --------------------------------------------------------------------------- +# Scenes +# --------------------------------------------------------------------------- + + +def _scene1_pack_lookup(*, show: bool, stream: bool) -> TurnRecord: + prompt = "What is truth?" + if show: + _scene_header(1, "Asking CORE to define a concept") + surface, grounding = _ask(prompt, planner=False) + note = ( + "CORE looked this up in its curated lexicon. Every word in the " + "answer traces to a reviewed source — same answer every time, no " + "internet, no guessing." + ) + if show: + _emit_turn(prompt, surface, note, stream=stream) + return TurnRecord( + scene="S1_pack_lookup", prompt=prompt, surface=surface, + grounding_source=grounding, note=note, + ) + + +def _scene2_teaching_chain(*, show: bool, stream: bool) -> TurnRecord: + prompt = "Walk me through recall." + if show: + _scene_header(2, "Asking CORE to walk through a concept") + surface, grounding = _ask(prompt, planner=True) + note = ( + "The second sentence wasn't memorised — CORE walked a reviewed " + "teaching chain: recall → reveals → memory. Each hop is a fact " + "an operator approved." + ) + if show: + _emit_turn(prompt, surface, note, stream=stream) + return TurnRecord( + scene="S2_teaching_chain", prompt=prompt, surface=surface, + grounding_source=grounding, note=note, + ) + + +def _scene3_compound(*, show: bool, stream: bool) -> TurnRecord: + prompt = "What is truth, and why does it matter?" + if show: + _scene_header(3, "Asking CORE a two-part question") + surface, grounding = _ask(prompt, planner=True) + note = ( + "CORE split the question at the comma, answered both halves, and " + "stitched them together in order — every sentence still grounded " + "in the lexicon or in a reviewed chain." + ) + if show: + _emit_turn(prompt, surface, note, stream=stream) + return TurnRecord( + scene="S3_compound", prompt=prompt, surface=surface, + grounding_source=grounding, note=note, + ) + + +def _scene4_learning_loop(*, show: bool, stream: bool) -> tuple[TurnRecord, TurnRecord, bool, bool]: + """Cold turn → operator teaches → re-ask. + + Reuses the production learning-loop demo so the underlying + propose/replay/accept machinery is exactly what ships. + """ + from evals.learning_loop.run_demo import run_demo as run_loop + + prompt = "Why does narrative exist?" + if show: + _scene_header(4, "Teaching CORE something new, then re-asking") + sys.stdout.write(" (This scene runs CORE's reviewed-learning loop end-to-end:\n") + sys.stdout.write(" cold turn → operator proposes a chain → safety/replay gate\n") + sys.stdout.write(" confirms no regression → operator accepts → same prompt is\n") + sys.stdout.write(" now grounded. The active corpus on disk is not mutated.)\n\n") + sys.stdout.flush() + + # Run the real learning-loop demo (suppressed output) to get the + # before/after surfaces deterministically. + import contextlib, io + with contextlib.redirect_stdout(io.StringIO()): + ll = run_loop(emit_json=True) + + before_surface = ll["before"]["surface"] + before_grounding = ll["before"]["grounding_source"] + after_surface = ll["after"]["surface"] + after_grounding = ll["after"]["grounding_source"] + loop_closed = bool(ll["learning_loop_closed"]) + byte_identical = bool(ll["active_corpus_byte_identical"]) + + before_note = ( + "CORE refuses to make something up. It says it hasn't learned this " + "yet and points to where a reviewed chain would help — instead of " + "fabricating an answer." + ) + after_note = ( + "An operator reviewed and accepted one new chain " + "(narrative → reveals → meaning). A replay gate first confirmed it " + "wouldn't regress anything CORE already knows. Now the same prompt " + "is answered — with full provenance back to that one accept." + ) + + if show: + _emit_turn(prompt, before_surface, before_note, stream=stream) + sys.stdout.write("\n") + sys.stdout.write(" ┄ ┄ ┄ operator teaches CORE one new fact ┄ ┄ ┄\n\n") + sys.stdout.flush() + if stream: + time.sleep(0.6) + _emit_turn(prompt, after_surface, after_note, stream=stream) + + before = TurnRecord( + scene="S4a_cold_turn", prompt=prompt, surface=before_surface, + grounding_source=before_grounding, note=before_note, + ) + after = TurnRecord( + scene="S4b_after_teaching", prompt=prompt, surface=after_surface, + grounding_source=after_grounding, note=after_note, + ) + return before, after, loop_closed, byte_identical + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + + +def run_demo(*, emit_json: bool = False, stream: bool = True) -> dict[str, Any]: + """Run all four scenes and return a structured report. + + ``emit_json=True`` suppresses every chat-style print; only the + final JSON object will be emitted by the caller. ``stream=False`` + keeps the chat layout but skips the per-character / per-word + delays (used by tests and ``--no-stream``). + """ + show = not emit_json + actual_stream = show and stream + + if show: + sys.stdout.write("\n") + sys.stdout.write("═" * 64 + "\n") + sys.stdout.write(" Conversation with CORE — live walkthrough\n") + sys.stdout.write("═" * 64 + "\n") + sys.stdout.write( + "\n CORE is a deterministic cognitive engine. It doesn't run\n" + " an LLM, it doesn't sample, it doesn't search the web. Every\n" + " word in every answer below traces to a reviewed source.\n" + " Run this demo twice — you'll get the same surfaces.\n" + ) + sys.stdout.flush() + + s1 = _scene1_pack_lookup(show=show, stream=actual_stream) + s2 = _scene2_teaching_chain(show=show, stream=actual_stream) + s3 = _scene3_compound(show=show, stream=actual_stream) + s4_before, s4_after, loop_closed, byte_identical = _scene4_learning_loop( + show=show, stream=actual_stream, + ) + + turns = (s1, s2, s3, s4_before, s4_after) + report = ConversationReport( + turns=turns, + learning_loop_closed=loop_closed, + active_corpus_byte_identical=byte_identical, + ) + + if show: + sys.stdout.write("\n") + sys.stdout.write("═" * 64 + "\n") + sys.stdout.write(" Done. Everything above is deterministic and replayable.\n") + sys.stdout.write("═" * 64 + "\n\n") + sys.stdout.flush() + + return report.as_dict() + + +__all__ = ["run_demo"] diff --git a/tests/test_conversation_demo.py b/tests/test_conversation_demo.py new file mode 100644 index 00000000..e3e9db8e --- /dev/null +++ b/tests/test_conversation_demo.py @@ -0,0 +1,90 @@ +"""Conversation demo — pins the layperson-facing chat transcript. + +These tests use ``stream=False`` so the demo runs instantly. They +verify the structured JSON report (which is what downstream +consumers integrate against), not the streamed visual layout. +""" + +from __future__ import annotations + +import pytest + +from evals.conversation.run_demo import run_demo + + +@pytest.fixture(scope="module") +def demo_report() -> dict: + return run_demo(emit_json=True, stream=False) + + +def test_demo_has_five_turns(demo_report: dict) -> None: + assert len(demo_report["turns"]) == 5 + + +def test_demo_closes_the_learning_loop(demo_report: dict) -> None: + assert demo_report["learning_loop_closed"] is True + assert demo_report["active_corpus_byte_identical"] is True + + +def test_scene1_pack_lookup_grounds_in_pack(demo_report: dict) -> None: + s1 = demo_report["turns"][0] + assert s1["scene"] == "S1_pack_lookup" + assert s1["prompt"] == "What is truth?" + assert s1["grounding_source"] == "pack" + assert "truth" in s1["surface"].lower() + assert "lexicon" in s1["note"].lower() + + +def test_scene2_teaching_chain_grounds_in_teaching(demo_report: dict) -> None: + s2 = demo_report["turns"][1] + assert s2["scene"] == "S2_teaching_chain" + assert s2["prompt"] == "Walk me through recall." + assert s2["grounding_source"] == "teaching" + assert "reveals memory" in s2["surface"].lower() + assert "chain" in s2["note"].lower() + + +def test_scene3_compound_handles_both_clauses(demo_report: dict) -> None: + s3 = demo_report["turns"][2] + assert s3["scene"] == "S3_compound" + assert s3["grounding_source"] in {"pack", "teaching"} + sentence_count = sum(1 for ch in s3["surface"] if ch in ".!?") + assert sentence_count >= 4 + assert "truth" in s3["surface"].lower() + + +def test_scene4_cold_turn_does_not_make_up_an_answer(demo_report: dict) -> None: + s4a = demo_report["turns"][3] + assert s4a["scene"] == "S4a_cold_turn" + assert s4a["grounding_source"] in {"none", "oov"} + surface_low = s4a["surface"].lower() + assert "don't know" in surface_low or "haven't learned" in surface_low or "insufficient" in surface_low + + +def test_scene4_after_teaching_is_grounded_with_new_chain(demo_report: dict) -> None: + s4b = demo_report["turns"][4] + assert s4b["scene"] == "S4b_after_teaching" + assert s4b["grounding_source"] == "teaching" + surface_low = s4b["surface"].lower() + assert "narrative" in surface_low + assert "meaning" in surface_low + + +def test_demo_json_shape_is_stable(demo_report: dict) -> None: + assert set(demo_report.keys()) == { + "turns", "learning_loop_closed", "active_corpus_byte_identical", + } + for turn in demo_report["turns"]: + assert set(turn.keys()) == { + "scene", "prompt", "surface", "grounding_source", "note", + } + + +def test_demo_does_not_mutate_active_teaching_corpus() -> None: + """The demo must be read-only against the live corpus.""" + from chat import teaching_grounding as _tg + + before = _tg._CORPUS_PATH.read_bytes() if _tg._CORPUS_PATH.exists() else b"" + run_demo(emit_json=True, stream=False) + after = _tg._CORPUS_PATH.read_bytes() if _tg._CORPUS_PATH.exists() else b"" + assert before == after