diff --git a/core/cli.py b/core/cli.py index 2c0cf659..79c833c5 100644 --- a/core/cli.py +++ b/core/cli.py @@ -23,7 +23,7 @@ _CORE_RS_DIR = _REPO_ROOT / "core-rs" _CORE_RS_MANIFEST = _CORE_RS_DIR / "Cargo.toml" DESCRIPTION = "CORE versor engine command suite." -EPILOG = "Examples:\n core chat\n core pulse \"What is truth?\"\n core pulse --no-glove --json \"Compare knowledge and wisdom\"\n core bench\n core bench --suite all\n core bench --suite all --json --report bench_all.json\n core bench --suite determinism --runs 50\n core bench --suite speedup --json\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core teaching audit\n core teaching audit --json\n core teaching gaps --top 10\n core teaching queue --threshold 3\n core teaching propose \n core teaching proposals --state pending\n core teaching review --accept --review-date 2026-05-18\n core teaching supersede cause_light_reveals_truth --subject light --intent cause --connective grounds --object truth --review-date 2026-05-18\n core teaching supersessions\n core teaching supersessions --json\n core test --suite fast -q\n core test --suite pulse -q\n core test --suite proof -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core demo audit-tour\n core demo pack-measurements\n core demo long-context-comparison\n core demo anti-regression\n core demo learning-loop\n core eval --list\n core eval cognition\n core eval cognition --json --save\n core eval cognition --split dev --version v1\n core eval cognition --split holdout" +EPILOG = "Examples:\n core chat\n core pulse \"What is truth?\"\n core pulse --no-glove --json \"Compare knowledge and wisdom\"\n core bench\n core bench --suite all\n core bench --suite all --json --report bench_all.json\n core bench --suite determinism --runs 50\n core bench --suite speedup --json\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core teaching audit\n core teaching audit --json\n core teaching gaps --top 10\n core teaching queue --threshold 3\n core teaching propose \n core teaching proposals --state pending\n core teaching review --accept --review-date 2026-05-18\n core teaching supersede cause_light_reveals_truth --subject light --intent cause --connective grounds --object truth --review-date 2026-05-18\n core teaching supersessions\n core teaching supersessions --json\n core test --suite fast -q\n core test --suite pulse -q\n core test --suite proof -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core demo audit-tour\n core demo pack-measurements\n core demo long-context-comparison\n core demo anti-regression\n core demo learning-loop\n core demo articulation\n core eval --list\n core eval cognition\n core eval cognition --json --save\n core eval cognition --split dev --version v1\n core eval cognition --split holdout" _TEST_SUITES: dict[str, tuple[str, ...]] = { "fast": ( @@ -1477,6 +1477,59 @@ Machine-readable output: """ +_ARTICULATION_PREAMBLE = """ +================================================================================ + Articulation — Discourse-Planner Spine, End-to-End +================================================================================ + +Reference: docs/evals/articulation_bench_2026-05-19.md, commits 7af7892 +(CompoundIntent), 4e3ddee (WALKTHROUGH v1), e985790 (planner-on bench), +07fefb9 (articulate/disclosure/unarticulate partition). + +The discourse-planner spine turns a classified intent + grounding bundle +into a deterministic multi-sentence surface without an LLM, without +sampling, and without approximate retrieval. Every sentence traces to a +pack lemma, a reviewed teaching chain, or a fixed connective vocabulary. + + S1. EXPLAIN — "Explain truth." + Flag-on: ANCHOR + SUPPORT multi-sentence paragraph + grounded in teaching (>=3 sentences). + Flag-off: BRIEF pack anchor only (2 sentences, + incl. pack-grounded tag). + + S2. COMPOUND — "What is truth, and why does it matter?" + Flag-on: source-ordered sub-plans + TRANSITION + bridge (>=4 sentences, teaching-grounded). + Flag-off: OOV disclosure (the flat classifier + cannot parse the second clause). + + S3. WALKTHROUGH — "Walk me through recall." + Flag-on: pack anchor + teaching-chain CLOSURE + ("Recall reveals memory."). + Flag-off: pack anchor only, no chain hop. + + S4. Determinism — Each prompt re-run N=3 with a fresh ChatRuntime; + unique(surface) == 1 for every prompt. + +Trust boundary: + This demo does not mutate any corpus, pack, or vault. Read-only + against live packs + active teaching corpus. + +What to expect: + Per-scene printout with CLAIM, prompt, flag-off baseline, flag-on + surface, sentence counts, grounding source. Final summary lists each + scene's claim_supported flag. + +Test gate: + tests/test_articulation_demo.py (7 tests — per-scene claim + + all_claims_supported + determinism invariant). + +Machine-readable output: + core demo articulation --json +================================================================================ +""" + + _ANTI_REGRESSION_PREAMBLE = """ ================================================================================ Anti-Regression — Three-Gate Defense Against Learning Harm (ADR-0057) @@ -1904,6 +1957,16 @@ def cmd_demo(args: argparse.Namespace) -> int: print(json.dumps(report, indent=2, sort_keys=True)) return 0 + if target == "articulation": + from evals.articulation.run_demo import run_demo as run_articulation_demo + + if not args.json: + _print_preamble(_ARTICULATION_PREAMBLE) + report = run_articulation_demo(emit_json=args.json) + if args.json: + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + if target == "long-context-comparison": from evals.long_context_cost.comparison_runner import ( run_comparison, @@ -2564,6 +2627,7 @@ def build_parser() -> argparse.ArgumentParser: "long-context-comparison", "anti-regression", "learning-loop", + "articulation", "list-results", ], help=( @@ -2580,6 +2644,8 @@ def build_parser() -> argparse.ArgumentParser: "harmful chains (eligibility / replay-equivalence / operator). " "learning-loop: ADR-0055..0057 — full cold-turn → discovery → " "propose → accept → same-prompt-now-grounded walkthrough. " + "articulation: discourse-planner spine — EXPLAIN / COMPOUND / " + "WALKTHROUGH multi-sentence articulation + determinism gate. " "list-results: index every JSON report in the results directory." ), ) diff --git a/evals/articulation/__init__.py b/evals/articulation/__init__.py new file mode 100644 index 00000000..fd8be40f --- /dev/null +++ b/evals/articulation/__init__.py @@ -0,0 +1,4 @@ +"""Articulation demo — discourse-planner spine end-to-end. + +See ``run_demo`` for the four-scene walkthrough. +""" diff --git a/evals/articulation/run_demo.py b/evals/articulation/run_demo.py new file mode 100644 index 00000000..091b4d6f --- /dev/null +++ b/evals/articulation/run_demo.py @@ -0,0 +1,406 @@ +"""Articulation demo — discourse-planner spine, end-to-end. + +The thesis (the demo's headline claim): + + > With ``RuntimeConfig.discourse_planner=True``, CORE produces + > deterministic, grounded, multi-sentence articulation across three + > distinct prompt shapes — EXPLAIN, COMPOUND, WALKTHROUGH — and the + > exact same prompts under the flag-off baseline collapse to + > single-sentence (or disclosure) surfaces. The lift is load-bearing, + > not cosmetic. Every multi-sentence surface is byte-identical across + > reruns. + +The discourse-planner spine is: + + DialogueIntent + ResponseMode + GroundingBundle + -> DiscoursePlan (canonical move ordering) + -> PropositionGraph (pack/teaching-resident atoms) + -> ArticulationTarget (selected facts + connectives) + -> RealizedPlan (deterministic surface) + +No LLM, no stochastic sampling, no approximate retrieval. Every +sentence traces to a pack lemma, a reviewed teaching chain, or a +fixed connective vocabulary. + +Four scenes, each on a real ``ChatRuntime`` against the live active +corpus and packs. The active corpus file bytes are byte-identical +pre/post — this demo does not mutate any corpus. + + S1. EXPLAIN — ``Explain truth.`` + Flag-on: ANCHOR + SUPPORT multi-sentence paragraph. + Flag-off: BRIEF single-sentence baseline. + S2. COMPOUND — ``What is truth, and why does it matter?`` + Flag-on: source-ordered sub-plans + TRANSITION bridge. + Flag-off: OOV disclosure (compound subject pollution). + S3. WALKTHROUGH — ``Walk me through recall.`` + Flag-on: sequential teaching-chain walk with CLOSURE. + Flag-off: BRIEF single-sentence baseline. + S4. Determinism — Each prompt re-run N times under flag-on; + unique(surface) == 1 for every prompt. + +The test gates pin each scene's load-bearing assertion. If any of them +break, the demo's headline claim no longer holds. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from chat.runtime import ChatRuntime +from core.config import RuntimeConfig + + +_EXPLAIN_PROMPT: str = "Explain truth." +_COMPOUND_PROMPT: str = "What is truth, and why does it matter?" +_WALKTHROUGH_PROMPT: str = "Walk me through recall." + +_DETERMINISM_RERUNS: int = 3 + +_VERBOSE = True + + +def _say(*args: Any, **kwargs: Any) -> None: + if _VERBOSE: + print(*args, **kwargs) + + +def _print_header(title: str, claim: str) -> None: + _say() + _say("─" * 72) + _say(f" {title}") + _say("─" * 72) + _say(f" CLAIM: {claim}") + _say() + + +def _sentence_count(surface: str) -> int: + """Sentence count by terminal punctuation. + + Matches the convention used by the articulation bench + (``benchmarks/articulation._sentence_count``) so demo claims and + bench claims compose without arithmetic drift. + """ + if not surface: + return 0 + text = surface.strip() + count = 0 + for ch in text: + if ch in ".!?": + count += 1 + return max(count, 1) + + +def _chat_once(prompt: str, *, flag: bool) -> tuple[str, str]: + """Single deterministic turn. Returns ``(surface, grounding_source)``.""" + rt = ChatRuntime(config=RuntimeConfig(discourse_planner=flag)) + response = rt.chat(prompt) + return response.surface, response.grounding_source + + +# --------------------------------------------------------------------------- +# Report shapes +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class SceneResult: + scene: str + claim: str + detail: dict[str, Any] + + def as_dict(self) -> dict[str, Any]: + return {"scene": self.scene, "claim": self.claim, "detail": self.detail} + + +@dataclass(frozen=True, slots=True) +class DemoReport: + scenes: tuple[SceneResult, ...] + all_claims_supported: bool + + def as_dict(self) -> dict[str, Any]: + return { + "scenes": [s.as_dict() for s in self.scenes], + "all_claims_supported": self.all_claims_supported, + } + + +# --------------------------------------------------------------------------- +# Scenes +# --------------------------------------------------------------------------- + + +def _scene1_explain() -> SceneResult: + _print_header( + "S1. EXPLAIN — ANCHOR + SUPPORT multi-sentence paragraph", + "Under discourse_planner=True, an EXPLAIN prompt produces a " + "grounded multi-sentence paragraph composed from pack atoms + " + "reviewed teaching chains. Under flag-off, the same prompt " + "collapses to a single-sentence baseline. The lift is the " + "discourse planner spine doing the work.", + ) + off_surface, off_grounding = _chat_once(_EXPLAIN_PROMPT, flag=False) + on_surface, on_grounding = _chat_once(_EXPLAIN_PROMPT, flag=True) + off_count = _sentence_count(off_surface) + on_count = _sentence_count(on_surface) + + _say(f" prompt : {_EXPLAIN_PROMPT}") + _say(f" flag=False (BRIEF) : [{off_grounding}] ({off_count} sent.) {off_surface}") + _say(f" flag=True (EXPLAIN): [{on_grounding}] ({on_count} sent.) {on_surface}") + + claim_supported = ( + on_count >= off_count + 2 + and on_count >= 3 + and on_grounding == "teaching" + and off_grounding == "pack" + and "truth" in on_surface.lower() + ) + if not claim_supported: + raise RuntimeError( + f"S1 invariant broken: on_count={on_count}, off_count={off_count}, " + f"on_grounding={on_grounding!r}, off_grounding={off_grounding!r}" + ) + return SceneResult( + scene="S1_explain", + claim=( + "Flag-on yields at least +2 sentences over flag-off and upgrades " + "grounding from pack to teaching by chaining reviewed chains " + "onto the pack anchor. The added sentences are pack/teaching-" + "grounded continuations, not template padding." + ), + detail={ + "prompt": _EXPLAIN_PROMPT, + "flag_on": { + "surface": on_surface, + "grounding_source": on_grounding, + "sentence_count": on_count, + }, + "flag_off": { + "surface": off_surface, + "grounding_source": off_grounding, + "sentence_count": off_count, + }, + "claim_supported": claim_supported, + }, + ) + + +def _scene2_compound() -> SceneResult: + _print_header( + "S2. COMPOUND — source-ordered sub-plans, no clause dropped", + "Under discourse_planner=True, a compound prompt decomposes via " + "classify_compound_intent() into ordered sub-intents. Each " + "sub-plan composes its own grounded surface, fact-deduped across " + "parts, joined with TRANSITION bridges. Under flag-off, the " + "flat classifier sees a polluted subject (\"truth, and why does " + "it matter\") and routes to OOV. Compound handling is therefore " + "load-bearing, not stylistic.", + ) + off_surface, off_grounding = _chat_once(_COMPOUND_PROMPT, flag=False) + on_surface, on_grounding = _chat_once(_COMPOUND_PROMPT, flag=True) + off_count = _sentence_count(off_surface) + on_count = _sentence_count(on_surface) + + _say(f" prompt : {_COMPOUND_PROMPT}") + _say(f" flag=False (flat) : [{off_grounding}] ({off_count} sent.) {off_surface[:140]}...") + _say(f" flag=True (compound): [{on_grounding}] ({on_count} sent.) {on_surface}") + + claim_supported = ( + on_count >= 4 + and on_grounding in {"pack", "teaching"} + and off_grounding in {"oov", "none"} + and "truth" in on_surface.lower() + and "haven't learned" in off_surface.lower() + ) + if not claim_supported: + raise RuntimeError( + f"S2 invariant broken: on_count={on_count}, " + f"on_grounding={on_grounding!r}, off_grounding={off_grounding!r}" + ) + return SceneResult( + scene="S2_compound", + claim=( + "Flag-on yields >=4 grounded sentences spanning both clauses " + "of the compound prompt; flag-off routes to OOV because the " + "flat classifier cannot parse the second clause. Compound " + "decomposition is the load-bearing step." + ), + detail={ + "prompt": _COMPOUND_PROMPT, + "flag_on": { + "surface": on_surface, + "grounding_source": on_grounding, + "sentence_count": on_count, + }, + "flag_off": { + "surface": off_surface, + "grounding_source": off_grounding, + "sentence_count": off_count, + }, + "claim_supported": claim_supported, + }, + ) + + +def _scene3_walkthrough() -> SceneResult: + _print_header( + "S3. WALKTHROUGH — sequential teaching-chain walk with CLOSURE", + "Under discourse_planner=True, a walkthrough prompt drives the " + "planner's WALKTHROUGH mode: anchor on the subject's pack " + "definition, then walk reviewed teaching chains " + "(subject, *, obj) -> (obj, *, *) up to 4 hops, terminating in " + "a CLOSURE move. Under flag-off, the same prompt collapses to " + "the brief definition only.", + ) + off_surface, off_grounding = _chat_once(_WALKTHROUGH_PROMPT, flag=False) + on_surface, on_grounding = _chat_once(_WALKTHROUGH_PROMPT, flag=True) + off_count = _sentence_count(off_surface) + on_count = _sentence_count(on_surface) + + _say(f" prompt : {_WALKTHROUGH_PROMPT}") + _say(f" flag=False (BRIEF) : [{off_grounding}] ({off_count} sent.) {off_surface}") + _say(f" flag=True (WALKTHROUGH): [{on_grounding}] ({on_count} sent.) {on_surface}") + + on_lower = on_surface.lower() + off_lower = off_surface.lower() + # Walkthrough load-bearing test: the chain-walk CLOSURE sentence + # ("Recall reveals memory.") appears flag-on but not flag-off. + # Flag-off emits only the pack anchor. + chain_hop_on = "reveals memory" in on_lower + chain_hop_off = "reveals memory" in off_lower + claim_supported = ( + on_grounding == "teaching" + and chain_hop_on + and not chain_hop_off + and "recall" in on_lower + ) + if not claim_supported: + raise RuntimeError( + f"S3 invariant broken: on_grounding={on_grounding!r}, " + f"chain_hop_on={chain_hop_on}, chain_hop_off={chain_hop_off}, " + f"surface={on_surface!r}" + ) + return SceneResult( + scene="S3_walkthrough", + claim=( + "Flag-on emits the chain-walk CLOSURE sentence " + "'Recall reveals memory.' from the reviewed teaching chain; " + "flag-off emits only the pack anchor. The chain walk is " + "the load-bearing step." + ), + detail={ + "prompt": _WALKTHROUGH_PROMPT, + "flag_on": { + "surface": on_surface, + "grounding_source": on_grounding, + "sentence_count": on_count, + }, + "flag_off": { + "surface": off_surface, + "grounding_source": off_grounding, + "sentence_count": off_count, + }, + "claim_supported": claim_supported, + }, + ) + + +def _scene4_determinism() -> SceneResult: + _print_header( + "S4. Determinism — byte-identical across reruns, every prompt", + "Each of the three discourse-planner prompts is re-run N times " + "with a fresh ChatRuntime per turn. unique(surface) must equal " + "1 for every prompt. No LLM, no sampling, no clock-time reads " + "in the articulation path — same plan, same proposition graph, " + "same realizer, same bytes.", + ) + prompts = [ + ("EXPLAIN", _EXPLAIN_PROMPT), + ("COMPOUND", _COMPOUND_PROMPT), + ("WALKTHROUGH", _WALKTHROUGH_PROMPT), + ] + per_prompt: list[dict[str, Any]] = [] + all_identical = True + for label, prompt in prompts: + seen: set[str] = set() + for _ in range(_DETERMINISM_RERUNS): + surface, _ = _chat_once(prompt, flag=True) + seen.add(surface) + unique = len(seen) + identical = unique == 1 + all_identical = all_identical and identical + _say(f" {label:<12} runs={_DETERMINISM_RERUNS} unique={unique} identical={identical}") + per_prompt.append({ + "label": label, + "prompt": prompt, + "runs": _DETERMINISM_RERUNS, + "unique_surfaces": unique, + "identical": identical, + }) + + if not all_identical: + raise RuntimeError( + f"S4 invariant broken: not every prompt produced unique=1; " + f"per_prompt={per_prompt}" + ) + return SceneResult( + scene="S4_determinism", + claim=( + "Every discourse-planner prompt produces byte-identical " + "surface across reruns. Replayability is architectural, " + "not configurational." + ), + detail={ + "reruns_per_prompt": _DETERMINISM_RERUNS, + "per_prompt": per_prompt, + "all_identical": all_identical, + }, + ) + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + + +def run_demo(*, emit_json: bool = False) -> dict[str, Any]: + """Run all four scenes and return a structured report.""" + global _VERBOSE + _VERBOSE = not emit_json + + s1 = _scene1_explain() + s2 = _scene2_compound() + s3 = _scene3_walkthrough() + s4 = _scene4_determinism() + scenes = (s1, s2, s3, s4) + + all_claims_supported = all( + bool(scene.detail.get("claim_supported", scene.detail.get("all_identical", False))) + for scene in scenes + ) + + report = DemoReport( + scenes=scenes, + all_claims_supported=all_claims_supported, + ) + + if _VERBOSE: + _say() + _say("═" * 72) + _say(" ARTICULATION DEMO — summary") + _say("═" * 72) + for scene in scenes: + supported = scene.detail.get( + "claim_supported", + scene.detail.get("all_identical", False), + ) + mark = "✓" if supported else "✗" + _say(f" {mark} {scene.scene}") + _say() + _say(f" all_claims_supported : {report.all_claims_supported}") + _say() + + return report.as_dict() + + +__all__ = ["run_demo"] diff --git a/tests/test_articulation_demo.py b/tests/test_articulation_demo.py new file mode 100644 index 00000000..eb095821 --- /dev/null +++ b/tests/test_articulation_demo.py @@ -0,0 +1,97 @@ +"""Articulation demo — pins the load-bearing claim per scene. + +The headline claim: ``RuntimeConfig.discourse_planner=True`` produces +deterministic, grounded, multi-sentence articulation across EXPLAIN, +COMPOUND, and WALKTHROUGH prompt shapes; the same prompts under the +flag-off baseline collapse to single-anchor or OOV surfaces. + +If any assertion below fails, the demo's headline claim no longer +holds. + +Performance: ``run_demo()`` instantiates ~13 ``ChatRuntime`` objects +(3 scenes x 2 flags + 3 prompts x 3 reruns for determinism). Module- +scoped fixture caches one run across every test in this file. +""" + +from __future__ import annotations + +import pytest + +from evals.articulation.run_demo import run_demo + + +@pytest.fixture(scope="module") +def demo_report() -> dict: + return run_demo(emit_json=True) + + +def test_demo_all_claims_supported(demo_report: dict) -> None: + assert demo_report["all_claims_supported"] is True + assert len(demo_report["scenes"]) == 4 + + +def test_s1_explain_lifts_to_multi_sentence_teaching(demo_report: dict) -> None: + s1 = demo_report["scenes"][0] + assert s1["scene"] == "S1_explain" + assert s1["detail"]["claim_supported"] is True + on = s1["detail"]["flag_on"] + off = s1["detail"]["flag_off"] + assert on["grounding_source"] == "teaching" + assert off["grounding_source"] == "pack" + assert on["sentence_count"] >= off["sentence_count"] + 2 + assert on["sentence_count"] >= 3 + assert "truth" in on["surface"].lower() + + +def test_s2_compound_lifts_oov_to_grounded(demo_report: dict) -> None: + s2 = demo_report["scenes"][1] + assert s2["scene"] == "S2_compound" + assert s2["detail"]["claim_supported"] is True + on = s2["detail"]["flag_on"] + off = s2["detail"]["flag_off"] + assert on["grounding_source"] in {"pack", "teaching"} + assert off["grounding_source"] in {"oov", "none"} + assert on["sentence_count"] >= 4 + assert "haven't learned" in off["surface"].lower() + assert "truth" in on["surface"].lower() + + +def test_s3_walkthrough_emits_chain_closure(demo_report: dict) -> None: + s3 = demo_report["scenes"][2] + assert s3["scene"] == "S3_walkthrough" + assert s3["detail"]["claim_supported"] is True + on = s3["detail"]["flag_on"] + off = s3["detail"]["flag_off"] + assert on["grounding_source"] == "teaching" + # The CLOSURE chain hop appears only flag-on. + assert "reveals memory" in on["surface"].lower() + assert "reveals memory" not in off["surface"].lower() + + +def test_s4_determinism_byte_identical_across_reruns(demo_report: dict) -> None: + s4 = demo_report["scenes"][3] + assert s4["scene"] == "S4_determinism" + assert s4["detail"]["all_identical"] is True + assert s4["detail"]["reruns_per_prompt"] == 3 + per_prompt = s4["detail"]["per_prompt"] + assert len(per_prompt) == 3 + for entry in per_prompt: + assert entry["unique_surfaces"] == 1 + assert entry["identical"] is True + + +def test_demo_does_not_mutate_active_teaching_corpus() -> None: + """Demo is read-only — re-running it twice must not change corpus bytes.""" + from chat import teaching_grounding as _tg + + before = _tg._CORPUS_PATH.read_bytes() if _tg._CORPUS_PATH.exists() else b"" + run_demo(emit_json=True) + after = _tg._CORPUS_PATH.read_bytes() if _tg._CORPUS_PATH.exists() else b"" + assert before == after + + +def test_demo_json_shape_is_stable(demo_report: dict) -> None: + """Stable JSON contract for downstream consumers.""" + assert set(demo_report.keys()) == {"scenes", "all_claims_supported"} + for scene in demo_report["scenes"]: + assert set(scene.keys()) == {"scene", "claim", "detail"}