Merge pull request #571 from AssetOverflow/feat/l10-shapeBplus-phaseD

feat(persistence): Shape B+ Phase D+E — reboot is transparent (resume-as-same-life)

chat/runtime.py

@@ -689,6 +689,15 @@ class ChatRuntime:
         self._recognizer_registry = RecognizerRegistry.from_recognizers(recognizers)
         self._pending_candidates = store.load_discovery_candidates()
         self._turn_count = int(manifest.get("turn_count", 0))
+        # Shape B+ (schema v2): restore the lived session state into the live
+        # context so a reboot resumes the SAME life (field/vault/anchor/graph/
+        # referents/dialogue). Opt-in (config.persist_session_state); None for a
+        # v1 checkpoint -> fresh session (the historical Shape B behavior), so old
+        # checkpoints stay loadable.
+        if self.config.persist_session_state and self._context is not None:
+            session_snapshot = store.load_session_state()
+            if session_snapshot is not None:
+                self._context.restore(session_snapshot)
         # W-024 / ADR-0158 — buffer reboot event for emission when sink attaches.
         from engine_state import _git_revision
         self._pending_reboot_payload = format_reboot_event_jsonl(
@@ -722,6 +731,13 @@ class ChatRuntime:
                 for c in candidates_to_save
             ]
         store.save_discovery_candidates(candidates_to_save)
+        # Shape B+ (schema v2): persist the lived session state (field, vault,
+        # anchor, graph, referents, dialogue) BEFORE the manifest, so the
+        # manifest stays the last durable act — the commit marker for the turn.
+        # Opt-in (config.persist_session_state): a deliberate resume mode, off by
+        # default so one-shot runtimes don't pay the per-turn snapshot cost.
+        if self._context is not None and self.config.persist_session_state:
+            store.save_session_state(self._context.snapshot())
         store.save_manifest(self._turn_count)
 
     def record_recognition_example(

core/config.py

@@ -276,6 +276,14 @@ class RuntimeConfig:
     # ADR-0151 — generate TeachingChainProposals from enriched candidates on load.
     auto_proposal_enabled: bool = False
 
+    # Shape B+ (L10 resume-as-same-life) — persist and restore the FULL lived
+    # session state (field, vault, anchor, graph, referents, dialogue) across
+    # reboot, not just recognizers/candidates (Shape B). OFF by default: it is a
+    # deliberate always-on-runtime mode, and per-turn snapshotting has an O(turns)
+    # cost, so demos/evals/one-shot runtimes must not pay for resume they don't
+    # use. Enabled by the L10 continuity lane and the production L10 process.
+    persist_session_state: bool = False
+
 
 DEFAULT_IDENTITY_PACK: str = "default_general_v1"
 DEFAULT_ETHICS_PACK: str = "default_general_ethics_v1"

engine_state/__init__.py

@@ -71,7 +71,7 @@ def _atomic_write_text(target: Path, content: str, *, encoding: str = "utf-8") -
                 pass
         raise
 
-_SCHEMA_VERSION = 1
+_SCHEMA_VERSION = 2  # v2 adds session_state.json (Shape B+ lived-state persistence)
 _DEFAULT_DIR = (
     Path(os.environ["CORE_ENGINE_STATE_DIR"])
     if os.environ.get("CORE_ENGINE_STATE_DIR")
@@ -203,6 +203,34 @@ class EngineStateStore:
             )
         return manifest
 
+    def save_session_state(self, snapshot: dict) -> None:
+        """Persist the lived session state (Shape B+ / schema v2).
+
+        ``snapshot`` is ``SessionContext.snapshot()`` — a bit-exact, JSON-safe
+        dict of the field, vault, anchor, graph, referents, and dialogue. Written
+        atomically (ADR-0156). Save this BEFORE ``save_manifest`` so the manifest
+        stays the last durable act of a checkpoint (the commit marker).
+        """
+        _atomic_write_text(
+            self.path / "session_state.json",
+            json.dumps(snapshot, sort_keys=True, separators=(",", ":")),
+        )
+
+    def load_session_state(self) -> dict | None:
+        """Load the lived session state, or None when absent (v1 checkpoint).
+
+        A v1 checkpoint has no ``session_state.json`` — returning None lets the
+        caller fall back to a fresh session (today's Shape B behavior), so old
+        checkpoints stay loadable.
+        """
+        p = self.path / "session_state.json"
+        if not p.exists():
+            return None
+        content = p.read_text(encoding="utf-8").strip()
+        if not content:
+            return None
+        return json.loads(content)
+
     def exists(self) -> bool:
         return (self.path / "manifest.json").exists()
 

evals/l10_continuity/contract.md

@@ -37,17 +37,20 @@ fail under the violation it nominally catches is decoration, not proof.
   semantics (the raw recall score is not a clean similarity). Recorded as
   `not_covered` in the report; a follow-up increment.
 
-## The headline diagnostic (P2b)
+## The headline result (P2b) — resume-as-same-life
 
-Today a reboot restores only recognizers / discovery candidates / `turn_count`
-(Shape B, ADR-0146) and discards the lived field / vault / anchor / graph /
-referents. So `post_reboot_transparent == False`: a reboot diverges from the
-uninterrupted run **at the first post-reboot turn**. This is the mechanical
-proof of "many lives sharing a checkpoint" and the precise definition of the
-Shape-B+ persistence work that closes resume-as-same-life. The
-`test_p2b_documents_current_resume_gap` test pins this reality and will **flip**
-if persistence is later added — forcing a doc update so the gap cannot close
-silently.
+A reboot is now **fully transparent**: `post_reboot_transparent == True`,
+`first_divergence is None`. With Shape B+ persistence wired
+(`SessionContext.snapshot/restore` → engine_state schema v2), the lived field /
+vault / anchor / graph / referents / dialogue survive a reboot, so
+`[run K → reboot → run M]` is byte-identical to the uninterrupted `[run K+M]`.
+`test_p2b_reboot_is_transparent` is the load-bearing guard.
+
+This **flipped** from the original Shape B (ADR-0146) behavior, where only
+recognizers / discovery candidates / `turn_count` survived and the lived
+field/vault were discarded — `post_reboot_transparent == False`, divergence at
+the first post-reboot turn ("many lives sharing a checkpoint"). The spike
+measured that gap, defined the persistence work, and now confirms it closed.
 
 ## Thresholds (empirical basis, not arbitrary)
 

evals/l10_continuity/predicates.py

@@ -116,12 +116,14 @@ def evaluate_p2b_reboot_transparency(
 ) -> tuple[PredicateOutcome, RebootTransparency]:
     """P2b — locate where a rebooted run diverges from an uninterrupted one.
 
-    The predicate PASSES on the structural invariant only: a reboot must not
-    change any turn *before* the reboot point (those are the same first
-    segment, so they must be identical — a failure here is a real determinism
-    or state-leak bug). Full post-reboot transparency is the *measured*
-    diagnostic, returned alongside; it is expected to be ``False`` until the
-    lived field/vault are persisted across reboot (the Shape-B+ work).
+    The predicate PASSES on the structural invariant: a reboot must not change
+    any turn *before* the reboot point (those are the same first segment, so they
+    must be identical — a failure here is a real determinism or state-leak bug).
+    Full post-reboot transparency is returned alongside as the *measured*
+    headline. With Shape B+ persistence wired (SessionContext.snapshot/restore ->
+    engine_state schema v2), it is now ``True`` — a reboot is byte-identical to
+    no reboot. It was ``False`` under Shape B (field/vault discarded); the flip is
+    the resume-as-same-life proof.
     """
     if not rebooted.reboot_at:
         raise ValueError("P2b expects a rebooted run (reboot_at non-empty).")

evals/l10_continuity/runner.py

@@ -15,18 +15,21 @@ and returns it. It makes NO pass/fail judgement — that is ``predicates.py``
 it never repairs, normalizes, or mutates field state (it only reads what the real
 pipeline produced).
 
-What a reboot restores (today, Shape B / ADR-0146): recognizers, discovery
-candidates, and ``turn_count`` — and NOTHING else. The lived field, vault,
-session graph, referents, and session anchor are process-local and discarded on
-exit. The ``booted_segment`` tag on each record exists precisely so the
-reboot-transparency predicate (P2b) can locate where a rebooted run diverges
-from an uninterrupted one.
+What a reboot restores (Shape B+ / engine_state schema v2): recognizers,
+discovery candidates, ``turn_count``, AND the full lived session state — field,
+vault, session graph, referents, session anchor, and dialogue — via
+``SessionContext.snapshot/restore``. So a reboot now resumes the SAME life and
+P2b is transparent. (Under the original Shape B / ADR-0146 only the first three
+survived and the lived field/vault were discarded — "many lives sharing a
+checkpoint".) The ``booted_segment`` tag on each record lets the
+reboot-transparency predicate (P2b) confirm a rebooted run is byte-identical to
+an uninterrupted one.
 """
 
 from __future__ import annotations
 
 import resource
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from pathlib import Path
 
 import numpy as np
@@ -91,10 +94,13 @@ def _new_runtime(config: RuntimeConfig, engine_state_dir: Path) -> ChatRuntime:
     """Construct a ChatRuntime bound to the checkpoint dir.
 
     Reconstruction is the reboot: ``ChatRuntime.__init__`` loads the on-disk
-    engine-state checkpoint when one exists (recognizers / candidates /
-    turn_count), so a second instance over the same directory resumes from the
-    last durable checkpoint.
+    engine-state checkpoint when one exists, so a second instance over the same
+    directory resumes from the last durable checkpoint. The continuity lane is
+    the resume-mode lane by definition, so it forces ``persist_session_state`` on
+    (the full lived field/vault/anchor/graph survive reboot — what P2b measures).
     """
+    if not config.persist_session_state:
+        config = replace(config, persist_session_state=True)
     return ChatRuntime(config=config, engine_state_path=engine_state_dir)
 
 

tests/test_adr_0146_engine_state.py

@@ -85,7 +85,7 @@ def test_engine_state_store_manifest_written(tmp_path) -> None:
     store.save_manifest(turn_count=7)
 
     manifest = json.loads((tmp_path / "manifest.json").read_text(encoding="utf-8"))
-    assert manifest["schema_version"] == 1
+    assert manifest["schema_version"] == 2  # Shape B+ (v2 adds session_state.json)
     assert manifest["turn_count"] == 7
     assert isinstance(manifest["written_at_revision"], str)
 

tests/test_engine_state_session_persistence.py

@@ -0,0 +1,108 @@
+"""Shape B+ Phase D — engine_state session-state I/O + ChatRuntime wiring.
+
+The payoff test (test_chat_runtime_restores_lived_state_across_reboot) is a
+direct preview of the L10 spike's P2b: after Phase D, a reboot restores the lived
+field/vault/anchor/graph — so resuming continues the same life, not a fresh one
+sharing only recognizers/candidates.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from chat.runtime import ChatRuntime
+from core.cognition.pipeline import CognitiveTurnPipeline
+from core.config import RuntimeConfig
+from engine_state import EngineStateStore
+
+_PROMPTS = ("What causes light?", "What is a concept?", "What causes rain?", "Hello.")
+# Shape B+ persistence is opt-in; the resume-mode config for these tests.
+_PERSIST = RuntimeConfig(persist_session_state=True)
+
+
+# --------------------------------------------------------------------------- #
+# EngineStateStore session-state I/O                                            #
+# --------------------------------------------------------------------------- #
+def test_save_load_session_state_round_trips(tmp_path: Path) -> None:
+    store = EngineStateStore(tmp_path)
+    snap = {"turn": 3, "state": None, "vault": {"versors": [], "metadata": []}}
+    store.save_session_state(snap)
+    assert store.load_session_state() == snap
+
+
+def test_load_session_state_is_none_when_absent(tmp_path: Path) -> None:
+    assert EngineStateStore(tmp_path).load_session_state() is None
+
+
+def test_schema_version_is_2(tmp_path: Path) -> None:
+    store = EngineStateStore(tmp_path)
+    store.save_manifest(turn_count=5)
+    assert store.load_manifest()["schema_version"] == 2
+
+
+def test_v1_checkpoint_still_loads_without_session_state(tmp_path: Path) -> None:
+    # A v1 checkpoint: manifest with schema_version 1 and NO session_state.json.
+    (tmp_path / "manifest.json").write_text(
+        json.dumps({"schema_version": 1, "turn_count": 4, "written_at_revision": "x"}),
+        encoding="utf-8",
+    )
+    store = EngineStateStore(tmp_path)
+    assert store.load_manifest() is not None  # tolerated (1 <= 2)
+    assert store.load_session_state() is None  # -> fresh session fallback
+
+
+# --------------------------------------------------------------------------- #
+# ChatRuntime reboot restores the lived state (the Phase D payoff)              #
+# --------------------------------------------------------------------------- #
+def _drive(state_dir: Path) -> ChatRuntime:
+    runtime = ChatRuntime(config=_PERSIST, engine_state_path=state_dir)
+    pipe = CognitiveTurnPipeline(runtime=runtime)
+    for p in _PROMPTS:
+        pipe.run(p)
+    return runtime
+
+
+def test_chat_runtime_restores_lived_state_across_reboot(tmp_path: Path) -> None:
+    state_dir = tmp_path / "es"
+    rt_a = _drive(state_dir)
+    ctx_a = rt_a._context
+
+    # Reboot: a fresh runtime over the same checkpoint dir (resume mode on).
+    rt_b = ChatRuntime(config=_PERSIST, engine_state_path=state_dir)
+    ctx_b = rt_b._context
+
+    # The lived field is restored bit-exactly (NOT fresh/None).
+    assert ctx_a.state is not None
+    assert ctx_b.state is not None
+    assert ctx_b.state.F.tobytes() == ctx_a.state.F.tobytes()
+    assert ctx_b.turn == ctx_a.turn
+    assert ctx_b.turn > 0  # we actually accumulated turns
+
+    # Anchor restored.
+    if ctx_a._anchor_field is not None:
+        assert ctx_b._anchor_field is not None
+        assert ctx_b._anchor_field.tobytes() == ctx_a._anchor_field.tobytes()
+
+    # Vault is restored (not empty) and recalls identically.
+    assert len(ctx_b.vault) == len(ctx_a.vault) > 0
+    q = ctx_a.state.F
+    assert [(r["index"], r["score"]) for r in ctx_b.vault.recall(q, 5)] == [
+        (r["index"], r["score"]) for r in ctx_a.vault.recall(q, 5)
+    ]
+
+    # Graph restored.
+    assert len(ctx_b.graph) == len(ctx_a.graph) > 0
+
+
+def test_no_load_state_runtime_starts_fresh(tmp_path: Path) -> None:
+    # no_load_state=True must NOT restore even if a checkpoint exists.
+    state_dir = tmp_path / "es"
+    _drive(state_dir)
+    fresh = ChatRuntime(
+        config=_PERSIST, engine_state_path=state_dir, no_load_state=True
+    )
+    assert fresh._context.state is None
+    assert fresh._context.turn == 0

tests/test_l10_continuity.py

@@ -140,24 +140,28 @@ def test_p2b_pre_reboot_invariant_holds_on_real_soak(tmp_path: Path) -> None:
         assert transparency.first_divergence >= reboot_turn
 
 
-def test_p2b_documents_current_resume_gap(tmp_path: Path) -> None:
-    """Diagnostic: TODAY (Shape B, field/vault not persisted) a reboot is NOT
-    transparent — the first post-reboot turn diverges because the lived field
-    and vault were discarded. This test pins that empirical reality so that if
-    persistence is later built, it flips and we are forced to update the claim.
+def test_p2b_reboot_is_transparent(tmp_path: Path) -> None:
+    """Resume-as-same-life: a reboot is FULLY transparent.
+
+    With Shape B+ persistence wired (SessionContext.snapshot/restore ->
+    engine_state schema v2), the lived field/vault/anchor/graph/referents survive
+    a reboot, so [run K -> reboot -> run M] is byte-identical to the
+    uninterrupted [run K+M]. This is the load-bearing L10 proof — it FLIPPED from
+    the Shape-B 'many lives sharing a checkpoint' gap the moment persistence
+    landed. If this regresses to non-transparent, resume-as-same-life broke.
     """
     reboot_turn = 3
     rebooted = run_soak(
         _SOAK_N, engine_state_dir=tmp_path / "r", reboot_at=(reboot_turn,)
     )
     baseline = run_soak(_SOAK_N, engine_state_dir=tmp_path / "base")
-    _, transparency = evaluate_p2b_reboot_transparency(rebooted, baseline)
-    assert not transparency.post_reboot_transparent, (
-        "Reboot transparency is unexpected under Shape B: if this fails, the "
-        "lived field/vault are now surviving reboot — update the L10 spike doc "
-        "and this assertion to assert transparency."
+    outcome, transparency = evaluate_p2b_reboot_transparency(rebooted, baseline)
+    assert outcome.passed, outcome.detail
+    assert transparency.post_reboot_transparent, (
+        "Reboot is no longer transparent — the lived field/vault stopped "
+        "surviving reboot. Resume-as-same-life regressed."
     )
-    assert transparency.first_divergence == reboot_turn
+    assert transparency.first_divergence is None
 
 
 def test_p2b_bites_on_pre_reboot_divergence() -> None: