From 5ed9fbb8e79ad4b06e0e392ec846b9d0e320ad5e Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 5 Jun 2026 13:17:30 -0700 Subject: [PATCH] =?UTF-8?q?feat(persistence):=20Shape=20B+=20Phase=20D+E?= =?UTF-8?q?=20=E2=80=94=20opt-in=20lived-state=20persistence;=20reboot=20t?= =?UTF-8?q?ransparent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The load-bearing L10 milestone: with resume mode enabled, a reboot resumes the SAME life. Wires SessionContext.snapshot/restore (Phases A-C) into the engine-state checkpoint and flips the L10 spike's P2b oracle to transparent. Persistence is OPT-IN (RuntimeConfig.persist_session_state, default False): it is a deliberate always-on-runtime mode, and per-turn snapshotting has an O(turns) cost, so demos / evals / one-shot runtimes do NOT pay for resume they don't use. This keeps every existing ChatRuntime byte-for-byte unchanged (no perf tax, no pinned-lane SHA drift, no test breakage); only the L10 continuity lane and the production L10 process enable it. Phase D (wiring): - core/config.py: persist_session_state flag (default False). - engine_state/__init__.py: bump _SCHEMA_VERSION 1->2; add save_session_state / load_session_state (atomic, ADR-0156). v1 checkpoints still load (1 <= 2) with no session_state -> fresh session. - chat/runtime.py: when persist_session_state, checkpoint_engine_state saves the session snapshot BEFORE the manifest (manifest = the commit marker / WAL force boundary); _load_engine_state restores it into self._context. Phase E (flip the oracle): - evals/l10_continuity/runner.py: the continuity lane forces persist on (it IS the resume-mode lane). - tests/test_l10_continuity.py: test_p2b_documents_current_resume_gap -> test_p2b_reboot_is_transparent (asserts post_reboot_transparent, divergence None). predicates.py / runner.py / contract.md: P2b is now the resume-as-same-life guard. - tests/test_adr_0146_engine_state.py: manifest schema_version 1 -> 2. Validation: full spike (P1 closure, P2a determinism, P2b NOW TRANSPARENT, P3 bounded, P4 crash-recovery determinism + commit point, P5b/P5c) + reboot-restores-lived-state + v1 back-compat + ADR-0146, all green; [run K -> reboot -> run M] byte-identical to [run K+M]. With persistence off (default), the curated smoke + showcase budget + pinned lanes are unchanged. Closes the A->E Shape B+ scope (docs/analysis/L10-shapeBplus-persistence-scope-2026-06-05.md). --- chat/runtime.py | 16 +++ core/config.py | 8 ++ engine_state/__init__.py | 30 ++++- evals/l10_continuity/contract.md | 23 ++-- evals/l10_continuity/predicates.py | 14 ++- evals/l10_continuity/runner.py | 26 +++-- tests/test_adr_0146_engine_state.py | 2 +- .../test_engine_state_session_persistence.py | 108 ++++++++++++++++++ tests/test_l10_continuity.py | 26 +++-- 9 files changed, 214 insertions(+), 39 deletions(-) create mode 100644 tests/test_engine_state_session_persistence.py diff --git a/chat/runtime.py b/chat/runtime.py index 46abec71..a3097561 100644 --- a/chat/runtime.py +++ b/chat/runtime.py @@ -689,6 +689,15 @@ class ChatRuntime: self._recognizer_registry = RecognizerRegistry.from_recognizers(recognizers) self._pending_candidates = store.load_discovery_candidates() self._turn_count = int(manifest.get("turn_count", 0)) + # Shape B+ (schema v2): restore the lived session state into the live + # context so a reboot resumes the SAME life (field/vault/anchor/graph/ + # referents/dialogue). Opt-in (config.persist_session_state); None for a + # v1 checkpoint -> fresh session (the historical Shape B behavior), so old + # checkpoints stay loadable. + if self.config.persist_session_state and self._context is not None: + session_snapshot = store.load_session_state() + if session_snapshot is not None: + self._context.restore(session_snapshot) # W-024 / ADR-0158 — buffer reboot event for emission when sink attaches. from engine_state import _git_revision self._pending_reboot_payload = format_reboot_event_jsonl( @@ -722,6 +731,13 @@ class ChatRuntime: for c in candidates_to_save ] store.save_discovery_candidates(candidates_to_save) + # Shape B+ (schema v2): persist the lived session state (field, vault, + # anchor, graph, referents, dialogue) BEFORE the manifest, so the + # manifest stays the last durable act — the commit marker for the turn. + # Opt-in (config.persist_session_state): a deliberate resume mode, off by + # default so one-shot runtimes don't pay the per-turn snapshot cost. + if self._context is not None and self.config.persist_session_state: + store.save_session_state(self._context.snapshot()) store.save_manifest(self._turn_count) def record_recognition_example( diff --git a/core/config.py b/core/config.py index 48f6160e..ccef7235 100644 --- a/core/config.py +++ b/core/config.py @@ -276,6 +276,14 @@ class RuntimeConfig: # ADR-0151 — generate TeachingChainProposals from enriched candidates on load. auto_proposal_enabled: bool = False + # Shape B+ (L10 resume-as-same-life) — persist and restore the FULL lived + # session state (field, vault, anchor, graph, referents, dialogue) across + # reboot, not just recognizers/candidates (Shape B). OFF by default: it is a + # deliberate always-on-runtime mode, and per-turn snapshotting has an O(turns) + # cost, so demos/evals/one-shot runtimes must not pay for resume they don't + # use. Enabled by the L10 continuity lane and the production L10 process. + persist_session_state: bool = False + DEFAULT_IDENTITY_PACK: str = "default_general_v1" DEFAULT_ETHICS_PACK: str = "default_general_ethics_v1" diff --git a/engine_state/__init__.py b/engine_state/__init__.py index 66fcbb3a..3936a388 100644 --- a/engine_state/__init__.py +++ b/engine_state/__init__.py @@ -71,7 +71,7 @@ def _atomic_write_text(target: Path, content: str, *, encoding: str = "utf-8") - pass raise -_SCHEMA_VERSION = 1 +_SCHEMA_VERSION = 2 # v2 adds session_state.json (Shape B+ lived-state persistence) _DEFAULT_DIR = ( Path(os.environ["CORE_ENGINE_STATE_DIR"]) if os.environ.get("CORE_ENGINE_STATE_DIR") @@ -203,6 +203,34 @@ class EngineStateStore: ) return manifest + def save_session_state(self, snapshot: dict) -> None: + """Persist the lived session state (Shape B+ / schema v2). + + ``snapshot`` is ``SessionContext.snapshot()`` — a bit-exact, JSON-safe + dict of the field, vault, anchor, graph, referents, and dialogue. Written + atomically (ADR-0156). Save this BEFORE ``save_manifest`` so the manifest + stays the last durable act of a checkpoint (the commit marker). + """ + _atomic_write_text( + self.path / "session_state.json", + json.dumps(snapshot, sort_keys=True, separators=(",", ":")), + ) + + def load_session_state(self) -> dict | None: + """Load the lived session state, or None when absent (v1 checkpoint). + + A v1 checkpoint has no ``session_state.json`` — returning None lets the + caller fall back to a fresh session (today's Shape B behavior), so old + checkpoints stay loadable. + """ + p = self.path / "session_state.json" + if not p.exists(): + return None + content = p.read_text(encoding="utf-8").strip() + if not content: + return None + return json.loads(content) + def exists(self) -> bool: return (self.path / "manifest.json").exists() diff --git a/evals/l10_continuity/contract.md b/evals/l10_continuity/contract.md index 1722afb4..7d7f2fc9 100644 --- a/evals/l10_continuity/contract.md +++ b/evals/l10_continuity/contract.md @@ -37,17 +37,20 @@ fail under the violation it nominally catches is decoration, not proof. semantics (the raw recall score is not a clean similarity). Recorded as `not_covered` in the report; a follow-up increment. -## The headline diagnostic (P2b) +## The headline result (P2b) — resume-as-same-life -Today a reboot restores only recognizers / discovery candidates / `turn_count` -(Shape B, ADR-0146) and discards the lived field / vault / anchor / graph / -referents. So `post_reboot_transparent == False`: a reboot diverges from the -uninterrupted run **at the first post-reboot turn**. This is the mechanical -proof of "many lives sharing a checkpoint" and the precise definition of the -Shape-B+ persistence work that closes resume-as-same-life. The -`test_p2b_documents_current_resume_gap` test pins this reality and will **flip** -if persistence is later added — forcing a doc update so the gap cannot close -silently. +A reboot is now **fully transparent**: `post_reboot_transparent == True`, +`first_divergence is None`. With Shape B+ persistence wired +(`SessionContext.snapshot/restore` → engine_state schema v2), the lived field / +vault / anchor / graph / referents / dialogue survive a reboot, so +`[run K → reboot → run M]` is byte-identical to the uninterrupted `[run K+M]`. +`test_p2b_reboot_is_transparent` is the load-bearing guard. + +This **flipped** from the original Shape B (ADR-0146) behavior, where only +recognizers / discovery candidates / `turn_count` survived and the lived +field/vault were discarded — `post_reboot_transparent == False`, divergence at +the first post-reboot turn ("many lives sharing a checkpoint"). The spike +measured that gap, defined the persistence work, and now confirms it closed. ## Thresholds (empirical basis, not arbitrary) diff --git a/evals/l10_continuity/predicates.py b/evals/l10_continuity/predicates.py index 5774d004..009e19cf 100644 --- a/evals/l10_continuity/predicates.py +++ b/evals/l10_continuity/predicates.py @@ -116,12 +116,14 @@ def evaluate_p2b_reboot_transparency( ) -> tuple[PredicateOutcome, RebootTransparency]: """P2b — locate where a rebooted run diverges from an uninterrupted one. - The predicate PASSES on the structural invariant only: a reboot must not - change any turn *before* the reboot point (those are the same first - segment, so they must be identical — a failure here is a real determinism - or state-leak bug). Full post-reboot transparency is the *measured* - diagnostic, returned alongside; it is expected to be ``False`` until the - lived field/vault are persisted across reboot (the Shape-B+ work). + The predicate PASSES on the structural invariant: a reboot must not change + any turn *before* the reboot point (those are the same first segment, so they + must be identical — a failure here is a real determinism or state-leak bug). + Full post-reboot transparency is returned alongside as the *measured* + headline. With Shape B+ persistence wired (SessionContext.snapshot/restore -> + engine_state schema v2), it is now ``True`` — a reboot is byte-identical to + no reboot. It was ``False`` under Shape B (field/vault discarded); the flip is + the resume-as-same-life proof. """ if not rebooted.reboot_at: raise ValueError("P2b expects a rebooted run (reboot_at non-empty).") diff --git a/evals/l10_continuity/runner.py b/evals/l10_continuity/runner.py index 9cb90515..69e2923d 100644 --- a/evals/l10_continuity/runner.py +++ b/evals/l10_continuity/runner.py @@ -15,18 +15,21 @@ and returns it. It makes NO pass/fail judgement — that is ``predicates.py`` it never repairs, normalizes, or mutates field state (it only reads what the real pipeline produced). -What a reboot restores (today, Shape B / ADR-0146): recognizers, discovery -candidates, and ``turn_count`` — and NOTHING else. The lived field, vault, -session graph, referents, and session anchor are process-local and discarded on -exit. The ``booted_segment`` tag on each record exists precisely so the -reboot-transparency predicate (P2b) can locate where a rebooted run diverges -from an uninterrupted one. +What a reboot restores (Shape B+ / engine_state schema v2): recognizers, +discovery candidates, ``turn_count``, AND the full lived session state — field, +vault, session graph, referents, session anchor, and dialogue — via +``SessionContext.snapshot/restore``. So a reboot now resumes the SAME life and +P2b is transparent. (Under the original Shape B / ADR-0146 only the first three +survived and the lived field/vault were discarded — "many lives sharing a +checkpoint".) The ``booted_segment`` tag on each record lets the +reboot-transparency predicate (P2b) confirm a rebooted run is byte-identical to +an uninterrupted one. """ from __future__ import annotations import resource -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path import numpy as np @@ -91,10 +94,13 @@ def _new_runtime(config: RuntimeConfig, engine_state_dir: Path) -> ChatRuntime: """Construct a ChatRuntime bound to the checkpoint dir. Reconstruction is the reboot: ``ChatRuntime.__init__`` loads the on-disk - engine-state checkpoint when one exists (recognizers / candidates / - turn_count), so a second instance over the same directory resumes from the - last durable checkpoint. + engine-state checkpoint when one exists, so a second instance over the same + directory resumes from the last durable checkpoint. The continuity lane is + the resume-mode lane by definition, so it forces ``persist_session_state`` on + (the full lived field/vault/anchor/graph survive reboot — what P2b measures). """ + if not config.persist_session_state: + config = replace(config, persist_session_state=True) return ChatRuntime(config=config, engine_state_path=engine_state_dir) diff --git a/tests/test_adr_0146_engine_state.py b/tests/test_adr_0146_engine_state.py index 65bb6c28..a5b0c4fd 100644 --- a/tests/test_adr_0146_engine_state.py +++ b/tests/test_adr_0146_engine_state.py @@ -85,7 +85,7 @@ def test_engine_state_store_manifest_written(tmp_path) -> None: store.save_manifest(turn_count=7) manifest = json.loads((tmp_path / "manifest.json").read_text(encoding="utf-8")) - assert manifest["schema_version"] == 1 + assert manifest["schema_version"] == 2 # Shape B+ (v2 adds session_state.json) assert manifest["turn_count"] == 7 assert isinstance(manifest["written_at_revision"], str) diff --git a/tests/test_engine_state_session_persistence.py b/tests/test_engine_state_session_persistence.py new file mode 100644 index 00000000..719acd8b --- /dev/null +++ b/tests/test_engine_state_session_persistence.py @@ -0,0 +1,108 @@ +"""Shape B+ Phase D — engine_state session-state I/O + ChatRuntime wiring. + +The payoff test (test_chat_runtime_restores_lived_state_across_reboot) is a +direct preview of the L10 spike's P2b: after Phase D, a reboot restores the lived +field/vault/anchor/graph — so resuming continues the same life, not a fresh one +sharing only recognizers/candidates. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from chat.runtime import ChatRuntime +from core.cognition.pipeline import CognitiveTurnPipeline +from core.config import RuntimeConfig +from engine_state import EngineStateStore + +_PROMPTS = ("What causes light?", "What is a concept?", "What causes rain?", "Hello.") +# Shape B+ persistence is opt-in; the resume-mode config for these tests. +_PERSIST = RuntimeConfig(persist_session_state=True) + + +# --------------------------------------------------------------------------- # +# EngineStateStore session-state I/O # +# --------------------------------------------------------------------------- # +def test_save_load_session_state_round_trips(tmp_path: Path) -> None: + store = EngineStateStore(tmp_path) + snap = {"turn": 3, "state": None, "vault": {"versors": [], "metadata": []}} + store.save_session_state(snap) + assert store.load_session_state() == snap + + +def test_load_session_state_is_none_when_absent(tmp_path: Path) -> None: + assert EngineStateStore(tmp_path).load_session_state() is None + + +def test_schema_version_is_2(tmp_path: Path) -> None: + store = EngineStateStore(tmp_path) + store.save_manifest(turn_count=5) + assert store.load_manifest()["schema_version"] == 2 + + +def test_v1_checkpoint_still_loads_without_session_state(tmp_path: Path) -> None: + # A v1 checkpoint: manifest with schema_version 1 and NO session_state.json. + (tmp_path / "manifest.json").write_text( + json.dumps({"schema_version": 1, "turn_count": 4, "written_at_revision": "x"}), + encoding="utf-8", + ) + store = EngineStateStore(tmp_path) + assert store.load_manifest() is not None # tolerated (1 <= 2) + assert store.load_session_state() is None # -> fresh session fallback + + +# --------------------------------------------------------------------------- # +# ChatRuntime reboot restores the lived state (the Phase D payoff) # +# --------------------------------------------------------------------------- # +def _drive(state_dir: Path) -> ChatRuntime: + runtime = ChatRuntime(config=_PERSIST, engine_state_path=state_dir) + pipe = CognitiveTurnPipeline(runtime=runtime) + for p in _PROMPTS: + pipe.run(p) + return runtime + + +def test_chat_runtime_restores_lived_state_across_reboot(tmp_path: Path) -> None: + state_dir = tmp_path / "es" + rt_a = _drive(state_dir) + ctx_a = rt_a._context + + # Reboot: a fresh runtime over the same checkpoint dir (resume mode on). + rt_b = ChatRuntime(config=_PERSIST, engine_state_path=state_dir) + ctx_b = rt_b._context + + # The lived field is restored bit-exactly (NOT fresh/None). + assert ctx_a.state is not None + assert ctx_b.state is not None + assert ctx_b.state.F.tobytes() == ctx_a.state.F.tobytes() + assert ctx_b.turn == ctx_a.turn + assert ctx_b.turn > 0 # we actually accumulated turns + + # Anchor restored. + if ctx_a._anchor_field is not None: + assert ctx_b._anchor_field is not None + assert ctx_b._anchor_field.tobytes() == ctx_a._anchor_field.tobytes() + + # Vault is restored (not empty) and recalls identically. + assert len(ctx_b.vault) == len(ctx_a.vault) > 0 + q = ctx_a.state.F + assert [(r["index"], r["score"]) for r in ctx_b.vault.recall(q, 5)] == [ + (r["index"], r["score"]) for r in ctx_a.vault.recall(q, 5) + ] + + # Graph restored. + assert len(ctx_b.graph) == len(ctx_a.graph) > 0 + + +def test_no_load_state_runtime_starts_fresh(tmp_path: Path) -> None: + # no_load_state=True must NOT restore even if a checkpoint exists. + state_dir = tmp_path / "es" + _drive(state_dir) + fresh = ChatRuntime( + config=_PERSIST, engine_state_path=state_dir, no_load_state=True + ) + assert fresh._context.state is None + assert fresh._context.turn == 0 diff --git a/tests/test_l10_continuity.py b/tests/test_l10_continuity.py index 62f9dce3..2b40a5e8 100644 --- a/tests/test_l10_continuity.py +++ b/tests/test_l10_continuity.py @@ -140,24 +140,28 @@ def test_p2b_pre_reboot_invariant_holds_on_real_soak(tmp_path: Path) -> None: assert transparency.first_divergence >= reboot_turn -def test_p2b_documents_current_resume_gap(tmp_path: Path) -> None: - """Diagnostic: TODAY (Shape B, field/vault not persisted) a reboot is NOT - transparent — the first post-reboot turn diverges because the lived field - and vault were discarded. This test pins that empirical reality so that if - persistence is later built, it flips and we are forced to update the claim. +def test_p2b_reboot_is_transparent(tmp_path: Path) -> None: + """Resume-as-same-life: a reboot is FULLY transparent. + + With Shape B+ persistence wired (SessionContext.snapshot/restore -> + engine_state schema v2), the lived field/vault/anchor/graph/referents survive + a reboot, so [run K -> reboot -> run M] is byte-identical to the + uninterrupted [run K+M]. This is the load-bearing L10 proof — it FLIPPED from + the Shape-B 'many lives sharing a checkpoint' gap the moment persistence + landed. If this regresses to non-transparent, resume-as-same-life broke. """ reboot_turn = 3 rebooted = run_soak( _SOAK_N, engine_state_dir=tmp_path / "r", reboot_at=(reboot_turn,) ) baseline = run_soak(_SOAK_N, engine_state_dir=tmp_path / "base") - _, transparency = evaluate_p2b_reboot_transparency(rebooted, baseline) - assert not transparency.post_reboot_transparent, ( - "Reboot transparency is unexpected under Shape B: if this fails, the " - "lived field/vault are now surviving reboot — update the L10 spike doc " - "and this assertion to assert transparency." + outcome, transparency = evaluate_p2b_reboot_transparency(rebooted, baseline) + assert outcome.passed, outcome.detail + assert transparency.post_reboot_transparent, ( + "Reboot is no longer transparent — the lived field/vault stopped " + "surviving reboot. Resume-as-same-life regressed." ) - assert transparency.first_divergence == reboot_turn + assert transparency.first_divergence is None def test_p2b_bites_on_pre_reboot_divergence() -> None: