From 5feedcebd9a927618fccb1601c1fcb186d36cd93 Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 5 Jun 2026 12:13:46 -0700 Subject: [PATCH] =?UTF-8?q?feat(persistence):=20Shape=20B+=20Phase=20C=20?= =?UTF-8?q?=E2=80=94=20SessionContext.snapshot/restore=20(full=20lived=20s?= =?UTF-8?q?tate)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Composes the FieldState (A) and VaultStore (B) codecs with new codecs for SessionGraph/TurnNode, ReferentRegistry/ReferentEntry, Proposition, and DialogueTurn into SessionContext.snapshot()/restore() — the complete lived session state that must survive reboot for resume-as-same-life. - session/graph.py: TurnNode + SessionGraph to_dict/from_dict (versors bit-exact). - session/referents.py: ReferentEntry + ReferentRegistry, preserving the _slots<->_history object aliasing via slot->history-index (update_turn_versor relies on `is` identity). - generate/proposition.py + generate/dialogue.py: Proposition + DialogueTurn codecs (relation_norm is derived in __post_init__, not persisted). - vault/store.py: complete the metadata codec — vault metadata can hold a Proposition ({"kind":"proposition",...} from generate/proposition.py), tagged on encode and reconstructed on decode (lazy import, cycle-free). This closes a gap Phase B assumed away ("metadata is primitives only"); surfaced by the Phase C JSON-safe integration test. - session/context.py: snapshot()/restore(). vocab/persona are NOT serialized (shared, supplied at restore); restore() mutates self by design (a load). Exit gate: a real 4-turn session, snapshotted and restored into a fresh context, is field-equal — field bit-exact, vault recall identical, graph/referents/ dialogue preserved (incl. the referent aliasing). 9 new tests; INV-02 + session-coherence regression green (68 passed). Part of the A->E Shape B+ scope (Phase C). --- generate/dialogue.py | 14 +++ generate/proposition.py | 35 +++++++ session/context.py | 63 ++++++++++++ session/graph.py | 38 ++++++- session/referents.py | 55 +++++++++- tests/test_session_context_codec.py | 152 ++++++++++++++++++++++++++++ tests/test_vaultstore_codec.py | 22 ++++ vault/store.py | 39 ++++++- 8 files changed, 413 insertions(+), 5 deletions(-) create mode 100644 tests/test_session_context_codec.py diff --git a/generate/dialogue.py b/generate/dialogue.py index 7f02afff..54720913 100644 --- a/generate/dialogue.py +++ b/generate/dialogue.py @@ -14,6 +14,7 @@ from typing import Literal import numpy as np from algebra.cga import cga_inner, outer_product +from core.array_codec import decode_array, encode_array from field.state import FieldState from generate.proposition import FrameRegistry, Proposition, propose @@ -33,6 +34,19 @@ class DialogueTurn: blade = np.asarray(self.outer_product_blade, dtype=np.float32).copy() object.__setattr__(self, "outer_product_blade", blade) + def to_dict(self) -> dict: + return { + "proposition": self.proposition.to_dict(), + "outer_product_blade": encode_array(self.outer_product_blade), + } + + @classmethod + def from_dict(cls, payload: dict) -> "DialogueTurn": + return cls( + proposition=Proposition.from_dict(payload["proposition"]), + outer_product_blade=decode_array(payload["outer_product_blade"]), + ) + def blade_alignment(blade: np.ndarray, reference: np.ndarray) -> float: """ diff --git a/generate/proposition.py b/generate/proposition.py index 1088df6d..a00e1e23 100644 --- a/generate/proposition.py +++ b/generate/proposition.py @@ -17,6 +17,12 @@ from typing import Iterable import numpy as np from algebra.cga import cga_inner, outer_product +from core.array_codec import ( + decode_array, + decode_optional_array, + encode_array, + encode_optional_array, +) from field.state import FieldState from generate.admissibility import AdmissibilityRegion, filter_candidates from generate.stream import _articulate @@ -75,6 +81,35 @@ class Proposition: object.__setattr__(self, "relation", relation) object.__setattr__(self, "relation_norm", float(np.linalg.norm(relation))) + def to_dict(self) -> dict: + # relation_norm is recomputed from `relation` in __post_init__, so it is + # derived, not persisted. + return { + "subject": self.subject, + "predicate": self.predicate, + "object_": self.object_, + "surface": self.surface, + "frame_id": self.frame_id, + "subject_versor": encode_array(self.subject_versor), + "predicate_versor": encode_array(self.predicate_versor), + "object_versor": encode_optional_array(self.object_versor), + "relation": encode_array(self.relation), + } + + @classmethod + def from_dict(cls, payload: dict) -> "Proposition": + return cls( + subject=payload["subject"], + predicate=payload["predicate"], + object_=payload["object_"], + surface=payload["surface"], + frame_id=payload["frame_id"], + subject_versor=decode_array(payload["subject_versor"]), + predicate_versor=decode_array(payload["predicate_versor"]), + object_versor=decode_optional_array(payload["object_versor"]), + relation=decode_array(payload["relation"]), + ) + class FrameRegistry: """Exact frame selection over precompiled frame relation blades.""" diff --git a/session/context.py b/session/context.py index 0307b095..e1901346 100644 --- a/session/context.py +++ b/session/context.py @@ -13,6 +13,7 @@ import numpy as np from algebra.backend import cga_inner, versor_apply from algebra.rotor import rotor_power, word_transition_rotor from algebra.versor import versor_condition as _versor_condition +from core.array_codec import decode_optional_array, encode_optional_array from field.state import FieldState from generate.dialogue import DialogueTurn from generate.proposition import Proposition @@ -340,3 +341,65 @@ class SessionContext: def recall(self, query_tokens: list, top_k: int = 5) -> list: query_state = inject(query_tokens, self.vocab) return self.vault.recall(query_state.F, top_k=top_k) + + def snapshot(self) -> dict: + """Serialize the lived session state for cross-reboot persistence. + + Captures EVERYTHING that accumulates across turns — field, vault, anchor, + graph, referents, dialogue blade/history, and the per-turn caches — so a + restore continues the same life. ``vocab`` and ``persona`` are NOT + serialized: they are shared, ratified surfaces supplied at restore time, + not session state. Bit-exact (arrays via the codec); JSON-safe. + """ + return { + "state": self.state.to_dict() if self.state is not None else None, + "vault": self.vault.to_dict(), + "turn": int(self.turn), + "graph": self.graph.to_dict(), + "referents": self.referents.to_dict(), + "anchor_field": encode_optional_array(self._anchor_field), + "running_dialogue_blade": encode_optional_array(self.running_dialogue_blade), + "dialogue_history": [t.to_dict() for t in self._dialogue_history_compat], + "last_input_tokens": list(self._last_input_tokens), + "last_resolved_input_tokens": list(self._last_resolved_input_tokens), + "last_input_versor": encode_optional_array(self._last_input_versor), + "last_response_tokens": ( + list(self._last_response_tokens) + if self._last_response_tokens is not None + else None + ), + } + + def restore(self, payload: dict) -> None: + """Load a snapshot into THIS context, replacing all lived state. + + Mutating by design — restoring a checkpoint is inherently a load. The + field, vault, graph, and referents are rebuilt from the exact persisted + bytes (no normalization / reprojection — that discipline lives in the + component codecs); ``vocab`` and ``persona`` already set on this context + are kept. + """ + self.state = ( + FieldState.from_dict(payload["state"]) + if payload["state"] is not None + else None + ) + self.vault = VaultStore.from_dict(payload["vault"]) + self.turn = int(payload["turn"]) + self.graph = SessionGraph.from_dict(payload["graph"]) + self.referents = ReferentRegistry.from_dict(payload["referents"]) + self._anchor_field = decode_optional_array(payload["anchor_field"]) + self.running_dialogue_blade = decode_optional_array( + payload["running_dialogue_blade"] + ) + self._dialogue_history_compat = [ + DialogueTurn.from_dict(t) for t in payload["dialogue_history"] + ] + self._last_input_tokens = tuple(payload["last_input_tokens"]) + self._last_resolved_input_tokens = tuple(payload["last_resolved_input_tokens"]) + self._last_input_versor = decode_optional_array(payload["last_input_versor"]) + self._last_response_tokens = ( + tuple(payload["last_response_tokens"]) + if payload["last_response_tokens"] is not None + else None + ) diff --git a/session/graph.py b/session/graph.py index 42741cc1..9440bde4 100644 --- a/session/graph.py +++ b/session/graph.py @@ -9,10 +9,12 @@ walk those edges with true BFS distance, not traversal ordinal. from __future__ import annotations from dataclasses import dataclass, field -from typing import Sequence +from typing import Any, Sequence import numpy as np +from core.array_codec import decode_array, encode_array + @dataclass(slots=True) class TurnNode: @@ -25,6 +27,31 @@ class TurnNode: referent_slots: dict[str, int] backward_edges: list[int] = field(default_factory=list) + def to_dict(self) -> dict[str, Any]: + return { + "turn_idx": int(self.turn_idx), + "input_versor": encode_array(self.input_versor), + "output_versor": encode_array(self.output_versor), + "tokens_in": list(self.tokens_in), + "tokens_out": list(self.tokens_out), + "dialogue_role": self.dialogue_role, + "referent_slots": dict(self.referent_slots), + "backward_edges": list(self.backward_edges), + } + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "TurnNode": + return cls( + turn_idx=int(payload["turn_idx"]), + input_versor=decode_array(payload["input_versor"]), + output_versor=decode_array(payload["output_versor"]), + tokens_in=tuple(payload["tokens_in"]), + tokens_out=tuple(payload["tokens_out"]), + dialogue_role=payload["dialogue_role"], + referent_slots=dict(payload["referent_slots"]), + backward_edges=list(payload["backward_edges"]), + ) + def copy_with_output( self, new_output_versor: np.ndarray, @@ -138,3 +165,12 @@ class SessionGraph: def __repr__(self) -> str: return f"SessionGraph(turns={len(self._nodes)})" + + def to_dict(self) -> dict[str, Any]: + return {"nodes": [n.to_dict() for n in self._nodes]} + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "SessionGraph": + graph = cls() + graph._nodes = [TurnNode.from_dict(n) for n in payload["nodes"]] + return graph diff --git a/session/referents.py b/session/referents.py index 7d245a5b..38b2bd3f 100644 --- a/session/referents.py +++ b/session/referents.py @@ -10,10 +10,12 @@ backward edges instead of broad historical guesses. from __future__ import annotations from dataclasses import dataclass -from typing import Sequence +from typing import Any, Sequence import numpy as np +from core.array_codec import decode_array, encode_array + _PRONOUN_SLOTS: dict[str, str] = { "it": "neut_sg", "this": "neut_sg", @@ -42,6 +44,23 @@ class ReferentEntry: turn: int slot: str + def to_dict(self) -> dict[str, Any]: + return { + "surface": self.surface, + "versor": encode_array(self.versor), + "turn": int(self.turn), + "slot": self.slot, + } + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "ReferentEntry": + return cls( + surface=payload["surface"], + versor=decode_array(payload["versor"]), + turn=int(payload["turn"]), + slot=payload["slot"], + ) + class ReferentRegistry: """Per-session registry of active discourse referents.""" @@ -150,3 +169,37 @@ class ReferentRegistry: def __repr__(self) -> str: active = {k: v.surface for k, v in self._slots.items()} return f"ReferentRegistry(active={active})" + + def to_dict(self) -> dict[str, Any]: + """Serialize, preserving the _slots <-> _history object aliasing. + + ``register`` puts the SAME ReferentEntry object in both ``_slots[slot]`` + and ``_history``; ``update_turn_versor`` relies on that identity + (``_slots.get(slot) is entry``). We persist ``_history`` as the source of + truth and ``_slots`` as slot -> history-index, so restore rebinds the + exact same objects rather than independent copies. + """ + slot_to_index: dict[str, int] = {} + for slot, entry in self._slots.items(): + for i, hist_entry in enumerate(self._history): + if hist_entry is entry: + slot_to_index[slot] = i + break + return { + "history": [e.to_dict() for e in self._history], + "slot_to_index": slot_to_index, + "last_resolved_sources": list(self._last_resolved_sources), + "last_resolved_slots": dict(self._last_resolved_slots), + } + + @classmethod + def from_dict(cls, payload: dict[str, Any]) -> "ReferentRegistry": + registry = cls() + registry._history = [ReferentEntry.from_dict(e) for e in payload["history"]] + registry._slots = { + slot: registry._history[i] + for slot, i in payload["slot_to_index"].items() + } + registry._last_resolved_sources = list(payload["last_resolved_sources"]) + registry._last_resolved_slots = dict(payload["last_resolved_slots"]) + return registry diff --git a/tests/test_session_context_codec.py b/tests/test_session_context_codec.py new file mode 100644 index 00000000..687c31de --- /dev/null +++ b/tests/test_session_context_codec.py @@ -0,0 +1,152 @@ +"""SessionContext.snapshot/restore — Shape B+ Phase C. + +Composes the FieldState (Phase A) and VaultStore (Phase B) codecs with new +codecs for SessionGraph, ReferentRegistry, Proposition, and DialogueTurn. Exit +gate: a real session, snapshotted and restored into a fresh context, is +field-equal — field bit-exact, vault recall identical, graph/referents/dialogue +preserved (including the _slots<->_history object aliasing). +""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np + +from chat.runtime import ChatRuntime +from core.config import RuntimeConfig +from core.cognition.pipeline import CognitiveTurnPipeline +from generate.dialogue import DialogueTurn +from generate.proposition import Proposition +from session.context import SessionContext +from session.graph import SessionGraph, TurnNode +from session.referents import ReferentEntry, ReferentRegistry + +_PROMPTS = ("What causes light?", "What is a concept?", "What causes rain?", "Hello.") + + +def _v(seed: int) -> np.ndarray: + v = np.zeros(32, dtype=np.float32) + v[0] = 1.0 + v[(seed % 31) + 1] = 0.01 * (seed + 1) + return v + + +# --------------------------------------------------------------------------- # +# Component round-trips # +# --------------------------------------------------------------------------- # +def test_turnnode_and_graph_round_trip() -> None: + g = SessionGraph() + g.add_turn(0, _v(1), _v(2), ("a", "b"), ("c",), "assert", {"neut_sg": 0}, []) + g.add_turn(1, _v(3), _v(4), ("d",), ("e", "f"), "question", {"neut_sg": 0}, [0]) + r = SessionGraph.from_dict(g.to_dict()) + assert len(r) == 2 + for orig, rec in zip(g.all_nodes(), r.all_nodes()): + assert rec.input_versor.tobytes() == orig.input_versor.tobytes() + assert rec.output_versor.tobytes() == orig.output_versor.tobytes() + assert rec.tokens_in == orig.tokens_in + assert rec.backward_edges == orig.backward_edges + assert rec.referent_slots == orig.referent_slots + + +def test_referent_registry_round_trip_preserves_slot_history_aliasing() -> None: + reg = ReferentRegistry() + reg.register("cat", _v(5), turn=0, slot="neut_sg") + reg.register("dog", _v(6), turn=1, slot="neut_sg") # same slot, twice + restored = ReferentRegistry.from_dict(reg.to_dict()) + # Active slot points to the latest ("dog") and is the SAME object as the + # corresponding history entry (the aliasing update_turn_versor relies on). + active = restored.active_referent("neut_sg") + assert active is not None and active.surface == "dog" + assert active is restored.history()[-1] + assert restored.history()[0].versor.tobytes() == reg.history()[0].versor.tobytes() + + +def test_proposition_and_dialogue_turn_round_trip() -> None: + prop = Proposition( + subject="s", predicate="p", object_="o", surface="s p o", + frame_id="f", subject_versor=_v(7), predicate_versor=_v(8), + object_versor=_v(9), relation=_v(10), + ) + turn = DialogueTurn(proposition=prop, outer_product_blade=_v(11)) + rec = DialogueTurn.from_dict(turn.to_dict()) + assert rec.proposition.subject == "s" + assert rec.proposition.subject_versor.tobytes() == prop.subject_versor.tobytes() + assert rec.proposition.object_versor is not None + assert rec.proposition.relation.tobytes() == prop.relation.tobytes() + assert rec.outer_product_blade.tobytes() == turn.outer_product_blade.tobytes() + + +def test_proposition_none_object_versor_round_trips() -> None: + prop = Proposition( + subject="s", predicate="p", object_=None, surface="s p", + frame_id="f", subject_versor=_v(1), predicate_versor=_v(2), + ) + rec = Proposition.from_dict(prop.to_dict()) + assert rec.object_versor is None + assert rec.object_ is None + + +# --------------------------------------------------------------------------- # +# Integration: a real session snapshot -> restore is field-equal # +# --------------------------------------------------------------------------- # +def _drive_session(tmp_path: Path) -> SessionContext: + runtime = ChatRuntime(config=RuntimeConfig(), engine_state_path=tmp_path / "es") + pipe = CognitiveTurnPipeline(runtime=runtime) + for p in _PROMPTS: + pipe.run(p) + return runtime._context + + +def test_session_context_snapshot_restore_is_field_equal(tmp_path: Path) -> None: + ctx = _drive_session(tmp_path) + snap = ctx.snapshot() + + restored = SessionContext(vocab=ctx.vocab, persona=ctx.persona) + restored.restore(snap) + + # Field bit-exact + closure preserved. + assert (ctx.state is None) == (restored.state is None) + if ctx.state is not None: + assert restored.state.F.tobytes() == ctx.state.F.tobytes() + assert restored.turn == ctx.turn + + # Anchor bit-exact. + if ctx._anchor_field is not None: + assert restored._anchor_field.tobytes() == ctx._anchor_field.tobytes() + + # Vault recall identical (exact CGA preserved through the whole compose). + query = ctx.state.F if ctx.state is not None else _v(0) + before = ctx.vault.recall(query, top_k=5) + after = restored.vault.recall(query, top_k=5) + assert [(r["index"], r["score"]) for r in before] == [ + (r["index"], r["score"]) for r in after + ] + + # Graph preserved. + assert len(restored.graph) == len(ctx.graph) + for orig, rec in zip(ctx.graph.all_nodes(), restored.graph.all_nodes()): + assert rec.output_versor.tobytes() == orig.output_versor.tobytes() + assert rec.tokens_in == orig.tokens_in + + # Referents + dialogue history preserved. + assert restored.referents.active_slots() == ctx.referents.active_slots() + assert len(restored._dialogue_history_compat) == len(ctx._dialogue_history_compat) + assert (ctx.running_dialogue_blade is None) == ( + restored.running_dialogue_blade is None + ) + if ctx.running_dialogue_blade is not None: + assert ( + restored.running_dialogue_blade.tobytes() + == ctx.running_dialogue_blade.tobytes() + ) + + +def test_session_context_snapshot_is_json_safe(tmp_path: Path) -> None: + import json + + ctx = _drive_session(tmp_path) + blob = json.dumps(ctx.snapshot()) + restored = SessionContext(vocab=ctx.vocab, persona=ctx.persona) + restored.restore(json.loads(blob)) + assert restored.turn == ctx.turn diff --git a/tests/test_vaultstore_codec.py b/tests/test_vaultstore_codec.py index c5e909fc..d7c98a86 100644 --- a/tests/test_vaultstore_codec.py +++ b/tests/test_vaultstore_codec.py @@ -83,6 +83,28 @@ def test_vaultstore_restore_rebuilds_exact_match_index() -> None: assert hits[0]["score"] == float("inf") # exact match found via rebuilt index +def test_vaultstore_round_trips_proposition_valued_metadata() -> None: + # generate/proposition.py stores {"kind":"proposition","proposition":} + # into vault metadata — the one structured (non-primitive) metadata value. + import json + + from generate.proposition import Proposition + + prop = Proposition( + subject="s", predicate="p", object_="o", surface="s p o", + frame_id="f", subject_versor=_versors()[0], predicate_versor=_versors()[1], + relation=_versors()[2], + ) + store = VaultStore(reproject_interval=0) + store.store(_versors()[0], {"kind": "proposition", "proposition": prop}) + + restored = VaultStore.from_dict(json.loads(json.dumps(store.to_dict()))) + recovered = restored._metadata[0]["proposition"] + assert isinstance(recovered, Proposition) + assert recovered.subject == "s" + assert recovered.relation.tobytes() == prop.relation.tobytes() + + def test_empty_vaultstore_round_trips() -> None: store = VaultStore(reproject_interval=50, max_entries=10) restored = VaultStore.from_dict(store.to_dict()) diff --git a/vault/store.py b/vault/store.py index ee762c50..758e7dda 100644 --- a/vault/store.py +++ b/vault/store.py @@ -58,6 +58,37 @@ def _versor_key(F: np.ndarray) -> bytes: return np.asarray(F, dtype=np.float32).tobytes() +# Metadata values are JSON primitives except for one structured value: a +# ``Proposition`` stored under the ``"proposition"`` key (generate/proposition.py). +# It is tagged on encode and reconstructed on decode. The Proposition import is +# lazy (inside the functions) so vault/store.py stays free of a load-time cycle. +_PROPOSITION_TAG = "__core_proposition__" + + +def _encode_metadata(metadata: dict) -> dict: + from generate.proposition import Proposition + + encoded: dict = {} + for key, value in metadata.items(): + if isinstance(value, Proposition): + encoded[key] = {_PROPOSITION_TAG: value.to_dict()} + else: + encoded[key] = value + return encoded + + +def _decode_metadata(metadata: dict) -> dict: + decoded: dict = {} + for key, value in metadata.items(): + if isinstance(value, dict) and _PROPOSITION_TAG in value: + from generate.proposition import Proposition + + decoded[key] = Proposition.from_dict(value[_PROPOSITION_TAG]) + else: + decoded[key] = value + return decoded + + def epistemic_state_for_vault_status(entry_status: EpistemicStatus) -> EpistemicState: """Map legacy vault review statuses onto the ratified state taxonomy.""" if entry_status is EpistemicStatus.COHERENT: @@ -398,11 +429,13 @@ class VaultStore: reproject boundary during the live session — encoded losslessly via the array codec. The derived ``_exact_index`` and the lazy ``_matrix_cache`` are NOT persisted; they are rebuilt deterministically on load. Metadata - is assumed JSON-safe (it is, by construction: primitives only). + is mostly primitives, with one structured value — a ``Proposition`` under + the ``"proposition"`` key (generate/proposition.py) — handled by + ``_encode_metadata`` so the snapshot stays JSON-safe. """ return { "versors": [encode_array(v) for v in self._versors], - "metadata": [dict(m) for m in self._metadata], + "metadata": [_encode_metadata(m) for m in self._metadata], "store_count": int(self._store_count), "reproject_interval": int(self._reproject_interval), "max_entries": self._max_entries, @@ -427,7 +460,7 @@ class VaultStore: maxlen=store._max_entries, ) store._metadata = deque( - (dict(m) for m in payload["metadata"]), + (_decode_metadata(m) for m in payload["metadata"]), maxlen=store._max_entries, ) store._store_count = int(payload["store_count"])