diff --git a/chat/runtime.py b/chat/runtime.py index ebfa56d2..1c71a3ce 100644 --- a/chat/runtime.py +++ b/chat/runtime.py @@ -101,6 +101,7 @@ class ChatRuntime: for manifest in self._manifests ): raise KeyError(f"OOV token requires vocab proposal: {token}") + kept.append(token) return kept def _syntactic_guard(self, tokens: tuple[str, ...]) -> list[str]: @@ -135,7 +136,7 @@ class ChatRuntime: ) proposition = propose_dialogue( field_state, - None, + self._context.vault, self._context.vocab, self._frame_registry, reference_blade, diff --git a/tests/test_chat_runtime.py b/tests/test_chat_runtime.py index dfb947b8..e83b7afc 100644 --- a/tests/test_chat_runtime.py +++ b/tests/test_chat_runtime.py @@ -21,3 +21,13 @@ def test_chat_runtime_keeps_live_session_across_two_turns() -> None: assert second.versor_condition < 1e-6 assert second.dialogue_role in {"elaborate", "assert"} assert not np.array_equal(second_field, first_field) + + +def test_chat_runtime_keeps_groundable_oov_tokens_in_open_pack() -> None: + runtime = ChatRuntime() + + response = runtime.chat("what is דברית", max_tokens=4) + + assert response.surface.strip() + assert runtime.session.state is not None + assert runtime.session.vault.recall(runtime.session.state.F, top_k=1)