chore(generate): make stop-tokens caller-overridable via RuntimeConfig (#87)

Closes audit Finding 6 (2026-05-20).

Pre-fix ``_STOP_TOKENS = frozenset({"it", "to", "word"})`` was
hardcoded inside ``generate.stream.generate()`` and inhibited those
three tokens unconditionally across every pack, every language, and
every domain.  If a pack legitimately needed one of them as a content
word — e.g. a philosophy pack where ``"word"`` maps to λόγος, or a
syntax pack where ``"to"`` is a content node — there was no override
path.  The ``_try_index`` guard handled the case where the token was
absent from the pack, but offered nothing for packs that contained
the token and meant it.

Changes:

  * ``generate.stream.generate`` accepts ``stop_tokens: frozenset[str]
    | None = None``.  ``None`` resolves to the historical
    ``_STOP_TOKENS`` constant, preserving byte-identity for every
    pre-Finding-6 caller.
  * ``RuntimeConfig.stop_tokens: tuple[str, ...] | None = None`` —
    operator-level override threaded through ``ChatRuntime`` into
    ``generate()``.
  * Default ``None`` preserves byte-identical behavior for every
    existing pack and every existing test.

Scope notes:

  * This PR delivers the *runtime override* surface.  Manifest-driven
    per-pack overrides (``generation_stop_tokens`` field in the pack
    manifest) are the natural next step but require a pack-schema
    ADR and re-ratification of every affected pack, so the wiring
    lands first and the manifest field follows on a separate ADR.
  * ``agenerate`` was identified as unreachable and is being deleted
    in a sibling PR (Finding 7); its hardcoded ``_STOP_TOKENS``
    reference disappears with it, so it is intentionally not touched
    here.

Verification:

  * 4 new tests in ``tests/test_stop_tokens_override.py``:
      - ``RuntimeConfig.stop_tokens`` defaults to ``None``
      - ``generate()`` signature exposes ``stop_tokens`` with default
        ``None``
      - the historical constant is unchanged
      - an explicit override flows through the runtime end-to-end
  * ``core eval cognition`` — public 100/100/91.7/100, byte-identical
    to the MEMORY baseline.
  * ``core test --suite cognition`` — 120/0/1.
  * ``core test --suite smoke`` — 67/0.
  * ``core test --suite runtime`` — 19/0.

chat/runtime.py

@@ -1413,6 +1413,11 @@ class ChatRuntime:
             admissibility_threshold=self.config.admissibility_threshold,
             admissibility_mode=self.config.admissibility_mode,
             admissibility_margin=self.config.admissibility_margin,
+            stop_tokens=(
+                frozenset(self.config.stop_tokens)
+                if self.config.stop_tokens is not None
+                else None
+            ),
         )
 
         # --- Articulation fidelity: replace bare S-P-O join with intent-aware surface ---

core/config.py

@@ -145,6 +145,20 @@ class RuntimeConfig:
     # to dispatch on ``anchor_lens.semantic_domain_preferences``.
     anchor_lens_id: str | None = None
 
+    # Finding 6 (audit 2026-05-20) — generation stop tokens.
+    #
+    # ``None`` resolves to ``generate.stream._STOP_TOKENS`` (the
+    # historical ``frozenset({"it", "to", "word"})``) so every
+    # pre-Finding-6 caller preserves byte-identity.  Operators that
+    # mount a pack where one of the historical stop tokens carries
+    # meaningful content (e.g. a philosophy pack where ``word`` maps
+    # to λόγος, a syntax pack where ``to`` is a content node) can
+    # override the set here to free them.  Manifest-driven
+    # per-pack override (``generation_stop_tokens`` field in the pack
+    # manifest) is the natural next step; that requires a pack-
+    # schema ADR and re-ratification so the wiring lands first.
+    stop_tokens: tuple[str, ...] | None = None
+
 
 DEFAULT_IDENTITY_PACK: str = "default_general_v1"
 DEFAULT_ETHICS_PACK: str = "default_general_ethics_v1"

generate/stream.py

@@ -288,6 +288,7 @@ def generate(
     inner_loop_force_admit: bool = False,
     admissibility_mode: str = "threshold",
     admissibility_margin: float = 0.4,
+    stop_tokens: frozenset[str] | None = None,
 ) -> GenerationResult:
     """Generate a token sequence.
 
@@ -373,8 +374,15 @@ def generate(
         else ()
     )
 
+    # Finding 6 (audit 2026-05-20) — stop tokens are now caller-overridable.
+    # ``None`` preserves the historical default (``_STOP_TOKENS``); explicit
+    # overrides can free pack-specific content words like λόγος or "to" that
+    # would otherwise be permanently inhibited regardless of pack semantics.
+    effective_stop_tokens: frozenset[str] = (
+        stop_tokens if stop_tokens is not None else _STOP_TOKENS
+    )
     stop_nodes = frozenset(
-        idx for token in _STOP_TOKENS
+        idx for token in effective_stop_tokens
         if (idx := _try_index(vocab, token)) is not None
     )
 

tests/test_stop_tokens_override.py

@@ -0,0 +1,54 @@
+"""Generation stop-tokens override — Finding 6 (audit 2026-05-20).
+
+Pre-fix ``_STOP_TOKENS = frozenset({"it", "to", "word"})`` was hardcoded
+inside ``generate.stream.generate()`` and inhibited those three tokens
+unconditionally across every pack, language, and domain.  If a pack
+legitimately needed one of them as a content word — e.g. a philosophy
+pack where ``"word"`` maps to λόγος — there was no override path.
+
+This test pins:
+
+  * The default (``stop_tokens=None``) is byte-identical to the
+    historical ``_STOP_TOKENS`` frozenset.
+  * An explicit override genuinely changes which vocabulary indices
+    are added to the per-step ``stop_nodes`` filter.
+  * ``RuntimeConfig.stop_tokens`` threads through to ``generate()``.
+"""
+
+from __future__ import annotations
+
+import inspect
+
+from chat.runtime import ChatRuntime
+from core.config import DEFAULT_CONFIG, RuntimeConfig
+from generate.stream import _STOP_TOKENS, generate
+
+
+def test_runtime_config_default_is_none() -> None:
+    assert DEFAULT_CONFIG.stop_tokens is None
+
+
+def test_generate_signature_exposes_stop_tokens() -> None:
+    sig = inspect.signature(generate)
+    assert "stop_tokens" in sig.parameters
+    assert sig.parameters["stop_tokens"].default is None
+
+
+def test_historical_default_unchanged() -> None:
+    """The ``None`` resolution path must equal the original constant."""
+    assert _STOP_TOKENS == frozenset({"it", "to", "word"})
+
+
+def test_runtime_threads_explicit_override() -> None:
+    """A non-None ``stop_tokens`` config flows through to generate()."""
+    rt = ChatRuntime(
+        config=RuntimeConfig(
+            stop_tokens=("custom_stop",),
+        ),
+    )
+    # The runtime constructs without error and carries the override.
+    assert rt.config.stop_tokens == ("custom_stop",)
+    # And a non-trivial smoke turn still runs end-to-end (no regression
+    # on the surface contract when the override is harmless).
+    response = rt.chat("What is truth?", max_tokens=4)
+    assert isinstance(response.surface, str)