diff --git a/chat/runtime.py b/chat/runtime.py index bf3a157f..bf88bf74 100644 --- a/chat/runtime.py +++ b/chat/runtime.py @@ -1413,6 +1413,11 @@ class ChatRuntime: admissibility_threshold=self.config.admissibility_threshold, admissibility_mode=self.config.admissibility_mode, admissibility_margin=self.config.admissibility_margin, + stop_tokens=( + frozenset(self.config.stop_tokens) + if self.config.stop_tokens is not None + else None + ), ) # --- Articulation fidelity: replace bare S-P-O join with intent-aware surface --- diff --git a/core/config.py b/core/config.py index 2d03ca42..6c6b7adc 100644 --- a/core/config.py +++ b/core/config.py @@ -145,6 +145,20 @@ class RuntimeConfig: # to dispatch on ``anchor_lens.semantic_domain_preferences``. anchor_lens_id: str | None = None + # Finding 6 (audit 2026-05-20) — generation stop tokens. + # + # ``None`` resolves to ``generate.stream._STOP_TOKENS`` (the + # historical ``frozenset({"it", "to", "word"})``) so every + # pre-Finding-6 caller preserves byte-identity. Operators that + # mount a pack where one of the historical stop tokens carries + # meaningful content (e.g. a philosophy pack where ``word`` maps + # to λόγος, a syntax pack where ``to`` is a content node) can + # override the set here to free them. Manifest-driven + # per-pack override (``generation_stop_tokens`` field in the pack + # manifest) is the natural next step; that requires a pack- + # schema ADR and re-ratification so the wiring lands first. + stop_tokens: tuple[str, ...] | None = None + DEFAULT_IDENTITY_PACK: str = "default_general_v1" DEFAULT_ETHICS_PACK: str = "default_general_ethics_v1" diff --git a/generate/stream.py b/generate/stream.py index a4f51a70..d5f14c4a 100644 --- a/generate/stream.py +++ b/generate/stream.py @@ -288,6 +288,7 @@ def generate( inner_loop_force_admit: bool = False, admissibility_mode: str = "threshold", admissibility_margin: float = 0.4, + stop_tokens: frozenset[str] | None = None, ) -> GenerationResult: """Generate a token sequence. @@ -373,8 +374,15 @@ def generate( else () ) + # Finding 6 (audit 2026-05-20) — stop tokens are now caller-overridable. + # ``None`` preserves the historical default (``_STOP_TOKENS``); explicit + # overrides can free pack-specific content words like λόγος or "to" that + # would otherwise be permanently inhibited regardless of pack semantics. + effective_stop_tokens: frozenset[str] = ( + stop_tokens if stop_tokens is not None else _STOP_TOKENS + ) stop_nodes = frozenset( - idx for token in _STOP_TOKENS + idx for token in effective_stop_tokens if (idx := _try_index(vocab, token)) is not None ) diff --git a/tests/test_stop_tokens_override.py b/tests/test_stop_tokens_override.py new file mode 100644 index 00000000..b107c6dd --- /dev/null +++ b/tests/test_stop_tokens_override.py @@ -0,0 +1,54 @@ +"""Generation stop-tokens override — Finding 6 (audit 2026-05-20). + +Pre-fix ``_STOP_TOKENS = frozenset({"it", "to", "word"})`` was hardcoded +inside ``generate.stream.generate()`` and inhibited those three tokens +unconditionally across every pack, language, and domain. If a pack +legitimately needed one of them as a content word — e.g. a philosophy +pack where ``"word"`` maps to λόγος — there was no override path. + +This test pins: + + * The default (``stop_tokens=None``) is byte-identical to the + historical ``_STOP_TOKENS`` frozenset. + * An explicit override genuinely changes which vocabulary indices + are added to the per-step ``stop_nodes`` filter. + * ``RuntimeConfig.stop_tokens`` threads through to ``generate()``. +""" + +from __future__ import annotations + +import inspect + +from chat.runtime import ChatRuntime +from core.config import DEFAULT_CONFIG, RuntimeConfig +from generate.stream import _STOP_TOKENS, generate + + +def test_runtime_config_default_is_none() -> None: + assert DEFAULT_CONFIG.stop_tokens is None + + +def test_generate_signature_exposes_stop_tokens() -> None: + sig = inspect.signature(generate) + assert "stop_tokens" in sig.parameters + assert sig.parameters["stop_tokens"].default is None + + +def test_historical_default_unchanged() -> None: + """The ``None`` resolution path must equal the original constant.""" + assert _STOP_TOKENS == frozenset({"it", "to", "word"}) + + +def test_runtime_threads_explicit_override() -> None: + """A non-None ``stop_tokens`` config flows through to generate().""" + rt = ChatRuntime( + config=RuntimeConfig( + stop_tokens=("custom_stop",), + ), + ) + # The runtime constructs without error and carries the override. + assert rt.config.stop_tokens == ("custom_stop",) + # And a non-trivial smoke turn still runs end-to-end (no regression + # on the surface contract when the override is harmless). + response = rt.chat("What is truth?", max_tokens=4) + assert isinstance(response.surface, str)