diff --git a/docs/evals/discourse_runtime_baseline_2026-05-19.md b/docs/evals/discourse_runtime_baseline_2026-05-19.md new file mode 100644 index 00000000..e3a0c466 --- /dev/null +++ b/docs/evals/discourse_runtime_baseline_2026-05-19.md @@ -0,0 +1,93 @@ +# Discourse Runtime Baseline — 2026-05-19 + +This note records the runtime evidence around the discourse-planner landing. +It preserves the pre-wiring baseline and the post-refinement predicate result +so future deltas are interpreted against the right measurement contract. + +## Pre-Wiring Baseline + +Measured before the five-step discourse planner sequence landed: + +```json +{ + "conversational_thread_coherence_public_v1": { + "cases": 6, + "is_grounded_rate": 0.9333, + "length_adequate_rate": 1.0, + "no_placeholder_rate": 1.0, + "no_topic_drift_rate": 0.8333, + "not_walk_fragment_rate": 1.0, + "topic_anchor_rate": 0.5, + "total_turns": 45 + }, + "discourse_paragraph_public_v2": { + "accuracy": 1.0, + "mean_sentence_count": 14.333, + "mean_subject_coverage": 1.0, + "passed": 6, + "per_sentence_grammar_pass_rate": 1.0, + "replay_determinism_rate": 1.0, + "total": 6 + }, + "multi_sentence_response_public_v1": { + "cases": 15, + "connective_present_rate": 0.1, + "grounded_rate": 0.4667, + "multi_sentence_rate": 0.5333, + "non_fragment_rate": 1.0, + "subject_named_rate": 0.5333 + } +} +``` + +The direct realizer path was already paragraph-capable: +`discourse_paragraph` passed at 100% with deterministic replay and +per-sentence grammar intact. The live runtime gap was upstream of +realization. + +## Predicate Refinement + +The original `multi_sentence_response` sentence splitter over-counted +structural punctuation: + +- dotted semantic-domain atoms such as `cognition.truth` +- lowercase domain continuations such as `logos.core. truth grounds ...` +- the fixed trust-boundary tail `No session evidence yet.` + +The refined predicate counts only substantive sentences: + +- strip trailing provenance / trust-boundary tails before counting +- do not split on dotted semantic-domain atoms +- split a terminal mark only when followed by an uppercase/digit sentence + opener or the end of the substantive surface + +Measured on `30948a1` after the discourse planner sequence landed and with +the refined predicate: + +```json +{ + "flag_off": { + "cases": 15, + "connective_present_rate": 0.1, + "grounded_rate": 0.4667, + "multi_sentence_rate": 0.2, + "non_fragment_rate": 1.0, + "subject_named_rate": 0.5333 + }, + "flag_on": { + "cases": 15, + "connective_present_rate": 0.1, + "grounded_rate": 0.4667, + "multi_sentence_rate": 0.2, + "non_fragment_rate": 1.0, + "subject_named_rate": 0.5333 + } +} +``` + +Interpretation: the earlier `0.5333` multi-sentence rate was inflated by +structural tails and domain punctuation. The flag-on planner work improved +form quality on surfaces where it engaged, but this one-shot lane still does +not isolate that hook. The next measurement should either prime the warm path +before scoring or move planner engagement into the cold pack/teaching-grounded +path and then compare flag-off versus flag-on again. diff --git a/evals/multi_sentence_response/contract.md b/evals/multi_sentence_response/contract.md index 38e153c4..8e74175c 100644 --- a/evals/multi_sentence_response/contract.md +++ b/evals/multi_sentence_response/contract.md @@ -20,10 +20,10 @@ as the *only* multi-sentence-capable code path. | Predicate | Definition | |---|---| -| `sentence_count_>=_2` | the surface contains at least 2 terminated sentences (`.`, `?`, `!`) | +| `sentence_count_>=_2` | the substantive surface contains at least 2 terminated sentences (`.`, `?`, `!`) | | `each_sentence_>=_4_tokens` | every sentence has ≥ 4 alphabetic tokens (no fragments) | | `connective_present` | the surface contains at least one connective (`and`, `because`, `therefore`, `which`, `since`, `also`, `furthermore`, `however`, `consequently`) — only enforced when `expects_connective=true` | -| `not_just_provenance_tag` | sentence_count counts BEFORE the trailing provenance tag (`pack-grounded (…).`) is treated as its own sentence | +| `not_just_provenance_tag` | sentence_count counts BEFORE trailing provenance / trust-boundary tails (`pack-grounded (…).`, `No session evidence yet.`) are treated as real sentences | | `grounded` | `grounding_source` ∈ {pack, teaching} | | `subject_named` | the prompt's subject lemma appears in the surface | @@ -37,8 +37,12 @@ connective_present_rate = cases_with_connective / cases_expecting_connective ## Doctrine constraints -- The "trailing provenance tag" is structural, not a real sentence — - predicate logic strips it before counting. +- The trailing provenance / trust-boundary tail is structural, not a real + sentence — predicate logic strips it before counting. +- Dotted semantic-domain atoms (`cognition.truth`, `logos.core`) are not + sentence boundaries by themselves. A terminal mark counts as a boundary + only when it is followed by a new uppercase/digit sentence opener or the + end of the substantive surface. - No LLM judge. Pure structural counting. - Red-on-creation expected: only NARRATIVE / EXAMPLE / cross-pack / composed_surface code paths can possibly satisfy `sentence_count_>=_2` diff --git a/evals/multi_sentence_response/runner.py b/evals/multi_sentence_response/runner.py index 1f5020c8..b1923b4a 100644 --- a/evals/multi_sentence_response/runner.py +++ b/evals/multi_sentence_response/runner.py @@ -28,6 +28,9 @@ from chat.runtime import ChatRuntime _PROVENANCE_TAIL_RE = re.compile( r"\s*(pack-grounded|teaching-grounded)\s*\([^)]+\)\.?\s*$" ) +_TRUST_DISCLOSURE_TAIL_RE = re.compile( + r"\s*No session evidence yet\.?\s*$" +) _CONNECTIVES = ( "and", "because", "therefore", "which", "since", "also", @@ -37,11 +40,27 @@ _CONNECTIVES = ( def _strip_provenance(surface: str) -> str: - return _PROVENANCE_TAIL_RE.sub("", surface).strip() + stripped = _PROVENANCE_TAIL_RE.sub("", surface).strip() + return _TRUST_DISCLOSURE_TAIL_RE.sub("", stripped).strip() def _split_sentences(text: str) -> list[str]: - parts = re.split(r"(?<=[.!?])\s+", text.strip()) + """Split substantive sentences without treating domain dots as stops. + + Pack and teaching surfaces often contain semantic-domain atoms such as + ``cognition.truth`` or ``logos.core``. A raw ``period + whitespace`` + splitter over-counts those atoms as sentence boundaries, especially in + older structured disclosures like ``logos.core. truth grounds ...``. + + Treat a stop as sentence-final only when it is followed by whitespace and + an uppercase/digit opener, or by the end of the text. This keeps + ``cognition.truth. In turn, ...`` as two sentences while preventing + lowercase domain continuations from inflating the metric. + """ + stripped = text.strip() + if not stripped: + return [] + parts = re.split(r"(?<=[.!?])\s+(?=[A-Z0-9])", stripped) return [p.strip() for p in parts if p.strip()] @@ -75,8 +94,8 @@ class LaneReport: case_details: list[dict[str, Any]] = field(default_factory=list) -def _run_case(case: dict[str, Any]) -> CaseResult: - rt = ChatRuntime() +def _run_case(case: dict[str, Any], config: Any = None) -> CaseResult: + rt = ChatRuntime(config=config) if config is not None else ChatRuntime() resp = rt.chat(case["prompt"]) surface = resp.surface grounding = resp.grounding_source or "none" @@ -103,11 +122,11 @@ def _run_case(case: dict[str, Any]) -> CaseResult: ) -def run_lane(cases: list[dict[str, Any]], config: Any = None) -> LaneReport: # noqa: ARG001 +def run_lane(cases: list[dict[str, Any]], config: Any = None) -> LaneReport: if not cases: return LaneReport(metrics={}, case_details=[]) - results = [_run_case(c) for c in cases] + results = [_run_case(c, config=config) for c in cases] total = len(results) multi = sum(1 for r in results if r.sentence_count >= 2) diff --git a/tests/test_multi_sentence_response_eval.py b/tests/test_multi_sentence_response_eval.py new file mode 100644 index 00000000..e370737a --- /dev/null +++ b/tests/test_multi_sentence_response_eval.py @@ -0,0 +1,86 @@ +"""Tests for the multi-sentence response eval lane predicates.""" + +from __future__ import annotations + +from core.config import RuntimeConfig +from evals.multi_sentence_response import runner +from evals.multi_sentence_response.runner import ( + _split_sentences, + _strip_provenance, + run_lane, +) + + +def test_strip_provenance_removes_trust_boundary_tail() -> None: + surface = ( + "truth — narrative-grounded: cognition.truth. " + "truth grounds knowledge. No session evidence yet." + ) + + stripped = _strip_provenance(surface) + + assert stripped == "truth — narrative-grounded: cognition.truth. truth grounds knowledge." + + +def test_sentence_splitter_ignores_lowercase_semantic_domain_continuation() -> None: + surface = ( + "truth — teaching-grounded: cognition.truth; logos.core. " + "truth grounds knowledge (cognition.knowledge). No session evidence yet." + ) + + sentences = _split_sentences(surface) + + assert sentences == [ + ( + "truth — teaching-grounded: cognition.truth; logos.core. " + "truth grounds knowledge (cognition.knowledge)." + ), + "No session evidence yet.", + ] + + +def test_sentence_splitter_keeps_uppercase_discourse_transition() -> None: + surface = ( + "Truth is a claim grounded by evidence. Furthermore, truth belongs " + "to cognition.truth. In turn, truth grounds knowledge." + ) + + sentences = _split_sentences(surface) + + assert sentences == [ + "Truth is a claim grounded by evidence.", + "Furthermore, truth belongs to cognition.truth.", + "In turn, truth grounds knowledge.", + ] + + +def test_run_lane_passes_runtime_config_to_chat_runtime(monkeypatch) -> None: + seen_configs: list[RuntimeConfig | None] = [] + + class _FakeResponse: + surface = "Truth is grounded. Furthermore, truth belongs to cognition.truth." + grounding_source = "pack" + + class _FakeRuntime: + def __init__(self, config=None): + seen_configs.append(config) + + def chat(self, prompt: str) -> _FakeResponse: # noqa: ARG002 + return _FakeResponse() + + monkeypatch.setattr(runner, "ChatRuntime", _FakeRuntime) + cases = [ + { + "id": "flag_on_truth", + "category": "explain", + "prompt": "Explain truth.", + "subject_lemma": "truth", + "expects_connective": True, + } + ] + cfg = RuntimeConfig(discourse_planner=True) + + report = run_lane(cases, config=cfg) + + assert seen_configs == [cfg] + assert report.case_details[0]["connective_present"] is True