core/evals/multi_sentence_response/runner.py

"""Multi-sentence response eval lane runner.

Measures whether ``ChatRuntime`` emits more than one sentence when
the prompt structurally calls for elaboration.  Strips the trailing
provenance tag (``pack-grounded (...).``) before counting sentences
so the metric reflects substantive content.

Framework contract: ``run_lane(cases, config=None) -> LaneReport``.

Case schema:
    {
      "id": "...",
      "category": "...",
      "prompt": "Tell me about truth.",
      "subject_lemma": "truth",
      "expects_connective": true,
      "priming_prompts": ["What is truth?"]   # optional
    }

``priming_prompts`` is an optional list run before the scored prompt
on the same ``ChatRuntime`` instance.  Their responses are discarded;
only ``prompt`` is scored.  Priming exists because the discourse
planner currently hooks the warm pack/teaching-grounded path (post-
vault), so a one-shot cold-start case cannot exercise it.  Cases
remain backward-compatible — missing or empty ``priming_prompts``
yields the original cold-start behavior.
"""
from __future__ import annotations

import re
from dataclasses import dataclass, field
from typing import Any

from chat.runtime import ChatRuntime


_PROVENANCE_TAIL_RE = re.compile(
    r"\s*(pack-grounded|teaching-grounded)\s*\([^)]+\)\.?\s*$"
)
_TRUST_DISCLOSURE_TAIL_RE = re.compile(
    r"\s*No session evidence yet\.?\s*$"
)

_CONNECTIVES = (
    "and", "because", "therefore", "which", "since", "also",
    "furthermore", "however", "consequently", "thus", "so", "while",
    "whereas", "moreover", "in turn",
)


def _strip_provenance(surface: str) -> str:
    stripped = _PROVENANCE_TAIL_RE.sub("", surface).strip()
    return _TRUST_DISCLOSURE_TAIL_RE.sub("", stripped).strip()


def _split_sentences(text: str) -> list[str]:
    """Split substantive sentences without treating domain dots as stops.

    Pack and teaching surfaces often contain semantic-domain atoms such as
    ``cognition.truth`` or ``logos.core``.  A raw ``period + whitespace``
    splitter over-counts those atoms as sentence boundaries, especially in
    older structured disclosures like ``logos.core. truth grounds ...``.

    Treat a stop as sentence-final only when it is followed by whitespace and
    an uppercase/digit opener, or by the end of the text.  This keeps
    ``cognition.truth. In turn, ...`` as two sentences while preventing
    lowercase domain continuations from inflating the metric.
    """
    stripped = text.strip()
    if not stripped:
        return []
    parts = re.split(r"(?<=[.!?])\s+(?=[A-Z0-9])", stripped)
    return [p.strip() for p in parts if p.strip()]


def _alpha_tokens(text: str) -> int:
    return len(re.findall(r"[A-Za-z]+", text))


def _has_connective(text: str) -> bool:
    low = text.lower()
    return any(re.search(rf"\b{re.escape(c)}\b", low) for c in _CONNECTIVES)


@dataclass(frozen=True, slots=True)
class CaseResult:
    case_id: str
    category: str
    prompt: str
    surface: str
    grounding_source: str
    sentence_count: int
    each_sentence_long_enough: bool
    connective_present: bool
    grounded: bool
    subject_named: bool
    expects_connective: bool
    primed: bool


@dataclass
class LaneReport:
    metrics: dict[str, Any] = field(default_factory=dict)
    case_details: list[dict[str, Any]] = field(default_factory=list)


def _run_case(case: dict[str, Any], config: Any = None) -> CaseResult:
    rt = ChatRuntime(config=config) if config is not None else ChatRuntime()

    # Run optional priming turns on the same runtime so the scored
    # prompt executes on the warm pack/teaching path.  Responses are
    # discarded; only the scored prompt's response is measured.
    priming = case.get("priming_prompts") or ()
    primed = False
    for prime in priming:
        if not isinstance(prime, str) or not prime.strip():
            continue
        rt.chat(prime)
        primed = True

    resp = rt.chat(case["prompt"])
    surface = resp.surface
    grounding = resp.grounding_source or "none"

    stripped = _strip_provenance(surface)
    sentences = _split_sentences(stripped)
    each_long = all(_alpha_tokens(s) >= 4 for s in sentences) if sentences else False

    subj = case.get("subject_lemma", "").lower()
    subj_named = (subj in surface.lower()) if subj else True

    return CaseResult(
        case_id=case["id"],
        category=case.get("category", "uncategorised"),
        prompt=case["prompt"],
        surface=surface,
        grounding_source=grounding,
        sentence_count=len(sentences),
        each_sentence_long_enough=each_long,
        connective_present=_has_connective(stripped),
        grounded=(grounding in {"pack", "teaching"}),
        subject_named=subj_named,
        expects_connective=bool(case.get("expects_connective", False)),
        primed=primed,
    )


def run_lane(cases: list[dict[str, Any]], config: Any = None) -> LaneReport:
    if not cases:
        return LaneReport(metrics={}, case_details=[])

    results = [_run_case(c, config=config) for c in cases]
    total = len(results)

    multi = sum(1 for r in results if r.sentence_count >= 2)
    non_frag = sum(1 for r in results if r.each_sentence_long_enough)
    grounded = sum(1 for r in results if r.grounded)
    named = sum(1 for r in results if r.subject_named)

    conn_expected = [r for r in results if r.expects_connective]
    conn_rate = (
        round(sum(1 for r in conn_expected if r.connective_present) / len(conn_expected), 4)
        if conn_expected else 1.0
    )

    # ``multi_sentence_rate`` historically counted any case with ≥ 2
    # sentences regardless of grounding source.  That admitted OOV
    # teaching invitations and refusal disclosures into the headline
    # capability number — fixed here by splitting into three honest
    # buckets:
    #
    #   articulate_sentence_rate  — ≥2 sentences AND grounded in pack
    #                               or teaching (real capability).
    #   disclosure_sentence_rate  — ≥2 sentences but grounded in oov,
    #                               refusal, none, etc. (structural
    #                               multi-sentence from disclosure
    #                               templates, not articulation).
    #   unarticulate_rate         — <2 sentences regardless of source.
    #
    # ``multi_sentence_rate`` is retained as a continuity metric.  The
    # doctrine-correct headline is ``articulate_sentence_rate``.
    _DISCLOSURE_SOURCES = {"oov", "refusal", "none"}
    articulate = sum(
        1 for r in results
        if r.sentence_count >= 2
        and r.grounding_source in {"pack", "teaching"}
    )
    disclosure = sum(
        1 for r in results
        if r.sentence_count >= 2
        and r.grounding_source in _DISCLOSURE_SOURCES
    )
    unarticulate = sum(1 for r in results if r.sentence_count < 2)

    metrics: dict[str, Any] = {
        "cases": total,
        "multi_sentence_rate":       round(multi / total, 4) if total else 0.0,
        "articulate_sentence_rate":  round(articulate / total, 4) if total else 0.0,
        "disclosure_sentence_rate":  round(disclosure / total, 4) if total else 0.0,
        "unarticulate_rate":         round(unarticulate / total, 4) if total else 0.0,
        "non_fragment_rate":         round(non_frag / total, 4) if total else 0.0,
        "grounded_rate":             round(grounded / total, 4) if total else 0.0,
        "subject_named_rate":        round(named / total, 4) if total else 0.0,
        "connective_present_rate":   conn_rate,
    }

    primed_results = [r for r in results if r.primed]
    metrics["primed_cases"] = len(primed_results)
    if primed_results:
        multi_primed = sum(1 for r in primed_results if r.sentence_count >= 2)
        metrics["primed_multi_sentence_rate"] = round(
            multi_primed / len(primed_results), 4
        )
    else:
        metrics["primed_multi_sentence_rate"] = 0.0

    case_details = [
        {
            "case_id": r.case_id,
            "category": r.category,
            "prompt": r.prompt,
            "surface": r.surface,
            "grounding_source": r.grounding_source,
            "sentence_count": r.sentence_count,
            "each_sentence_long_enough": r.each_sentence_long_enough,
            "connective_present": r.connective_present,
            "grounded": r.grounded,
            "subject_named": r.subject_named,
            "expects_connective": r.expects_connective,
            "primed": r.primed,
        }
        for r in results
    ]

    return LaneReport(metrics=metrics, case_details=case_details)