core/evals/articulation_of_status/runner.py

"""articulation-of-status lane runner.

Teaches a SPECULATIVE correction against a primed prior, then probes
the system and inspects the surface for an explicit status marker.

This lane is expected to fail at v1. The realizer today does not
consult `pack_mutation_proposal.epistemic_status` when forming the
surface, so SPECULATIVE-backed surfaces are indistinguishable from
COHERENT-backed ones. The lane makes that gap visible.

Conforms to the framework interface: run_lane(cases, config=None) -> report.
"""

from __future__ import annotations

import re
from dataclasses import dataclass, field
from typing import Any

from chat.runtime import ChatRuntime
from core.cognition.pipeline import CognitiveTurnPipeline
from core.config import RuntimeConfig
from evals.parallel import run_cases_parallel


SPECULATIVE_MARKERS: tuple[str, ...] = (
    "speculative",
    "not yet reviewed",
    "unconfirmed",
    "pending review",
    "tentatively",
    "as taught but unreviewed",
    "awaiting coherence judgment",
)

_BARE_ASSERTION_PATTERNS: tuple[re.Pattern[str], ...] = (
    re.compile(r"\bis defined as\b", re.IGNORECASE),
    re.compile(r"\bmeans\b", re.IGNORECASE),
    re.compile(r"^[A-Z][a-z]+ is [a-z]", re.MULTILINE),
)


@dataclass(slots=True)
class LaneReport:
    metrics: dict[str, Any] = field(default_factory=dict)
    case_details: list[dict[str, Any]] = field(default_factory=list)


def _marks_speculative(surface: str) -> bool:
    s = surface.lower()
    return any(m in s for m in SPECULATIVE_MARKERS)


def _bare_assertion(surface: str) -> bool:
    return any(p.search(surface) for p in _BARE_ASSERTION_PATTERNS)


def _run_case(case: dict[str, Any]) -> dict[str, Any]:
    runtime = ChatRuntime()
    pipeline = CognitiveTurnPipeline(runtime)

    prior = case.get("prior", "")
    teach = case.get("teach", "")
    probe = case["probe"]
    kind = case.get("kind", "")

    teach_status = ""
    if prior:
        try:
            pipeline.run(prior, max_tokens=8)
        except ValueError:
            pass
    if teach:
        try:
            teach_result = pipeline.run(teach, max_tokens=8)
            proposal = teach_result.pack_mutation_proposal
            teach_status = proposal.epistemic_status.value if proposal is not None else ""
        except ValueError:
            pass

    try:
        probe_result = pipeline.run(probe, max_tokens=12)
        surface = probe_result.surface
    except ValueError:
        surface = ""

    marks_spec = _marks_speculative(surface)
    bare = _bare_assertion(surface) and not marks_spec

    if kind in {"speculative_definition", "speculative_query"}:
        passed = marks_spec and not bare
    elif kind == "no_teaching_control":
        passed = not bare
    else:
        passed = False

    return {
        "id": case.get("id", ""),
        "kind": kind,
        "teach_status": teach_status,
        "surface": surface,
        "marks_speculative": marks_spec,
        "bare_assertion": bare,
        "passed": passed,
    }


def run_lane(
    cases: list[dict[str, Any]],
    *,
    config: RuntimeConfig | None = None,
    workers: int | None = None,
) -> LaneReport:
    if not cases:
        return LaneReport(metrics={}, case_details=[])
    _ = config

    case_details = run_cases_parallel(cases, _run_case, workers=workers)

    spec_cases = [
        d for d in case_details
        if d["kind"] in {"speculative_definition", "speculative_query"}
    ]
    control_cases = [d for d in case_details if d["kind"] == "no_teaching_control"]

    spec_rate = (
        sum(1 for d in spec_cases if d["marks_speculative"]) / len(spec_cases)
        if spec_cases else 0.0
    )
    false_certainty = (
        sum(1 for d in spec_cases if d["bare_assertion"]) / len(spec_cases)
        if spec_cases else 0.0
    )
    control_ok_rate = (
        sum(1 for d in control_cases if d["passed"]) / len(control_cases)
        if control_cases else 1.0
    )

    overall_pass = (
        spec_rate >= 0.90
        and false_certainty == 0.0
        and control_ok_rate >= 0.90
    )

    metrics: dict[str, Any] = {
        "speculative_articulation_rate": round(spec_rate, 4),
        "false_certainty_rate": round(false_certainty, 4),
        "control_ok_rate": round(control_ok_rate, 4),
        "speculative_case_count": len(spec_cases),
        "control_case_count": len(control_cases),
        "overall_pass": overall_pass,
    }
    return LaneReport(metrics=metrics, case_details=case_details)