core/evals/l10_continuity/report.py

"""Assemble the L10 continuity panel into a structured, freeze-gateable report.

The panel runs the soaks the predicates need (an uninterrupted baseline, a
reboot leg, and two crash-recoveries), evaluates every predicate, and emits a
structured report with per-predicate PASS/FAIL, metrics, the explicitly
*not-covered* legs (no silent skips — CLAUDE.md), and a **deterministic digest**.

The digest is a SHA-256 over only the hardware-stable evidence: the canonical
``trace_hash`` sequence (``core.cognition.trace`` already rounds floats so the
hash is stable across hardware) and each predicate's ``(name, passed)`` verdict.
It deliberately EXCLUDES RSS, wall-clock, and raw float metrics, which are not
reproducible across machines. The digest is the freeze handle: pin it once the
lane is trusted and a regression flips it.
"""

from __future__ import annotations

import hashlib
import json
from dataclasses import asdict, dataclass
from pathlib import Path

from core.config import RuntimeConfig

from evals.l10_continuity.predicates import (
    PredicateOutcome,
    evaluate_p1_closure,
    evaluate_p2a_determinism,
    evaluate_p2b_reboot_transparency,
    evaluate_p3_bounded_resources,
    evaluate_p4_commit_point,
    evaluate_p4_recovery_determinism,
    evaluate_p5b_anchor_stability,
    evaluate_p5c_coherence,
)
from evals.l10_continuity.runner import (
    SoakResult,
    _inject_orphan_tmp,
    read_recovered_turn_count,
    run_soak,
)

# Legs the spec names but this lane does not yet cover, recorded explicitly so a
# PASS is never read as "everything was checked".
NOT_COVERED: tuple[tuple[str, str], ...] = (
    (
        "P5a_recall_stability",
        "recall precision@k over a held-out probe set requires a probe set with "
        "known-relevant entries and a metric grounded in the vault's scoring "
        "semantics (the raw recall score is not a clean similarity); deferred.",
    ),
)


@dataclass(frozen=True)
class L10ContinuityReport:
    n_turns: int
    reboot_turn: int
    predicates: tuple[PredicateOutcome, ...]
    not_covered: tuple[tuple[str, str], ...]
    deterministic_digest: str

    def all_gates_pass(self) -> bool:
        return all(p.passed for p in self.predicates)

    def to_dict(self) -> dict:
        return {
            "n_turns": self.n_turns,
            "reboot_turn": self.reboot_turn,
            "all_gates_pass": self.all_gates_pass(),
            "deterministic_digest": self.deterministic_digest,
            "predicates": [asdict(p) for p in self.predicates],
            "not_covered": [
                {"leg": leg, "reason": reason} for leg, reason in self.not_covered
            ],
        }


def deterministic_digest(
    baseline: SoakResult, predicates: tuple[PredicateOutcome, ...]
) -> str:
    """SHA-256 over hardware-stable evidence: trace_hash sequence + verdicts."""
    payload = {
        "trace_hashes": list(baseline.trace_hashes()),
        "verdicts": [[p.name, p.passed] for p in predicates],
        "not_covered": [leg for leg, _ in NOT_COVERED],
    }
    serialized = json.dumps(payload, sort_keys=True, ensure_ascii=False)
    return hashlib.sha256(serialized.encode("utf-8")).hexdigest()


def build_report(
    *,
    n_turns: int = 12,
    reboot_turn: int = 3,
    engine_state_root: Path,
    config: RuntimeConfig | None = None,
) -> L10ContinuityReport:
    """Run the full panel and assemble the report.

    Soaks: an uninterrupted ``baseline``; a second independent ``run_b`` (P2a);
    a ``reboot`` leg (P2b); and two orphan-crash recoveries (P4).
    """
    config = config or RuntimeConfig()
    root = engine_state_root

    baseline = run_soak(n_turns, engine_state_dir=root / "baseline", config=config)
    run_b = run_soak(n_turns, engine_state_dir=root / "run_b", config=config)
    reboot = run_soak(
        n_turns, engine_state_dir=root / "reboot", reboot_at=(reboot_turn,), config=config
    )
    rec_a = run_soak(
        n_turns,
        engine_state_dir=root / "rec_a",
        reboot_at=(reboot_turn,),
        inject_orphan_tmp_at_reboot=True,
        config=config,
    )
    rec_b = run_soak(
        n_turns,
        engine_state_dir=root / "rec_b",
        reboot_at=(reboot_turn,),
        inject_orphan_tmp_at_reboot=True,
        config=config,
    )
    # Commit-point probe: run exactly ``reboot_turn`` turns, simulate the torn
    # write, and read the recovered turn_count AT the crash boundary (not after
    # the recovery continues and re-checkpoints).
    probe_dir = root / "commit_probe"
    run_soak(reboot_turn, engine_state_dir=probe_dir, config=config)
    _inject_orphan_tmp(probe_dir)
    recovered = read_recovered_turn_count(probe_dir)

    p2b_outcome, _ = evaluate_p2b_reboot_transparency(reboot, baseline)
    predicates: tuple[PredicateOutcome, ...] = (
        evaluate_p1_closure(baseline),
        evaluate_p2a_determinism(baseline, run_b),
        p2b_outcome,
        evaluate_p3_bounded_resources(baseline),
        evaluate_p4_recovery_determinism(rec_a, rec_b),
        evaluate_p4_commit_point(recovered, expected_turn_count=reboot_turn),
        evaluate_p5b_anchor_stability(baseline),
        evaluate_p5c_coherence(baseline),
    )
    digest = deterministic_digest(baseline, predicates)
    return L10ContinuityReport(
        n_turns=n_turns,
        reboot_turn=reboot_turn,
        predicates=predicates,
        not_covered=NOT_COVERED,
        deterministic_digest=digest,
    )