core/evals/l10_continuity/report.py

"""Assemble the L10 continuity panel into a structured, freeze-gateable report.

The panel runs the soaks the predicates need (an uninterrupted baseline, a
reboot leg, and two crash-recoveries), evaluates every predicate, and emits a
structured report with per-predicate PASS/FAIL, metrics, the explicitly
*not-covered* legs (no silent skips — CLAUDE.md), and a **deterministic digest**.

The digest is a SHA-256 over only the hardware-stable evidence: the canonical
``trace_hash`` sequence (``core.cognition.trace`` already rounds floats so the
hash is stable across hardware) and each predicate's ``(name, passed)`` verdict.
It deliberately EXCLUDES RSS, wall-clock, and raw float metrics, which are not
reproducible across machines. The digest is the freeze handle: pin it once the
lane is trusted and a regression flips it.
"""

from __future__ import annotations

import hashlib
import json
from dataclasses import asdict, dataclass
from pathlib import Path

from core.config import RuntimeConfig

from evals.l10_continuity.predicates import (
    CutPointEvidence,
    PredicateOutcome,
    evaluate_p1_closure,
    evaluate_p2a_determinism,
    evaluate_p2b_reboot_transparency,
    evaluate_p3_bounded_resources,
    evaluate_p4_arbitrary_interruption,
    evaluate_p4_commit_point,
    evaluate_p4_recovery_determinism,
    evaluate_p5a_recall_precision,
    evaluate_p5b_anchor_stability,
    evaluate_p5c_coherence,
)
from evals.l10_continuity.runner import (
    InterruptionCutPoint,
    SoakResult,
    _inject_at_cutpoint,
    _inject_orphan_tmp,
    read_recovered_turn_count,
    run_soak,
)

# Legs the spec names but this lane does not yet cover, recorded explicitly so a
# PASS is never read as "everything was checked".  P5a is now covered.
NOT_COVERED: tuple[tuple[str, str], ...] = ()


@dataclass(frozen=True)
class L10ContinuityReport:
    n_turns: int
    reboot_turn: int
    predicates: tuple[PredicateOutcome, ...]
    not_covered: tuple[tuple[str, str], ...]
    deterministic_digest: str

    def all_gates_pass(self) -> bool:
        return all(p.passed for p in self.predicates)

    def to_dict(self) -> dict:
        return {
            "n_turns": self.n_turns,
            "reboot_turn": self.reboot_turn,
            "all_gates_pass": self.all_gates_pass(),
            "deterministic_digest": self.deterministic_digest,
            "predicates": [asdict(p) for p in self.predicates],
            "not_covered": [
                {"leg": leg, "reason": reason} for leg, reason in self.not_covered
            ],
        }


def deterministic_digest(
    baseline: SoakResult, predicates: tuple[PredicateOutcome, ...]
) -> str:
    """SHA-256 over hardware-stable evidence: trace_hash sequence + verdicts."""
    payload = {
        "trace_hashes": list(baseline.trace_hashes()),
        "verdicts": [[p.name, p.passed] for p in predicates],
        "not_covered": [leg for leg, _ in NOT_COVERED],
    }
    serialized = json.dumps(payload, sort_keys=True, ensure_ascii=False)
    return hashlib.sha256(serialized.encode("utf-8")).hexdigest()


def build_report(
    *,
    n_turns: int = 12,
    reboot_turn: int = 3,
    engine_state_root: Path,
    config: RuntimeConfig | None = None,
) -> L10ContinuityReport:
    """Run the full panel and assemble the report.

    Soaks: an uninterrupted ``baseline``; a second independent ``run_b`` (P2a);
    a ``reboot`` leg (P2b); and two orphan-crash recoveries (P4).
    """
    config = config or RuntimeConfig()
    root = engine_state_root

    baseline = run_soak(n_turns, engine_state_dir=root / "baseline", config=config)
    run_b = run_soak(n_turns, engine_state_dir=root / "run_b", config=config)
    # P5a probe: register at turn 1 (pre-reboot), verify two turns after the reboot.
    # Verification is intentionally placed BEFORE the vault's first auto-reproject
    # cycle (every ``vault_reproject_interval=20`` stores): after null_project fires,
    # the stored versors change and all CGA inner-product scores drop to 0.0 — a
    # real finding documented in the L10 continuity hardening brief pack, deferred
    # to a follow-up increment. The two-turns-after-reboot window gives 2–3 distractor
    # turns post-reboot while staying well below the reproject boundary.
    p5a_probe_turn = min(1, reboot_turn - 1) if reboot_turn > 1 else 0
    p5a_verify_turn = reboot_turn + 2
    reboot = run_soak(
        n_turns,
        engine_state_dir=root / "reboot",
        reboot_at=(reboot_turn,),
        config=config,
        probe_at=(p5a_probe_turn,),
        verify_probes_at=(p5a_verify_turn,),
    )
    rec_a = run_soak(
        n_turns,
        engine_state_dir=root / "rec_a",
        reboot_at=(reboot_turn,),
        inject_orphan_tmp_at_reboot=True,
        config=config,
    )
    rec_b = run_soak(
        n_turns,
        engine_state_dir=root / "rec_b",
        reboot_at=(reboot_turn,),
        inject_orphan_tmp_at_reboot=True,
        config=config,
    )
    # Commit-point probe: run exactly ``reboot_turn`` turns, simulate the torn
    # write, and read the recovered turn_count AT the crash boundary (not after
    # the recovery continues and re-checkpoints).
    probe_dir = root / "commit_probe"
    run_soak(reboot_turn, engine_state_dir=probe_dir, config=config)
    _inject_orphan_tmp(probe_dir)
    recovered = read_recovered_turn_count(probe_dir)

    # W2-R arbitrary-interruption soaks: two cut-points × two recovery runs each.
    # Each pair exercises one ADR-0219 gate bullet empirically through the real
    # turn loop.  AFTER_SWAP is the clean-commit case covered by rec_a/rec_b above.
    w2r_evidence: list[CutPointEvidence] = []
    for cp in (InterruptionCutPoint.PARTIAL_GEN, InterruptionCutPoint.FULL_GEN_BEFORE_SWAP):
        # Commit-probe: run reboot_turn turns, inject cut-point, read committed gen.
        cp_probe_dir = root / f"cp_probe_{cp.value}"
        run_soak(reboot_turn, engine_state_dir=cp_probe_dir, config=config)
        _inject_at_cutpoint(cp_probe_dir, cp)
        cp_recovered = read_recovered_turn_count(cp_probe_dir)

        # Two independent recovery soaks for the determinism gate.
        cp_rec_a = run_soak(
            n_turns,
            engine_state_dir=root / f"cp_{cp.value}_a",
            reboot_at=(reboot_turn,),
            cutpoint_at_reboot=cp,
            config=config,
        )
        cp_rec_b = run_soak(
            n_turns,
            engine_state_dir=root / f"cp_{cp.value}_b",
            reboot_at=(reboot_turn,),
            cutpoint_at_reboot=cp,
            config=config,
        )
        w2r_evidence.append(
            CutPointEvidence(
                cut_point=cp.value,
                recovered_turn_count=cp_recovered,
                expected_turn_count=reboot_turn,
                tail_hashes_a=tuple(
                    r.trace_hash for r in cp_rec_a.post_reboot_records()
                ),
                tail_hashes_b=tuple(
                    r.trace_hash for r in cp_rec_b.post_reboot_records()
                ),
                all_versor_conditions=(
                    cp_rec_a.versor_conditions() + cp_rec_b.versor_conditions()
                ),
            )
        )

    p2b_outcome, _ = evaluate_p2b_reboot_transparency(reboot, baseline)
    predicates: tuple[PredicateOutcome, ...] = (
        evaluate_p1_closure(baseline),
        evaluate_p2a_determinism(baseline, run_b),
        p2b_outcome,
        evaluate_p3_bounded_resources(baseline),
        evaluate_p4_recovery_determinism(rec_a, rec_b),
        evaluate_p4_commit_point(recovered, expected_turn_count=reboot_turn),
        evaluate_p4_arbitrary_interruption(tuple(w2r_evidence)),
        evaluate_p5a_recall_precision(reboot.probe_records),
        evaluate_p5b_anchor_stability(baseline),
        evaluate_p5c_coherence(baseline),
    )
    digest = deterministic_digest(baseline, predicates)
    return L10ContinuityReport(
        n_turns=n_turns,
        reboot_turn=reboot_turn,
        predicates=predicates,
        not_covered=NOT_COVERED,
        deterministic_digest=digest,
    )