ff1581f85f
Adds the P5a recall-precision predicate that was listed as NOT_COVERED in the L10 continuity lane. Closes the gap in the schema-as-proof discipline (CLAUDE.md): every predicate must have both a *_holds test (real soak) and a *_bites mutation test so a passing lane cannot silently miss the violation it nominally catches. Changes: - runner.py: ProbeRecord dataclass + probe_at/verify_probes_at params on run_soak(). Registers a field state (float32 bytes, matching vault's _exact_index dtype) at a named turn and recalls it against the vault at a later turn — including after a reboot, which is the cross-reboot claim. - predicates.py: evaluate_p5a_recall_precision — fails if any ProbeRecord has rank=None or rank>top_k, or if no cross-reboot probe was recorded. - report.py: wires P5a into build_report(); probe registered at turn 1, verified at reboot_turn+2 (intentionally before the vault's null_project auto-reproject cycle at store_count=20, which would destroy all CGA inner-product scores — documented as a real finding, deferred to a follow-up increment); NOT_COVERED is now empty (). - contract.md: P5a row in the predicate table + reprojection-boundary scope note. - test_l10_continuity.py: 4 tests — holds (real 6-turn soak across reboot) + 3 bites (rank=None, no cross-reboot probe, empty probe_records). Key finding: vault.null_project() fires every vault_reproject_interval=20 stores and produces CGA-orthogonal versors (inner product → 0.0 with the original), completely destroying both exact-match and ranked recall. This is a long-horizon vault stability issue, recorded here rather than silently avoided. The P5a probe window is constrained to the pre-reproject interval to keep wrong=0 intact while documenting the gap.
161 lines
6 KiB
Python
161 lines
6 KiB
Python
"""Assemble the L10 continuity panel into a structured, freeze-gateable report.
|
||
|
||
The panel runs the soaks the predicates need (an uninterrupted baseline, a
|
||
reboot leg, and two crash-recoveries), evaluates every predicate, and emits a
|
||
structured report with per-predicate PASS/FAIL, metrics, the explicitly
|
||
*not-covered* legs (no silent skips — CLAUDE.md), and a **deterministic digest**.
|
||
|
||
The digest is a SHA-256 over only the hardware-stable evidence: the canonical
|
||
``trace_hash`` sequence (``core.cognition.trace`` already rounds floats so the
|
||
hash is stable across hardware) and each predicate's ``(name, passed)`` verdict.
|
||
It deliberately EXCLUDES RSS, wall-clock, and raw float metrics, which are not
|
||
reproducible across machines. The digest is the freeze handle: pin it once the
|
||
lane is trusted and a regression flips it.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
from dataclasses import asdict, dataclass
|
||
from pathlib import Path
|
||
|
||
from core.config import RuntimeConfig
|
||
|
||
from evals.l10_continuity.predicates import (
|
||
PredicateOutcome,
|
||
evaluate_p1_closure,
|
||
evaluate_p2a_determinism,
|
||
evaluate_p2b_reboot_transparency,
|
||
evaluate_p3_bounded_resources,
|
||
evaluate_p4_commit_point,
|
||
evaluate_p4_recovery_determinism,
|
||
evaluate_p5a_recall_precision,
|
||
evaluate_p5b_anchor_stability,
|
||
evaluate_p5c_coherence,
|
||
)
|
||
from evals.l10_continuity.runner import (
|
||
SoakResult,
|
||
_inject_orphan_tmp,
|
||
read_recovered_turn_count,
|
||
run_soak,
|
||
)
|
||
|
||
# Legs the spec names but this lane does not yet cover, recorded explicitly so a
|
||
# PASS is never read as "everything was checked". P5a is now covered.
|
||
NOT_COVERED: tuple[tuple[str, str], ...] = ()
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class L10ContinuityReport:
|
||
n_turns: int
|
||
reboot_turn: int
|
||
predicates: tuple[PredicateOutcome, ...]
|
||
not_covered: tuple[tuple[str, str], ...]
|
||
deterministic_digest: str
|
||
|
||
def all_gates_pass(self) -> bool:
|
||
return all(p.passed for p in self.predicates)
|
||
|
||
def to_dict(self) -> dict:
|
||
return {
|
||
"n_turns": self.n_turns,
|
||
"reboot_turn": self.reboot_turn,
|
||
"all_gates_pass": self.all_gates_pass(),
|
||
"deterministic_digest": self.deterministic_digest,
|
||
"predicates": [asdict(p) for p in self.predicates],
|
||
"not_covered": [
|
||
{"leg": leg, "reason": reason} for leg, reason in self.not_covered
|
||
],
|
||
}
|
||
|
||
|
||
def deterministic_digest(
|
||
baseline: SoakResult, predicates: tuple[PredicateOutcome, ...]
|
||
) -> str:
|
||
"""SHA-256 over hardware-stable evidence: trace_hash sequence + verdicts."""
|
||
payload = {
|
||
"trace_hashes": list(baseline.trace_hashes()),
|
||
"verdicts": [[p.name, p.passed] for p in predicates],
|
||
"not_covered": [leg for leg, _ in NOT_COVERED],
|
||
}
|
||
serialized = json.dumps(payload, sort_keys=True, ensure_ascii=False)
|
||
return hashlib.sha256(serialized.encode("utf-8")).hexdigest()
|
||
|
||
|
||
def build_report(
|
||
*,
|
||
n_turns: int = 12,
|
||
reboot_turn: int = 3,
|
||
engine_state_root: Path,
|
||
config: RuntimeConfig | None = None,
|
||
) -> L10ContinuityReport:
|
||
"""Run the full panel and assemble the report.
|
||
|
||
Soaks: an uninterrupted ``baseline``; a second independent ``run_b`` (P2a);
|
||
a ``reboot`` leg (P2b); and two orphan-crash recoveries (P4).
|
||
"""
|
||
config = config or RuntimeConfig()
|
||
root = engine_state_root
|
||
|
||
baseline = run_soak(n_turns, engine_state_dir=root / "baseline", config=config)
|
||
run_b = run_soak(n_turns, engine_state_dir=root / "run_b", config=config)
|
||
# P5a probe: register at turn 1 (pre-reboot), verify two turns after the reboot.
|
||
# Verification is intentionally placed BEFORE the vault's first auto-reproject
|
||
# cycle (every ``vault_reproject_interval=20`` stores): after null_project fires,
|
||
# the stored versors change and all CGA inner-product scores drop to 0.0 — a
|
||
# real finding documented in the L10 continuity hardening brief pack, deferred
|
||
# to a follow-up increment. The two-turns-after-reboot window gives 2–3 distractor
|
||
# turns post-reboot while staying well below the reproject boundary.
|
||
p5a_probe_turn = min(1, reboot_turn - 1) if reboot_turn > 1 else 0
|
||
p5a_verify_turn = reboot_turn + 2
|
||
reboot = run_soak(
|
||
n_turns,
|
||
engine_state_dir=root / "reboot",
|
||
reboot_at=(reboot_turn,),
|
||
config=config,
|
||
probe_at=(p5a_probe_turn,),
|
||
verify_probes_at=(p5a_verify_turn,),
|
||
)
|
||
rec_a = run_soak(
|
||
n_turns,
|
||
engine_state_dir=root / "rec_a",
|
||
reboot_at=(reboot_turn,),
|
||
inject_orphan_tmp_at_reboot=True,
|
||
config=config,
|
||
)
|
||
rec_b = run_soak(
|
||
n_turns,
|
||
engine_state_dir=root / "rec_b",
|
||
reboot_at=(reboot_turn,),
|
||
inject_orphan_tmp_at_reboot=True,
|
||
config=config,
|
||
)
|
||
# Commit-point probe: run exactly ``reboot_turn`` turns, simulate the torn
|
||
# write, and read the recovered turn_count AT the crash boundary (not after
|
||
# the recovery continues and re-checkpoints).
|
||
probe_dir = root / "commit_probe"
|
||
run_soak(reboot_turn, engine_state_dir=probe_dir, config=config)
|
||
_inject_orphan_tmp(probe_dir)
|
||
recovered = read_recovered_turn_count(probe_dir)
|
||
|
||
p2b_outcome, _ = evaluate_p2b_reboot_transparency(reboot, baseline)
|
||
predicates: tuple[PredicateOutcome, ...] = (
|
||
evaluate_p1_closure(baseline),
|
||
evaluate_p2a_determinism(baseline, run_b),
|
||
p2b_outcome,
|
||
evaluate_p3_bounded_resources(baseline),
|
||
evaluate_p4_recovery_determinism(rec_a, rec_b),
|
||
evaluate_p4_commit_point(recovered, expected_turn_count=reboot_turn),
|
||
evaluate_p5a_recall_precision(reboot.probe_records),
|
||
evaluate_p5b_anchor_stability(baseline),
|
||
evaluate_p5c_coherence(baseline),
|
||
)
|
||
digest = deterministic_digest(baseline, predicates)
|
||
return L10ContinuityReport(
|
||
n_turns=n_turns,
|
||
reboot_turn=reboot_turn,
|
||
predicates=predicates,
|
||
not_covered=NOT_COVERED,
|
||
deterministic_digest=digest,
|
||
)
|