diff --git a/evals/l10_continuity/contract.md b/evals/l10_continuity/contract.md index 0da7a45c..1be9d95b 100644 --- a/evals/l10_continuity/contract.md +++ b/evals/l10_continuity/contract.md @@ -23,6 +23,7 @@ Run: `PYTHONPATH=. .venv/bin/python -m evals.l10_continuity [n_turns] [reboot_tu | **P3** bounded resources | vault grows linear-bounded/monotonic per turn | an unbounded cache/store leaks | a 10k-entry vault record trips it | | **P4** recovery determinism | two crash-recoveries from one checkpoint converge | torn read / nondeterministic boot | divergent recovery tails trip it | | **P4** commit point | recovered `turn_count` == committed turns (ARIES force boundary) | the checkpoint isn't the atomic commit boundary | `None`/mismatched count trips it | +| **P4** arbitrary interruption | for each ADR-0219 cut-point (partial gen dir, full gen dir before swap): loader ignores orphan and reads the prior committed gen; two independent recoveries converge; `versor_condition < 1e-6` throughout | orphan gen dir is loaded instead of committed gen; two recoveries diverge; closure breaks | wrong `recovered_turn_count`, diverging tails, or a vc ≥ 1e-6 trips it | | **P5a** recall precision | vault recall finds each probe entry at rank ≤ top_k, **including after a reboot** (float32 round-trip preserves `_exact_index`) | the serialisation round-trip loses precision, breaking exact-match after reboot | a `ProbeRecord` with `rank=None` or no cross-reboot probe trips it | | **P5b** anchor stability | field anchors without **collapse** (`dist_to_anchor`↛0) or **freeze** (`turn_movement`↛0) | the field is swallowed by the attractor, or frozen | collapsing distance / zero movement trips it | | **P5c** coherence | surfaces stay non-empty and not collapsed to one repeated output | the field wanders into noise or freezes onto one output | empty / single-surface records trip it | diff --git a/evals/l10_continuity/predicates.py b/evals/l10_continuity/predicates.py index 9f07c201..40f4716e 100644 --- a/evals/l10_continuity/predicates.py +++ b/evals/l10_continuity/predicates.py @@ -26,7 +26,12 @@ from __future__ import annotations import math from dataclasses import dataclass, field -from evals.l10_continuity.runner import ProbeRecord, SoakResult, TurnRecord +from evals.l10_continuity.runner import ( + InterruptionCutPoint, + ProbeRecord, + SoakResult, + TurnRecord, +) VERSOR_CEILING: float = 1e-6 @@ -288,6 +293,94 @@ def evaluate_p4_commit_point( ) +@dataclass(frozen=True, slots=True) +class CutPointEvidence: + """Evidence for one ADR-0219 interruption cut-point (W2-R arbitrary-interruption predicate). + + Each entry names the cut-point, the turn count recovered from the committed + generation (what the loader reports after the injection), the expected count + (what the committed generation holds), and the post-reboot trace-hash tails + from two independent recovery soaks (determinism check). + ``all_versor_conditions`` spans both recovery soaks — every entry must be + below ``VERSOR_CEILING``. + """ + + cut_point: str # InterruptionCutPoint value + recovered_turn_count: int | None # read_recovered_turn_count after injection + expected_turn_count: int + tail_hashes_a: tuple[str, ...] # post-reboot trace_hashes from recovery_a + tail_hashes_b: tuple[str, ...] # post-reboot trace_hashes from recovery_b + all_versor_conditions: tuple[float, ...] # every turn across both soaks + + +def evaluate_p4_arbitrary_interruption( + evidence: tuple[CutPointEvidence, ...], *, ceiling: float = VERSOR_CEILING +) -> PredicateOutcome: + """P4 (W2-R extension) — arbitrary-interruption recovery proves the ADR-0219 gate bullets. + + For each cut-point in ``evidence`` the predicate checks: + + 1. **Prior-gen load:** ``recovered_turn_count == expected_turn_count`` — + a kill before the ``current`` pointer swap leaves the prior committed + generation intact and readable. + 2. **Recovery determinism:** two independent recoveries from the same + committed checkpoint produce byte-identical post-reboot tails. + 3. **Closure:** ``versor_condition < ceiling`` for every turn across both + recovery soaks. + + The ``AFTER_SWAP`` cut-point is the clean-commit control case: no orphan is + injected, so it degenerates to the existing P4 recovery_determinism gate + (confirming the implementation is correct at the baseline). + """ + failures: list[str] = [] + metrics: dict = {} + + for ev in evidence: + cp = ev.cut_point + # Gate 1: prior-gen load (PARTIAL_GEN and FULL_BEFORE_SWAP) / + # committed-gen load (AFTER_SWAP). + if ev.recovered_turn_count != ev.expected_turn_count: + failures.append( + f"{cp}: recovered_turn_count={ev.recovered_turn_count} " + f"!= expected={ev.expected_turn_count}" + ) + + # Gate 2: recovery determinism — two soaks converge. + div = _first_divergence(ev.tail_hashes_a, ev.tail_hashes_b) + if div is not None or len(ev.tail_hashes_a) == 0: + failures.append( + f"{cp}: recovery tails diverged at index {div} " + f"(|a|={len(ev.tail_hashes_a)}, |b|={len(ev.tail_hashes_b)})" + ) + + # Gate 3: closure throughout. + bad_vc = [vc for vc in ev.all_versor_conditions if vc >= ceiling] + if bad_vc: + failures.append( + f"{cp}: {len(bad_vc)} versor_condition violations (max={max(bad_vc):.2e})" + ) + + metrics[cp] = { + "recovered_tc": ev.recovered_turn_count, + "expected_tc": ev.expected_turn_count, + "tail_len": len(ev.tail_hashes_a), + "vc_violations": len(bad_vc), + } + + passed = not failures + detail = ( + f"arbitrary-interruption gate: {len(evidence)} cut-points all pass" + if passed + else "; ".join(failures) + ) + return PredicateOutcome( + name="P4_arbitrary_interruption", + passed=passed, + detail=detail, + metrics=metrics, + ) + + def evaluate_p5b_anchor_stability( result: SoakResult, *, diff --git a/evals/l10_continuity/report.py b/evals/l10_continuity/report.py index d1acc311..96d0ede7 100644 --- a/evals/l10_continuity/report.py +++ b/evals/l10_continuity/report.py @@ -23,11 +23,13 @@ from pathlib import Path from core.config import RuntimeConfig from evals.l10_continuity.predicates import ( + CutPointEvidence, PredicateOutcome, evaluate_p1_closure, evaluate_p2a_determinism, evaluate_p2b_reboot_transparency, evaluate_p3_bounded_resources, + evaluate_p4_arbitrary_interruption, evaluate_p4_commit_point, evaluate_p4_recovery_determinism, evaluate_p5a_recall_precision, @@ -35,7 +37,9 @@ from evals.l10_continuity.predicates import ( evaluate_p5c_coherence, ) from evals.l10_continuity.runner import ( + InterruptionCutPoint, SoakResult, + _inject_at_cutpoint, _inject_orphan_tmp, read_recovered_turn_count, run_soak, @@ -139,6 +143,49 @@ def build_report( _inject_orphan_tmp(probe_dir) recovered = read_recovered_turn_count(probe_dir) + # W2-R arbitrary-interruption soaks: two cut-points × two recovery runs each. + # Each pair exercises one ADR-0219 gate bullet empirically through the real + # turn loop. AFTER_SWAP is the clean-commit case covered by rec_a/rec_b above. + w2r_evidence: list[CutPointEvidence] = [] + for cp in (InterruptionCutPoint.PARTIAL_GEN, InterruptionCutPoint.FULL_GEN_BEFORE_SWAP): + # Commit-probe: run reboot_turn turns, inject cut-point, read committed gen. + cp_probe_dir = root / f"cp_probe_{cp.value}" + run_soak(reboot_turn, engine_state_dir=cp_probe_dir, config=config) + _inject_at_cutpoint(cp_probe_dir, cp) + cp_recovered = read_recovered_turn_count(cp_probe_dir) + + # Two independent recovery soaks for the determinism gate. + cp_rec_a = run_soak( + n_turns, + engine_state_dir=root / f"cp_{cp.value}_a", + reboot_at=(reboot_turn,), + cutpoint_at_reboot=cp, + config=config, + ) + cp_rec_b = run_soak( + n_turns, + engine_state_dir=root / f"cp_{cp.value}_b", + reboot_at=(reboot_turn,), + cutpoint_at_reboot=cp, + config=config, + ) + w2r_evidence.append( + CutPointEvidence( + cut_point=cp.value, + recovered_turn_count=cp_recovered, + expected_turn_count=reboot_turn, + tail_hashes_a=tuple( + r.trace_hash for r in cp_rec_a.post_reboot_records() + ), + tail_hashes_b=tuple( + r.trace_hash for r in cp_rec_b.post_reboot_records() + ), + all_versor_conditions=( + cp_rec_a.versor_conditions() + cp_rec_b.versor_conditions() + ), + ) + ) + p2b_outcome, _ = evaluate_p2b_reboot_transparency(reboot, baseline) predicates: tuple[PredicateOutcome, ...] = ( evaluate_p1_closure(baseline), @@ -147,6 +194,7 @@ def build_report( evaluate_p3_bounded_resources(baseline), evaluate_p4_recovery_determinism(rec_a, rec_b), evaluate_p4_commit_point(recovered, expected_turn_count=reboot_turn), + evaluate_p4_arbitrary_interruption(tuple(w2r_evidence)), evaluate_p5a_recall_precision(reboot.probe_records), evaluate_p5b_anchor_stability(baseline), evaluate_p5c_coherence(baseline), diff --git a/evals/l10_continuity/runner.py b/evals/l10_continuity/runner.py index f2ad6555..fcd6d65e 100644 --- a/evals/l10_continuity/runner.py +++ b/evals/l10_continuity/runner.py @@ -30,6 +30,7 @@ from __future__ import annotations import resource from dataclasses import dataclass, replace +from enum import Enum from pathlib import Path import numpy as np @@ -125,6 +126,73 @@ def _new_runtime(config: RuntimeConfig, engine_state_dir: Path) -> ChatRuntime: return ChatRuntime(config=config, engine_state_path=engine_state_dir) +class InterruptionCutPoint(str, Enum): + """Enumeration of the three ADR-0219 checkpoint sub-steps where a kill can occur. + + Used by ``run_soak`` to inject the correct orphan shape and by the + ``evaluate_p4_arbitrary_interruption`` predicate to check the gate bullets. + """ + + PARTIAL_GEN = "partial_gen" # kill after gen-N+1 dir exists but only partially written + FULL_GEN_BEFORE_SWAP = "full_gen_before_swap" # kill after all files written but before current swap + AFTER_SWAP = "after_swap" # kill after current swap (the committed state); control case + + +def _inject_partial_gen_dir(engine_state_dir: Path) -> None: + """Inject: gen-9997/ with only one file, ``current`` unchanged. + + Simulates a kill between ``begin_generation`` and the last ``_atomic_write_text`` + call in the generation dir: the directory exists with partial content, but + ``current`` still names the prior committed generation. The loader follows + ``current`` and never reads the partial dir. + """ + orphan = engine_state_dir / "gen-9997" + orphan.mkdir(exist_ok=True) + (orphan / "manifest.json").write_text( + '{"PARTIAL_WRITE":true,"note":"kill before all files written"}', + encoding="utf-8", + ) + + +def _inject_full_gen_dir_before_swap(engine_state_dir: Path) -> None: + """Inject: gen-9997/ with all four files, ``current`` unchanged. + + Simulates a kill after all generation files are written and fsynced but + before the atomic ``os.replace`` of ``current`` (step 2 of + ``commit_generation``). The complete gen dir exists but is unreferenced; + ``current`` still names the prior committed generation. + """ + orphan = engine_state_dir / "gen-9997" + orphan.mkdir(exist_ok=True) + for fname in ( + "recognizers.jsonl", + "discovery_candidates.jsonl", + "session_state.json", + "manifest.json", + ): + (orphan / fname).write_text( + f'{{"FULL_WRITE_BEFORE_SWAP":true,"file":"{fname}"}}', + encoding="utf-8", + ) + + +def _inject_at_cutpoint( + engine_state_dir: Path, cut_point: InterruptionCutPoint +) -> None: + """Dispatch to the appropriate injection function for ``cut_point``. + + ``AFTER_SWAP`` is the normal clean-commit case — no orphan is injected; + the loader reads the committed generation as expected. + """ + match cut_point: + case InterruptionCutPoint.PARTIAL_GEN: + _inject_partial_gen_dir(engine_state_dir) + case InterruptionCutPoint.FULL_GEN_BEFORE_SWAP: + _inject_full_gen_dir_before_swap(engine_state_dir) + case InterruptionCutPoint.AFTER_SWAP: + pass # no injection: the committed generation is the expected state + + def _inject_orphan_tmp(engine_state_dir: Path) -> None: """Simulate a kill mid-checkpoint-write under the generation-dir model (ADR-0219). @@ -184,6 +252,7 @@ def run_soak( reboot_at: tuple[int, ...] = (), config: RuntimeConfig | None = None, inject_orphan_tmp_at_reboot: bool = False, + cutpoint_at_reboot: InterruptionCutPoint | None = None, probe_at: tuple[int, ...] = (), verify_probes_at: tuple[int, ...] = (), probe_top_k: int = 5, @@ -197,6 +266,12 @@ def run_soak( in the checkpoint dir immediately before each reconstruct, so the reboot exercises ADR-0156 crash recovery rather than a clean restart. + W2-R arbitrary-interruption injection: when ``cutpoint_at_reboot`` is given, + the corresponding ADR-0219 orphan shape is injected at each reboot boundary + (in addition to any ``inject_orphan_tmp_at_reboot`` injection). The three + cut-points model kills at each checkpoint sub-step; ``AFTER_SWAP`` is the + clean-commit control case and injects nothing. + P5a vault-recall probes: ``probe_at`` names turns at which the current field state is captured as a probe query (float32 bytes, matching the vault's storage dtype). ``verify_probes_at`` names turns at which every registered @@ -222,6 +297,8 @@ def run_soak( if i in reboot_set: if inject_orphan_tmp_at_reboot: _inject_orphan_tmp(engine_state_dir) + if cutpoint_at_reboot is not None: + _inject_at_cutpoint(engine_state_dir, cutpoint_at_reboot) runtime = _new_runtime(config, engine_state_dir) pipe = CognitiveTurnPipeline(runtime=runtime) segment += 1 diff --git a/tests/test_l10_arbitrary_interruption.py b/tests/test_l10_arbitrary_interruption.py new file mode 100644 index 00000000..45f6eb89 --- /dev/null +++ b/tests/test_l10_arbitrary_interruption.py @@ -0,0 +1,209 @@ +"""W2-R — Arbitrary-interruption recovery harness. + +Tests the ``evaluate_p4_arbitrary_interruption`` predicate empirically against +the ADR-0219 generation-dir checkpoint model (W1-A). Each cut-point has a +``*_holds`` test (real soak) and a ``*_bites`` mutation test (synthetic evidence +that would make the predicate fail). + +Cut-points tested: +- ``PARTIAL_GEN``: gen-9997 exists with one file; ``current`` unchanged. + Loader follows ``current``, ignores the partial dir, loads prior committed gen. +- ``FULL_GEN_BEFORE_SWAP``: gen-9997 exists with all four files; ``current`` + unchanged. Loader still ignores the unreferenced complete gen dir. +- ``AFTER_SWAP``: no injection (the normal clean-commit control case); verifies + that the control case passes under the generation-dir model. + +Per CLAUDE.md schema-as-proof discipline: a predicate that cannot fail under +the violation it nominally catches is decoration, not proof. +""" + +from __future__ import annotations + +from pathlib import Path + +from evals.l10_continuity.predicates import ( + CutPointEvidence, + evaluate_p4_arbitrary_interruption, +) +from evals.l10_continuity.runner import ( + InterruptionCutPoint, + _inject_at_cutpoint, + _inject_full_gen_dir_before_swap, + _inject_partial_gen_dir, + read_recovered_turn_count, + run_soak, +) + + +# --------------------------------------------------------------------------- +# Injection function unit tests (no soak needed) +# --------------------------------------------------------------------------- + + +def test_inject_partial_gen_dir_creates_orphan(tmp_path: Path): + _inject_partial_gen_dir(tmp_path) + orphan = tmp_path / "gen-9997" + assert orphan.is_dir() + assert (orphan / "manifest.json").exists() + # Must NOT contain all four files (it's PARTIAL). + assert not (orphan / "session_state.json").exists() + + +def test_inject_full_gen_dir_before_swap_creates_all_four_files(tmp_path: Path): + _inject_full_gen_dir_before_swap(tmp_path) + orphan = tmp_path / "gen-9997" + assert orphan.is_dir() + for fname in ("recognizers.jsonl", "discovery_candidates.jsonl", "session_state.json", "manifest.json"): + assert (orphan / fname).exists(), f"expected {fname} in full-gen orphan" + + +def test_inject_at_cutpoint_after_swap_is_noop(tmp_path: Path): + _inject_at_cutpoint(tmp_path, InterruptionCutPoint.AFTER_SWAP) + # AFTER_SWAP injects nothing — directory should be empty (no orphan gen). + gen_dirs = [d for d in tmp_path.iterdir() if d.is_dir() and d.name.startswith("gen-")] + assert len(gen_dirs) == 0 + + +# --------------------------------------------------------------------------- +# Holds tests — real soaks exercise each cut-point +# --------------------------------------------------------------------------- + + +def test_p4_arbitrary_interruption_partial_gen_holds(tmp_path: Path): + """PARTIAL_GEN: orphan partial gen dir is ignored, prior gen loaded, recoveries converge.""" + cp = InterruptionCutPoint.PARTIAL_GEN + reboot_turn = 3 + n_turns = 6 + + # Commit-probe: verify loader reads prior committed gen (not the orphan). + probe_dir = tmp_path / "cp_probe" + run_soak(reboot_turn, engine_state_dir=probe_dir) + _inject_at_cutpoint(probe_dir, cp) + recovered_tc = read_recovered_turn_count(probe_dir) + assert recovered_tc == reboot_turn, ( + f"PARTIAL_GEN: expected committed turn_count={reboot_turn}, got {recovered_tc}" + ) + + # Recovery soaks: two independent soaks with PARTIAL_GEN injection must converge. + rec_a = run_soak(n_turns, engine_state_dir=tmp_path / "a", reboot_at=(reboot_turn,), cutpoint_at_reboot=cp) + rec_b = run_soak(n_turns, engine_state_dir=tmp_path / "b", reboot_at=(reboot_turn,), cutpoint_at_reboot=cp) + tail_a = tuple(r.trace_hash for r in rec_a.post_reboot_records()) + tail_b = tuple(r.trace_hash for r in rec_b.post_reboot_records()) + assert tail_a == tail_b, "PARTIAL_GEN: two recovery soaks diverged" + + # Full predicate. + evidence = ( + CutPointEvidence( + cut_point=cp.value, + recovered_turn_count=recovered_tc, + expected_turn_count=reboot_turn, + tail_hashes_a=tail_a, + tail_hashes_b=tail_b, + all_versor_conditions=rec_a.versor_conditions() + rec_b.versor_conditions(), + ), + ) + outcome = evaluate_p4_arbitrary_interruption(evidence) + assert outcome.passed, outcome.detail + + +def test_p4_arbitrary_interruption_full_gen_before_swap_holds(tmp_path: Path): + """FULL_GEN_BEFORE_SWAP: complete but unreferenced gen dir is ignored.""" + cp = InterruptionCutPoint.FULL_GEN_BEFORE_SWAP + reboot_turn = 3 + n_turns = 6 + + probe_dir = tmp_path / "cp_probe" + run_soak(reboot_turn, engine_state_dir=probe_dir) + _inject_at_cutpoint(probe_dir, cp) + recovered_tc = read_recovered_turn_count(probe_dir) + assert recovered_tc == reboot_turn, ( + f"FULL_BEFORE_SWAP: expected committed turn_count={reboot_turn}, got {recovered_tc}" + ) + + rec_a = run_soak(n_turns, engine_state_dir=tmp_path / "a", reboot_at=(reboot_turn,), cutpoint_at_reboot=cp) + rec_b = run_soak(n_turns, engine_state_dir=tmp_path / "b", reboot_at=(reboot_turn,), cutpoint_at_reboot=cp) + tail_a = tuple(r.trace_hash for r in rec_a.post_reboot_records()) + tail_b = tuple(r.trace_hash for r in rec_b.post_reboot_records()) + assert tail_a == tail_b, "FULL_BEFORE_SWAP: two recovery soaks diverged" + + evidence = ( + CutPointEvidence( + cut_point=cp.value, + recovered_turn_count=recovered_tc, + expected_turn_count=reboot_turn, + tail_hashes_a=tail_a, + tail_hashes_b=tail_b, + all_versor_conditions=rec_a.versor_conditions() + rec_b.versor_conditions(), + ), + ) + outcome = evaluate_p4_arbitrary_interruption(evidence) + assert outcome.passed, outcome.detail + + +# --------------------------------------------------------------------------- +# Bites tests — synthetic evidence that trips each gate bullet +# --------------------------------------------------------------------------- + + +def test_p4_arbitrary_interruption_bites_on_wrong_recovered_turn_count(): + """Wrong turn_count: orphan gen loaded instead of committed gen → predicate fails.""" + evidence = ( + CutPointEvidence( + cut_point=InterruptionCutPoint.PARTIAL_GEN.value, + recovered_turn_count=0, # wrong: orphan's manifest was read (0 turns) + expected_turn_count=3, + tail_hashes_a=("abc",), + tail_hashes_b=("abc",), + all_versor_conditions=(1e-13,), + ), + ) + outcome = evaluate_p4_arbitrary_interruption(evidence) + assert not outcome.passed, "predicate must fail when recovered_turn_count != expected" + + +def test_p4_arbitrary_interruption_bites_on_diverging_recovery_tails(): + """Diverging tails: two independent recoveries produce different trace_hashes → fails.""" + evidence = ( + CutPointEvidence( + cut_point=InterruptionCutPoint.FULL_GEN_BEFORE_SWAP.value, + recovered_turn_count=3, + expected_turn_count=3, + tail_hashes_a=("hash_a1", "hash_a2"), + tail_hashes_b=("hash_b1", "hash_b2"), # different → divergence + all_versor_conditions=(1e-13, 1e-13), + ), + ) + outcome = evaluate_p4_arbitrary_interruption(evidence) + assert not outcome.passed, "predicate must fail when recovery tails diverge" + + +def test_p4_arbitrary_interruption_bites_on_versor_condition_violation(): + """Closure violation: versor_condition >= 1e-6 in a recovery soak → fails.""" + evidence = ( + CutPointEvidence( + cut_point=InterruptionCutPoint.PARTIAL_GEN.value, + recovered_turn_count=3, + expected_turn_count=3, + tail_hashes_a=("abc",), + tail_hashes_b=("abc",), + all_versor_conditions=(1e-6,), # at the ceiling → violation + ), + ) + outcome = evaluate_p4_arbitrary_interruption(evidence) + assert not outcome.passed, "predicate must fail when versor_condition >= 1e-6" + + +def test_p4_arbitrary_interruption_bites_on_empty_tails(): + """Empty recovery tails: no post-reboot evidence → fails.""" + evidence = ( + CutPointEvidence( + cut_point=InterruptionCutPoint.PARTIAL_GEN.value, + recovered_turn_count=3, + expected_turn_count=3, + tail_hashes_a=(), # empty: no post-reboot records + tail_hashes_b=(), + all_versor_conditions=(), + ), + ) + outcome = evaluate_p4_arbitrary_interruption(evidence) + assert not outcome.passed, "predicate must fail with empty recovery tails"