core-labs/core
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions
core/tests/test_l10_continuity.py

460 lines
19 KiB
Python
Raw Blame History

"""L10 continuity spike — foundation lane (P1, P2a, P2b, P3).
Two kinds of test per predicate:
- a ``*_holds`` test that drives the REAL turn loop over a short soak and asserts
the predicate passes on genuine evidence, and
- a ``*_bites`` test that feeds the predicate a single mutated record and asserts
it FAILS — the schema-as-proof obligation (CLAUDE.md): a predicate that cannot
fail under the violation it nominally catches is decoration, not proof.
The soak-running tests use a small N and a tmp engine-state dir; they are NOT in
the default smoke suite (this is a soak lane, run on demand / nightly).
"""
from __future__ import annotations
from pathlib import Path
import pytest
from evals.l10_continuity.corpus import prompt_at, scripted_corpus
from evals.l10_continuity.predicates import (
VERSOR_CEILING,
evaluate_p1_closure,
evaluate_p2a_determinism,
evaluate_p2b_reboot_transparency,
evaluate_p3_bounded_resources,
evaluate_p4_commit_point,
evaluate_p4_recovery_determinism,
evaluate_p5a_recall_precision,
evaluate_p5b_anchor_stability,
evaluate_p5c_coherence,
)
from evals.l10_continuity.runner import (
ProbeRecord,
SoakResult,
TurnRecord,
read_recovered_turn_count,
run_soak,
)
_SOAK_N = 6 # short horizon: enough to cycle the corpus and cross a reboot
# --------------------------------------------------------------------------- #
# Synthetic-evidence helpers (fast; no pipeline) — used by the *_bites tests. #
# --------------------------------------------------------------------------- #
def _rec(
i: int,
*,
trace_hash: str | None = None,
versor_condition: float = 1e-13,
vault_size: int | None = None,
booted_segment: int = 0,
surface: str | None = None,
dist_to_anchor: float = 5.0,
turn_movement: float = 1.0,
) -> TurnRecord:
return TurnRecord(
turn_index=i,
input_text=prompt_at(i),
trace_hash=trace_hash if trace_hash is not None else f"hash-{i}",
versor_condition=versor_condition,
surface=surface if surface is not None else f"surface-{i}",
vault_size=vault_size if vault_size is not None else 2 * (i + 1),
peak_rss_raw=1_000_000,
booted_segment=booted_segment,
dist_to_anchor=dist_to_anchor,
turn_movement=turn_movement,
)
def _synthetic(
records: list[TurnRecord],
reboot_at: tuple[int, ...] = (),
probe_records: tuple[ProbeRecord, ...] = (),
) -> SoakResult:
return SoakResult(
n_turns=len(records),
reboot_at=reboot_at,
records=tuple(records),
probe_records=probe_records,
)
# --------------------------------------------------------------------------- #
# Corpus determinism #
# --------------------------------------------------------------------------- #
def test_corpus_is_deterministic_and_total() -> None:
assert scripted_corpus(10) == scripted_corpus(10)
assert prompt_at(0) == prompt_at(6) # ring of length 6 cycles
assert scripted_corpus(3) == tuple(prompt_at(i) for i in range(3))
with pytest.raises(ValueError):
prompt_at(-1)
# --------------------------------------------------------------------------- #
# P1 — closure #
# --------------------------------------------------------------------------- #
def test_p1_closure_holds_on_real_soak(tmp_path: Path) -> None:
result = run_soak(_SOAK_N, engine_state_dir=tmp_path / "es")
outcome = evaluate_p1_closure(result)
assert outcome.passed, outcome.detail
assert outcome.metrics["worst_versor_condition"] < VERSOR_CEILING
def test_p1_closure_bites_on_breached_versor() -> None:
bad = _synthetic([_rec(0), _rec(1, versor_condition=1e-3), _rec(2)])
outcome = evaluate_p1_closure(bad)
assert not outcome.passed
assert (1, 1e-3) in outcome.metrics["violations"]
# --------------------------------------------------------------------------- #
# P2a — pipeline determinism (two independent no-reboot runs) #
# --------------------------------------------------------------------------- #
def test_p2a_determinism_holds_across_independent_runtimes(tmp_path: Path) -> None:
run_a = run_soak(_SOAK_N, engine_state_dir=tmp_path / "a")
run_b = run_soak(_SOAK_N, engine_state_dir=tmp_path / "b")
outcome = evaluate_p2a_determinism(run_a, run_b)
assert outcome.passed, outcome.detail
# And the trace_hashes are genuinely populated (not all empty strings).
assert all(h for h in run_a.trace_hashes()), "pipeline must produce trace_hashes"
def test_p2a_determinism_bites_on_perturbed_hash() -> None:
base = [_rec(i) for i in range(4)]
perturbed = [_rec(i) for i in range(4)]
perturbed[2] = _rec(2, trace_hash="DIVERGED")
outcome = evaluate_p2a_determinism(_synthetic(base), _synthetic(perturbed))
assert not outcome.passed
assert outcome.metrics["first_divergence"] == 2
# --------------------------------------------------------------------------- #
# P2b — reboot transparency (the diagnostic) #
# --------------------------------------------------------------------------- #
def test_p2b_pre_reboot_invariant_holds_on_real_soak(tmp_path: Path) -> None:
reboot_turn = 3
rebooted = run_soak(
_SOAK_N, engine_state_dir=tmp_path / "r", reboot_at=(reboot_turn,)
)
baseline = run_soak(_SOAK_N, engine_state_dir=tmp_path / "base")
outcome, transparency = evaluate_p2b_reboot_transparency(rebooted, baseline)
# The structural invariant ALWAYS holds: a reboot cannot change earlier turns.
assert outcome.passed, outcome.detail
assert transparency.pre_reboot_identical
# Diagnostic record: whatever the persistence story, a divergence (if any)
# must not appear before the reboot turn.
if transparency.first_divergence is not None:
assert transparency.first_divergence >= reboot_turn
def test_p2b_reboot_is_transparent(tmp_path: Path) -> None:
"""Resume-as-same-life: a reboot is FULLY transparent.
With Shape B+ persistence wired (SessionContext.snapshot/restore ->
engine_state schema v2), the lived field/vault/anchor/graph/referents survive
a reboot, so [run K -> reboot -> run M] is byte-identical to the
uninterrupted [run K+M]. This is the load-bearing L10 proof — it FLIPPED from
the Shape-B 'many lives sharing a checkpoint' gap the moment persistence
landed. If this regresses to non-transparent, resume-as-same-life broke.
"""
reboot_turn = 3
rebooted = run_soak(
_SOAK_N, engine_state_dir=tmp_path / "r", reboot_at=(reboot_turn,)
)
baseline = run_soak(_SOAK_N, engine_state_dir=tmp_path / "base")
outcome, transparency = evaluate_p2b_reboot_transparency(rebooted, baseline)
assert outcome.passed, outcome.detail
assert transparency.post_reboot_transparent, (
"Reboot is no longer transparent — the lived field/vault stopped "
"surviving reboot. Resume-as-same-life regressed."
)
assert transparency.first_divergence is None
def test_p2b_bites_on_pre_reboot_divergence() -> None:
reboot_turn = 3
baseline = [_rec(i) for i in range(6)]
# A rebooted run that (wrongly) differs BEFORE the reboot point.
rebooted_records = [_rec(i) for i in range(6)]
rebooted_records[1] = _rec(1, trace_hash="LEAKED-BACKWARD")
outcome, transparency = evaluate_p2b_reboot_transparency(
_synthetic(rebooted_records, reboot_at=(reboot_turn,)),
_synthetic(baseline),
)
assert not outcome.passed
assert not transparency.pre_reboot_identical
assert transparency.first_divergence == 1
# --------------------------------------------------------------------------- #
# P3 — bounded resources #
# --------------------------------------------------------------------------- #
def test_p3_bounded_resources_holds_on_real_soak(tmp_path: Path) -> None:
result = run_soak(_SOAK_N, engine_state_dir=tmp_path / "es")
outcome = evaluate_p3_bounded_resources(result)
assert outcome.passed, outcome.detail
assert outcome.metrics["vault_monotonic"]
def test_p3_bounded_resources_bites_on_unbounded_vault() -> None:
records = [_rec(i) for i in range(5)]
# Simulate an unbounded store: turn 4 holds far more than ceiling*turns.
records[4] = _rec(4, vault_size=10_000)
outcome = evaluate_p3_bounded_resources(_synthetic(records))
assert not outcome.passed
assert (4, 10_000) in outcome.metrics["vault_breaches"]
# --------------------------------------------------------------------------- #
# P4 — kill-9 crash recovery (ADR-0156 atomicity + WAL commit boundary) #
# --------------------------------------------------------------------------- #
def test_p4_atomic_write_survives_mid_replace_kill(tmp_path: Path, monkeypatch) -> None:
"""ADR-0156: a kill at the os.replace instant leaves the PRIOR checkpoint
fully intact and no partial target behind."""
import engine_state
from engine_state import EngineStateStore
store = EngineStateStore(tmp_path)
store.save_manifest(turn_count=7) # a good prior checkpoint
good = (tmp_path / "manifest.json").read_bytes()
def _boom(*_args, **_kwargs): # simulate SIGKILL between fsync and rename
raise OSError("killed mid-replace")
monkeypatch.setattr(engine_state.os, "replace", _boom)
with pytest.raises(OSError):
store.save_manifest(turn_count=8)
# Prior target intact; no orphan .tmp left in place by the except cleanup.
assert (tmp_path / "manifest.json").read_bytes() == good
assert not list(tmp_path.glob(".manifest.json.*.tmp"))
def test_p4_recovered_checkpoint_is_valid_prior(tmp_path: Path) -> None:
"""A reboot after an orphan-leaving crash loads the valid prior turn_count."""
state_dir = tmp_path / "es"
k = 4
run_soak(k, engine_state_dir=state_dir) # K committed turns
# Simulate the torn write, then verify the loader recovers turn_count == K.
from evals.l10_continuity.runner import _inject_orphan_tmp
_inject_orphan_tmp(state_dir)
recovered = read_recovered_turn_count(state_dir)
outcome = evaluate_p4_commit_point(recovered, expected_turn_count=k)
assert outcome.passed, outcome.detail
def test_p4_recovery_is_deterministic_across_orphan_crash(tmp_path: Path) -> None:
reboot_turn = 3
rec_a = run_soak(
_SOAK_N,
engine_state_dir=tmp_path / "a",
reboot_at=(reboot_turn,),
inject_orphan_tmp_at_reboot=True,
)
rec_b = run_soak(
_SOAK_N,
engine_state_dir=tmp_path / "b",
reboot_at=(reboot_turn,),
inject_orphan_tmp_at_reboot=True,
)
outcome = evaluate_p4_recovery_determinism(rec_a, rec_b)
assert outcome.passed, outcome.detail
assert outcome.metrics["recovered_tail_len"] == _SOAK_N - reboot_turn
def test_p4_recovery_determinism_bites_on_divergent_tail() -> None:
"""Synthetic bite for the predicate itself: two recoveries whose post-reboot
tails differ MUST fail (the corrupt-checkpoint test below exercises system
atomicity, not this predicate's logic)."""
reboot_turn = 2
base = [_rec(i, booted_segment=0 if i < reboot_turn else 1) for i in range(5)]
div = [_rec(i, booted_segment=0 if i < reboot_turn else 1) for i in range(5)]
div[3] = _rec(3, trace_hash="DIVERGED", booted_segment=1) # post-reboot index 1
outcome = evaluate_p4_recovery_determinism(
_synthetic(base, reboot_at=(reboot_turn,)),
_synthetic(div, reboot_at=(reboot_turn,)),
)
assert not outcome.passed
assert outcome.metrics["first_divergence"] == 1
def test_p4_commit_point_bites_on_missing_or_partial_checkpoint() -> None:
# No checkpoint at all → recovered count is None → not the committed count.
assert not evaluate_p4_commit_point(None, expected_turn_count=5).passed
# A checkpoint that recorded fewer turns than were committed (non-atomic).
assert not evaluate_p4_commit_point(3, expected_turn_count=5).passed
def test_p4_recovery_bites_on_corrupt_checkpoint(tmp_path: Path) -> None:
"""In-place corruption (the thing atomicity prevents) MUST fail recovery
loudly — proving the atomic write is load-bearing, not decoration."""
from chat.runtime import ChatRuntime
from core.config import RuntimeConfig
state_dir = tmp_path / "es"
run_soak(3, engine_state_dir=state_dir)
current = (state_dir / "current").read_text(encoding="utf-8").strip()
(state_dir / current / "manifest.json").write_text("{ torn-write garbage <<<", encoding="utf-8")
with pytest.raises(Exception):
ChatRuntime(config=RuntimeConfig(), engine_state_path=state_dir)
# --------------------------------------------------------------------------- #
# P5 — semantic quality over the horizon (the T-experience gate) #
# --------------------------------------------------------------------------- #
# P5a — vault recall precision (cross-reboot)
def test_p5a_recall_precision_holds_on_real_soak(tmp_path: Path) -> None:
"""P5a *holds*: a probe registered pre-reboot is recalled at rank ≤ top_k
after a reboot, confirming the float32 serialisation round-trip preserves
the exact-match guarantee in ``_exact_index``."""
n = _SOAK_N
reboot_turn = 3
result = run_soak(
n,
engine_state_dir=tmp_path / "es",
reboot_at=(reboot_turn,),
probe_at=(1,),
verify_probes_at=(n - 1,),
)
assert result.probe_records, "run_soak must collect at least one ProbeRecord"
outcome = evaluate_p5a_recall_precision(result.probe_records)
assert outcome.passed, outcome.detail
# Every probe must be found within top_k.
assert all(p.rank is not None and p.rank <= p.top_k for p in result.probe_records)
# At least one probe crosses the reboot boundary.
assert any(p.across_reboot for p in result.probe_records)
def test_p5a_recall_precision_bites_on_missing_entry() -> None:
"""P5a *bites*: a ProbeRecord with rank=None (entry not found in vault)
makes the predicate fail — it is not decoration."""
missing = ProbeRecord(
registered_at=1,
verified_at=5,
rank=None,
top_k=5,
across_reboot=True,
)
outcome = evaluate_p5a_recall_precision((missing,))
assert not outcome.passed
assert outcome.metrics["failures"]
def test_p5a_recall_precision_bites_on_no_reboot_probe() -> None:
"""P5a *bites*: if all probes are same-segment (no reboot crossed), the
predicate fails — cross-reboot verification is the primary claim."""
same_segment = ProbeRecord(
registered_at=0,
verified_at=2,
rank=1,
top_k=5,
across_reboot=False, # no reboot between registration and verification
)
outcome = evaluate_p5a_recall_precision((same_segment,))
assert not outcome.passed
assert "cross-reboot" in outcome.detail
def test_p5a_recall_precision_bites_on_empty_probe_records() -> None:
"""P5a *bites*: an empty probe_records tuple fails rather than trivially
passing — the runner must be configured to collect evidence."""
outcome = evaluate_p5a_recall_precision(())
assert not outcome.passed
def test_p5b_anchor_stability_holds_on_real_soak(tmp_path: Path) -> None:
result = run_soak(12, engine_state_dir=tmp_path / "es")
outcome = evaluate_p5b_anchor_stability(result)
assert outcome.passed, outcome.detail
assert outcome.metrics["min_steady_dist_to_anchor"] > 1.0
def test_p5b_bites_on_anchor_collapse() -> None:
# dist_to_anchor monotonically collapsing to ~0 → field swallowed by anchor.
records = [
_rec(i, dist_to_anchor=max(0.0, 5.0 - i), turn_movement=1.0) for i in range(8)
]
outcome = evaluate_p5b_anchor_stability(_synthetic(records))
assert not outcome.passed
assert "COLLAPSE" in outcome.detail
def test_p5b_bites_on_field_freeze() -> None:
# turn_movement ~0 → field stopped moving with content (frozen attractor).
records = [_rec(i, dist_to_anchor=5.0, turn_movement=0.0) for i in range(8)]
outcome = evaluate_p5b_anchor_stability(_synthetic(records))
assert not outcome.passed
assert "FREEZE" in outcome.detail
def test_p5c_coherence_holds_over_multiple_corpus_cycles(tmp_path: Path) -> None:
# Span >2 corpus cycles (ring length 6) so the horizon exercises REPETITION,
# not just 6 unique prompts — a total output collapse across cycles would
# drop distinct_surfaces toward 1 and trip the predicate.
from evals.l10_continuity.corpus import base_prompts
n = len(base_prompts()) * 2 + 2 # 14 turns over a 6-prompt ring
result = run_soak(n, engine_state_dir=tmp_path / "es")
outcome = evaluate_p5c_coherence(result)
assert outcome.passed, outcome.detail
assert n > len(base_prompts()), "horizon must exceed one cycle to be meaningful"
def test_p5c_bites_on_empty_surfaces() -> None:
records = [_rec(i, surface="") for i in range(4)]
outcome = evaluate_p5c_coherence(_synthetic(records))
assert not outcome.passed
def test_p5c_bites_on_frozen_single_surface() -> None:
records = [_rec(i, surface="the same thing") for i in range(6)]
outcome = evaluate_p5c_coherence(_synthetic(records))
assert not outcome.passed
assert outcome.metrics["distinct_surfaces"] == 1
# --------------------------------------------------------------------------- #
# Report panel + freeze-gate digest #
# --------------------------------------------------------------------------- #
def test_report_panel_passes_and_records_not_covered(tmp_path: Path) -> None:
from evals.l10_continuity.report import NOT_COVERED, build_report
report = build_report(n_turns=8, reboot_turn=3, engine_state_root=tmp_path)
assert report.all_gates_pass(), [
(p.name, p.detail) for p in report.predicates if not p.passed
]
# P5a is now a live predicate — NOT_COVERED is empty.
assert NOT_COVERED == ()
assert report.not_covered == ()
# P5a must appear in the predicate panel and pass.
p5a = next((p for p in report.predicates if p.name == "P5a_recall_precision"), None)
assert p5a is not None, "P5a_recall_precision must be in the predicate panel"
assert p5a.passed, p5a.detail
# The deterministic digest is a 64-hex SHA-256.
assert len(report.deterministic_digest) == 64
# The report serializes cleanly (for the on-disk artifact).
d = report.to_dict()
assert d["all_gates_pass"] is True
assert any(p["name"] == "P2b_reboot_transparency" for p in d["predicates"])
assert any(p["name"] == "P5a_recall_precision" for p in d["predicates"])
def test_report_digest_is_pure_and_bites() -> None:
from evals.l10_continuity.predicates import evaluate_p1_closure
from evals.l10_continuity.report import deterministic_digest
baseline = _synthetic([_rec(i) for i in range(5)])
outcomes = (evaluate_p1_closure(baseline),)
a = deterministic_digest(baseline, outcomes)
b = deterministic_digest(baseline, outcomes)
assert a == b # pure
# A flipped verdict changes the digest (the freeze handle bites).
flipped = (evaluate_p1_closure(_synthetic([_rec(0, versor_condition=1e-3)])),)
assert deterministic_digest(baseline, flipped) != a
Reference in a new issue View git blame Copy permalink