core-labs/core
core-labs/core
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

feat(adr-0243): Phase 4 falsifiability benchmark — metrics eval + CLI dispatcher

Browse source 
Completes ADR-0243 Phase 4 (plan §5). Adds the metrics benchmark scoring the
cognitive lifecycle against concrete falsifiable comparison classes, joining the
already-committed decisive propositional falsifier (bdf8146a) under one eval package.

evals/adr_0243_cognitive_lifecycle/benchmark.py — five falsifiable metrics, each
grounded in a live lifecycle primitive (no decorative numbers); typed
BenchmarkVerdict / MetricResult; deterministic, JSON-safe:
  - fidelity: decode overlap |<psi_steady, target>| after relax_to_ground on a
    quadratic well from a perturbed start (min >= 0.999; measured 1.0).
  - surprise separation: energy-above-ground of ID (small rotations of the
    identity axis) vs OOD (near-orthogonal rotations into distinct Cl(4,1)
    planes) against a fixed identity well; strict margined separation
    min_ood - max_id > 0.05 (measured 0.834); lam0 verified ~0, not assumed.
  - insertion cost: relaxation certificate.steps_taken — all certified, bounded
    (<= 256), real work (max_steps > 0; measured 22-23).
  - f32 drift over T=1000: unit versor iterated by a fixed rotor with no renorm;
    f64 holds versor closure to 1e-9 (~8e-14 measured) while f32 truncates to
    ~4.7e-5 (ratio ~5.9e8) — the gap motivating ADR-0244 §2.5/§2.6.
  - falsifier: run_propositional_falsifier wrong == 0 (1008 ID + 18 refusal-parity).

evals/adr_0243_cognitive_lifecycle/__main__.py — subcommand dispatcher
(benchmark [default] / corridor / falsifier); non-zero exit on falsification.

tests/test_adr_0243_benchmark.py — pins overall pass, each metric's falsifiable
claim, the genuine f32/f64 drift gap, CLI routing + exit codes, and the A-04
off-serving quarantine.

[Verification]: in-worktree smoke gate 176 passed; fast lane
(-m "not quarantine and not slow" -n auto) 11808 passed, 108 skipped;
serve-quarantine + third-door cohesion + dispatch hygiene 22 passed; Phase 4
tests (benchmark + falsifier) 17 passed; benchmark overall_passed True
(deterministic across runs).
This commit is contained in:
Shay 2026-07-17 12:59:24 -07:00
parent bdf8146a2b
commit 2bb57a868f
3 changed files with 548 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
evals/adr_0243_cognitive_lifecycle/__main__.py
View file

@ -1,7 +1,14 @@
"""CLI: python -m evals.adr_0243_cognitive_lifecycle [--out PATH]
"""CLI: python -m evals.adr_0243_cognitive_lifecycle <subcommand> [--out PATH]
Writes the fixed-replay sensorium corridor artifact as JSON.
Research/OFF-SERVING only; never imported from chat/runtime.py.
Subcommands (research / OFF-SERVING only; never imported from chat/runtime.py):
benchmark ADR-0243 Phase 4 falsifiability metrics (default) — fidelity,
surprise separation, insertion cost, f32 drift, decisive falsifier;
exits non-zero if the overall gate fails.
corridor Fixed-replay sensorium corridor artifact (Lane B, I-04 consumer).
falsifier Decisive propositional field-vs-ROBDD-gold artifact (wrong == 0).
Each writes a JSON artifact to ``--out`` (default: stdout).
"""
from __future__ import annotations
@ -11,26 +18,51 @@ import json
import sys
from pathlib import Path
from evals.adr_0243_cognitive_lifecycle import run_fixed_replay
def _emit(payload: dict, out: Path | None) -> None:
text = json.dumps(payload, indent=2, sort_keys=True) + "\n"
if out is not None:
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(text, encoding="utf-8")
else:
sys.stdout.write(text)
def main(argv: list[str] | None = None) -> int:
p = argparse.ArgumentParser(description=__doc__)
p.add_argument(
"--out",
type=Path,
default=None,
help="Optional output path (default: stdout)",
)
args = p.parse_args(argv)
artifact = run_fixed_replay()
text = json.dumps(artifact, indent=2, sort_keys=True) + "\n"
if args.out is not None:
args.out.parent.mkdir(parents=True, exist_ok=True)
args.out.write_text(text, encoding="utf-8")
else:
sys.stdout.write(text)
return 0
parser = argparse.ArgumentParser(prog="evals.adr_0243_cognitive_lifecycle")
sub = parser.add_subparsers(dest="command")
for name in ("benchmark", "corridor", "falsifier"):
p = sub.add_parser(name)
p.add_argument("--out", type=Path, default=None, help="Output path (default: stdout)")
args = parser.parse_args(argv)
command = args.command or "benchmark"
out = getattr(args, "out", None)
if command == "benchmark":
from evals.adr_0243_cognitive_lifecycle.benchmark import run_benchmark
verdict = run_benchmark()
_emit(verdict.as_dict(), out)
# Non-zero exit on falsification so the gate is scriptable.
return 0 if verdict.overall_passed else 1
if command == "corridor":
from evals.adr_0243_cognitive_lifecycle import run_fixed_replay
_emit(run_fixed_replay(), out)
return 0
if command == "falsifier":
from evals.adr_0243_cognitive_lifecycle.propositional_falsifier import (
run_propositional_falsifier,
)
artifact = run_propositional_falsifier()
_emit(artifact, out)
return 0 if int(artifact["wrong"]) == 0 else 1
parser.error(f"unknown command: {command!r}") # NoReturn: argparse exits (code 2)
if __name__ == "__main__":

360
evals/adr_0243_cognitive_lifecycle/benchmark.py Normal file
View file

@ -0,0 +1,360 @@
"""ADR-0243 Phase 4 — falsifiability benchmark (metrics eval, OFF-SERVING).
The plan (``docs/plans/adr-0243-implementation-plan.md`` §5 Phase 4) requires the
cognitive lifecycle to be measured against *concrete, falsifiable* comparison
classes — not described in architecture prose. This module is that measurement.
Every metric is grounded in a live lifecycle primitive; none is decorative:
* **fidelity** — decode overlap ``|⟨ψ_steady, target⟩|`` after
:func:`relax_to_ground` on :func:`compile_quadratic_well` from a perturbed
start. The well's ground space is exactly ``span(target)`` at energy 0 with a
gap of ``curvature``; a genuine decoder must land back on the target it was
perturbed from. Falsified if any panel case decodes below
:data:`FIDELITY_MIN`.
* **surprise separation (ID vs OOD)** — energy-above-ground ``ψᵀHψ − λ0`` of an
incoming field against a fixed identity well. In-distribution fields (small
rotations of the identity) sit low; out-of-distribution fields (large-angle
rotations into distinct Cl(4,1) planes) sit high. The operator must separate
the two classes: ``min(OOD) − max(ID) > `` :data:`SURPRISE_MIN_SEPARATION`.
Falsified if the classes overlap in energy.
* **insertion cost** — relaxation ``certificate.steps_taken`` to decode. Every
panel case must *certify* convergence (never mis-certified) within
:data:`INSERTION_STEP_BOUND` steps, and the panel must exercise real decoding
work (``max steps_taken > 0``).
* **f32 drift over T=1000** — a unit versor iterated ``T`` times by a fixed unit
rotor via :func:`algebra.cl41.geometric_product` with **no renormalization**.
Right-multiplication by a unit versor conserves the reverse-norm ``ψψ̃`` and,
for these spatial-plane rotors, the Euclidean norm — *exactly* in f64, up to
rounding in f32. The metric reports both, quantifying the ``float32``
truncation gap that motivates the serving-boundary cast contract (ADR-0244
§2.5) and the f64 fast-path (ADR-0244 §2.6). f64 must hold closure to
:data:`F64_DRIFT_MAX`; the f32 gap is reported for evidence. Ties to the
no-f32-truncation invariant (``docs/…/cl41-algebra-pitfalls``).
* **falsifier** — the decisive propositional field-vs-ROBDD-gold check
(:func:`run_propositional_falsifier`); ``wrong`` must be 0.
Off-serving: lives under ``evals/`` only; never imported by ``chat/runtime.py``
(A-04 quarantine, inherited transitively through
``core.physics.cognitive_lifecycle``). Deterministic: fixed construction, no
wall-clock, no unseeded randomness — the artifact is byte-stable across runs.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
import numpy as np
from algebra.cl41 import N_COMPONENTS, geometric_product, reverse
from algebra.rotor import make_rotor_from_angle
from core.physics.cognitive_lifecycle import (
compile_quadratic_well,
relax_to_ground,
)
from core.physics.wave_manifold import WaveManifold
from evals.adr_0243_cognitive_lifecycle.propositional_falsifier import (
run_propositional_falsifier,
)
__all__ = [
"FIDELITY_MIN",
"SURPRISE_MIN_SEPARATION",
"INSERTION_STEP_BOUND",
"F64_DRIFT_MAX",
"DRIFT_STEPS",
"MetricResult",
"BenchmarkVerdict",
"run_benchmark",
]
# --- Falsifiable thresholds (one place; the pass/fail contract) --------------------
FIDELITY_MIN: float = 0.999
SURPRISE_MIN_SEPARATION: float = 0.05
INSERTION_STEP_BOUND: int = 256
F64_DRIFT_MAX: float = 1e-9
DRIFT_STEPS: int = 1000
# Identity axis for the surprise metric. Rotation planes e12/e13/e14 (indices
# 6/7/8) all *contain* e1, so a rotor built on one genuinely moves the axis — a
# rotor on a disjoint plane would commute past e1 and leave it invariant, the trap
# that makes a naive OOD field read as ID.
_E1_AXIS: int = 0
def _euclidean_unit(psi: np.ndarray) -> np.ndarray:
arr = np.asarray(psi, dtype=np.float64)
norm = float(np.linalg.norm(arr))
if norm <= 0.0 or not np.isfinite(norm):
raise ValueError("degenerate state has no Euclidean-unit direction")
return arr / norm
def _basis_vector(axis: int) -> np.ndarray:
"""Unit grade-1 basis vector e_{axis} (0-indexed) as a 32-component field."""
v = np.zeros(N_COMPONENTS, dtype=np.float64)
v[1 + int(axis)] = 1.0
return v
def _sandwich(rotor: np.ndarray, field: np.ndarray) -> np.ndarray:
"""Rotate ``field`` by ``rotor``: R X R̃ (pure algebra, no backend dispatch).
``make_rotor_from_angle(θ, B)`` rotates a vector in plane ``B`` by exactly
``θ``; when ``B`` contains the vector's axis the overlap becomes ``cos θ``.
"""
return geometric_product(geometric_product(rotor, field), reverse(rotor))
@dataclass(frozen=True, slots=True)
class MetricResult:
"""One falsifiable metric: its measured evidence and whether it passed."""
name: str
passed: bool
detail: dict[str, Any]
def as_dict(self) -> dict[str, Any]:
return {"name": self.name, "passed": self.passed, "detail": self.detail}
@dataclass(frozen=True, slots=True)
class BenchmarkVerdict:
"""Typed benchmark outcome: per-metric results and the overall pass gate."""
metrics: tuple[MetricResult, ...]
overall_passed: bool
def as_dict(self) -> dict[str, Any]:
return {
"kind": "ADR0243BenchmarkVerdict",
"overall_passed": self.overall_passed,
"metrics": [m.as_dict() for m in self.metrics],
}
# --- Metric 1: fidelity (decode overlap) -------------------------------------------
def _fidelity_and_insertion(
*,
curvature: float = 1.0,
) -> tuple[MetricResult, MetricResult]:
"""Decode a perturbed field back to its well's target; measure overlap + steps.
One pass produces both the fidelity metric and the insertion-cost metric so
the (expensive) relaxations run once. Each target is a unit basis axis; each
perturbed start is that axis rotated by a small angle in a plane that
*contains* it (so the start genuinely differs from the target and relaxation
must do real work to decode back).
"""
# (target axis, rotation plane containing it, perturbation angles)
target_specs = (
(0, 6, (0.15, 0.30)), # e1 rotated in e12
(1, 6, (0.20, 0.35)), # e2 rotated in e12
(2, 7, (0.25, 0.40)), # e3 rotated in e13
(3, 8, (0.30, 0.18)), # e4 rotated in e14
)
fidelities: list[float] = []
steps: list[int] = []
all_converged = True
cases: list[dict[str, Any]] = []
for axis, plane, perturb_angles in target_specs:
target = _basis_vector(axis)
well = compile_quadratic_well(target, curvature=curvature)
for p_angle in perturb_angles:
perturb = make_rotor_from_angle(p_angle, plane)
start = _euclidean_unit(_sandwich(perturb, target))
result = relax_to_ground(start, well)
cert = result.certificate
overlap = abs(float(np.dot(result.psi_steady, target)))
fidelities.append(overlap)
steps.append(int(cert.steps_taken))
all_converged = all_converged and bool(cert.converged)
cases.append(
{
"axis": axis,
"plane": plane,
"perturb_angle": round(p_angle, 4),
"fidelity": overlap,
"steps_taken": int(cert.steps_taken),
"converged": bool(cert.converged),
"reason": cert.reason,
}
)
min_fidelity = min(fidelities)
max_steps = max(steps)
fidelity_metric = MetricResult(
name="fidelity",
passed=min_fidelity >= FIDELITY_MIN,
detail={
"min_fidelity": min_fidelity,
"threshold": FIDELITY_MIN,
"n_cases": len(fidelities),
"cases": cases,
},
)
insertion_metric = MetricResult(
name="insertion_cost",
passed=all_converged and 0 < max_steps <= INSERTION_STEP_BOUND,
detail={
"all_converged": all_converged,
"max_steps_taken": max_steps,
"min_steps_taken": min(steps),
"step_bound": INSERTION_STEP_BOUND,
"n_cases": len(steps),
},
)
return fidelity_metric, insertion_metric
# --- Metric 2: surprise separation (ID vs OOD) -------------------------------------
def _energy_above_ground(psi: np.ndarray, hamiltonian_matrix: np.ndarray, lam0: float) -> float:
return float(psi @ hamiltonian_matrix @ psi) - lam0
def _surprise_separation(*, curvature: float = 1.0) -> MetricResult:
"""Separate small-rotation (ID) from large-rotation (OOD) fields by energy.
The identity well targets the fixed e1 axis; incoming fields are rotations
of e1 in planes that contain it (so the rotation is effective — a rotor on a
disjoint plane commutes past e1 and would leave a genuine OOD field reading
as ID). ID = small angles; OOD = large angles toward orthogonality (energy
``c·sin²θ`` is monotone in θ, so this falsifies a miscompiled well, not a
hand-built number). λ0 is read from the spectrum and verified ≈ 0, not
assumed.
"""
target = _basis_vector(_E1_AXIS)
well = compile_quadratic_well(target, curvature=curvature)
H = well.matrix
lam0 = float(np.linalg.eigvalsh(H)[0])
if abs(lam0) > 1e-9:
# The quadratic well is constructed with ground energy 0; a nonzero λ0
# would mean the primitive drifted. Fail-closed rather than mask it.
return MetricResult(
name="surprise_separation",
passed=False,
detail={"error": "well_ground_energy_nonzero", "lam0": lam0},
)
# Small rotations (ID) vs near-orthogonal rotations (OOD) into distinct planes.
id_specs = ((0.15, 6), (0.25, 7), (0.35, 8), (0.20, 6))
ood_specs = ((1.40, 6), (1.45, 7), (1.50, 8), (1.35, 7))
def _energies(specs: tuple[tuple[float, int], ...]) -> list[float]:
out: list[float] = []
for angle, biv in specs:
rotor = make_rotor_from_angle(angle, biv)
field = _euclidean_unit(_sandwich(rotor, target))
out.append(_energy_above_ground(field, H, lam0))
return out
id_energies = _energies(id_specs)
ood_energies = _energies(ood_specs)
separation = min(ood_energies) - max(id_energies)
return MetricResult(
name="surprise_separation",
passed=separation > SURPRISE_MIN_SEPARATION,
detail={
"separation": separation,
"threshold": SURPRISE_MIN_SEPARATION,
"max_id_energy": max(id_energies),
"min_ood_energy": min(ood_energies),
"id_energies": id_energies,
"ood_energies": ood_energies,
"lam0": lam0,
},
)
# --- Metric 3: f32 drift over T steps (no renormalization) -------------------------
def _closure_drift(origin_dtype: np.dtype, *, steps: int) -> tuple[float, float]:
"""Max Euclidean-norm deviation and max versor residual over ``steps`` products.
ψ ← ψ · R_step, R_step a fixed unit rotor, no renormalization. Both operands
share ``origin_dtype`` so ``geometric_product`` keeps the trajectory in that
precision — the whole point is to let ``float32`` rounding accumulate.
"""
r_step = make_rotor_from_angle(0.05, 6).astype(origin_dtype)
psi = make_rotor_from_angle(0.30, 7).astype(origin_dtype)
manifold = WaveManifold()
max_norm_dev = abs(float(np.linalg.norm(psi)) - 1.0)
max_residual = float(manifold.measure_unitary_residual(psi))
for _ in range(int(steps)):
psi = geometric_product(psi, r_step)
max_norm_dev = max(max_norm_dev, abs(float(np.linalg.norm(psi)) - 1.0))
max_residual = max(max_residual, float(manifold.measure_unitary_residual(psi)))
return max_norm_dev, max_residual
def _drift_metric(*, steps: int = DRIFT_STEPS) -> MetricResult:
f64_norm_dev, f64_residual = _closure_drift(np.dtype(np.float64), steps=steps)
f32_norm_dev, f32_residual = _closure_drift(np.dtype(np.float32), steps=steps)
return MetricResult(
name="f32_drift",
# The falsifiable claim is on the f64 substrate (the source of truth):
# versor closure holds to F64_DRIFT_MAX over T steps with no renorm. The
# f32 figures are reported as the truncation-gap evidence, not gated.
passed=(f64_norm_dev <= F64_DRIFT_MAX and f64_residual <= F64_DRIFT_MAX),
detail={
"steps": int(steps),
"f64_max_norm_dev": f64_norm_dev,
"f64_max_versor_residual": f64_residual,
"f32_max_norm_dev": f32_norm_dev,
"f32_max_versor_residual": f32_residual,
"f64_threshold": F64_DRIFT_MAX,
"f32_over_f64_residual_ratio": (
f32_residual / f64_residual if f64_residual > 0.0 else float("inf")
),
},
)
# --- Metric 5: decisive propositional falsifier ------------------------------------
def _falsifier_metric() -> MetricResult:
artifact = run_propositional_falsifier()
wrong = int(artifact["wrong"])
return MetricResult(
name="falsifier",
passed=(
wrong == 0
and artifact["id_case_count"] > 0
and artifact["refusal_parity_count"] > 0
and bool(artifact["ood_field_refused"])
and bool(artifact["ood_gold_decided"])
),
detail={
"wrong": wrong,
"id_case_count": int(artifact["id_case_count"]),
"refusal_parity_count": int(artifact["refusal_parity_count"]),
"ood_field_refused": bool(artifact["ood_field_refused"]),
"ood_gold_decided": bool(artifact["ood_gold_decided"]),
},
)
def run_benchmark(*, drift_steps: int = DRIFT_STEPS) -> BenchmarkVerdict:
"""Run all five falsifiable metrics; return a typed, JSON-safe verdict.
Deterministic and side-effect-free. ``drift_steps`` is exposed only so tests
can exercise a shorter trajectory; the shipped contract is ``DRIFT_STEPS``.
"""
fidelity, insertion = _fidelity_and_insertion()
metrics = (
fidelity,
_surprise_separation(),
insertion,
_drift_metric(steps=drift_steps),
_falsifier_metric(),
)
overall = all(m.passed for m in metrics)
return BenchmarkVerdict(metrics=metrics, overall_passed=overall)

136
tests/test_adr_0243_benchmark.py Normal file
View file

@ -0,0 +1,136 @@
"""ADR-0243 Phase 4 — falsifiability benchmark tests (plan §5 Phase 4).
Pins that the lifecycle passes every falsifiable metric, that each metric is
actually exercised (not a vacuous pass), that the f32/f64 drift gap is real
evidence (both are not silently zero), that the CLI dispatcher routes and
signals failure by exit code, and that the eval stays off the serve path.
"""
from __future__ import annotations
import ast
import json
from pathlib import Path
import pytest
from evals.adr_0243_cognitive_lifecycle import benchmark as bench
from evals.adr_0243_cognitive_lifecycle.__main__ import main
_ROOT = Path(__file__).resolve().parents[1]
@pytest.fixture(scope="module")
def verdict() -> bench.BenchmarkVerdict:
return bench.run_benchmark()
def _metric(verdict: bench.BenchmarkVerdict, name: str) -> bench.MetricResult:
for m in verdict.metrics:
if m.name == name:
return m
raise AssertionError(f"metric {name!r} absent from verdict")
def test_overall_benchmark_passes(verdict: bench.BenchmarkVerdict) -> None:
"""The load-bearing gate: every falsifiable metric passes on the lifecycle."""
assert verdict.overall_passed is True
names = {m.name for m in verdict.metrics}
assert names == {
"fidelity",
"surprise_separation",
"insertion_cost",
"f32_drift",
"falsifier",
}
assert all(m.passed for m in verdict.metrics)
def test_fidelity_decodes_near_perfect(verdict: bench.BenchmarkVerdict) -> None:
m = _metric(verdict, "fidelity")
assert m.detail["n_cases"] > 0
assert m.detail["min_fidelity"] >= bench.FIDELITY_MIN
def test_surprise_separates_id_from_ood(verdict: bench.BenchmarkVerdict) -> None:
m = _metric(verdict, "surprise_separation")
# Strict, margined separation — every OOD field sits above every ID field.
assert m.detail["separation"] > bench.SURPRISE_MIN_SEPARATION
assert m.detail["min_ood_energy"] > m.detail["max_id_energy"]
# The well's ground energy must actually be ~0 (verified, not assumed).
assert abs(m.detail["lam0"]) <= 1e-9
def test_insertion_cost_bounded_and_certified(verdict: bench.BenchmarkVerdict) -> None:
m = _metric(verdict, "insertion_cost")
assert m.detail["all_converged"] is True
# Real decoding work happened (not a start-already-at-ground no-op)...
assert m.detail["max_steps_taken"] > 0
# ...and it stayed bounded.
assert m.detail["max_steps_taken"] <= bench.INSERTION_STEP_BOUND
def test_f32_drift_gap_is_real_evidence(verdict: bench.BenchmarkVerdict) -> None:
"""f64 holds versor closure tight; f32 truncates measurably (the gap is the point)."""
m = _metric(verdict, "f32_drift")
assert m.detail["steps"] == bench.DRIFT_STEPS
# f64 substrate holds closure to the shipped bound.
assert m.detail["f64_max_versor_residual"] <= bench.F64_DRIFT_MAX
assert m.detail["f64_max_norm_dev"] <= bench.F64_DRIFT_MAX
# f32 is not silently zero — the truncation gap is genuine, motivating the
# serving-boundary cast (ADR-0244 §2.5) and f64 fast-path (§2.6).
assert m.detail["f32_max_versor_residual"] > m.detail["f64_max_versor_residual"]
assert m.detail["f32_over_f64_residual_ratio"] > 1_000.0
def test_falsifier_wrong_zero(verdict: bench.BenchmarkVerdict) -> None:
m = _metric(verdict, "falsifier")
assert m.detail["wrong"] == 0
assert m.detail["id_case_count"] > 0
assert m.detail["refusal_parity_count"] > 0
assert m.detail["ood_field_refused"] is True
assert m.detail["ood_gold_decided"] is True
def test_verdict_is_deterministic_and_json_safe() -> None:
a = json.dumps(bench.run_benchmark().as_dict(), sort_keys=True)
b = json.dumps(bench.run_benchmark().as_dict(), sort_keys=True)
assert a == b
assert json.loads(a)["overall_passed"] is True
def test_cli_benchmark_default_and_exit_code() -> None:
# No subcommand → benchmark (default); passing gate → exit 0.
assert main([]) == 0
assert main(["benchmark"]) == 0
def test_cli_writes_artifact(tmp_path: Path) -> None:
out = tmp_path / "artifact.json"
assert main(["benchmark", "--out", str(out)]) == 0
payload = json.loads(out.read_text(encoding="utf-8"))
assert payload["kind"] == "ADR0243BenchmarkVerdict"
assert payload["overall_passed"] is True
def test_cli_falsifier_and_corridor_subcommands(tmp_path: Path) -> None:
fout = tmp_path / "falsifier.json"
assert main(["falsifier", "--out", str(fout)]) == 0
assert json.loads(fout.read_text(encoding="utf-8"))["wrong"] == 0
cout = tmp_path / "corridor.json"
assert main(["corridor", "--out", str(cout)]) == 0
corridor = json.loads(cout.read_text(encoding="utf-8"))
assert corridor["outcome_id"] # non-empty content-addressed id
def test_benchmark_is_not_serve_wired() -> None:
"""Off-serving: chat/runtime.py must never import this eval package."""
runtime_src = (_ROOT / "chat" / "runtime.py").read_text(encoding="utf-8")
tree = ast.parse(runtime_src)
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom) and node.module:
assert "adr_0243_cognitive_lifecycle" not in node.module
if isinstance(node, ast.Import):
for alias in node.names:
assert "adr_0243_cognitive_lifecycle" not in alias.name