From 2bb57a868f5d39c94ab726ab594817b2636563e7 Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 17 Jul 2026 12:59:24 -0700 Subject: [PATCH] =?UTF-8?q?feat(adr-0243):=20Phase=204=20falsifiability=20?= =?UTF-8?q?benchmark=20=E2=80=94=20metrics=20eval=20+=20CLI=20dispatcher?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes ADR-0243 Phase 4 (plan §5). Adds the metrics benchmark scoring the cognitive lifecycle against concrete falsifiable comparison classes, joining the already-committed decisive propositional falsifier (bdf8146a) under one eval package. evals/adr_0243_cognitive_lifecycle/benchmark.py — five falsifiable metrics, each grounded in a live lifecycle primitive (no decorative numbers); typed BenchmarkVerdict / MetricResult; deterministic, JSON-safe: - fidelity: decode overlap || after relax_to_ground on a quadratic well from a perturbed start (min >= 0.999; measured 1.0). - surprise separation: energy-above-ground of ID (small rotations of the identity axis) vs OOD (near-orthogonal rotations into distinct Cl(4,1) planes) against a fixed identity well; strict margined separation min_ood - max_id > 0.05 (measured 0.834); lam0 verified ~0, not assumed. - insertion cost: relaxation certificate.steps_taken — all certified, bounded (<= 256), real work (max_steps > 0; measured 22-23). - f32 drift over T=1000: unit versor iterated by a fixed rotor with no renorm; f64 holds versor closure to 1e-9 (~8e-14 measured) while f32 truncates to ~4.7e-5 (ratio ~5.9e8) — the gap motivating ADR-0244 §2.5/§2.6. - falsifier: run_propositional_falsifier wrong == 0 (1008 ID + 18 refusal-parity). evals/adr_0243_cognitive_lifecycle/__main__.py — subcommand dispatcher (benchmark [default] / corridor / falsifier); non-zero exit on falsification. tests/test_adr_0243_benchmark.py — pins overall pass, each metric's falsifiable claim, the genuine f32/f64 drift gap, CLI routing + exit codes, and the A-04 off-serving quarantine. [Verification]: in-worktree smoke gate 176 passed; fast lane (-m "not quarantine and not slow" -n auto) 11808 passed, 108 skipped; serve-quarantine + third-door cohesion + dispatch hygiene 22 passed; Phase 4 tests (benchmark + falsifier) 17 passed; benchmark overall_passed True (deterministic across runs). --- .../adr_0243_cognitive_lifecycle/__main__.py | 72 +++- .../adr_0243_cognitive_lifecycle/benchmark.py | 360 ++++++++++++++++++ tests/test_adr_0243_benchmark.py | 136 +++++++ 3 files changed, 548 insertions(+), 20 deletions(-) create mode 100644 evals/adr_0243_cognitive_lifecycle/benchmark.py create mode 100644 tests/test_adr_0243_benchmark.py diff --git a/evals/adr_0243_cognitive_lifecycle/__main__.py b/evals/adr_0243_cognitive_lifecycle/__main__.py index 4f101cbd..5462278c 100644 --- a/evals/adr_0243_cognitive_lifecycle/__main__.py +++ b/evals/adr_0243_cognitive_lifecycle/__main__.py @@ -1,7 +1,14 @@ -"""CLI: python -m evals.adr_0243_cognitive_lifecycle [--out PATH] +"""CLI: python -m evals.adr_0243_cognitive_lifecycle [--out PATH] -Writes the fixed-replay sensorium corridor artifact as JSON. -Research/OFF-SERVING only; never imported from chat/runtime.py. +Subcommands (research / OFF-SERVING only; never imported from chat/runtime.py): + + benchmark ADR-0243 Phase 4 falsifiability metrics (default) — fidelity, + surprise separation, insertion cost, f32 drift, decisive falsifier; + exits non-zero if the overall gate fails. + corridor Fixed-replay sensorium corridor artifact (Lane B, I-04 consumer). + falsifier Decisive propositional field-vs-ROBDD-gold artifact (wrong == 0). + +Each writes a JSON artifact to ``--out`` (default: stdout). """ from __future__ import annotations @@ -11,26 +18,51 @@ import json import sys from pathlib import Path -from evals.adr_0243_cognitive_lifecycle import run_fixed_replay + +def _emit(payload: dict, out: Path | None) -> None: + text = json.dumps(payload, indent=2, sort_keys=True) + "\n" + if out is not None: + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(text, encoding="utf-8") + else: + sys.stdout.write(text) def main(argv: list[str] | None = None) -> int: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument( - "--out", - type=Path, - default=None, - help="Optional output path (default: stdout)", - ) - args = p.parse_args(argv) - artifact = run_fixed_replay() - text = json.dumps(artifact, indent=2, sort_keys=True) + "\n" - if args.out is not None: - args.out.parent.mkdir(parents=True, exist_ok=True) - args.out.write_text(text, encoding="utf-8") - else: - sys.stdout.write(text) - return 0 + parser = argparse.ArgumentParser(prog="evals.adr_0243_cognitive_lifecycle") + sub = parser.add_subparsers(dest="command") + for name in ("benchmark", "corridor", "falsifier"): + p = sub.add_parser(name) + p.add_argument("--out", type=Path, default=None, help="Output path (default: stdout)") + + args = parser.parse_args(argv) + command = args.command or "benchmark" + out = getattr(args, "out", None) + + if command == "benchmark": + from evals.adr_0243_cognitive_lifecycle.benchmark import run_benchmark + + verdict = run_benchmark() + _emit(verdict.as_dict(), out) + # Non-zero exit on falsification so the gate is scriptable. + return 0 if verdict.overall_passed else 1 + + if command == "corridor": + from evals.adr_0243_cognitive_lifecycle import run_fixed_replay + + _emit(run_fixed_replay(), out) + return 0 + + if command == "falsifier": + from evals.adr_0243_cognitive_lifecycle.propositional_falsifier import ( + run_propositional_falsifier, + ) + + artifact = run_propositional_falsifier() + _emit(artifact, out) + return 0 if int(artifact["wrong"]) == 0 else 1 + + parser.error(f"unknown command: {command!r}") # NoReturn: argparse exits (code 2) if __name__ == "__main__": diff --git a/evals/adr_0243_cognitive_lifecycle/benchmark.py b/evals/adr_0243_cognitive_lifecycle/benchmark.py new file mode 100644 index 00000000..0a85561f --- /dev/null +++ b/evals/adr_0243_cognitive_lifecycle/benchmark.py @@ -0,0 +1,360 @@ +"""ADR-0243 Phase 4 — falsifiability benchmark (metrics eval, OFF-SERVING). + +The plan (``docs/plans/adr-0243-implementation-plan.md`` §5 Phase 4) requires the +cognitive lifecycle to be measured against *concrete, falsifiable* comparison +classes — not described in architecture prose. This module is that measurement. +Every metric is grounded in a live lifecycle primitive; none is decorative: + +* **fidelity** — decode overlap ``|⟨ψ_steady, target⟩|`` after + :func:`relax_to_ground` on :func:`compile_quadratic_well` from a perturbed + start. The well's ground space is exactly ``span(target)`` at energy 0 with a + gap of ``curvature``; a genuine decoder must land back on the target it was + perturbed from. Falsified if any panel case decodes below + :data:`FIDELITY_MIN`. +* **surprise separation (ID vs OOD)** — energy-above-ground ``ψᵀHψ − λ0`` of an + incoming field against a fixed identity well. In-distribution fields (small + rotations of the identity) sit low; out-of-distribution fields (large-angle + rotations into distinct Cl(4,1) planes) sit high. The operator must separate + the two classes: ``min(OOD) − max(ID) > `` :data:`SURPRISE_MIN_SEPARATION`. + Falsified if the classes overlap in energy. +* **insertion cost** — relaxation ``certificate.steps_taken`` to decode. Every + panel case must *certify* convergence (never mis-certified) within + :data:`INSERTION_STEP_BOUND` steps, and the panel must exercise real decoding + work (``max steps_taken > 0``). +* **f32 drift over T=1000** — a unit versor iterated ``T`` times by a fixed unit + rotor via :func:`algebra.cl41.geometric_product` with **no renormalization**. + Right-multiplication by a unit versor conserves the reverse-norm ``ψψ̃`` and, + for these spatial-plane rotors, the Euclidean norm — *exactly* in f64, up to + rounding in f32. The metric reports both, quantifying the ``float32`` + truncation gap that motivates the serving-boundary cast contract (ADR-0244 + §2.5) and the f64 fast-path (ADR-0244 §2.6). f64 must hold closure to + :data:`F64_DRIFT_MAX`; the f32 gap is reported for evidence. Ties to the + no-f32-truncation invariant (``docs/…/cl41-algebra-pitfalls``). +* **falsifier** — the decisive propositional field-vs-ROBDD-gold check + (:func:`run_propositional_falsifier`); ``wrong`` must be 0. + +Off-serving: lives under ``evals/`` only; never imported by ``chat/runtime.py`` +(A-04 quarantine, inherited transitively through +``core.physics.cognitive_lifecycle``). Deterministic: fixed construction, no +wall-clock, no unseeded randomness — the artifact is byte-stable across runs. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import numpy as np + +from algebra.cl41 import N_COMPONENTS, geometric_product, reverse +from algebra.rotor import make_rotor_from_angle +from core.physics.cognitive_lifecycle import ( + compile_quadratic_well, + relax_to_ground, +) +from core.physics.wave_manifold import WaveManifold +from evals.adr_0243_cognitive_lifecycle.propositional_falsifier import ( + run_propositional_falsifier, +) + +__all__ = [ + "FIDELITY_MIN", + "SURPRISE_MIN_SEPARATION", + "INSERTION_STEP_BOUND", + "F64_DRIFT_MAX", + "DRIFT_STEPS", + "MetricResult", + "BenchmarkVerdict", + "run_benchmark", +] + +# --- Falsifiable thresholds (one place; the pass/fail contract) -------------------- +FIDELITY_MIN: float = 0.999 +SURPRISE_MIN_SEPARATION: float = 0.05 +INSERTION_STEP_BOUND: int = 256 +F64_DRIFT_MAX: float = 1e-9 +DRIFT_STEPS: int = 1000 + +# Identity axis for the surprise metric. Rotation planes e12/e13/e14 (indices +# 6/7/8) all *contain* e1, so a rotor built on one genuinely moves the axis — a +# rotor on a disjoint plane would commute past e1 and leave it invariant, the trap +# that makes a naive OOD field read as ID. +_E1_AXIS: int = 0 + + +def _euclidean_unit(psi: np.ndarray) -> np.ndarray: + arr = np.asarray(psi, dtype=np.float64) + norm = float(np.linalg.norm(arr)) + if norm <= 0.0 or not np.isfinite(norm): + raise ValueError("degenerate state has no Euclidean-unit direction") + return arr / norm + + +def _basis_vector(axis: int) -> np.ndarray: + """Unit grade-1 basis vector e_{axis} (0-indexed) as a 32-component field.""" + v = np.zeros(N_COMPONENTS, dtype=np.float64) + v[1 + int(axis)] = 1.0 + return v + + +def _sandwich(rotor: np.ndarray, field: np.ndarray) -> np.ndarray: + """Rotate ``field`` by ``rotor``: R X R̃ (pure algebra, no backend dispatch). + + ``make_rotor_from_angle(θ, B)`` rotates a vector in plane ``B`` by exactly + ``θ``; when ``B`` contains the vector's axis the overlap becomes ``cos θ``. + """ + return geometric_product(geometric_product(rotor, field), reverse(rotor)) + + +@dataclass(frozen=True, slots=True) +class MetricResult: + """One falsifiable metric: its measured evidence and whether it passed.""" + + name: str + passed: bool + detail: dict[str, Any] + + def as_dict(self) -> dict[str, Any]: + return {"name": self.name, "passed": self.passed, "detail": self.detail} + + +@dataclass(frozen=True, slots=True) +class BenchmarkVerdict: + """Typed benchmark outcome: per-metric results and the overall pass gate.""" + + metrics: tuple[MetricResult, ...] + overall_passed: bool + + def as_dict(self) -> dict[str, Any]: + return { + "kind": "ADR0243BenchmarkVerdict", + "overall_passed": self.overall_passed, + "metrics": [m.as_dict() for m in self.metrics], + } + + +# --- Metric 1: fidelity (decode overlap) ------------------------------------------- + + +def _fidelity_and_insertion( + *, + curvature: float = 1.0, +) -> tuple[MetricResult, MetricResult]: + """Decode a perturbed field back to its well's target; measure overlap + steps. + + One pass produces both the fidelity metric and the insertion-cost metric so + the (expensive) relaxations run once. Each target is a unit basis axis; each + perturbed start is that axis rotated by a small angle in a plane that + *contains* it (so the start genuinely differs from the target and relaxation + must do real work to decode back). + """ + # (target axis, rotation plane containing it, perturbation angles) + target_specs = ( + (0, 6, (0.15, 0.30)), # e1 rotated in e12 + (1, 6, (0.20, 0.35)), # e2 rotated in e12 + (2, 7, (0.25, 0.40)), # e3 rotated in e13 + (3, 8, (0.30, 0.18)), # e4 rotated in e14 + ) + + fidelities: list[float] = [] + steps: list[int] = [] + all_converged = True + cases: list[dict[str, Any]] = [] + + for axis, plane, perturb_angles in target_specs: + target = _basis_vector(axis) + well = compile_quadratic_well(target, curvature=curvature) + for p_angle in perturb_angles: + perturb = make_rotor_from_angle(p_angle, plane) + start = _euclidean_unit(_sandwich(perturb, target)) + result = relax_to_ground(start, well) + cert = result.certificate + overlap = abs(float(np.dot(result.psi_steady, target))) + fidelities.append(overlap) + steps.append(int(cert.steps_taken)) + all_converged = all_converged and bool(cert.converged) + cases.append( + { + "axis": axis, + "plane": plane, + "perturb_angle": round(p_angle, 4), + "fidelity": overlap, + "steps_taken": int(cert.steps_taken), + "converged": bool(cert.converged), + "reason": cert.reason, + } + ) + + min_fidelity = min(fidelities) + max_steps = max(steps) + fidelity_metric = MetricResult( + name="fidelity", + passed=min_fidelity >= FIDELITY_MIN, + detail={ + "min_fidelity": min_fidelity, + "threshold": FIDELITY_MIN, + "n_cases": len(fidelities), + "cases": cases, + }, + ) + insertion_metric = MetricResult( + name="insertion_cost", + passed=all_converged and 0 < max_steps <= INSERTION_STEP_BOUND, + detail={ + "all_converged": all_converged, + "max_steps_taken": max_steps, + "min_steps_taken": min(steps), + "step_bound": INSERTION_STEP_BOUND, + "n_cases": len(steps), + }, + ) + return fidelity_metric, insertion_metric + + +# --- Metric 2: surprise separation (ID vs OOD) ------------------------------------- + + +def _energy_above_ground(psi: np.ndarray, hamiltonian_matrix: np.ndarray, lam0: float) -> float: + return float(psi @ hamiltonian_matrix @ psi) - lam0 + + +def _surprise_separation(*, curvature: float = 1.0) -> MetricResult: + """Separate small-rotation (ID) from large-rotation (OOD) fields by energy. + + The identity well targets the fixed e1 axis; incoming fields are rotations + of e1 in planes that contain it (so the rotation is effective — a rotor on a + disjoint plane commutes past e1 and would leave a genuine OOD field reading + as ID). ID = small angles; OOD = large angles toward orthogonality (energy + ``c·sin²θ`` is monotone in θ, so this falsifies a miscompiled well, not a + hand-built number). λ0 is read from the spectrum and verified ≈ 0, not + assumed. + """ + target = _basis_vector(_E1_AXIS) + well = compile_quadratic_well(target, curvature=curvature) + H = well.matrix + lam0 = float(np.linalg.eigvalsh(H)[0]) + if abs(lam0) > 1e-9: + # The quadratic well is constructed with ground energy 0; a nonzero λ0 + # would mean the primitive drifted. Fail-closed rather than mask it. + return MetricResult( + name="surprise_separation", + passed=False, + detail={"error": "well_ground_energy_nonzero", "lam0": lam0}, + ) + + # Small rotations (ID) vs near-orthogonal rotations (OOD) into distinct planes. + id_specs = ((0.15, 6), (0.25, 7), (0.35, 8), (0.20, 6)) + ood_specs = ((1.40, 6), (1.45, 7), (1.50, 8), (1.35, 7)) + + def _energies(specs: tuple[tuple[float, int], ...]) -> list[float]: + out: list[float] = [] + for angle, biv in specs: + rotor = make_rotor_from_angle(angle, biv) + field = _euclidean_unit(_sandwich(rotor, target)) + out.append(_energy_above_ground(field, H, lam0)) + return out + + id_energies = _energies(id_specs) + ood_energies = _energies(ood_specs) + separation = min(ood_energies) - max(id_energies) + return MetricResult( + name="surprise_separation", + passed=separation > SURPRISE_MIN_SEPARATION, + detail={ + "separation": separation, + "threshold": SURPRISE_MIN_SEPARATION, + "max_id_energy": max(id_energies), + "min_ood_energy": min(ood_energies), + "id_energies": id_energies, + "ood_energies": ood_energies, + "lam0": lam0, + }, + ) + + +# --- Metric 3: f32 drift over T steps (no renormalization) ------------------------- + + +def _closure_drift(origin_dtype: np.dtype, *, steps: int) -> tuple[float, float]: + """Max Euclidean-norm deviation and max versor residual over ``steps`` products. + + ψ ← ψ · R_step, R_step a fixed unit rotor, no renormalization. Both operands + share ``origin_dtype`` so ``geometric_product`` keeps the trajectory in that + precision — the whole point is to let ``float32`` rounding accumulate. + """ + r_step = make_rotor_from_angle(0.05, 6).astype(origin_dtype) + psi = make_rotor_from_angle(0.30, 7).astype(origin_dtype) + manifold = WaveManifold() + max_norm_dev = abs(float(np.linalg.norm(psi)) - 1.0) + max_residual = float(manifold.measure_unitary_residual(psi)) + for _ in range(int(steps)): + psi = geometric_product(psi, r_step) + max_norm_dev = max(max_norm_dev, abs(float(np.linalg.norm(psi)) - 1.0)) + max_residual = max(max_residual, float(manifold.measure_unitary_residual(psi))) + return max_norm_dev, max_residual + + +def _drift_metric(*, steps: int = DRIFT_STEPS) -> MetricResult: + f64_norm_dev, f64_residual = _closure_drift(np.dtype(np.float64), steps=steps) + f32_norm_dev, f32_residual = _closure_drift(np.dtype(np.float32), steps=steps) + return MetricResult( + name="f32_drift", + # The falsifiable claim is on the f64 substrate (the source of truth): + # versor closure holds to F64_DRIFT_MAX over T steps with no renorm. The + # f32 figures are reported as the truncation-gap evidence, not gated. + passed=(f64_norm_dev <= F64_DRIFT_MAX and f64_residual <= F64_DRIFT_MAX), + detail={ + "steps": int(steps), + "f64_max_norm_dev": f64_norm_dev, + "f64_max_versor_residual": f64_residual, + "f32_max_norm_dev": f32_norm_dev, + "f32_max_versor_residual": f32_residual, + "f64_threshold": F64_DRIFT_MAX, + "f32_over_f64_residual_ratio": ( + f32_residual / f64_residual if f64_residual > 0.0 else float("inf") + ), + }, + ) + + +# --- Metric 5: decisive propositional falsifier ------------------------------------ + + +def _falsifier_metric() -> MetricResult: + artifact = run_propositional_falsifier() + wrong = int(artifact["wrong"]) + return MetricResult( + name="falsifier", + passed=( + wrong == 0 + and artifact["id_case_count"] > 0 + and artifact["refusal_parity_count"] > 0 + and bool(artifact["ood_field_refused"]) + and bool(artifact["ood_gold_decided"]) + ), + detail={ + "wrong": wrong, + "id_case_count": int(artifact["id_case_count"]), + "refusal_parity_count": int(artifact["refusal_parity_count"]), + "ood_field_refused": bool(artifact["ood_field_refused"]), + "ood_gold_decided": bool(artifact["ood_gold_decided"]), + }, + ) + + +def run_benchmark(*, drift_steps: int = DRIFT_STEPS) -> BenchmarkVerdict: + """Run all five falsifiable metrics; return a typed, JSON-safe verdict. + + Deterministic and side-effect-free. ``drift_steps`` is exposed only so tests + can exercise a shorter trajectory; the shipped contract is ``DRIFT_STEPS``. + """ + fidelity, insertion = _fidelity_and_insertion() + metrics = ( + fidelity, + _surprise_separation(), + insertion, + _drift_metric(steps=drift_steps), + _falsifier_metric(), + ) + overall = all(m.passed for m in metrics) + return BenchmarkVerdict(metrics=metrics, overall_passed=overall) diff --git a/tests/test_adr_0243_benchmark.py b/tests/test_adr_0243_benchmark.py new file mode 100644 index 00000000..91a8cc2b --- /dev/null +++ b/tests/test_adr_0243_benchmark.py @@ -0,0 +1,136 @@ +"""ADR-0243 Phase 4 — falsifiability benchmark tests (plan §5 Phase 4). + +Pins that the lifecycle passes every falsifiable metric, that each metric is +actually exercised (not a vacuous pass), that the f32/f64 drift gap is real +evidence (both are not silently zero), that the CLI dispatcher routes and +signals failure by exit code, and that the eval stays off the serve path. +""" + +from __future__ import annotations + +import ast +import json +from pathlib import Path + +import pytest + +from evals.adr_0243_cognitive_lifecycle import benchmark as bench +from evals.adr_0243_cognitive_lifecycle.__main__ import main + +_ROOT = Path(__file__).resolve().parents[1] + + +@pytest.fixture(scope="module") +def verdict() -> bench.BenchmarkVerdict: + return bench.run_benchmark() + + +def _metric(verdict: bench.BenchmarkVerdict, name: str) -> bench.MetricResult: + for m in verdict.metrics: + if m.name == name: + return m + raise AssertionError(f"metric {name!r} absent from verdict") + + +def test_overall_benchmark_passes(verdict: bench.BenchmarkVerdict) -> None: + """The load-bearing gate: every falsifiable metric passes on the lifecycle.""" + assert verdict.overall_passed is True + names = {m.name for m in verdict.metrics} + assert names == { + "fidelity", + "surprise_separation", + "insertion_cost", + "f32_drift", + "falsifier", + } + assert all(m.passed for m in verdict.metrics) + + +def test_fidelity_decodes_near_perfect(verdict: bench.BenchmarkVerdict) -> None: + m = _metric(verdict, "fidelity") + assert m.detail["n_cases"] > 0 + assert m.detail["min_fidelity"] >= bench.FIDELITY_MIN + + +def test_surprise_separates_id_from_ood(verdict: bench.BenchmarkVerdict) -> None: + m = _metric(verdict, "surprise_separation") + # Strict, margined separation — every OOD field sits above every ID field. + assert m.detail["separation"] > bench.SURPRISE_MIN_SEPARATION + assert m.detail["min_ood_energy"] > m.detail["max_id_energy"] + # The well's ground energy must actually be ~0 (verified, not assumed). + assert abs(m.detail["lam0"]) <= 1e-9 + + +def test_insertion_cost_bounded_and_certified(verdict: bench.BenchmarkVerdict) -> None: + m = _metric(verdict, "insertion_cost") + assert m.detail["all_converged"] is True + # Real decoding work happened (not a start-already-at-ground no-op)... + assert m.detail["max_steps_taken"] > 0 + # ...and it stayed bounded. + assert m.detail["max_steps_taken"] <= bench.INSERTION_STEP_BOUND + + +def test_f32_drift_gap_is_real_evidence(verdict: bench.BenchmarkVerdict) -> None: + """f64 holds versor closure tight; f32 truncates measurably (the gap is the point).""" + m = _metric(verdict, "f32_drift") + assert m.detail["steps"] == bench.DRIFT_STEPS + # f64 substrate holds closure to the shipped bound. + assert m.detail["f64_max_versor_residual"] <= bench.F64_DRIFT_MAX + assert m.detail["f64_max_norm_dev"] <= bench.F64_DRIFT_MAX + # f32 is not silently zero — the truncation gap is genuine, motivating the + # serving-boundary cast (ADR-0244 §2.5) and f64 fast-path (§2.6). + assert m.detail["f32_max_versor_residual"] > m.detail["f64_max_versor_residual"] + assert m.detail["f32_over_f64_residual_ratio"] > 1_000.0 + + +def test_falsifier_wrong_zero(verdict: bench.BenchmarkVerdict) -> None: + m = _metric(verdict, "falsifier") + assert m.detail["wrong"] == 0 + assert m.detail["id_case_count"] > 0 + assert m.detail["refusal_parity_count"] > 0 + assert m.detail["ood_field_refused"] is True + assert m.detail["ood_gold_decided"] is True + + +def test_verdict_is_deterministic_and_json_safe() -> None: + a = json.dumps(bench.run_benchmark().as_dict(), sort_keys=True) + b = json.dumps(bench.run_benchmark().as_dict(), sort_keys=True) + assert a == b + assert json.loads(a)["overall_passed"] is True + + +def test_cli_benchmark_default_and_exit_code() -> None: + # No subcommand → benchmark (default); passing gate → exit 0. + assert main([]) == 0 + assert main(["benchmark"]) == 0 + + +def test_cli_writes_artifact(tmp_path: Path) -> None: + out = tmp_path / "artifact.json" + assert main(["benchmark", "--out", str(out)]) == 0 + payload = json.loads(out.read_text(encoding="utf-8")) + assert payload["kind"] == "ADR0243BenchmarkVerdict" + assert payload["overall_passed"] is True + + +def test_cli_falsifier_and_corridor_subcommands(tmp_path: Path) -> None: + fout = tmp_path / "falsifier.json" + assert main(["falsifier", "--out", str(fout)]) == 0 + assert json.loads(fout.read_text(encoding="utf-8"))["wrong"] == 0 + + cout = tmp_path / "corridor.json" + assert main(["corridor", "--out", str(cout)]) == 0 + corridor = json.loads(cout.read_text(encoding="utf-8")) + assert corridor["outcome_id"] # non-empty content-addressed id + + +def test_benchmark_is_not_serve_wired() -> None: + """Off-serving: chat/runtime.py must never import this eval package.""" + runtime_src = (_ROOT / "chat" / "runtime.py").read_text(encoding="utf-8") + tree = ast.parse(runtime_src) + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.module: + assert "adr_0243_cognitive_lifecycle" not in node.module + if isinstance(node, ast.Import): + for alias in node.names: + assert "adr_0243_cognitive_lifecycle" not in alias.name