From 66b8c7c431a924bdfea576b2aad355cbbc24985b Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 6 Jun 2026 10:27:10 -0700 Subject: [PATCH] =?UTF-8?q?feat(edge):=20edge-deployment=20budget=20gate?= =?UTF-8?q?=20=E2=80=94=20deterministic=20per-turn=20persistence=20cost?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A2 of the refined sequencing. Proves (deterministically, not by assertion) what a long-running CORE life costs to persist per turn on a constrained offline device. Measures the Shape B+ checkpoint BYTES per turn (session_state.json) over the real turn loop — bytes, not wall-clock latency (machine-dependent → flaky). Reuses the L10 continuity corpus. Measured cliff: save_session_state re-serializes the FULL snapshot every turn, so per-turn bytes are O(n) in the accumulated life — 3,811 → 88,189 bytes (23x) over 24 turns, ~1.3KB/vault-entry re-written every turn. That blocks continuous-life at the edge. The gate encodes the edge REQUIREMENT (≤16 KiB/turn regardless of session length) as xfail(strict): it fails today (documenting the cliff), runs green in CI, and flips to a hard failure the moment incremental/append-only persistence (O(Δ)/turn) lands — forcing us to retire it. Plus a regression ceiling (passes today) and a determinism check (the byte metric is reproducible → a valid gate). The fix is algorithmic (incremental persistence, Python/Ring-2), NOT a language rewrite. Tagged core/array_codec.py as the locked reference contract for a future gated Ring-1 Zig byte-exact codec (ADR-0196 G0-G8) — step 3, only after the O(Δ) fix and only if this gate proves the codec is still the bottleneck. See contract.md. --- core/array_codec.py | 10 ++++ evals/edge_budget/__init__.py | 27 ++++++++++ evals/edge_budget/__main__.py | 22 ++++++++ evals/edge_budget/contract.md | 39 ++++++++++++++ evals/edge_budget/runner.py | 93 ++++++++++++++++++++++++++++++++++ tests/test_edge_budget_gate.py | 74 +++++++++++++++++++++++++++ 6 files changed, 265 insertions(+) create mode 100644 evals/edge_budget/__init__.py create mode 100644 evals/edge_budget/__main__.py create mode 100644 evals/edge_budget/contract.md create mode 100644 evals/edge_budget/runner.py create mode 100644 tests/test_edge_budget_gate.py diff --git a/core/array_codec.py b/core/array_codec.py index 1e5cb99b..811ac18f 100644 --- a/core/array_codec.py +++ b/core/array_codec.py @@ -12,6 +12,16 @@ the encoding is portable, and ``float32`` is never conflated with ``float64``. This module is a leaf: it imports only numpy + base64, so every layer (field, vault, session, engine_state) can use it without an import cycle. + +Zig-codec follow-up (tagged — NOT authorized). This bit-exact codec is the natural +locked **reference contract** (ADR-0196 decision rule 1) for a future Ring-1 Zig +byte-exact serialization component: deterministic buffer ownership, stable layout, and +edge-native build are exactly Zig's profile. It is gated behind the G0–G8 ladder and +is **only** worth proposing AFTER (1) persistence becomes incremental/append-only +(O(Δ)/turn — the algorithmic fix, in Python), and (2) the edge-budget gate +(``evals/edge_budget/``) proves the bounded per-turn codec is still the device +bottleneck. A Zig rewrite of today's O(n) snapshot would only speed up the wrong +asymptotics. See ``evals/edge_budget/contract.md``. """ from __future__ import annotations diff --git a/evals/edge_budget/__init__.py b/evals/edge_budget/__init__.py new file mode 100644 index 00000000..d20eccdb --- /dev/null +++ b/evals/edge_budget/__init__.py @@ -0,0 +1,27 @@ +"""Edge-deployment budget lane (A2 of the refined sequencing). + +Proves — deterministically, not by assertion — what a long-running CORE life costs to +persist per turn on a constrained, offline, no-GPU device. The gate encodes the edge +REQUIREMENT (bounded per-turn checkpoint cost) and currently fails it (the O(n) +persistence cliff), flipping green only when incremental/append-only persistence lands. +""" + +from evals.edge_budget.runner import ( + DEFAULT_TURNS, + EDGE_PER_TURN_CEILING_BYTES, + REGRESSION_PER_TURN_CEILING_BYTES, + REGRESSION_TOTAL_CEILING_BYTES, + TurnCost, + measure, + run, +) + +__all__ = [ + "DEFAULT_TURNS", + "EDGE_PER_TURN_CEILING_BYTES", + "REGRESSION_PER_TURN_CEILING_BYTES", + "REGRESSION_TOTAL_CEILING_BYTES", + "TurnCost", + "measure", + "run", +] diff --git a/evals/edge_budget/__main__.py b/evals/edge_budget/__main__.py new file mode 100644 index 00000000..0112d125 --- /dev/null +++ b/evals/edge_budget/__main__.py @@ -0,0 +1,22 @@ +"""CLI: print the edge-budget cost report. + + python -m evals.edge_budget [n_turns] +""" + +from __future__ import annotations + +import json +import sys + +from evals.edge_budget.runner import run + + +def main() -> int: + n_turns = int(sys.argv[1]) if len(sys.argv) > 1 else None + report = run() if n_turns is None else run(n_turns) + print(json.dumps(report, indent=2)) + return 0 if report["edge_budget_met"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/edge_budget/contract.md b/evals/edge_budget/contract.md new file mode 100644 index 00000000..9427b968 --- /dev/null +++ b/evals/edge_budget/contract.md @@ -0,0 +1,39 @@ +# Edge-budget lane — contract (A2) + +**Status:** GATE (the edge axis made falsifiable). **Telos:** [[project-core-is-one-continuous-life]] deployed at the edge — offline, no-GPU, deterministic, on a constrained device (clinic / disaster-center / rural-school box). + +## What it proves + +That a long-running CORE life stays **affordable to persist per turn** on a constrained device — *measured deterministically*, not asserted. The metric is the bytes the Shape B+ checkpoint writes each turn (`engine_state/session_state.json`), captured over the real turn loop (`CognitiveTurnPipeline` + `ChatRuntime(persist_session_state=True)`). Bytes, **not wall-clock latency**: latency is machine-dependent and would make the gate flaky in CI; the snapshot bytes are reproducible (proven by `test_cost_metric_is_deterministic`). + +## The cliff (measured, 24-turn soak) + +`save_session_state` re-serializes the **full** snapshot every turn, so per-turn cost is **O(n) in the accumulated life** (the vault): + +| turn | vault | `session_state.json` bytes | +|----:|----:|----:| +| 0 | 2 | 3,811 | +| 2 | 8 | 11,884 | +| 4 | 14 | 20,228 | +| 8 | 25 | 32,831 | +| 12 | 37 | 48,993 | +| 16 | 48 | 62,965 | +| 20 | 60 | 78,564 | +| 23 | 68 | 88,189 | + +Per-turn bytes grow ~linearly with vault size (~1.3 KB/entry, re-written *every* turn): **growth ratio 23× over 24 turns**, cumulative ~1.1 MB. Extrapolated, a life of 1,000 turns writes multiple MB **per turn**; 10,000 turns, tens of MB per turn. That is the edge-deployability blocker for continuous life. + +## The gate + +- **`test_per_turn_checkpoint_cost_is_within_edge_budget`** — `xfail(strict=True)`. The edge **requirement**: `max_per_turn_bytes ≤ 16 KiB` regardless of session length (a bounded device budget; an O(Δ) implementation writes only the turn's delta, ~a few KB). Today's O(n) snapshot breaches it by turn ~4, so it is an **expected failure that documents the cliff**. When incremental/append-only persistence lands and per-turn bytes go flat, this **xpasses** → `strict` turns it into a hard CI failure → we retire the xfail. That is the falsifiable handle: the cliff is a red gate that the fix turns green. +- **`test_persistence_cost_regression_ceiling`** — passes today; guards against making the cliff *worse* (per-turn ≤ 160 KiB, total ≤ 4 MiB). +- **`test_cost_grows_with_accumulated_state_today`** — records the current O(n) signature on the record (so the fix is a visible delta). +- **`test_cost_metric_is_deterministic`** — the byte series is reproducible across runs. + +## The fix this gate is waiting for + +**Incremental / append-only persistence — algorithmic, in Python (Ring 2).** Persist only the turn's **delta** (new vault entries + the fixed-size field/anchor/scalar state) instead of re-serializing all history; periodic compaction; preserve bit-exact resume (Shape B+) and torn-write atomicity. The vault is append-mostly and the field is fixed-size, so O(Δ)/turn is natural, not a fight against the architecture. This is *not* a micro-optimization and *not* a language rewrite. + +## Zig-codec follow-up (tagged — NOT authorized) + +Once persistence is O(Δ) and this gate is green, **if** the bounded per-turn codec is still the device bottleneck, `core/array_codec.py` is the **locked reference contract** (ADR-0196 decision rule 1) for a Ring-1 Zig byte-exact codec component — gated through the G0–G8 ladder with a parity + determinism + mechanical-advantage proof, behind an explicit selector. A Zig rewrite of *today's* O(n) snapshot would only accelerate the wrong asymptotics, so it is **step 3**, after the algorithmic fix and after this gate proves it's needed. Tag lives in `core/array_codec.py`. diff --git a/evals/edge_budget/runner.py b/evals/edge_budget/runner.py new file mode 100644 index 00000000..5ab28bfb --- /dev/null +++ b/evals/edge_budget/runner.py @@ -0,0 +1,93 @@ +"""Edge-deployment budget lane — deterministic per-turn persistence cost. + +Runs the REAL turn loop with ``persist_session_state=True`` and measures the BYTES the +Shape B+ checkpoint writes each turn (``session_state.json``). The metric is +DETERMINISTIC (snapshot bytes, not wall-clock latency, which is machine-dependent and +would make an edge gate flaky in CI) — so it is a falsifiable handle, not a vibe. + +Today persistence is O(n) per turn: ``save_session_state`` re-serializes the FULL +snapshot every turn, so per-turn bytes grow linearly with the accumulated life (the +vault). This lane makes that cliff visible and gated; it is the falsification lane for +the incremental/append-only persistence fix (O(Δ)/turn → bounded per-turn bytes). + +Reuses the L10 continuity corpus (``prompt_at``) — the same deterministic, always-in- +vocabulary turn ring the lived-spine soak uses — so the cost series is reproducible. +""" + +from __future__ import annotations + +import tempfile +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Any + +from chat.runtime import ChatRuntime +from core.cognition.pipeline import CognitiveTurnPipeline +from core.config import RuntimeConfig +from evals.l10_continuity.corpus import prompt_at + +#: Default soak length — enough turns that an O(n)-per-turn implementation visibly +#: breaches the bounded edge budget, kept small enough to stay fast in CI. +DEFAULT_TURNS = 20 + +#: The edge budget: the most a constrained device (clinic/disaster-center box) can +#: afford to write to durable storage PER TURN, for a life that runs indefinitely. +#: A bounded (O(Δ)) implementation writes only the turn's delta (~a few KB); 16 KiB is +#: generous for that. Today's O(n) snapshot blows through it within a handful of turns. +EDGE_PER_TURN_CEILING_BYTES = 16 * 1024 + +#: Regression guard (passes today): current max per-turn (~86 KiB at 20 turns) + head- +#: room. Catches a change that makes the cliff materially WORSE before the fix lands. +REGRESSION_PER_TURN_CEILING_BYTES = 160 * 1024 +REGRESSION_TOTAL_CEILING_BYTES = 4 * 1024 * 1024 + + +@dataclass(frozen=True, slots=True) +class TurnCost: + turn_index: int + vault_size: int + checkpoint_bytes: int + + +def measure(n_turns: int = DEFAULT_TURNS, engine_state_dir: Path | None = None) -> list[TurnCost]: + """Run the real turn loop and capture the per-turn checkpoint byte cost. + + If ``engine_state_dir`` is None a TemporaryDirectory is used (and cleaned up). + """ + if engine_state_dir is not None: + return _measure_into(n_turns, engine_state_dir) + with tempfile.TemporaryDirectory() as tmp: + return _measure_into(n_turns, Path(tmp)) + + +def _measure_into(n_turns: int, engine_state_dir: Path) -> list[TurnCost]: + config = replace(RuntimeConfig(), persist_session_state=True) + runtime = ChatRuntime(config=config, engine_state_path=engine_state_dir) + pipe = CognitiveTurnPipeline(runtime=runtime) + session_file = engine_state_dir / "session_state.json" + + costs: list[TurnCost] = [] + for i in range(n_turns): + pipe.run(prompt_at(i)) + size = session_file.stat().st_size if session_file.exists() else 0 + costs.append(TurnCost(i, len(runtime._context.vault), size)) + return costs + + +def run(n_turns: int = DEFAULT_TURNS) -> dict[str, Any]: + """Measure and summarize the per-turn persistence cost (JSON-safe report).""" + costs = measure(n_turns) + per_turn = [c.checkpoint_bytes for c in costs] + first = per_turn[0] if per_turn else 0 + return { + "n_turns": n_turns, + "per_turn_bytes": per_turn, + "vault_sizes": [c.vault_size for c in costs], + "first_per_turn_bytes": first, + "final_per_turn_bytes": per_turn[-1] if per_turn else 0, + "max_per_turn_bytes": max(per_turn) if per_turn else 0, + "total_bytes_written": sum(per_turn), + "growth_ratio": round(per_turn[-1] / first, 3) if first else 0.0, + "edge_per_turn_ceiling_bytes": EDGE_PER_TURN_CEILING_BYTES, + "edge_budget_met": (max(per_turn) if per_turn else 0) <= EDGE_PER_TURN_CEILING_BYTES, + } diff --git a/tests/test_edge_budget_gate.py b/tests/test_edge_budget_gate.py new file mode 100644 index 00000000..f11b59df --- /dev/null +++ b/tests/test_edge_budget_gate.py @@ -0,0 +1,74 @@ +"""Edge-deployment budget gate (A2) — deterministic per-turn persistence cost. + +Three obligations: + - EDGE REQUIREMENT (xfail today, strict): per-turn checkpoint bytes must stay under a + fixed budget regardless of session length — what a constrained offline device can + afford for an indefinitely-running life. The current O(n) snapshot breaches it, so + this is an EXPECTED failure that documents the cliff; it flips to a hard failure + (xpass, strict) the moment incremental/append-only persistence (O(Δ)/turn) lands, + forcing us to retire the xfail. This is the gate that makes the fix falsifiable. + - REGRESSION CEILING (passes today): catches a change that makes the cliff worse. + - DETERMINISM: the byte metric is reproducible (same corpus → identical series), which + is why it is a valid gate rather than a flaky wall-clock measurement. + +Each pipeline turn is ~3s, so the soak runs ONCE (module-scoped) and is kept short — +the cliff already breaches the 16 KiB edge budget by turn ~4. The full 24-turn measured +series lives in ``evals/edge_budget/contract.md``. +""" + +from __future__ import annotations + +import pytest + +from evals.edge_budget.runner import ( + EDGE_PER_TURN_CEILING_BYTES, + REGRESSION_PER_TURN_CEILING_BYTES, + REGRESSION_TOTAL_CEILING_BYTES, + measure, + run, +) + +_TURNS = 8 + + +@pytest.fixture(scope="module") +def report() -> dict: + return run(_TURNS) # one soak, shared across the cost assertions + + +@pytest.mark.xfail( + strict=True, + reason=( + "O(n) per-turn persistence cliff: save_session_state re-serializes the FULL " + "snapshot every turn, so per-turn bytes grow with the accumulated life. Flips " + "green when incremental/append-only persistence lands (O(Δ)/turn). See " + "evals/edge_budget/contract.md." + ), +) +def test_per_turn_checkpoint_cost_is_within_edge_budget(report) -> None: + # The edge requirement: bounded per-turn write cost on a constrained device. + assert report["max_per_turn_bytes"] <= EDGE_PER_TURN_CEILING_BYTES, ( + f"per-turn checkpoint peaked at {report['max_per_turn_bytes']} bytes " + f"(budget {EDGE_PER_TURN_CEILING_BYTES}); growth_ratio={report['growth_ratio']}" + ) + + +def test_persistence_cost_regression_ceiling(report) -> None: + # Passes today; guards against making the cliff materially worse before the fix. + assert report["max_per_turn_bytes"] <= REGRESSION_PER_TURN_CEILING_BYTES + assert report["total_bytes_written"] <= REGRESSION_TOTAL_CEILING_BYTES + + +def test_cost_grows_with_accumulated_state_today(report) -> None: + # Documents the CURRENT defect: per-turn cost is NOT bounded — it tracks vault + # growth. (When the fix lands this becomes ~flat; update the assertion then.) + assert report["final_per_turn_bytes"] > report["first_per_turn_bytes"] + assert report["growth_ratio"] > 1.0 + assert report["edge_budget_met"] is False # the cliff is real, on the record + + +def test_cost_metric_is_deterministic() -> None: + # The whole point of measuring BYTES (not latency): reproducible → a valid gate. + a = [c.checkpoint_bytes for c in measure(2)] + b = [c.checkpoint_bytes for c in measure(2)] + assert a == b and len(a) == 2