core/benchmarks/footprint.py

"""Footprint bench — total on-disk and in-memory bytes required to run CORE.

Anchors the "Smaller" claim in evals/CLAIMS.md. Reports:

- On-disk bytes for the active language pack, persistent vault state,
  the compiled Rust backend artifact, and the Python modules actually
  loaded by a single pulse (via sys.modules walk, not a directory scan).
- Resident memory before and after one pulse (RSS delta).
- A deployment-profile flag set: runs_offline, requires_gpu,
  requires_network, requires_api_key.
- A frontier-model context table sourced from published model cards so
  the comparison is reproducible without calling any provider API.

The measurement is honest: it counts only what the cognition lane needs
to run, not the whole repository. If you add an optional subsystem and
it is not loaded by a default pulse, it does not count toward this
number — and that is correct, because deployment does not need it.

Usage:

    from benchmarks.footprint import run_footprint
    report = run_footprint()
    print(report.summary())

CLI surface (added to core/cli.py separately):

    core bench footprint --json
"""

from __future__ import annotations

import json
import resource
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any

PROJECT_ROOT = Path(__file__).resolve().parent.parent


@dataclass(frozen=True, slots=True)
class FrontierReference:
    """A published frontier-model size for context.

    Sources are cited inline (in `source_note`). Do not add a row
    without a public source — speculation is not evidence.
    """
    name: str
    parameters_billion: float
    weights_bytes_estimate: int
    source_note: str


FRONTIER_REFERENCES: tuple[FrontierReference, ...] = (
    FrontierReference(
        name="Llama 3.1 8B (fp16)",
        parameters_billion=8.0,
        weights_bytes_estimate=8_000_000_000 * 2,
        source_note="Meta model card, 2024 — 2 bytes/param at fp16.",
    ),
    FrontierReference(
        name="Llama 3.1 70B (fp16)",
        parameters_billion=70.0,
        weights_bytes_estimate=70_000_000_000 * 2,
        source_note="Meta model card, 2024 — 2 bytes/param at fp16.",
    ),
    FrontierReference(
        name="Llama 3.1 405B (fp16)",
        parameters_billion=405.0,
        weights_bytes_estimate=405_000_000_000 * 2,
        source_note="Meta model card, 2024 — 2 bytes/param at fp16.",
    ),
    FrontierReference(
        name="GPT-3.5 (175B, fp16 estimate)",
        parameters_billion=175.0,
        weights_bytes_estimate=175_000_000_000 * 2,
        source_note="Brown et al. 2020, GPT-3 paper; size is public, fp16 storage assumed.",
    ),
)


@dataclass(frozen=True, slots=True)
class ArtifactSize:
    name: str
    path: str
    bytes_on_disk: int
    present: bool


@dataclass(frozen=True, slots=True)
class DeploymentProfile:
    runs_offline: bool
    requires_gpu: bool
    requires_network: bool
    requires_api_key: bool

    def as_dict(self) -> dict[str, bool]:
        return {
            "runs_offline": self.runs_offline,
            "requires_gpu": self.requires_gpu,
            "requires_network": self.requires_network,
            "requires_api_key": self.requires_api_key,
        }


@dataclass(frozen=True, slots=True)
class FootprintReport:
    artifacts: tuple[ArtifactSize, ...]
    python_runtime_bytes: int
    python_runtime_module_count: int
    rss_idle_bytes: int
    rss_post_pulse_bytes: int
    deployment: DeploymentProfile
    frontier_context: tuple[FrontierReference, ...]

    @property
    def total_disk_bytes(self) -> int:
        return sum(a.bytes_on_disk for a in self.artifacts if a.present) + self.python_runtime_bytes

    @property
    def rss_pulse_delta_bytes(self) -> int:
        return max(0, self.rss_post_pulse_bytes - self.rss_idle_bytes)

    def smaller_than(self, ref: FrontierReference) -> float:
        if self.total_disk_bytes <= 0:
            return 0.0
        return ref.weights_bytes_estimate / self.total_disk_bytes

    def as_dict(self) -> dict[str, Any]:
        return {
            "total_disk_bytes": self.total_disk_bytes,
            "total_disk_human": _human_bytes(self.total_disk_bytes),
            "python_runtime_bytes": self.python_runtime_bytes,
            "python_runtime_module_count": self.python_runtime_module_count,
            "rss_idle_bytes": self.rss_idle_bytes,
            "rss_post_pulse_bytes": self.rss_post_pulse_bytes,
            "rss_pulse_delta_bytes": self.rss_pulse_delta_bytes,
            "deployment": self.deployment.as_dict(),
            "artifacts": [
                {
                    "name": a.name,
                    "path": a.path,
                    "bytes_on_disk": a.bytes_on_disk,
                    "human": _human_bytes(a.bytes_on_disk),
                    "present": a.present,
                }
                for a in self.artifacts
            ],
            "frontier_context": [
                {
                    "name": r.name,
                    "parameters_billion": r.parameters_billion,
                    "weights_bytes_estimate": r.weights_bytes_estimate,
                    "weights_human": _human_bytes(r.weights_bytes_estimate),
                    "core_is_smaller_by_x": round(self.smaller_than(r), 1),
                    "source_note": r.source_note,
                }
                for r in self.frontier_context
            ],
        }

    def summary(self) -> str:
        lines = [
            f"footprint  total_disk={_human_bytes(self.total_disk_bytes)}  "
            f"rss_idle={_human_bytes(self.rss_idle_bytes)}  "
            f"rss_after_pulse={_human_bytes(self.rss_post_pulse_bytes)}  "
            f"(+{_human_bytes(self.rss_pulse_delta_bytes)})",
            "  artifacts:",
        ]
        for a in self.artifacts:
            mark = "ok" if a.present else "--"
            lines.append(f"    [{mark}] {a.name:<28} {_human_bytes(a.bytes_on_disk):>10}  {a.path}")
        lines.append(f"  python_runtime_modules: {self.python_runtime_module_count} "
                     f"({_human_bytes(self.python_runtime_bytes)})")
        lines.append("  deployment: " + ", ".join(
            f"{k}={v}" for k, v in self.deployment.as_dict().items()
        ))
        lines.append("  vs published frontier model sizes:")
        for r in self.frontier_context:
            x = self.smaller_than(r)
            lines.append(f"    {r.name:<32} {_human_bytes(r.weights_bytes_estimate):>10}   "
                         f"CORE is {x:,.0f}x smaller")
        return "\n".join(lines)


def _human_bytes(n: int) -> str:
    """Format bytes for humans. Uses binary units (KiB, MiB) at full
    precision and falls back to plain int below 1 KiB.
    """
    if n < 1024:
        return f"{n} B"
    units = ("KiB", "MiB", "GiB", "TiB", "PiB")
    size = float(n)
    unit = units[0]
    for unit in units:
        size /= 1024.0
        if size < 1024.0:
            break
    return f"{size:.2f} {unit}"


def _dir_bytes(path: Path) -> int:
    if not path.exists():
        return 0
    if path.is_file():
        return path.stat().st_size
    total = 0
    for entry in path.rglob("*"):
        if entry.is_file():
            try:
                total += entry.stat().st_size
            except OSError:
                continue
    return total


def _rust_artifact_path() -> Path:
    """Locate the compiled Rust backend shared library, if present.

    Searches the release directory only — debug artifacts are not what
    deployment ships. Returns the first matching file or a sentinel
    non-existent path so the report still renders cleanly when Rust
    has not been built.
    """
    release = PROJECT_ROOT / "core-rs" / "target" / "release"
    if not release.exists():
        return release / "libcore_rs.dylib"
    for ext in ("dylib", "so", "dll"):
        for candidate in release.glob(f"libcore_rs.{ext}"):
            return candidate
    return release / "libcore_rs.dylib"


def _measure_python_runtime() -> tuple[int, int]:
    """Sum the byte size of every loaded module whose file is under
    PROJECT_ROOT. This is the actual import closure for the current
    process — not a directory scan that would over-count optional
    subsystems the lane never imports.
    """
    seen: set[Path] = set()
    total_bytes = 0
    for module in list(sys.modules.values()):
        if module is None:
            continue
        file_attr = getattr(module, "__file__", None)
        if not file_attr:
            continue
        try:
            p = Path(file_attr).resolve()
        except (OSError, ValueError):
            continue
        try:
            p.relative_to(PROJECT_ROOT)
        except ValueError:
            continue
        if p in seen or not p.exists():
            continue
        seen.add(p)
        try:
            total_bytes += p.stat().st_size
        except OSError:
            continue
    return total_bytes, len(seen)


def _rss_bytes() -> int:
    """Return resident set size in bytes. On Linux ru_maxrss is KiB; on
    macOS it is bytes. The conversion below covers both.
    """
    rusage = resource.getrusage(resource.RUSAGE_SELF)
    raw = int(rusage.ru_maxrss)
    scale = 1 if sys.platform == "darwin" else 1024
    return raw * scale


def run_footprint(*, pack_id: str = "en_core_cognition_v1") -> FootprintReport:
    """Measure CORE's deployed footprint and return a FootprintReport.

    Triggers one pulse via `scripts.run_pulse.run_pulse` so the import
    closure and post-pulse RSS reflect what real deployment looks like.
    """
    rss_idle = _rss_bytes()

    from scripts.run_pulse import run_pulse
    run_pulse("What is truth?", use_glove=False)

    rss_post = _rss_bytes()
    py_bytes, py_modules = _measure_python_runtime()

    pack_path = PROJECT_ROOT / "packs" / "data" / pack_id
    vault_path = PROJECT_ROOT / "vault"
    rust_path = _rust_artifact_path()
    seed_packs_path = PROJECT_ROOT / "packs"

    artifacts = (
        ArtifactSize(
            name=f"language_pack:{pack_id}",
            path=str(pack_path.relative_to(PROJECT_ROOT)),
            bytes_on_disk=_dir_bytes(pack_path),
            present=pack_path.exists(),
        ),
        ArtifactSize(
            name="seed_packs",
            path=str(seed_packs_path.relative_to(PROJECT_ROOT)),
            bytes_on_disk=_dir_bytes(seed_packs_path),
            present=seed_packs_path.exists(),
        ),
        ArtifactSize(
            name="vault_module",
            path=str(vault_path.relative_to(PROJECT_ROOT)),
            bytes_on_disk=_dir_bytes(vault_path),
            present=vault_path.exists(),
        ),
        ArtifactSize(
            name="rust_backend",
            path=str(rust_path.relative_to(PROJECT_ROOT)) if rust_path.exists() else str(rust_path),
            bytes_on_disk=rust_path.stat().st_size if rust_path.exists() else 0,
            present=rust_path.exists(),
        ),
    )

    deployment = DeploymentProfile(
        runs_offline=True,
        requires_gpu=False,
        requires_network=False,
        requires_api_key=False,
    )

    return FootprintReport(
        artifacts=artifacts,
        python_runtime_bytes=py_bytes,
        python_runtime_module_count=py_modules,
        rss_idle_bytes=rss_idle,
        rss_post_pulse_bytes=rss_post,
        deployment=deployment,
        frontier_context=FRONTIER_REFERENCES,
    )


def write_report(report: FootprintReport, root: Path | None = None) -> Path:
    base = root or PROJECT_ROOT / "evals" / "reports"
    base.mkdir(parents=True, exist_ok=True)
    path = base / "footprint_latest.json"
    path.write_text(
        json.dumps(report.as_dict(), ensure_ascii=False, indent=2, sort_keys=True) + "\n"
    )
    return path