feat(frontier): add replay variability suite and token-cost telemetry (#66)

Agent-Logs-Url: https://github.com/AssetOverflow/core/sessions/f88b48fa-0c2a-4f9d-a42b-d275596e43b8

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: AssetOverflow <109810776+AssetOverflow@users.noreply.github.com>

.env.example

@@ -6,7 +6,7 @@
 # ---------------------------------------------------------------------------
 # OpenAI
 # Used by: evals/frontier_compare/providers.py (OpenAIAdapter)
-# Benchmarks: determinism, truth_lock, axis_orthogonality (provider=openai)
+# Benchmarks: prompt_battery, replay_variability (provider=openai)
 # ---------------------------------------------------------------------------
 OPENAI_API_KEY=
 
@@ -21,7 +21,7 @@ OPENAI_MODEL=gpt-4o
 # ---------------------------------------------------------------------------
 # Anthropic
 # Used by: evals/frontier_compare/providers.py (AnthropicAdapter)
-# Benchmarks: determinism, truth_lock, axis_orthogonality (provider=anthropic)
+# Benchmarks: prompt_battery, replay_variability (provider=anthropic)
 # ---------------------------------------------------------------------------
 ANTHROPIC_API_KEY=
 

evals/frontier_compare/README.md

@@ -10,15 +10,18 @@ If CORE can solve something the LLM cannot structurally audit, CORE must prove i
 If both solve it, compare correctness, determinism, traceability, latency, cost, memory, and failure mode.
 ```
 
-Wave 1 is deliberately local and CORE-native.  It does **not** call external frontier APIs, does **not** require provider keys, and does **not** change runtime behavior.  It creates the benchmark harness, report schema, recording UI, and first suites that measure the things CORE should already be able to defend:
+Wave 1 now supports both CORE-native and cross-provider execution through adapter wiring.  CORE-only suites remain local and deterministic; cross-provider suites can call external APIs when credentials are configured.
+
+Current suites focus on:
 
 * deterministic replay
 * truth-lock / groundedness behavior
 * register vs anchor-lens axis discipline
-* compact machine-readable reports suitable for later head-to-head frontier runs
+* compact machine-readable reports suitable for head-to-head frontier runs
 * a static visual report viewer for clean recordings and demos
+* usage-token and formula-based cost telemetry when provider APIs expose usage
 
-Provider adapters for GPT / Claude / Gemini / open-weight baselines are intentionally deferred to a later wave so this PR remains testable without secrets.
+Provider adapters for OpenAI / Anthropic / Ollama are available; CORE still remains runnable with no API keys.
 
 ---
 
@@ -74,6 +77,29 @@ surface_variation_observed
 anchor_lens_engagement_observed
 ```
 
+### `prompt_battery` (cross-provider)
+
+Provider-agnostic prompt battery over the adapter interface.
+
+Primary metrics:
+
+```text
+non_empty_surface_rate
+mean_latency_ms
+token_usage_capture_rate
+formula_cost_estimate (when pricing + usage are available)
+```
+
+### `replay_variability` (cross-provider)
+
+Runs repeated calls per prompt and scores stability.
+
+Primary metric:
+
+```text
+stability_score = 1 / unique_surface_count
+```
+
 ---
 
 ## Run
@@ -98,6 +124,12 @@ Human-readable table:
 uv run python -m evals.frontier_compare --suite all
 ```
 
+Cross-provider (example):
+
+```bash
+uv run python -m evals.frontier_compare --provider openai --suite all --json
+```
+
 ---
 
 ## Recording UI

evals/frontier_compare/__main__.py

@@ -5,9 +5,9 @@ import json
 import sys
 from pathlib import Path
 
-from .cross_provider import run_prompt_battery
+from .cross_provider import run_prompt_battery, run_replay_variability
 from .model_registry import require_model_card
-from .providers import ProviderConfig, build_adapter, load_dotenv_if_present
+from .providers import ProviderConfig, build_observing_adapter, load_dotenv_if_present
 from .runner import (
     BenchmarkReport,
     format_human_report,
@@ -18,7 +18,7 @@ from .runner import (
 
 
 _CORE_ONLY_SUITES = ("determinism", "truth_lock", "axis_orthogonality", "all")
-_CROSS_PROVIDER_SUITES = ("prompt_battery",)
+_CROSS_PROVIDER_SUITES = ("prompt_battery", "replay_variability")
 _VALID_SUITES = _CORE_ONLY_SUITES + _CROSS_PROVIDER_SUITES
 
 
@@ -39,7 +39,7 @@ def build_parser() -> argparse.ArgumentParser:
             "benchmark suite to run.  CORE-only suites: "
             f"{', '.join(_CORE_ONLY_SUITES)}.  Cross-provider suites: "
             f"{', '.join(_CROSS_PROVIDER_SUITES)}.  Default 'all' runs every "
-            "CORE-only suite when --provider=core, otherwise prompt_battery."
+            "CORE-only suite when --provider=core, otherwise all cross-provider suites."
         ),
     )
     parser.add_argument(
@@ -86,6 +86,15 @@ def build_parser() -> argparse.ArgumentParser:
         default=Path(".env"),
         help="path to a .env file with provider credentials (default: ./.env).",
     )
+    parser.add_argument(
+        "--repeats",
+        type=int,
+        default=3,
+        help=(
+            "repeat count for repeat-sensitive cross-provider suites "
+            "(currently replay_variability)."
+        ),
+    )
     return parser
 
 
@@ -147,13 +156,31 @@ def main(argv: list[str] | None = None) -> int:
     else:
         # Cross-provider path — runs over the provider adapter.
         cfg = _build_cfg(args)
-        adapter = build_adapter(cfg)
+        adapter = build_observing_adapter(cfg)
-        suite_report = run_prompt_battery(adapter, cfg=cfg)
+        if args.suite == "all" and args.provider != "core":
+            suite_reports = (
+                run_prompt_battery(adapter, cfg=cfg),
+                run_replay_variability(
+                    adapter,
+                    cfg=cfg,
+                    repeats=max(1, int(args.repeats)),
+                ),
+            )
+        elif args.suite == "replay_variability":
+            suite_reports = (
+                run_replay_variability(
+                    adapter,
+                    cfg=cfg,
+                    repeats=max(1, int(args.repeats)),
+                ),
+            )
+        else:
+            suite_reports = (run_prompt_battery(adapter, cfg=cfg),)
         report = BenchmarkReport(
             benchmark_family="frontier_compare_wave1",
             model=cfg.model,
             mode=cfg.provider,
-            suites=(suite_report,),
+            suites=suite_reports,
         )
         # Always persist non-CORE runs — they're rate-limited / paid, so
         # losing the artifact is genuinely costly.  CORE adapter runs of

evals/frontier_compare/cross_provider.py

@@ -36,6 +36,7 @@ import time
 from dataclasses import dataclass
 from typing import Callable
 
+from .model_registry import resolve_model_card
 from .providers import ProviderConfig
 from .runner import CaseResult, SuiteReport
 
@@ -78,6 +79,10 @@ class ProviderObservation:
     provider: str
     model: str
     elapsed_ms: float
+    input_tokens: int | None = None
+    output_tokens: int | None = None
+    total_tokens: int | None = None
+    estimated_cost_usd: float | None = None
     error_type: str = ""
     error_message: str = ""
 
@@ -92,6 +97,10 @@ class ProviderObservation:
             "provider": self.provider,
             "model": self.model,
             "elapsed_ms": self.elapsed_ms,
+            "input_tokens": self.input_tokens,
+            "output_tokens": self.output_tokens,
+            "total_tokens": self.total_tokens,
+            "estimated_cost_usd": self.estimated_cost_usd,
             "error_type": self.error_type,
             "error_message": self.error_message,
         }
@@ -103,13 +112,14 @@ class ProviderObservation:
 
 
 def _observe_one(
-    adapter: Callable[[str], str],
+    adapter: Callable[[str], object],
     cfg: ProviderConfig,
     prompt: str,
 ) -> ProviderObservation:
+    card = resolve_model_card(cfg.provider, cfg.model)
     start = time.perf_counter()
     try:
-        surface = adapter(prompt)
+        raw = adapter(prompt)
     except Exception as exc:  # noqa: BLE001 - record failure, never abort the suite
         return ProviderObservation(
             prompt=prompt,
@@ -121,17 +131,45 @@ def _observe_one(
             error_message=str(exc),
         )
     elapsed_ms = (time.perf_counter() - start) * 1000.0
+    surface = ""
+    input_tokens: int | None = None
+    output_tokens: int | None = None
+    total_tokens: int | None = None
+    if isinstance(raw, str):
+        surface = raw
+    else:
+        surface = str(getattr(raw, "surface", "") or "")
+        in_val = getattr(raw, "input_tokens", None)
+        out_val = getattr(raw, "output_tokens", None)
+        total_val = getattr(raw, "total_tokens", None)
+        input_tokens = int(in_val) if in_val is not None else None
+        output_tokens = int(out_val) if out_val is not None else None
+        total_tokens = int(total_val) if total_val is not None else None
+    estimated_cost_usd: float | None = None
+    if (
+        card is not None
+        and input_tokens is not None
+        and output_tokens is not None
+    ):
+        estimated_cost_usd = card.estimate_cost_usd(
+            input_tokens=input_tokens,
+            output_tokens=output_tokens,
+        )
     return ProviderObservation(
         prompt=prompt,
         surface=surface,
         provider=cfg.provider,
         model=cfg.model,
         elapsed_ms=elapsed_ms,
+        input_tokens=input_tokens,
+        output_tokens=output_tokens,
+        total_tokens=total_tokens,
+        estimated_cost_usd=estimated_cost_usd,
     )
 
 
 def run_prompt_battery(
-    adapter: Callable[[str], str],
+    adapter: Callable[[str], object],
     *,
     cfg: ProviderConfig,
     prompts: tuple[tuple[str, str], ...] = _PROMPT_BATTERY,
@@ -175,7 +213,89 @@ def run_prompt_battery(
     )
 
 
+def run_replay_variability(
+    adapter: Callable[[str], object],
+    *,
+    cfg: ProviderConfig,
+    repeats: int = 3,
+    prompts: tuple[tuple[str, str], ...] = _PROMPT_BATTERY,
+) -> SuiteReport:
+    """Run repeated calls per prompt and score surface stability.
+
+    Score formula per case:
+        stability_score = 1 / unique_surface_count
+    where unique_surface_count >= 1 over `repeats` runs.
+    """
+    runs = max(1, int(repeats))
+    cases: list[CaseResult] = []
+    for case_id, prompt in prompts:
+        observations = [_observe_one(adapter, cfg, prompt) for _ in range(runs)]
+        successful = [o for o in observations if not o.error_type]
+        failures: list[str] = []
+        if not successful:
+            failures.append("adapter_error")
+            unique_count = 0
+            score = 0.0
+            elapsed_ms = sum(o.elapsed_ms for o in observations)
+            details = {
+                "repeats": runs,
+                "observations": [o.as_dict() for o in observations],
+            }
+        else:
+            surfaces = {
+                o.surface.strip()
+                for o in successful
+                if o.surface.strip()
+            }
+            if not surfaces:
+                failures.append("empty_surface")
+            unique_count = len(surfaces) if surfaces else 0
+            score = 0.0 if unique_count == 0 else (1.0 / float(unique_count))
+            if any(o.error_type for o in observations):
+                failures.append("partial_adapter_error")
+            elapsed_ms = sum(o.elapsed_ms for o in observations)
+            costs = [
+                float(o.estimated_cost_usd)
+                for o in successful
+                if o.estimated_cost_usd is not None
+            ]
+            details = {
+                "repeats": runs,
+                "successful_runs": len(successful),
+                "unique_surface_count": unique_count,
+                "mean_elapsed_ms": (elapsed_ms / runs) if runs else 0.0,
+                "mean_estimated_cost_usd": (
+                    (sum(costs) / len(costs))
+                    if costs else None
+                ),
+                "observations": [o.as_dict() for o in observations],
+            }
+        passed = not failures and unique_count == 1
+        cases.append(
+            CaseResult(
+                suite="replay_variability",
+                case_id=case_id,
+                prompt=prompt,
+                passed=passed,
+                score=score,
+                elapsed_ms=elapsed_ms,
+                details=details,
+                failures=tuple(failures),
+            )
+        )
+    primary = (
+        sum(c.score for c in cases) / len(cases) if cases else 0.0
+    )
+    return SuiteReport(
+        suite="replay_variability",
+        cases=tuple(cases),
+        primary_score=primary,
+        passed=all(c.passed for c in cases),
+    )
+
+
 __all__ = [
     "ProviderObservation",
     "run_prompt_battery",
+    "run_replay_variability",
 ]

evals/frontier_compare/model_registry.py

@@ -61,6 +61,15 @@ class ModelCard:
     notes: str = ""
     """Free-form notes: known quirks, benchmark-specific observations, version history."""
 
+    input_usd_per_million_tokens: float | None = None
+    """Public list price for input tokens in USD per 1M tokens."""
+
+    output_usd_per_million_tokens: float | None = None
+    """Public list price for output tokens in USD per 1M tokens."""
+
+    pricing_source: str = ""
+    """Source URL/note for pricing metadata."""
+
     tags: tuple[str, ...] = field(default_factory=tuple)
     """Searchable tags, e.g. ('reasoning', 'code', 'vision')."""
 
@@ -69,6 +78,35 @@ class ModelCard:
         d["tags"] = list(self.tags)
         return d
 
+    @property
+    def has_pricing(self) -> bool:
+        return (
+            self.input_usd_per_million_tokens is not None
+            and self.output_usd_per_million_tokens is not None
+        )
+
+    def estimate_cost_usd(
+        self,
+        *,
+        input_tokens: int | float,
+        output_tokens: int | float,
+    ) -> float | None:
+        """Compute provider list-price cost for one request.
+
+        Formula:
+            cost_usd =
+              (input_tokens / 1_000_000) * input_usd_per_million_tokens +
+              (output_tokens / 1_000_000) * output_usd_per_million_tokens
+        """
+        if not self.has_pricing:
+            return None
+        in_rate = float(self.input_usd_per_million_tokens or 0.0)
+        out_rate = float(self.output_usd_per_million_tokens or 0.0)
+        return (
+            (float(input_tokens) / 1_000_000.0) * in_rate
+            + (float(output_tokens) / 1_000_000.0) * out_rate
+        )
+
 
 # ---------------------------------------------------------------------------
 # Registry
@@ -107,6 +145,9 @@ _REGISTRY: dict[str, ModelCard] = {
             "Use a dated snapshot (e.g. gpt-4o-2024-08-06) for reproducible benchmarks. "
             "Set OPENAI_MODEL=gpt-4o-2024-08-06 in .env."
         ),
+        input_usd_per_million_tokens=2.50,
+        output_usd_per_million_tokens=10.00,
+        pricing_source="https://openai.com/api/pricing (captured 2026-05-20)",
         tags=("frontier", "multimodal", "reasoning", "code"),
     ),
     "openai/gpt-4o-2024-08-06": ModelCard(
@@ -119,6 +160,9 @@ _REGISTRY: dict[str, ModelCard] = {
         architecture="GPT-4 class transformer, multimodal (text + vision).",
         sampling="Stochastic at T>0. T=0 is near-deterministic for a fixed snapshot but backend routing can still vary.",
         notes="Pinned snapshot. Preferred for reproducible benchmark comparisons.",
+        input_usd_per_million_tokens=2.50,
+        output_usd_per_million_tokens=10.00,
+        pricing_source="https://openai.com/api/pricing (captured 2026-05-20)",
         tags=("frontier", "multimodal", "reasoning", "code", "pinned"),
     ),
     "openai/gpt-4o-mini": ModelCard(
@@ -131,6 +175,9 @@ _REGISTRY: dict[str, ModelCard] = {
         architecture="Smaller GPT-4o class model optimised for latency and cost.",
         sampling="Stochastic at T>0.",
         notes="Useful for cost/latency baseline comparisons in the benchmark cost suite.",
+        input_usd_per_million_tokens=0.15,
+        output_usd_per_million_tokens=0.60,
+        pricing_source="https://openai.com/api/pricing (captured 2026-05-20)",
         tags=("frontier", "fast", "cost-efficient"),
     ),
     "openai/o3": ModelCard(
@@ -146,6 +193,7 @@ _REGISTRY: dict[str, ModelCard] = {
             "o-series models use a reasoning_effort parameter instead of temperature. "
             "Pass via ProviderConfig.extra = {'reasoning_effort': 'high'} for benchmark use."
         ),
+        pricing_source="https://openai.com/api/pricing (captured 2026-05-20)",
         tags=("frontier", "reasoning", "chain-of-thought"),
     ),
 
@@ -164,6 +212,9 @@ _REGISTRY: dict[str, ModelCard] = {
             "Default ANTHROPIC_MODEL in .env.example. "
             "For extended thinking benchmarks, pass extra={'thinking': {'type': 'enabled', 'budget_tokens': 10000}}."
         ),
+        input_usd_per_million_tokens=15.00,
+        output_usd_per_million_tokens=75.00,
+        pricing_source="https://www.anthropic.com/pricing (captured 2026-05-20)",
         tags=("frontier", "reasoning", "code", "extended-thinking"),
     ),
     "anthropic/claude-sonnet-4-5": ModelCard(
@@ -176,6 +227,9 @@ _REGISTRY: dict[str, ModelCard] = {
         architecture="Claude 4 class transformer (Anthropic). Balanced speed/capability.",
         sampling="Stochastic at T>0.",
         notes="Good default for high-volume benchmark sweeps where Opus cost is prohibitive.",
+        input_usd_per_million_tokens=3.00,
+        output_usd_per_million_tokens=15.00,
+        pricing_source="https://www.anthropic.com/pricing (captured 2026-05-20)",
         tags=("frontier", "balanced", "cost-efficient"),
     ),
     "anthropic/claude-haiku-3-5": ModelCard(
@@ -188,6 +242,9 @@ _REGISTRY: dict[str, ModelCard] = {
         architecture="Claude 3 class transformer (Anthropic). Optimised for latency.",
         sampling="Stochastic at T>0.",
         notes="Useful for latency/cost baseline comparisons. Lower capability ceiling than Sonnet/Opus.",
+        input_usd_per_million_tokens=0.80,
+        output_usd_per_million_tokens=4.00,
+        pricing_source="https://www.anthropic.com/pricing (captured 2026-05-20)",
         tags=("frontier", "fast", "cost-efficient"),
     ),
 

evals/frontier_compare/providers.py

@@ -164,23 +164,41 @@ class ProviderConfig:
         }
 
 
+@dataclass(frozen=True, slots=True)
+class AdapterResponse:
+    """Unified provider response payload for benchmark telemetry."""
+
+    surface: str
+    input_tokens: int | None = None
+    output_tokens: int | None = None
+    total_tokens: int | None = None
+
+    def as_dict(self) -> dict:
+        return {
+            "surface": self.surface,
+            "input_tokens": self.input_tokens,
+            "output_tokens": self.output_tokens,
+            "total_tokens": self.total_tokens,
+        }
+
+
 # ---------------------------------------------------------------------------
 # Adapter builders
 # ---------------------------------------------------------------------------
 
-def _build_core_adapter(cfg: ProviderConfig) -> Callable[[str], str]:
+def _build_core_adapter(cfg: ProviderConfig) -> Callable[[str], AdapterResponse]:
     """CORE ChatRuntime adapter.  Fresh runtime per call — no session bleed."""
     from chat.runtime import ChatRuntime
 
-    def adapter(prompt: str) -> str:
+    def adapter(prompt: str) -> AdapterResponse:
         rt = ChatRuntime()
         resp = rt.chat(prompt, max_tokens=cfg.max_tokens)
-        return resp.surface or ""
+        return AdapterResponse(surface=resp.surface or "")
 
     return adapter
 
 
-def _build_openai_adapter(cfg: ProviderConfig) -> Callable[[str], str]:
+def _build_openai_adapter(cfg: ProviderConfig) -> Callable[[str], AdapterResponse]:
     """OpenAI Chat Completions adapter.
 
     Requires: ``pip install openai``
@@ -200,19 +218,28 @@ def _build_openai_adapter(cfg: ProviderConfig) -> Callable[[str], str]:
         client_kwargs["base_url"] = cfg.extra["base_url"]
     client = openai.OpenAI(**client_kwargs)
 
-    def adapter(prompt: str) -> str:
+    def adapter(prompt: str) -> AdapterResponse:
         response = client.chat.completions.create(
             model=cfg.model,
             messages=[{"role": "user", "content": prompt}],
             temperature=cfg.temperature,
             max_tokens=cfg.max_tokens,
         )
-        return (response.choices[0].message.content or "").strip()
+        usage = getattr(response, "usage", None)
+        input_tokens = int(getattr(usage, "prompt_tokens", 0)) if usage else None
+        output_tokens = int(getattr(usage, "completion_tokens", 0)) if usage else None
+        total_tokens = int(getattr(usage, "total_tokens", 0)) if usage else None
+        return AdapterResponse(
+            surface=(response.choices[0].message.content or "").strip(),
+            input_tokens=input_tokens,
+            output_tokens=output_tokens,
+            total_tokens=total_tokens,
+        )
 
     return adapter
 
 
-def _build_anthropic_adapter(cfg: ProviderConfig) -> Callable[[str], str]:
+def _build_anthropic_adapter(cfg: ProviderConfig) -> Callable[[str], AdapterResponse]:
     """Anthropic Messages adapter.
 
     Requires: ``pip install anthropic``
@@ -229,7 +256,7 @@ def _build_anthropic_adapter(cfg: ProviderConfig) -> Callable[[str], str]:
     api_key = _require_env("ANTHROPIC_API_KEY")
     client = ant.Anthropic(api_key=api_key)
 
-    def adapter(prompt: str) -> str:
+    def adapter(prompt: str) -> AdapterResponse:
         message = client.messages.create(
             model=cfg.model,
             max_tokens=cfg.max_tokens,
@@ -239,12 +266,25 @@ def _build_anthropic_adapter(cfg: ProviderConfig) -> Callable[[str], str]:
             temperature=max(0.0, min(1.0, cfg.temperature)),
         )
         block = message.content[0] if message.content else None
-        return (getattr(block, "text", "") or "").strip()
+        usage = getattr(message, "usage", None)
+        input_tokens = int(getattr(usage, "input_tokens", 0)) if usage else None
+        output_tokens = int(getattr(usage, "output_tokens", 0)) if usage else None
+        total_tokens = (
+            (input_tokens or 0) + (output_tokens or 0)
+            if (input_tokens is not None or output_tokens is not None)
+            else None
+        )
+        return AdapterResponse(
+            surface=(getattr(block, "text", "") or "").strip(),
+            input_tokens=input_tokens,
+            output_tokens=output_tokens,
+            total_tokens=total_tokens,
+        )
 
     return adapter
 
 
-def _build_ollama_adapter(cfg: ProviderConfig) -> Callable[[str], str]:
+def _build_ollama_adapter(cfg: ProviderConfig) -> Callable[[str], AdapterResponse]:
     """Ollama local model adapter (HTTP /api/chat endpoint).
 
     Requires: ``pip install httpx``  (already present in most Python envs)
@@ -267,7 +307,7 @@ def _build_ollama_adapter(cfg: ProviderConfig) -> Callable[[str], str]:
     if api_key:
         headers["Authorization"] = f"Bearer {api_key}"
 
-    def adapter(prompt: str) -> str:
+    def adapter(prompt: str) -> AdapterResponse:
         payload = {
             "model": cfg.model,
             "messages": [{"role": "user", "content": prompt}],
@@ -282,7 +322,19 @@ def _build_ollama_adapter(cfg: ProviderConfig) -> Callable[[str], str]:
         )
         resp.raise_for_status()
         data = resp.json()
-        return (data.get("message", {}).get("content") or "").strip()
+        input_tokens = data.get("prompt_eval_count")
+        output_tokens = data.get("eval_count")
+        total_tokens = (
+            int(input_tokens or 0) + int(output_tokens or 0)
+            if input_tokens is not None or output_tokens is not None
+            else None
+        )
+        return AdapterResponse(
+            surface=(data.get("message", {}).get("content") or "").strip(),
+            input_tokens=int(input_tokens) if input_tokens is not None else None,
+            output_tokens=int(output_tokens) if output_tokens is not None else None,
+            total_tokens=total_tokens,
+        )
 
     return adapter
 
@@ -307,6 +359,16 @@ def build_adapter(cfg: ProviderConfig) -> Callable[[str], str]:
     it is safe to pass to ``compare_to_llm`` and the frontier_compare
     runner suites.
     """
+    observing = build_observing_adapter(cfg)
+
+    def _surface_only(prompt: str) -> str:
+        return observing(prompt).surface
+
+    return _surface_only
+
+
+def build_observing_adapter(cfg: ProviderConfig) -> Callable[[str], AdapterResponse]:
+    """Return a provider adapter that includes usage telemetry when available."""
     builder = _BUILDERS.get(cfg.provider)
     if builder is None:
         raise ValueError(

tests/test_frontier_compare_cross_provider.py

@@ -113,6 +113,28 @@ def test_cli_prompt_battery_with_core_provider(tmp_path: Path, capsys) -> None:
     assert report_path.exists()
 
 
+def test_cli_replay_variability_with_core_provider(tmp_path: Path, capsys) -> None:
+    """Replay-variability suite should run through cross-provider path."""
+    report_path = tmp_path / "cross_core_replay.json"
+    code = main(
+        [
+            "--provider", "core",
+            "--suite", "replay_variability",
+            "--repeats", "2",
+            "--json",
+            "--report", str(report_path),
+        ]
+    )
+    out = capsys.readouterr().out
+    payload = json.loads(out)
+    assert payload["model"] == "core-native"
+    assert payload["mode"] == "core"
+    assert len(payload["suites"]) == 1
+    assert payload["suites"][0]["suite"] == "replay_variability"
+    assert code == 0
+    assert report_path.exists()
+
+
 def test_cli_rejects_core_only_suite_with_non_core_provider(capsys) -> None:
     """Loud failure when an operator asks for a CORE-only suite with a
     non-CORE provider — no silent telemetry degradation."""

tests/test_frontier_model_registry_pricing.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from evals.frontier_compare.cross_provider import run_prompt_battery
+from evals.frontier_compare.model_registry import require_model_card
+from evals.frontier_compare.providers import AdapterResponse, ProviderConfig
+
+
+def test_model_card_estimate_cost_formula() -> None:
+    card = require_model_card("openai", "gpt-4o-2024-08-06")
+    cost = card.estimate_cost_usd(input_tokens=2000, output_tokens=1000)
+    assert cost is not None
+    # (2000 / 1e6 * 2.5) + (1000 / 1e6 * 10.0) = 0.015
+    assert round(cost, 6) == 0.015
+
+
+def test_prompt_battery_records_usage_and_cost_when_available() -> None:
+    cfg = ProviderConfig(provider="openai", model="gpt-4o-2024-08-06")
+
+    def fake_adapter(_: str) -> AdapterResponse:
+        return AdapterResponse(
+            surface="ok",
+            input_tokens=120,
+            output_tokens=80,
+            total_tokens=200,
+        )
+
+    report = run_prompt_battery(
+        fake_adapter,
+        cfg=cfg,
+        prompts=(("case", "What is truth?"),),
+    )
+    obs = report.cases[0].details["observation"]
+    assert obs["input_tokens"] == 120
+    assert obs["output_tokens"] == 80
+    assert obs["total_tokens"] == 200
+    assert obs["estimated_cost_usd"] is not None
+    assert obs["estimated_cost_usd"] > 0