diff --git a/.env.example b/.env.example index f17ff442..491d2fa2 100644 --- a/.env.example +++ b/.env.example @@ -6,7 +6,7 @@ # --------------------------------------------------------------------------- # OpenAI # Used by: evals/frontier_compare/providers.py (OpenAIAdapter) -# Benchmarks: determinism, truth_lock, axis_orthogonality (provider=openai) +# Benchmarks: prompt_battery, replay_variability (provider=openai) # --------------------------------------------------------------------------- OPENAI_API_KEY= @@ -21,7 +21,7 @@ OPENAI_MODEL=gpt-4o # --------------------------------------------------------------------------- # Anthropic # Used by: evals/frontier_compare/providers.py (AnthropicAdapter) -# Benchmarks: determinism, truth_lock, axis_orthogonality (provider=anthropic) +# Benchmarks: prompt_battery, replay_variability (provider=anthropic) # --------------------------------------------------------------------------- ANTHROPIC_API_KEY= diff --git a/evals/frontier_compare/README.md b/evals/frontier_compare/README.md index d7715330..9300e926 100644 --- a/evals/frontier_compare/README.md +++ b/evals/frontier_compare/README.md @@ -10,15 +10,18 @@ If CORE can solve something the LLM cannot structurally audit, CORE must prove i If both solve it, compare correctness, determinism, traceability, latency, cost, memory, and failure mode. ``` -Wave 1 is deliberately local and CORE-native. It does **not** call external frontier APIs, does **not** require provider keys, and does **not** change runtime behavior. It creates the benchmark harness, report schema, recording UI, and first suites that measure the things CORE should already be able to defend: +Wave 1 now supports both CORE-native and cross-provider execution through adapter wiring. CORE-only suites remain local and deterministic; cross-provider suites can call external APIs when credentials are configured. + +Current suites focus on: * deterministic replay * truth-lock / groundedness behavior * register vs anchor-lens axis discipline -* compact machine-readable reports suitable for later head-to-head frontier runs +* compact machine-readable reports suitable for head-to-head frontier runs * a static visual report viewer for clean recordings and demos +* usage-token and formula-based cost telemetry when provider APIs expose usage -Provider adapters for GPT / Claude / Gemini / open-weight baselines are intentionally deferred to a later wave so this PR remains testable without secrets. +Provider adapters for OpenAI / Anthropic / Ollama are available; CORE still remains runnable with no API keys. --- @@ -74,6 +77,29 @@ surface_variation_observed anchor_lens_engagement_observed ``` +### `prompt_battery` (cross-provider) + +Provider-agnostic prompt battery over the adapter interface. + +Primary metrics: + +```text +non_empty_surface_rate +mean_latency_ms +token_usage_capture_rate +formula_cost_estimate (when pricing + usage are available) +``` + +### `replay_variability` (cross-provider) + +Runs repeated calls per prompt and scores stability. + +Primary metric: + +```text +stability_score = 1 / unique_surface_count +``` + --- ## Run @@ -98,6 +124,12 @@ Human-readable table: uv run python -m evals.frontier_compare --suite all ``` +Cross-provider (example): + +```bash +uv run python -m evals.frontier_compare --provider openai --suite all --json +``` + --- ## Recording UI diff --git a/evals/frontier_compare/__main__.py b/evals/frontier_compare/__main__.py index 631264ab..5fd67c05 100644 --- a/evals/frontier_compare/__main__.py +++ b/evals/frontier_compare/__main__.py @@ -5,9 +5,9 @@ import json import sys from pathlib import Path -from .cross_provider import run_prompt_battery +from .cross_provider import run_prompt_battery, run_replay_variability from .model_registry import require_model_card -from .providers import ProviderConfig, build_adapter, load_dotenv_if_present +from .providers import ProviderConfig, build_observing_adapter, load_dotenv_if_present from .runner import ( BenchmarkReport, format_human_report, @@ -18,7 +18,7 @@ from .runner import ( _CORE_ONLY_SUITES = ("determinism", "truth_lock", "axis_orthogonality", "all") -_CROSS_PROVIDER_SUITES = ("prompt_battery",) +_CROSS_PROVIDER_SUITES = ("prompt_battery", "replay_variability") _VALID_SUITES = _CORE_ONLY_SUITES + _CROSS_PROVIDER_SUITES @@ -39,7 +39,7 @@ def build_parser() -> argparse.ArgumentParser: "benchmark suite to run. CORE-only suites: " f"{', '.join(_CORE_ONLY_SUITES)}. Cross-provider suites: " f"{', '.join(_CROSS_PROVIDER_SUITES)}. Default 'all' runs every " - "CORE-only suite when --provider=core, otherwise prompt_battery." + "CORE-only suite when --provider=core, otherwise all cross-provider suites." ), ) parser.add_argument( @@ -86,6 +86,15 @@ def build_parser() -> argparse.ArgumentParser: default=Path(".env"), help="path to a .env file with provider credentials (default: ./.env).", ) + parser.add_argument( + "--repeats", + type=int, + default=3, + help=( + "repeat count for repeat-sensitive cross-provider suites " + "(currently replay_variability)." + ), + ) return parser @@ -147,13 +156,31 @@ def main(argv: list[str] | None = None) -> int: else: # Cross-provider path — runs over the provider adapter. cfg = _build_cfg(args) - adapter = build_adapter(cfg) - suite_report = run_prompt_battery(adapter, cfg=cfg) + adapter = build_observing_adapter(cfg) + if args.suite == "all" and args.provider != "core": + suite_reports = ( + run_prompt_battery(adapter, cfg=cfg), + run_replay_variability( + adapter, + cfg=cfg, + repeats=max(1, int(args.repeats)), + ), + ) + elif args.suite == "replay_variability": + suite_reports = ( + run_replay_variability( + adapter, + cfg=cfg, + repeats=max(1, int(args.repeats)), + ), + ) + else: + suite_reports = (run_prompt_battery(adapter, cfg=cfg),) report = BenchmarkReport( benchmark_family="frontier_compare_wave1", model=cfg.model, mode=cfg.provider, - suites=(suite_report,), + suites=suite_reports, ) # Always persist non-CORE runs — they're rate-limited / paid, so # losing the artifact is genuinely costly. CORE adapter runs of diff --git a/evals/frontier_compare/cross_provider.py b/evals/frontier_compare/cross_provider.py index 60fc4567..e4ce7ad1 100644 --- a/evals/frontier_compare/cross_provider.py +++ b/evals/frontier_compare/cross_provider.py @@ -36,6 +36,7 @@ import time from dataclasses import dataclass from typing import Callable +from .model_registry import resolve_model_card from .providers import ProviderConfig from .runner import CaseResult, SuiteReport @@ -78,6 +79,10 @@ class ProviderObservation: provider: str model: str elapsed_ms: float + input_tokens: int | None = None + output_tokens: int | None = None + total_tokens: int | None = None + estimated_cost_usd: float | None = None error_type: str = "" error_message: str = "" @@ -92,6 +97,10 @@ class ProviderObservation: "provider": self.provider, "model": self.model, "elapsed_ms": self.elapsed_ms, + "input_tokens": self.input_tokens, + "output_tokens": self.output_tokens, + "total_tokens": self.total_tokens, + "estimated_cost_usd": self.estimated_cost_usd, "error_type": self.error_type, "error_message": self.error_message, } @@ -103,13 +112,14 @@ class ProviderObservation: def _observe_one( - adapter: Callable[[str], str], + adapter: Callable[[str], object], cfg: ProviderConfig, prompt: str, ) -> ProviderObservation: + card = resolve_model_card(cfg.provider, cfg.model) start = time.perf_counter() try: - surface = adapter(prompt) + raw = adapter(prompt) except Exception as exc: # noqa: BLE001 - record failure, never abort the suite return ProviderObservation( prompt=prompt, @@ -121,17 +131,45 @@ def _observe_one( error_message=str(exc), ) elapsed_ms = (time.perf_counter() - start) * 1000.0 + surface = "" + input_tokens: int | None = None + output_tokens: int | None = None + total_tokens: int | None = None + if isinstance(raw, str): + surface = raw + else: + surface = str(getattr(raw, "surface", "") or "") + in_val = getattr(raw, "input_tokens", None) + out_val = getattr(raw, "output_tokens", None) + total_val = getattr(raw, "total_tokens", None) + input_tokens = int(in_val) if in_val is not None else None + output_tokens = int(out_val) if out_val is not None else None + total_tokens = int(total_val) if total_val is not None else None + estimated_cost_usd: float | None = None + if ( + card is not None + and input_tokens is not None + and output_tokens is not None + ): + estimated_cost_usd = card.estimate_cost_usd( + input_tokens=input_tokens, + output_tokens=output_tokens, + ) return ProviderObservation( prompt=prompt, surface=surface, provider=cfg.provider, model=cfg.model, elapsed_ms=elapsed_ms, + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=total_tokens, + estimated_cost_usd=estimated_cost_usd, ) def run_prompt_battery( - adapter: Callable[[str], str], + adapter: Callable[[str], object], *, cfg: ProviderConfig, prompts: tuple[tuple[str, str], ...] = _PROMPT_BATTERY, @@ -175,7 +213,89 @@ def run_prompt_battery( ) +def run_replay_variability( + adapter: Callable[[str], object], + *, + cfg: ProviderConfig, + repeats: int = 3, + prompts: tuple[tuple[str, str], ...] = _PROMPT_BATTERY, +) -> SuiteReport: + """Run repeated calls per prompt and score surface stability. + + Score formula per case: + stability_score = 1 / unique_surface_count + where unique_surface_count >= 1 over `repeats` runs. + """ + runs = max(1, int(repeats)) + cases: list[CaseResult] = [] + for case_id, prompt in prompts: + observations = [_observe_one(adapter, cfg, prompt) for _ in range(runs)] + successful = [o for o in observations if not o.error_type] + failures: list[str] = [] + if not successful: + failures.append("adapter_error") + unique_count = 0 + score = 0.0 + elapsed_ms = sum(o.elapsed_ms for o in observations) + details = { + "repeats": runs, + "observations": [o.as_dict() for o in observations], + } + else: + surfaces = { + o.surface.strip() + for o in successful + if o.surface.strip() + } + if not surfaces: + failures.append("empty_surface") + unique_count = len(surfaces) if surfaces else 0 + score = 0.0 if unique_count == 0 else (1.0 / float(unique_count)) + if any(o.error_type for o in observations): + failures.append("partial_adapter_error") + elapsed_ms = sum(o.elapsed_ms for o in observations) + costs = [ + float(o.estimated_cost_usd) + for o in successful + if o.estimated_cost_usd is not None + ] + details = { + "repeats": runs, + "successful_runs": len(successful), + "unique_surface_count": unique_count, + "mean_elapsed_ms": (elapsed_ms / runs) if runs else 0.0, + "mean_estimated_cost_usd": ( + (sum(costs) / len(costs)) + if costs else None + ), + "observations": [o.as_dict() for o in observations], + } + passed = not failures and unique_count == 1 + cases.append( + CaseResult( + suite="replay_variability", + case_id=case_id, + prompt=prompt, + passed=passed, + score=score, + elapsed_ms=elapsed_ms, + details=details, + failures=tuple(failures), + ) + ) + primary = ( + sum(c.score for c in cases) / len(cases) if cases else 0.0 + ) + return SuiteReport( + suite="replay_variability", + cases=tuple(cases), + primary_score=primary, + passed=all(c.passed for c in cases), + ) + + __all__ = [ "ProviderObservation", "run_prompt_battery", + "run_replay_variability", ] diff --git a/evals/frontier_compare/model_registry.py b/evals/frontier_compare/model_registry.py index 1c3497f9..f4e86d0f 100644 --- a/evals/frontier_compare/model_registry.py +++ b/evals/frontier_compare/model_registry.py @@ -61,6 +61,15 @@ class ModelCard: notes: str = "" """Free-form notes: known quirks, benchmark-specific observations, version history.""" + input_usd_per_million_tokens: float | None = None + """Public list price for input tokens in USD per 1M tokens.""" + + output_usd_per_million_tokens: float | None = None + """Public list price for output tokens in USD per 1M tokens.""" + + pricing_source: str = "" + """Source URL/note for pricing metadata.""" + tags: tuple[str, ...] = field(default_factory=tuple) """Searchable tags, e.g. ('reasoning', 'code', 'vision').""" @@ -69,6 +78,35 @@ class ModelCard: d["tags"] = list(self.tags) return d + @property + def has_pricing(self) -> bool: + return ( + self.input_usd_per_million_tokens is not None + and self.output_usd_per_million_tokens is not None + ) + + def estimate_cost_usd( + self, + *, + input_tokens: int | float, + output_tokens: int | float, + ) -> float | None: + """Compute provider list-price cost for one request. + + Formula: + cost_usd = + (input_tokens / 1_000_000) * input_usd_per_million_tokens + + (output_tokens / 1_000_000) * output_usd_per_million_tokens + """ + if not self.has_pricing: + return None + in_rate = float(self.input_usd_per_million_tokens or 0.0) + out_rate = float(self.output_usd_per_million_tokens or 0.0) + return ( + (float(input_tokens) / 1_000_000.0) * in_rate + + (float(output_tokens) / 1_000_000.0) * out_rate + ) + # --------------------------------------------------------------------------- # Registry @@ -107,6 +145,9 @@ _REGISTRY: dict[str, ModelCard] = { "Use a dated snapshot (e.g. gpt-4o-2024-08-06) for reproducible benchmarks. " "Set OPENAI_MODEL=gpt-4o-2024-08-06 in .env." ), + input_usd_per_million_tokens=2.50, + output_usd_per_million_tokens=10.00, + pricing_source="https://openai.com/api/pricing (captured 2026-05-20)", tags=("frontier", "multimodal", "reasoning", "code"), ), "openai/gpt-4o-2024-08-06": ModelCard( @@ -119,6 +160,9 @@ _REGISTRY: dict[str, ModelCard] = { architecture="GPT-4 class transformer, multimodal (text + vision).", sampling="Stochastic at T>0. T=0 is near-deterministic for a fixed snapshot but backend routing can still vary.", notes="Pinned snapshot. Preferred for reproducible benchmark comparisons.", + input_usd_per_million_tokens=2.50, + output_usd_per_million_tokens=10.00, + pricing_source="https://openai.com/api/pricing (captured 2026-05-20)", tags=("frontier", "multimodal", "reasoning", "code", "pinned"), ), "openai/gpt-4o-mini": ModelCard( @@ -131,6 +175,9 @@ _REGISTRY: dict[str, ModelCard] = { architecture="Smaller GPT-4o class model optimised for latency and cost.", sampling="Stochastic at T>0.", notes="Useful for cost/latency baseline comparisons in the benchmark cost suite.", + input_usd_per_million_tokens=0.15, + output_usd_per_million_tokens=0.60, + pricing_source="https://openai.com/api/pricing (captured 2026-05-20)", tags=("frontier", "fast", "cost-efficient"), ), "openai/o3": ModelCard( @@ -146,6 +193,7 @@ _REGISTRY: dict[str, ModelCard] = { "o-series models use a reasoning_effort parameter instead of temperature. " "Pass via ProviderConfig.extra = {'reasoning_effort': 'high'} for benchmark use." ), + pricing_source="https://openai.com/api/pricing (captured 2026-05-20)", tags=("frontier", "reasoning", "chain-of-thought"), ), @@ -164,6 +212,9 @@ _REGISTRY: dict[str, ModelCard] = { "Default ANTHROPIC_MODEL in .env.example. " "For extended thinking benchmarks, pass extra={'thinking': {'type': 'enabled', 'budget_tokens': 10000}}." ), + input_usd_per_million_tokens=15.00, + output_usd_per_million_tokens=75.00, + pricing_source="https://www.anthropic.com/pricing (captured 2026-05-20)", tags=("frontier", "reasoning", "code", "extended-thinking"), ), "anthropic/claude-sonnet-4-5": ModelCard( @@ -176,6 +227,9 @@ _REGISTRY: dict[str, ModelCard] = { architecture="Claude 4 class transformer (Anthropic). Balanced speed/capability.", sampling="Stochastic at T>0.", notes="Good default for high-volume benchmark sweeps where Opus cost is prohibitive.", + input_usd_per_million_tokens=3.00, + output_usd_per_million_tokens=15.00, + pricing_source="https://www.anthropic.com/pricing (captured 2026-05-20)", tags=("frontier", "balanced", "cost-efficient"), ), "anthropic/claude-haiku-3-5": ModelCard( @@ -188,6 +242,9 @@ _REGISTRY: dict[str, ModelCard] = { architecture="Claude 3 class transformer (Anthropic). Optimised for latency.", sampling="Stochastic at T>0.", notes="Useful for latency/cost baseline comparisons. Lower capability ceiling than Sonnet/Opus.", + input_usd_per_million_tokens=0.80, + output_usd_per_million_tokens=4.00, + pricing_source="https://www.anthropic.com/pricing (captured 2026-05-20)", tags=("frontier", "fast", "cost-efficient"), ), diff --git a/evals/frontier_compare/providers.py b/evals/frontier_compare/providers.py index 205b3ac3..26776d24 100644 --- a/evals/frontier_compare/providers.py +++ b/evals/frontier_compare/providers.py @@ -164,23 +164,41 @@ class ProviderConfig: } +@dataclass(frozen=True, slots=True) +class AdapterResponse: + """Unified provider response payload for benchmark telemetry.""" + + surface: str + input_tokens: int | None = None + output_tokens: int | None = None + total_tokens: int | None = None + + def as_dict(self) -> dict: + return { + "surface": self.surface, + "input_tokens": self.input_tokens, + "output_tokens": self.output_tokens, + "total_tokens": self.total_tokens, + } + + # --------------------------------------------------------------------------- # Adapter builders # --------------------------------------------------------------------------- -def _build_core_adapter(cfg: ProviderConfig) -> Callable[[str], str]: +def _build_core_adapter(cfg: ProviderConfig) -> Callable[[str], AdapterResponse]: """CORE ChatRuntime adapter. Fresh runtime per call — no session bleed.""" from chat.runtime import ChatRuntime - def adapter(prompt: str) -> str: + def adapter(prompt: str) -> AdapterResponse: rt = ChatRuntime() resp = rt.chat(prompt, max_tokens=cfg.max_tokens) - return resp.surface or "" + return AdapterResponse(surface=resp.surface or "") return adapter -def _build_openai_adapter(cfg: ProviderConfig) -> Callable[[str], str]: +def _build_openai_adapter(cfg: ProviderConfig) -> Callable[[str], AdapterResponse]: """OpenAI Chat Completions adapter. Requires: ``pip install openai`` @@ -200,19 +218,28 @@ def _build_openai_adapter(cfg: ProviderConfig) -> Callable[[str], str]: client_kwargs["base_url"] = cfg.extra["base_url"] client = openai.OpenAI(**client_kwargs) - def adapter(prompt: str) -> str: + def adapter(prompt: str) -> AdapterResponse: response = client.chat.completions.create( model=cfg.model, messages=[{"role": "user", "content": prompt}], temperature=cfg.temperature, max_tokens=cfg.max_tokens, ) - return (response.choices[0].message.content or "").strip() + usage = getattr(response, "usage", None) + input_tokens = int(getattr(usage, "prompt_tokens", 0)) if usage else None + output_tokens = int(getattr(usage, "completion_tokens", 0)) if usage else None + total_tokens = int(getattr(usage, "total_tokens", 0)) if usage else None + return AdapterResponse( + surface=(response.choices[0].message.content or "").strip(), + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=total_tokens, + ) return adapter -def _build_anthropic_adapter(cfg: ProviderConfig) -> Callable[[str], str]: +def _build_anthropic_adapter(cfg: ProviderConfig) -> Callable[[str], AdapterResponse]: """Anthropic Messages adapter. Requires: ``pip install anthropic`` @@ -229,7 +256,7 @@ def _build_anthropic_adapter(cfg: ProviderConfig) -> Callable[[str], str]: api_key = _require_env("ANTHROPIC_API_KEY") client = ant.Anthropic(api_key=api_key) - def adapter(prompt: str) -> str: + def adapter(prompt: str) -> AdapterResponse: message = client.messages.create( model=cfg.model, max_tokens=cfg.max_tokens, @@ -239,12 +266,25 @@ def _build_anthropic_adapter(cfg: ProviderConfig) -> Callable[[str], str]: temperature=max(0.0, min(1.0, cfg.temperature)), ) block = message.content[0] if message.content else None - return (getattr(block, "text", "") or "").strip() + usage = getattr(message, "usage", None) + input_tokens = int(getattr(usage, "input_tokens", 0)) if usage else None + output_tokens = int(getattr(usage, "output_tokens", 0)) if usage else None + total_tokens = ( + (input_tokens or 0) + (output_tokens or 0) + if (input_tokens is not None or output_tokens is not None) + else None + ) + return AdapterResponse( + surface=(getattr(block, "text", "") or "").strip(), + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=total_tokens, + ) return adapter -def _build_ollama_adapter(cfg: ProviderConfig) -> Callable[[str], str]: +def _build_ollama_adapter(cfg: ProviderConfig) -> Callable[[str], AdapterResponse]: """Ollama local model adapter (HTTP /api/chat endpoint). Requires: ``pip install httpx`` (already present in most Python envs) @@ -267,7 +307,7 @@ def _build_ollama_adapter(cfg: ProviderConfig) -> Callable[[str], str]: if api_key: headers["Authorization"] = f"Bearer {api_key}" - def adapter(prompt: str) -> str: + def adapter(prompt: str) -> AdapterResponse: payload = { "model": cfg.model, "messages": [{"role": "user", "content": prompt}], @@ -282,7 +322,19 @@ def _build_ollama_adapter(cfg: ProviderConfig) -> Callable[[str], str]: ) resp.raise_for_status() data = resp.json() - return (data.get("message", {}).get("content") or "").strip() + input_tokens = data.get("prompt_eval_count") + output_tokens = data.get("eval_count") + total_tokens = ( + int(input_tokens or 0) + int(output_tokens or 0) + if input_tokens is not None or output_tokens is not None + else None + ) + return AdapterResponse( + surface=(data.get("message", {}).get("content") or "").strip(), + input_tokens=int(input_tokens) if input_tokens is not None else None, + output_tokens=int(output_tokens) if output_tokens is not None else None, + total_tokens=total_tokens, + ) return adapter @@ -307,6 +359,16 @@ def build_adapter(cfg: ProviderConfig) -> Callable[[str], str]: it is safe to pass to ``compare_to_llm`` and the frontier_compare runner suites. """ + observing = build_observing_adapter(cfg) + + def _surface_only(prompt: str) -> str: + return observing(prompt).surface + + return _surface_only + + +def build_observing_adapter(cfg: ProviderConfig) -> Callable[[str], AdapterResponse]: + """Return a provider adapter that includes usage telemetry when available.""" builder = _BUILDERS.get(cfg.provider) if builder is None: raise ValueError( diff --git a/tests/test_frontier_compare_cross_provider.py b/tests/test_frontier_compare_cross_provider.py index ccbf3753..8d84188f 100644 --- a/tests/test_frontier_compare_cross_provider.py +++ b/tests/test_frontier_compare_cross_provider.py @@ -113,6 +113,28 @@ def test_cli_prompt_battery_with_core_provider(tmp_path: Path, capsys) -> None: assert report_path.exists() +def test_cli_replay_variability_with_core_provider(tmp_path: Path, capsys) -> None: + """Replay-variability suite should run through cross-provider path.""" + report_path = tmp_path / "cross_core_replay.json" + code = main( + [ + "--provider", "core", + "--suite", "replay_variability", + "--repeats", "2", + "--json", + "--report", str(report_path), + ] + ) + out = capsys.readouterr().out + payload = json.loads(out) + assert payload["model"] == "core-native" + assert payload["mode"] == "core" + assert len(payload["suites"]) == 1 + assert payload["suites"][0]["suite"] == "replay_variability" + assert code == 0 + assert report_path.exists() + + def test_cli_rejects_core_only_suite_with_non_core_provider(capsys) -> None: """Loud failure when an operator asks for a CORE-only suite with a non-CORE provider — no silent telemetry degradation.""" diff --git a/tests/test_frontier_model_registry_pricing.py b/tests/test_frontier_model_registry_pricing.py new file mode 100644 index 00000000..171b5715 --- /dev/null +++ b/tests/test_frontier_model_registry_pricing.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from evals.frontier_compare.cross_provider import run_prompt_battery +from evals.frontier_compare.model_registry import require_model_card +from evals.frontier_compare.providers import AdapterResponse, ProviderConfig + + +def test_model_card_estimate_cost_formula() -> None: + card = require_model_card("openai", "gpt-4o-2024-08-06") + cost = card.estimate_cost_usd(input_tokens=2000, output_tokens=1000) + assert cost is not None + # (2000 / 1e6 * 2.5) + (1000 / 1e6 * 10.0) = 0.015 + assert round(cost, 6) == 0.015 + + +def test_prompt_battery_records_usage_and_cost_when_available() -> None: + cfg = ProviderConfig(provider="openai", model="gpt-4o-2024-08-06") + + def fake_adapter(_: str) -> AdapterResponse: + return AdapterResponse( + surface="ok", + input_tokens=120, + output_tokens=80, + total_tokens=200, + ) + + report = run_prompt_battery( + fake_adapter, + cfg=cfg, + prompts=(("case", "What is truth?"),), + ) + obs = report.cases[0].details["observation"] + assert obs["input_tokens"] == 120 + assert obs["output_tokens"] == 80 + assert obs["total_tokens"] == 200 + assert obs["estimated_cost_usd"] is not None + assert obs["estimated_cost_usd"] > 0