feat(frontier): add replay variability suite and token-cost telemetry (#66)
Agent-Logs-Url: https://github.com/AssetOverflow/core/sessions/f88b48fa-0c2a-4f9d-a42b-d275596e43b8 Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: AssetOverflow <109810776+AssetOverflow@users.noreply.github.com>
This commit is contained in:
Copilot
GitHub
parent
9e6fa4be75
commit
dedf05565d
8 changed files with 384 additions and 27 deletions
|
|
@ -6,7 +6,7 @@
|
|||
# ---------------------------------------------------------------------------
|
||||
# OpenAI
|
||||
# Used by: evals/frontier_compare/providers.py (OpenAIAdapter)
|
||||
# Benchmarks: determinism, truth_lock, axis_orthogonality (provider=openai)
|
||||
# Benchmarks: prompt_battery, replay_variability (provider=openai)
|
||||
# ---------------------------------------------------------------------------
|
||||
OPENAI_API_KEY=
|
||||
|
||||
|
|
@ -21,7 +21,7 @@ OPENAI_MODEL=gpt-4o
|
|||
# ---------------------------------------------------------------------------
|
||||
# Anthropic
|
||||
# Used by: evals/frontier_compare/providers.py (AnthropicAdapter)
|
||||
# Benchmarks: determinism, truth_lock, axis_orthogonality (provider=anthropic)
|
||||
# Benchmarks: prompt_battery, replay_variability (provider=anthropic)
|
||||
# ---------------------------------------------------------------------------
|
||||
ANTHROPIC_API_KEY=
|
||||
|
||||
|
|
|
|||
|
|
@ -10,15 +10,18 @@ If CORE can solve something the LLM cannot structurally audit, CORE must prove i
|
|||
If both solve it, compare correctness, determinism, traceability, latency, cost, memory, and failure mode.
|
||||
```
|
||||
|
||||
Wave 1 is deliberately local and CORE-native. It does **not** call external frontier APIs, does **not** require provider keys, and does **not** change runtime behavior. It creates the benchmark harness, report schema, recording UI, and first suites that measure the things CORE should already be able to defend:
|
||||
Wave 1 now supports both CORE-native and cross-provider execution through adapter wiring. CORE-only suites remain local and deterministic; cross-provider suites can call external APIs when credentials are configured.
|
||||
|
||||
Current suites focus on:
|
||||
|
||||
* deterministic replay
|
||||
* truth-lock / groundedness behavior
|
||||
* register vs anchor-lens axis discipline
|
||||
* compact machine-readable reports suitable for later head-to-head frontier runs
|
||||
* compact machine-readable reports suitable for head-to-head frontier runs
|
||||
* a static visual report viewer for clean recordings and demos
|
||||
* usage-token and formula-based cost telemetry when provider APIs expose usage
|
||||
|
||||
Provider adapters for GPT / Claude / Gemini / open-weight baselines are intentionally deferred to a later wave so this PR remains testable without secrets.
|
||||
Provider adapters for OpenAI / Anthropic / Ollama are available; CORE still remains runnable with no API keys.
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -74,6 +77,29 @@ surface_variation_observed
|
|||
anchor_lens_engagement_observed
|
||||
```
|
||||
|
||||
### `prompt_battery` (cross-provider)
|
||||
|
||||
Provider-agnostic prompt battery over the adapter interface.
|
||||
|
||||
Primary metrics:
|
||||
|
||||
```text
|
||||
non_empty_surface_rate
|
||||
mean_latency_ms
|
||||
token_usage_capture_rate
|
||||
formula_cost_estimate (when pricing + usage are available)
|
||||
```
|
||||
|
||||
### `replay_variability` (cross-provider)
|
||||
|
||||
Runs repeated calls per prompt and scores stability.
|
||||
|
||||
Primary metric:
|
||||
|
||||
```text
|
||||
stability_score = 1 / unique_surface_count
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Run
|
||||
|
|
@ -98,6 +124,12 @@ Human-readable table:
|
|||
uv run python -m evals.frontier_compare --suite all
|
||||
```
|
||||
|
||||
Cross-provider (example):
|
||||
|
||||
```bash
|
||||
uv run python -m evals.frontier_compare --provider openai --suite all --json
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Recording UI
|
||||
|
|
|
|||
|
|
@ -5,9 +5,9 @@ import json
|
|||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from .cross_provider import run_prompt_battery
|
||||
from .cross_provider import run_prompt_battery, run_replay_variability
|
||||
from .model_registry import require_model_card
|
||||
from .providers import ProviderConfig, build_adapter, load_dotenv_if_present
|
||||
from .providers import ProviderConfig, build_observing_adapter, load_dotenv_if_present
|
||||
from .runner import (
|
||||
BenchmarkReport,
|
||||
format_human_report,
|
||||
|
|
@ -18,7 +18,7 @@ from .runner import (
|
|||
|
||||
|
||||
_CORE_ONLY_SUITES = ("determinism", "truth_lock", "axis_orthogonality", "all")
|
||||
_CROSS_PROVIDER_SUITES = ("prompt_battery",)
|
||||
_CROSS_PROVIDER_SUITES = ("prompt_battery", "replay_variability")
|
||||
_VALID_SUITES = _CORE_ONLY_SUITES + _CROSS_PROVIDER_SUITES
|
||||
|
||||
|
||||
|
|
@ -39,7 +39,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
"benchmark suite to run. CORE-only suites: "
|
||||
f"{', '.join(_CORE_ONLY_SUITES)}. Cross-provider suites: "
|
||||
f"{', '.join(_CROSS_PROVIDER_SUITES)}. Default 'all' runs every "
|
||||
"CORE-only suite when --provider=core, otherwise prompt_battery."
|
||||
"CORE-only suite when --provider=core, otherwise all cross-provider suites."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
|
|
@ -86,6 +86,15 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
default=Path(".env"),
|
||||
help="path to a .env file with provider credentials (default: ./.env).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--repeats",
|
||||
type=int,
|
||||
default=3,
|
||||
help=(
|
||||
"repeat count for repeat-sensitive cross-provider suites "
|
||||
"(currently replay_variability)."
|
||||
),
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
|
|
@ -147,13 +156,31 @@ def main(argv: list[str] | None = None) -> int:
|
|||
else:
|
||||
# Cross-provider path — runs over the provider adapter.
|
||||
cfg = _build_cfg(args)
|
||||
adapter = build_adapter(cfg)
|
||||
suite_report = run_prompt_battery(adapter, cfg=cfg)
|
||||
adapter = build_observing_adapter(cfg)
|
||||
if args.suite == "all" and args.provider != "core":
|
||||
suite_reports = (
|
||||
run_prompt_battery(adapter, cfg=cfg),
|
||||
run_replay_variability(
|
||||
adapter,
|
||||
cfg=cfg,
|
||||
repeats=max(1, int(args.repeats)),
|
||||
),
|
||||
)
|
||||
elif args.suite == "replay_variability":
|
||||
suite_reports = (
|
||||
run_replay_variability(
|
||||
adapter,
|
||||
cfg=cfg,
|
||||
repeats=max(1, int(args.repeats)),
|
||||
),
|
||||
)
|
||||
else:
|
||||
suite_reports = (run_prompt_battery(adapter, cfg=cfg),)
|
||||
report = BenchmarkReport(
|
||||
benchmark_family="frontier_compare_wave1",
|
||||
model=cfg.model,
|
||||
mode=cfg.provider,
|
||||
suites=(suite_report,),
|
||||
suites=suite_reports,
|
||||
)
|
||||
# Always persist non-CORE runs — they're rate-limited / paid, so
|
||||
# losing the artifact is genuinely costly. CORE adapter runs of
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ import time
|
|||
from dataclasses import dataclass
|
||||
from typing import Callable
|
||||
|
||||
from .model_registry import resolve_model_card
|
||||
from .providers import ProviderConfig
|
||||
from .runner import CaseResult, SuiteReport
|
||||
|
||||
|
|
@ -78,6 +79,10 @@ class ProviderObservation:
|
|||
provider: str
|
||||
model: str
|
||||
elapsed_ms: float
|
||||
input_tokens: int | None = None
|
||||
output_tokens: int | None = None
|
||||
total_tokens: int | None = None
|
||||
estimated_cost_usd: float | None = None
|
||||
error_type: str = ""
|
||||
error_message: str = ""
|
||||
|
||||
|
|
@ -92,6 +97,10 @@ class ProviderObservation:
|
|||
"provider": self.provider,
|
||||
"model": self.model,
|
||||
"elapsed_ms": self.elapsed_ms,
|
||||
"input_tokens": self.input_tokens,
|
||||
"output_tokens": self.output_tokens,
|
||||
"total_tokens": self.total_tokens,
|
||||
"estimated_cost_usd": self.estimated_cost_usd,
|
||||
"error_type": self.error_type,
|
||||
"error_message": self.error_message,
|
||||
}
|
||||
|
|
@ -103,13 +112,14 @@ class ProviderObservation:
|
|||
|
||||
|
||||
def _observe_one(
|
||||
adapter: Callable[[str], str],
|
||||
adapter: Callable[[str], object],
|
||||
cfg: ProviderConfig,
|
||||
prompt: str,
|
||||
) -> ProviderObservation:
|
||||
card = resolve_model_card(cfg.provider, cfg.model)
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
surface = adapter(prompt)
|
||||
raw = adapter(prompt)
|
||||
except Exception as exc: # noqa: BLE001 - record failure, never abort the suite
|
||||
return ProviderObservation(
|
||||
prompt=prompt,
|
||||
|
|
@ -121,17 +131,45 @@ def _observe_one(
|
|||
error_message=str(exc),
|
||||
)
|
||||
elapsed_ms = (time.perf_counter() - start) * 1000.0
|
||||
surface = ""
|
||||
input_tokens: int | None = None
|
||||
output_tokens: int | None = None
|
||||
total_tokens: int | None = None
|
||||
if isinstance(raw, str):
|
||||
surface = raw
|
||||
else:
|
||||
surface = str(getattr(raw, "surface", "") or "")
|
||||
in_val = getattr(raw, "input_tokens", None)
|
||||
out_val = getattr(raw, "output_tokens", None)
|
||||
total_val = getattr(raw, "total_tokens", None)
|
||||
input_tokens = int(in_val) if in_val is not None else None
|
||||
output_tokens = int(out_val) if out_val is not None else None
|
||||
total_tokens = int(total_val) if total_val is not None else None
|
||||
estimated_cost_usd: float | None = None
|
||||
if (
|
||||
card is not None
|
||||
and input_tokens is not None
|
||||
and output_tokens is not None
|
||||
):
|
||||
estimated_cost_usd = card.estimate_cost_usd(
|
||||
input_tokens=input_tokens,
|
||||
output_tokens=output_tokens,
|
||||
)
|
||||
return ProviderObservation(
|
||||
prompt=prompt,
|
||||
surface=surface,
|
||||
provider=cfg.provider,
|
||||
model=cfg.model,
|
||||
elapsed_ms=elapsed_ms,
|
||||
input_tokens=input_tokens,
|
||||
output_tokens=output_tokens,
|
||||
total_tokens=total_tokens,
|
||||
estimated_cost_usd=estimated_cost_usd,
|
||||
)
|
||||
|
||||
|
||||
def run_prompt_battery(
|
||||
adapter: Callable[[str], str],
|
||||
adapter: Callable[[str], object],
|
||||
*,
|
||||
cfg: ProviderConfig,
|
||||
prompts: tuple[tuple[str, str], ...] = _PROMPT_BATTERY,
|
||||
|
|
@ -175,7 +213,89 @@ def run_prompt_battery(
|
|||
)
|
||||
|
||||
|
||||
def run_replay_variability(
|
||||
adapter: Callable[[str], object],
|
||||
*,
|
||||
cfg: ProviderConfig,
|
||||
repeats: int = 3,
|
||||
prompts: tuple[tuple[str, str], ...] = _PROMPT_BATTERY,
|
||||
) -> SuiteReport:
|
||||
"""Run repeated calls per prompt and score surface stability.
|
||||
|
||||
Score formula per case:
|
||||
stability_score = 1 / unique_surface_count
|
||||
where unique_surface_count >= 1 over `repeats` runs.
|
||||
"""
|
||||
runs = max(1, int(repeats))
|
||||
cases: list[CaseResult] = []
|
||||
for case_id, prompt in prompts:
|
||||
observations = [_observe_one(adapter, cfg, prompt) for _ in range(runs)]
|
||||
successful = [o for o in observations if not o.error_type]
|
||||
failures: list[str] = []
|
||||
if not successful:
|
||||
failures.append("adapter_error")
|
||||
unique_count = 0
|
||||
score = 0.0
|
||||
elapsed_ms = sum(o.elapsed_ms for o in observations)
|
||||
details = {
|
||||
"repeats": runs,
|
||||
"observations": [o.as_dict() for o in observations],
|
||||
}
|
||||
else:
|
||||
surfaces = {
|
||||
o.surface.strip()
|
||||
for o in successful
|
||||
if o.surface.strip()
|
||||
}
|
||||
if not surfaces:
|
||||
failures.append("empty_surface")
|
||||
unique_count = len(surfaces) if surfaces else 0
|
||||
score = 0.0 if unique_count == 0 else (1.0 / float(unique_count))
|
||||
if any(o.error_type for o in observations):
|
||||
failures.append("partial_adapter_error")
|
||||
elapsed_ms = sum(o.elapsed_ms for o in observations)
|
||||
costs = [
|
||||
float(o.estimated_cost_usd)
|
||||
for o in successful
|
||||
if o.estimated_cost_usd is not None
|
||||
]
|
||||
details = {
|
||||
"repeats": runs,
|
||||
"successful_runs": len(successful),
|
||||
"unique_surface_count": unique_count,
|
||||
"mean_elapsed_ms": (elapsed_ms / runs) if runs else 0.0,
|
||||
"mean_estimated_cost_usd": (
|
||||
(sum(costs) / len(costs))
|
||||
if costs else None
|
||||
),
|
||||
"observations": [o.as_dict() for o in observations],
|
||||
}
|
||||
passed = not failures and unique_count == 1
|
||||
cases.append(
|
||||
CaseResult(
|
||||
suite="replay_variability",
|
||||
case_id=case_id,
|
||||
prompt=prompt,
|
||||
passed=passed,
|
||||
score=score,
|
||||
elapsed_ms=elapsed_ms,
|
||||
details=details,
|
||||
failures=tuple(failures),
|
||||
)
|
||||
)
|
||||
primary = (
|
||||
sum(c.score for c in cases) / len(cases) if cases else 0.0
|
||||
)
|
||||
return SuiteReport(
|
||||
suite="replay_variability",
|
||||
cases=tuple(cases),
|
||||
primary_score=primary,
|
||||
passed=all(c.passed for c in cases),
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ProviderObservation",
|
||||
"run_prompt_battery",
|
||||
"run_replay_variability",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -61,6 +61,15 @@ class ModelCard:
|
|||
notes: str = ""
|
||||
"""Free-form notes: known quirks, benchmark-specific observations, version history."""
|
||||
|
||||
input_usd_per_million_tokens: float | None = None
|
||||
"""Public list price for input tokens in USD per 1M tokens."""
|
||||
|
||||
output_usd_per_million_tokens: float | None = None
|
||||
"""Public list price for output tokens in USD per 1M tokens."""
|
||||
|
||||
pricing_source: str = ""
|
||||
"""Source URL/note for pricing metadata."""
|
||||
|
||||
tags: tuple[str, ...] = field(default_factory=tuple)
|
||||
"""Searchable tags, e.g. ('reasoning', 'code', 'vision')."""
|
||||
|
||||
|
|
@ -69,6 +78,35 @@ class ModelCard:
|
|||
d["tags"] = list(self.tags)
|
||||
return d
|
||||
|
||||
@property
|
||||
def has_pricing(self) -> bool:
|
||||
return (
|
||||
self.input_usd_per_million_tokens is not None
|
||||
and self.output_usd_per_million_tokens is not None
|
||||
)
|
||||
|
||||
def estimate_cost_usd(
|
||||
self,
|
||||
*,
|
||||
input_tokens: int | float,
|
||||
output_tokens: int | float,
|
||||
) -> float | None:
|
||||
"""Compute provider list-price cost for one request.
|
||||
|
||||
Formula:
|
||||
cost_usd =
|
||||
(input_tokens / 1_000_000) * input_usd_per_million_tokens +
|
||||
(output_tokens / 1_000_000) * output_usd_per_million_tokens
|
||||
"""
|
||||
if not self.has_pricing:
|
||||
return None
|
||||
in_rate = float(self.input_usd_per_million_tokens or 0.0)
|
||||
out_rate = float(self.output_usd_per_million_tokens or 0.0)
|
||||
return (
|
||||
(float(input_tokens) / 1_000_000.0) * in_rate
|
||||
+ (float(output_tokens) / 1_000_000.0) * out_rate
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registry
|
||||
|
|
@ -107,6 +145,9 @@ _REGISTRY: dict[str, ModelCard] = {
|
|||
"Use a dated snapshot (e.g. gpt-4o-2024-08-06) for reproducible benchmarks. "
|
||||
"Set OPENAI_MODEL=gpt-4o-2024-08-06 in .env."
|
||||
),
|
||||
input_usd_per_million_tokens=2.50,
|
||||
output_usd_per_million_tokens=10.00,
|
||||
pricing_source="https://openai.com/api/pricing (captured 2026-05-20)",
|
||||
tags=("frontier", "multimodal", "reasoning", "code"),
|
||||
),
|
||||
"openai/gpt-4o-2024-08-06": ModelCard(
|
||||
|
|
@ -119,6 +160,9 @@ _REGISTRY: dict[str, ModelCard] = {
|
|||
architecture="GPT-4 class transformer, multimodal (text + vision).",
|
||||
sampling="Stochastic at T>0. T=0 is near-deterministic for a fixed snapshot but backend routing can still vary.",
|
||||
notes="Pinned snapshot. Preferred for reproducible benchmark comparisons.",
|
||||
input_usd_per_million_tokens=2.50,
|
||||
output_usd_per_million_tokens=10.00,
|
||||
pricing_source="https://openai.com/api/pricing (captured 2026-05-20)",
|
||||
tags=("frontier", "multimodal", "reasoning", "code", "pinned"),
|
||||
),
|
||||
"openai/gpt-4o-mini": ModelCard(
|
||||
|
|
@ -131,6 +175,9 @@ _REGISTRY: dict[str, ModelCard] = {
|
|||
architecture="Smaller GPT-4o class model optimised for latency and cost.",
|
||||
sampling="Stochastic at T>0.",
|
||||
notes="Useful for cost/latency baseline comparisons in the benchmark cost suite.",
|
||||
input_usd_per_million_tokens=0.15,
|
||||
output_usd_per_million_tokens=0.60,
|
||||
pricing_source="https://openai.com/api/pricing (captured 2026-05-20)",
|
||||
tags=("frontier", "fast", "cost-efficient"),
|
||||
),
|
||||
"openai/o3": ModelCard(
|
||||
|
|
@ -146,6 +193,7 @@ _REGISTRY: dict[str, ModelCard] = {
|
|||
"o-series models use a reasoning_effort parameter instead of temperature. "
|
||||
"Pass via ProviderConfig.extra = {'reasoning_effort': 'high'} for benchmark use."
|
||||
),
|
||||
pricing_source="https://openai.com/api/pricing (captured 2026-05-20)",
|
||||
tags=("frontier", "reasoning", "chain-of-thought"),
|
||||
),
|
||||
|
||||
|
|
@ -164,6 +212,9 @@ _REGISTRY: dict[str, ModelCard] = {
|
|||
"Default ANTHROPIC_MODEL in .env.example. "
|
||||
"For extended thinking benchmarks, pass extra={'thinking': {'type': 'enabled', 'budget_tokens': 10000}}."
|
||||
),
|
||||
input_usd_per_million_tokens=15.00,
|
||||
output_usd_per_million_tokens=75.00,
|
||||
pricing_source="https://www.anthropic.com/pricing (captured 2026-05-20)",
|
||||
tags=("frontier", "reasoning", "code", "extended-thinking"),
|
||||
),
|
||||
"anthropic/claude-sonnet-4-5": ModelCard(
|
||||
|
|
@ -176,6 +227,9 @@ _REGISTRY: dict[str, ModelCard] = {
|
|||
architecture="Claude 4 class transformer (Anthropic). Balanced speed/capability.",
|
||||
sampling="Stochastic at T>0.",
|
||||
notes="Good default for high-volume benchmark sweeps where Opus cost is prohibitive.",
|
||||
input_usd_per_million_tokens=3.00,
|
||||
output_usd_per_million_tokens=15.00,
|
||||
pricing_source="https://www.anthropic.com/pricing (captured 2026-05-20)",
|
||||
tags=("frontier", "balanced", "cost-efficient"),
|
||||
),
|
||||
"anthropic/claude-haiku-3-5": ModelCard(
|
||||
|
|
@ -188,6 +242,9 @@ _REGISTRY: dict[str, ModelCard] = {
|
|||
architecture="Claude 3 class transformer (Anthropic). Optimised for latency.",
|
||||
sampling="Stochastic at T>0.",
|
||||
notes="Useful for latency/cost baseline comparisons. Lower capability ceiling than Sonnet/Opus.",
|
||||
input_usd_per_million_tokens=0.80,
|
||||
output_usd_per_million_tokens=4.00,
|
||||
pricing_source="https://www.anthropic.com/pricing (captured 2026-05-20)",
|
||||
tags=("frontier", "fast", "cost-efficient"),
|
||||
),
|
||||
|
||||
|
|
|
|||
|
|
@ -164,23 +164,41 @@ class ProviderConfig:
|
|||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AdapterResponse:
|
||||
"""Unified provider response payload for benchmark telemetry."""
|
||||
|
||||
surface: str
|
||||
input_tokens: int | None = None
|
||||
output_tokens: int | None = None
|
||||
total_tokens: int | None = None
|
||||
|
||||
def as_dict(self) -> dict:
|
||||
return {
|
||||
"surface": self.surface,
|
||||
"input_tokens": self.input_tokens,
|
||||
"output_tokens": self.output_tokens,
|
||||
"total_tokens": self.total_tokens,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Adapter builders
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _build_core_adapter(cfg: ProviderConfig) -> Callable[[str], str]:
|
||||
def _build_core_adapter(cfg: ProviderConfig) -> Callable[[str], AdapterResponse]:
|
||||
"""CORE ChatRuntime adapter. Fresh runtime per call — no session bleed."""
|
||||
from chat.runtime import ChatRuntime
|
||||
|
||||
def adapter(prompt: str) -> str:
|
||||
def adapter(prompt: str) -> AdapterResponse:
|
||||
rt = ChatRuntime()
|
||||
resp = rt.chat(prompt, max_tokens=cfg.max_tokens)
|
||||
return resp.surface or ""
|
||||
return AdapterResponse(surface=resp.surface or "")
|
||||
|
||||
return adapter
|
||||
|
||||
|
||||
def _build_openai_adapter(cfg: ProviderConfig) -> Callable[[str], str]:
|
||||
def _build_openai_adapter(cfg: ProviderConfig) -> Callable[[str], AdapterResponse]:
|
||||
"""OpenAI Chat Completions adapter.
|
||||
|
||||
Requires: ``pip install openai``
|
||||
|
|
@ -200,19 +218,28 @@ def _build_openai_adapter(cfg: ProviderConfig) -> Callable[[str], str]:
|
|||
client_kwargs["base_url"] = cfg.extra["base_url"]
|
||||
client = openai.OpenAI(**client_kwargs)
|
||||
|
||||
def adapter(prompt: str) -> str:
|
||||
def adapter(prompt: str) -> AdapterResponse:
|
||||
response = client.chat.completions.create(
|
||||
model=cfg.model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=cfg.temperature,
|
||||
max_tokens=cfg.max_tokens,
|
||||
)
|
||||
return (response.choices[0].message.content or "").strip()
|
||||
usage = getattr(response, "usage", None)
|
||||
input_tokens = int(getattr(usage, "prompt_tokens", 0)) if usage else None
|
||||
output_tokens = int(getattr(usage, "completion_tokens", 0)) if usage else None
|
||||
total_tokens = int(getattr(usage, "total_tokens", 0)) if usage else None
|
||||
return AdapterResponse(
|
||||
surface=(response.choices[0].message.content or "").strip(),
|
||||
input_tokens=input_tokens,
|
||||
output_tokens=output_tokens,
|
||||
total_tokens=total_tokens,
|
||||
)
|
||||
|
||||
return adapter
|
||||
|
||||
|
||||
def _build_anthropic_adapter(cfg: ProviderConfig) -> Callable[[str], str]:
|
||||
def _build_anthropic_adapter(cfg: ProviderConfig) -> Callable[[str], AdapterResponse]:
|
||||
"""Anthropic Messages adapter.
|
||||
|
||||
Requires: ``pip install anthropic``
|
||||
|
|
@ -229,7 +256,7 @@ def _build_anthropic_adapter(cfg: ProviderConfig) -> Callable[[str], str]:
|
|||
api_key = _require_env("ANTHROPIC_API_KEY")
|
||||
client = ant.Anthropic(api_key=api_key)
|
||||
|
||||
def adapter(prompt: str) -> str:
|
||||
def adapter(prompt: str) -> AdapterResponse:
|
||||
message = client.messages.create(
|
||||
model=cfg.model,
|
||||
max_tokens=cfg.max_tokens,
|
||||
|
|
@ -239,12 +266,25 @@ def _build_anthropic_adapter(cfg: ProviderConfig) -> Callable[[str], str]:
|
|||
temperature=max(0.0, min(1.0, cfg.temperature)),
|
||||
)
|
||||
block = message.content[0] if message.content else None
|
||||
return (getattr(block, "text", "") or "").strip()
|
||||
usage = getattr(message, "usage", None)
|
||||
input_tokens = int(getattr(usage, "input_tokens", 0)) if usage else None
|
||||
output_tokens = int(getattr(usage, "output_tokens", 0)) if usage else None
|
||||
total_tokens = (
|
||||
(input_tokens or 0) + (output_tokens or 0)
|
||||
if (input_tokens is not None or output_tokens is not None)
|
||||
else None
|
||||
)
|
||||
return AdapterResponse(
|
||||
surface=(getattr(block, "text", "") or "").strip(),
|
||||
input_tokens=input_tokens,
|
||||
output_tokens=output_tokens,
|
||||
total_tokens=total_tokens,
|
||||
)
|
||||
|
||||
return adapter
|
||||
|
||||
|
||||
def _build_ollama_adapter(cfg: ProviderConfig) -> Callable[[str], str]:
|
||||
def _build_ollama_adapter(cfg: ProviderConfig) -> Callable[[str], AdapterResponse]:
|
||||
"""Ollama local model adapter (HTTP /api/chat endpoint).
|
||||
|
||||
Requires: ``pip install httpx`` (already present in most Python envs)
|
||||
|
|
@ -267,7 +307,7 @@ def _build_ollama_adapter(cfg: ProviderConfig) -> Callable[[str], str]:
|
|||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
|
||||
def adapter(prompt: str) -> str:
|
||||
def adapter(prompt: str) -> AdapterResponse:
|
||||
payload = {
|
||||
"model": cfg.model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
|
|
@ -282,7 +322,19 @@ def _build_ollama_adapter(cfg: ProviderConfig) -> Callable[[str], str]:
|
|||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
return (data.get("message", {}).get("content") or "").strip()
|
||||
input_tokens = data.get("prompt_eval_count")
|
||||
output_tokens = data.get("eval_count")
|
||||
total_tokens = (
|
||||
int(input_tokens or 0) + int(output_tokens or 0)
|
||||
if input_tokens is not None or output_tokens is not None
|
||||
else None
|
||||
)
|
||||
return AdapterResponse(
|
||||
surface=(data.get("message", {}).get("content") or "").strip(),
|
||||
input_tokens=int(input_tokens) if input_tokens is not None else None,
|
||||
output_tokens=int(output_tokens) if output_tokens is not None else None,
|
||||
total_tokens=total_tokens,
|
||||
)
|
||||
|
||||
return adapter
|
||||
|
||||
|
|
@ -307,6 +359,16 @@ def build_adapter(cfg: ProviderConfig) -> Callable[[str], str]:
|
|||
it is safe to pass to ``compare_to_llm`` and the frontier_compare
|
||||
runner suites.
|
||||
"""
|
||||
observing = build_observing_adapter(cfg)
|
||||
|
||||
def _surface_only(prompt: str) -> str:
|
||||
return observing(prompt).surface
|
||||
|
||||
return _surface_only
|
||||
|
||||
|
||||
def build_observing_adapter(cfg: ProviderConfig) -> Callable[[str], AdapterResponse]:
|
||||
"""Return a provider adapter that includes usage telemetry when available."""
|
||||
builder = _BUILDERS.get(cfg.provider)
|
||||
if builder is None:
|
||||
raise ValueError(
|
||||
|
|
|
|||
|
|
@ -113,6 +113,28 @@ def test_cli_prompt_battery_with_core_provider(tmp_path: Path, capsys) -> None:
|
|||
assert report_path.exists()
|
||||
|
||||
|
||||
def test_cli_replay_variability_with_core_provider(tmp_path: Path, capsys) -> None:
|
||||
"""Replay-variability suite should run through cross-provider path."""
|
||||
report_path = tmp_path / "cross_core_replay.json"
|
||||
code = main(
|
||||
[
|
||||
"--provider", "core",
|
||||
"--suite", "replay_variability",
|
||||
"--repeats", "2",
|
||||
"--json",
|
||||
"--report", str(report_path),
|
||||
]
|
||||
)
|
||||
out = capsys.readouterr().out
|
||||
payload = json.loads(out)
|
||||
assert payload["model"] == "core-native"
|
||||
assert payload["mode"] == "core"
|
||||
assert len(payload["suites"]) == 1
|
||||
assert payload["suites"][0]["suite"] == "replay_variability"
|
||||
assert code == 0
|
||||
assert report_path.exists()
|
||||
|
||||
|
||||
def test_cli_rejects_core_only_suite_with_non_core_provider(capsys) -> None:
|
||||
"""Loud failure when an operator asks for a CORE-only suite with a
|
||||
non-CORE provider — no silent telemetry degradation."""
|
||||
|
|
|
|||
37
tests/test_frontier_model_registry_pricing.py
Normal file
37
tests/test_frontier_model_registry_pricing.py
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from evals.frontier_compare.cross_provider import run_prompt_battery
|
||||
from evals.frontier_compare.model_registry import require_model_card
|
||||
from evals.frontier_compare.providers import AdapterResponse, ProviderConfig
|
||||
|
||||
|
||||
def test_model_card_estimate_cost_formula() -> None:
|
||||
card = require_model_card("openai", "gpt-4o-2024-08-06")
|
||||
cost = card.estimate_cost_usd(input_tokens=2000, output_tokens=1000)
|
||||
assert cost is not None
|
||||
# (2000 / 1e6 * 2.5) + (1000 / 1e6 * 10.0) = 0.015
|
||||
assert round(cost, 6) == 0.015
|
||||
|
||||
|
||||
def test_prompt_battery_records_usage_and_cost_when_available() -> None:
|
||||
cfg = ProviderConfig(provider="openai", model="gpt-4o-2024-08-06")
|
||||
|
||||
def fake_adapter(_: str) -> AdapterResponse:
|
||||
return AdapterResponse(
|
||||
surface="ok",
|
||||
input_tokens=120,
|
||||
output_tokens=80,
|
||||
total_tokens=200,
|
||||
)
|
||||
|
||||
report = run_prompt_battery(
|
||||
fake_adapter,
|
||||
cfg=cfg,
|
||||
prompts=(("case", "What is truth?"),),
|
||||
)
|
||||
obs = report.cases[0].details["observation"]
|
||||
assert obs["input_tokens"] == 120
|
||||
assert obs["output_tokens"] == 80
|
||||
assert obs["total_tokens"] == 200
|
||||
assert obs["estimated_cost_usd"] is not None
|
||||
assert obs["estimated_cost_usd"] > 0
|
||||
Loading…
Reference in a new issue