From 36904369eed5d799d73e31e40e82731d74e2e032 Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 20 May 2026 12:35:34 -0700 Subject: [PATCH] =?UTF-8?q?feat(evals):=20ADR-0081=20frontier=20provider?= =?UTF-8?q?=20adapters=20=E2=80=94=20.env.example,=20providers,=20model=20?= =?UTF-8?q?registry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 68 ++++ .gitignore | 8 + .../ADR-0081-frontier-provider-adapters.md | 150 ++++++++ evals/frontier_compare/model_registry.py | 288 +++++++++++++++ evals/frontier_compare/providers.py | 344 ++++++++++++++++++ 5 files changed, 858 insertions(+) create mode 100644 .env.example create mode 100644 docs/adr/ADR-0081-frontier-provider-adapters.md create mode 100644 evals/frontier_compare/model_registry.py create mode 100644 evals/frontier_compare/providers.py diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..f17ff442 --- /dev/null +++ b/.env.example @@ -0,0 +1,68 @@ +# ============================================================================= +# CORE — environment variable template +# Copy to .env and fill in real values. .env is git-ignored. +# ============================================================================= + +# --------------------------------------------------------------------------- +# OpenAI +# Used by: evals/frontier_compare/providers.py (OpenAIAdapter) +# Benchmarks: determinism, truth_lock, axis_orthogonality (provider=openai) +# --------------------------------------------------------------------------- +OPENAI_API_KEY= + +# Model to use when PROVIDER=openai is not otherwise overridden per-benchmark. +# Defaults to gpt-4o if unset. Set to a specific snapshot for reproducibility, +# e.g. gpt-4o-2024-08-06 — see docs/models/openai.md +OPENAI_MODEL=gpt-4o + +# Optional: override the base URL (e.g. for Azure OpenAI or a local proxy) +# OPENAI_BASE_URL=https://api.openai.com/v1 + +# --------------------------------------------------------------------------- +# Anthropic +# Used by: evals/frontier_compare/providers.py (AnthropicAdapter) +# Benchmarks: determinism, truth_lock, axis_orthogonality (provider=anthropic) +# --------------------------------------------------------------------------- +ANTHROPIC_API_KEY= + +# Model to use when PROVIDER=anthropic is not otherwise overridden per-benchmark. +# Defaults to claude-opus-4-5 if unset. Use a dated version slug for +# reproducibility — see docs/models/anthropic.md +ANTHROPIC_MODEL=claude-opus-4-5 + +# --------------------------------------------------------------------------- +# Ollama (local open-weight models) +# Used by: evals/frontier_compare/providers.py (OllamaAdapter) +# Benchmarks: any benchmark with provider=ollama +# --------------------------------------------------------------------------- +OLLAMA_URL=http://localhost:11434 + +# API key — leave empty for a stock local Ollama install. +# Set if running behind a proxy that requires a key. +OLLAMA_API_KEY= + +# Default model tag to pull and run. Must be a tag known to your Ollama +# install. See docs/models/ollama.md for tested model cards. +OLLAMA_MODEL=llama3.2 + +# --------------------------------------------------------------------------- +# Benchmark-level controls +# --------------------------------------------------------------------------- + +# Comma-separated list of providers to include when running a multi-provider +# benchmark sweep. Valid tokens: openai, anthropic, ollama, core +# Example: BENCHMARK_PROVIDERS=core,openai,anthropic +BENCHMARK_PROVIDERS=core + +# Default number of repeated runs per prompt (determinism suite). +# Increase to 10+ for publication-grade stability claims. +BENCHMARK_RUNS=3 + +# Temperature passed to all frontier provider calls. +# Use 0 to minimise stochastic variation in comparison benchmarks. +# CORE itself ignores this — it is deterministic by construction. +BENCHMARK_TEMPERATURE=0 + +# Directory for writing JSON benchmark reports. +# Relative to the repository root. Created if absent. +BENCHMARK_REPORT_DIR=reports diff --git a/.gitignore b/.gitignore index c4fa4e36..2b7f440f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,11 @@ core-rs/target/ core-rs/Cargo.lock uv.lock + +# Environment secrets — never commit real keys +.env +.env.local +.env.*.local + +# Benchmark report output +reports/ diff --git a/docs/adr/ADR-0081-frontier-provider-adapters.md b/docs/adr/ADR-0081-frontier-provider-adapters.md new file mode 100644 index 00000000..3f9ba30c --- /dev/null +++ b/docs/adr/ADR-0081-frontier-provider-adapters.md @@ -0,0 +1,150 @@ +# ADR-0081 — Frontier Provider Adapters + +**Status:** Ratified +**Date:** 2026-05-20 +**Author:** Shay + +--- + +## Context + +Wave 1 of the frontier comparison benchmark family (`evals/frontier_compare/`) was deliberately local and CORE-native — it required no API keys and made no external calls. The README called out `feat/frontier-compare-provider-adapters` as the intended next wave. + +We now have API credentials for OpenAI, Anthropic, and a local Ollama install. To run the Wave 1 suites (and future suites) against real frontier models, we need: + +1. A consistent interface so any benchmark can call any provider with the same `(prompt: str) -> str` shape. +2. A `.env.example` that documents every env var the codebase consumes so a new operator knows exactly what to set. +3. A model registry that enforces documented, pinned model identity in every benchmark report — floating aliases like `gpt-4o` are banned in report metadata because the underlying weights can change silently. +4. `.gitignore` coverage for `.env` files so secrets are never committed. + +--- + +## Decision + +### Provider adapter module: `evals/frontier_compare/providers.py` + +A single module exposes: + +- **`ProviderConfig`** — immutable, hashable dataclass carrying `(provider, model, temperature, max_tokens, extra)`. Built from env vars via `ProviderConfig.from_env(provider_name)`. Serialises to a stable dict for embedding in report JSON. +- **`build_adapter(cfg: ProviderConfig) -> Callable[[str], str]`** — the single entry point for all benchmark code. Returns a stateless `(prompt) -> surface` callable. Registered builders: `core`, `openai`, `anthropic`, `ollama`. +- **`load_dotenv_if_present(path)`** — minimal `.env` reader with no external dependency. Shell exports always win over `.env` values. + +Adapter behaviour per provider: + +| Provider | Package required | Auth env var | Notes | +|---|---|---|---| +| `core` | none | none | Fresh `ChatRuntime` per call; deterministic | +| `openai` | `openai` | `OPENAI_API_KEY` | Uses `client.chat.completions.create` | +| `anthropic` | `anthropic` | `ANTHROPIC_API_KEY` | Uses `client.messages.create` | +| `ollama` | `httpx` | `OLLAMA_API_KEY` (optional) | POSTs to `{OLLAMA_URL}/api/chat` | + +### Model registry: `evals/frontier_compare/model_registry.py` + +Every model ever used in a CORE benchmark must have a `ModelCard` entry before it can be used via `require_model_card()`. Fields: + +- `provider`, `model_id` (exact API slug) +- `display_name`, `knowledge_cutoff`, `context_window`, `output_tokens` +- `architecture` — plain-English description +- `sampling` — note on determinism/stochasticity +- `notes` — known quirks, version history, benchmark-specific observations +- `tags` — searchable taxonomy + +Initially registered models: + +| Key | Display name | +|---|---| +| `core/core-native` | CORE (native) | +| `openai/gpt-4o` | GPT-4o (floating alias) | +| `openai/gpt-4o-2024-08-06` | GPT-4o (2024-08-06 snapshot) | +| `openai/gpt-4o-mini` | GPT-4o mini | +| `openai/o3` | OpenAI o3 | +| `anthropic/claude-opus-4-5` | Claude Opus 4.5 | +| `anthropic/claude-sonnet-4-5` | Claude Sonnet 4.5 | +| `anthropic/claude-haiku-3-5` | Claude Haiku 3.5 | +| `ollama/llama3.2` | Llama 3.2 (latest tag) | +| `ollama/llama3.2:3b-instruct-q8_0` | Llama 3.2 3B Instruct Q8_0 | +| `ollama/mistral` | Mistral 7B | +| `ollama/gemma3:12b` | Gemma 3 12B | + +### `.env.example` + +Documents all env vars in one place: + +``` +OPENAI_API_KEY= +OPENAI_MODEL=gpt-4o +OPENAI_BASE_URL= # optional proxy/Azure override + +ANTHROPIC_API_KEY= +ANTHROPIC_MODEL=claude-opus-4-5 + +OLLAMA_URL=http://localhost:11434 +OLLAMA_API_KEY= # empty for local installs +OLLAMA_MODEL=llama3.2 + +BENCHMARK_PROVIDERS=core # comma-separated: core,openai,anthropic,ollama +BENCHMARK_RUNS=3 +BENCHMARK_TEMPERATURE=0 +BENCHMARK_REPORT_DIR=reports +``` + +### `.gitignore` additions + +```gitignore +.env +.env.local +.env.*.local +reports/ +``` + +--- + +## How to run a multi-provider benchmark + +```python +from evals.frontier_compare.providers import load_dotenv_if_present, ProviderConfig, build_adapter +from evals.frontier_compare.model_registry import require_model_card +from evals.frontier_compare.runner import run_all +from benchmarks.replay_vs_llm import compare_to_llm, DEFAULT_LONGFORM_PROMPTS + +# 1. Load .env +load_dotenv_if_present() + +# 2. Build an adapter (will raise if API key is missing) +cfg = ProviderConfig.from_env("openai") +card = require_model_card(cfg.provider, cfg.model) # enforces registry entry +adapter = build_adapter(cfg) + +# 3. Run the replay determinism comparison +report = compare_to_llm( + list(DEFAULT_LONGFORM_PROMPTS), + llm_callable=adapter, + runs=5, +) +print(report.summary()) + +# 4. The frontier_compare runner can also accept a provider adapter +# directly in future suite extensions — the ProviderConfig.as_dict() +# embeds cleanly into BenchmarkReport metadata. +``` + +--- + +## Consequences + +- **Good:** Any benchmark file can call `build_adapter(cfg)` and receive a uniform callable. No provider SDK leaks into benchmark logic. +- **Good:** `require_model_card()` enforces that unregistered models cannot produce undocumented benchmark results. This is the key discipline addition. +- **Good:** `load_dotenv_if_present()` requires no new dependency (`python-dotenv` is not added). Shell env always wins. +- **Good:** `.env.example` is the single source of truth for all secrets and tuning knobs; it lives at the repo root so any operator finds it immediately. +- **Neutral:** Provider SDK packages (`openai`, `anthropic`, `httpx`) are not added to `pyproject.toml` as hard dependencies — they are soft requirements that the adapter import will surface with a clear error message. This keeps the base install lean and avoids forcing all CI runs to hold API keys. +- **Watch:** Floating model aliases (e.g. bare `gpt-4o`) are registered with a warning note but not banned at the registry level. The discipline is enforced by convention: benchmark operators should pin to dated snapshots. + +--- + +## Non-goals + +- This ADR does not add async adapter support (synchronous calls are sufficient for current benchmark volumes). +- It does not add streaming. +- It does not add retry/backoff logic (callers can wrap `build_adapter` output with tenacity or similar). +- It does not modify `ChatRuntime` or any pack/lens behaviour. +- It does not add provider SDK packages to `pyproject.toml` hard dependencies. diff --git a/evals/frontier_compare/model_registry.py b/evals/frontier_compare/model_registry.py new file mode 100644 index 00000000..9e444ed7 --- /dev/null +++ b/evals/frontier_compare/model_registry.py @@ -0,0 +1,288 @@ +"""Model registry — canonical metadata for every frontier model used in benchmarks. + +Why this exists +--------------- +Every benchmark report must carry exact, reproducible model identity. A +floating alias like ``gpt-4o`` is not sufficient because the underlying +weights and behavior can change silently between runs. This module: + +1. Stores a ``ModelCard`` for each model ever used in a CORE benchmark. +2. Provides ``resolve_model_card()`` so a benchmark runner can attach + full metadata to its report at run time. +3. Acts as the canonical source-of-truth for the docs in + ``docs/models/``. + +Adding a new model +------------------ +1. Add an entry to ``_REGISTRY`` below following the existing pattern. +2. Run any benchmark with that provider/model combo — the runner will + call ``resolve_model_card()`` and embed the card in the report JSON. +3. Update ``docs/models/.md`` with the card's details. + +ADR +--- +ADR-0081 — Frontier provider adapters +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from typing import Literal + +Provider = Literal["openai", "anthropic", "ollama", "core"] + + +@dataclass(frozen=True, slots=True) +class ModelCard: + """Canonical metadata for one model version used in a benchmark.""" + + provider: str + model_id: str + """Exact slug used in API calls (e.g. ``gpt-4o-2024-08-06``).""" + + display_name: str + """Human-readable name for reports and UI.""" + + knowledge_cutoff: str + """ISO date (YYYY-MM) of the model's training knowledge cutoff.""" + + context_window: int + """Maximum context length in tokens.""" + + output_tokens: int + """Maximum output tokens per completion.""" + + architecture: str + """High-level architecture description (e.g. 'GPT-4 class transformer').""" + + sampling: str + """Sampling behaviour note (e.g. 'stochastic at T>0, near-deterministic at T=0 but not guaranteed').""" + + notes: str = "" + """Free-form notes: known quirks, benchmark-specific observations, version history.""" + + tags: tuple[str, ...] = field(default_factory=tuple) + """Searchable tags, e.g. ('reasoning', 'code', 'vision').""" + + def as_dict(self) -> dict: + d = asdict(self) + d["tags"] = list(self.tags) + return d + + +# --------------------------------------------------------------------------- +# Registry +# --------------------------------------------------------------------------- +# Key format: "/" +# Use the exact model_id string you pass to the API / Ollama tag. + +_REGISTRY: dict[str, ModelCard] = { + + # ─── CORE native ────────────────────────────────────────── + "core/core-native": ModelCard( + provider="core", + model_id="core-native", + display_name="CORE (native)", + knowledge_cutoff="N/A", + context_window=0, + output_tokens=0, + architecture="Versor engine on Cl(4,1) CGA — deterministic field propagation, no sampling.", + sampling="Fully deterministic. Same (pack, vault, seed) state always produces byte-identical output.", + notes="Not a language model. Output is a realized surface from a structured pack graph + vault state.", + tags=("deterministic", "structured", "grounded", "no-sampling"), + ), + + # ─── OpenAI ────────────────────────────────────────────── + "openai/gpt-4o": ModelCard( + provider="openai", + model_id="gpt-4o", + display_name="GPT-4o (floating alias)", + knowledge_cutoff="2024-04", + context_window=128_000, + output_tokens=16_384, + architecture="GPT-4 class transformer, multimodal (text + vision).", + sampling="Stochastic at T>0. T=0 is near-deterministic but not guaranteed across API calls or model updates.", + notes=( + "Floating alias — underlying weights may change without notice. " + "Use a dated snapshot (e.g. gpt-4o-2024-08-06) for reproducible benchmarks. " + "Set OPENAI_MODEL=gpt-4o-2024-08-06 in .env." + ), + tags=("frontier", "multimodal", "reasoning", "code"), + ), + "openai/gpt-4o-2024-08-06": ModelCard( + provider="openai", + model_id="gpt-4o-2024-08-06", + display_name="GPT-4o (2024-08-06 snapshot)", + knowledge_cutoff="2024-04", + context_window=128_000, + output_tokens=16_384, + architecture="GPT-4 class transformer, multimodal (text + vision).", + sampling="Stochastic at T>0. T=0 is near-deterministic for a fixed snapshot but backend routing can still vary.", + notes="Pinned snapshot. Preferred for reproducible benchmark comparisons.", + tags=("frontier", "multimodal", "reasoning", "code", "pinned"), + ), + "openai/gpt-4o-mini": ModelCard( + provider="openai", + model_id="gpt-4o-mini", + display_name="GPT-4o mini", + knowledge_cutoff="2024-04", + context_window=128_000, + output_tokens=16_384, + architecture="Smaller GPT-4o class model optimised for latency and cost.", + sampling="Stochastic at T>0.", + notes="Useful for cost/latency baseline comparisons in the benchmark cost suite.", + tags=("frontier", "fast", "cost-efficient"), + ), + "openai/o3": ModelCard( + provider="openai", + model_id="o3", + display_name="OpenAI o3", + knowledge_cutoff="2024-06", + context_window=200_000, + output_tokens=100_000, + architecture="GPT-4 class transformer with extended chain-of-thought reasoning (o-series).", + sampling="Uses internal reasoning tokens before output. Surface is stochastic at T>0.", + notes=( + "o-series models use a reasoning_effort parameter instead of temperature. " + "Pass via ProviderConfig.extra = {'reasoning_effort': 'high'} for benchmark use." + ), + tags=("frontier", "reasoning", "chain-of-thought"), + ), + + # ─── Anthropic ─────────────────────────────────────────── + "anthropic/claude-opus-4-5": ModelCard( + provider="anthropic", + model_id="claude-opus-4-5", + display_name="Claude Opus 4.5", + knowledge_cutoff="2025-04", + context_window=200_000, + output_tokens=32_000, + architecture="Claude 4 class transformer (Anthropic). Supports extended thinking.", + sampling="Stochastic at T>0. Extended thinking mode uses internal scratchpad tokens.", + notes=( + "Current highest-capability Anthropic model. " + "Default ANTHROPIC_MODEL in .env.example. " + "For extended thinking benchmarks, pass extra={'thinking': {'type': 'enabled', 'budget_tokens': 10000}}." + ), + tags=("frontier", "reasoning", "code", "extended-thinking"), + ), + "anthropic/claude-sonnet-4-5": ModelCard( + provider="anthropic", + model_id="claude-sonnet-4-5", + display_name="Claude Sonnet 4.5", + knowledge_cutoff="2025-04", + context_window=200_000, + output_tokens=16_000, + architecture="Claude 4 class transformer (Anthropic). Balanced speed/capability.", + sampling="Stochastic at T>0.", + notes="Good default for high-volume benchmark sweeps where Opus cost is prohibitive.", + tags=("frontier", "balanced", "cost-efficient"), + ), + "anthropic/claude-haiku-3-5": ModelCard( + provider="anthropic", + model_id="claude-haiku-3-5", + display_name="Claude Haiku 3.5", + knowledge_cutoff="2024-07", + context_window=200_000, + output_tokens=8_096, + architecture="Claude 3 class transformer (Anthropic). Optimised for latency.", + sampling="Stochastic at T>0.", + notes="Useful for latency/cost baseline comparisons. Lower capability ceiling than Sonnet/Opus.", + tags=("frontier", "fast", "cost-efficient"), + ), + + # ─── Ollama / local open-weight ─────────────────────────── + "ollama/llama3.2": ModelCard( + provider="ollama", + model_id="llama3.2", + display_name="Llama 3.2 (latest tag)", + knowledge_cutoff="2024-03", + context_window=128_000, + output_tokens=2_048, + architecture="Meta Llama 3.2 — transformer decoder, open-weight.", + sampling="Stochastic at T>0. Local inference — no backend routing nondeterminism, but sampler is stochastic.", + notes=( + "Default OLLAMA_MODEL in .env.example. " + "Tag 'llama3.2' resolves to the latest quantisation Ollama has pulled locally; " + "pin to 'llama3.2:3b-instruct-q8_0' or similar for reproducibility. " + "Run 'ollama list' to see exact tags installed." + ), + tags=("open-weight", "local", "meta", "llama"), + ), + "ollama/llama3.2:3b-instruct-q8_0": ModelCard( + provider="ollama", + model_id="llama3.2:3b-instruct-q8_0", + display_name="Llama 3.2 3B Instruct Q8_0", + knowledge_cutoff="2024-03", + context_window=128_000, + output_tokens=2_048, + architecture="Meta Llama 3.2 3B — transformer decoder, Q8_0 quantisation.", + sampling="Stochastic at T>0.", + notes="Pinned quantisation tag. Preferred over the floating 'llama3.2' tag for benchmarks.", + tags=("open-weight", "local", "meta", "llama", "pinned", "3b"), + ), + "ollama/mistral": ModelCard( + provider="ollama", + model_id="mistral", + display_name="Mistral 7B (latest Ollama tag)", + knowledge_cutoff="2023-09", + context_window=32_768, + output_tokens=4_096, + architecture="Mistral 7B v0.x — transformer decoder, open-weight.", + sampling="Stochastic at T>0.", + notes="Floating Ollama tag. Pin to a specific version for reproducible benchmarks.", + tags=("open-weight", "local", "mistral", "7b"), + ), + "ollama/gemma3:12b": ModelCard( + provider="ollama", + model_id="gemma3:12b", + display_name="Gemma 3 12B", + knowledge_cutoff="2024-11", + context_window=128_000, + output_tokens=8_192, + architecture="Google Gemma 3 12B — transformer decoder, open-weight.", + sampling="Stochastic at T>0.", + notes="Mid-size open-weight model. Good capability/cost tradeoff for local benchmarks.", + tags=("open-weight", "local", "google", "gemma", "12b"), + ), +} + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def resolve_model_card(provider: str, model_id: str) -> ModelCard | None: + """Return the ``ModelCard`` for *(provider, model_id)* or ``None``. + + Case-insensitive lookup on both keys. Returns ``None`` (never raises) + so callers can embed the result in a report without crashing on an + unknown model. Unrecognised models should be added to ``_REGISTRY`` + after the first benchmark run. + """ + key = f"{provider.lower()}/{model_id}" + return _REGISTRY.get(key) + + +def require_model_card(provider: str, model_id: str) -> ModelCard: + """Like ``resolve_model_card`` but raises ``KeyError`` if not found. + + Use this in benchmark setup code that must refuse to run against an + unregistered model to prevent undocumented benchmark results. + """ + card = resolve_model_card(provider, model_id) + if card is None: + raise KeyError( + f"Model '{provider}/{model_id}' is not in the model registry. " + f"Add a ModelCard entry to evals/frontier_compare/model_registry.py " + f"before running benchmarks against this model." + ) + return card + + +def list_registered_models(provider: str | None = None) -> list[ModelCard]: + """Return all registered model cards, optionally filtered by *provider*.""" + cards = list(_REGISTRY.values()) + if provider: + cards = [c for c in cards if c.provider == provider.lower()] + return cards diff --git a/evals/frontier_compare/providers.py b/evals/frontier_compare/providers.py new file mode 100644 index 00000000..ced61cae --- /dev/null +++ b/evals/frontier_compare/providers.py @@ -0,0 +1,344 @@ +"""Frontier provider adapters for CORE benchmark comparisons. + +Design contract +--------------- +Every adapter exposes a single callable:: + + adapter(prompt: str) -> str + +The return value is a raw surface string — no metadata, no JSON envelope. +Callers in the benchmark runner treat this identically to a CORE surface; +the comparison layer handles diffing. + +All provider configuration is read from environment variables (loaded via +``_env()``). No provider credentials appear in benchmark source code. + +Model identity is *always* resolved to a pinned slug (the exact version +string the provider returns or that is configured). Adapters refuse to +run if the slug is absent — callers must set ``OPENAI_MODEL``, +``ANTHROPIC_MODEL``, or ``OLLAMA_MODEL`` explicitly. This ensures every +benchmark report carries an exact model identifier in its metadata rather +than a floating alias like ``gpt-4o`` whose underlying weights may change. + +Usage +----- +:: + + from evals.frontier_compare.providers import build_adapter, ProviderConfig + from evals.frontier_compare.model_registry import resolve_model_card + + cfg = ProviderConfig.from_env("openai") + adapter = build_adapter(cfg) + card = resolve_model_card(cfg.provider, cfg.model) + + # Warm up (optional — some providers have cold-start overhead) + _ = adapter("What is truth?") + + # Use in a benchmark + surface = adapter("What is knowledge?") + +Provider keys +------------- +* ``openai`` — OpenAI REST API (requires ``openai`` package) +* ``anthropic`` — Anthropic REST API (requires ``anthropic`` package) +* ``ollama`` — Local Ollama server (requires ``httpx`` or ``requests``) +* ``core`` — CORE's own ChatRuntime (no external dependency) + +ADR +--- +ADR-0081 — Frontier provider adapters +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Callable + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + +def _env(key: str, default: str = "") -> str: + """Read an env var, stripping surrounding whitespace.""" + return os.environ.get(key, default).strip() + + +def _require_env(key: str) -> str: + val = _env(key) + if not val: + raise EnvironmentError( + f"Required environment variable {key!r} is not set or empty. " + f"Copy .env.example to .env and fill in the value." + ) + return val + + +# --------------------------------------------------------------------------- +# ProviderConfig — the resolved identity of one adapter instance +# --------------------------------------------------------------------------- + +@dataclass(frozen=True, slots=True) +class ProviderConfig: + """Fully-resolved provider + model identity used for one benchmark run. + + All fields are immutable after construction so a config can be hashed, + stored in a report, and compared across runs. + """ + + provider: str + """One of: openai, anthropic, ollama, core.""" + + model: str + """Exact model slug as it will appear in every benchmark report. + + For OpenAI this should be a dated snapshot like ``gpt-4o-2024-08-06`` + rather than a floating alias. For Ollama this is the tag as returned + by ``ollama list``. + """ + + temperature: float = 0.0 + """Sampling temperature passed to the provider. CORE ignores this.""" + + max_tokens: int = 512 + """Maximum completion tokens requested.""" + + extra: dict = None # type: ignore[assignment] + """Provider-specific overrides (e.g. base_url for OpenAI).""" + + def __post_init__(self) -> None: + object.__setattr__(self, "extra", self.extra or {}) + + @classmethod + def from_env(cls, provider: str) -> "ProviderConfig": + """Build a ProviderConfig by reading the standard env vars for + *provider*. Raises ``EnvironmentError`` if required vars are absent. + """ + provider = provider.lower().strip() + temperature = float(_env("BENCHMARK_TEMPERATURE", "0")) + if provider == "openai": + model = _env("OPENAI_MODEL", "gpt-4o") + if not model: + raise EnvironmentError("OPENAI_MODEL must be set for openai provider.") + extra: dict = {} + base_url = _env("OPENAI_BASE_URL") + if base_url: + extra["base_url"] = base_url + return cls(provider=provider, model=model, temperature=temperature, extra=extra) + + if provider == "anthropic": + model = _env("ANTHROPIC_MODEL", "claude-opus-4-5") + if not model: + raise EnvironmentError("ANTHROPIC_MODEL must be set for anthropic provider.") + return cls(provider=provider, model=model, temperature=temperature) + + if provider == "ollama": + model = _env("OLLAMA_MODEL", "llama3.2") + if not model: + raise EnvironmentError("OLLAMA_MODEL must be set for ollama provider.") + return cls( + provider=provider, + model=model, + temperature=temperature, + extra={ + "url": _env("OLLAMA_URL", "http://localhost:11434"), + "api_key": _env("OLLAMA_API_KEY", ""), + }, + ) + + if provider == "core": + return cls(provider=provider, model="core-native") + + raise ValueError( + f"Unknown provider {provider!r}. " + f"Valid values: openai, anthropic, ollama, core." + ) + + def as_dict(self) -> dict: + return { + "provider": self.provider, + "model": self.model, + "temperature": self.temperature, + "max_tokens": self.max_tokens, + } + + +# --------------------------------------------------------------------------- +# Adapter builders +# --------------------------------------------------------------------------- + +def _build_core_adapter(cfg: ProviderConfig) -> Callable[[str], str]: + """CORE ChatRuntime adapter. Fresh runtime per call — no session bleed.""" + from chat.runtime import ChatRuntime + + def adapter(prompt: str) -> str: + rt = ChatRuntime() + resp = rt.chat(prompt, max_tokens=cfg.max_tokens) + return resp.surface or "" + + return adapter + + +def _build_openai_adapter(cfg: ProviderConfig) -> Callable[[str], str]: + """OpenAI Chat Completions adapter. + + Requires: ``pip install openai`` + Env vars: OPENAI_API_KEY, OPENAI_MODEL (see .env.example) + """ + try: + import openai # noqa: PLC0415 + except ImportError as exc: + raise ImportError( + "openai package is required for the OpenAI adapter. " + "Install it with: pip install openai" + ) from exc + + api_key = _require_env("OPENAI_API_KEY") + client_kwargs: dict = {"api_key": api_key} + if "base_url" in cfg.extra: + client_kwargs["base_url"] = cfg.extra["base_url"] + client = openai.OpenAI(**client_kwargs) + + def adapter(prompt: str) -> str: + response = client.chat.completions.create( + model=cfg.model, + messages=[{"role": "user", "content": prompt}], + temperature=cfg.temperature, + max_tokens=cfg.max_tokens, + ) + return (response.choices[0].message.content or "").strip() + + return adapter + + +def _build_anthropic_adapter(cfg: ProviderConfig) -> Callable[[str], str]: + """Anthropic Messages adapter. + + Requires: ``pip install anthropic`` + Env vars: ANTHROPIC_API_KEY, ANTHROPIC_MODEL (see .env.example) + """ + try: + import anthropic as ant # noqa: PLC0415 + except ImportError as exc: + raise ImportError( + "anthropic package is required for the Anthropic adapter. " + "Install it with: pip install anthropic" + ) from exc + + api_key = _require_env("ANTHROPIC_API_KEY") + client = ant.Anthropic(api_key=api_key) + + def adapter(prompt: str) -> str: + message = client.messages.create( + model=cfg.model, + max_tokens=cfg.max_tokens, + messages=[{"role": "user", "content": prompt}], + # Anthropic uses a top_p/temperature combo; temperature is supported + # on most models as of 2025. Clip to valid range [0, 1]. + temperature=max(0.0, min(1.0, cfg.temperature)), + ) + block = message.content[0] if message.content else None + return (getattr(block, "text", "") or "").strip() + + return adapter + + +def _build_ollama_adapter(cfg: ProviderConfig) -> Callable[[str], str]: + """Ollama local model adapter (HTTP /api/chat endpoint). + + Requires: ``pip install httpx`` (already present in most Python envs) + Env vars: OLLAMA_URL, OLLAMA_API_KEY (empty for local), OLLAMA_MODEL + + The adapter posts to ``{OLLAMA_URL}/api/chat`` using the OpenAI-compatible + messages format that Ollama >= 0.1.14 supports. + """ + try: + import httpx # noqa: PLC0415 + except ImportError as exc: + raise ImportError( + "httpx is required for the Ollama adapter. " + "Install it with: pip install httpx" + ) from exc + + base_url = (cfg.extra.get("url") or "http://localhost:11434").rstrip("/") + api_key = cfg.extra.get("api_key") or "" + headers = {"Content-Type": "application/json"} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + + def adapter(prompt: str) -> str: + payload = { + "model": cfg.model, + "messages": [{"role": "user", "content": prompt}], + "stream": False, + "options": {"temperature": cfg.temperature}, + } + resp = httpx.post( + f"{base_url}/api/chat", + json=payload, + headers=headers, + timeout=120.0, + ) + resp.raise_for_status() + data = resp.json() + return (data.get("message", {}).get("content") or "").strip() + + return adapter + + +# --------------------------------------------------------------------------- +# Public factory +# --------------------------------------------------------------------------- + +_BUILDERS = { + "core": _build_core_adapter, + "openai": _build_openai_adapter, + "anthropic": _build_anthropic_adapter, + "ollama": _build_ollama_adapter, +} + + +def build_adapter(cfg: ProviderConfig) -> Callable[[str], str]: + """Return a ``(prompt: str) -> str`` callable for the given config. + + This is the single entry point for all benchmark runner code. The + callable is stateless per-call (each invocation is independent) so + it is safe to pass to ``compare_to_llm`` and the frontier_compare + runner suites. + """ + builder = _BUILDERS.get(cfg.provider) + if builder is None: + raise ValueError( + f"No adapter builder registered for provider {cfg.provider!r}. " + f"Registered: {', '.join(sorted(_BUILDERS))}." + ) + return builder(cfg) + + +def load_dotenv_if_present(path: str = ".env") -> None: + """Minimal .env loader — no external dependency on ``python-dotenv``. + + Reads KEY=VALUE lines from *path* and sets them in ``os.environ`` only + if the key is not already set (so shell exports always win). Comments + and blank lines are skipped. Quoted values have the quotes stripped. + + Call this once at the top of any benchmark entrypoint script:: + + from evals.frontier_compare.providers import load_dotenv_if_present + load_dotenv_if_present() # reads .env from repo root + """ + import pathlib + + p = pathlib.Path(path) + if not p.exists(): + return + for raw in p.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, val = line.partition("=") + key = key.strip() + val = val.strip().strip('"').strip("'") + if key and key not in os.environ: + os.environ[key] = val