diff --git a/.gitignore b/.gitignore index 2b7f440f..224e8808 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ uv.lock # Benchmark report output reports/ +frontier_wave1.json diff --git a/evals/frontier_compare/__main__.py b/evals/frontier_compare/__main__.py index 02326e91..631264ab 100644 --- a/evals/frontier_compare/__main__.py +++ b/evals/frontier_compare/__main__.py @@ -5,40 +5,165 @@ import json import sys from pathlib import Path -from .runner import format_human_report, run_all, run_suite, write_report +from .cross_provider import run_prompt_battery +from .model_registry import require_model_card +from .providers import ProviderConfig, build_adapter, load_dotenv_if_present +from .runner import ( + BenchmarkReport, + format_human_report, + run_all, + run_suite, + write_report, +) + + +_CORE_ONLY_SUITES = ("determinism", "truth_lock", "axis_orthogonality", "all") +_CROSS_PROVIDER_SUITES = ("prompt_battery",) +_VALID_SUITES = _CORE_ONLY_SUITES + _CROSS_PROVIDER_SUITES def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="python -m evals.frontier_compare", - description="Run CORE's Wave-1 frontier comparison benchmark suites.", + description=( + "Run frontier_compare benchmark suites. Default --provider=core " + "runs the existing CORE-native suites; specify a non-CORE " + "--provider to run the cross-provider prompt_battery." + ), ) parser.add_argument( "--suite", - choices=("determinism", "truth_lock", "axis_orthogonality", "all"), + choices=_VALID_SUITES, default="all", - help="Benchmark suite to run. Defaults to all.", + help=( + "benchmark suite to run. CORE-only suites: " + f"{', '.join(_CORE_ONLY_SUITES)}. Cross-provider suites: " + f"{', '.join(_CROSS_PROVIDER_SUITES)}. Default 'all' runs every " + "CORE-only suite when --provider=core, otherwise prompt_battery." + ), + ) + parser.add_argument( + "--provider", + choices=("core", "openai", "anthropic", "ollama"), + default="core", + help=( + "provider to benchmark. 'core' (default) runs the existing " + "CORE-native suites with full telemetry. Other providers run " + "the cross-provider prompt_battery only — CORE-only suites " + "(determinism/truth_lock/axis_orthogonality) are rejected with " + "a clear error rather than silently producing degraded telemetry." + ), + ) + parser.add_argument( + "--model", + type=str, + default=None, + help=( + "explicit model id override (e.g. gpt-4o-2024-08-06). When " + "omitted, uses the provider's env-var default. Validated " + "against the model_registry — floating aliases are rejected." + ), ) parser.add_argument( "--json", action="store_true", - help="Emit the stable machine-readable JSON report.", + help="emit the stable machine-readable JSON report on stdout.", ) parser.add_argument( "--report", type=Path, default=None, - help="Optional path to write the JSON report.", + help=( + "optional path to write the JSON report. When --provider is " + "non-core and --report is not set, the report is auto-written " + "to evals/frontier_compare/results/__.json " + "so cross-provider runs always leave a durable artifact." + ), + ) + parser.add_argument( + "--env-file", + type=Path, + default=Path(".env"), + help="path to a .env file with provider credentials (default: ./.env).", ) return parser +def _build_cfg(args: argparse.Namespace) -> ProviderConfig: + """Build a validated ProviderConfig from --provider / --model flags.""" + load_dotenv_if_present(str(args.env_file)) + cfg = ProviderConfig.from_env(args.provider) + if args.model is not None: + # Operator-supplied model override — re-resolve through the registry. + cfg = ProviderConfig( + provider=cfg.provider, + model=args.model, + temperature=cfg.temperature, + max_tokens=cfg.max_tokens, + extra=cfg.extra, + ) + # require_model_card raises on floating aliases / unregistered models. + require_model_card(cfg.provider, cfg.model) + return cfg + + +def _auto_report_path(cfg: ProviderConfig) -> Path: + """Deterministic results-dir path when operator didn't supply --report.""" + from datetime import datetime, timezone + + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + safe_model = cfg.model.replace("/", "_").replace(":", "_") + return Path("evals/frontier_compare/results") / ( + f"{cfg.provider}_{safe_model}_{stamp}.json" + ) + + def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) - report = run_all() if args.suite == "all" else run_suite(args.suite) + + # Decide which lane this run belongs to. Two orthogonal axes: + # - cross-provider suites (prompt_battery) ALWAYS go through the + # provider-adapter path, even when --provider=core (CORE is just + # one adapter among many for those suites). + # - CORE-only suites (determinism/truth_lock/axis_orthogonality) + # require --provider=core and pull CORE-internal telemetry that + # no other adapter can supply. + cross_provider = args.suite in _CROSS_PROVIDER_SUITES or ( + args.suite == "all" and args.provider != "core" + ) + + if not cross_provider: + # Existing CORE-native path. Reject the obvious operator error + # of asking for a CORE-only suite with a non-CORE provider. + if args.provider != "core": + print( + f"error: suite {args.suite!r} is CORE-only; pass " + f"--suite prompt_battery (the cross-provider suite) when " + f"--provider={args.provider!r}.", + file=sys.stderr, + ) + return 2 + report = run_all() if args.suite == "all" else run_suite(args.suite) + else: + # Cross-provider path — runs over the provider adapter. + cfg = _build_cfg(args) + adapter = build_adapter(cfg) + suite_report = run_prompt_battery(adapter, cfg=cfg) + report = BenchmarkReport( + benchmark_family="frontier_compare_wave1", + model=cfg.model, + mode=cfg.provider, + suites=(suite_report,), + ) + # Always persist non-CORE runs — they're rate-limited / paid, so + # losing the artifact is genuinely costly. CORE adapter runs of + # prompt_battery only persist when --report is explicit. + if args.report is None and cfg.provider != "core": + args.report = _auto_report_path(cfg) if args.report is not None: write_report(report, args.report) + print(f"report written: {args.report}", file=sys.stderr) if args.json: print(json.dumps(report.as_dict(), ensure_ascii=False, indent=2, sort_keys=True)) diff --git a/evals/frontier_compare/cross_provider.py b/evals/frontier_compare/cross_provider.py new file mode 100644 index 00000000..60fc4567 --- /dev/null +++ b/evals/frontier_compare/cross_provider.py @@ -0,0 +1,181 @@ +"""Cross-provider benchmark suite for frontier_compare. + +The existing suites (``determinism``, ``truth_lock``, +``axis_orthogonality``) pull CORE-only telemetry (``trace_hash``, +``versor_condition``, ``register_id``, ``anchor_lens_id``) — they +cannot run cross-provider as-is. + +This module is the cross-provider lane: + + - Pure ``(prompt) -> str`` over any provider adapter. + - No CORE-internal telemetry expected. + - Per-case ``passed`` is loose by design — non-empty surface within + the elapsed-ms budget — because we are not in a position to + semantically judge GPT-4o vs Claude vs CORE here. The point of + the suite is **operator-visible, side-by-side surface evidence** + across providers on a fixed prompt battery, not automatic + quality scoring. + +The prompt battery (``_PROMPT_BATTERY``) is the load-bearing data — +edit it to expand cross-provider coverage. Each entry pairs a +``case_id`` (stable across runs, used by reviewers to diff results) +with the prompt itself. + +Trust boundary +-------------- +- Read-only. Never writes packs, vault, or runtime state. +- Provider adapters are constructed via + ``evals.frontier_compare.providers.build_adapter`` and made of a + single ``(prompt) -> str`` callable each. +- Per-call exceptions are recorded as case failures, never propagated. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass +from typing import Callable + +from .providers import ProviderConfig +from .runner import CaseResult, SuiteReport + + +# --------------------------------------------------------------------------- +# Prompt battery +# --------------------------------------------------------------------------- +# Stable case_ids so a future re-run on the same provider produces +# diffable JSON. Prompts span definitional / causal / verification / +# comparison / procedural / unknown shapes — enough to surface obvious +# provider behavior differences without ballooning credit cost. + +_PROMPT_BATTERY: tuple[tuple[str, str], ...] = ( + ("definition_truth", "What is truth?"), + ("definition_knowledge", "What is knowledge?"), + ("cause_understanding", "What causes understanding?"), + ("verification_evidence", "Does evidence ground knowledge?"), + ("comparison_knowledge_wisdom", "Compare knowledge and wisdom."), + ("procedure_recall", "Walk me through recall."), + ("unknown_term", "What is xylomorphic?"), +) + + +# --------------------------------------------------------------------------- +# Observation shape — provider-agnostic +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class ProviderObservation: + """One ``(prompt, provider) -> surface`` observation. + + Cross-provider sibling of :class:`runner.RuntimeObservation`. + Carries only fields any provider can supply; CORE-only telemetry + deliberately omitted. + """ + + prompt: str + surface: str + provider: str + model: str + elapsed_ms: float + error_type: str = "" + error_message: str = "" + + @property + def succeeded(self) -> bool: + return not self.error_type and bool(self.surface.strip()) + + def as_dict(self) -> dict: + return { + "prompt": self.prompt, + "surface": self.surface, + "provider": self.provider, + "model": self.model, + "elapsed_ms": self.elapsed_ms, + "error_type": self.error_type, + "error_message": self.error_message, + } + + +# --------------------------------------------------------------------------- +# Suite runner +# --------------------------------------------------------------------------- + + +def _observe_one( + adapter: Callable[[str], str], + cfg: ProviderConfig, + prompt: str, +) -> ProviderObservation: + start = time.perf_counter() + try: + surface = adapter(prompt) + except Exception as exc: # noqa: BLE001 - record failure, never abort the suite + return ProviderObservation( + prompt=prompt, + surface="", + provider=cfg.provider, + model=cfg.model, + elapsed_ms=(time.perf_counter() - start) * 1000.0, + error_type=exc.__class__.__name__, + error_message=str(exc), + ) + elapsed_ms = (time.perf_counter() - start) * 1000.0 + return ProviderObservation( + prompt=prompt, + surface=surface, + provider=cfg.provider, + model=cfg.model, + elapsed_ms=elapsed_ms, + ) + + +def run_prompt_battery( + adapter: Callable[[str], str], + *, + cfg: ProviderConfig, + prompts: tuple[tuple[str, str], ...] = _PROMPT_BATTERY, +) -> SuiteReport: + """Run the cross-provider prompt battery against one adapter. + + Per-case ``passed`` is loose by design (non-empty surface, no + exception). Reviewers should diff ``details.observation.surface`` + side-by-side rather than rely on the boolean. + """ + cases: list[CaseResult] = [] + for case_id, prompt in prompts: + obs = _observe_one(adapter, cfg, prompt) + passed = obs.succeeded + score = 1.0 if passed else 0.0 + failures: tuple[str, ...] = () + if not passed: + failures = ( + ("adapter_error",) if obs.error_type else ("empty_surface",) + ) + cases.append( + CaseResult( + suite="prompt_battery", + case_id=case_id, + prompt=prompt, + passed=passed, + score=score, + elapsed_ms=obs.elapsed_ms, + details={"observation": obs.as_dict()}, + failures=failures, + ) + ) + primary = ( + sum(c.score for c in cases) / len(cases) if cases else 0.0 + ) + return SuiteReport( + suite="prompt_battery", + cases=tuple(cases), + primary_score=primary, + passed=all(c.passed for c in cases), + ) + + +__all__ = [ + "ProviderObservation", + "run_prompt_battery", +] diff --git a/evals/frontier_compare/results/.gitkeep b/evals/frontier_compare/results/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_frontier_compare_cross_provider.py b/tests/test_frontier_compare_cross_provider.py new file mode 100644 index 00000000..ccbf3753 --- /dev/null +++ b/tests/test_frontier_compare_cross_provider.py @@ -0,0 +1,180 @@ +"""Tests for the frontier_compare cross-provider lane (ADR-0082 wiring). + +Pins the integration between providers.py + model_registry.py + +runner.py that was unwired when PR #58 landed. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from evals.frontier_compare.__main__ import main +from evals.frontier_compare.cross_provider import ( + ProviderObservation, + run_prompt_battery, +) +from evals.frontier_compare.providers import ProviderConfig, build_adapter + + +# --------------------------------------------------------------------------- +# Cross-provider suite — CORE adapter only (no API needed) +# --------------------------------------------------------------------------- + + +def test_prompt_battery_runs_with_core_adapter() -> None: + """The cross-provider suite must work with CORE as one provider among + many — that's the whole point of routing CORE through the same + adapter abstraction as OpenAI/Anthropic/Ollama.""" + cfg = ProviderConfig(provider="core", model="core-native") + adapter = build_adapter(cfg) + report = run_prompt_battery(adapter, cfg=cfg) + + assert report.suite == "prompt_battery" + assert report.cases, "prompt battery must emit at least one case" + assert report.passed is True + # Every case carries a ProviderObservation in details. + for case in report.cases: + obs = case.details.get("observation") + assert obs is not None + assert obs["provider"] == "core" + assert obs["model"] == "core-native" + assert obs["surface"], "CORE adapter should produce non-empty surface" + + +def test_provider_observation_records_failures_not_exceptions() -> None: + """Adapter exceptions must be recorded as failed observations, never + propagated — a single provider hiccup must not abort the suite.""" + cfg = ProviderConfig(provider="core", model="core-native") + + def broken_adapter(prompt: str) -> str: + raise RuntimeError("simulated provider failure") + + report = run_prompt_battery(broken_adapter, cfg=cfg) + assert not report.passed + for case in report.cases: + assert not case.passed + assert case.failures == ("adapter_error",) + obs = case.details["observation"] + assert obs["error_type"] == "RuntimeError" + assert "simulated" in obs["error_message"] + + +def test_provider_observation_records_empty_surface() -> None: + """An adapter that returns an empty string must be flagged as + 'empty_surface', distinct from an adapter that throws.""" + cfg = ProviderConfig(provider="core", model="core-native") + report = run_prompt_battery(lambda p: "", cfg=cfg) + assert not report.passed + for case in report.cases: + assert case.failures == ("empty_surface",) + + +# --------------------------------------------------------------------------- +# CLI routing — load-bearing dispatch logic +# --------------------------------------------------------------------------- + + +def test_cli_default_runs_core_native_path(tmp_path: Path, capsys) -> None: + """`--suite determinism` (CORE-only) with default provider runs the + legacy CORE-native path — no breaking change for existing operators.""" + report_path = tmp_path / "core_run.json" + code = main(["--suite", "determinism", "--json", "--report", str(report_path)]) + out = capsys.readouterr().out + payload = json.loads(out) + # Existing SuiteReport shape: top-level 'suite' key, not 'suites'. + assert payload.get("suite") == "determinism" + assert code == 0 or code == 1 # determinism may pass or fail; both valid + assert report_path.exists() + + +def test_cli_prompt_battery_with_core_provider(tmp_path: Path, capsys) -> None: + """--provider core --suite prompt_battery must route through the + cross-provider adapter path even though provider is CORE.""" + report_path = tmp_path / "cross_core.json" + code = main( + [ + "--provider", "core", + "--suite", "prompt_battery", + "--json", + "--report", str(report_path), + ] + ) + out = capsys.readouterr().out + payload = json.loads(out) + # Cross-provider runs always produce a BenchmarkReport with model + mode. + assert payload["model"] == "core-native" + assert payload["mode"] == "core" + assert len(payload["suites"]) == 1 + assert payload["suites"][0]["suite"] == "prompt_battery" + assert code == 0 # CORE adapter on a fixed battery should pass + assert report_path.exists() + + +def test_cli_rejects_core_only_suite_with_non_core_provider(capsys) -> None: + """Loud failure when an operator asks for a CORE-only suite with a + non-CORE provider — no silent telemetry degradation.""" + code = main(["--provider", "openai", "--suite", "determinism"]) + err = capsys.readouterr().err + assert code == 2 + assert "CORE-only" in err + assert "prompt_battery" in err # tells operator the right alternative + + +def test_cli_help_lists_both_suite_families() -> None: + """--help must surface both CORE-only and cross-provider suites so + operators discover the cross-provider lane without reading source.""" + with pytest.raises(SystemExit): + main(["--help"]) + + +# --------------------------------------------------------------------------- +# Model registry validation — the guardrail against silent drift +# --------------------------------------------------------------------------- + + +def test_unregistered_model_is_rejected(monkeypatch, tmp_path: Path) -> None: + """ADR-0082's load-bearing guarantee: floating or unregistered models + are rejected before any benchmark cycles burn. Operators cannot + silently report results against an alias the registry doesn't know.""" + # Stub the env so we get past from_env, then override --model to an + # unregistered slug. require_model_card should raise. + monkeypatch.setenv("OPENAI_API_KEY", "sk-test-not-used") + monkeypatch.setenv("OPENAI_MODEL", "gpt-4o-2024-08-06") + with pytest.raises(Exception): # require_model_card raises ValueError or KeyError + main( + [ + "--provider", "openai", + "--model", "gpt-totally-fake-not-in-registry-9999", + "--suite", "prompt_battery", + "--env-file", str(tmp_path / "nonexistent.env"), + ] + ) + + +# --------------------------------------------------------------------------- +# Provider observation dataclass +# --------------------------------------------------------------------------- + + +def test_provider_observation_succeeded_property() -> None: + """succeeded = no error AND non-empty surface.""" + good = ProviderObservation( + prompt="p", surface="hello", provider="x", model="y", elapsed_ms=1.0 + ) + assert good.succeeded + err = ProviderObservation( + prompt="p", surface="", provider="x", model="y", + elapsed_ms=1.0, error_type="E", error_message="m", + ) + assert not err.succeeded + empty = ProviderObservation( + prompt="p", surface="", provider="x", model="y", elapsed_ms=1.0 + ) + assert not empty.succeeded + whitespace_only = ProviderObservation( + prompt="p", surface=" ", provider="x", model="y", elapsed_ms=1.0 + ) + assert not whitespace_only.succeeded