1e01f7794e
Implement the eval infrastructure defined in ADR-0016 before building new eval lanes. This establishes the discipline that governs the entire capability roadmap. - Generic eval framework (evals/framework.py): lane discovery, versioned scoring, result persistence - Cognition lane retrofitted into new convention: 45 cases split into stratified dev (13) / public v1 (13) / holdout (19) sets with contract, runner, and recorded results - Generalized `core eval <lane>` CLI: dynamic lane discovery, --list, --version, --split, --save, --json flags - Holdout runner scaffold: plaintext fallback, encryption interface ready - Baseline runner scaffold: pluggable frontier model interface - Fix: CognitiveTurnPipeline.run() crashed on turn_log[-1] when the unknown-domain gate returned a stub without appending to turn_log - ADR-0016, eval_methodology.md, PROGRESS.md, capability gates session log Phase 0 exit audit found two methodology issues: 1. Pipeline turn_log crash (fixed here) 2. Versor drift in multi-turn sessions (pre-existing, under investigation)
76 lines
2.3 KiB
Python
76 lines
2.3 KiB
Python
"""Baseline runner — scores frontier models on eval lane public test sets.
|
|
|
|
Queries a frontier model API on the same public test set that CORE is scored on,
|
|
using the eval task as the prompt (no prompt engineering, no tuning).
|
|
|
|
Current implementation is a scaffold with a pluggable model interface. Actual
|
|
API calls are deferred until API keys are configured.
|
|
|
|
Trust boundary: this module calls external APIs. It sends only eval prompts
|
|
(which are not sensitive) and writes scores to ``evals/<lane>/baselines/``.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Protocol
|
|
|
|
|
|
class BaselineModel(Protocol):
|
|
"""Interface for a frontier model baseline."""
|
|
|
|
@property
|
|
def model_id(self) -> str: ...
|
|
|
|
def score_case(self, case: dict[str, Any]) -> dict[str, Any]:
|
|
"""Score a single case. Returns a dict with at minimum 'passed': bool."""
|
|
...
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class BaselineResult:
|
|
lane: str
|
|
version: str
|
|
model_id: str
|
|
metrics: dict[str, Any]
|
|
timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"lane": self.lane,
|
|
"version": self.version,
|
|
"model_id": self.model_id,
|
|
"timestamp": self.timestamp,
|
|
"metrics": self.metrics,
|
|
}
|
|
|
|
|
|
class StubBaseline:
|
|
"""Placeholder baseline that records 'not scored' for all cases."""
|
|
|
|
@property
|
|
def model_id(self) -> str:
|
|
return "stub-not-configured"
|
|
|
|
def score_case(self, case: dict[str, Any]) -> dict[str, Any]:
|
|
return {"passed": False, "reason": "baseline model not configured"}
|
|
|
|
|
|
def write_baseline(
|
|
lane_root: Path,
|
|
result: BaselineResult,
|
|
) -> Path:
|
|
"""Write a baseline result to the lane's baselines directory."""
|
|
baselines_dir = lane_root / "baselines"
|
|
baselines_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
filename = f"{result.version}_{result.model_id}_{ts}.json"
|
|
path = baselines_dir / filename
|
|
path.write_text(
|
|
json.dumps(result.as_dict(), ensure_ascii=False, indent=2, sort_keys=True)
|
|
+ "\n"
|
|
)
|
|
return path
|