c4f056c44c
Records the architectural floor for frontier-LLM performance on each
Phase 2 v1 lane.
The baseline is structural: every lane's scoring rubric measures a
property that frontier LLMs do not architecturally emit (Provenance
typed sources, pack_mutation_proposal, vault_hits, REJECTED_IDENTITY
outcome, deterministic trace_hash). The frontier score on each of
those sub-metrics is 0.0 by construction, not by failure — even a
live-API run would still record 0.0 on these typed-signal checks
because the evidence is absent regardless of prose quality.
Artifacts:
docs/frontier_baselines.md
Full per-lane analysis: what each sub-metric scores, why the
frontier value is 0, and where a live-API baseline would or
would not add information.
evals/<lane>/baselines/v1_structural_zero.json (× 5)
Per-lane baseline records in the same shape as lane reports.
Encodes 0.0 / None on each sub-metric with rationale.
evals/baseline_runner.py
Adds StructuralZeroBaseline adapter conforming to the
BaselineModel protocol — a real, non-stub adapter that returns
the deterministic floor. Live-API adapters (Anthropic, OpenAI)
can be wired alongside when API keys are configured; the
structural floor remains the comparison baseline.
Across 5 lanes / 14 typed-signal sub-metrics:
CORE v1: 1.0 (each)
frontier structural: 0.0 (each)
The gap is "CORE measures a property frontier output does not
expose", not "CORE outperforms on a shared benchmark". v2 lanes may
add content-level sub-metrics where direct comparison via live-API
runs becomes meaningful.
103 lines
3.2 KiB
Python
103 lines
3.2 KiB
Python
"""Baseline runner — scores frontier models on eval lane public test sets.
|
|
|
|
Queries a frontier model API on the same public test set that CORE is scored on,
|
|
using the eval task as the prompt (no prompt engineering, no tuning).
|
|
|
|
Current implementation is a scaffold with a pluggable model interface. Actual
|
|
API calls are deferred until API keys are configured.
|
|
|
|
Trust boundary: this module calls external APIs. It sends only eval prompts
|
|
(which are not sensitive) and writes scores to ``evals/<lane>/baselines/``.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Protocol
|
|
|
|
|
|
class BaselineModel(Protocol):
|
|
"""Interface for a frontier model baseline."""
|
|
|
|
@property
|
|
def model_id(self) -> str: ...
|
|
|
|
def score_case(self, case: dict[str, Any]) -> dict[str, Any]:
|
|
"""Score a single case. Returns a dict with at minimum 'passed': bool."""
|
|
...
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class BaselineResult:
|
|
lane: str
|
|
version: str
|
|
model_id: str
|
|
metrics: dict[str, Any]
|
|
timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"lane": self.lane,
|
|
"version": self.version,
|
|
"model_id": self.model_id,
|
|
"timestamp": self.timestamp,
|
|
"metrics": self.metrics,
|
|
}
|
|
|
|
|
|
class StubBaseline:
|
|
"""Placeholder baseline that records 'not scored' for all cases."""
|
|
|
|
@property
|
|
def model_id(self) -> str:
|
|
return "stub-not-configured"
|
|
|
|
def score_case(self, case: dict[str, Any]) -> dict[str, Any]:
|
|
return {"passed": False, "reason": "baseline model not configured"}
|
|
|
|
|
|
class StructuralZeroBaseline:
|
|
"""Structural-zero frontier baseline.
|
|
|
|
Encodes the architectural fact that frontier LLMs do not emit the
|
|
typed signals CORE's lane rubrics score against (Provenance.sources,
|
|
pack_mutation_proposal, vault_hits, REJECTED_IDENTITY outcome,
|
|
deterministic trace_hash). Every case scores ``passed=False`` with
|
|
a reason identifying the missing typed evidence.
|
|
|
|
This is not a stub: it is the deterministic floor against which any
|
|
live-API baseline (if/when configured) must be compared. See
|
|
``docs/frontier_baselines.md`` for the full analysis.
|
|
"""
|
|
|
|
_REASON = (
|
|
"frontier outputs do not emit the typed signal this rubric scores "
|
|
"(see docs/frontier_baselines.md)"
|
|
)
|
|
|
|
@property
|
|
def model_id(self) -> str:
|
|
return "frontier-structural-zero"
|
|
|
|
def score_case(self, case: dict[str, Any]) -> dict[str, Any]:
|
|
return {"passed": False, "reason": self._REASON}
|
|
|
|
|
|
def write_baseline(
|
|
lane_root: Path,
|
|
result: BaselineResult,
|
|
) -> Path:
|
|
"""Write a baseline result to the lane's baselines directory."""
|
|
baselines_dir = lane_root / "baselines"
|
|
baselines_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
filename = f"{result.version}_{result.model_id}_{ts}.json"
|
|
path = baselines_dir / filename
|
|
path.write_text(
|
|
json.dumps(result.as_dict(), ensure_ascii=False, indent=2, sort_keys=True)
|
|
+ "\n"
|
|
)
|
|
return path
|