1e01f7794e
Implement the eval infrastructure defined in ADR-0016 before building new eval lanes. This establishes the discipline that governs the entire capability roadmap. - Generic eval framework (evals/framework.py): lane discovery, versioned scoring, result persistence - Cognition lane retrofitted into new convention: 45 cases split into stratified dev (13) / public v1 (13) / holdout (19) sets with contract, runner, and recorded results - Generalized `core eval <lane>` CLI: dynamic lane discovery, --list, --version, --split, --save, --json flags - Holdout runner scaffold: plaintext fallback, encryption interface ready - Baseline runner scaffold: pluggable frontier model interface - Fix: CognitiveTurnPipeline.run() crashed on turn_log[-1] when the unknown-domain gate returned a stub without appending to turn_log - ADR-0016, eval_methodology.md, PROGRESS.md, capability gates session log Phase 0 exit audit found two methodology issues: 1. Pipeline turn_log crash (fixed here) 2. Versor drift in multi-turn sessions (pre-existing, under investigation)
76 lines
2.4 KiB
Python
76 lines
2.4 KiB
Python
"""Holdout runner — scores sealed test sets without exposing item-level results.
|
|
|
|
The holdout set lives encrypted in ``evals/<lane>/holdouts/``. The decryption
|
|
key is held by the human reviewer and supplied via environment variable.
|
|
|
|
Current implementation is a scaffold: it reads plaintext holdouts for initial
|
|
development. The encryption layer (age or GPG) is wired in before any holdout
|
|
set is considered "sealed."
|
|
|
|
Trust boundary: this module reads encrypted files and writes only aggregate
|
|
scores. It must never write item-level results, case details, or raw case
|
|
content to the working tree.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from evals.framework import LaneInfo, load_lane_runner, load_cases
|
|
|
|
|
|
HOLDOUT_KEY_ENV = "CORE_HOLDOUT_KEY"
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class HoldoutResult:
|
|
lane: str
|
|
metrics: dict[str, Any]
|
|
sealed: bool
|
|
|
|
|
|
def _decrypt_holdout(encrypted_path: Path) -> list[dict[str, Any]]:
|
|
"""Decrypt a holdout file and return cases.
|
|
|
|
Currently: reads plaintext fallback if no encryption key is set.
|
|
Future: decrypt with age/GPG using CORE_HOLDOUT_KEY.
|
|
"""
|
|
key = os.environ.get(HOLDOUT_KEY_ENV)
|
|
|
|
plaintext_path = encrypted_path.parent / "cases_plaintext.jsonl"
|
|
if key is None and plaintext_path.exists():
|
|
return load_cases(plaintext_path)
|
|
|
|
if key is None:
|
|
raise EnvironmentError(
|
|
f"Set {HOLDOUT_KEY_ENV} to decrypt holdout, or provide "
|
|
f"cases_plaintext.jsonl for unsealed development."
|
|
)
|
|
|
|
# TODO: implement actual decryption (age -d -i <key> <encrypted_path>)
|
|
raise NotImplementedError(
|
|
"Encrypted holdout decryption not yet implemented. "
|
|
"Use cases_plaintext.jsonl for development."
|
|
)
|
|
|
|
|
|
def run_holdout(lane: LaneInfo, *, config: Any = None) -> HoldoutResult:
|
|
"""Score a lane's holdout set and return only aggregate metrics."""
|
|
holdout_dir = lane.root / "holdouts"
|
|
if not holdout_dir.exists():
|
|
raise FileNotFoundError(f"No holdouts directory: {holdout_dir}")
|
|
|
|
cases = _decrypt_holdout(holdout_dir / "cases.jsonl.age")
|
|
|
|
runner_module = load_lane_runner(lane)
|
|
report = runner_module.run_lane(cases, config=config)
|
|
|
|
sealed = os.environ.get(HOLDOUT_KEY_ENV) is not None
|
|
|
|
return HoldoutResult(
|
|
lane=lane.name,
|
|
metrics=report.metrics,
|
|
sealed=sealed,
|
|
)
|