366f7a08c4
* feat: add cognitive eval harness with CLI integration 20 eval cases across 8 categories (definition, comparison, cause, procedure, recall, correction, verification, unknown). Metrics: intent accuracy, term capture, surface groundedness, versor closure, trace determinism. CLI: `core eval cognition [--json] [--report PATH]`. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * feat: add operator calibration replay with deterministic grid search Bounded parameter tuning via eval replay evidence. Grid search over salience_top_k and inhibition_threshold with invariant regression guard (versor closure must not regress). Frozen CalibrationParams, before/after metrics, no pack or identity mutation. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
31 lines
1.2 KiB
Python
31 lines
1.2 KiB
Python
"""Calibration report generation."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from calibration.tune import CalibrationResult
|
|
|
|
|
|
def write_report(result: CalibrationResult, path: str | Path) -> Path:
|
|
p = Path(path)
|
|
p.parent.mkdir(parents=True, exist_ok=True)
|
|
p.write_text(json.dumps(result.as_dict(), ensure_ascii=False, indent=2, sort_keys=True))
|
|
return p
|
|
|
|
|
|
def print_report(result: CalibrationResult) -> None:
|
|
bl = result.baseline_report
|
|
br = result.best_report
|
|
print("=== Calibration Report ===")
|
|
print(f"baseline: {result.baseline_params.as_dict()}")
|
|
print(f" intent_accuracy : {bl.intent_accuracy:.1%}")
|
|
print(f" versor_closure : {bl.versor_closure_rate:.1%}")
|
|
print(f" surface_ground : {bl.surface_groundedness:.1%}")
|
|
print(f"best : {result.best_params.as_dict()}")
|
|
print(f" intent_accuracy : {br.intent_accuracy:.1%}")
|
|
print(f" versor_closure : {br.versor_closure_rate:.1%}")
|
|
print(f" surface_ground : {br.surface_groundedness:.1%}")
|
|
print(f"candidates evaluated: {len(result.candidates)}")
|
|
print(f"candidates accepted : {sum(1 for c in result.candidates if c.accepted)}")
|