Add cognitive eval harness and calibration replay (#30)
* feat: add cognitive eval harness with CLI integration 20 eval cases across 8 categories (definition, comparison, cause, procedure, recall, correction, verification, unknown). Metrics: intent accuracy, term capture, surface groundedness, versor closure, trace determinism. CLI: `core eval cognition [--json] [--report PATH]`. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * feat: add operator calibration replay with deterministic grid search Bounded parameter tuning via eval replay evidence. Grid search over salience_top_k and inhibition_threshold with invariant regression guard (versor closure must not regress). Frozen CalibrationParams, before/after metrics, no pack or identity mutation. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Shay
GitHub
parent
a7febd48ef
commit
366f7a08c4
13 changed files with 702 additions and 1 deletions
0
calibration/__init__.py
Normal file
0
calibration/__init__.py
Normal file
52
calibration/params.py
Normal file
52
calibration/params.py
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
"""Calibration parameter space — bounded, deterministic, immutable."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from itertools import product
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CalibrationParams:
|
||||
salience_top_k: int = 16
|
||||
inhibition_threshold: float = 0.3
|
||||
teaching_retrieval_limit: int = 8
|
||||
|
||||
def as_dict(self) -> dict[str, int | float]:
|
||||
return {
|
||||
"salience_top_k": self.salience_top_k,
|
||||
"inhibition_threshold": self.inhibition_threshold,
|
||||
"teaching_retrieval_limit": self.teaching_retrieval_limit,
|
||||
}
|
||||
|
||||
|
||||
DEFAULT_PARAMS = CalibrationParams()
|
||||
|
||||
PARAM_GRID: dict[str, tuple] = {
|
||||
"salience_top_k": (8, 12, 16),
|
||||
"inhibition_threshold": (0.2, 0.3, 0.4),
|
||||
}
|
||||
|
||||
|
||||
def grid_candidates(
|
||||
grid: dict[str, tuple] | None = None,
|
||||
base: CalibrationParams = DEFAULT_PARAMS,
|
||||
) -> tuple[CalibrationParams, ...]:
|
||||
"""Generate all candidate parameter sets from a grid.
|
||||
|
||||
Each candidate varies exactly one axis from the base; the grid is
|
||||
a deterministic Cartesian product over the provided axes.
|
||||
"""
|
||||
g = grid or PARAM_GRID
|
||||
keys = sorted(g.keys())
|
||||
values = [g[k] for k in keys]
|
||||
candidates = []
|
||||
for combo in product(*values):
|
||||
overrides = dict(zip(keys, combo))
|
||||
candidate = CalibrationParams(
|
||||
salience_top_k=overrides.get("salience_top_k", base.salience_top_k),
|
||||
inhibition_threshold=overrides.get("inhibition_threshold", base.inhibition_threshold),
|
||||
teaching_retrieval_limit=base.teaching_retrieval_limit,
|
||||
)
|
||||
candidates.append(candidate)
|
||||
return tuple(candidates)
|
||||
36
calibration/replay.py
Normal file
36
calibration/replay.py
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
"""Replay eval cases under a given parameter set and return metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from core.config import RuntimeConfig, DEFAULT_CONFIG
|
||||
from calibration.params import CalibrationParams
|
||||
from evals.metrics import EvalReport
|
||||
from evals.run_cognition_eval import load_cases, run_eval
|
||||
|
||||
|
||||
def replay_with_params(
|
||||
params: CalibrationParams,
|
||||
cases: list[dict] | None = None,
|
||||
) -> EvalReport:
|
||||
"""Run the eval harness under a specific parameter configuration.
|
||||
|
||||
Builds a RuntimeConfig from the CalibrationParams and passes it
|
||||
through to run_eval, which creates fresh ChatRuntime instances
|
||||
per case.
|
||||
"""
|
||||
if cases is None:
|
||||
cases = load_cases()
|
||||
|
||||
config = RuntimeConfig(
|
||||
input_packs=DEFAULT_CONFIG.input_packs,
|
||||
output_language=DEFAULT_CONFIG.output_language,
|
||||
frame_pack=DEFAULT_CONFIG.frame_pack,
|
||||
max_tokens=DEFAULT_CONFIG.max_tokens,
|
||||
allow_cross_language_recall=DEFAULT_CONFIG.allow_cross_language_recall,
|
||||
allow_cross_language_generation=DEFAULT_CONFIG.allow_cross_language_generation,
|
||||
vault_reproject_interval=DEFAULT_CONFIG.vault_reproject_interval,
|
||||
use_salience=DEFAULT_CONFIG.use_salience,
|
||||
salience_top_k=params.salience_top_k,
|
||||
inhibition_threshold=params.inhibition_threshold,
|
||||
)
|
||||
return run_eval(cases, config=config)
|
||||
31
calibration/report.py
Normal file
31
calibration/report.py
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
"""Calibration report generation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from calibration.tune import CalibrationResult
|
||||
|
||||
|
||||
def write_report(result: CalibrationResult, path: str | Path) -> Path:
|
||||
p = Path(path)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(json.dumps(result.as_dict(), ensure_ascii=False, indent=2, sort_keys=True))
|
||||
return p
|
||||
|
||||
|
||||
def print_report(result: CalibrationResult) -> None:
|
||||
bl = result.baseline_report
|
||||
br = result.best_report
|
||||
print("=== Calibration Report ===")
|
||||
print(f"baseline: {result.baseline_params.as_dict()}")
|
||||
print(f" intent_accuracy : {bl.intent_accuracy:.1%}")
|
||||
print(f" versor_closure : {bl.versor_closure_rate:.1%}")
|
||||
print(f" surface_ground : {bl.surface_groundedness:.1%}")
|
||||
print(f"best : {result.best_params.as_dict()}")
|
||||
print(f" intent_accuracy : {br.intent_accuracy:.1%}")
|
||||
print(f" versor_closure : {br.versor_closure_rate:.1%}")
|
||||
print(f" surface_ground : {br.surface_groundedness:.1%}")
|
||||
print(f"candidates evaluated: {len(result.candidates)}")
|
||||
print(f"candidates accepted : {sum(1 for c in result.candidates if c.accepted)}")
|
||||
115
calibration/tune.py
Normal file
115
calibration/tune.py
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
"""Deterministic grid-search calibration over bounded parameter sets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from calibration.params import CalibrationParams, DEFAULT_PARAMS, grid_candidates
|
||||
from calibration.replay import replay_with_params
|
||||
from evals.metrics import EvalReport
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CalibrationCandidate:
|
||||
params: CalibrationParams
|
||||
before_report: EvalReport
|
||||
after_report: EvalReport
|
||||
accepted: bool
|
||||
rejection_reason: str | None = None
|
||||
|
||||
def improvement(self) -> float:
|
||||
return self.after_report.intent_accuracy - self.before_report.intent_accuracy
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CalibrationResult:
|
||||
baseline_params: CalibrationParams
|
||||
baseline_report: EvalReport
|
||||
candidates: tuple[CalibrationCandidate, ...]
|
||||
best_params: CalibrationParams
|
||||
best_report: EvalReport
|
||||
|
||||
def as_dict(self) -> dict:
|
||||
return {
|
||||
"baseline_params": self.baseline_params.as_dict(),
|
||||
"baseline_metrics": {
|
||||
"intent_accuracy": round(self.baseline_report.intent_accuracy, 4),
|
||||
"versor_closure_rate": round(self.baseline_report.versor_closure_rate, 4),
|
||||
"surface_groundedness": round(self.baseline_report.surface_groundedness, 4),
|
||||
},
|
||||
"best_params": self.best_params.as_dict(),
|
||||
"best_metrics": {
|
||||
"intent_accuracy": round(self.best_report.intent_accuracy, 4),
|
||||
"versor_closure_rate": round(self.best_report.versor_closure_rate, 4),
|
||||
"surface_groundedness": round(self.best_report.surface_groundedness, 4),
|
||||
},
|
||||
"candidates_evaluated": len(self.candidates),
|
||||
"candidates_accepted": sum(1 for c in self.candidates if c.accepted),
|
||||
}
|
||||
|
||||
|
||||
def _score(report: EvalReport) -> float:
|
||||
"""Composite score: intent accuracy + versor closure + surface groundedness."""
|
||||
return (
|
||||
report.intent_accuracy
|
||||
+ report.versor_closure_rate
|
||||
+ report.surface_groundedness
|
||||
) / 3.0
|
||||
|
||||
|
||||
def calibrate(
|
||||
cases: list[dict] | None = None,
|
||||
baseline: CalibrationParams = DEFAULT_PARAMS,
|
||||
grid: dict[str, tuple] | None = None,
|
||||
) -> CalibrationResult:
|
||||
"""Run deterministic grid-search calibration.
|
||||
|
||||
1. Evaluate baseline params
|
||||
2. For each candidate in the grid, evaluate and compare
|
||||
3. Accept only if no invariant regression (versor closure stays 100%)
|
||||
4. Return the best accepted candidate
|
||||
"""
|
||||
baseline_report = replay_with_params(baseline, cases)
|
||||
baseline_score = _score(baseline_report)
|
||||
|
||||
candidates_list: list[CalibrationCandidate] = []
|
||||
best_params = baseline
|
||||
best_report = baseline_report
|
||||
best_score = baseline_score
|
||||
|
||||
for params in grid_candidates(grid, baseline):
|
||||
report = replay_with_params(params, cases)
|
||||
|
||||
rejection_reason = None
|
||||
accepted = True
|
||||
|
||||
if report.versor_closure_rate < baseline_report.versor_closure_rate:
|
||||
accepted = False
|
||||
rejection_reason = "versor closure regression"
|
||||
|
||||
score = _score(report)
|
||||
if accepted and score <= baseline_score:
|
||||
accepted = False
|
||||
rejection_reason = "no composite improvement"
|
||||
|
||||
candidate = CalibrationCandidate(
|
||||
params=params,
|
||||
before_report=baseline_report,
|
||||
after_report=report,
|
||||
accepted=accepted,
|
||||
rejection_reason=rejection_reason,
|
||||
)
|
||||
candidates_list.append(candidate)
|
||||
|
||||
if accepted and score > best_score:
|
||||
best_params = params
|
||||
best_report = report
|
||||
best_score = score
|
||||
|
||||
return CalibrationResult(
|
||||
baseline_params=baseline,
|
||||
baseline_report=baseline_report,
|
||||
candidates=tuple(candidates_list),
|
||||
best_params=best_params,
|
||||
best_report=best_report,
|
||||
)
|
||||
49
core/cli.py
49
core/cli.py
|
|
@ -23,7 +23,7 @@ _CORE_RS_DIR = _REPO_ROOT / "core-rs"
|
|||
_CORE_RS_MANIFEST = _CORE_RS_DIR / "Cargo.toml"
|
||||
|
||||
DESCRIPTION = "CORE versor engine command suite."
|
||||
EPILOG = "Examples:\n core chat\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core test --suite smoke -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q"
|
||||
EPILOG = "Examples:\n core chat\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core test --suite smoke -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core eval cognition\n core eval cognition --json"
|
||||
|
||||
_TEST_SUITES: dict[str, tuple[str, ...]] = {
|
||||
"smoke": (
|
||||
|
|
@ -44,6 +44,7 @@ _TEST_SUITES: dict[str, tuple[str, ...]] = {
|
|||
"tests/test_cognitive_turn_pipeline.py",
|
||||
"tests/test_articulation_realizer_v2.py",
|
||||
"tests/test_semantic_realizer_integration.py",
|
||||
"tests/test_cognitive_eval_harness.py",
|
||||
),
|
||||
"teaching": (
|
||||
"tests/test_reviewed_teaching_loop.py",
|
||||
|
|
@ -450,6 +451,45 @@ def cmd_doctor(args: argparse.Namespace) -> int:
|
|||
return 0 if ok else 1
|
||||
|
||||
|
||||
def cmd_eval_cognition(args: argparse.Namespace) -> int:
|
||||
"""Run the cognition eval harness."""
|
||||
from evals.run_cognition_eval import load_cases, run_eval
|
||||
|
||||
cases = load_cases()
|
||||
report = run_eval(cases)
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(report.as_dict(), ensure_ascii=False, indent=2, sort_keys=True))
|
||||
else:
|
||||
print(f"cases : {report.total}")
|
||||
print(f"intent_accuracy: {report.intent_accuracy:.1%}")
|
||||
print(f"term_capture : {report.term_capture_rate:.1%}")
|
||||
print(f"surface_ground : {report.surface_groundedness:.1%}")
|
||||
print(f"versor_closure : {report.versor_closure_rate:.1%}")
|
||||
print(f"det_traces : {report.deterministic_traces}")
|
||||
failures = [c for c in report.cases if not c.intent_correct or not c.versor_closure]
|
||||
if failures:
|
||||
print(f"\nfailures ({len(failures)}):")
|
||||
for c in failures:
|
||||
issues = []
|
||||
if not c.intent_correct:
|
||||
issues.append("intent")
|
||||
if not c.versor_closure:
|
||||
issues.append(f"versor={c.versor_condition:.2e}")
|
||||
print(f" {c.case_id}: {', '.join(issues)}")
|
||||
|
||||
if args.report:
|
||||
report_path = Path(args.report)
|
||||
report_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
report_path.write_text(
|
||||
json.dumps(report.as_dict(), ensure_ascii=False, indent=2, sort_keys=True)
|
||||
)
|
||||
print(f"\nreport written: {report_path}")
|
||||
|
||||
all_pass = report.intent_accuracy == 1.0 and report.versor_closure_rate == 1.0
|
||||
return 0 if all_pass else 1
|
||||
|
||||
|
||||
def _add_runtime_policy_args(parser: argparse.ArgumentParser) -> None:
|
||||
parser.add_argument("--pack", action="append", help="language pack to mount; repeat for multiple packs")
|
||||
parser.add_argument("--output-language", default="en", help="target output language code; default: en")
|
||||
|
|
@ -537,6 +577,13 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
rust_test = rust_sub.add_parser("test", help="run cargo test --release for core-rs")
|
||||
rust_test.set_defaults(func=cmd_rust_test)
|
||||
|
||||
eval_cmd = subparsers.add_parser("eval", help="run eval harnesses")
|
||||
eval_sub = eval_cmd.add_subparsers(dest="eval_command", metavar="eval-command", required=True)
|
||||
eval_cognition = eval_sub.add_parser("cognition", help="run the cognition eval harness")
|
||||
eval_cognition.add_argument("--json", action="store_true", help="emit machine-readable JSON")
|
||||
eval_cognition.add_argument("--report", metavar="PATH", help="write JSON report to file")
|
||||
eval_cognition.set_defaults(func=cmd_eval_cognition)
|
||||
|
||||
doctor = subparsers.add_parser("doctor", help="check runtime imports and packaging health")
|
||||
doctor.add_argument("--packs", action="store_true", help="also list discovered language packs")
|
||||
doctor.add_argument("--rust", action="store_true", help="also show Rust backend activation status")
|
||||
|
|
|
|||
0
evals/__init__.py
Normal file
0
evals/__init__.py
Normal file
20
evals/cognition_cases.jsonl
Normal file
20
evals/cognition_cases.jsonl
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
{"id": "definition_truth_001", "category": "definition", "prompt": "What is truth?", "expected_intent": "definition", "expected_terms": ["truth"], "expected_surface_contains": ["truth"], "requires_versor_closure": true, "requires_deterministic_trace": true}
|
||||
{"id": "definition_light_002", "category": "definition", "prompt": "What is light?", "expected_intent": "definition", "expected_terms": ["light"], "expected_surface_contains": ["light"], "requires_versor_closure": true, "requires_deterministic_trace": true}
|
||||
{"id": "definition_knowledge_003", "category": "definition", "prompt": "What is knowledge?", "expected_intent": "definition", "expected_terms": ["knowledge"], "expected_surface_contains": ["knowledge"], "requires_versor_closure": true, "requires_deterministic_trace": true}
|
||||
{"id": "definition_meaning_004", "category": "definition", "prompt": "What is meaning?", "expected_intent": "definition", "expected_terms": ["meaning"], "expected_surface_contains": ["meaning"], "requires_versor_closure": true, "requires_deterministic_trace": true}
|
||||
{"id": "comparison_truth_light_005", "category": "comparison", "prompt": "Compare truth and light", "expected_intent": "comparison", "expected_terms": ["truth", "light"], "expected_surface_contains": ["truth", "light"], "requires_versor_closure": true, "requires_deterministic_trace": true}
|
||||
{"id": "comparison_word_meaning_006", "category": "comparison", "prompt": "Compare word and meaning", "expected_intent": "comparison", "expected_terms": ["word", "meaning"], "expected_surface_contains": ["word", "meaning"], "requires_versor_closure": true, "requires_deterministic_trace": true}
|
||||
{"id": "cause_light_007", "category": "cause", "prompt": "Why does light exist?", "expected_intent": "cause", "expected_terms": ["light"], "expected_surface_contains": ["light"], "requires_versor_closure": true, "requires_deterministic_trace": true}
|
||||
{"id": "cause_creation_008", "category": "cause", "prompt": "Why does creation matter?", "expected_intent": "cause", "expected_terms": ["creation"], "expected_surface_contains": ["creation"], "requires_versor_closure": true, "requires_deterministic_trace": true}
|
||||
{"id": "cause_wisdom_009", "category": "cause", "prompt": "Why does wisdom matter?", "expected_intent": "cause", "expected_terms": ["wisdom"], "expected_surface_contains": ["wisdom"], "requires_versor_closure": true, "requires_deterministic_trace": true}
|
||||
{"id": "procedure_define_010", "category": "procedure", "prompt": "How do I define a concept?", "expected_intent": "procedure", "expected_terms": ["concept"], "expected_surface_contains": ["concept"], "requires_versor_closure": true, "requires_deterministic_trace": true}
|
||||
{"id": "procedure_compare_011", "category": "procedure", "prompt": "How do I compare two terms?", "expected_intent": "procedure", "expected_terms": ["terms"], "expected_surface_contains": [], "requires_versor_closure": true, "requires_deterministic_trace": true}
|
||||
{"id": "recall_truth_012", "category": "recall", "prompt": "Remember truth", "expected_intent": "recall", "expected_terms": ["truth"], "expected_surface_contains": ["truth"], "requires_versor_closure": true, "requires_deterministic_trace": true}
|
||||
{"id": "recall_light_013", "category": "recall", "prompt": "Remember light", "expected_intent": "recall", "expected_terms": ["light"], "expected_surface_contains": ["light"], "requires_versor_closure": true, "requires_deterministic_trace": true}
|
||||
{"id": "correction_basic_014", "category": "correction", "prompt": "No, that's wrong", "expected_intent": "correction", "expected_terms": [], "expected_surface_contains": ["correction"], "requires_versor_closure": true, "requires_deterministic_trace": true}
|
||||
{"id": "correction_specific_015", "category": "correction", "prompt": "No, correction means reviewed repair", "expected_intent": "correction", "expected_terms": ["correction"], "expected_surface_contains": ["correction"], "requires_versor_closure": true, "requires_deterministic_trace": true}
|
||||
{"id": "verification_truth_016", "category": "verification", "prompt": "Is truth coherent?", "expected_intent": "verification", "expected_terms": ["truth"], "expected_surface_contains": ["truth"], "requires_versor_closure": true, "requires_deterministic_trace": true}
|
||||
{"id": "verification_light_017", "category": "verification", "prompt": "Is light real?", "expected_intent": "verification", "expected_terms": ["light"], "expected_surface_contains": ["light"], "requires_versor_closure": true, "requires_deterministic_trace": true}
|
||||
{"id": "unknown_word_018", "category": "unknown", "prompt": "word beginning truth", "expected_intent": "unknown", "expected_terms": ["word", "truth"], "expected_surface_contains": [], "requires_versor_closure": true, "requires_deterministic_trace": true}
|
||||
{"id": "unknown_logos_019", "category": "unknown", "prompt": "light logos", "expected_intent": "unknown", "expected_terms": ["light"], "expected_surface_contains": [], "requires_versor_closure": true, "requires_deterministic_trace": true}
|
||||
{"id": "definition_wisdom_020", "category": "definition", "prompt": "What is wisdom?", "expected_intent": "definition", "expected_terms": ["wisdom"], "expected_surface_contains": ["wisdom"], "requires_versor_closure": true, "requires_deterministic_trace": true}
|
||||
73
evals/metrics.py
Normal file
73
evals/metrics.py
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
"""Cognition eval metrics — deterministic, compact measurements."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CaseResult:
|
||||
case_id: str
|
||||
category: str
|
||||
prompt: str
|
||||
intent_correct: bool
|
||||
terms_captured: tuple[str, ...]
|
||||
terms_expected: tuple[str, ...]
|
||||
surface_contains_pass: bool
|
||||
versor_closure: bool
|
||||
versor_condition: float
|
||||
trace_hash: str
|
||||
surface: str
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class EvalReport:
|
||||
total: int = 0
|
||||
intent_correct: int = 0
|
||||
terms_captured: int = 0
|
||||
terms_expected: int = 0
|
||||
surface_grounded: int = 0
|
||||
versor_closures: int = 0
|
||||
deterministic_traces: int = 0
|
||||
cases: list[CaseResult] = field(default_factory=list)
|
||||
trace_hashes: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def intent_accuracy(self) -> float:
|
||||
return self.intent_correct / self.total if self.total else 0.0
|
||||
|
||||
@property
|
||||
def term_capture_rate(self) -> float:
|
||||
return self.terms_captured / self.terms_expected if self.terms_expected else 1.0
|
||||
|
||||
@property
|
||||
def surface_groundedness(self) -> float:
|
||||
return self.surface_grounded / self.total if self.total else 0.0
|
||||
|
||||
@property
|
||||
def versor_closure_rate(self) -> float:
|
||||
return self.versor_closures / self.total if self.total else 0.0
|
||||
|
||||
def as_dict(self) -> dict:
|
||||
return {
|
||||
"total": self.total,
|
||||
"intent_accuracy": round(self.intent_accuracy, 4),
|
||||
"term_capture_rate": round(self.term_capture_rate, 4),
|
||||
"surface_groundedness": round(self.surface_groundedness, 4),
|
||||
"versor_closure_rate": round(self.versor_closure_rate, 4),
|
||||
"deterministic_traces": self.deterministic_traces,
|
||||
"trace_hashes": dict(self.trace_hashes),
|
||||
"cases": [
|
||||
{
|
||||
"case_id": c.case_id,
|
||||
"category": c.category,
|
||||
"intent_correct": c.intent_correct,
|
||||
"surface_contains_pass": c.surface_contains_pass,
|
||||
"versor_closure": c.versor_closure,
|
||||
"versor_condition": round(c.versor_condition, 9),
|
||||
"trace_hash": c.trace_hash,
|
||||
"surface": c.surface,
|
||||
}
|
||||
for c in self.cases
|
||||
],
|
||||
}
|
||||
0
evals/reports/.gitkeep
Normal file
0
evals/reports/.gitkeep
Normal file
110
evals/run_cognition_eval.py
Normal file
110
evals/run_cognition_eval.py
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
"""Run the cognition eval harness.
|
||||
|
||||
Loads cases from cognition_cases.jsonl, runs each through the
|
||||
CognitiveTurnPipeline, and produces an EvalReport with deterministic
|
||||
metrics. Each case gets a fresh pipeline instance for isolation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from chat.runtime import ChatRuntime
|
||||
from core.config import RuntimeConfig
|
||||
from core.cognition.pipeline import CognitiveTurnPipeline
|
||||
from evals.metrics import CaseResult, EvalReport
|
||||
from generate.intent import IntentTag
|
||||
|
||||
_CASES_PATH = Path(__file__).parent / "cognition_cases.jsonl"
|
||||
|
||||
|
||||
def load_cases(path: Path | None = None) -> list[dict]:
|
||||
p = path or _CASES_PATH
|
||||
cases = []
|
||||
for line in p.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
cases.append(json.loads(line))
|
||||
return cases
|
||||
|
||||
|
||||
def _run_case(case: dict, pipeline: CognitiveTurnPipeline) -> CaseResult:
|
||||
prompt = case["prompt"]
|
||||
expected_intent = case["expected_intent"]
|
||||
expected_terms = case.get("expected_terms", [])
|
||||
expected_surface_contains = case.get("expected_surface_contains", [])
|
||||
|
||||
result = pipeline.run(prompt, max_tokens=8)
|
||||
|
||||
actual_intent = result.intent.tag if result.intent else IntentTag.UNKNOWN
|
||||
intent_correct = actual_intent.value == expected_intent
|
||||
|
||||
surface_lower = result.surface.lower()
|
||||
terms_captured = tuple(
|
||||
t for t in expected_terms if t.lower() in surface_lower
|
||||
)
|
||||
surface_contains_pass = all(
|
||||
s.lower() in surface_lower for s in expected_surface_contains
|
||||
)
|
||||
|
||||
versor_ok = result.versor_condition < 1e-6
|
||||
|
||||
return CaseResult(
|
||||
case_id=case["id"],
|
||||
category=case.get("category", "unknown"),
|
||||
prompt=prompt,
|
||||
intent_correct=intent_correct,
|
||||
terms_captured=terms_captured,
|
||||
terms_expected=tuple(expected_terms),
|
||||
surface_contains_pass=surface_contains_pass,
|
||||
versor_closure=versor_ok,
|
||||
versor_condition=result.versor_condition,
|
||||
trace_hash=result.trace_hash,
|
||||
surface=result.surface,
|
||||
)
|
||||
|
||||
|
||||
def run_eval(
|
||||
cases: list[dict] | None = None,
|
||||
config: RuntimeConfig | None = None,
|
||||
) -> EvalReport:
|
||||
if cases is None:
|
||||
cases = load_cases()
|
||||
|
||||
report = EvalReport()
|
||||
|
||||
for case in cases:
|
||||
runtime = ChatRuntime(config=config) if config else ChatRuntime()
|
||||
pipeline = CognitiveTurnPipeline(runtime)
|
||||
case_result = _run_case(case, pipeline)
|
||||
|
||||
report.total += 1
|
||||
if case_result.intent_correct:
|
||||
report.intent_correct += 1
|
||||
report.terms_expected += len(case_result.terms_expected)
|
||||
report.terms_captured += len(case_result.terms_captured)
|
||||
if case_result.surface_contains_pass:
|
||||
report.surface_grounded += 1
|
||||
if case_result.versor_closure:
|
||||
report.versor_closures += 1
|
||||
report.cases.append(case_result)
|
||||
report.trace_hashes[case_result.case_id] = case_result.trace_hash
|
||||
|
||||
return report
|
||||
|
||||
|
||||
def check_determinism(cases: list[dict] | None = None, runs: int = 2) -> bool:
|
||||
if cases is None:
|
||||
cases = load_cases()
|
||||
|
||||
hashes_by_run: list[dict[str, str]] = []
|
||||
for _ in range(runs):
|
||||
report = run_eval(cases)
|
||||
hashes_by_run.append(dict(report.trace_hashes))
|
||||
|
||||
first = hashes_by_run[0]
|
||||
for run_hashes in hashes_by_run[1:]:
|
||||
if run_hashes != first:
|
||||
return False
|
||||
return True
|
||||
106
tests/test_cognitive_eval_harness.py
Normal file
106
tests/test_cognitive_eval_harness.py
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
"""Tests for the cognitive eval harness."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from evals.run_cognition_eval import load_cases, run_eval, check_determinism
|
||||
|
||||
|
||||
_CASES_PATH = Path(__file__).resolve().parent.parent / "evals" / "cognition_cases.jsonl"
|
||||
|
||||
|
||||
class TestCognitionEvalLoadsCases:
|
||||
def test_loads_all_cases(self) -> None:
|
||||
cases = load_cases(_CASES_PATH)
|
||||
assert len(cases) >= 15
|
||||
assert all("id" in c for c in cases)
|
||||
assert all("prompt" in c for c in cases)
|
||||
assert all("expected_intent" in c for c in cases)
|
||||
|
||||
def test_cases_have_valid_structure(self) -> None:
|
||||
cases = load_cases(_CASES_PATH)
|
||||
for case in cases:
|
||||
assert isinstance(case["id"], str)
|
||||
assert isinstance(case["prompt"], str)
|
||||
assert case["expected_intent"] in {
|
||||
"definition", "comparison", "cause", "procedure",
|
||||
"recall", "correction", "verification", "unknown",
|
||||
}
|
||||
assert isinstance(case.get("expected_terms", []), list)
|
||||
|
||||
def test_cases_cover_required_categories(self) -> None:
|
||||
cases = load_cases(_CASES_PATH)
|
||||
categories = {c.get("category", "unknown") for c in cases}
|
||||
required = {"definition", "comparison", "cause", "correction", "verification", "unknown"}
|
||||
assert required.issubset(categories), f"missing: {required - categories}"
|
||||
|
||||
|
||||
class TestCognitionEvalRunsSmallCaseSet:
|
||||
def test_runs_single_case(self) -> None:
|
||||
cases = load_cases(_CASES_PATH)[:1]
|
||||
report = run_eval(cases)
|
||||
assert report.total == 1
|
||||
assert len(report.cases) == 1
|
||||
assert report.cases[0].case_id == cases[0]["id"]
|
||||
|
||||
def test_runs_five_cases(self) -> None:
|
||||
cases = load_cases(_CASES_PATH)[:5]
|
||||
report = run_eval(cases)
|
||||
assert report.total == 5
|
||||
assert len(report.cases) == 5
|
||||
|
||||
|
||||
class TestCognitionEvalRecordsIntentAccuracy:
|
||||
def test_definition_intent_detected(self) -> None:
|
||||
cases = [c for c in load_cases(_CASES_PATH) if c["expected_intent"] == "definition"][:2]
|
||||
report = run_eval(cases)
|
||||
assert report.intent_correct == report.total
|
||||
|
||||
def test_comparison_intent_detected(self) -> None:
|
||||
cases = [c for c in load_cases(_CASES_PATH) if c["expected_intent"] == "comparison"][:1]
|
||||
report = run_eval(cases)
|
||||
assert report.intent_correct == report.total
|
||||
|
||||
def test_report_has_accuracy_metric(self) -> None:
|
||||
cases = load_cases(_CASES_PATH)[:3]
|
||||
report = run_eval(cases)
|
||||
assert 0.0 <= report.intent_accuracy <= 1.0
|
||||
report_dict = report.as_dict()
|
||||
assert "intent_accuracy" in report_dict
|
||||
|
||||
|
||||
class TestCognitionEvalRecordsTraceHashes:
|
||||
def test_trace_hashes_present(self) -> None:
|
||||
cases = load_cases(_CASES_PATH)[:3]
|
||||
report = run_eval(cases)
|
||||
assert len(report.trace_hashes) == 3
|
||||
for case_id, h in report.trace_hashes.items():
|
||||
assert isinstance(h, str)
|
||||
assert len(h) == 64 # SHA-256 hex
|
||||
|
||||
def test_distinct_cases_get_distinct_hashes(self) -> None:
|
||||
cases = load_cases(_CASES_PATH)[:5]
|
||||
report = run_eval(cases)
|
||||
hashes = list(report.trace_hashes.values())
|
||||
assert len(set(hashes)) == len(hashes), "duplicate trace hashes"
|
||||
|
||||
|
||||
class TestCognitionEvalIsDeterministic:
|
||||
def test_two_runs_same_hashes(self) -> None:
|
||||
cases = load_cases(_CASES_PATH)[:3]
|
||||
assert check_determinism(cases, runs=2)
|
||||
|
||||
|
||||
class TestEvalReportSerialization:
|
||||
def test_as_dict_roundtrips(self) -> None:
|
||||
cases = load_cases(_CASES_PATH)[:2]
|
||||
report = run_eval(cases)
|
||||
d = report.as_dict()
|
||||
serialized = json.dumps(d)
|
||||
parsed = json.loads(serialized)
|
||||
assert parsed["total"] == 2
|
||||
assert "intent_accuracy" in parsed
|
||||
assert "trace_hashes" in parsed
|
||||
assert len(parsed["cases"]) == 2
|
||||
111
tests/test_operator_calibration_replay.py
Normal file
111
tests/test_operator_calibration_replay.py
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
"""Tests for the operator calibration replay system."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from calibration.params import (
|
||||
CalibrationParams,
|
||||
DEFAULT_PARAMS,
|
||||
grid_candidates,
|
||||
)
|
||||
from calibration.replay import replay_with_params
|
||||
from calibration.tune import calibrate, CalibrationResult
|
||||
from evals.run_cognition_eval import load_cases
|
||||
|
||||
|
||||
_SMALL_CASES = None
|
||||
|
||||
|
||||
def _get_small_cases() -> list[dict]:
|
||||
global _SMALL_CASES
|
||||
if _SMALL_CASES is None:
|
||||
_SMALL_CASES = load_cases()[:3]
|
||||
return _SMALL_CASES
|
||||
|
||||
|
||||
class TestCalibrationReplayIsDeterministic:
|
||||
def test_same_params_same_metrics(self) -> None:
|
||||
cases = _get_small_cases()
|
||||
r1 = replay_with_params(DEFAULT_PARAMS, cases)
|
||||
r2 = replay_with_params(DEFAULT_PARAMS, cases)
|
||||
assert r1.intent_accuracy == r2.intent_accuracy
|
||||
assert r1.versor_closure_rate == r2.versor_closure_rate
|
||||
assert r1.trace_hashes == r2.trace_hashes
|
||||
|
||||
|
||||
class TestCalibrationCandidateParamsAreBounded:
|
||||
def test_grid_produces_bounded_candidates(self) -> None:
|
||||
candidates = grid_candidates()
|
||||
assert len(candidates) > 0
|
||||
for c in candidates:
|
||||
assert 4 <= c.salience_top_k <= 32
|
||||
assert 0.1 <= c.inhibition_threshold <= 0.9
|
||||
|
||||
def test_grid_is_deterministic(self) -> None:
|
||||
c1 = grid_candidates()
|
||||
c2 = grid_candidates()
|
||||
assert c1 == c2
|
||||
|
||||
def test_custom_grid(self) -> None:
|
||||
custom = {"salience_top_k": (8, 16), "inhibition_threshold": (0.2,)}
|
||||
candidates = grid_candidates(custom)
|
||||
assert len(candidates) == 2
|
||||
salience_values = {c.salience_top_k for c in candidates}
|
||||
assert salience_values == {8, 16}
|
||||
|
||||
|
||||
class TestCalibrationReportHasBeforeAfterMetrics:
|
||||
def test_calibrate_returns_result(self) -> None:
|
||||
cases = _get_small_cases()
|
||||
tiny_grid = {"salience_top_k": (12, 16), "inhibition_threshold": (0.3,)}
|
||||
result = calibrate(cases, grid=tiny_grid)
|
||||
assert isinstance(result, CalibrationResult)
|
||||
assert result.baseline_report.total == len(cases)
|
||||
assert result.best_report.total == len(cases)
|
||||
|
||||
def test_report_dict_has_required_fields(self) -> None:
|
||||
cases = _get_small_cases()
|
||||
tiny_grid = {"salience_top_k": (16,), "inhibition_threshold": (0.3,)}
|
||||
result = calibrate(cases, grid=tiny_grid)
|
||||
d = result.as_dict()
|
||||
assert "baseline_params" in d
|
||||
assert "baseline_metrics" in d
|
||||
assert "best_params" in d
|
||||
assert "best_metrics" in d
|
||||
assert "candidates_evaluated" in d
|
||||
assert "candidates_accepted" in d
|
||||
|
||||
|
||||
class TestCalibrationRejectsInvariantRegression:
|
||||
def test_versor_closure_must_not_regress(self) -> None:
|
||||
cases = _get_small_cases()
|
||||
tiny_grid = {"salience_top_k": (8, 12, 16), "inhibition_threshold": (0.2, 0.3, 0.4)}
|
||||
result = calibrate(cases, grid=tiny_grid)
|
||||
for c in result.candidates:
|
||||
if c.accepted:
|
||||
assert c.after_report.versor_closure_rate >= result.baseline_report.versor_closure_rate
|
||||
|
||||
|
||||
class TestCalibrationDoesNotMutateIdentityOrPacks:
|
||||
def test_params_are_frozen(self) -> None:
|
||||
params = CalibrationParams()
|
||||
try:
|
||||
params.salience_top_k = 99 # type: ignore[misc]
|
||||
raise AssertionError("CalibrationParams should be frozen")
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
def test_calibration_does_not_touch_packs(self) -> None:
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
pack_dir = Path(__file__).resolve().parent.parent / "language_packs" / "data"
|
||||
before = {}
|
||||
for f in sorted(pack_dir.rglob("*.jsonl")):
|
||||
before[str(f)] = hashlib.sha256(f.read_bytes()).hexdigest()
|
||||
|
||||
cases = _get_small_cases()
|
||||
tiny_grid = {"salience_top_k": (16,), "inhibition_threshold": (0.3,)}
|
||||
calibrate(cases, grid=tiny_grid)
|
||||
|
||||
for f in sorted(pack_dir.rglob("*.jsonl")):
|
||||
assert before[str(f)] == hashlib.sha256(f.read_bytes()).hexdigest()
|
||||
Loading…
Reference in a new issue