366f7a08c4
* feat: add cognitive eval harness with CLI integration 20 eval cases across 8 categories (definition, comparison, cause, procedure, recall, correction, verification, unknown). Metrics: intent accuracy, term capture, surface groundedness, versor closure, trace determinism. CLI: `core eval cognition [--json] [--report PATH]`. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * feat: add operator calibration replay with deterministic grid search Bounded parameter tuning via eval replay evidence. Grid search over salience_top_k and inhibition_threshold with invariant regression guard (versor closure must not regress). Frozen CalibrationParams, before/after metrics, no pack or identity mutation. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
106 lines
3.8 KiB
Python
106 lines
3.8 KiB
Python
"""Tests for the cognitive eval harness."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from evals.run_cognition_eval import load_cases, run_eval, check_determinism
|
|
|
|
|
|
_CASES_PATH = Path(__file__).resolve().parent.parent / "evals" / "cognition_cases.jsonl"
|
|
|
|
|
|
class TestCognitionEvalLoadsCases:
|
|
def test_loads_all_cases(self) -> None:
|
|
cases = load_cases(_CASES_PATH)
|
|
assert len(cases) >= 15
|
|
assert all("id" in c for c in cases)
|
|
assert all("prompt" in c for c in cases)
|
|
assert all("expected_intent" in c for c in cases)
|
|
|
|
def test_cases_have_valid_structure(self) -> None:
|
|
cases = load_cases(_CASES_PATH)
|
|
for case in cases:
|
|
assert isinstance(case["id"], str)
|
|
assert isinstance(case["prompt"], str)
|
|
assert case["expected_intent"] in {
|
|
"definition", "comparison", "cause", "procedure",
|
|
"recall", "correction", "verification", "unknown",
|
|
}
|
|
assert isinstance(case.get("expected_terms", []), list)
|
|
|
|
def test_cases_cover_required_categories(self) -> None:
|
|
cases = load_cases(_CASES_PATH)
|
|
categories = {c.get("category", "unknown") for c in cases}
|
|
required = {"definition", "comparison", "cause", "correction", "verification", "unknown"}
|
|
assert required.issubset(categories), f"missing: {required - categories}"
|
|
|
|
|
|
class TestCognitionEvalRunsSmallCaseSet:
|
|
def test_runs_single_case(self) -> None:
|
|
cases = load_cases(_CASES_PATH)[:1]
|
|
report = run_eval(cases)
|
|
assert report.total == 1
|
|
assert len(report.cases) == 1
|
|
assert report.cases[0].case_id == cases[0]["id"]
|
|
|
|
def test_runs_five_cases(self) -> None:
|
|
cases = load_cases(_CASES_PATH)[:5]
|
|
report = run_eval(cases)
|
|
assert report.total == 5
|
|
assert len(report.cases) == 5
|
|
|
|
|
|
class TestCognitionEvalRecordsIntentAccuracy:
|
|
def test_definition_intent_detected(self) -> None:
|
|
cases = [c for c in load_cases(_CASES_PATH) if c["expected_intent"] == "definition"][:2]
|
|
report = run_eval(cases)
|
|
assert report.intent_correct == report.total
|
|
|
|
def test_comparison_intent_detected(self) -> None:
|
|
cases = [c for c in load_cases(_CASES_PATH) if c["expected_intent"] == "comparison"][:1]
|
|
report = run_eval(cases)
|
|
assert report.intent_correct == report.total
|
|
|
|
def test_report_has_accuracy_metric(self) -> None:
|
|
cases = load_cases(_CASES_PATH)[:3]
|
|
report = run_eval(cases)
|
|
assert 0.0 <= report.intent_accuracy <= 1.0
|
|
report_dict = report.as_dict()
|
|
assert "intent_accuracy" in report_dict
|
|
|
|
|
|
class TestCognitionEvalRecordsTraceHashes:
|
|
def test_trace_hashes_present(self) -> None:
|
|
cases = load_cases(_CASES_PATH)[:3]
|
|
report = run_eval(cases)
|
|
assert len(report.trace_hashes) == 3
|
|
for case_id, h in report.trace_hashes.items():
|
|
assert isinstance(h, str)
|
|
assert len(h) == 64 # SHA-256 hex
|
|
|
|
def test_distinct_cases_get_distinct_hashes(self) -> None:
|
|
cases = load_cases(_CASES_PATH)[:5]
|
|
report = run_eval(cases)
|
|
hashes = list(report.trace_hashes.values())
|
|
assert len(set(hashes)) == len(hashes), "duplicate trace hashes"
|
|
|
|
|
|
class TestCognitionEvalIsDeterministic:
|
|
def test_two_runs_same_hashes(self) -> None:
|
|
cases = load_cases(_CASES_PATH)[:3]
|
|
assert check_determinism(cases, runs=2)
|
|
|
|
|
|
class TestEvalReportSerialization:
|
|
def test_as_dict_roundtrips(self) -> None:
|
|
cases = load_cases(_CASES_PATH)[:2]
|
|
report = run_eval(cases)
|
|
d = report.as_dict()
|
|
serialized = json.dumps(d)
|
|
parsed = json.loads(serialized)
|
|
assert parsed["total"] == 2
|
|
assert "intent_accuracy" in parsed
|
|
assert "trace_hashes" in parsed
|
|
assert len(parsed["cases"]) == 2
|