Add cognitive eval harness and calibration replay (#30)

* feat: add cognitive eval harness with CLI integration

20 eval cases across 8 categories (definition, comparison, cause,
procedure, recall, correction, verification, unknown). Metrics:
intent accuracy, term capture, surface groundedness, versor closure,
trace determinism. CLI: `core eval cognition [--json] [--report PATH]`.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* feat: add operator calibration replay with deterministic grid search

Bounded parameter tuning via eval replay evidence. Grid search over
salience_top_k and inhibition_threshold with invariant regression
guard (versor closure must not regress). Frozen CalibrationParams,
before/after metrics, no pack or identity mutation.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

calibration/params.py

@@ -0,0 +1,52 @@
+"""Calibration parameter space — bounded, deterministic, immutable."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from itertools import product
+
+
+@dataclass(frozen=True, slots=True)
+class CalibrationParams:
+    salience_top_k: int = 16
+    inhibition_threshold: float = 0.3
+    teaching_retrieval_limit: int = 8
+
+    def as_dict(self) -> dict[str, int | float]:
+        return {
+            "salience_top_k": self.salience_top_k,
+            "inhibition_threshold": self.inhibition_threshold,
+            "teaching_retrieval_limit": self.teaching_retrieval_limit,
+        }
+
+
+DEFAULT_PARAMS = CalibrationParams()
+
+PARAM_GRID: dict[str, tuple] = {
+    "salience_top_k": (8, 12, 16),
+    "inhibition_threshold": (0.2, 0.3, 0.4),
+}
+
+
+def grid_candidates(
+    grid: dict[str, tuple] | None = None,
+    base: CalibrationParams = DEFAULT_PARAMS,
+) -> tuple[CalibrationParams, ...]:
+    """Generate all candidate parameter sets from a grid.
+
+    Each candidate varies exactly one axis from the base; the grid is
+    a deterministic Cartesian product over the provided axes.
+    """
+    g = grid or PARAM_GRID
+    keys = sorted(g.keys())
+    values = [g[k] for k in keys]
+    candidates = []
+    for combo in product(*values):
+        overrides = dict(zip(keys, combo))
+        candidate = CalibrationParams(
+            salience_top_k=overrides.get("salience_top_k", base.salience_top_k),
+            inhibition_threshold=overrides.get("inhibition_threshold", base.inhibition_threshold),
+            teaching_retrieval_limit=base.teaching_retrieval_limit,
+        )
+        candidates.append(candidate)
+    return tuple(candidates)

calibration/replay.py

@@ -0,0 +1,36 @@
+"""Replay eval cases under a given parameter set and return metrics."""
+
+from __future__ import annotations
+
+from core.config import RuntimeConfig, DEFAULT_CONFIG
+from calibration.params import CalibrationParams
+from evals.metrics import EvalReport
+from evals.run_cognition_eval import load_cases, run_eval
+
+
+def replay_with_params(
+    params: CalibrationParams,
+    cases: list[dict] | None = None,
+) -> EvalReport:
+    """Run the eval harness under a specific parameter configuration.
+
+    Builds a RuntimeConfig from the CalibrationParams and passes it
+    through to run_eval, which creates fresh ChatRuntime instances
+    per case.
+    """
+    if cases is None:
+        cases = load_cases()
+
+    config = RuntimeConfig(
+        input_packs=DEFAULT_CONFIG.input_packs,
+        output_language=DEFAULT_CONFIG.output_language,
+        frame_pack=DEFAULT_CONFIG.frame_pack,
+        max_tokens=DEFAULT_CONFIG.max_tokens,
+        allow_cross_language_recall=DEFAULT_CONFIG.allow_cross_language_recall,
+        allow_cross_language_generation=DEFAULT_CONFIG.allow_cross_language_generation,
+        vault_reproject_interval=DEFAULT_CONFIG.vault_reproject_interval,
+        use_salience=DEFAULT_CONFIG.use_salience,
+        salience_top_k=params.salience_top_k,
+        inhibition_threshold=params.inhibition_threshold,
+    )
+    return run_eval(cases, config=config)

calibration/report.py

@@ -0,0 +1,31 @@
+"""Calibration report generation."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from calibration.tune import CalibrationResult
+
+
+def write_report(result: CalibrationResult, path: str | Path) -> Path:
+    p = Path(path)
+    p.parent.mkdir(parents=True, exist_ok=True)
+    p.write_text(json.dumps(result.as_dict(), ensure_ascii=False, indent=2, sort_keys=True))
+    return p
+
+
+def print_report(result: CalibrationResult) -> None:
+    bl = result.baseline_report
+    br = result.best_report
+    print("=== Calibration Report ===")
+    print(f"baseline: {result.baseline_params.as_dict()}")
+    print(f"  intent_accuracy  : {bl.intent_accuracy:.1%}")
+    print(f"  versor_closure   : {bl.versor_closure_rate:.1%}")
+    print(f"  surface_ground   : {bl.surface_groundedness:.1%}")
+    print(f"best    : {result.best_params.as_dict()}")
+    print(f"  intent_accuracy  : {br.intent_accuracy:.1%}")
+    print(f"  versor_closure   : {br.versor_closure_rate:.1%}")
+    print(f"  surface_ground   : {br.surface_groundedness:.1%}")
+    print(f"candidates evaluated: {len(result.candidates)}")
+    print(f"candidates accepted : {sum(1 for c in result.candidates if c.accepted)}")

calibration/tune.py

@@ -0,0 +1,115 @@
+"""Deterministic grid-search calibration over bounded parameter sets."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from calibration.params import CalibrationParams, DEFAULT_PARAMS, grid_candidates
+from calibration.replay import replay_with_params
+from evals.metrics import EvalReport
+
+
+@dataclass(frozen=True, slots=True)
+class CalibrationCandidate:
+    params: CalibrationParams
+    before_report: EvalReport
+    after_report: EvalReport
+    accepted: bool
+    rejection_reason: str | None = None
+
+    def improvement(self) -> float:
+        return self.after_report.intent_accuracy - self.before_report.intent_accuracy
+
+
+@dataclass(frozen=True, slots=True)
+class CalibrationResult:
+    baseline_params: CalibrationParams
+    baseline_report: EvalReport
+    candidates: tuple[CalibrationCandidate, ...]
+    best_params: CalibrationParams
+    best_report: EvalReport
+
+    def as_dict(self) -> dict:
+        return {
+            "baseline_params": self.baseline_params.as_dict(),
+            "baseline_metrics": {
+                "intent_accuracy": round(self.baseline_report.intent_accuracy, 4),
+                "versor_closure_rate": round(self.baseline_report.versor_closure_rate, 4),
+                "surface_groundedness": round(self.baseline_report.surface_groundedness, 4),
+            },
+            "best_params": self.best_params.as_dict(),
+            "best_metrics": {
+                "intent_accuracy": round(self.best_report.intent_accuracy, 4),
+                "versor_closure_rate": round(self.best_report.versor_closure_rate, 4),
+                "surface_groundedness": round(self.best_report.surface_groundedness, 4),
+            },
+            "candidates_evaluated": len(self.candidates),
+            "candidates_accepted": sum(1 for c in self.candidates if c.accepted),
+        }
+
+
+def _score(report: EvalReport) -> float:
+    """Composite score: intent accuracy + versor closure + surface groundedness."""
+    return (
+        report.intent_accuracy
+        + report.versor_closure_rate
+        + report.surface_groundedness
+    ) / 3.0
+
+
+def calibrate(
+    cases: list[dict] | None = None,
+    baseline: CalibrationParams = DEFAULT_PARAMS,
+    grid: dict[str, tuple] | None = None,
+) -> CalibrationResult:
+    """Run deterministic grid-search calibration.
+
+    1. Evaluate baseline params
+    2. For each candidate in the grid, evaluate and compare
+    3. Accept only if no invariant regression (versor closure stays 100%)
+    4. Return the best accepted candidate
+    """
+    baseline_report = replay_with_params(baseline, cases)
+    baseline_score = _score(baseline_report)
+
+    candidates_list: list[CalibrationCandidate] = []
+    best_params = baseline
+    best_report = baseline_report
+    best_score = baseline_score
+
+    for params in grid_candidates(grid, baseline):
+        report = replay_with_params(params, cases)
+
+        rejection_reason = None
+        accepted = True
+
+        if report.versor_closure_rate < baseline_report.versor_closure_rate:
+            accepted = False
+            rejection_reason = "versor closure regression"
+
+        score = _score(report)
+        if accepted and score <= baseline_score:
+            accepted = False
+            rejection_reason = "no composite improvement"
+
+        candidate = CalibrationCandidate(
+            params=params,
+            before_report=baseline_report,
+            after_report=report,
+            accepted=accepted,
+            rejection_reason=rejection_reason,
+        )
+        candidates_list.append(candidate)
+
+        if accepted and score > best_score:
+            best_params = params
+            best_report = report
+            best_score = score
+
+    return CalibrationResult(
+        baseline_params=baseline,
+        baseline_report=baseline_report,
+        candidates=tuple(candidates_list),
+        best_params=best_params,
+        best_report=best_report,
+    )

core/cli.py

@@ -23,7 +23,7 @@ _CORE_RS_DIR = _REPO_ROOT / "core-rs"
 _CORE_RS_MANIFEST = _CORE_RS_DIR / "Cargo.toml"
 
 DESCRIPTION = "CORE versor engine command suite."
-EPILOG = "Examples:\n  core chat\n  core trace \"word beginning truth\"\n  core trace --output-language grc --frame-pack grc --json \"logos\"\n  core rust status\n  core rust build\n  core oov covenant\n  core pack list\n  core pack verify en_minimal_v1\n  core test --suite smoke -q\n  core test --suite cognition -q\n  core test -- tests/test_alignment_graph.py -q"
+EPILOG = "Examples:\n  core chat\n  core trace \"word beginning truth\"\n  core trace --output-language grc --frame-pack grc --json \"logos\"\n  core rust status\n  core rust build\n  core oov covenant\n  core pack list\n  core pack verify en_minimal_v1\n  core test --suite smoke -q\n  core test --suite cognition -q\n  core test -- tests/test_alignment_graph.py -q\n  core eval cognition\n  core eval cognition --json"
 
 _TEST_SUITES: dict[str, tuple[str, ...]] = {
     "smoke": (
@@ -44,6 +44,7 @@ _TEST_SUITES: dict[str, tuple[str, ...]] = {
         "tests/test_cognitive_turn_pipeline.py",
         "tests/test_articulation_realizer_v2.py",
         "tests/test_semantic_realizer_integration.py",
+        "tests/test_cognitive_eval_harness.py",
     ),
     "teaching": (
         "tests/test_reviewed_teaching_loop.py",
@@ -450,6 +451,45 @@ def cmd_doctor(args: argparse.Namespace) -> int:
     return 0 if ok else 1
 
 
+def cmd_eval_cognition(args: argparse.Namespace) -> int:
+    """Run the cognition eval harness."""
+    from evals.run_cognition_eval import load_cases, run_eval
+
+    cases = load_cases()
+    report = run_eval(cases)
+
+    if args.json:
+        print(json.dumps(report.as_dict(), ensure_ascii=False, indent=2, sort_keys=True))
+    else:
+        print(f"cases          : {report.total}")
+        print(f"intent_accuracy: {report.intent_accuracy:.1%}")
+        print(f"term_capture   : {report.term_capture_rate:.1%}")
+        print(f"surface_ground : {report.surface_groundedness:.1%}")
+        print(f"versor_closure : {report.versor_closure_rate:.1%}")
+        print(f"det_traces     : {report.deterministic_traces}")
+        failures = [c for c in report.cases if not c.intent_correct or not c.versor_closure]
+        if failures:
+            print(f"\nfailures ({len(failures)}):")
+            for c in failures:
+                issues = []
+                if not c.intent_correct:
+                    issues.append("intent")
+                if not c.versor_closure:
+                    issues.append(f"versor={c.versor_condition:.2e}")
+                print(f"  {c.case_id}: {', '.join(issues)}")
+
+    if args.report:
+        report_path = Path(args.report)
+        report_path.parent.mkdir(parents=True, exist_ok=True)
+        report_path.write_text(
+            json.dumps(report.as_dict(), ensure_ascii=False, indent=2, sort_keys=True)
+        )
+        print(f"\nreport written: {report_path}")
+
+    all_pass = report.intent_accuracy == 1.0 and report.versor_closure_rate == 1.0
+    return 0 if all_pass else 1
+
+
 def _add_runtime_policy_args(parser: argparse.ArgumentParser) -> None:
     parser.add_argument("--pack", action="append", help="language pack to mount; repeat for multiple packs")
     parser.add_argument("--output-language", default="en", help="target output language code; default: en")
@@ -537,6 +577,13 @@ def build_parser() -> argparse.ArgumentParser:
     rust_test = rust_sub.add_parser("test", help="run cargo test --release for core-rs")
     rust_test.set_defaults(func=cmd_rust_test)
 
+    eval_cmd = subparsers.add_parser("eval", help="run eval harnesses")
+    eval_sub = eval_cmd.add_subparsers(dest="eval_command", metavar="eval-command", required=True)
+    eval_cognition = eval_sub.add_parser("cognition", help="run the cognition eval harness")
+    eval_cognition.add_argument("--json", action="store_true", help="emit machine-readable JSON")
+    eval_cognition.add_argument("--report", metavar="PATH", help="write JSON report to file")
+    eval_cognition.set_defaults(func=cmd_eval_cognition)
+
     doctor = subparsers.add_parser("doctor", help="check runtime imports and packaging health")
     doctor.add_argument("--packs", action="store_true", help="also list discovered language packs")
     doctor.add_argument("--rust", action="store_true", help="also show Rust backend activation status")

evals/cognition_cases.jsonl

@@ -0,0 +1,20 @@
+{"id": "definition_truth_001", "category": "definition", "prompt": "What is truth?", "expected_intent": "definition", "expected_terms": ["truth"], "expected_surface_contains": ["truth"], "requires_versor_closure": true, "requires_deterministic_trace": true}
+{"id": "definition_light_002", "category": "definition", "prompt": "What is light?", "expected_intent": "definition", "expected_terms": ["light"], "expected_surface_contains": ["light"], "requires_versor_closure": true, "requires_deterministic_trace": true}
+{"id": "definition_knowledge_003", "category": "definition", "prompt": "What is knowledge?", "expected_intent": "definition", "expected_terms": ["knowledge"], "expected_surface_contains": ["knowledge"], "requires_versor_closure": true, "requires_deterministic_trace": true}
+{"id": "definition_meaning_004", "category": "definition", "prompt": "What is meaning?", "expected_intent": "definition", "expected_terms": ["meaning"], "expected_surface_contains": ["meaning"], "requires_versor_closure": true, "requires_deterministic_trace": true}
+{"id": "comparison_truth_light_005", "category": "comparison", "prompt": "Compare truth and light", "expected_intent": "comparison", "expected_terms": ["truth", "light"], "expected_surface_contains": ["truth", "light"], "requires_versor_closure": true, "requires_deterministic_trace": true}
+{"id": "comparison_word_meaning_006", "category": "comparison", "prompt": "Compare word and meaning", "expected_intent": "comparison", "expected_terms": ["word", "meaning"], "expected_surface_contains": ["word", "meaning"], "requires_versor_closure": true, "requires_deterministic_trace": true}
+{"id": "cause_light_007", "category": "cause", "prompt": "Why does light exist?", "expected_intent": "cause", "expected_terms": ["light"], "expected_surface_contains": ["light"], "requires_versor_closure": true, "requires_deterministic_trace": true}
+{"id": "cause_creation_008", "category": "cause", "prompt": "Why does creation matter?", "expected_intent": "cause", "expected_terms": ["creation"], "expected_surface_contains": ["creation"], "requires_versor_closure": true, "requires_deterministic_trace": true}
+{"id": "cause_wisdom_009", "category": "cause", "prompt": "Why does wisdom matter?", "expected_intent": "cause", "expected_terms": ["wisdom"], "expected_surface_contains": ["wisdom"], "requires_versor_closure": true, "requires_deterministic_trace": true}
+{"id": "procedure_define_010", "category": "procedure", "prompt": "How do I define a concept?", "expected_intent": "procedure", "expected_terms": ["concept"], "expected_surface_contains": ["concept"], "requires_versor_closure": true, "requires_deterministic_trace": true}
+{"id": "procedure_compare_011", "category": "procedure", "prompt": "How do I compare two terms?", "expected_intent": "procedure", "expected_terms": ["terms"], "expected_surface_contains": [], "requires_versor_closure": true, "requires_deterministic_trace": true}
+{"id": "recall_truth_012", "category": "recall", "prompt": "Remember truth", "expected_intent": "recall", "expected_terms": ["truth"], "expected_surface_contains": ["truth"], "requires_versor_closure": true, "requires_deterministic_trace": true}
+{"id": "recall_light_013", "category": "recall", "prompt": "Remember light", "expected_intent": "recall", "expected_terms": ["light"], "expected_surface_contains": ["light"], "requires_versor_closure": true, "requires_deterministic_trace": true}
+{"id": "correction_basic_014", "category": "correction", "prompt": "No, that's wrong", "expected_intent": "correction", "expected_terms": [], "expected_surface_contains": ["correction"], "requires_versor_closure": true, "requires_deterministic_trace": true}
+{"id": "correction_specific_015", "category": "correction", "prompt": "No, correction means reviewed repair", "expected_intent": "correction", "expected_terms": ["correction"], "expected_surface_contains": ["correction"], "requires_versor_closure": true, "requires_deterministic_trace": true}
+{"id": "verification_truth_016", "category": "verification", "prompt": "Is truth coherent?", "expected_intent": "verification", "expected_terms": ["truth"], "expected_surface_contains": ["truth"], "requires_versor_closure": true, "requires_deterministic_trace": true}
+{"id": "verification_light_017", "category": "verification", "prompt": "Is light real?", "expected_intent": "verification", "expected_terms": ["light"], "expected_surface_contains": ["light"], "requires_versor_closure": true, "requires_deterministic_trace": true}
+{"id": "unknown_word_018", "category": "unknown", "prompt": "word beginning truth", "expected_intent": "unknown", "expected_terms": ["word", "truth"], "expected_surface_contains": [], "requires_versor_closure": true, "requires_deterministic_trace": true}
+{"id": "unknown_logos_019", "category": "unknown", "prompt": "light logos", "expected_intent": "unknown", "expected_terms": ["light"], "expected_surface_contains": [], "requires_versor_closure": true, "requires_deterministic_trace": true}
+{"id": "definition_wisdom_020", "category": "definition", "prompt": "What is wisdom?", "expected_intent": "definition", "expected_terms": ["wisdom"], "expected_surface_contains": ["wisdom"], "requires_versor_closure": true, "requires_deterministic_trace": true}

evals/metrics.py

@@ -0,0 +1,73 @@
+"""Cognition eval metrics — deterministic, compact measurements."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True, slots=True)
+class CaseResult:
+    case_id: str
+    category: str
+    prompt: str
+    intent_correct: bool
+    terms_captured: tuple[str, ...]
+    terms_expected: tuple[str, ...]
+    surface_contains_pass: bool
+    versor_closure: bool
+    versor_condition: float
+    trace_hash: str
+    surface: str
+
+
+@dataclass(slots=True)
+class EvalReport:
+    total: int = 0
+    intent_correct: int = 0
+    terms_captured: int = 0
+    terms_expected: int = 0
+    surface_grounded: int = 0
+    versor_closures: int = 0
+    deterministic_traces: int = 0
+    cases: list[CaseResult] = field(default_factory=list)
+    trace_hashes: dict[str, str] = field(default_factory=dict)
+
+    @property
+    def intent_accuracy(self) -> float:
+        return self.intent_correct / self.total if self.total else 0.0
+
+    @property
+    def term_capture_rate(self) -> float:
+        return self.terms_captured / self.terms_expected if self.terms_expected else 1.0
+
+    @property
+    def surface_groundedness(self) -> float:
+        return self.surface_grounded / self.total if self.total else 0.0
+
+    @property
+    def versor_closure_rate(self) -> float:
+        return self.versor_closures / self.total if self.total else 0.0
+
+    def as_dict(self) -> dict:
+        return {
+            "total": self.total,
+            "intent_accuracy": round(self.intent_accuracy, 4),
+            "term_capture_rate": round(self.term_capture_rate, 4),
+            "surface_groundedness": round(self.surface_groundedness, 4),
+            "versor_closure_rate": round(self.versor_closure_rate, 4),
+            "deterministic_traces": self.deterministic_traces,
+            "trace_hashes": dict(self.trace_hashes),
+            "cases": [
+                {
+                    "case_id": c.case_id,
+                    "category": c.category,
+                    "intent_correct": c.intent_correct,
+                    "surface_contains_pass": c.surface_contains_pass,
+                    "versor_closure": c.versor_closure,
+                    "versor_condition": round(c.versor_condition, 9),
+                    "trace_hash": c.trace_hash,
+                    "surface": c.surface,
+                }
+                for c in self.cases
+            ],
+        }

evals/run_cognition_eval.py

@@ -0,0 +1,110 @@
+"""Run the cognition eval harness.
+
+Loads cases from cognition_cases.jsonl, runs each through the
+CognitiveTurnPipeline, and produces an EvalReport with deterministic
+metrics. Each case gets a fresh pipeline instance for isolation.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from chat.runtime import ChatRuntime
+from core.config import RuntimeConfig
+from core.cognition.pipeline import CognitiveTurnPipeline
+from evals.metrics import CaseResult, EvalReport
+from generate.intent import IntentTag
+
+_CASES_PATH = Path(__file__).parent / "cognition_cases.jsonl"
+
+
+def load_cases(path: Path | None = None) -> list[dict]:
+    p = path or _CASES_PATH
+    cases = []
+    for line in p.read_text().splitlines():
+        line = line.strip()
+        if line:
+            cases.append(json.loads(line))
+    return cases
+
+
+def _run_case(case: dict, pipeline: CognitiveTurnPipeline) -> CaseResult:
+    prompt = case["prompt"]
+    expected_intent = case["expected_intent"]
+    expected_terms = case.get("expected_terms", [])
+    expected_surface_contains = case.get("expected_surface_contains", [])
+
+    result = pipeline.run(prompt, max_tokens=8)
+
+    actual_intent = result.intent.tag if result.intent else IntentTag.UNKNOWN
+    intent_correct = actual_intent.value == expected_intent
+
+    surface_lower = result.surface.lower()
+    terms_captured = tuple(
+        t for t in expected_terms if t.lower() in surface_lower
+    )
+    surface_contains_pass = all(
+        s.lower() in surface_lower for s in expected_surface_contains
+    )
+
+    versor_ok = result.versor_condition < 1e-6
+
+    return CaseResult(
+        case_id=case["id"],
+        category=case.get("category", "unknown"),
+        prompt=prompt,
+        intent_correct=intent_correct,
+        terms_captured=terms_captured,
+        terms_expected=tuple(expected_terms),
+        surface_contains_pass=surface_contains_pass,
+        versor_closure=versor_ok,
+        versor_condition=result.versor_condition,
+        trace_hash=result.trace_hash,
+        surface=result.surface,
+    )
+
+
+def run_eval(
+    cases: list[dict] | None = None,
+    config: RuntimeConfig | None = None,
+) -> EvalReport:
+    if cases is None:
+        cases = load_cases()
+
+    report = EvalReport()
+
+    for case in cases:
+        runtime = ChatRuntime(config=config) if config else ChatRuntime()
+        pipeline = CognitiveTurnPipeline(runtime)
+        case_result = _run_case(case, pipeline)
+
+        report.total += 1
+        if case_result.intent_correct:
+            report.intent_correct += 1
+        report.terms_expected += len(case_result.terms_expected)
+        report.terms_captured += len(case_result.terms_captured)
+        if case_result.surface_contains_pass:
+            report.surface_grounded += 1
+        if case_result.versor_closure:
+            report.versor_closures += 1
+        report.cases.append(case_result)
+        report.trace_hashes[case_result.case_id] = case_result.trace_hash
+
+    return report
+
+
+def check_determinism(cases: list[dict] | None = None, runs: int = 2) -> bool:
+    if cases is None:
+        cases = load_cases()
+
+    hashes_by_run: list[dict[str, str]] = []
+    for _ in range(runs):
+        report = run_eval(cases)
+        hashes_by_run.append(dict(report.trace_hashes))
+
+    first = hashes_by_run[0]
+    for run_hashes in hashes_by_run[1:]:
+        if run_hashes != first:
+            return False
+    return True

tests/test_cognitive_eval_harness.py

@@ -0,0 +1,106 @@
+"""Tests for the cognitive eval harness."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from evals.run_cognition_eval import load_cases, run_eval, check_determinism
+
+
+_CASES_PATH = Path(__file__).resolve().parent.parent / "evals" / "cognition_cases.jsonl"
+
+
+class TestCognitionEvalLoadsCases:
+    def test_loads_all_cases(self) -> None:
+        cases = load_cases(_CASES_PATH)
+        assert len(cases) >= 15
+        assert all("id" in c for c in cases)
+        assert all("prompt" in c for c in cases)
+        assert all("expected_intent" in c for c in cases)
+
+    def test_cases_have_valid_structure(self) -> None:
+        cases = load_cases(_CASES_PATH)
+        for case in cases:
+            assert isinstance(case["id"], str)
+            assert isinstance(case["prompt"], str)
+            assert case["expected_intent"] in {
+                "definition", "comparison", "cause", "procedure",
+                "recall", "correction", "verification", "unknown",
+            }
+            assert isinstance(case.get("expected_terms", []), list)
+
+    def test_cases_cover_required_categories(self) -> None:
+        cases = load_cases(_CASES_PATH)
+        categories = {c.get("category", "unknown") for c in cases}
+        required = {"definition", "comparison", "cause", "correction", "verification", "unknown"}
+        assert required.issubset(categories), f"missing: {required - categories}"
+
+
+class TestCognitionEvalRunsSmallCaseSet:
+    def test_runs_single_case(self) -> None:
+        cases = load_cases(_CASES_PATH)[:1]
+        report = run_eval(cases)
+        assert report.total == 1
+        assert len(report.cases) == 1
+        assert report.cases[0].case_id == cases[0]["id"]
+
+    def test_runs_five_cases(self) -> None:
+        cases = load_cases(_CASES_PATH)[:5]
+        report = run_eval(cases)
+        assert report.total == 5
+        assert len(report.cases) == 5
+
+
+class TestCognitionEvalRecordsIntentAccuracy:
+    def test_definition_intent_detected(self) -> None:
+        cases = [c for c in load_cases(_CASES_PATH) if c["expected_intent"] == "definition"][:2]
+        report = run_eval(cases)
+        assert report.intent_correct == report.total
+
+    def test_comparison_intent_detected(self) -> None:
+        cases = [c for c in load_cases(_CASES_PATH) if c["expected_intent"] == "comparison"][:1]
+        report = run_eval(cases)
+        assert report.intent_correct == report.total
+
+    def test_report_has_accuracy_metric(self) -> None:
+        cases = load_cases(_CASES_PATH)[:3]
+        report = run_eval(cases)
+        assert 0.0 <= report.intent_accuracy <= 1.0
+        report_dict = report.as_dict()
+        assert "intent_accuracy" in report_dict
+
+
+class TestCognitionEvalRecordsTraceHashes:
+    def test_trace_hashes_present(self) -> None:
+        cases = load_cases(_CASES_PATH)[:3]
+        report = run_eval(cases)
+        assert len(report.trace_hashes) == 3
+        for case_id, h in report.trace_hashes.items():
+            assert isinstance(h, str)
+            assert len(h) == 64  # SHA-256 hex
+
+    def test_distinct_cases_get_distinct_hashes(self) -> None:
+        cases = load_cases(_CASES_PATH)[:5]
+        report = run_eval(cases)
+        hashes = list(report.trace_hashes.values())
+        assert len(set(hashes)) == len(hashes), "duplicate trace hashes"
+
+
+class TestCognitionEvalIsDeterministic:
+    def test_two_runs_same_hashes(self) -> None:
+        cases = load_cases(_CASES_PATH)[:3]
+        assert check_determinism(cases, runs=2)
+
+
+class TestEvalReportSerialization:
+    def test_as_dict_roundtrips(self) -> None:
+        cases = load_cases(_CASES_PATH)[:2]
+        report = run_eval(cases)
+        d = report.as_dict()
+        serialized = json.dumps(d)
+        parsed = json.loads(serialized)
+        assert parsed["total"] == 2
+        assert "intent_accuracy" in parsed
+        assert "trace_hashes" in parsed
+        assert len(parsed["cases"]) == 2

tests/test_operator_calibration_replay.py

@@ -0,0 +1,111 @@
+"""Tests for the operator calibration replay system."""
+
+from __future__ import annotations
+
+from calibration.params import (
+    CalibrationParams,
+    DEFAULT_PARAMS,
+    grid_candidates,
+)
+from calibration.replay import replay_with_params
+from calibration.tune import calibrate, CalibrationResult
+from evals.run_cognition_eval import load_cases
+
+
+_SMALL_CASES = None
+
+
+def _get_small_cases() -> list[dict]:
+    global _SMALL_CASES
+    if _SMALL_CASES is None:
+        _SMALL_CASES = load_cases()[:3]
+    return _SMALL_CASES
+
+
+class TestCalibrationReplayIsDeterministic:
+    def test_same_params_same_metrics(self) -> None:
+        cases = _get_small_cases()
+        r1 = replay_with_params(DEFAULT_PARAMS, cases)
+        r2 = replay_with_params(DEFAULT_PARAMS, cases)
+        assert r1.intent_accuracy == r2.intent_accuracy
+        assert r1.versor_closure_rate == r2.versor_closure_rate
+        assert r1.trace_hashes == r2.trace_hashes
+
+
+class TestCalibrationCandidateParamsAreBounded:
+    def test_grid_produces_bounded_candidates(self) -> None:
+        candidates = grid_candidates()
+        assert len(candidates) > 0
+        for c in candidates:
+            assert 4 <= c.salience_top_k <= 32
+            assert 0.1 <= c.inhibition_threshold <= 0.9
+
+    def test_grid_is_deterministic(self) -> None:
+        c1 = grid_candidates()
+        c2 = grid_candidates()
+        assert c1 == c2
+
+    def test_custom_grid(self) -> None:
+        custom = {"salience_top_k": (8, 16), "inhibition_threshold": (0.2,)}
+        candidates = grid_candidates(custom)
+        assert len(candidates) == 2
+        salience_values = {c.salience_top_k for c in candidates}
+        assert salience_values == {8, 16}
+
+
+class TestCalibrationReportHasBeforeAfterMetrics:
+    def test_calibrate_returns_result(self) -> None:
+        cases = _get_small_cases()
+        tiny_grid = {"salience_top_k": (12, 16), "inhibition_threshold": (0.3,)}
+        result = calibrate(cases, grid=tiny_grid)
+        assert isinstance(result, CalibrationResult)
+        assert result.baseline_report.total == len(cases)
+        assert result.best_report.total == len(cases)
+
+    def test_report_dict_has_required_fields(self) -> None:
+        cases = _get_small_cases()
+        tiny_grid = {"salience_top_k": (16,), "inhibition_threshold": (0.3,)}
+        result = calibrate(cases, grid=tiny_grid)
+        d = result.as_dict()
+        assert "baseline_params" in d
+        assert "baseline_metrics" in d
+        assert "best_params" in d
+        assert "best_metrics" in d
+        assert "candidates_evaluated" in d
+        assert "candidates_accepted" in d
+
+
+class TestCalibrationRejectsInvariantRegression:
+    def test_versor_closure_must_not_regress(self) -> None:
+        cases = _get_small_cases()
+        tiny_grid = {"salience_top_k": (8, 12, 16), "inhibition_threshold": (0.2, 0.3, 0.4)}
+        result = calibrate(cases, grid=tiny_grid)
+        for c in result.candidates:
+            if c.accepted:
+                assert c.after_report.versor_closure_rate >= result.baseline_report.versor_closure_rate
+
+
+class TestCalibrationDoesNotMutateIdentityOrPacks:
+    def test_params_are_frozen(self) -> None:
+        params = CalibrationParams()
+        try:
+            params.salience_top_k = 99  # type: ignore[misc]
+            raise AssertionError("CalibrationParams should be frozen")
+        except AttributeError:
+            pass
+
+    def test_calibration_does_not_touch_packs(self) -> None:
+        import hashlib
+        from pathlib import Path
+
+        pack_dir = Path(__file__).resolve().parent.parent / "language_packs" / "data"
+        before = {}
+        for f in sorted(pack_dir.rglob("*.jsonl")):
+            before[str(f)] = hashlib.sha256(f.read_bytes()).hexdigest()
+
+        cases = _get_small_cases()
+        tiny_grid = {"salience_top_k": (16,), "inhibition_threshold": (0.3,)}
+        calibrate(cases, grid=tiny_grid)
+
+        for f in sorted(pack_dir.rglob("*.jsonl")):
+            assert before[str(f)] == hashlib.sha256(f.read_bytes()).hexdigest()