diff --git a/calibration/__init__.py b/calibration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/calibration/params.py b/calibration/params.py new file mode 100644 index 00000000..aad7f3b7 --- /dev/null +++ b/calibration/params.py @@ -0,0 +1,52 @@ +"""Calibration parameter space — bounded, deterministic, immutable.""" + +from __future__ import annotations + +from dataclasses import dataclass +from itertools import product + + +@dataclass(frozen=True, slots=True) +class CalibrationParams: + salience_top_k: int = 16 + inhibition_threshold: float = 0.3 + teaching_retrieval_limit: int = 8 + + def as_dict(self) -> dict[str, int | float]: + return { + "salience_top_k": self.salience_top_k, + "inhibition_threshold": self.inhibition_threshold, + "teaching_retrieval_limit": self.teaching_retrieval_limit, + } + + +DEFAULT_PARAMS = CalibrationParams() + +PARAM_GRID: dict[str, tuple] = { + "salience_top_k": (8, 12, 16), + "inhibition_threshold": (0.2, 0.3, 0.4), +} + + +def grid_candidates( + grid: dict[str, tuple] | None = None, + base: CalibrationParams = DEFAULT_PARAMS, +) -> tuple[CalibrationParams, ...]: + """Generate all candidate parameter sets from a grid. + + Each candidate varies exactly one axis from the base; the grid is + a deterministic Cartesian product over the provided axes. + """ + g = grid or PARAM_GRID + keys = sorted(g.keys()) + values = [g[k] for k in keys] + candidates = [] + for combo in product(*values): + overrides = dict(zip(keys, combo)) + candidate = CalibrationParams( + salience_top_k=overrides.get("salience_top_k", base.salience_top_k), + inhibition_threshold=overrides.get("inhibition_threshold", base.inhibition_threshold), + teaching_retrieval_limit=base.teaching_retrieval_limit, + ) + candidates.append(candidate) + return tuple(candidates) diff --git a/calibration/replay.py b/calibration/replay.py new file mode 100644 index 00000000..20305593 --- /dev/null +++ b/calibration/replay.py @@ -0,0 +1,36 @@ +"""Replay eval cases under a given parameter set and return metrics.""" + +from __future__ import annotations + +from core.config import RuntimeConfig, DEFAULT_CONFIG +from calibration.params import CalibrationParams +from evals.metrics import EvalReport +from evals.run_cognition_eval import load_cases, run_eval + + +def replay_with_params( + params: CalibrationParams, + cases: list[dict] | None = None, +) -> EvalReport: + """Run the eval harness under a specific parameter configuration. + + Builds a RuntimeConfig from the CalibrationParams and passes it + through to run_eval, which creates fresh ChatRuntime instances + per case. + """ + if cases is None: + cases = load_cases() + + config = RuntimeConfig( + input_packs=DEFAULT_CONFIG.input_packs, + output_language=DEFAULT_CONFIG.output_language, + frame_pack=DEFAULT_CONFIG.frame_pack, + max_tokens=DEFAULT_CONFIG.max_tokens, + allow_cross_language_recall=DEFAULT_CONFIG.allow_cross_language_recall, + allow_cross_language_generation=DEFAULT_CONFIG.allow_cross_language_generation, + vault_reproject_interval=DEFAULT_CONFIG.vault_reproject_interval, + use_salience=DEFAULT_CONFIG.use_salience, + salience_top_k=params.salience_top_k, + inhibition_threshold=params.inhibition_threshold, + ) + return run_eval(cases, config=config) diff --git a/calibration/report.py b/calibration/report.py new file mode 100644 index 00000000..531d6886 --- /dev/null +++ b/calibration/report.py @@ -0,0 +1,31 @@ +"""Calibration report generation.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from calibration.tune import CalibrationResult + + +def write_report(result: CalibrationResult, path: str | Path) -> Path: + p = Path(path) + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(json.dumps(result.as_dict(), ensure_ascii=False, indent=2, sort_keys=True)) + return p + + +def print_report(result: CalibrationResult) -> None: + bl = result.baseline_report + br = result.best_report + print("=== Calibration Report ===") + print(f"baseline: {result.baseline_params.as_dict()}") + print(f" intent_accuracy : {bl.intent_accuracy:.1%}") + print(f" versor_closure : {bl.versor_closure_rate:.1%}") + print(f" surface_ground : {bl.surface_groundedness:.1%}") + print(f"best : {result.best_params.as_dict()}") + print(f" intent_accuracy : {br.intent_accuracy:.1%}") + print(f" versor_closure : {br.versor_closure_rate:.1%}") + print(f" surface_ground : {br.surface_groundedness:.1%}") + print(f"candidates evaluated: {len(result.candidates)}") + print(f"candidates accepted : {sum(1 for c in result.candidates if c.accepted)}") diff --git a/calibration/tune.py b/calibration/tune.py new file mode 100644 index 00000000..38355994 --- /dev/null +++ b/calibration/tune.py @@ -0,0 +1,115 @@ +"""Deterministic grid-search calibration over bounded parameter sets.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from calibration.params import CalibrationParams, DEFAULT_PARAMS, grid_candidates +from calibration.replay import replay_with_params +from evals.metrics import EvalReport + + +@dataclass(frozen=True, slots=True) +class CalibrationCandidate: + params: CalibrationParams + before_report: EvalReport + after_report: EvalReport + accepted: bool + rejection_reason: str | None = None + + def improvement(self) -> float: + return self.after_report.intent_accuracy - self.before_report.intent_accuracy + + +@dataclass(frozen=True, slots=True) +class CalibrationResult: + baseline_params: CalibrationParams + baseline_report: EvalReport + candidates: tuple[CalibrationCandidate, ...] + best_params: CalibrationParams + best_report: EvalReport + + def as_dict(self) -> dict: + return { + "baseline_params": self.baseline_params.as_dict(), + "baseline_metrics": { + "intent_accuracy": round(self.baseline_report.intent_accuracy, 4), + "versor_closure_rate": round(self.baseline_report.versor_closure_rate, 4), + "surface_groundedness": round(self.baseline_report.surface_groundedness, 4), + }, + "best_params": self.best_params.as_dict(), + "best_metrics": { + "intent_accuracy": round(self.best_report.intent_accuracy, 4), + "versor_closure_rate": round(self.best_report.versor_closure_rate, 4), + "surface_groundedness": round(self.best_report.surface_groundedness, 4), + }, + "candidates_evaluated": len(self.candidates), + "candidates_accepted": sum(1 for c in self.candidates if c.accepted), + } + + +def _score(report: EvalReport) -> float: + """Composite score: intent accuracy + versor closure + surface groundedness.""" + return ( + report.intent_accuracy + + report.versor_closure_rate + + report.surface_groundedness + ) / 3.0 + + +def calibrate( + cases: list[dict] | None = None, + baseline: CalibrationParams = DEFAULT_PARAMS, + grid: dict[str, tuple] | None = None, +) -> CalibrationResult: + """Run deterministic grid-search calibration. + + 1. Evaluate baseline params + 2. For each candidate in the grid, evaluate and compare + 3. Accept only if no invariant regression (versor closure stays 100%) + 4. Return the best accepted candidate + """ + baseline_report = replay_with_params(baseline, cases) + baseline_score = _score(baseline_report) + + candidates_list: list[CalibrationCandidate] = [] + best_params = baseline + best_report = baseline_report + best_score = baseline_score + + for params in grid_candidates(grid, baseline): + report = replay_with_params(params, cases) + + rejection_reason = None + accepted = True + + if report.versor_closure_rate < baseline_report.versor_closure_rate: + accepted = False + rejection_reason = "versor closure regression" + + score = _score(report) + if accepted and score <= baseline_score: + accepted = False + rejection_reason = "no composite improvement" + + candidate = CalibrationCandidate( + params=params, + before_report=baseline_report, + after_report=report, + accepted=accepted, + rejection_reason=rejection_reason, + ) + candidates_list.append(candidate) + + if accepted and score > best_score: + best_params = params + best_report = report + best_score = score + + return CalibrationResult( + baseline_params=baseline, + baseline_report=baseline_report, + candidates=tuple(candidates_list), + best_params=best_params, + best_report=best_report, + ) diff --git a/core/cli.py b/core/cli.py index fb36b711..1587b47d 100644 --- a/core/cli.py +++ b/core/cli.py @@ -23,7 +23,7 @@ _CORE_RS_DIR = _REPO_ROOT / "core-rs" _CORE_RS_MANIFEST = _CORE_RS_DIR / "Cargo.toml" DESCRIPTION = "CORE versor engine command suite." -EPILOG = "Examples:\n core chat\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core test --suite smoke -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q" +EPILOG = "Examples:\n core chat\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core test --suite smoke -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core eval cognition\n core eval cognition --json" _TEST_SUITES: dict[str, tuple[str, ...]] = { "smoke": ( @@ -44,6 +44,7 @@ _TEST_SUITES: dict[str, tuple[str, ...]] = { "tests/test_cognitive_turn_pipeline.py", "tests/test_articulation_realizer_v2.py", "tests/test_semantic_realizer_integration.py", + "tests/test_cognitive_eval_harness.py", ), "teaching": ( "tests/test_reviewed_teaching_loop.py", @@ -450,6 +451,45 @@ def cmd_doctor(args: argparse.Namespace) -> int: return 0 if ok else 1 +def cmd_eval_cognition(args: argparse.Namespace) -> int: + """Run the cognition eval harness.""" + from evals.run_cognition_eval import load_cases, run_eval + + cases = load_cases() + report = run_eval(cases) + + if args.json: + print(json.dumps(report.as_dict(), ensure_ascii=False, indent=2, sort_keys=True)) + else: + print(f"cases : {report.total}") + print(f"intent_accuracy: {report.intent_accuracy:.1%}") + print(f"term_capture : {report.term_capture_rate:.1%}") + print(f"surface_ground : {report.surface_groundedness:.1%}") + print(f"versor_closure : {report.versor_closure_rate:.1%}") + print(f"det_traces : {report.deterministic_traces}") + failures = [c for c in report.cases if not c.intent_correct or not c.versor_closure] + if failures: + print(f"\nfailures ({len(failures)}):") + for c in failures: + issues = [] + if not c.intent_correct: + issues.append("intent") + if not c.versor_closure: + issues.append(f"versor={c.versor_condition:.2e}") + print(f" {c.case_id}: {', '.join(issues)}") + + if args.report: + report_path = Path(args.report) + report_path.parent.mkdir(parents=True, exist_ok=True) + report_path.write_text( + json.dumps(report.as_dict(), ensure_ascii=False, indent=2, sort_keys=True) + ) + print(f"\nreport written: {report_path}") + + all_pass = report.intent_accuracy == 1.0 and report.versor_closure_rate == 1.0 + return 0 if all_pass else 1 + + def _add_runtime_policy_args(parser: argparse.ArgumentParser) -> None: parser.add_argument("--pack", action="append", help="language pack to mount; repeat for multiple packs") parser.add_argument("--output-language", default="en", help="target output language code; default: en") @@ -537,6 +577,13 @@ def build_parser() -> argparse.ArgumentParser: rust_test = rust_sub.add_parser("test", help="run cargo test --release for core-rs") rust_test.set_defaults(func=cmd_rust_test) + eval_cmd = subparsers.add_parser("eval", help="run eval harnesses") + eval_sub = eval_cmd.add_subparsers(dest="eval_command", metavar="eval-command", required=True) + eval_cognition = eval_sub.add_parser("cognition", help="run the cognition eval harness") + eval_cognition.add_argument("--json", action="store_true", help="emit machine-readable JSON") + eval_cognition.add_argument("--report", metavar="PATH", help="write JSON report to file") + eval_cognition.set_defaults(func=cmd_eval_cognition) + doctor = subparsers.add_parser("doctor", help="check runtime imports and packaging health") doctor.add_argument("--packs", action="store_true", help="also list discovered language packs") doctor.add_argument("--rust", action="store_true", help="also show Rust backend activation status") diff --git a/evals/__init__.py b/evals/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/cognition_cases.jsonl b/evals/cognition_cases.jsonl new file mode 100644 index 00000000..a3e40a06 --- /dev/null +++ b/evals/cognition_cases.jsonl @@ -0,0 +1,20 @@ +{"id": "definition_truth_001", "category": "definition", "prompt": "What is truth?", "expected_intent": "definition", "expected_terms": ["truth"], "expected_surface_contains": ["truth"], "requires_versor_closure": true, "requires_deterministic_trace": true} +{"id": "definition_light_002", "category": "definition", "prompt": "What is light?", "expected_intent": "definition", "expected_terms": ["light"], "expected_surface_contains": ["light"], "requires_versor_closure": true, "requires_deterministic_trace": true} +{"id": "definition_knowledge_003", "category": "definition", "prompt": "What is knowledge?", "expected_intent": "definition", "expected_terms": ["knowledge"], "expected_surface_contains": ["knowledge"], "requires_versor_closure": true, "requires_deterministic_trace": true} +{"id": "definition_meaning_004", "category": "definition", "prompt": "What is meaning?", "expected_intent": "definition", "expected_terms": ["meaning"], "expected_surface_contains": ["meaning"], "requires_versor_closure": true, "requires_deterministic_trace": true} +{"id": "comparison_truth_light_005", "category": "comparison", "prompt": "Compare truth and light", "expected_intent": "comparison", "expected_terms": ["truth", "light"], "expected_surface_contains": ["truth", "light"], "requires_versor_closure": true, "requires_deterministic_trace": true} +{"id": "comparison_word_meaning_006", "category": "comparison", "prompt": "Compare word and meaning", "expected_intent": "comparison", "expected_terms": ["word", "meaning"], "expected_surface_contains": ["word", "meaning"], "requires_versor_closure": true, "requires_deterministic_trace": true} +{"id": "cause_light_007", "category": "cause", "prompt": "Why does light exist?", "expected_intent": "cause", "expected_terms": ["light"], "expected_surface_contains": ["light"], "requires_versor_closure": true, "requires_deterministic_trace": true} +{"id": "cause_creation_008", "category": "cause", "prompt": "Why does creation matter?", "expected_intent": "cause", "expected_terms": ["creation"], "expected_surface_contains": ["creation"], "requires_versor_closure": true, "requires_deterministic_trace": true} +{"id": "cause_wisdom_009", "category": "cause", "prompt": "Why does wisdom matter?", "expected_intent": "cause", "expected_terms": ["wisdom"], "expected_surface_contains": ["wisdom"], "requires_versor_closure": true, "requires_deterministic_trace": true} +{"id": "procedure_define_010", "category": "procedure", "prompt": "How do I define a concept?", "expected_intent": "procedure", "expected_terms": ["concept"], "expected_surface_contains": ["concept"], "requires_versor_closure": true, "requires_deterministic_trace": true} +{"id": "procedure_compare_011", "category": "procedure", "prompt": "How do I compare two terms?", "expected_intent": "procedure", "expected_terms": ["terms"], "expected_surface_contains": [], "requires_versor_closure": true, "requires_deterministic_trace": true} +{"id": "recall_truth_012", "category": "recall", "prompt": "Remember truth", "expected_intent": "recall", "expected_terms": ["truth"], "expected_surface_contains": ["truth"], "requires_versor_closure": true, "requires_deterministic_trace": true} +{"id": "recall_light_013", "category": "recall", "prompt": "Remember light", "expected_intent": "recall", "expected_terms": ["light"], "expected_surface_contains": ["light"], "requires_versor_closure": true, "requires_deterministic_trace": true} +{"id": "correction_basic_014", "category": "correction", "prompt": "No, that's wrong", "expected_intent": "correction", "expected_terms": [], "expected_surface_contains": ["correction"], "requires_versor_closure": true, "requires_deterministic_trace": true} +{"id": "correction_specific_015", "category": "correction", "prompt": "No, correction means reviewed repair", "expected_intent": "correction", "expected_terms": ["correction"], "expected_surface_contains": ["correction"], "requires_versor_closure": true, "requires_deterministic_trace": true} +{"id": "verification_truth_016", "category": "verification", "prompt": "Is truth coherent?", "expected_intent": "verification", "expected_terms": ["truth"], "expected_surface_contains": ["truth"], "requires_versor_closure": true, "requires_deterministic_trace": true} +{"id": "verification_light_017", "category": "verification", "prompt": "Is light real?", "expected_intent": "verification", "expected_terms": ["light"], "expected_surface_contains": ["light"], "requires_versor_closure": true, "requires_deterministic_trace": true} +{"id": "unknown_word_018", "category": "unknown", "prompt": "word beginning truth", "expected_intent": "unknown", "expected_terms": ["word", "truth"], "expected_surface_contains": [], "requires_versor_closure": true, "requires_deterministic_trace": true} +{"id": "unknown_logos_019", "category": "unknown", "prompt": "light logos", "expected_intent": "unknown", "expected_terms": ["light"], "expected_surface_contains": [], "requires_versor_closure": true, "requires_deterministic_trace": true} +{"id": "definition_wisdom_020", "category": "definition", "prompt": "What is wisdom?", "expected_intent": "definition", "expected_terms": ["wisdom"], "expected_surface_contains": ["wisdom"], "requires_versor_closure": true, "requires_deterministic_trace": true} diff --git a/evals/metrics.py b/evals/metrics.py new file mode 100644 index 00000000..f46b1261 --- /dev/null +++ b/evals/metrics.py @@ -0,0 +1,73 @@ +"""Cognition eval metrics — deterministic, compact measurements.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass(frozen=True, slots=True) +class CaseResult: + case_id: str + category: str + prompt: str + intent_correct: bool + terms_captured: tuple[str, ...] + terms_expected: tuple[str, ...] + surface_contains_pass: bool + versor_closure: bool + versor_condition: float + trace_hash: str + surface: str + + +@dataclass(slots=True) +class EvalReport: + total: int = 0 + intent_correct: int = 0 + terms_captured: int = 0 + terms_expected: int = 0 + surface_grounded: int = 0 + versor_closures: int = 0 + deterministic_traces: int = 0 + cases: list[CaseResult] = field(default_factory=list) + trace_hashes: dict[str, str] = field(default_factory=dict) + + @property + def intent_accuracy(self) -> float: + return self.intent_correct / self.total if self.total else 0.0 + + @property + def term_capture_rate(self) -> float: + return self.terms_captured / self.terms_expected if self.terms_expected else 1.0 + + @property + def surface_groundedness(self) -> float: + return self.surface_grounded / self.total if self.total else 0.0 + + @property + def versor_closure_rate(self) -> float: + return self.versor_closures / self.total if self.total else 0.0 + + def as_dict(self) -> dict: + return { + "total": self.total, + "intent_accuracy": round(self.intent_accuracy, 4), + "term_capture_rate": round(self.term_capture_rate, 4), + "surface_groundedness": round(self.surface_groundedness, 4), + "versor_closure_rate": round(self.versor_closure_rate, 4), + "deterministic_traces": self.deterministic_traces, + "trace_hashes": dict(self.trace_hashes), + "cases": [ + { + "case_id": c.case_id, + "category": c.category, + "intent_correct": c.intent_correct, + "surface_contains_pass": c.surface_contains_pass, + "versor_closure": c.versor_closure, + "versor_condition": round(c.versor_condition, 9), + "trace_hash": c.trace_hash, + "surface": c.surface, + } + for c in self.cases + ], + } diff --git a/evals/reports/.gitkeep b/evals/reports/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/evals/run_cognition_eval.py b/evals/run_cognition_eval.py new file mode 100644 index 00000000..2e6776c7 --- /dev/null +++ b/evals/run_cognition_eval.py @@ -0,0 +1,110 @@ +"""Run the cognition eval harness. + +Loads cases from cognition_cases.jsonl, runs each through the +CognitiveTurnPipeline, and produces an EvalReport with deterministic +metrics. Each case gets a fresh pipeline instance for isolation. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from chat.runtime import ChatRuntime +from core.config import RuntimeConfig +from core.cognition.pipeline import CognitiveTurnPipeline +from evals.metrics import CaseResult, EvalReport +from generate.intent import IntentTag + +_CASES_PATH = Path(__file__).parent / "cognition_cases.jsonl" + + +def load_cases(path: Path | None = None) -> list[dict]: + p = path or _CASES_PATH + cases = [] + for line in p.read_text().splitlines(): + line = line.strip() + if line: + cases.append(json.loads(line)) + return cases + + +def _run_case(case: dict, pipeline: CognitiveTurnPipeline) -> CaseResult: + prompt = case["prompt"] + expected_intent = case["expected_intent"] + expected_terms = case.get("expected_terms", []) + expected_surface_contains = case.get("expected_surface_contains", []) + + result = pipeline.run(prompt, max_tokens=8) + + actual_intent = result.intent.tag if result.intent else IntentTag.UNKNOWN + intent_correct = actual_intent.value == expected_intent + + surface_lower = result.surface.lower() + terms_captured = tuple( + t for t in expected_terms if t.lower() in surface_lower + ) + surface_contains_pass = all( + s.lower() in surface_lower for s in expected_surface_contains + ) + + versor_ok = result.versor_condition < 1e-6 + + return CaseResult( + case_id=case["id"], + category=case.get("category", "unknown"), + prompt=prompt, + intent_correct=intent_correct, + terms_captured=terms_captured, + terms_expected=tuple(expected_terms), + surface_contains_pass=surface_contains_pass, + versor_closure=versor_ok, + versor_condition=result.versor_condition, + trace_hash=result.trace_hash, + surface=result.surface, + ) + + +def run_eval( + cases: list[dict] | None = None, + config: RuntimeConfig | None = None, +) -> EvalReport: + if cases is None: + cases = load_cases() + + report = EvalReport() + + for case in cases: + runtime = ChatRuntime(config=config) if config else ChatRuntime() + pipeline = CognitiveTurnPipeline(runtime) + case_result = _run_case(case, pipeline) + + report.total += 1 + if case_result.intent_correct: + report.intent_correct += 1 + report.terms_expected += len(case_result.terms_expected) + report.terms_captured += len(case_result.terms_captured) + if case_result.surface_contains_pass: + report.surface_grounded += 1 + if case_result.versor_closure: + report.versor_closures += 1 + report.cases.append(case_result) + report.trace_hashes[case_result.case_id] = case_result.trace_hash + + return report + + +def check_determinism(cases: list[dict] | None = None, runs: int = 2) -> bool: + if cases is None: + cases = load_cases() + + hashes_by_run: list[dict[str, str]] = [] + for _ in range(runs): + report = run_eval(cases) + hashes_by_run.append(dict(report.trace_hashes)) + + first = hashes_by_run[0] + for run_hashes in hashes_by_run[1:]: + if run_hashes != first: + return False + return True diff --git a/tests/test_cognitive_eval_harness.py b/tests/test_cognitive_eval_harness.py new file mode 100644 index 00000000..c534723a --- /dev/null +++ b/tests/test_cognitive_eval_harness.py @@ -0,0 +1,106 @@ +"""Tests for the cognitive eval harness.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from evals.run_cognition_eval import load_cases, run_eval, check_determinism + + +_CASES_PATH = Path(__file__).resolve().parent.parent / "evals" / "cognition_cases.jsonl" + + +class TestCognitionEvalLoadsCases: + def test_loads_all_cases(self) -> None: + cases = load_cases(_CASES_PATH) + assert len(cases) >= 15 + assert all("id" in c for c in cases) + assert all("prompt" in c for c in cases) + assert all("expected_intent" in c for c in cases) + + def test_cases_have_valid_structure(self) -> None: + cases = load_cases(_CASES_PATH) + for case in cases: + assert isinstance(case["id"], str) + assert isinstance(case["prompt"], str) + assert case["expected_intent"] in { + "definition", "comparison", "cause", "procedure", + "recall", "correction", "verification", "unknown", + } + assert isinstance(case.get("expected_terms", []), list) + + def test_cases_cover_required_categories(self) -> None: + cases = load_cases(_CASES_PATH) + categories = {c.get("category", "unknown") for c in cases} + required = {"definition", "comparison", "cause", "correction", "verification", "unknown"} + assert required.issubset(categories), f"missing: {required - categories}" + + +class TestCognitionEvalRunsSmallCaseSet: + def test_runs_single_case(self) -> None: + cases = load_cases(_CASES_PATH)[:1] + report = run_eval(cases) + assert report.total == 1 + assert len(report.cases) == 1 + assert report.cases[0].case_id == cases[0]["id"] + + def test_runs_five_cases(self) -> None: + cases = load_cases(_CASES_PATH)[:5] + report = run_eval(cases) + assert report.total == 5 + assert len(report.cases) == 5 + + +class TestCognitionEvalRecordsIntentAccuracy: + def test_definition_intent_detected(self) -> None: + cases = [c for c in load_cases(_CASES_PATH) if c["expected_intent"] == "definition"][:2] + report = run_eval(cases) + assert report.intent_correct == report.total + + def test_comparison_intent_detected(self) -> None: + cases = [c for c in load_cases(_CASES_PATH) if c["expected_intent"] == "comparison"][:1] + report = run_eval(cases) + assert report.intent_correct == report.total + + def test_report_has_accuracy_metric(self) -> None: + cases = load_cases(_CASES_PATH)[:3] + report = run_eval(cases) + assert 0.0 <= report.intent_accuracy <= 1.0 + report_dict = report.as_dict() + assert "intent_accuracy" in report_dict + + +class TestCognitionEvalRecordsTraceHashes: + def test_trace_hashes_present(self) -> None: + cases = load_cases(_CASES_PATH)[:3] + report = run_eval(cases) + assert len(report.trace_hashes) == 3 + for case_id, h in report.trace_hashes.items(): + assert isinstance(h, str) + assert len(h) == 64 # SHA-256 hex + + def test_distinct_cases_get_distinct_hashes(self) -> None: + cases = load_cases(_CASES_PATH)[:5] + report = run_eval(cases) + hashes = list(report.trace_hashes.values()) + assert len(set(hashes)) == len(hashes), "duplicate trace hashes" + + +class TestCognitionEvalIsDeterministic: + def test_two_runs_same_hashes(self) -> None: + cases = load_cases(_CASES_PATH)[:3] + assert check_determinism(cases, runs=2) + + +class TestEvalReportSerialization: + def test_as_dict_roundtrips(self) -> None: + cases = load_cases(_CASES_PATH)[:2] + report = run_eval(cases) + d = report.as_dict() + serialized = json.dumps(d) + parsed = json.loads(serialized) + assert parsed["total"] == 2 + assert "intent_accuracy" in parsed + assert "trace_hashes" in parsed + assert len(parsed["cases"]) == 2 diff --git a/tests/test_operator_calibration_replay.py b/tests/test_operator_calibration_replay.py new file mode 100644 index 00000000..32c964f4 --- /dev/null +++ b/tests/test_operator_calibration_replay.py @@ -0,0 +1,111 @@ +"""Tests for the operator calibration replay system.""" + +from __future__ import annotations + +from calibration.params import ( + CalibrationParams, + DEFAULT_PARAMS, + grid_candidates, +) +from calibration.replay import replay_with_params +from calibration.tune import calibrate, CalibrationResult +from evals.run_cognition_eval import load_cases + + +_SMALL_CASES = None + + +def _get_small_cases() -> list[dict]: + global _SMALL_CASES + if _SMALL_CASES is None: + _SMALL_CASES = load_cases()[:3] + return _SMALL_CASES + + +class TestCalibrationReplayIsDeterministic: + def test_same_params_same_metrics(self) -> None: + cases = _get_small_cases() + r1 = replay_with_params(DEFAULT_PARAMS, cases) + r2 = replay_with_params(DEFAULT_PARAMS, cases) + assert r1.intent_accuracy == r2.intent_accuracy + assert r1.versor_closure_rate == r2.versor_closure_rate + assert r1.trace_hashes == r2.trace_hashes + + +class TestCalibrationCandidateParamsAreBounded: + def test_grid_produces_bounded_candidates(self) -> None: + candidates = grid_candidates() + assert len(candidates) > 0 + for c in candidates: + assert 4 <= c.salience_top_k <= 32 + assert 0.1 <= c.inhibition_threshold <= 0.9 + + def test_grid_is_deterministic(self) -> None: + c1 = grid_candidates() + c2 = grid_candidates() + assert c1 == c2 + + def test_custom_grid(self) -> None: + custom = {"salience_top_k": (8, 16), "inhibition_threshold": (0.2,)} + candidates = grid_candidates(custom) + assert len(candidates) == 2 + salience_values = {c.salience_top_k for c in candidates} + assert salience_values == {8, 16} + + +class TestCalibrationReportHasBeforeAfterMetrics: + def test_calibrate_returns_result(self) -> None: + cases = _get_small_cases() + tiny_grid = {"salience_top_k": (12, 16), "inhibition_threshold": (0.3,)} + result = calibrate(cases, grid=tiny_grid) + assert isinstance(result, CalibrationResult) + assert result.baseline_report.total == len(cases) + assert result.best_report.total == len(cases) + + def test_report_dict_has_required_fields(self) -> None: + cases = _get_small_cases() + tiny_grid = {"salience_top_k": (16,), "inhibition_threshold": (0.3,)} + result = calibrate(cases, grid=tiny_grid) + d = result.as_dict() + assert "baseline_params" in d + assert "baseline_metrics" in d + assert "best_params" in d + assert "best_metrics" in d + assert "candidates_evaluated" in d + assert "candidates_accepted" in d + + +class TestCalibrationRejectsInvariantRegression: + def test_versor_closure_must_not_regress(self) -> None: + cases = _get_small_cases() + tiny_grid = {"salience_top_k": (8, 12, 16), "inhibition_threshold": (0.2, 0.3, 0.4)} + result = calibrate(cases, grid=tiny_grid) + for c in result.candidates: + if c.accepted: + assert c.after_report.versor_closure_rate >= result.baseline_report.versor_closure_rate + + +class TestCalibrationDoesNotMutateIdentityOrPacks: + def test_params_are_frozen(self) -> None: + params = CalibrationParams() + try: + params.salience_top_k = 99 # type: ignore[misc] + raise AssertionError("CalibrationParams should be frozen") + except AttributeError: + pass + + def test_calibration_does_not_touch_packs(self) -> None: + import hashlib + from pathlib import Path + + pack_dir = Path(__file__).resolve().parent.parent / "language_packs" / "data" + before = {} + for f in sorted(pack_dir.rglob("*.jsonl")): + before[str(f)] = hashlib.sha256(f.read_bytes()).hexdigest() + + cases = _get_small_cases() + tiny_grid = {"salience_top_k": (16,), "inhibition_threshold": (0.3,)} + calibrate(cases, grid=tiny_grid) + + for f in sorted(pack_dir.rglob("*.jsonl")): + assert before[str(f)] == hashlib.sha256(f.read_bytes()).hexdigest()