1e01f7794e
Implement the eval infrastructure defined in ADR-0016 before building new eval lanes. This establishes the discipline that governs the entire capability roadmap. - Generic eval framework (evals/framework.py): lane discovery, versioned scoring, result persistence - Cognition lane retrofitted into new convention: 45 cases split into stratified dev (13) / public v1 (13) / holdout (19) sets with contract, runner, and recorded results - Generalized `core eval <lane>` CLI: dynamic lane discovery, --list, --version, --split, --save, --json flags - Holdout runner scaffold: plaintext fallback, encryption interface ready - Baseline runner scaffold: pluggable frontier model interface - Fix: CognitiveTurnPipeline.run() crashed on turn_log[-1] when the unknown-domain gate returned a stub without appending to turn_log - ADR-0016, eval_methodology.md, PROGRESS.md, capability gates session log Phase 0 exit audit found two methodology issues: 1. Pipeline turn_log crash (fixed here) 2. Versor drift in multi-turn sessions (pre-existing, under investigation)
19 lines
5 KiB
JSON
19 lines
5 KiB
JSON
{"category": "cause", "expected_intent": "cause", "expected_surface_contains": ["truth"], "expected_terms": ["truth"], "id": "cause_truth_031", "prompt": "Why does truth matter?", "requires_deterministic_trace": true, "requires_versor_closure": true}
|
|
{"category": "cause", "expected_intent": "cause", "expected_surface_contains": ["wisdom"], "expected_terms": ["wisdom"], "id": "cause_wisdom_009", "prompt": "Why does wisdom matter?", "requires_deterministic_trace": true, "requires_versor_closure": true}
|
|
{"category": "comparison", "expected_intent": "comparison", "expected_surface_contains": ["reason", "cause"], "expected_terms": ["reason", "cause"], "id": "comparison_reason_cause_028", "prompt": "Compare reason and cause", "requires_deterministic_trace": true, "requires_versor_closure": true}
|
|
{"category": "comparison", "expected_intent": "comparison", "expected_surface_contains": ["truth", "light"], "expected_terms": ["truth", "light"], "id": "comparison_truth_light_005", "prompt": "Compare truth and light", "requires_deterministic_trace": true, "requires_versor_closure": true}
|
|
{"category": "comparison", "expected_intent": "comparison", "expected_surface_contains": ["word", "meaning"], "expected_terms": ["word", "meaning"], "id": "comparison_word_meaning_006", "prompt": "Compare word and meaning", "requires_deterministic_trace": true, "requires_versor_closure": true}
|
|
{"category": "correction", "expected_intent": "correction", "expected_surface_contains": ["correction"], "expected_terms": ["truth"], "id": "correction_truth_040", "prompt": "Actually, truth requires evidence", "requires_deterministic_trace": true, "requires_versor_closure": true}
|
|
{"category": "definition", "expected_intent": "definition", "expected_surface_contains": ["thought"], "expected_terms": ["thought"], "id": "definition_thought_043", "prompt": "What is thought?", "requires_deterministic_trace": true, "requires_versor_closure": true}
|
|
{"category": "definition", "expected_intent": "definition", "expected_surface_contains": ["truth"], "expected_terms": ["truth"], "id": "definition_truth_001", "prompt": "What is truth?", "requires_deterministic_trace": true, "requires_versor_closure": true}
|
|
{"category": "definition", "expected_intent": "definition", "expected_surface_contains": ["understanding"], "expected_terms": ["understanding"], "id": "definition_understanding_045", "prompt": "What is understanding?", "requires_deterministic_trace": true, "requires_versor_closure": true}
|
|
{"category": "definition", "expected_intent": "definition", "expected_surface_contains": ["verification"], "expected_terms": ["verification"], "id": "definition_verification_024", "prompt": "What is verification?", "requires_deterministic_trace": true, "requires_versor_closure": true}
|
|
{"category": "definition", "expected_intent": "definition", "expected_surface_contains": ["wisdom"], "expected_terms": ["wisdom"], "id": "definition_wisdom_020", "prompt": "What is wisdom?", "requires_deterministic_trace": true, "requires_versor_closure": true}
|
|
{"category": "procedure", "expected_intent": "procedure", "expected_surface_contains": ["concept"], "expected_terms": ["concept"], "id": "procedure_define_010", "prompt": "How do I define a concept?", "requires_deterministic_trace": true, "requires_versor_closure": true}
|
|
{"category": "procedure", "expected_intent": "procedure", "expected_surface_contains": [], "expected_terms": [], "id": "procedure_verify_034", "prompt": "How do I verify a claim?", "requires_deterministic_trace": true, "requires_versor_closure": true}
|
|
{"category": "recall", "expected_intent": "recall", "expected_surface_contains": ["truth"], "expected_terms": ["truth"], "id": "recall_truth_012", "prompt": "Remember truth", "requires_deterministic_trace": true, "requires_versor_closure": true}
|
|
{"category": "recall", "expected_intent": "recall", "expected_surface_contains": ["wisdom"], "expected_terms": ["wisdom"], "id": "recall_wisdom_038", "prompt": "Remember wisdom", "requires_deterministic_trace": true, "requires_versor_closure": true}
|
|
{"category": "unknown", "expected_intent": "unknown", "expected_surface_contains": [], "expected_terms": ["wisdom", "truth"], "id": "unknown_spirit_041", "prompt": "spirit wisdom truth", "requires_deterministic_trace": true, "requires_versor_closure": true}
|
|
{"category": "unknown", "expected_intent": "unknown", "expected_surface_contains": [], "expected_terms": ["word", "truth"], "id": "unknown_word_018", "prompt": "word beginning truth", "requires_deterministic_trace": true, "requires_versor_closure": true}
|
|
{"category": "verification", "expected_intent": "verification", "expected_surface_contains": ["truth"], "expected_terms": ["truth"], "id": "verification_truth_016", "prompt": "Is truth coherent?", "requires_deterministic_trace": true, "requires_versor_closure": true}
|
|
{"category": "verification", "expected_intent": "verification", "expected_surface_contains": ["wisdom"], "expected_terms": ["wisdom", "knowledge"], "id": "verification_wisdom_036", "prompt": "Is wisdom the same as knowledge?", "requires_deterministic_trace": true, "requires_versor_closure": true}
|