bec76d4889
Correctsbf17f42, which claimed a fix that did not work. That commit added hash_surface to evals/cognition/runner.py::CaseResult. The test imports run_eval from evals/run_cognition_eval.py, which builds evals/metrics.py::CaseResult — a DIFFERENT class with the same name. Thirteen files in this repo define one. I matched on the class name instead of following the import, so the field never reached the assertion and _diff_rows raised AttributeError: 'CaseResult' object has no attribute 'hash_surface'. That is why the count went 99 -> 100: the same 99 divergences, plus one error. The diagnosis inbf17f42was right; only its aim was wrong. Confirmed directly at the pipeline seam: register=None surface = 'Truth is what is true. pack-grounded...' hash_surface = 'Truth is what is true. pack-grounded...' register=socratic_v1 surface = 'Consider the question: Truth is what is...' hash_surface = 'Truth is what is true. pack-grounded...' surface varies across the register axis (as it must), hash_surface does not (as it must not), trace_hash matches. So the field added to CognitiveTurnResult inbf17f42is correct and stays; this moves the eval-side plumbing to the class the test really consumes, and reverts the edit to the one it does not. [Verification]: PARTIAL. Three registers green — socratic_v1, terse_v1, manifesto_v1, 24 passed, where all three failed before. The full 99-register run (~17 min) was still executing when the tree had to be committed; result to follow, and bf17f42's DO NOT MERGE stands until it is green. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FduW6Krm3PPQv3P5iwBYtx
79 lines
2.7 KiB
Python
79 lines
2.7 KiB
Python
"""Cognition eval metrics — deterministic, compact measurements."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class CaseResult:
|
|
case_id: str
|
|
category: str
|
|
prompt: str
|
|
intent_correct: bool
|
|
terms_captured: tuple[str, ...]
|
|
terms_expected: tuple[str, ...]
|
|
surface_contains_pass: bool
|
|
versor_closure: bool
|
|
versor_condition: float
|
|
trace_hash: str
|
|
#: The SERVED surface — register-decorated, what the user reads.
|
|
surface: str
|
|
#: The register-INVARIANT truth-path bytes that ``compute_trace_hash``
|
|
#: folds (ADR-0069 inv C). A different field with a different contract:
|
|
#: ``surface`` is *expected* to vary across the register axis, this is
|
|
#: *required* not to. Defaulted so older constructors stay valid.
|
|
hash_surface: str = ""
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class EvalReport:
|
|
total: int = 0
|
|
intent_correct: int = 0
|
|
terms_captured: int = 0
|
|
terms_expected: int = 0
|
|
surface_grounded: int = 0
|
|
versor_closures: int = 0
|
|
deterministic_traces: int = 0
|
|
cases: list[CaseResult] = field(default_factory=list)
|
|
trace_hashes: dict[str, str] = field(default_factory=dict)
|
|
|
|
@property
|
|
def intent_accuracy(self) -> float:
|
|
return self.intent_correct / self.total if self.total else 0.0
|
|
|
|
@property
|
|
def term_capture_rate(self) -> float:
|
|
return self.terms_captured / self.terms_expected if self.terms_expected else 1.0
|
|
|
|
@property
|
|
def surface_groundedness(self) -> float:
|
|
return self.surface_grounded / self.total if self.total else 0.0
|
|
|
|
@property
|
|
def versor_closure_rate(self) -> float:
|
|
return self.versor_closures / self.total if self.total else 0.0
|
|
|
|
def as_dict(self) -> dict:
|
|
return {
|
|
"total": self.total,
|
|
"intent_accuracy": round(self.intent_accuracy, 4),
|
|
"term_capture_rate": round(self.term_capture_rate, 4),
|
|
"surface_groundedness": round(self.surface_groundedness, 4),
|
|
"versor_closure_rate": round(self.versor_closure_rate, 4),
|
|
"deterministic_traces": self.deterministic_traces,
|
|
"trace_hashes": dict(self.trace_hashes),
|
|
"cases": [
|
|
{
|
|
"case_id": c.case_id,
|
|
"category": c.category,
|
|
"intent_correct": c.intent_correct,
|
|
"surface_contains_pass": c.surface_contains_pass,
|
|
"versor_closure": c.versor_closure,
|
|
"versor_condition": round(c.versor_condition, 9),
|
|
"trace_hash": c.trace_hash,
|
|
"surface": c.surface,
|
|
}
|
|
for c in self.cases
|
|
],
|
|
}
|