diff --git a/teaching/__init__.py b/teaching/__init__.py index ff79b46c..17a03798 100644 --- a/teaching/__init__.py +++ b/teaching/__init__.py @@ -14,6 +14,7 @@ approval before they touch the vocabulary manifold. from teaching.correction import CorrectionCandidate, extract_correction from teaching.epistemic import ADMISSIBLE_AS_EVIDENCE, EpistemicStatus, parse_status +from teaching.metric_set import MetricSet from teaching.review import ReviewedTeachingExample, ReviewOutcome, review_correction from teaching.store import TeachingStore, PackMutationProposal @@ -21,6 +22,7 @@ __all__ = [ "ADMISSIBLE_AS_EVIDENCE", "CorrectionCandidate", "EpistemicStatus", + "MetricSet", "PackMutationProposal", "ReviewedTeachingExample", "ReviewOutcome", diff --git a/teaching/metric_set.py b/teaching/metric_set.py new file mode 100644 index 00000000..a47397ff --- /dev/null +++ b/teaching/metric_set.py @@ -0,0 +1,14 @@ +"""Versioned metric carrier for replay evidence gates.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class MetricSet: + version: int + metrics: tuple[str, ...] + + +__all__ = ["MetricSet"] diff --git a/teaching/replay.py b/teaching/replay.py index 497cbc6b..d357a294 100644 --- a/teaching/replay.py +++ b/teaching/replay.py @@ -26,16 +26,20 @@ from pathlib import Path from typing import Any, Iterator from chat import teaching_grounding as _tg +from teaching.metric_set import MetricSet from teaching.proposals import ReplayEvidence # Metrics watched for regression. Any metric whose candidate value # is strictly less than the baseline value counts as a regression. -_WATCHED_METRICS: tuple[str, ...] = ( - "intent_accuracy", - "surface_groundedness", - "term_capture_rate", - "versor_closure_rate", +_WATCHED_METRICS = MetricSet( + version=1, + metrics=( + "intent_accuracy", + "surface_groundedness", + "term_capture_rate", + "versor_closure_rate", + ), ) @@ -83,7 +87,7 @@ def _run_cognition_public() -> dict[str, float]: lane = get_lane("cognition") result = run_lane(lane, version="v1", split="public") out: dict[str, float] = {} - for k in _WATCHED_METRICS: + for k in _WATCHED_METRICS.metrics: v = result.metrics.get(k) if isinstance(v, (int, float)): out[k] = float(v) @@ -143,7 +147,7 @@ def run_replay_equivalence(chain: dict[str, Any]) -> ReplayEvidence: candidate = _run_cognition_public() regressed: list[str] = [] - for metric in _WATCHED_METRICS: + for metric in _WATCHED_METRICS.metrics: b = baseline.get(metric) c = candidate.get(metric) if b is None or c is None: