parent
c7c21e6acf
commit
a118977f0f
3 changed files with 27 additions and 7 deletions
|
|
@ -14,6 +14,7 @@ approval before they touch the vocabulary manifold.
|
||||||
|
|
||||||
from teaching.correction import CorrectionCandidate, extract_correction
|
from teaching.correction import CorrectionCandidate, extract_correction
|
||||||
from teaching.epistemic import ADMISSIBLE_AS_EVIDENCE, EpistemicStatus, parse_status
|
from teaching.epistemic import ADMISSIBLE_AS_EVIDENCE, EpistemicStatus, parse_status
|
||||||
|
from teaching.metric_set import MetricSet
|
||||||
from teaching.review import ReviewedTeachingExample, ReviewOutcome, review_correction
|
from teaching.review import ReviewedTeachingExample, ReviewOutcome, review_correction
|
||||||
from teaching.store import TeachingStore, PackMutationProposal
|
from teaching.store import TeachingStore, PackMutationProposal
|
||||||
|
|
||||||
|
|
@ -21,6 +22,7 @@ __all__ = [
|
||||||
"ADMISSIBLE_AS_EVIDENCE",
|
"ADMISSIBLE_AS_EVIDENCE",
|
||||||
"CorrectionCandidate",
|
"CorrectionCandidate",
|
||||||
"EpistemicStatus",
|
"EpistemicStatus",
|
||||||
|
"MetricSet",
|
||||||
"PackMutationProposal",
|
"PackMutationProposal",
|
||||||
"ReviewedTeachingExample",
|
"ReviewedTeachingExample",
|
||||||
"ReviewOutcome",
|
"ReviewOutcome",
|
||||||
|
|
|
||||||
14
teaching/metric_set.py
Normal file
14
teaching/metric_set.py
Normal file
|
|
@ -0,0 +1,14 @@
|
||||||
|
"""Versioned metric carrier for replay evidence gates."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class MetricSet:
|
||||||
|
version: int
|
||||||
|
metrics: tuple[str, ...]
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["MetricSet"]
|
||||||
|
|
@ -26,16 +26,20 @@ from pathlib import Path
|
||||||
from typing import Any, Iterator
|
from typing import Any, Iterator
|
||||||
|
|
||||||
from chat import teaching_grounding as _tg
|
from chat import teaching_grounding as _tg
|
||||||
|
from teaching.metric_set import MetricSet
|
||||||
from teaching.proposals import ReplayEvidence
|
from teaching.proposals import ReplayEvidence
|
||||||
|
|
||||||
|
|
||||||
# Metrics watched for regression. Any metric whose candidate value
|
# Metrics watched for regression. Any metric whose candidate value
|
||||||
# is strictly less than the baseline value counts as a regression.
|
# is strictly less than the baseline value counts as a regression.
|
||||||
_WATCHED_METRICS: tuple[str, ...] = (
|
_WATCHED_METRICS = MetricSet(
|
||||||
"intent_accuracy",
|
version=1,
|
||||||
"surface_groundedness",
|
metrics=(
|
||||||
"term_capture_rate",
|
"intent_accuracy",
|
||||||
"versor_closure_rate",
|
"surface_groundedness",
|
||||||
|
"term_capture_rate",
|
||||||
|
"versor_closure_rate",
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -83,7 +87,7 @@ def _run_cognition_public() -> dict[str, float]:
|
||||||
lane = get_lane("cognition")
|
lane = get_lane("cognition")
|
||||||
result = run_lane(lane, version="v1", split="public")
|
result = run_lane(lane, version="v1", split="public")
|
||||||
out: dict[str, float] = {}
|
out: dict[str, float] = {}
|
||||||
for k in _WATCHED_METRICS:
|
for k in _WATCHED_METRICS.metrics:
|
||||||
v = result.metrics.get(k)
|
v = result.metrics.get(k)
|
||||||
if isinstance(v, (int, float)):
|
if isinstance(v, (int, float)):
|
||||||
out[k] = float(v)
|
out[k] = float(v)
|
||||||
|
|
@ -143,7 +147,7 @@ def run_replay_equivalence(chain: dict[str, Any]) -> ReplayEvidence:
|
||||||
candidate = _run_cognition_public()
|
candidate = _run_cognition_public()
|
||||||
|
|
||||||
regressed: list[str] = []
|
regressed: list[str] = []
|
||||||
for metric in _WATCHED_METRICS:
|
for metric in _WATCHED_METRICS.metrics:
|
||||||
b = baseline.get(metric)
|
b = baseline.get(metric)
|
||||||
c = candidate.get(metric)
|
c = candidate.get(metric)
|
||||||
if b is None or c is None:
|
if b is None or c is None:
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue