e985790a03
Sharpens the measurement layer to match the runtime spine landed in07fefb9/7af7892/4e3ddee. Pure eval/benchmark/holdout work — no runtime or planner code changed. New isolation lanes ------------------- * ``evals/compound_intent_decomposition/`` — single-purpose lane for the new ``classify_compound_intent`` decomposer. Metrics: ``decomposition_accuracy``, ``atom_precision``, ``subject_accuracy``. Public: ``decomposition=1.0`` on4e3ddee. * ``evals/walkthrough_chain/`` — single-purpose lane for the new WALKTHROUGH sequential teaching-chain walk. Metrics: ``path_exact_rate``, ``anchor_rate``, ``min_hop_rate``, ``bounded_rate``. Public: ``path_exact=1.0`` on4e3ddee. Without these, regressions in compound decomposition or the walkthrough walk would show up as noise in ``multi_sentence_response``. Each capability now has a single load-bearing metric on its own lane. Cold-start lane sharpened ------------------------- * ``evals/cold_start_grounding/public/v1/cases.jsonl`` extended with expository, compound, and walkthrough cases (48 total cases across 19 categories including new ``expository_definition``, ``compound_definition_cause``, ``walkthrough_definition``). * ``evals/cold_start_grounding/runner.py`` uses ``classify_compound_intent(...).primary`` for compound subject scoring — previously misattributed subjects on multi-part prompts. Holdouts for the long-span lanes -------------------------------- Until now only the cognition lane had a holdout split. Adding holdouts to the long-span lanes gives the planner work somewhere to fail honestly when we widen: * ``evals/cold_start_grounding/holdouts/v1/cases.jsonl`` (5 cases) * ``evals/multi_sentence_response/holdouts/v1/cases.jsonl`` (5 cases) * ``evals/conversational_thread_coherence/holdouts/v1/cases.jsonl`` (3 cases) * ``evals/warmed_session_consistency/holdouts/v1/cases.jsonl`` (2 cases) Discourse-planner-on bench sub-bench ------------------------------------ * ``benchmarks/articulation.py`` adds a planner-on sub-bench that reports ``articulate_sentence_rate`` alongside the existing throughput metrics. Baselines articulation under load before any follow-up touches ``compute_trace_hash``. Test coverage ------------- * ``tests/test_compound_walkthrough_eval_lanes.py`` — new file pinning the two new lane runners. * ``tests/test_articulation_bench.py``, ``tests/test_cold_start_grounding_lane.py``, ``tests/test_intent_explain_paragraph.py``, ``tests/test_response_mode_classifier.py`` — updated for new cases and assertions. Validation ---------- * 152/152 active tests pass on the listed surfaces (2 skipped). * smoke suite 67/67. * cognition eval byte-identical: public 100/100/91.7/100. * multi_sentence flag_on: articulate=1.0, disclosure=0.0, unarticulate=0.0 * compound_intent_decomp public: decomposition=1.0 * walkthrough_chain public: path_exact=1.0 * cold_start_grounding public (48 cases): intent=1.0, grounding=1.0, subject=1.0
84 lines
2.5 KiB
Python
84 lines
2.5 KiB
Python
"""Walkthrough chain eval lane."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from chat.teaching_grounding import _all_chains_index
|
|
|
|
|
|
@dataclass
|
|
class LaneReport:
|
|
metrics: dict[str, Any] = field(default_factory=dict)
|
|
case_details: list[dict[str, Any]] = field(default_factory=list)
|
|
|
|
|
|
def _walk(subject: str, *, max_hops: int) -> tuple[str, ...]:
|
|
corpus = _all_chains_index()
|
|
path: list[str] = [subject.strip().lower()]
|
|
seen = {path[0]}
|
|
cursor = path[0]
|
|
for _ in range(max(0, max_hops)):
|
|
chain = corpus.get((cursor, "cause")) or corpus.get((cursor, "verification"))
|
|
if chain is None:
|
|
break
|
|
nxt = chain.object.strip().lower()
|
|
if not nxt or nxt in seen:
|
|
break
|
|
path.append(nxt)
|
|
seen.add(nxt)
|
|
cursor = nxt
|
|
return tuple(path)
|
|
|
|
|
|
def run_lane(cases: list[dict[str, Any]], config: Any = None) -> LaneReport: # noqa: ARG001
|
|
details: list[dict[str, Any]] = []
|
|
exact = 0
|
|
anchored = 0
|
|
min_hop = 0
|
|
bounded = 0
|
|
|
|
for case in cases:
|
|
subject = str(case["subject"]).strip().lower()
|
|
max_hops = int(case.get("max_hops", 2))
|
|
expected = tuple(str(x).strip().lower() for x in case.get("expected_path", ()))
|
|
actual = _walk(subject, max_hops=max_hops)
|
|
exact_match = actual == expected
|
|
anchor_match = bool(actual) and actual[0] == subject
|
|
has_hop = len(actual) >= 2
|
|
is_bounded = len(actual) <= max_hops + 1
|
|
|
|
exact += int(exact_match)
|
|
anchored += int(anchor_match)
|
|
min_hop += int(has_hop)
|
|
bounded += int(is_bounded)
|
|
|
|
details.append({
|
|
"case_id": case["id"],
|
|
"prompt": case.get("prompt", ""),
|
|
"subject": subject,
|
|
"max_hops": max_hops,
|
|
"expected_path": list(expected),
|
|
"actual_path": list(actual),
|
|
"path_exact": exact_match,
|
|
"anchor_match": anchor_match,
|
|
"min_hop": has_hop,
|
|
"bounded": is_bounded,
|
|
})
|
|
|
|
total = len(cases)
|
|
return LaneReport(
|
|
metrics={
|
|
"cases": total,
|
|
"path_exact_rate": round(exact / total, 4) if total else 0.0,
|
|
"anchor_rate": round(anchored / total, 4) if total else 0.0,
|
|
"min_hop_rate": round(min_hop / total, 4) if total else 0.0,
|
|
"bounded_rate": round(bounded / total, 4) if total else 0.0,
|
|
},
|
|
case_details=details,
|
|
)
|
|
|
|
|
|
__all__ = ["run_lane", "LaneReport"]
|
|
|