e985790a03
Sharpens the measurement layer to match the runtime spine landed in07fefb9/7af7892/4e3ddee. Pure eval/benchmark/holdout work — no runtime or planner code changed. New isolation lanes ------------------- * ``evals/compound_intent_decomposition/`` — single-purpose lane for the new ``classify_compound_intent`` decomposer. Metrics: ``decomposition_accuracy``, ``atom_precision``, ``subject_accuracy``. Public: ``decomposition=1.0`` on4e3ddee. * ``evals/walkthrough_chain/`` — single-purpose lane for the new WALKTHROUGH sequential teaching-chain walk. Metrics: ``path_exact_rate``, ``anchor_rate``, ``min_hop_rate``, ``bounded_rate``. Public: ``path_exact=1.0`` on4e3ddee. Without these, regressions in compound decomposition or the walkthrough walk would show up as noise in ``multi_sentence_response``. Each capability now has a single load-bearing metric on its own lane. Cold-start lane sharpened ------------------------- * ``evals/cold_start_grounding/public/v1/cases.jsonl`` extended with expository, compound, and walkthrough cases (48 total cases across 19 categories including new ``expository_definition``, ``compound_definition_cause``, ``walkthrough_definition``). * ``evals/cold_start_grounding/runner.py`` uses ``classify_compound_intent(...).primary`` for compound subject scoring — previously misattributed subjects on multi-part prompts. Holdouts for the long-span lanes -------------------------------- Until now only the cognition lane had a holdout split. Adding holdouts to the long-span lanes gives the planner work somewhere to fail honestly when we widen: * ``evals/cold_start_grounding/holdouts/v1/cases.jsonl`` (5 cases) * ``evals/multi_sentence_response/holdouts/v1/cases.jsonl`` (5 cases) * ``evals/conversational_thread_coherence/holdouts/v1/cases.jsonl`` (3 cases) * ``evals/warmed_session_consistency/holdouts/v1/cases.jsonl`` (2 cases) Discourse-planner-on bench sub-bench ------------------------------------ * ``benchmarks/articulation.py`` adds a planner-on sub-bench that reports ``articulate_sentence_rate`` alongside the existing throughput metrics. Baselines articulation under load before any follow-up touches ``compute_trace_hash``. Test coverage ------------- * ``tests/test_compound_walkthrough_eval_lanes.py`` — new file pinning the two new lane runners. * ``tests/test_articulation_bench.py``, ``tests/test_cold_start_grounding_lane.py``, ``tests/test_intent_explain_paragraph.py``, ``tests/test_response_mode_classifier.py`` — updated for new cases and assertions. Validation ---------- * 152/152 active tests pass on the listed surfaces (2 skipped). * smoke suite 67/67. * cognition eval byte-identical: public 100/100/91.7/100. * multi_sentence flag_on: articulate=1.0, disclosure=0.0, unarticulate=0.0 * compound_intent_decomp public: decomposition=1.0 * walkthrough_chain public: path_exact=1.0 * cold_start_grounding public (48 cases): intent=1.0, grounding=1.0, subject=1.0
87 lines
2.5 KiB
Python
87 lines
2.5 KiB
Python
"""Compound intent decomposition eval lane."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from generate.intent import classify_compound_intent
|
|
|
|
|
|
@dataclass
|
|
class LaneReport:
|
|
metrics: dict[str, Any] = field(default_factory=dict)
|
|
case_details: list[dict[str, Any]] = field(default_factory=list)
|
|
|
|
|
|
def _expected_atoms(case: dict[str, Any]) -> list[dict[str, str]]:
|
|
atoms = case.get("expected_atoms")
|
|
if not isinstance(atoms, list):
|
|
return []
|
|
out: list[dict[str, str]] = []
|
|
for atom in atoms:
|
|
if not isinstance(atom, dict):
|
|
continue
|
|
intent = str(atom.get("intent", "")).strip().lower()
|
|
subject = str(atom.get("subject", "")).strip().lower()
|
|
out.append({"intent": intent, "subject": subject})
|
|
return out
|
|
|
|
|
|
def run_lane(cases: list[dict[str, Any]], config: Any = None) -> LaneReport: # noqa: ARG001
|
|
details: list[dict[str, Any]] = []
|
|
exact = 0
|
|
atom_positions = 0
|
|
atom_correct = 0
|
|
subject_positions = 0
|
|
subject_correct = 0
|
|
|
|
for case in cases:
|
|
expected = _expected_atoms(case)
|
|
actual = [
|
|
{"intent": atom.tag.value, "subject": atom.subject.strip().lower()}
|
|
for atom in classify_compound_intent(case["prompt"]).parts
|
|
]
|
|
exact_match = actual == expected
|
|
if exact_match:
|
|
exact += 1
|
|
|
|
for idx, exp in enumerate(expected):
|
|
if idx >= len(actual):
|
|
atom_positions += 1
|
|
subject_positions += 1
|
|
continue
|
|
got = actual[idx]
|
|
atom_positions += 1
|
|
subject_positions += 1
|
|
if got == exp:
|
|
atom_correct += 1
|
|
if got["subject"] == exp["subject"]:
|
|
subject_correct += 1
|
|
|
|
details.append({
|
|
"case_id": case["id"],
|
|
"prompt": case["prompt"],
|
|
"expected_atoms": expected,
|
|
"actual_atoms": actual,
|
|
"exact_match": exact_match,
|
|
})
|
|
|
|
total = len(cases)
|
|
return LaneReport(
|
|
metrics={
|
|
"cases": total,
|
|
"decomposition_accuracy": round(exact / total, 4) if total else 0.0,
|
|
"atom_precision": (
|
|
round(atom_correct / atom_positions, 4) if atom_positions else 1.0
|
|
),
|
|
"subject_accuracy": (
|
|
round(subject_correct / subject_positions, 4)
|
|
if subject_positions else 1.0
|
|
),
|
|
},
|
|
case_details=details,
|
|
)
|
|
|
|
|
|
__all__ = ["run_lane", "LaneReport"]
|