Sharpens the measurement layer to match the runtime spine landed in07fefb9/7af7892/4e3ddee. Pure eval/benchmark/holdout work — no runtime or planner code changed. New isolation lanes ------------------- * ``evals/compound_intent_decomposition/`` — single-purpose lane for the new ``classify_compound_intent`` decomposer. Metrics: ``decomposition_accuracy``, ``atom_precision``, ``subject_accuracy``. Public: ``decomposition=1.0`` on4e3ddee. * ``evals/walkthrough_chain/`` — single-purpose lane for the new WALKTHROUGH sequential teaching-chain walk. Metrics: ``path_exact_rate``, ``anchor_rate``, ``min_hop_rate``, ``bounded_rate``. Public: ``path_exact=1.0`` on4e3ddee. Without these, regressions in compound decomposition or the walkthrough walk would show up as noise in ``multi_sentence_response``. Each capability now has a single load-bearing metric on its own lane. Cold-start lane sharpened ------------------------- * ``evals/cold_start_grounding/public/v1/cases.jsonl`` extended with expository, compound, and walkthrough cases (48 total cases across 19 categories including new ``expository_definition``, ``compound_definition_cause``, ``walkthrough_definition``). * ``evals/cold_start_grounding/runner.py`` uses ``classify_compound_intent(...).primary`` for compound subject scoring — previously misattributed subjects on multi-part prompts. Holdouts for the long-span lanes -------------------------------- Until now only the cognition lane had a holdout split. Adding holdouts to the long-span lanes gives the planner work somewhere to fail honestly when we widen: * ``evals/cold_start_grounding/holdouts/v1/cases.jsonl`` (5 cases) * ``evals/multi_sentence_response/holdouts/v1/cases.jsonl`` (5 cases) * ``evals/conversational_thread_coherence/holdouts/v1/cases.jsonl`` (3 cases) * ``evals/warmed_session_consistency/holdouts/v1/cases.jsonl`` (2 cases) Discourse-planner-on bench sub-bench ------------------------------------ * ``benchmarks/articulation.py`` adds a planner-on sub-bench that reports ``articulate_sentence_rate`` alongside the existing throughput metrics. Baselines articulation under load before any follow-up touches ``compute_trace_hash``. Test coverage ------------- * ``tests/test_compound_walkthrough_eval_lanes.py`` — new file pinning the two new lane runners. * ``tests/test_articulation_bench.py``, ``tests/test_cold_start_grounding_lane.py``, ``tests/test_intent_explain_paragraph.py``, ``tests/test_response_mode_classifier.py`` — updated for new cases and assertions. Validation ---------- * 152/152 active tests pass on the listed surfaces (2 skipped). * smoke suite 67/67. * cognition eval byte-identical: public 100/100/91.7/100. * multi_sentence flag_on: articulate=1.0, disclosure=0.0, unarticulate=0.0 * compound_intent_decomp public: decomposition=1.0 * walkthrough_chain public: path_exact=1.0 * cold_start_grounding public (48 cases): intent=1.0, grounding=1.0, subject=1.0
141 lines
4.7 KiB
Python
141 lines
4.7 KiB
Python
"""Tests for the expository-DEFINITION classifier rules.
|
|
|
|
Pins:
|
|
* ``Explain X`` / ``Explain X.`` → DEFINITION (subject=X)
|
|
* ``Write a paragraph about X`` → DEFINITION (subject=X)
|
|
* ``Compose/draft a short paragraph about X`` → DEFINITION (subject=X)
|
|
* ``Paragraph about X`` → DEFINITION (subject=X)
|
|
|
|
Orthogonality preservation:
|
|
* ``Tell me about X`` stays NARRATIVE.
|
|
* ``Describe X`` stays NARRATIVE.
|
|
* ``ResponseMode`` classification for the same prompts stays
|
|
EXPLAIN / PARAGRAPH (independent axis).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from generate.intent import (
|
|
DialogueIntent,
|
|
IntentTag,
|
|
ResponseMode,
|
|
classify_compound_intent,
|
|
classify_intent,
|
|
classify_response_mode,
|
|
)
|
|
|
|
|
|
class TestExplainDefinition:
|
|
@pytest.mark.parametrize(
|
|
"prompt,subject",
|
|
[
|
|
("Explain truth.", "truth"),
|
|
("Explain truth", "truth"),
|
|
("explain memory", "memory"),
|
|
("Explain the parent.", "parent"),
|
|
("Explain a parent.", "parent"),
|
|
],
|
|
)
|
|
def test_explain_classifies_to_definition(
|
|
self, prompt: str, subject: str
|
|
) -> None:
|
|
result = classify_intent(prompt)
|
|
assert result.tag is IntentTag.DEFINITION
|
|
assert result.subject == subject
|
|
|
|
def test_explain_response_mode_stays_explain(self) -> None:
|
|
assert classify_response_mode("Explain truth.") is ResponseMode.EXPLAIN
|
|
|
|
|
|
class TestParagraphRequestDefinition:
|
|
@pytest.mark.parametrize(
|
|
"prompt,subject",
|
|
[
|
|
("Write a paragraph about truth.", "truth"),
|
|
("Write a short paragraph about truth.", "truth"),
|
|
("Write a brief paragraph about memory.", "memory"),
|
|
("Compose a paragraph about light.", "light"),
|
|
("Draft a paragraph on knowledge.", "knowledge"),
|
|
("Paragraph about wisdom.", "wisdom"),
|
|
("Paragraph on understanding", "understanding"),
|
|
],
|
|
)
|
|
def test_paragraph_request_classifies_to_definition(
|
|
self, prompt: str, subject: str
|
|
) -> None:
|
|
result = classify_intent(prompt)
|
|
assert result.tag is IntentTag.DEFINITION
|
|
assert result.subject == subject
|
|
|
|
def test_paragraph_request_response_mode_stays_paragraph(self) -> None:
|
|
assert (
|
|
classify_response_mode("Write a paragraph about truth.")
|
|
is ResponseMode.PARAGRAPH
|
|
)
|
|
assert (
|
|
classify_response_mode("Compose a paragraph about light.")
|
|
is ResponseMode.PARAGRAPH
|
|
)
|
|
|
|
|
|
class TestNarrativeRulesUntouched:
|
|
@pytest.mark.parametrize(
|
|
"prompt,subject",
|
|
[
|
|
("Tell me about truth.", "truth"),
|
|
("Describe wisdom.", "wisdom"),
|
|
("Describe memory", "memory"),
|
|
],
|
|
)
|
|
def test_narrative_rules_still_win(
|
|
self, prompt: str, subject: str
|
|
) -> None:
|
|
result = classify_intent(prompt)
|
|
assert result == DialogueIntent(
|
|
tag=IntentTag.NARRATIVE, subject=subject
|
|
)
|
|
|
|
|
|
class TestExampleRulesUntouched:
|
|
def test_example_still_routes_correctly(self) -> None:
|
|
result = classify_intent("Give me an example of truth.")
|
|
assert result.tag is IntentTag.EXAMPLE
|
|
assert result.subject == "truth"
|
|
|
|
|
|
class TestExistingDefinitionRulesUntouched:
|
|
@pytest.mark.parametrize(
|
|
"prompt,subject",
|
|
[
|
|
("What is truth?", "truth"),
|
|
("Define memory.", "memory"),
|
|
],
|
|
)
|
|
def test_existing_definition_rules_unchanged(
|
|
self, prompt: str, subject: str
|
|
) -> None:
|
|
result = classify_intent(prompt)
|
|
assert result.tag is IntentTag.DEFINITION
|
|
assert result.subject == subject
|
|
|
|
|
|
class TestCompoundAndWalkthroughAnchors:
|
|
def test_compound_definition_strips_causal_tail_from_subject(self) -> None:
|
|
result = classify_compound_intent("What is truth, and why does it matter?")
|
|
assert result.primary.tag is IntentTag.DEFINITION
|
|
assert result.primary.subject == "truth"
|
|
|
|
def test_compound_definition_cause_decomposes_to_two_atoms(self) -> None:
|
|
atoms = classify_compound_intent("What is truth, and why does it matter?")
|
|
assert atoms.parts == (
|
|
DialogueIntent(tag=IntentTag.DEFINITION, subject="truth"),
|
|
DialogueIntent(tag=IntentTag.CAUSE, subject="truth"),
|
|
)
|
|
|
|
def test_simple_walkthrough_gets_grounded_definition_anchor(self) -> None:
|
|
result = classify_intent("Walk me through recall.")
|
|
assert result.tag is IntentTag.DEFINITION
|
|
assert result.subject == "recall"
|
|
assert classify_response_mode("Walk me through recall.") is ResponseMode.WALKTHROUGH
|