feat(evals+bench): isolation lanes, holdouts, planner-on bench sub-bench

Sharpens the measurement layer to match the runtime spine landed in
07fefb9 / 7af7892 / 4e3ddee.  Pure eval/benchmark/holdout work —
no runtime or planner code changed.

New isolation lanes
-------------------

* ``evals/compound_intent_decomposition/`` — single-purpose lane for
  the new ``classify_compound_intent`` decomposer.  Metrics:
  ``decomposition_accuracy``, ``atom_precision``, ``subject_accuracy``.
  Public: ``decomposition=1.0`` on 4e3ddee.
* ``evals/walkthrough_chain/`` — single-purpose lane for the new
  WALKTHROUGH sequential teaching-chain walk.  Metrics:
  ``path_exact_rate``, ``anchor_rate``, ``min_hop_rate``, ``bounded_rate``.
  Public: ``path_exact=1.0`` on 4e3ddee.

Without these, regressions in compound decomposition or the
walkthrough walk would show up as noise in ``multi_sentence_response``.
Each capability now has a single load-bearing metric on its own lane.

Cold-start lane sharpened
-------------------------

* ``evals/cold_start_grounding/public/v1/cases.jsonl`` extended with
  expository, compound, and walkthrough cases (48 total cases across
  19 categories including new ``expository_definition``,
  ``compound_definition_cause``, ``walkthrough_definition``).
* ``evals/cold_start_grounding/runner.py`` uses
  ``classify_compound_intent(...).primary`` for compound subject
  scoring — previously misattributed subjects on multi-part prompts.

Holdouts for the long-span lanes
--------------------------------

Until now only the cognition lane had a holdout split.  Adding
holdouts to the long-span lanes gives the planner work somewhere to
fail honestly when we widen:

* ``evals/cold_start_grounding/holdouts/v1/cases.jsonl`` (5 cases)
* ``evals/multi_sentence_response/holdouts/v1/cases.jsonl`` (5 cases)
* ``evals/conversational_thread_coherence/holdouts/v1/cases.jsonl`` (3 cases)
* ``evals/warmed_session_consistency/holdouts/v1/cases.jsonl`` (2 cases)

Discourse-planner-on bench sub-bench
------------------------------------

* ``benchmarks/articulation.py`` adds a planner-on sub-bench that
  reports ``articulate_sentence_rate`` alongside the existing
  throughput metrics.  Baselines articulation under load before any
  follow-up touches ``compute_trace_hash``.

Test coverage
-------------

* ``tests/test_compound_walkthrough_eval_lanes.py`` — new file pinning
  the two new lane runners.
* ``tests/test_articulation_bench.py``, ``tests/test_cold_start_grounding_lane.py``,
  ``tests/test_intent_explain_paragraph.py``,
  ``tests/test_response_mode_classifier.py`` — updated for new cases
  and assertions.

Validation
----------

* 152/152 active tests pass on the listed surfaces (2 skipped).
* smoke suite 67/67.
* cognition eval byte-identical: public 100/100/91.7/100.
* multi_sentence flag_on: articulate=1.0, disclosure=0.0, unarticulate=0.0
* compound_intent_decomp public: decomposition=1.0
* walkthrough_chain public: path_exact=1.0
* cold_start_grounding public (48 cases): intent=1.0, grounding=1.0, subject=1.0
This commit is contained in:
Shay 2026-05-19 12:42:55 -07:00
parent 4e3ddee91f
commit e985790a03
20 changed files with 436 additions and 15 deletions

View file

@ -36,6 +36,12 @@ Sub-benches:
prompts. Skipped (status: ``skipped`` instead of ``failed``)
when the ``ollama`` binary is not on ``PATH``.
6. **discourse-planner** Runs expository, compound, and
walkthrough prompts with ``RuntimeConfig(discourse_planner=True)``
and reports honest sentence buckets. This keeps the benchmark
aligned with the multi-clause articulation spine instead of only
the older intent-breadth probes.
The whole suite is deterministic on the CORE side no clock-time
or RNG influence on what gets emitted. Walltime sampling lives in
``benchmarks.cost``; this module focuses on capability + identity.
@ -88,6 +94,13 @@ DETERMINISM_PROMPTS: tuple[str, ...] = (
"Give me an example of memory.",
)
DISCOURSE_PLANNER_PROMPTS: tuple[tuple[str, str], ...] = (
("EXPLAIN", "Explain truth."),
("PARAGRAPH", "Write a paragraph about truth."),
("COMPOUND", "What is truth, and why does it matter?"),
("WALKTHROUGH", "Walk me through recall."),
)
# ---------------------------------------------------------------------------
# Report shapes
@ -127,6 +140,18 @@ class CrossTopicTurn:
surface_snippet: str
@dataclass(frozen=True)
class DiscoursePlannerProbe:
label: str
prompt: str
intent_tag: str
grounding_source: str
sentence_count: int
articulate_sentence: bool
disclosure_sentence: bool
surface_snippet: str
@dataclass(frozen=True)
class OllamaPair:
prompt: str
@ -148,6 +173,8 @@ class ArticulationReport:
footprint_per_turn_delta_bytes: float = 0.0
cross_topic: list[CrossTopicTurn] = field(default_factory=list)
anaphora_fire_count: int = 0
discourse_planner: list[DiscoursePlannerProbe] = field(default_factory=list)
discourse_planner_metrics: dict[str, Any] = field(default_factory=dict)
ollama: dict[str, Any] = field(default_factory=dict)
def as_dict(self) -> dict[str, Any]:
@ -164,6 +191,8 @@ class ArticulationReport:
),
"cross_topic": [t.__dict__ for t in self.cross_topic],
"anaphora_fire_count": self.anaphora_fire_count,
"discourse_planner": [p.__dict__ for p in self.discourse_planner],
"discourse_planner_metrics": self.discourse_planner_metrics,
"ollama": self.ollama,
}
@ -178,6 +207,12 @@ def _snippet(s: str, n: int = 120) -> str:
return s if len(s) <= n else s[: n - 1] + ""
def _sentence_count(surface: str) -> int:
from evals.multi_sentence_response.runner import _split_sentences, _strip_provenance
return len(_split_sentences(_strip_provenance(surface)))
def _classify_prompt(prompt: str) -> str:
"""Re-derive the intent label from the prompt text for the report.
@ -293,6 +328,48 @@ def bench_cross_topic() -> tuple[list[CrossTopicTurn], int]:
return out, fires
def bench_discourse_planner() -> tuple[list[DiscoursePlannerProbe], dict[str, Any]]:
from chat.runtime import ChatRuntime
from core.config import RuntimeConfig
out: list[DiscoursePlannerProbe] = []
for label, prompt in DISCOURSE_PLANNER_PROMPTS:
rt = ChatRuntime(config=RuntimeConfig(discourse_planner=True))
resp = rt.chat(prompt)
grounding = getattr(resp, "grounding_source", "unknown")
sentence_count = _sentence_count(resp.surface)
articulate = sentence_count >= 2 and grounding in {"pack", "teaching"}
disclosure = sentence_count >= 2 and grounding in {"oov", "refusal", "none"}
out.append(DiscoursePlannerProbe(
label=label,
prompt=prompt,
intent_tag=_classify_prompt(prompt),
grounding_source=grounding,
sentence_count=sentence_count,
articulate_sentence=articulate,
disclosure_sentence=disclosure,
surface_snippet=_snippet(resp.surface),
))
total = len(out)
metrics = {
"cases": total,
"articulate_sentence_rate": (
round(sum(1 for p in out if p.articulate_sentence) / total, 4)
if total else 0.0
),
"disclosure_sentence_rate": (
round(sum(1 for p in out if p.disclosure_sentence) / total, 4)
if total else 0.0
),
"multi_sentence_rate": (
round(sum(1 for p in out if p.sentence_count >= 2) / total, 4)
if total else 0.0
),
}
return out, metrics
def _have_ollama() -> bool:
return shutil.which("ollama") is not None
@ -409,6 +486,9 @@ def run_articulation_suite(
ct_turns, ct_fires = bench_cross_topic()
report.cross_topic = ct_turns
report.anaphora_fire_count = ct_fires
dp_probes, dp_metrics = bench_discourse_planner()
report.discourse_planner = dp_probes
report.discourse_planner_metrics = dp_metrics
report.ollama = bench_ollama_compare(
model=ollama_model,
prompts=DETERMINISM_PROMPTS[:3], # subset — ollama is slow
@ -425,14 +505,14 @@ def format_summary(report: ArticulationReport) -> str:
out.append("Articulation benchmark suite")
out.append("=" * 76)
out.append("")
out.append("[1/5] Intent breadth — every supported intent shape:")
out.append("[1/6] Intent breadth — every supported intent shape:")
for p in report.breadth:
out.append(
f" {p.label:30s} {p.intent_tag:14s} {p.grounding_source:9s} "
f"{_snippet(p.surface_snippet, 80)}"
)
out.append("")
out.append("[2/5] Determinism — same prompt → byte-identical surface:")
out.append("[2/6] Determinism — same prompt → byte-identical surface:")
for c in report.determinism:
flag = "OK" if c.unique_surfaces == 1 else "FAIL"
out.append(
@ -443,7 +523,7 @@ def format_summary(report: ArticulationReport) -> str:
f" all_identical = {report.determinism_all_identical}"
)
out.append("")
out.append("[3/5] Memory footprint — single runtime, repeated turns:")
out.append("[3/6] Memory footprint — single runtime, repeated turns:")
if report.footprint:
out.append(
f" start = {report.footprint_start_bytes / 1024 / 1024:.1f} MiB "
@ -455,7 +535,7 @@ def format_summary(report: ArticulationReport) -> str:
f"{report.footprint_per_turn_delta_bytes / 1024:.2f} KiB"
)
out.append("")
out.append("[4/5] Cross-topic context — thread anaphora across subjects:")
out.append("[4/6] Cross-topic context — thread anaphora across subjects:")
for t in report.cross_topic:
marker = "" if t.anaphora_fired else " "
out.append(
@ -472,7 +552,16 @@ def format_summary(report: ArticulationReport) -> str:
"fire rate (which is the architectural ceiling, not a defect)."
)
out.append("")
out.append("[5/5] Ollama side-by-side:")
out.append("[5/6] Discourse planner — flag-on articulation spine:")
for p in report.discourse_planner:
marker = "A" if p.articulate_sentence else ("D" if p.disclosure_sentence else " ")
out.append(
f" [{marker}] {p.label:12s} {p.intent_tag:12s} {p.grounding_source:9s} "
f"{p.sentence_count} sentence(s) {_snippet(p.prompt, 46)}"
)
out.append(f" metrics = {report.discourse_planner_metrics}")
out.append("")
out.append("[6/6] Ollama side-by-side:")
status = report.ollama.get("status", "skipped")
if status == "skipped":
out.append(f" skipped — {report.ollama.get('reason', '')}")
@ -502,10 +591,12 @@ __all__ = [
"INTENT_PROBE_PROMPTS",
"CROSS_TOPIC_PROMPTS",
"DETERMINISM_PROMPTS",
"DISCOURSE_PLANNER_PROMPTS",
"bench_breadth",
"bench_determinism",
"bench_footprint",
"bench_cross_topic",
"bench_discourse_planner",
"bench_ollama_compare",
"run_articulation_suite",
"format_summary",

View file

@ -0,0 +1,5 @@
{"id":"hold_def_evidence_001","prompt":"What is evidence?","category":"definition_cognition","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"evidence"}
{"id":"hold_explain_knowledge_002","prompt":"Explain knowledge.","category":"expository_definition","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"knowledge"}
{"id":"hold_paragraph_truth_003","prompt":"Write a paragraph about truth.","category":"expository_definition","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"truth"}
{"id":"hold_compound_memory_004","prompt":"What is memory, and why does it matter?","category":"compound_definition_cause","expected_intent":"definition","expected_grounding_source":"oov","expected_subject":"memory"}
{"id":"hold_walk_inference_005","prompt":"Walk me through inference.","category":"walkthrough_definition","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"inference"}

View file

@ -42,3 +42,7 @@
{"id":"cause_why_truth_042","prompt":"Why is truth important?","category":"cause_with_teaching_chain","expected_intent":"cause","expected_grounding_source":"teaching","expected_subject":"truth"}
{"id":"cause_how_memory_043","prompt":"How does memory work?","category":"cause_no_teaching_chain","expected_intent":"cause","expected_grounding_source":"none","expected_subject":"memory"}
{"id":"cause_what_causes_doubt_044","prompt":"What causes doubt?","category":"cause_no_teaching_chain","expected_intent":"cause","expected_grounding_source":"none","expected_subject":"doubt"}
{"id":"explain_truth_045","prompt":"Explain truth.","category":"expository_definition","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"truth"}
{"id":"paragraph_memory_046","prompt":"Write a paragraph about memory.","category":"expository_definition","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"memory"}
{"id":"compound_truth_matter_047","prompt":"What is truth, and why does it matter?","category":"compound_definition_cause","expected_intent":"definition","expected_grounding_source":"oov","expected_subject":"truth"}
{"id":"walkthrough_recall_048","prompt":"Walk me through recall.","category":"walkthrough_definition","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"recall"}

View file

@ -15,7 +15,7 @@ from dataclasses import dataclass, field
from typing import Any
from chat.runtime import ChatRuntime
from generate.intent import classify_intent
from generate.intent import classify_compound_intent, classify_intent
@dataclass(frozen=True, slots=True)
@ -58,7 +58,8 @@ def _run_case(case: dict[str, Any]) -> CaseResult:
# Classify intent independently for the subject-match check —
# avoids round-tripping through the runtime when the prompt
# bypasses pack-grounding for an OOV/none case.
classified = classify_intent(prompt)
compound = classify_compound_intent(prompt)
classified = compound.primary if compound.is_compound() else classify_intent(prompt)
actual_subject = (classified.subject or "").strip().lower()
# Fresh runtime — cold-start invariant.

View file

@ -0,0 +1,27 @@
# Compound Intent Decomposition
**Lane:** `compound_intent_decomposition`
Scores whether a compound conversational prompt is decomposed into the
intended semantic atoms before generation. This lane is structural: it
does not grade paragraph fluency or final surface length.
## Case Schema
```json
{
"id": "compound_truth_001",
"prompt": "What is truth, and why does it matter?",
"expected_atoms": [
{"intent": "definition", "subject": "truth"},
{"intent": "cause", "subject": "truth"}
]
}
```
## Metrics
- `decomposition_accuracy`: exact ordered atom match.
- `atom_precision`: expected atoms found in the same position.
- `subject_accuracy`: expected subjects recovered in the same position.

View file

@ -0,0 +1,2 @@
{"id":"dev_compound_knowledge_definition_cause","prompt":"What is knowledge, and why does it matter?","expected_atoms":[{"intent":"definition","subject":"knowledge"},{"intent":"cause","subject":"knowledge"}]}

View file

@ -0,0 +1,3 @@
{"id":"compound_truth_definition_cause_001","prompt":"What is truth, and why does it matter?","expected_atoms":[{"intent":"definition","subject":"truth"},{"intent":"cause","subject":"truth"}]}
{"id":"compound_memory_definition_cause_002","prompt":"What is memory, and why does it matter?","expected_atoms":[{"intent":"definition","subject":"memory"},{"intent":"cause","subject":"memory"}]}

View file

@ -0,0 +1,87 @@
"""Compound intent decomposition eval lane."""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
from generate.intent import classify_compound_intent
@dataclass
class LaneReport:
metrics: dict[str, Any] = field(default_factory=dict)
case_details: list[dict[str, Any]] = field(default_factory=list)
def _expected_atoms(case: dict[str, Any]) -> list[dict[str, str]]:
atoms = case.get("expected_atoms")
if not isinstance(atoms, list):
return []
out: list[dict[str, str]] = []
for atom in atoms:
if not isinstance(atom, dict):
continue
intent = str(atom.get("intent", "")).strip().lower()
subject = str(atom.get("subject", "")).strip().lower()
out.append({"intent": intent, "subject": subject})
return out
def run_lane(cases: list[dict[str, Any]], config: Any = None) -> LaneReport: # noqa: ARG001
details: list[dict[str, Any]] = []
exact = 0
atom_positions = 0
atom_correct = 0
subject_positions = 0
subject_correct = 0
for case in cases:
expected = _expected_atoms(case)
actual = [
{"intent": atom.tag.value, "subject": atom.subject.strip().lower()}
for atom in classify_compound_intent(case["prompt"]).parts
]
exact_match = actual == expected
if exact_match:
exact += 1
for idx, exp in enumerate(expected):
if idx >= len(actual):
atom_positions += 1
subject_positions += 1
continue
got = actual[idx]
atom_positions += 1
subject_positions += 1
if got == exp:
atom_correct += 1
if got["subject"] == exp["subject"]:
subject_correct += 1
details.append({
"case_id": case["id"],
"prompt": case["prompt"],
"expected_atoms": expected,
"actual_atoms": actual,
"exact_match": exact_match,
})
total = len(cases)
return LaneReport(
metrics={
"cases": total,
"decomposition_accuracy": round(exact / total, 4) if total else 0.0,
"atom_precision": (
round(atom_correct / atom_positions, 4) if atom_positions else 1.0
),
"subject_accuracy": (
round(subject_correct / subject_positions, 4)
if subject_positions else 1.0
),
},
case_details=details,
)
__all__ = ["run_lane", "LaneReport"]

View file

@ -0,0 +1,3 @@
{"id":"hold_thread_compound_walk_001","category":"compound_walk","turns":[{"prompt":"What is truth?","subject_lemma":"truth"},{"prompt":"What is truth, and why does it matter?","subject_lemma":"truth"},{"prompt":"Walk me through inference.","subject_lemma":"inference"},{"prompt":"What is truth?","subject_lemma":"truth","is_replay_of_prompt_at_turn":0}]}
{"id":"hold_thread_topic_return_002","category":"topic_shift","turns":[{"prompt":"What is evidence?","subject_lemma":"evidence"},{"prompt":"What is recall?","subject_lemma":"recall"},{"prompt":"Write a paragraph about knowledge.","subject_lemma":"knowledge"},{"prompt":"What is evidence?","subject_lemma":"evidence","is_replay_of_prompt_at_turn":0}]}

View file

@ -0,0 +1,5 @@
{"id":"hold_multi_explain_evidence_001","category":"explain","prompt":"Explain evidence.","subject_lemma":"evidence","expects_connective":true}
{"id":"hold_multi_paragraph_knowledge_002","category":"essay","prompt":"Write a paragraph about knowledge.","subject_lemma":"knowledge","expects_connective":true}
{"id":"hold_multi_compound_memory_003","category":"compose","prompt":"What is memory, and why does it matter?","subject_lemma":"memory","expects_connective":true}
{"id":"hold_multi_walk_inference_004","category":"walkthrough","prompt":"Walk me through inference.","subject_lemma":"inference","expects_connective":true}

View file

@ -0,0 +1,27 @@
# Walkthrough Chain
**Lane:** `walkthrough_chain`
Scores bounded relation walks over the reviewed teaching-chain substrate.
This lane tests path structure: an anchor subject plus deterministic
relation hops. It is separate from paragraph or multi-sentence fluency.
## Case Schema
```json
{
"id": "walk_truth_001",
"prompt": "Walk me through truth.",
"subject": "truth",
"max_hops": 2,
"expected_path": ["truth", "knowledge", "evidence"]
}
```
## Metrics
- `path_exact_rate`: actual path equals expected path.
- `anchor_rate`: first path element equals expected subject.
- `min_hop_rate`: actual path contains at least one relation hop.
- `bounded_rate`: path length never exceeds `max_hops + 1`.

View file

@ -0,0 +1,2 @@
{"id":"dev_walk_understanding_chain","prompt":"Walk me through understanding.","subject":"understanding","max_hops":2,"expected_path":["understanding","knowledge","evidence"]}

View file

@ -0,0 +1,3 @@
{"id":"walk_truth_chain_001","prompt":"Walk me through truth.","subject":"truth","max_hops":2,"expected_path":["truth","knowledge","evidence"]}
{"id":"walk_inference_chain_002","prompt":"Walk me through inference.","subject":"inference","max_hops":2,"expected_path":["inference","evidence","knowledge"]}
{"id":"walk_recall_chain_003","prompt":"Walk me through recall.","subject":"recall","max_hops":2,"expected_path":["recall","memory"]}

View file

@ -0,0 +1,84 @@
"""Walkthrough chain eval lane."""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
from chat.teaching_grounding import _all_chains_index
@dataclass
class LaneReport:
metrics: dict[str, Any] = field(default_factory=dict)
case_details: list[dict[str, Any]] = field(default_factory=list)
def _walk(subject: str, *, max_hops: int) -> tuple[str, ...]:
corpus = _all_chains_index()
path: list[str] = [subject.strip().lower()]
seen = {path[0]}
cursor = path[0]
for _ in range(max(0, max_hops)):
chain = corpus.get((cursor, "cause")) or corpus.get((cursor, "verification"))
if chain is None:
break
nxt = chain.object.strip().lower()
if not nxt or nxt in seen:
break
path.append(nxt)
seen.add(nxt)
cursor = nxt
return tuple(path)
def run_lane(cases: list[dict[str, Any]], config: Any = None) -> LaneReport: # noqa: ARG001
details: list[dict[str, Any]] = []
exact = 0
anchored = 0
min_hop = 0
bounded = 0
for case in cases:
subject = str(case["subject"]).strip().lower()
max_hops = int(case.get("max_hops", 2))
expected = tuple(str(x).strip().lower() for x in case.get("expected_path", ()))
actual = _walk(subject, max_hops=max_hops)
exact_match = actual == expected
anchor_match = bool(actual) and actual[0] == subject
has_hop = len(actual) >= 2
is_bounded = len(actual) <= max_hops + 1
exact += int(exact_match)
anchored += int(anchor_match)
min_hop += int(has_hop)
bounded += int(is_bounded)
details.append({
"case_id": case["id"],
"prompt": case.get("prompt", ""),
"subject": subject,
"max_hops": max_hops,
"expected_path": list(expected),
"actual_path": list(actual),
"path_exact": exact_match,
"anchor_match": anchor_match,
"min_hop": has_hop,
"bounded": is_bounded,
})
total = len(cases)
return LaneReport(
metrics={
"cases": total,
"path_exact_rate": round(exact / total, 4) if total else 0.0,
"anchor_rate": round(anchored / total, 4) if total else 0.0,
"min_hop_rate": round(min_hop / total, 4) if total else 0.0,
"bounded_rate": round(bounded / total, 4) if total else 0.0,
},
case_details=details,
)
__all__ = ["run_lane", "LaneReport"]

View file

@ -0,0 +1,2 @@
{"id":"hold_warm_compound_truth_001","category":"compound_no_drift","turns":[{"prompt":"What is truth, and why does it matter?","expected_grounding_source":"oov"},{"prompt":"What is truth, and why does it matter?","expected_grounding_source":"oov"}],"warm_invariants":["no_placeholder","warm_grounding_stability"]}
{"id":"hold_warm_walk_recall_002","category":"walkthrough_no_drift","turns":[{"prompt":"Walk me through recall.","expected_grounding_source":"pack"},{"prompt":"Walk me through recall.","expected_grounding_source":"pack"}],"warm_invariants":["no_placeholder","warm_grounding_stability"]}

View file

@ -13,9 +13,11 @@ import pytest
from benchmarks.articulation import (
INTENT_PROBE_PROMPTS,
CROSS_TOPIC_PROMPTS,
DISCOURSE_PLANNER_PROMPTS,
bench_breadth,
bench_cross_topic,
bench_determinism,
bench_discourse_planner,
bench_footprint,
bench_ollama_compare,
run_articulation_suite,
@ -117,6 +119,20 @@ def test_cross_topic_visits_every_prompt() -> None:
}
# ---------------------------------------------------------------------------
# Discourse planner
# ---------------------------------------------------------------------------
def test_discourse_planner_bench_covers_new_prompt_shapes() -> None:
probes, metrics = bench_discourse_planner()
assert [p.label for p in probes] == [label for label, _ in DISCOURSE_PLANNER_PROMPTS]
assert metrics["cases"] == len(DISCOURSE_PLANNER_PROMPTS)
assert "articulate_sentence_rate" in metrics
labels = {p.label for p in probes}
assert {"COMPOUND", "WALKTHROUGH"} <= labels
# ---------------------------------------------------------------------------
# Ollama (skipped when binary absent)
# ---------------------------------------------------------------------------
@ -145,5 +161,7 @@ def test_run_articulation_suite_emits_shaped_report() -> None:
assert d["determinism_all_identical"] is True
assert isinstance(d["footprint_samples"], list)
assert d["ollama"]["status"] == "skipped"
assert isinstance(d["discourse_planner"], list)
assert d["discourse_planner_metrics"]["cases"] == len(DISCOURSE_PLANNER_PROMPTS)
# Cross-topic walk runs every entry.
assert len(d["cross_topic"]) == len(CROSS_TOPIC_PROMPTS)

View file

@ -1,9 +1,10 @@
"""Contract tests for the ``cold_start_grounding`` eval lane.
This lane commits the 44-prompt routing probe described in
This lane commits the 48-prompt routing probe described in
``evals/cold_start_grounding/contract.md``. The probe is the durable,
replayable artifact behind the 2026-05-19 lift from 52% "I don't know"
responses to 0% (out of 44 realistic conversational prompts).
responses to 0% (out of 44 realistic conversational prompts), then
extended it with expository, compound, and walkthrough surfaces.
These tests pin:
@ -48,7 +49,7 @@ class TestCaseSetIntegrity:
def test_public_case_count(self) -> None:
cases = load_cases(_PUBLIC_CASES)
assert len(cases) == 44
assert len(cases) == 48
def test_every_case_has_required_fields(self) -> None:
for case in load_cases(_PUBLIC_CASES):
@ -179,4 +180,4 @@ class TestResultSerialization:
result = run_lane(lane, version="v1", split="public")
payload = json.dumps(result.as_dict(), sort_keys=True)
reloaded = json.loads(payload)
assert reloaded["metrics"]["cases"] == 44
assert reloaded["metrics"]["cases"] == 48

View file

@ -0,0 +1,33 @@
"""Contract tests for compound and walkthrough articulation eval lanes."""
from __future__ import annotations
from evals.framework import get_lane, run_lane
def test_compound_intent_decomposition_public_passes() -> None:
lane = get_lane("compound_intent_decomposition")
result = run_lane(lane, version="v1", split="public")
assert result.metrics["decomposition_accuracy"] == 1.0
assert result.metrics["subject_accuracy"] == 1.0
def test_walkthrough_chain_public_passes() -> None:
lane = get_lane("walkthrough_chain")
result = run_lane(lane, version="v1", split="public")
assert result.metrics["path_exact_rate"] == 1.0
assert result.metrics["anchor_rate"] == 1.0
assert result.metrics["bounded_rate"] == 1.0
def test_chat_spine_holdout_splits_are_runnable() -> None:
for lane_name in (
"multi_sentence_response",
"cold_start_grounding",
"conversational_thread_coherence",
"warmed_session_consistency",
):
lane = get_lane(lane_name)
result = run_lane(lane, version="v1", split="holdout")
assert result.metrics["cases"] >= 1

View file

@ -21,6 +21,7 @@ from generate.intent import (
DialogueIntent,
IntentTag,
ResponseMode,
classify_compound_intent,
classify_intent,
classify_response_mode,
)
@ -118,3 +119,23 @@ class TestExistingDefinitionRulesUntouched:
result = classify_intent(prompt)
assert result.tag is IntentTag.DEFINITION
assert result.subject == subject
class TestCompoundAndWalkthroughAnchors:
def test_compound_definition_strips_causal_tail_from_subject(self) -> None:
result = classify_compound_intent("What is truth, and why does it matter?")
assert result.primary.tag is IntentTag.DEFINITION
assert result.primary.subject == "truth"
def test_compound_definition_cause_decomposes_to_two_atoms(self) -> None:
atoms = classify_compound_intent("What is truth, and why does it matter?")
assert atoms.parts == (
DialogueIntent(tag=IntentTag.DEFINITION, subject="truth"),
DialogueIntent(tag=IntentTag.CAUSE, subject="truth"),
)
def test_simple_walkthrough_gets_grounded_definition_anchor(self) -> None:
result = classify_intent("Walk me through recall.")
assert result.tag is IntentTag.DEFINITION
assert result.subject == "recall"
assert classify_response_mode("Walk me through recall.") is ResponseMode.WALKTHROUGH

View file

@ -179,11 +179,13 @@ class TestClassifyIntentUnchanged:
class TestIntentModeOrthogonality:
def test_definition_plus_paragraph(self) -> None:
prompt = "Write a paragraph about truth"
# "Write a paragraph about" isn't a DEFINITION trigger, so the
# intent falls through to UNKNOWN — but ResponseMode still picks
# up PARAGRAPH. This documents the orthogonality: mode does not
# *cause* a particular intent.
# The semantic intent and presentation mode are still distinct:
# the intent anchors the subject as a definition, while
# ResponseMode carries the paragraph shape.
intent = classify_intent(prompt)
mode = classify_response_mode(prompt)
assert intent.tag is IntentTag.DEFINITION
assert intent.subject == "truth"
assert mode is ResponseMode.PARAGRAPH
def test_narrative_plus_explain(self) -> None: