feat(evals+bench): isolation lanes, holdouts, planner-on bench sub-bench
Sharpens the measurement layer to match the runtime spine landed in07fefb9/7af7892/4e3ddee. Pure eval/benchmark/holdout work — no runtime or planner code changed. New isolation lanes ------------------- * ``evals/compound_intent_decomposition/`` — single-purpose lane for the new ``classify_compound_intent`` decomposer. Metrics: ``decomposition_accuracy``, ``atom_precision``, ``subject_accuracy``. Public: ``decomposition=1.0`` on4e3ddee. * ``evals/walkthrough_chain/`` — single-purpose lane for the new WALKTHROUGH sequential teaching-chain walk. Metrics: ``path_exact_rate``, ``anchor_rate``, ``min_hop_rate``, ``bounded_rate``. Public: ``path_exact=1.0`` on4e3ddee. Without these, regressions in compound decomposition or the walkthrough walk would show up as noise in ``multi_sentence_response``. Each capability now has a single load-bearing metric on its own lane. Cold-start lane sharpened ------------------------- * ``evals/cold_start_grounding/public/v1/cases.jsonl`` extended with expository, compound, and walkthrough cases (48 total cases across 19 categories including new ``expository_definition``, ``compound_definition_cause``, ``walkthrough_definition``). * ``evals/cold_start_grounding/runner.py`` uses ``classify_compound_intent(...).primary`` for compound subject scoring — previously misattributed subjects on multi-part prompts. Holdouts for the long-span lanes -------------------------------- Until now only the cognition lane had a holdout split. Adding holdouts to the long-span lanes gives the planner work somewhere to fail honestly when we widen: * ``evals/cold_start_grounding/holdouts/v1/cases.jsonl`` (5 cases) * ``evals/multi_sentence_response/holdouts/v1/cases.jsonl`` (5 cases) * ``evals/conversational_thread_coherence/holdouts/v1/cases.jsonl`` (3 cases) * ``evals/warmed_session_consistency/holdouts/v1/cases.jsonl`` (2 cases) Discourse-planner-on bench sub-bench ------------------------------------ * ``benchmarks/articulation.py`` adds a planner-on sub-bench that reports ``articulate_sentence_rate`` alongside the existing throughput metrics. Baselines articulation under load before any follow-up touches ``compute_trace_hash``. Test coverage ------------- * ``tests/test_compound_walkthrough_eval_lanes.py`` — new file pinning the two new lane runners. * ``tests/test_articulation_bench.py``, ``tests/test_cold_start_grounding_lane.py``, ``tests/test_intent_explain_paragraph.py``, ``tests/test_response_mode_classifier.py`` — updated for new cases and assertions. Validation ---------- * 152/152 active tests pass on the listed surfaces (2 skipped). * smoke suite 67/67. * cognition eval byte-identical: public 100/100/91.7/100. * multi_sentence flag_on: articulate=1.0, disclosure=0.0, unarticulate=0.0 * compound_intent_decomp public: decomposition=1.0 * walkthrough_chain public: path_exact=1.0 * cold_start_grounding public (48 cases): intent=1.0, grounding=1.0, subject=1.0
This commit is contained in:
parent
4e3ddee91f
commit
e985790a03
20 changed files with 436 additions and 15 deletions
|
|
@ -36,6 +36,12 @@ Sub-benches:
|
|||
prompts. Skipped (status: ``skipped`` instead of ``failed``)
|
||||
when the ``ollama`` binary is not on ``PATH``.
|
||||
|
||||
6. **discourse-planner** — Runs expository, compound, and
|
||||
walkthrough prompts with ``RuntimeConfig(discourse_planner=True)``
|
||||
and reports honest sentence buckets. This keeps the benchmark
|
||||
aligned with the multi-clause articulation spine instead of only
|
||||
the older intent-breadth probes.
|
||||
|
||||
The whole suite is deterministic on the CORE side — no clock-time
|
||||
or RNG influence on what gets emitted. Walltime sampling lives in
|
||||
``benchmarks.cost``; this module focuses on capability + identity.
|
||||
|
|
@ -88,6 +94,13 @@ DETERMINISM_PROMPTS: tuple[str, ...] = (
|
|||
"Give me an example of memory.",
|
||||
)
|
||||
|
||||
DISCOURSE_PLANNER_PROMPTS: tuple[tuple[str, str], ...] = (
|
||||
("EXPLAIN", "Explain truth."),
|
||||
("PARAGRAPH", "Write a paragraph about truth."),
|
||||
("COMPOUND", "What is truth, and why does it matter?"),
|
||||
("WALKTHROUGH", "Walk me through recall."),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Report shapes
|
||||
|
|
@ -127,6 +140,18 @@ class CrossTopicTurn:
|
|||
surface_snippet: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DiscoursePlannerProbe:
|
||||
label: str
|
||||
prompt: str
|
||||
intent_tag: str
|
||||
grounding_source: str
|
||||
sentence_count: int
|
||||
articulate_sentence: bool
|
||||
disclosure_sentence: bool
|
||||
surface_snippet: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OllamaPair:
|
||||
prompt: str
|
||||
|
|
@ -148,6 +173,8 @@ class ArticulationReport:
|
|||
footprint_per_turn_delta_bytes: float = 0.0
|
||||
cross_topic: list[CrossTopicTurn] = field(default_factory=list)
|
||||
anaphora_fire_count: int = 0
|
||||
discourse_planner: list[DiscoursePlannerProbe] = field(default_factory=list)
|
||||
discourse_planner_metrics: dict[str, Any] = field(default_factory=dict)
|
||||
ollama: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
|
|
@ -164,6 +191,8 @@ class ArticulationReport:
|
|||
),
|
||||
"cross_topic": [t.__dict__ for t in self.cross_topic],
|
||||
"anaphora_fire_count": self.anaphora_fire_count,
|
||||
"discourse_planner": [p.__dict__ for p in self.discourse_planner],
|
||||
"discourse_planner_metrics": self.discourse_planner_metrics,
|
||||
"ollama": self.ollama,
|
||||
}
|
||||
|
||||
|
|
@ -178,6 +207,12 @@ def _snippet(s: str, n: int = 120) -> str:
|
|||
return s if len(s) <= n else s[: n - 1] + "…"
|
||||
|
||||
|
||||
def _sentence_count(surface: str) -> int:
|
||||
from evals.multi_sentence_response.runner import _split_sentences, _strip_provenance
|
||||
|
||||
return len(_split_sentences(_strip_provenance(surface)))
|
||||
|
||||
|
||||
def _classify_prompt(prompt: str) -> str:
|
||||
"""Re-derive the intent label from the prompt text for the report.
|
||||
|
||||
|
|
@ -293,6 +328,48 @@ def bench_cross_topic() -> tuple[list[CrossTopicTurn], int]:
|
|||
return out, fires
|
||||
|
||||
|
||||
def bench_discourse_planner() -> tuple[list[DiscoursePlannerProbe], dict[str, Any]]:
|
||||
from chat.runtime import ChatRuntime
|
||||
from core.config import RuntimeConfig
|
||||
|
||||
out: list[DiscoursePlannerProbe] = []
|
||||
for label, prompt in DISCOURSE_PLANNER_PROMPTS:
|
||||
rt = ChatRuntime(config=RuntimeConfig(discourse_planner=True))
|
||||
resp = rt.chat(prompt)
|
||||
grounding = getattr(resp, "grounding_source", "unknown")
|
||||
sentence_count = _sentence_count(resp.surface)
|
||||
articulate = sentence_count >= 2 and grounding in {"pack", "teaching"}
|
||||
disclosure = sentence_count >= 2 and grounding in {"oov", "refusal", "none"}
|
||||
out.append(DiscoursePlannerProbe(
|
||||
label=label,
|
||||
prompt=prompt,
|
||||
intent_tag=_classify_prompt(prompt),
|
||||
grounding_source=grounding,
|
||||
sentence_count=sentence_count,
|
||||
articulate_sentence=articulate,
|
||||
disclosure_sentence=disclosure,
|
||||
surface_snippet=_snippet(resp.surface),
|
||||
))
|
||||
|
||||
total = len(out)
|
||||
metrics = {
|
||||
"cases": total,
|
||||
"articulate_sentence_rate": (
|
||||
round(sum(1 for p in out if p.articulate_sentence) / total, 4)
|
||||
if total else 0.0
|
||||
),
|
||||
"disclosure_sentence_rate": (
|
||||
round(sum(1 for p in out if p.disclosure_sentence) / total, 4)
|
||||
if total else 0.0
|
||||
),
|
||||
"multi_sentence_rate": (
|
||||
round(sum(1 for p in out if p.sentence_count >= 2) / total, 4)
|
||||
if total else 0.0
|
||||
),
|
||||
}
|
||||
return out, metrics
|
||||
|
||||
|
||||
def _have_ollama() -> bool:
|
||||
return shutil.which("ollama") is not None
|
||||
|
||||
|
|
@ -409,6 +486,9 @@ def run_articulation_suite(
|
|||
ct_turns, ct_fires = bench_cross_topic()
|
||||
report.cross_topic = ct_turns
|
||||
report.anaphora_fire_count = ct_fires
|
||||
dp_probes, dp_metrics = bench_discourse_planner()
|
||||
report.discourse_planner = dp_probes
|
||||
report.discourse_planner_metrics = dp_metrics
|
||||
report.ollama = bench_ollama_compare(
|
||||
model=ollama_model,
|
||||
prompts=DETERMINISM_PROMPTS[:3], # subset — ollama is slow
|
||||
|
|
@ -425,14 +505,14 @@ def format_summary(report: ArticulationReport) -> str:
|
|||
out.append("Articulation benchmark suite")
|
||||
out.append("=" * 76)
|
||||
out.append("")
|
||||
out.append("[1/5] Intent breadth — every supported intent shape:")
|
||||
out.append("[1/6] Intent breadth — every supported intent shape:")
|
||||
for p in report.breadth:
|
||||
out.append(
|
||||
f" {p.label:30s} {p.intent_tag:14s} {p.grounding_source:9s} "
|
||||
f"{_snippet(p.surface_snippet, 80)}"
|
||||
)
|
||||
out.append("")
|
||||
out.append("[2/5] Determinism — same prompt → byte-identical surface:")
|
||||
out.append("[2/6] Determinism — same prompt → byte-identical surface:")
|
||||
for c in report.determinism:
|
||||
flag = "OK" if c.unique_surfaces == 1 else "FAIL"
|
||||
out.append(
|
||||
|
|
@ -443,7 +523,7 @@ def format_summary(report: ArticulationReport) -> str:
|
|||
f" all_identical = {report.determinism_all_identical}"
|
||||
)
|
||||
out.append("")
|
||||
out.append("[3/5] Memory footprint — single runtime, repeated turns:")
|
||||
out.append("[3/6] Memory footprint — single runtime, repeated turns:")
|
||||
if report.footprint:
|
||||
out.append(
|
||||
f" start = {report.footprint_start_bytes / 1024 / 1024:.1f} MiB "
|
||||
|
|
@ -455,7 +535,7 @@ def format_summary(report: ArticulationReport) -> str:
|
|||
f"{report.footprint_per_turn_delta_bytes / 1024:.2f} KiB"
|
||||
)
|
||||
out.append("")
|
||||
out.append("[4/5] Cross-topic context — thread anaphora across subjects:")
|
||||
out.append("[4/6] Cross-topic context — thread anaphora across subjects:")
|
||||
for t in report.cross_topic:
|
||||
marker = "↩" if t.anaphora_fired else " "
|
||||
out.append(
|
||||
|
|
@ -472,7 +552,16 @@ def format_summary(report: ArticulationReport) -> str:
|
|||
"fire rate (which is the architectural ceiling, not a defect)."
|
||||
)
|
||||
out.append("")
|
||||
out.append("[5/5] Ollama side-by-side:")
|
||||
out.append("[5/6] Discourse planner — flag-on articulation spine:")
|
||||
for p in report.discourse_planner:
|
||||
marker = "A" if p.articulate_sentence else ("D" if p.disclosure_sentence else " ")
|
||||
out.append(
|
||||
f" [{marker}] {p.label:12s} {p.intent_tag:12s} {p.grounding_source:9s} "
|
||||
f"{p.sentence_count} sentence(s) {_snippet(p.prompt, 46)}"
|
||||
)
|
||||
out.append(f" metrics = {report.discourse_planner_metrics}")
|
||||
out.append("")
|
||||
out.append("[6/6] Ollama side-by-side:")
|
||||
status = report.ollama.get("status", "skipped")
|
||||
if status == "skipped":
|
||||
out.append(f" skipped — {report.ollama.get('reason', '')}")
|
||||
|
|
@ -502,10 +591,12 @@ __all__ = [
|
|||
"INTENT_PROBE_PROMPTS",
|
||||
"CROSS_TOPIC_PROMPTS",
|
||||
"DETERMINISM_PROMPTS",
|
||||
"DISCOURSE_PLANNER_PROMPTS",
|
||||
"bench_breadth",
|
||||
"bench_determinism",
|
||||
"bench_footprint",
|
||||
"bench_cross_topic",
|
||||
"bench_discourse_planner",
|
||||
"bench_ollama_compare",
|
||||
"run_articulation_suite",
|
||||
"format_summary",
|
||||
|
|
|
|||
5
evals/cold_start_grounding/holdouts/v1/cases.jsonl
Normal file
5
evals/cold_start_grounding/holdouts/v1/cases.jsonl
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{"id":"hold_def_evidence_001","prompt":"What is evidence?","category":"definition_cognition","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"evidence"}
|
||||
{"id":"hold_explain_knowledge_002","prompt":"Explain knowledge.","category":"expository_definition","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"knowledge"}
|
||||
{"id":"hold_paragraph_truth_003","prompt":"Write a paragraph about truth.","category":"expository_definition","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"truth"}
|
||||
{"id":"hold_compound_memory_004","prompt":"What is memory, and why does it matter?","category":"compound_definition_cause","expected_intent":"definition","expected_grounding_source":"oov","expected_subject":"memory"}
|
||||
{"id":"hold_walk_inference_005","prompt":"Walk me through inference.","category":"walkthrough_definition","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"inference"}
|
||||
|
|
@ -42,3 +42,7 @@
|
|||
{"id":"cause_why_truth_042","prompt":"Why is truth important?","category":"cause_with_teaching_chain","expected_intent":"cause","expected_grounding_source":"teaching","expected_subject":"truth"}
|
||||
{"id":"cause_how_memory_043","prompt":"How does memory work?","category":"cause_no_teaching_chain","expected_intent":"cause","expected_grounding_source":"none","expected_subject":"memory"}
|
||||
{"id":"cause_what_causes_doubt_044","prompt":"What causes doubt?","category":"cause_no_teaching_chain","expected_intent":"cause","expected_grounding_source":"none","expected_subject":"doubt"}
|
||||
{"id":"explain_truth_045","prompt":"Explain truth.","category":"expository_definition","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"truth"}
|
||||
{"id":"paragraph_memory_046","prompt":"Write a paragraph about memory.","category":"expository_definition","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"memory"}
|
||||
{"id":"compound_truth_matter_047","prompt":"What is truth, and why does it matter?","category":"compound_definition_cause","expected_intent":"definition","expected_grounding_source":"oov","expected_subject":"truth"}
|
||||
{"id":"walkthrough_recall_048","prompt":"Walk me through recall.","category":"walkthrough_definition","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"recall"}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ from dataclasses import dataclass, field
|
|||
from typing import Any
|
||||
|
||||
from chat.runtime import ChatRuntime
|
||||
from generate.intent import classify_intent
|
||||
from generate.intent import classify_compound_intent, classify_intent
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -58,7 +58,8 @@ def _run_case(case: dict[str, Any]) -> CaseResult:
|
|||
# Classify intent independently for the subject-match check —
|
||||
# avoids round-tripping through the runtime when the prompt
|
||||
# bypasses pack-grounding for an OOV/none case.
|
||||
classified = classify_intent(prompt)
|
||||
compound = classify_compound_intent(prompt)
|
||||
classified = compound.primary if compound.is_compound() else classify_intent(prompt)
|
||||
actual_subject = (classified.subject or "").strip().lower()
|
||||
|
||||
# Fresh runtime — cold-start invariant.
|
||||
|
|
|
|||
27
evals/compound_intent_decomposition/contract.md
Normal file
27
evals/compound_intent_decomposition/contract.md
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
# Compound Intent Decomposition
|
||||
|
||||
**Lane:** `compound_intent_decomposition`
|
||||
|
||||
Scores whether a compound conversational prompt is decomposed into the
|
||||
intended semantic atoms before generation. This lane is structural: it
|
||||
does not grade paragraph fluency or final surface length.
|
||||
|
||||
## Case Schema
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "compound_truth_001",
|
||||
"prompt": "What is truth, and why does it matter?",
|
||||
"expected_atoms": [
|
||||
{"intent": "definition", "subject": "truth"},
|
||||
{"intent": "cause", "subject": "truth"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Metrics
|
||||
|
||||
- `decomposition_accuracy`: exact ordered atom match.
|
||||
- `atom_precision`: expected atoms found in the same position.
|
||||
- `subject_accuracy`: expected subjects recovered in the same position.
|
||||
|
||||
2
evals/compound_intent_decomposition/dev/cases.jsonl
Normal file
2
evals/compound_intent_decomposition/dev/cases.jsonl
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
{"id":"dev_compound_knowledge_definition_cause","prompt":"What is knowledge, and why does it matter?","expected_atoms":[{"intent":"definition","subject":"knowledge"},{"intent":"cause","subject":"knowledge"}]}
|
||||
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
{"id":"compound_truth_definition_cause_001","prompt":"What is truth, and why does it matter?","expected_atoms":[{"intent":"definition","subject":"truth"},{"intent":"cause","subject":"truth"}]}
|
||||
{"id":"compound_memory_definition_cause_002","prompt":"What is memory, and why does it matter?","expected_atoms":[{"intent":"definition","subject":"memory"},{"intent":"cause","subject":"memory"}]}
|
||||
|
||||
87
evals/compound_intent_decomposition/runner.py
Normal file
87
evals/compound_intent_decomposition/runner.py
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
"""Compound intent decomposition eval lane."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from generate.intent import classify_compound_intent
|
||||
|
||||
|
||||
@dataclass
|
||||
class LaneReport:
|
||||
metrics: dict[str, Any] = field(default_factory=dict)
|
||||
case_details: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
|
||||
def _expected_atoms(case: dict[str, Any]) -> list[dict[str, str]]:
|
||||
atoms = case.get("expected_atoms")
|
||||
if not isinstance(atoms, list):
|
||||
return []
|
||||
out: list[dict[str, str]] = []
|
||||
for atom in atoms:
|
||||
if not isinstance(atom, dict):
|
||||
continue
|
||||
intent = str(atom.get("intent", "")).strip().lower()
|
||||
subject = str(atom.get("subject", "")).strip().lower()
|
||||
out.append({"intent": intent, "subject": subject})
|
||||
return out
|
||||
|
||||
|
||||
def run_lane(cases: list[dict[str, Any]], config: Any = None) -> LaneReport: # noqa: ARG001
|
||||
details: list[dict[str, Any]] = []
|
||||
exact = 0
|
||||
atom_positions = 0
|
||||
atom_correct = 0
|
||||
subject_positions = 0
|
||||
subject_correct = 0
|
||||
|
||||
for case in cases:
|
||||
expected = _expected_atoms(case)
|
||||
actual = [
|
||||
{"intent": atom.tag.value, "subject": atom.subject.strip().lower()}
|
||||
for atom in classify_compound_intent(case["prompt"]).parts
|
||||
]
|
||||
exact_match = actual == expected
|
||||
if exact_match:
|
||||
exact += 1
|
||||
|
||||
for idx, exp in enumerate(expected):
|
||||
if idx >= len(actual):
|
||||
atom_positions += 1
|
||||
subject_positions += 1
|
||||
continue
|
||||
got = actual[idx]
|
||||
atom_positions += 1
|
||||
subject_positions += 1
|
||||
if got == exp:
|
||||
atom_correct += 1
|
||||
if got["subject"] == exp["subject"]:
|
||||
subject_correct += 1
|
||||
|
||||
details.append({
|
||||
"case_id": case["id"],
|
||||
"prompt": case["prompt"],
|
||||
"expected_atoms": expected,
|
||||
"actual_atoms": actual,
|
||||
"exact_match": exact_match,
|
||||
})
|
||||
|
||||
total = len(cases)
|
||||
return LaneReport(
|
||||
metrics={
|
||||
"cases": total,
|
||||
"decomposition_accuracy": round(exact / total, 4) if total else 0.0,
|
||||
"atom_precision": (
|
||||
round(atom_correct / atom_positions, 4) if atom_positions else 1.0
|
||||
),
|
||||
"subject_accuracy": (
|
||||
round(subject_correct / subject_positions, 4)
|
||||
if subject_positions else 1.0
|
||||
),
|
||||
},
|
||||
case_details=details,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["run_lane", "LaneReport"]
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
{"id":"hold_thread_compound_walk_001","category":"compound_walk","turns":[{"prompt":"What is truth?","subject_lemma":"truth"},{"prompt":"What is truth, and why does it matter?","subject_lemma":"truth"},{"prompt":"Walk me through inference.","subject_lemma":"inference"},{"prompt":"What is truth?","subject_lemma":"truth","is_replay_of_prompt_at_turn":0}]}
|
||||
{"id":"hold_thread_topic_return_002","category":"topic_shift","turns":[{"prompt":"What is evidence?","subject_lemma":"evidence"},{"prompt":"What is recall?","subject_lemma":"recall"},{"prompt":"Write a paragraph about knowledge.","subject_lemma":"knowledge"},{"prompt":"What is evidence?","subject_lemma":"evidence","is_replay_of_prompt_at_turn":0}]}
|
||||
|
||||
5
evals/multi_sentence_response/holdouts/v1/cases.jsonl
Normal file
5
evals/multi_sentence_response/holdouts/v1/cases.jsonl
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{"id":"hold_multi_explain_evidence_001","category":"explain","prompt":"Explain evidence.","subject_lemma":"evidence","expects_connective":true}
|
||||
{"id":"hold_multi_paragraph_knowledge_002","category":"essay","prompt":"Write a paragraph about knowledge.","subject_lemma":"knowledge","expects_connective":true}
|
||||
{"id":"hold_multi_compound_memory_003","category":"compose","prompt":"What is memory, and why does it matter?","subject_lemma":"memory","expects_connective":true}
|
||||
{"id":"hold_multi_walk_inference_004","category":"walkthrough","prompt":"Walk me through inference.","subject_lemma":"inference","expects_connective":true}
|
||||
|
||||
27
evals/walkthrough_chain/contract.md
Normal file
27
evals/walkthrough_chain/contract.md
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
# Walkthrough Chain
|
||||
|
||||
**Lane:** `walkthrough_chain`
|
||||
|
||||
Scores bounded relation walks over the reviewed teaching-chain substrate.
|
||||
This lane tests path structure: an anchor subject plus deterministic
|
||||
relation hops. It is separate from paragraph or multi-sentence fluency.
|
||||
|
||||
## Case Schema
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "walk_truth_001",
|
||||
"prompt": "Walk me through truth.",
|
||||
"subject": "truth",
|
||||
"max_hops": 2,
|
||||
"expected_path": ["truth", "knowledge", "evidence"]
|
||||
}
|
||||
```
|
||||
|
||||
## Metrics
|
||||
|
||||
- `path_exact_rate`: actual path equals expected path.
|
||||
- `anchor_rate`: first path element equals expected subject.
|
||||
- `min_hop_rate`: actual path contains at least one relation hop.
|
||||
- `bounded_rate`: path length never exceeds `max_hops + 1`.
|
||||
|
||||
2
evals/walkthrough_chain/dev/cases.jsonl
Normal file
2
evals/walkthrough_chain/dev/cases.jsonl
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
{"id":"dev_walk_understanding_chain","prompt":"Walk me through understanding.","subject":"understanding","max_hops":2,"expected_path":["understanding","knowledge","evidence"]}
|
||||
|
||||
3
evals/walkthrough_chain/public/v1/cases.jsonl
Normal file
3
evals/walkthrough_chain/public/v1/cases.jsonl
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
{"id":"walk_truth_chain_001","prompt":"Walk me through truth.","subject":"truth","max_hops":2,"expected_path":["truth","knowledge","evidence"]}
|
||||
{"id":"walk_inference_chain_002","prompt":"Walk me through inference.","subject":"inference","max_hops":2,"expected_path":["inference","evidence","knowledge"]}
|
||||
{"id":"walk_recall_chain_003","prompt":"Walk me through recall.","subject":"recall","max_hops":2,"expected_path":["recall","memory"]}
|
||||
84
evals/walkthrough_chain/runner.py
Normal file
84
evals/walkthrough_chain/runner.py
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
"""Walkthrough chain eval lane."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from chat.teaching_grounding import _all_chains_index
|
||||
|
||||
|
||||
@dataclass
|
||||
class LaneReport:
|
||||
metrics: dict[str, Any] = field(default_factory=dict)
|
||||
case_details: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
|
||||
def _walk(subject: str, *, max_hops: int) -> tuple[str, ...]:
|
||||
corpus = _all_chains_index()
|
||||
path: list[str] = [subject.strip().lower()]
|
||||
seen = {path[0]}
|
||||
cursor = path[0]
|
||||
for _ in range(max(0, max_hops)):
|
||||
chain = corpus.get((cursor, "cause")) or corpus.get((cursor, "verification"))
|
||||
if chain is None:
|
||||
break
|
||||
nxt = chain.object.strip().lower()
|
||||
if not nxt or nxt in seen:
|
||||
break
|
||||
path.append(nxt)
|
||||
seen.add(nxt)
|
||||
cursor = nxt
|
||||
return tuple(path)
|
||||
|
||||
|
||||
def run_lane(cases: list[dict[str, Any]], config: Any = None) -> LaneReport: # noqa: ARG001
|
||||
details: list[dict[str, Any]] = []
|
||||
exact = 0
|
||||
anchored = 0
|
||||
min_hop = 0
|
||||
bounded = 0
|
||||
|
||||
for case in cases:
|
||||
subject = str(case["subject"]).strip().lower()
|
||||
max_hops = int(case.get("max_hops", 2))
|
||||
expected = tuple(str(x).strip().lower() for x in case.get("expected_path", ()))
|
||||
actual = _walk(subject, max_hops=max_hops)
|
||||
exact_match = actual == expected
|
||||
anchor_match = bool(actual) and actual[0] == subject
|
||||
has_hop = len(actual) >= 2
|
||||
is_bounded = len(actual) <= max_hops + 1
|
||||
|
||||
exact += int(exact_match)
|
||||
anchored += int(anchor_match)
|
||||
min_hop += int(has_hop)
|
||||
bounded += int(is_bounded)
|
||||
|
||||
details.append({
|
||||
"case_id": case["id"],
|
||||
"prompt": case.get("prompt", ""),
|
||||
"subject": subject,
|
||||
"max_hops": max_hops,
|
||||
"expected_path": list(expected),
|
||||
"actual_path": list(actual),
|
||||
"path_exact": exact_match,
|
||||
"anchor_match": anchor_match,
|
||||
"min_hop": has_hop,
|
||||
"bounded": is_bounded,
|
||||
})
|
||||
|
||||
total = len(cases)
|
||||
return LaneReport(
|
||||
metrics={
|
||||
"cases": total,
|
||||
"path_exact_rate": round(exact / total, 4) if total else 0.0,
|
||||
"anchor_rate": round(anchored / total, 4) if total else 0.0,
|
||||
"min_hop_rate": round(min_hop / total, 4) if total else 0.0,
|
||||
"bounded_rate": round(bounded / total, 4) if total else 0.0,
|
||||
},
|
||||
case_details=details,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["run_lane", "LaneReport"]
|
||||
|
||||
2
evals/warmed_session_consistency/holdouts/v1/cases.jsonl
Normal file
2
evals/warmed_session_consistency/holdouts/v1/cases.jsonl
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
{"id":"hold_warm_compound_truth_001","category":"compound_no_drift","turns":[{"prompt":"What is truth, and why does it matter?","expected_grounding_source":"oov"},{"prompt":"What is truth, and why does it matter?","expected_grounding_source":"oov"}],"warm_invariants":["no_placeholder","warm_grounding_stability"]}
|
||||
{"id":"hold_warm_walk_recall_002","category":"walkthrough_no_drift","turns":[{"prompt":"Walk me through recall.","expected_grounding_source":"pack"},{"prompt":"Walk me through recall.","expected_grounding_source":"pack"}],"warm_invariants":["no_placeholder","warm_grounding_stability"]}
|
||||
|
|
@ -13,9 +13,11 @@ import pytest
|
|||
from benchmarks.articulation import (
|
||||
INTENT_PROBE_PROMPTS,
|
||||
CROSS_TOPIC_PROMPTS,
|
||||
DISCOURSE_PLANNER_PROMPTS,
|
||||
bench_breadth,
|
||||
bench_cross_topic,
|
||||
bench_determinism,
|
||||
bench_discourse_planner,
|
||||
bench_footprint,
|
||||
bench_ollama_compare,
|
||||
run_articulation_suite,
|
||||
|
|
@ -117,6 +119,20 @@ def test_cross_topic_visits_every_prompt() -> None:
|
|||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Discourse planner
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_discourse_planner_bench_covers_new_prompt_shapes() -> None:
|
||||
probes, metrics = bench_discourse_planner()
|
||||
assert [p.label for p in probes] == [label for label, _ in DISCOURSE_PLANNER_PROMPTS]
|
||||
assert metrics["cases"] == len(DISCOURSE_PLANNER_PROMPTS)
|
||||
assert "articulate_sentence_rate" in metrics
|
||||
labels = {p.label for p in probes}
|
||||
assert {"COMPOUND", "WALKTHROUGH"} <= labels
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ollama (skipped when binary absent)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -145,5 +161,7 @@ def test_run_articulation_suite_emits_shaped_report() -> None:
|
|||
assert d["determinism_all_identical"] is True
|
||||
assert isinstance(d["footprint_samples"], list)
|
||||
assert d["ollama"]["status"] == "skipped"
|
||||
assert isinstance(d["discourse_planner"], list)
|
||||
assert d["discourse_planner_metrics"]["cases"] == len(DISCOURSE_PLANNER_PROMPTS)
|
||||
# Cross-topic walk runs every entry.
|
||||
assert len(d["cross_topic"]) == len(CROSS_TOPIC_PROMPTS)
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
"""Contract tests for the ``cold_start_grounding`` eval lane.
|
||||
|
||||
This lane commits the 44-prompt routing probe described in
|
||||
This lane commits the 48-prompt routing probe described in
|
||||
``evals/cold_start_grounding/contract.md``. The probe is the durable,
|
||||
replayable artifact behind the 2026-05-19 lift from 52% "I don't know"
|
||||
responses to 0% (out of 44 realistic conversational prompts).
|
||||
responses to 0% (out of 44 realistic conversational prompts), then
|
||||
extended it with expository, compound, and walkthrough surfaces.
|
||||
|
||||
These tests pin:
|
||||
|
||||
|
|
@ -48,7 +49,7 @@ class TestCaseSetIntegrity:
|
|||
|
||||
def test_public_case_count(self) -> None:
|
||||
cases = load_cases(_PUBLIC_CASES)
|
||||
assert len(cases) == 44
|
||||
assert len(cases) == 48
|
||||
|
||||
def test_every_case_has_required_fields(self) -> None:
|
||||
for case in load_cases(_PUBLIC_CASES):
|
||||
|
|
@ -179,4 +180,4 @@ class TestResultSerialization:
|
|||
result = run_lane(lane, version="v1", split="public")
|
||||
payload = json.dumps(result.as_dict(), sort_keys=True)
|
||||
reloaded = json.loads(payload)
|
||||
assert reloaded["metrics"]["cases"] == 44
|
||||
assert reloaded["metrics"]["cases"] == 48
|
||||
|
|
|
|||
33
tests/test_compound_walkthrough_eval_lanes.py
Normal file
33
tests/test_compound_walkthrough_eval_lanes.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
"""Contract tests for compound and walkthrough articulation eval lanes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from evals.framework import get_lane, run_lane
|
||||
|
||||
|
||||
def test_compound_intent_decomposition_public_passes() -> None:
|
||||
lane = get_lane("compound_intent_decomposition")
|
||||
result = run_lane(lane, version="v1", split="public")
|
||||
assert result.metrics["decomposition_accuracy"] == 1.0
|
||||
assert result.metrics["subject_accuracy"] == 1.0
|
||||
|
||||
|
||||
def test_walkthrough_chain_public_passes() -> None:
|
||||
lane = get_lane("walkthrough_chain")
|
||||
result = run_lane(lane, version="v1", split="public")
|
||||
assert result.metrics["path_exact_rate"] == 1.0
|
||||
assert result.metrics["anchor_rate"] == 1.0
|
||||
assert result.metrics["bounded_rate"] == 1.0
|
||||
|
||||
|
||||
def test_chat_spine_holdout_splits_are_runnable() -> None:
|
||||
for lane_name in (
|
||||
"multi_sentence_response",
|
||||
"cold_start_grounding",
|
||||
"conversational_thread_coherence",
|
||||
"warmed_session_consistency",
|
||||
):
|
||||
lane = get_lane(lane_name)
|
||||
result = run_lane(lane, version="v1", split="holdout")
|
||||
assert result.metrics["cases"] >= 1
|
||||
|
||||
|
|
@ -21,6 +21,7 @@ from generate.intent import (
|
|||
DialogueIntent,
|
||||
IntentTag,
|
||||
ResponseMode,
|
||||
classify_compound_intent,
|
||||
classify_intent,
|
||||
classify_response_mode,
|
||||
)
|
||||
|
|
@ -118,3 +119,23 @@ class TestExistingDefinitionRulesUntouched:
|
|||
result = classify_intent(prompt)
|
||||
assert result.tag is IntentTag.DEFINITION
|
||||
assert result.subject == subject
|
||||
|
||||
|
||||
class TestCompoundAndWalkthroughAnchors:
|
||||
def test_compound_definition_strips_causal_tail_from_subject(self) -> None:
|
||||
result = classify_compound_intent("What is truth, and why does it matter?")
|
||||
assert result.primary.tag is IntentTag.DEFINITION
|
||||
assert result.primary.subject == "truth"
|
||||
|
||||
def test_compound_definition_cause_decomposes_to_two_atoms(self) -> None:
|
||||
atoms = classify_compound_intent("What is truth, and why does it matter?")
|
||||
assert atoms.parts == (
|
||||
DialogueIntent(tag=IntentTag.DEFINITION, subject="truth"),
|
||||
DialogueIntent(tag=IntentTag.CAUSE, subject="truth"),
|
||||
)
|
||||
|
||||
def test_simple_walkthrough_gets_grounded_definition_anchor(self) -> None:
|
||||
result = classify_intent("Walk me through recall.")
|
||||
assert result.tag is IntentTag.DEFINITION
|
||||
assert result.subject == "recall"
|
||||
assert classify_response_mode("Walk me through recall.") is ResponseMode.WALKTHROUGH
|
||||
|
|
|
|||
|
|
@ -179,11 +179,13 @@ class TestClassifyIntentUnchanged:
|
|||
class TestIntentModeOrthogonality:
|
||||
def test_definition_plus_paragraph(self) -> None:
|
||||
prompt = "Write a paragraph about truth"
|
||||
# "Write a paragraph about" isn't a DEFINITION trigger, so the
|
||||
# intent falls through to UNKNOWN — but ResponseMode still picks
|
||||
# up PARAGRAPH. This documents the orthogonality: mode does not
|
||||
# *cause* a particular intent.
|
||||
# The semantic intent and presentation mode are still distinct:
|
||||
# the intent anchors the subject as a definition, while
|
||||
# ResponseMode carries the paragraph shape.
|
||||
intent = classify_intent(prompt)
|
||||
mode = classify_response_mode(prompt)
|
||||
assert intent.tag is IntentTag.DEFINITION
|
||||
assert intent.subject == "truth"
|
||||
assert mode is ResponseMode.PARAGRAPH
|
||||
|
||||
def test_narrative_plus_explain(self) -> None:
|
||||
|
|
|
|||
Loading…
Reference in a new issue