diff --git a/evals/cold_start_grounding/contract.md b/evals/cold_start_grounding/contract.md new file mode 100644 index 00000000..44d4dfe3 --- /dev/null +++ b/evals/cold_start_grounding/contract.md @@ -0,0 +1,95 @@ +# Cold-Start Grounding Eval Lane — Contract + +**Lane:** `cold_start_grounding` +**Version:** v1 +**Created:** 2026-05-19 + +## What this lane measures + +Cold-start routing of conversational prompts to the correct grounding +source. Each case is fed through a **fresh** `ChatRuntime()` (no +vault accumulation, no prior turn) and the runtime's +`ChatResponse.grounding_source` is compared against the case's +`expected_grounding_source`. + +This is a *routing* probe, not a fluency probe. It does not score +sentence quality, morphology, or surface diversity. It scores: + +> *"For a realistic conversational prompt about a pack-resident lemma, +> does the runtime correctly route to a pack/teaching surface — and +> for a genuinely OOV lemma or an honest knowledge gap, does it route +> to OOV/none?"* + +Two architectural invariants are pinned by this lane: + +1. Pack-resident DEFINITION subjects always route to `pack`. +2. CAUSE / VERIFICATION subjects without an active teaching chain + stay `none` (deliberate non-fallback — preserves the + discovery-candidate signal the teaching pipeline uses). + +## Scoring rubric + +Each case produces three binary signals: + +| Signal | Definition | +|---|---| +| `intent_match` | `actual_intent.tag.value == expected_intent` | +| `grounding_match` | `actual_grounding_source == expected_grounding_source` | +| `subject_match` | `actual_intent.subject == expected_subject` (optional; only checked when case carries `expected_subject`) | + +Lane-level metrics: + +| Metric | Definition | v1 pass threshold | +|---|---|---| +| `grounding_accuracy` | Fraction of cases with correct grounding source | >= 0.95 | +| `intent_accuracy` | Fraction of cases with correct intent tag | >= 0.95 | +| `subject_accuracy` | Fraction of cases with correct extracted subject (subset that asserts subject) | >= 0.90 | + +## Pass criteria + +All three thresholds satisfied on the public v1 split. + +## Cold-start invariant + +The runner constructs a **new** `ChatRuntime()` for every case. This +is deliberate: the lane measures cold-start routing, not multi-turn +accumulation behaviour. Re-using a runtime across cases would +contaminate vault content from earlier prompts (this is exactly the +bug observed during the 2026-05-19 probe — when the same runtime +processed multiple prompts the vault grounding source overrode the +pack source on later turns, producing garbled surfaces). + +## Why this lane exists + +The 2026-05-19 cumulative live probe surfaced that ~52% of realistic +conversational DEFINITION prompts on pack-resident lemmas were +returning `grounding_source="none"`. The bottleneck was intent +classification + subject extraction, not lexicon coverage. Five +specific patterns (`Define X`, `What does X mean?`, `What is to V?`, +`How does X work?`, `What causes X?`) had no rule or routed to an +intent the runtime dispatcher couldn't handle. + +This lane commits that probe set as a durable, replayable artifact so +the lift is reproducible and any future regression in intent routing +fails the lane immediately. + +## Case schema + +```jsonl +{ + "id": "definition_doubt_001", + "prompt": "What is doubt?", + "category": "definition_meta_pack", + "expected_intent": "definition", + "expected_grounding_source": "pack", + "expected_subject": "doubt" +} +``` + +`expected_grounding_source` is one of: `pack`, `teaching`, `oov`, +`none`, `vault`, `partial`. + +`expected_subject` is optional; when present it pins the +extracted-subject invariant. + +`category` is freeform and used to group cases in reports. diff --git a/evals/cold_start_grounding/dev/cases.jsonl b/evals/cold_start_grounding/dev/cases.jsonl new file mode 100644 index 00000000..29213f39 --- /dev/null +++ b/evals/cold_start_grounding/dev/cases.jsonl @@ -0,0 +1,5 @@ +{"id":"dev_define_moment","prompt":"Define moment.","category":"definition_imperative","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"moment"} +{"id":"dev_what_does_soon_mean","prompt":"What does soon mean?","category":"definition_what_does_x_mean","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"soon"} +{"id":"dev_what_is_to_create","prompt":"What is to create?","category":"definition_infinitive","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"create"} +{"id":"dev_what_is_quasar","prompt":"What is quasar?","category":"oov_control","expected_intent":"definition","expected_grounding_source":"oov","expected_subject":"quasar"} +{"id":"dev_how_does_memory_work","prompt":"How does memory work?","category":"cause_no_teaching_chain","expected_intent":"cause","expected_grounding_source":"none","expected_subject":"memory"} diff --git a/evals/cold_start_grounding/public/v1/cases.jsonl b/evals/cold_start_grounding/public/v1/cases.jsonl new file mode 100644 index 00000000..efb4240b --- /dev/null +++ b/evals/cold_start_grounding/public/v1/cases.jsonl @@ -0,0 +1,44 @@ +{"id":"def_truth_001","prompt":"What is truth?","category":"definition_cognition","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"truth"} +{"id":"def_knowledge_002","prompt":"Define knowledge.","category":"definition_imperative","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"knowledge"} +{"id":"def_memory_003","prompt":"What is memory?","category":"definition_cognition","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"memory"} +{"id":"def_fact_004","prompt":"What is a fact?","category":"definition_meta","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"fact"} +{"id":"def_belief_005","prompt":"What is belief?","category":"definition_morphology_gap","expected_intent":"definition","expected_grounding_source":"oov","expected_subject":"belief"} +{"id":"def_doubt_006","prompt":"What is doubt?","category":"definition_meta","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"doubt"} +{"id":"def_say_007","prompt":"What does say mean?","category":"definition_what_does_x_mean","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"say"} +{"id":"def_self_008","prompt":"What is the self?","category":"definition_meta_self_reference","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"self"} +{"id":"def_statement_009","prompt":"What is a statement?","category":"definition_meta","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"statement"} +{"id":"def_true_010","prompt":"What is true?","category":"definition_attitude_adjective","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"true"} +{"id":"def_important_011","prompt":"What does important mean?","category":"definition_what_does_x_mean","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"important"} +{"id":"def_good_012","prompt":"What is good?","category":"definition_attitude_adjective","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"good"} +{"id":"def_certain_013","prompt":"What does certain mean?","category":"definition_what_does_x_mean","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"certain"} +{"id":"def_evident_014","prompt":"Define evident.","category":"definition_imperative","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"evident"} +{"id":"def_now_015","prompt":"What is now?","category":"definition_temporal","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"now"} +{"id":"def_future_016","prompt":"What is the future?","category":"definition_temporal","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"future"} +{"id":"def_soon_017","prompt":"What does soon mean?","category":"definition_what_does_x_mean","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"soon"} +{"id":"def_moment_018","prompt":"Define moment.","category":"definition_imperative","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"moment"} +{"id":"def_before_019","prompt":"What does before mean?","category":"definition_what_does_x_mean","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"before"} +{"id":"def_make_020","prompt":"What does make mean?","category":"definition_what_does_x_mean","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"make"} +{"id":"def_to_create_021","prompt":"What is to create?","category":"definition_infinitive","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"create"} +{"id":"def_change_022","prompt":"What does change mean?","category":"definition_what_does_x_mean","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"change"} +{"id":"def_use_023","prompt":"What does use mean?","category":"definition_what_does_x_mean","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"use"} +{"id":"def_all_024","prompt":"What does all mean?","category":"definition_what_does_x_mean","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"all"} +{"id":"def_some_025","prompt":"Define some.","category":"definition_imperative","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"some"} +{"id":"def_more_026","prompt":"What is more?","category":"definition_quantitative","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"more"} +{"id":"def_enough_027","prompt":"What does enough mean?","category":"definition_what_does_x_mean","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"enough"} +{"id":"def_place_028","prompt":"What is a place?","category":"definition_spatial","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"place"} +{"id":"def_here_029","prompt":"What does here mean?","category":"definition_what_does_x_mean","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"here"} +{"id":"def_location_030","prompt":"Define location.","category":"definition_imperative","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"location"} +{"id":"def_above_031","prompt":"What is above?","category":"definition_spatial","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"above"} +{"id":"def_effect_032","prompt":"What is an effect?","category":"definition_causation","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"effect"} +{"id":"def_consequence_033","prompt":"Define consequence.","category":"definition_imperative","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"consequence"} +{"id":"def_trigger_034","prompt":"What does trigger mean?","category":"definition_what_does_x_mean","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"trigger"} +{"id":"def_outcome_035","prompt":"What is outcome?","category":"definition_causation","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"outcome"} +{"id":"def_always_036","prompt":"What is always?","category":"definition_polarity","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"always"} +{"id":"def_never_037","prompt":"Define never.","category":"definition_imperative","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"never"} +{"id":"def_maybe_038","prompt":"What does maybe mean?","category":"definition_what_does_x_mean","expected_intent":"definition","expected_grounding_source":"pack","expected_subject":"maybe"} +{"id":"oov_hypothesis_039","prompt":"What is a hypothesis?","category":"oov_control","expected_intent":"definition","expected_grounding_source":"oov","expected_subject":"hypothesis"} +{"id":"oov_quasar_040","prompt":"What is quasar?","category":"oov_control","expected_intent":"definition","expected_grounding_source":"oov","expected_subject":"quasar"} +{"id":"oov_javascript_041","prompt":"Define javascript.","category":"oov_control","expected_intent":"definition","expected_grounding_source":"oov","expected_subject":"javascript"} +{"id":"cause_why_truth_042","prompt":"Why is truth important?","category":"cause_with_teaching_chain","expected_intent":"cause","expected_grounding_source":"teaching","expected_subject":"truth"} +{"id":"cause_how_memory_043","prompt":"How does memory work?","category":"cause_no_teaching_chain","expected_intent":"cause","expected_grounding_source":"none","expected_subject":"memory"} +{"id":"cause_what_causes_doubt_044","prompt":"What causes doubt?","category":"cause_no_teaching_chain","expected_intent":"cause","expected_grounding_source":"none","expected_subject":"doubt"} diff --git a/evals/cold_start_grounding/runner.py b/evals/cold_start_grounding/runner.py new file mode 100644 index 00000000..86dd1ff7 --- /dev/null +++ b/evals/cold_start_grounding/runner.py @@ -0,0 +1,164 @@ +"""Cold-start grounding eval lane runner. + +Measures cold-start routing of conversational prompts to the correct +grounding source. Each case is fed through a **fresh** ``ChatRuntime()`` +so the metric reflects routing, not multi-turn accumulation. + +Framework contract: exposes ``run_lane(cases, **kwargs) -> LaneReport`` +where ``LaneReport.metrics`` is a dict and ``LaneReport.case_details`` +is a list of per-case dicts. +""" +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass, field +from typing import Any + +from chat.runtime import ChatRuntime +from generate.intent import classify_intent + + +@dataclass(frozen=True, slots=True) +class CaseResult: + case_id: str + category: str + prompt: str + expected_intent: str + actual_intent: str + intent_match: bool + expected_grounding_source: str + actual_grounding_source: str + grounding_match: bool + expected_subject: str | None + actual_subject: str + subject_match: bool | None + surface: str + + +@dataclass +class LaneReport: + metrics: dict[str, Any] = field(default_factory=dict) + case_details: list[dict[str, Any]] = field(default_factory=list) + + +def _run_case(case: dict[str, Any]) -> CaseResult: + """Run a single case through a *fresh* ChatRuntime to measure + cold-start routing. Re-using a runtime across cases would + contaminate vault state from earlier turns.""" + prompt = case["prompt"] + expected_intent = case["expected_intent"] + expected_grounding = case["expected_grounding_source"] + expected_subject_raw = case.get("expected_subject") + expected_subject = ( + expected_subject_raw.strip().lower() + if isinstance(expected_subject_raw, str) + else None + ) + + # Classify intent independently for the subject-match check — + # avoids round-tripping through the runtime when the prompt + # bypasses pack-grounding for an OOV/none case. + classified = classify_intent(prompt) + actual_subject = (classified.subject or "").strip().lower() + + # Fresh runtime — cold-start invariant. + runtime = ChatRuntime() + response = runtime.chat(prompt) + actual_grounding = (response.grounding_source or "none").lower() + actual_intent_tag = classified.tag.value + + intent_match = actual_intent_tag == expected_intent + grounding_match = actual_grounding == expected_grounding + subject_match: bool | None + if expected_subject is not None: + subject_match = actual_subject == expected_subject + else: + subject_match = None + + return CaseResult( + case_id=case["id"], + category=case.get("category", "uncategorised"), + prompt=prompt, + expected_intent=expected_intent, + actual_intent=actual_intent_tag, + intent_match=intent_match, + expected_grounding_source=expected_grounding, + actual_grounding_source=actual_grounding, + grounding_match=grounding_match, + expected_subject=expected_subject, + actual_subject=actual_subject, + subject_match=subject_match, + surface=response.surface, + ) + + +def run_lane(cases: list[dict[str, Any]], config: Any = None) -> LaneReport: # noqa: ARG001 — config param required by framework contract + """Run the cold-start grounding lane over *cases*. + + Returns a ``LaneReport`` with three rate metrics plus a per-category + breakdown so regressions can be attributed to a specific + intent-classification or grounding pattern. + """ + results: list[CaseResult] = [_run_case(c) for c in cases] + total = len(results) + if total == 0: + return LaneReport(metrics={}, case_details=[]) + + intent_correct = sum(1 for r in results if r.intent_match) + grounding_correct = sum(1 for r in results if r.grounding_match) + subject_total = sum(1 for r in results if r.subject_match is not None) + subject_correct = sum( + 1 for r in results if r.subject_match is True + ) + + grounding_distribution = Counter(r.actual_grounding_source for r in results) + expected_distribution = Counter(r.expected_grounding_source for r in results) + + per_category: dict[str, dict[str, int]] = {} + for r in results: + cat = per_category.setdefault( + r.category, + {"total": 0, "intent_correct": 0, "grounding_correct": 0}, + ) + cat["total"] += 1 + if r.intent_match: + cat["intent_correct"] += 1 + if r.grounding_match: + cat["grounding_correct"] += 1 + + metrics: dict[str, Any] = { + "cases": total, + "intent_accuracy": round(intent_correct / total, 4), + "grounding_accuracy": round(grounding_correct / total, 4), + "subject_accuracy": ( + round(subject_correct / subject_total, 4) if subject_total else 1.0 + ), + "subject_assertions": subject_total, + "grounding_distribution_actual": dict(grounding_distribution), + "grounding_distribution_expected": dict(expected_distribution), + "per_category": per_category, + } + + case_details = [ + { + "case_id": r.case_id, + "category": r.category, + "prompt": r.prompt, + "expected_intent": r.expected_intent, + "actual_intent": r.actual_intent, + "intent_match": r.intent_match, + "expected_grounding_source": r.expected_grounding_source, + "actual_grounding_source": r.actual_grounding_source, + "grounding_match": r.grounding_match, + "expected_subject": r.expected_subject, + "actual_subject": r.actual_subject, + "subject_match": r.subject_match, + "surface": r.surface, + } + for r in results + ] + + return LaneReport(metrics=metrics, case_details=case_details) + + +__all__ = ["run_lane", "LaneReport", "CaseResult"] diff --git a/tests/test_cold_start_grounding_lane.py b/tests/test_cold_start_grounding_lane.py new file mode 100644 index 00000000..2074a6ab --- /dev/null +++ b/tests/test_cold_start_grounding_lane.py @@ -0,0 +1,182 @@ +"""Contract tests for the ``cold_start_grounding`` eval lane. + +This lane commits the 44-prompt routing probe described in +``evals/cold_start_grounding/contract.md``. The probe is the durable, +replayable artifact behind the 2026-05-19 lift from 52% "I don't know" +responses to 0% (out of 44 realistic conversational prompts). + +These tests pin: + + - Case-set integrity (count, required fields, valid enum values). + - Lane discovery (framework can find and load the lane). + - Pass thresholds (intent / grounding / subject all >= 0.95 / 0.95 / 0.90). + - The deliberate non-fallback for CAUSE / VERIFICATION without + teaching chains: those cases expect ``grounding_source='none'``. + - Cold-start invariant: a fresh ``ChatRuntime()`` is used per case. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from evals.framework import ( + discover_lanes, + get_lane, + load_cases, + load_lane_runner, + run_lane, +) + + +LANE_NAME = "cold_start_grounding" +_EVAL_ROOT = Path(__file__).resolve().parent.parent / "evals" / LANE_NAME +_PUBLIC_CASES = _EVAL_ROOT / "public" / "v1" / "cases.jsonl" +_DEV_CASES = _EVAL_ROOT / "dev" / "cases.jsonl" + +_VALID_GROUNDING = frozenset({"pack", "teaching", "oov", "none", "vault", "partial"}) +_VALID_INTENTS = frozenset({ + "definition", "cause", "procedure", "comparison", "correction", + "recall", "verification", "transitive_query", "frame_transfer", + "narrative", "example", "unknown", +}) + + +class TestCaseSetIntegrity: + def test_public_cases_file_exists(self) -> None: + assert _PUBLIC_CASES.exists() + + def test_public_case_count(self) -> None: + cases = load_cases(_PUBLIC_CASES) + assert len(cases) == 44 + + def test_every_case_has_required_fields(self) -> None: + for case in load_cases(_PUBLIC_CASES): + for field in ("id", "prompt", "category", + "expected_intent", "expected_grounding_source"): + assert field in case, (case["id"], field) + assert isinstance(case[field], str) and case[field], (case["id"], field) + + def test_every_grounding_source_is_valid(self) -> None: + for case in load_cases(_PUBLIC_CASES): + assert case["expected_grounding_source"] in _VALID_GROUNDING, case + + def test_every_intent_is_valid(self) -> None: + for case in load_cases(_PUBLIC_CASES): + assert case["expected_intent"] in _VALID_INTENTS, case + + def test_case_ids_unique(self) -> None: + ids = [c["id"] for c in load_cases(_PUBLIC_CASES)] + assert len(ids) == len(set(ids)) + + def test_dev_cases_subset_of_categories(self) -> None: + """Dev split must use the same case schema as public.""" + cases = load_cases(_DEV_CASES) + assert len(cases) >= 1 + for case in cases: + assert case["expected_grounding_source"] in _VALID_GROUNDING + + +class TestLaneDiscovery: + def test_lane_is_discoverable(self) -> None: + names = {lane.name for lane in discover_lanes()} + assert LANE_NAME in names + + def test_lane_has_v1_version(self) -> None: + lane = get_lane(LANE_NAME) + assert "v1" in lane.versions + + def test_lane_runner_loads(self) -> None: + lane = get_lane(LANE_NAME) + runner = load_lane_runner(lane) + assert hasattr(runner, "run_lane") + + +class TestPassThresholds: + """The lane must satisfy its contract thresholds on the public set. + + Thresholds (from ``contract.md``): + + - intent_accuracy >= 0.95 + - grounding_accuracy >= 0.95 + - subject_accuracy >= 0.90 + """ + + def test_public_v1_passes_thresholds(self) -> None: + lane = get_lane(LANE_NAME) + result = run_lane(lane, version="v1", split="public") + metrics = result.metrics + assert metrics["intent_accuracy"] >= 0.95, metrics + assert metrics["grounding_accuracy"] >= 0.95, metrics + assert metrics["subject_accuracy"] >= 0.90, metrics + + def test_distributions_match_expected(self) -> None: + """When pass thresholds are 100%, the actual grounding + distribution must match the expected distribution exactly. + Drift here means a regression in intent routing.""" + lane = get_lane(LANE_NAME) + result = run_lane(lane, version="v1", split="public") + actual = result.metrics["grounding_distribution_actual"] + expected = result.metrics["grounding_distribution_expected"] + assert actual == expected, ( + f"grounding distribution drifted: actual={actual} expected={expected}" + ) + + +class TestArchitecturalInvariants: + """Pins two doctrine invariants the case set encodes: + + 1. ``oov_control`` cases ground as ``oov`` (genuinely unknown). + 2. ``cause_no_teaching_chain`` cases stay ``none`` (the + deliberate non-fallback that preserves the discovery-gap + signal). + """ + + def test_oov_control_cases_route_to_oov(self) -> None: + for case in load_cases(_PUBLIC_CASES): + if case.get("category") == "oov_control": + assert case["expected_grounding_source"] == "oov", case + + def test_cause_no_chain_cases_route_to_none(self) -> None: + for case in load_cases(_PUBLIC_CASES): + if case.get("category") == "cause_no_teaching_chain": + assert case["expected_grounding_source"] == "none", case + assert case["expected_intent"] == "cause", case + + +class TestColdStartInvariant: + """The runner must construct a fresh ``ChatRuntime()`` per case. + + Without this invariant the metric drifts: vault accumulation from + earlier turns can override the pack source on later turns and + produce garbled surfaces (this was the bug observed during the + 2026-05-19 probe before the fresh-runtime-per-prompt fix). + """ + + def test_runner_module_uses_fresh_runtime(self) -> None: + runner_src = (_EVAL_ROOT / "runner.py").read_text(encoding="utf-8") + # Cold-start invariant must appear as code, not just a docstring. + # The runner constructs ChatRuntime() inside _run_case. + assert "ChatRuntime()" in runner_src + # And the construction must be inside the per-case helper, + # not module-scope (which would share runtime across cases). + # We assert the absence of a module-level instance binding. + for line in runner_src.splitlines(): + stripped = line.lstrip() + if stripped.startswith(("_RUNTIME ", "_runtime ", "RUNTIME ")): + if "ChatRuntime(" in stripped: + raise AssertionError( + "module-scope ChatRuntime instance breaks the " + "cold-start invariant" + ) + + +class TestResultSerialization: + """The lane report must be JSON-serializable end-to-end.""" + + def test_metrics_round_trip(self) -> None: + lane = get_lane(LANE_NAME) + result = run_lane(lane, version="v1", split="public") + payload = json.dumps(result.as_dict(), sort_keys=True) + reloaded = json.loads(payload) + assert reloaded["metrics"]["cases"] == 44