From 9367209d0492f512d51c473bf339b49171ce9ac9 Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 19 May 2026 11:51:21 -0700 Subject: [PATCH] feat(evals): priming_prompts on multi_sentence_response lane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Option 1 of the lane-isolation work after the 8d1aeec predicate refinement. Adds optional ``priming_prompts: [str, ...]`` to each case in ``multi_sentence_response``. The runner runs priming prompts on the same ``ChatRuntime`` instance before the scored prompt and discards their responses; only the scored prompt is measured. This isolates code paths (notably the discourse planner hook) that engage only on the warm pack/teaching path from cold-start one-shot paths. Cold-start measurement is preserved: cases without ``priming_prompts`` (or with an empty list) keep the old behavior. New metric ``primed_multi_sentence_rate`` reports only on primed cases. ``primed`` is also exposed per-case in case_details. Six primed cases added to ``public/v1/cases.jsonl`` (Explain truth / Tell about truth / Explain knowledge / Tell about light / Tell about parent / Write a short paragraph about truth). Each is the cold- start variant of an existing case plus a single "What is X?" priming prompt. 3 new tests: * Priming prompts run in order on the same runtime before the scored prompt; primed=True on the result. * Default cold-start behavior: no priming key OR empty list ⇒ primed=False; aggregate untouched. * ``primed_multi_sentence_rate`` separates from aggregate so cold cases never inflate/depress the warm-path metric. A/B measurement on the live runtime (21 cases): flag off: multi=0.1429, primed_multi=0.0000, primed_cases=6 flag on : multi=0.2857, primed_multi=0.5000, primed_cases=6 Lift is real and exclusively on the substrate the planner can actually serve (teaching-grounded narrative). The three primed "Explain X" and "Write a short paragraph about X" cases stay vault-grounded (Explain / Write are not DEFINITION / NARRATIVE intents and so don't fire pack-grounded warm), so they don't lift. That gap is what option 2 will close. contract.md updated to document priming and the new metric. --- evals/multi_sentence_response/contract.md | 21 ++- .../public/v1/cases.jsonl | 6 + evals/multi_sentence_response/runner.py | 36 ++++- tests/test_multi_sentence_response_eval.py | 136 ++++++++++++++++++ 4 files changed, 195 insertions(+), 4 deletions(-) diff --git a/evals/multi_sentence_response/contract.md b/evals/multi_sentence_response/contract.md index 8e74175c..ac502a7a 100644 --- a/evals/multi_sentence_response/contract.md +++ b/evals/multi_sentence_response/contract.md @@ -30,11 +30,26 @@ as the *only* multi-sentence-capable code path. ## Scoring rubric ```text -multi_sentence_rate = cases_with_>=2_sentences / total_cases -non_fragment_rate = cases_where_every_sentence_>=4_tokens / total_cases -connective_present_rate = cases_with_connective / cases_expecting_connective +multi_sentence_rate = cases_with_>=2_sentences / total_cases +non_fragment_rate = cases_where_every_sentence_>=4_tokens / total_cases +connective_present_rate = cases_with_connective / cases_expecting_connective +primed_cases = cases_where_priming_prompts_engaged +primed_multi_sentence_rate = primed_cases_with_>=2_sentences / primed_cases ``` +## Priming (warm-path measurement) + +A case may carry an optional `priming_prompts: [str, ...]` array. The +runner runs each priming prompt on the same `ChatRuntime` instance +before the scored prompt, discards their responses, and then measures +the scored prompt. This isolates code paths that engage only on the +warm vault/pack/teaching path (e.g. the discourse planner hook at +`chat/runtime.py`) from cold-start one-shot paths. + +`primed_multi_sentence_rate` reports only on primed cases, so cold +cases never inflate or depress it. The aggregate +`multi_sentence_rate` includes both. + ## Doctrine constraints - The trailing provenance / trust-boundary tail is structural, not a real diff --git a/evals/multi_sentence_response/public/v1/cases.jsonl b/evals/multi_sentence_response/public/v1/cases.jsonl index 5e8794d0..233b9f86 100644 --- a/evals/multi_sentence_response/public/v1/cases.jsonl +++ b/evals/multi_sentence_response/public/v1/cases.jsonl @@ -13,3 +13,9 @@ {"id":"multi_essay_truth_013","category":"essay","prompt":"Write a short paragraph about truth.","subject_lemma":"truth","expects_connective":true} {"id":"multi_essay_memory_014","category":"essay","prompt":"Write a short paragraph about memory.","subject_lemma":"memory","expects_connective":true} {"id":"multi_compose_def_cause_015","category":"compose","prompt":"What is truth, and why does it matter?","subject_lemma":"truth","expects_connective":true} +{"id":"multi_primed_explain_truth_016","category":"explain","prompt":"Explain truth.","subject_lemma":"truth","expects_connective":true,"priming_prompts":["What is truth?"]} +{"id":"multi_primed_tell_truth_017","category":"narrative","prompt":"Tell me about truth.","subject_lemma":"truth","expects_connective":false,"priming_prompts":["What is truth?"]} +{"id":"multi_primed_explain_knowledge_018","category":"explain","prompt":"Explain knowledge.","subject_lemma":"knowledge","expects_connective":true,"priming_prompts":["What is knowledge?"]} +{"id":"multi_primed_tell_light_019","category":"narrative","prompt":"Tell me about light.","subject_lemma":"light","expects_connective":false,"priming_prompts":["What is light?"]} +{"id":"multi_primed_tell_parent_020","category":"narrative","prompt":"Tell me about parent.","subject_lemma":"parent","expects_connective":false,"priming_prompts":["What is a parent?"]} +{"id":"multi_primed_essay_truth_021","category":"essay","prompt":"Write a short paragraph about truth.","subject_lemma":"truth","expects_connective":true,"priming_prompts":["What is truth?"]} diff --git a/evals/multi_sentence_response/runner.py b/evals/multi_sentence_response/runner.py index b1923b4a..be0f0fe7 100644 --- a/evals/multi_sentence_response/runner.py +++ b/evals/multi_sentence_response/runner.py @@ -13,8 +13,17 @@ Case schema: "category": "...", "prompt": "Tell me about truth.", "subject_lemma": "truth", - "expects_connective": true + "expects_connective": true, + "priming_prompts": ["What is truth?"] # optional } + +``priming_prompts`` is an optional list run before the scored prompt +on the same ``ChatRuntime`` instance. Their responses are discarded; +only ``prompt`` is scored. Priming exists because the discourse +planner currently hooks the warm pack/teaching-grounded path (post- +vault), so a one-shot cold-start case cannot exercise it. Cases +remain backward-compatible — missing or empty ``priming_prompts`` +yields the original cold-start behavior. """ from __future__ import annotations @@ -86,6 +95,7 @@ class CaseResult: grounded: bool subject_named: bool expects_connective: bool + primed: bool @dataclass @@ -96,6 +106,18 @@ class LaneReport: def _run_case(case: dict[str, Any], config: Any = None) -> CaseResult: rt = ChatRuntime(config=config) if config is not None else ChatRuntime() + + # Run optional priming turns on the same runtime so the scored + # prompt executes on the warm pack/teaching path. Responses are + # discarded; only the scored prompt's response is measured. + priming = case.get("priming_prompts") or () + primed = False + for prime in priming: + if not isinstance(prime, str) or not prime.strip(): + continue + rt.chat(prime) + primed = True + resp = rt.chat(case["prompt"]) surface = resp.surface grounding = resp.grounding_source or "none" @@ -119,6 +141,7 @@ def _run_case(case: dict[str, Any], config: Any = None) -> CaseResult: grounded=(grounding in {"pack", "teaching"}), subject_named=subj_named, expects_connective=bool(case.get("expects_connective", False)), + primed=primed, ) @@ -149,6 +172,16 @@ def run_lane(cases: list[dict[str, Any]], config: Any = None) -> LaneReport: "connective_present_rate": conn_rate, } + primed_results = [r for r in results if r.primed] + metrics["primed_cases"] = len(primed_results) + if primed_results: + multi_primed = sum(1 for r in primed_results if r.sentence_count >= 2) + metrics["primed_multi_sentence_rate"] = round( + multi_primed / len(primed_results), 4 + ) + else: + metrics["primed_multi_sentence_rate"] = 0.0 + case_details = [ { "case_id": r.case_id, @@ -162,6 +195,7 @@ def run_lane(cases: list[dict[str, Any]], config: Any = None) -> LaneReport: "grounded": r.grounded, "subject_named": r.subject_named, "expects_connective": r.expects_connective, + "primed": r.primed, } for r in results ] diff --git a/tests/test_multi_sentence_response_eval.py b/tests/test_multi_sentence_response_eval.py index e370737a..ba931b41 100644 --- a/tests/test_multi_sentence_response_eval.py +++ b/tests/test_multi_sentence_response_eval.py @@ -84,3 +84,139 @@ def test_run_lane_passes_runtime_config_to_chat_runtime(monkeypatch) -> None: assert seen_configs == [cfg] assert report.case_details[0]["connective_present"] is True + + +def test_priming_prompts_run_before_scored_prompt(monkeypatch) -> None: + """Priming turns must run on the same runtime instance and only + the scored prompt may be measured. The ``primed`` field on the + case result must record whether priming engaged. + """ + + prompts_seen: list[str] = [] + + class _FakeResponse: + surface = "Truth is grounded. Furthermore, truth belongs to cognition.truth." + grounding_source = "teaching" + + class _FakeRuntime: + def __init__(self, config=None): # noqa: ARG002 + self.id = id(self) + + def chat(self, prompt: str) -> _FakeResponse: + prompts_seen.append(prompt) + return _FakeResponse() + + monkeypatch.setattr(runner, "ChatRuntime", _FakeRuntime) + cases = [ + { + "id": "primed_case", + "category": "narrative", + "prompt": "Tell me about truth.", + "subject_lemma": "truth", + "expects_connective": False, + "priming_prompts": ["What is truth?", "Hello"], + } + ] + + report = run_lane(cases, config=RuntimeConfig(discourse_planner=True)) + + # Both priming prompts ran before the scored prompt — in order. + assert prompts_seen == ["What is truth?", "Hello", "Tell me about truth."] + detail = report.case_details[0] + assert detail["primed"] is True + # The scored surface is what was returned for the last chat call. + assert "Furthermore" in detail["surface"] + + +def test_priming_default_is_cold_start(monkeypatch) -> None: + """A case without ``priming_prompts`` (or with an empty list) must + run cold-start; ``primed`` is False. + """ + + prompts_seen: list[str] = [] + + class _FakeResponse: + surface = "Truth." + grounding_source = "vault" + + class _FakeRuntime: + def __init__(self, config=None): # noqa: ARG002 + pass + + def chat(self, prompt: str) -> _FakeResponse: + prompts_seen.append(prompt) + return _FakeResponse() + + monkeypatch.setattr(runner, "ChatRuntime", _FakeRuntime) + cases = [ + { + "id": "cold_case", + "category": "explain", + "prompt": "Explain truth.", + "subject_lemma": "truth", + "expects_connective": True, + }, + { + "id": "empty_priming_case", + "category": "narrative", + "prompt": "Tell me about truth.", + "subject_lemma": "truth", + "expects_connective": False, + "priming_prompts": [], + }, + ] + + report = run_lane(cases) + + assert prompts_seen == ["Explain truth.", "Tell me about truth."] + for detail in report.case_details: + assert detail["primed"] is False + + +def test_primed_multi_sentence_rate_separates_from_aggregate(monkeypatch) -> None: + """The ``primed_multi_sentence_rate`` metric reports only on cases + that actually exercised priming, so cold-start cases never inflate + or depress it. + """ + + class _FakeResponse: + def __init__(self, surface: str) -> None: + self.surface = surface + self.grounding_source = "teaching" + + class _FakeRuntime: + def __init__(self, config=None): # noqa: ARG002 + self._turn = 0 + + def chat(self, prompt: str) -> _FakeResponse: # noqa: ARG002 + self._turn += 1 + if self._turn <= 1: + # priming turn — single sentence + return _FakeResponse("Truth is X.") + return _FakeResponse( + "Truth is X. Furthermore, truth belongs to cognition.truth." + ) + + monkeypatch.setattr(runner, "ChatRuntime", _FakeRuntime) + cases = [ + { + "id": "cold", + "category": "explain", + "prompt": "Explain truth.", + "subject_lemma": "truth", + "expects_connective": True, + }, + { + "id": "primed", + "category": "narrative", + "prompt": "Tell me about truth.", + "subject_lemma": "truth", + "expects_connective": False, + "priming_prompts": ["What is truth?"], + }, + ] + + report = run_lane(cases) + + assert report.metrics["primed_cases"] == 1 + assert report.metrics["primed_multi_sentence_rate"] == 1.0