From e06fda5b8bf74ec55a2d2e841ac729903658d61b Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 19 May 2026 08:26:38 -0700 Subject: [PATCH] feat(runtime+evals): warm-path pack grounding + three long-span lanes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 1 — warm_grounding_stability targeted patch - chat/runtime.py:_maybe_pack_grounded_surface accepts allow_warm=True; warm path invokes it after articulation and overrides response_surface / articulation / grounding_source when pack-grounded or teaching-grounded. - CAUSE / VERIFICATION without a teaching chain on warm path emits the unknown-domain disclosure (matches cold-path discovery-signal doctrine — no fabricated vault content). - warmed_session_consistency public lane: warm_grounding_stability 0.0 → 1.0, grounding_match_rate 1.0, telemetry_consistency 1.0. - Cognition lane byte-identical (public 100/100/91.7/100, holdout 100/100/83.3/100). Full suite 2294 passed. Step 2 — three new red eval lanes (measurement substrate) - conversational_thread_coherence: 6 cases / 45 turns; per-turn no_placeholder / not_walk_fragment / length / is_grounded predicates + per-case topic_anchor and no_topic_drift. Baseline: grounded 0.93, topic_anchor 0.50, no_topic_drift 0.83. - multi_sentence_response: 15 cases over Explain/Tell/Describe/Walk/ Example/Essay shapes; predicates sentence_count >= 2, non-fragment, connective_present, subject_named. Baseline: multi_sentence 0.53, connective 0.10 — biggest architectural gap. - self_consistency_over_time: 7 cases; same probe at multiple turn indices with unrelated fillers interleaved. Baseline: byte_identical 0.86 (one CAUSE-no-chain disclosure drifts under accumulation). All three lanes deterministic, lexical-predicate-only — no LLM judge, no embedding similarity. Red-on-creation by design. See notes/long_span_fluency_baseline_2026-05-19.md. --- chat/runtime.py | 61 +++++- .../contract.md | 48 +++++ .../dev/cases.jsonl | 2 + .../public/v1/cases.jsonl | 6 + .../conversational_thread_coherence/runner.py | 198 ++++++++++++++++++ evals/multi_sentence_response/contract.md | 45 ++++ evals/multi_sentence_response/dev/cases.jsonl | 3 + .../public/v1/cases.jsonl | 15 ++ evals/multi_sentence_response/runner.py | 150 +++++++++++++ evals/self_consistency_over_time/contract.md | 46 ++++ .../dev/cases.jsonl | 2 + .../public/v1/cases.jsonl | 7 + evals/self_consistency_over_time/runner.py | 143 +++++++++++++ .../long_span_fluency_baseline_2026-05-19.md | 103 +++++++++ 14 files changed, 822 insertions(+), 7 deletions(-) create mode 100644 evals/conversational_thread_coherence/contract.md create mode 100644 evals/conversational_thread_coherence/dev/cases.jsonl create mode 100644 evals/conversational_thread_coherence/public/v1/cases.jsonl create mode 100644 evals/conversational_thread_coherence/runner.py create mode 100644 evals/multi_sentence_response/contract.md create mode 100644 evals/multi_sentence_response/dev/cases.jsonl create mode 100644 evals/multi_sentence_response/public/v1/cases.jsonl create mode 100644 evals/multi_sentence_response/runner.py create mode 100644 evals/self_consistency_over_time/contract.md create mode 100644 evals/self_consistency_over_time/dev/cases.jsonl create mode 100644 evals/self_consistency_over_time/public/v1/cases.jsonl create mode 100644 evals/self_consistency_over_time/runner.py create mode 100644 notes/long_span_fluency_baseline_2026-05-19.md diff --git a/chat/runtime.py b/chat/runtime.py index bf933061..94fb0f1a 100644 --- a/chat/runtime.py +++ b/chat/runtime.py @@ -611,14 +611,22 @@ class ChatRuntime: ) def _maybe_pack_grounded_surface( - self, text: str, gate_source: str + self, text: str, gate_source: str, *, allow_warm: bool = False ) -> tuple[str, str] | None: """Return ``(surface, grounding_source)`` or ``None``. ADR-0048 / ADR-0050 / ADR-0052 — three reviewed sources of cold-start grounding share this dispatcher. + + ``allow_warm=True`` bypasses the empty-vault gate so the warm + path can engage pack-grounding for pack-resident DEFINITION / + RECALL / NARRATIVE / EXAMPLE / COMPARISON / PROCEDURE intents + — addresses ``warm_grounding_stability`` regression where + turn-2 of the same prompt drifted from a coherent pack surface + to a walk fragment. CAUSE / VERIFICATION still return None + when no teaching chain exists, preserving the discovery signal. """ - if gate_source != "empty_vault": + if not allow_warm and gate_source != "empty_vault": return None if self.config.output_language != "en": return None @@ -1034,11 +1042,50 @@ class ChatRuntime: ) refusal_emitted = refusal_surface is not None hedge_injected = False + warm_grounding_source: str | None = None + warm_pack_subject: str | None = None + warm_pack_intent_tag: Any = None if refusal_emitted: response_surface = refusal_surface self._last_refusal_was_typed = True else: response_surface = walk_surface + warm_pack_result = self._maybe_pack_grounded_surface( + text, "warm", allow_warm=True + ) + if warm_pack_result is None: + from generate.intent import IntentTag + from generate.intent_bridge import classify_intent_from_input + _wintent = classify_intent_from_input(text) + # Discovery-signal preservation on warm path: when CAUSE / + # VERIFICATION lacks both a teaching chain and a cross-pack + # chain, the cold path emits the unknown-domain disclosure. + # The warm path must match — fabricating a vault-grounded + # walk fragment ("Work infer.") would mask the very gap + # the discovery layer is meant to surface. + if _wintent.tag in (IntentTag.CAUSE, IntentTag.VERIFICATION): + response_surface = _UNKNOWN_DOMAIN_SURFACE + articulation = replace(articulation, surface=_UNKNOWN_DOMAIN_SURFACE) + warm_grounding_source = "none" + elif warm_pack_result is not None: + warm_pack_surface, warm_grounding_source = warm_pack_result + if self.config.thread_anaphora and warm_grounding_source in {"pack", "teaching"}: + from chat.anaphora import thread_anaphora_prefix + from generate.intent_bridge import classify_intent_from_input + _wintent = classify_intent_from_input(text) + warm_pack_intent_tag = _wintent.tag + warm_pack_subject = _wintent.subject + if warm_pack_subject and warm_pack_intent_tag is not None: + prefix = thread_anaphora_prefix( + self.thread_context, + warm_pack_subject, + warm_pack_intent_tag.name.lower(), + warm_grounding_source, + ) + if prefix is not None: + warm_pack_surface = prefix + warm_pack_surface + response_surface = warm_pack_surface + articulation = replace(articulation, surface=warm_pack_surface) if should_inject_hedge(ethics_verdict, self.ethics_pack): hedge_prefix = build_hedge_prefix(self.identity_manifold) before = response_surface @@ -1067,15 +1114,15 @@ class ChatRuntime: safety_verdict=safety_verdict, ethics_verdict=ethics_verdict, verdicts=verdicts_bundle, - grounding_source="vault", + grounding_source=warm_grounding_source or "vault", ) self.turn_log.append(turn_event) self._emit_turn_event(turn_event) self._push_thread_summary( turn_event=turn_event, - intent_tag=None, - intent_subject=articulation.subject, - grounding_source="vault", + intent_tag=warm_pack_intent_tag, + intent_subject=warm_pack_subject or articulation.subject, + grounding_source=warm_grounding_source or "vault", surface=response_surface, ) return ChatResponse( @@ -1099,7 +1146,7 @@ class ChatRuntime: safety_verdict=safety_verdict, ethics_verdict=ethics_verdict, verdicts=verdicts_bundle, - grounding_source="vault", + grounding_source=warm_grounding_source or "vault", ) def _unknown_domain_response(self, field_state: FieldState, filtered: list[str]) -> ChatResponse: diff --git a/evals/conversational_thread_coherence/contract.md b/evals/conversational_thread_coherence/contract.md new file mode 100644 index 00000000..75e93179 --- /dev/null +++ b/evals/conversational_thread_coherence/contract.md @@ -0,0 +1,48 @@ +# Conversational Thread Coherence Eval Lane — Contract + +**Lane:** `conversational_thread_coherence` +**Version:** v1 +**Created:** 2026-05-19 +**Status:** Red on creation — measurement substrate for long-span fluency. + +## What this lane measures + +Whether `ChatRuntime` maintains coherent grounding and topic continuity +across an 8–12 turn thread that includes topic shifts, follow-up +questions, and re-introductions of earlier subjects. + +The asymmetric pair to `warmed_session_consistency`: that lane pins +*replay* stability of identical prompts; this lane pins *evolving* +conversation across structurally-different turns. + +## Per-turn predicates + +| Predicate | Definition | +|---|---| +| `no_placeholder` | surface contains none of: `...`, ``, ``, `` | +| `is_grounded` | `grounding_source` ∈ {pack, teaching, vault, oov, partial} (not `none`) on turns whose prompt is expected to ground | +| `not_walk_fragment` | surface has ≥ 4 alphabetic tokens AND ≥ 1 finite verb form | +| `length_adequate` | `len(surface.strip()) ≥ 20` | + +## Per-case predicates + +| Predicate | Definition | +|---|---| +| `topic_anchor_present` | When a follow-up prompt does not name the prior topic explicitly (e.g. "What about that?"), the response surface mentions the prior topic's subject lemma OR explicitly discloses | +| `no_topic_drift_to_none` | After any `pack`/`teaching` turn, no subsequent turn on the SAME prompt-subject drops to `none` (would indicate state corruption) | + +## Scoring rubric + +```text +per_turn_grounded_rate = grounded_turns / total_turns +per_turn_not_fragment_rate = non_fragment_turns / total_turns +case_topic_anchor_rate = cases_passing_anchor / cases_with_anaphora +case_no_drift_rate = cases_passing_no_drift / replay_cases +``` + +## Doctrine constraints + +- No LLM judge. All predicates are deterministic, lexical / structural. +- No paraphrase equivalence by embedding. Lexical overlap only. +- Red-on-creation is acceptable and expected — this lane is a target, + not a regression gate (yet). diff --git a/evals/conversational_thread_coherence/dev/cases.jsonl b/evals/conversational_thread_coherence/dev/cases.jsonl new file mode 100644 index 00000000..5c1dc700 --- /dev/null +++ b/evals/conversational_thread_coherence/dev/cases.jsonl @@ -0,0 +1,2 @@ +{"id":"dev_thread_oov_recovery_001","category":"oov_during_thread","turns":[{"prompt":"What is truth?","subject_lemma":"truth"},{"prompt":"What is photosynthesis?","subject_lemma":"photosynthesis","expects_grounded":true},{"prompt":"What is truth?","subject_lemma":"truth","is_replay_of_prompt_at_turn":0},{"prompt":"What is knowledge?","subject_lemma":"knowledge"}]} +{"id":"dev_thread_correction_mid_002","category":"correction_during_thread","turns":[{"prompt":"What is light?","subject_lemma":"light"},{"prompt":"Actually no, that was wrong.","expects_grounded":false},{"prompt":"What is light?","subject_lemma":"light","is_replay_of_prompt_at_turn":0}]} diff --git a/evals/conversational_thread_coherence/public/v1/cases.jsonl b/evals/conversational_thread_coherence/public/v1/cases.jsonl new file mode 100644 index 00000000..35cf83d3 --- /dev/null +++ b/evals/conversational_thread_coherence/public/v1/cases.jsonl @@ -0,0 +1,6 @@ +{"id":"thread_truth_knowledge_001","category":"topic_persistence","turns":[{"prompt":"What is truth?","subject_lemma":"truth"},{"prompt":"What is knowledge?","subject_lemma":"knowledge"},{"prompt":"What is truth?","subject_lemma":"truth","is_replay_of_prompt_at_turn":0},{"prompt":"What is evidence?","subject_lemma":"evidence"},{"prompt":"What is knowledge?","subject_lemma":"knowledge","is_replay_of_prompt_at_turn":1},{"prompt":"What is memory?","subject_lemma":"memory"},{"prompt":"What is truth?","subject_lemma":"truth","is_replay_of_prompt_at_turn":0},{"prompt":"What is wisdom?","subject_lemma":"wisdom"}]} +{"id":"thread_definitions_chain_002","category":"definition_chain","turns":[{"prompt":"Define light.","subject_lemma":"light"},{"prompt":"Define meaning.","subject_lemma":"meaning"},{"prompt":"Define understanding.","subject_lemma":"understanding"},{"prompt":"Define judgment.","subject_lemma":"judgment"},{"prompt":"Define inference.","subject_lemma":"inference"},{"prompt":"Define recall.","subject_lemma":"recall"},{"prompt":"Define evidence.","subject_lemma":"evidence"},{"prompt":"Define light.","subject_lemma":"light","is_replay_of_prompt_at_turn":0}]} +{"id":"thread_mixed_intent_003","category":"intent_variety","turns":[{"prompt":"What is truth?","subject_lemma":"truth"},{"prompt":"Why does truth matter?","subject_lemma":"truth","anaphora_anchor_to":"truth"},{"prompt":"Tell me about truth.","subject_lemma":"truth","anaphora_anchor_to":"truth"},{"prompt":"Does truth require evidence?","subject_lemma":"truth"},{"prompt":"Give me an example of truth.","subject_lemma":"truth"},{"prompt":"What is truth?","subject_lemma":"truth","is_replay_of_prompt_at_turn":0},{"prompt":"Compare truth and knowledge.","subject_lemma":"truth"}]} +{"id":"thread_followup_anaphora_004","category":"anaphora","turns":[{"prompt":"What is light?","subject_lemma":"light"},{"prompt":"Why does it exist?","anaphora_anchor_to":"light","expects_grounded":false},{"prompt":"What is memory?","subject_lemma":"memory"},{"prompt":"Why does it require recall?","anaphora_anchor_to":"memory","expects_grounded":false},{"prompt":"What is light?","subject_lemma":"light","is_replay_of_prompt_at_turn":0}]} +{"id":"thread_cognition_walk_005","category":"long_walk","turns":[{"prompt":"What is meaning?","subject_lemma":"meaning"},{"prompt":"What is understanding?","subject_lemma":"understanding"},{"prompt":"What is recall?","subject_lemma":"recall"},{"prompt":"What is memory?","subject_lemma":"memory"},{"prompt":"What is judgment?","subject_lemma":"judgment"},{"prompt":"What is wisdom?","subject_lemma":"wisdom"},{"prompt":"What is inference?","subject_lemma":"inference"},{"prompt":"What is evidence?","subject_lemma":"evidence"},{"prompt":"What is truth?","subject_lemma":"truth"},{"prompt":"What is knowledge?","subject_lemma":"knowledge"}]} +{"id":"thread_topic_shift_recover_006","category":"topic_shift","turns":[{"prompt":"What is parent?","subject_lemma":"parent"},{"prompt":"What is child?","subject_lemma":"child"},{"prompt":"What is family?","subject_lemma":"family"},{"prompt":"What is truth?","subject_lemma":"truth"},{"prompt":"What is parent?","subject_lemma":"parent","is_replay_of_prompt_at_turn":0},{"prompt":"What is knowledge?","subject_lemma":"knowledge"},{"prompt":"What is family?","subject_lemma":"family","is_replay_of_prompt_at_turn":2}]} diff --git a/evals/conversational_thread_coherence/runner.py b/evals/conversational_thread_coherence/runner.py new file mode 100644 index 00000000..6a8e2cde --- /dev/null +++ b/evals/conversational_thread_coherence/runner.py @@ -0,0 +1,198 @@ +"""Conversational thread coherence eval lane runner. + +Measures whether ``ChatRuntime`` maintains coherent grounding and +topic continuity across an 8-12 turn thread. Predicates are +deterministic and lexical — no LLM judge, no embedding similarity. + +Framework contract: ``run_lane(cases, config=None) -> LaneReport``. + +Case schema (``cases.jsonl`` line): + + { + "id": "...", + "category": "...", + "turns": [ + { + "prompt": "...", + "subject_lemma": "truth", # optional — for topic-anchor check + "expects_grounded": true, # default true + "anaphora_anchor_to": "truth", # optional — prior subject expected to appear + "is_replay_of_prompt_at_turn": 0 # optional — drift check + } + ] + } +""" +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any + +from chat.runtime import ChatRuntime + + +_PLACEHOLDER_MARKERS = ("...", "", "", "") + +_FINITE_VERB_RE = re.compile( + r"\b(is|are|was|were|has|have|had|does|do|did|will|would|can|could|" + r"should|might|may|must|shall|been|being|[a-z]+(?:es|ed|ing)s?)\b", + re.IGNORECASE, +) + + +def _check_no_placeholder(surface: str) -> bool: + return not any(m in surface for m in _PLACEHOLDER_MARKERS) + + +def _check_not_fragment(surface: str) -> bool: + tokens = [t for t in re.findall(r"[A-Za-z]+", surface)] + if len(tokens) < 4: + return False + return bool(_FINITE_VERB_RE.search(surface)) + + +def _check_length_adequate(surface: str) -> bool: + return len(surface.strip()) >= 20 + + +def _check_is_grounded(grounding_source: str, expects: bool) -> bool: + if not expects: + return True + return grounding_source in {"pack", "teaching", "vault", "oov", "partial"} + + +@dataclass(frozen=True, slots=True) +class TurnResult: + turn_index: int + prompt: str + surface: str + grounding_source: str + no_placeholder: bool + not_walk_fragment: bool + length_adequate: bool + is_grounded: bool + topic_anchor_satisfied: bool | None # None when not applicable + + +@dataclass(frozen=True, slots=True) +class CaseResult: + case_id: str + category: str + turn_results: tuple[TurnResult, ...] + no_topic_drift: bool + + +@dataclass +class LaneReport: + metrics: dict[str, Any] = field(default_factory=dict) + case_details: list[dict[str, Any]] = field(default_factory=list) + + +def _run_case(case: dict[str, Any]) -> CaseResult: + rt = ChatRuntime() + turns: list[TurnResult] = [] + grounding_by_prompt: dict[str, list[str]] = {} + + for idx, turn in enumerate(case["turns"]): + prompt = turn["prompt"] + expects_grounded = bool(turn.get("expects_grounded", True)) + anaphora_anchor = turn.get("anaphora_anchor_to") + + resp = rt.chat(prompt) + surface = resp.surface + grounding = resp.grounding_source or "none" + + anchor_ok: bool | None = None + if anaphora_anchor: + anchor_ok = anaphora_anchor.lower() in surface.lower() + + turns.append(TurnResult( + turn_index=idx, + prompt=prompt, + surface=surface, + grounding_source=grounding, + no_placeholder=_check_no_placeholder(surface), + not_walk_fragment=_check_not_fragment(surface), + length_adequate=_check_length_adequate(surface), + is_grounded=_check_is_grounded(grounding, expects_grounded), + topic_anchor_satisfied=anchor_ok, + )) + grounding_by_prompt.setdefault(prompt, []).append(grounding) + + # No topic drift: any prompt that repeats must produce the SAME + # grounding tier on every firing (pack/teaching once → pack/teaching + # always). Drops to `none` after a successful grounding indicate + # state corruption. + no_drift = True + for srcs in grounding_by_prompt.values(): + if len(srcs) <= 1: + continue + strong = {"pack", "teaching"} + if any(s in strong for s in srcs) and any(s == "none" for s in srcs): + no_drift = False + break + + return CaseResult( + case_id=case["id"], + category=case.get("category", "uncategorised"), + turn_results=tuple(turns), + no_topic_drift=no_drift, + ) + + +def run_lane(cases: list[dict[str, Any]], config: Any = None) -> LaneReport: # noqa: ARG001 + if not cases: + return LaneReport(metrics={}, case_details=[]) + + results = [_run_case(c) for c in cases] + total_turns = sum(len(r.turn_results) for r in results) + + def _rate(pred: str) -> float: + passing = sum(1 for r in results for t in r.turn_results if getattr(t, pred)) + return round(passing / total_turns, 4) if total_turns else 1.0 + + anchor_turns = [t for r in results for t in r.turn_results + if t.topic_anchor_satisfied is not None] + anchor_rate = ( + round(sum(1 for t in anchor_turns if t.topic_anchor_satisfied) / len(anchor_turns), 4) + if anchor_turns else 1.0 + ) + + metrics: dict[str, Any] = { + "cases": len(results), + "total_turns": total_turns, + "no_placeholder_rate": _rate("no_placeholder"), + "not_walk_fragment_rate": _rate("not_walk_fragment"), + "length_adequate_rate": _rate("length_adequate"), + "is_grounded_rate": _rate("is_grounded"), + "topic_anchor_rate": anchor_rate, + "no_topic_drift_rate": ( + round(sum(1 for r in results if r.no_topic_drift) / len(results), 4) + if results else 1.0 + ), + } + + case_details = [ + { + "case_id": r.case_id, + "category": r.category, + "no_topic_drift": r.no_topic_drift, + "turns": [ + { + "turn_index": t.turn_index, + "prompt": t.prompt, + "surface": t.surface, + "grounding_source": t.grounding_source, + "no_placeholder": t.no_placeholder, + "not_walk_fragment": t.not_walk_fragment, + "length_adequate": t.length_adequate, + "is_grounded": t.is_grounded, + "topic_anchor_satisfied": t.topic_anchor_satisfied, + } + for t in r.turn_results + ], + } + for r in results + ] + + return LaneReport(metrics=metrics, case_details=case_details) diff --git a/evals/multi_sentence_response/contract.md b/evals/multi_sentence_response/contract.md new file mode 100644 index 00000000..38e153c4 --- /dev/null +++ b/evals/multi_sentence_response/contract.md @@ -0,0 +1,45 @@ +# Multi-Sentence Response Eval Lane — Contract + +**Lane:** `multi_sentence_response` +**Version:** v1 +**Created:** 2026-05-19 +**Status:** Red on creation — measurement substrate for compositional surface. + +## What this lane measures + +Whether `ChatRuntime` can emit a response that is more than a single +sentence when the prompt structurally calls for elaboration +("Explain X", "Tell me about X", "Describe X", "Walk me through X"). + +Currently every pack-grounded surface is a single sentence emitted +by `_frame_gloss`. NARRATIVE and EXAMPLE intents already compose +multi-clause output via teaching chains, so they are tested here too +as the *only* multi-sentence-capable code path. + +## Per-case predicates + +| Predicate | Definition | +|---|---| +| `sentence_count_>=_2` | the surface contains at least 2 terminated sentences (`.`, `?`, `!`) | +| `each_sentence_>=_4_tokens` | every sentence has ≥ 4 alphabetic tokens (no fragments) | +| `connective_present` | the surface contains at least one connective (`and`, `because`, `therefore`, `which`, `since`, `also`, `furthermore`, `however`, `consequently`) — only enforced when `expects_connective=true` | +| `not_just_provenance_tag` | sentence_count counts BEFORE the trailing provenance tag (`pack-grounded (…).`) is treated as its own sentence | +| `grounded` | `grounding_source` ∈ {pack, teaching} | +| `subject_named` | the prompt's subject lemma appears in the surface | + +## Scoring rubric + +```text +multi_sentence_rate = cases_with_>=2_sentences / total_cases +non_fragment_rate = cases_where_every_sentence_>=4_tokens / total_cases +connective_present_rate = cases_with_connective / cases_expecting_connective +``` + +## Doctrine constraints + +- The "trailing provenance tag" is structural, not a real sentence — + predicate logic strips it before counting. +- No LLM judge. Pure structural counting. +- Red-on-creation expected: only NARRATIVE / EXAMPLE / cross-pack / + composed_surface code paths can possibly satisfy `sentence_count_>=_2` + today. diff --git a/evals/multi_sentence_response/dev/cases.jsonl b/evals/multi_sentence_response/dev/cases.jsonl new file mode 100644 index 00000000..b5817d31 --- /dev/null +++ b/evals/multi_sentence_response/dev/cases.jsonl @@ -0,0 +1,3 @@ +{"id":"dev_multi_exam_001","category":"exam","prompt":"In your own words, what is the relationship between truth and evidence?","subject_lemma":"truth","expects_connective":true} +{"id":"dev_multi_poem_002","category":"poem","prompt":"Write a short poem about light.","subject_lemma":"light","expects_connective":false} +{"id":"dev_multi_summary_003","category":"summary","prompt":"Summarize what knowledge is and how it relates to evidence.","subject_lemma":"knowledge","expects_connective":true} diff --git a/evals/multi_sentence_response/public/v1/cases.jsonl b/evals/multi_sentence_response/public/v1/cases.jsonl new file mode 100644 index 00000000..5e8794d0 --- /dev/null +++ b/evals/multi_sentence_response/public/v1/cases.jsonl @@ -0,0 +1,15 @@ +{"id":"multi_explain_truth_001","category":"explain","prompt":"Explain truth.","subject_lemma":"truth","expects_connective":true} +{"id":"multi_explain_knowledge_002","category":"explain","prompt":"Explain knowledge.","subject_lemma":"knowledge","expects_connective":true} +{"id":"multi_explain_memory_003","category":"explain","prompt":"Explain memory.","subject_lemma":"memory","expects_connective":true} +{"id":"multi_tell_truth_004","category":"narrative","prompt":"Tell me about truth.","subject_lemma":"truth","expects_connective":false} +{"id":"multi_tell_light_005","category":"narrative","prompt":"Tell me about light.","subject_lemma":"light","expects_connective":false} +{"id":"multi_tell_parent_006","category":"narrative","prompt":"Tell me about parent.","subject_lemma":"parent","expects_connective":false} +{"id":"multi_describe_wisdom_007","category":"describe","prompt":"Describe wisdom.","subject_lemma":"wisdom","expects_connective":true} +{"id":"multi_describe_understanding_008","category":"describe","prompt":"Describe understanding.","subject_lemma":"understanding","expects_connective":true} +{"id":"multi_walk_inference_009","category":"walkthrough","prompt":"Walk me through inference.","subject_lemma":"inference","expects_connective":true} +{"id":"multi_walk_recall_010","category":"walkthrough","prompt":"Walk me through recall.","subject_lemma":"recall","expects_connective":true} +{"id":"multi_example_truth_011","category":"example","prompt":"Give me an example of truth.","subject_lemma":"truth","expects_connective":false} +{"id":"multi_example_parent_012","category":"example","prompt":"Give me an example of parent.","subject_lemma":"parent","expects_connective":false} +{"id":"multi_essay_truth_013","category":"essay","prompt":"Write a short paragraph about truth.","subject_lemma":"truth","expects_connective":true} +{"id":"multi_essay_memory_014","category":"essay","prompt":"Write a short paragraph about memory.","subject_lemma":"memory","expects_connective":true} +{"id":"multi_compose_def_cause_015","category":"compose","prompt":"What is truth, and why does it matter?","subject_lemma":"truth","expects_connective":true} diff --git a/evals/multi_sentence_response/runner.py b/evals/multi_sentence_response/runner.py new file mode 100644 index 00000000..1f5020c8 --- /dev/null +++ b/evals/multi_sentence_response/runner.py @@ -0,0 +1,150 @@ +"""Multi-sentence response eval lane runner. + +Measures whether ``ChatRuntime`` emits more than one sentence when +the prompt structurally calls for elaboration. Strips the trailing +provenance tag (``pack-grounded (...).``) before counting sentences +so the metric reflects substantive content. + +Framework contract: ``run_lane(cases, config=None) -> LaneReport``. + +Case schema: + { + "id": "...", + "category": "...", + "prompt": "Tell me about truth.", + "subject_lemma": "truth", + "expects_connective": true + } +""" +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any + +from chat.runtime import ChatRuntime + + +_PROVENANCE_TAIL_RE = re.compile( + r"\s*(pack-grounded|teaching-grounded)\s*\([^)]+\)\.?\s*$" +) + +_CONNECTIVES = ( + "and", "because", "therefore", "which", "since", "also", + "furthermore", "however", "consequently", "thus", "so", "while", + "whereas", "moreover", "in turn", +) + + +def _strip_provenance(surface: str) -> str: + return _PROVENANCE_TAIL_RE.sub("", surface).strip() + + +def _split_sentences(text: str) -> list[str]: + parts = re.split(r"(?<=[.!?])\s+", text.strip()) + return [p.strip() for p in parts if p.strip()] + + +def _alpha_tokens(text: str) -> int: + return len(re.findall(r"[A-Za-z]+", text)) + + +def _has_connective(text: str) -> bool: + low = text.lower() + return any(re.search(rf"\b{re.escape(c)}\b", low) for c in _CONNECTIVES) + + +@dataclass(frozen=True, slots=True) +class CaseResult: + case_id: str + category: str + prompt: str + surface: str + grounding_source: str + sentence_count: int + each_sentence_long_enough: bool + connective_present: bool + grounded: bool + subject_named: bool + expects_connective: bool + + +@dataclass +class LaneReport: + metrics: dict[str, Any] = field(default_factory=dict) + case_details: list[dict[str, Any]] = field(default_factory=list) + + +def _run_case(case: dict[str, Any]) -> CaseResult: + rt = ChatRuntime() + resp = rt.chat(case["prompt"]) + surface = resp.surface + grounding = resp.grounding_source or "none" + + stripped = _strip_provenance(surface) + sentences = _split_sentences(stripped) + each_long = all(_alpha_tokens(s) >= 4 for s in sentences) if sentences else False + + subj = case.get("subject_lemma", "").lower() + subj_named = (subj in surface.lower()) if subj else True + + return CaseResult( + case_id=case["id"], + category=case.get("category", "uncategorised"), + prompt=case["prompt"], + surface=surface, + grounding_source=grounding, + sentence_count=len(sentences), + each_sentence_long_enough=each_long, + connective_present=_has_connective(stripped), + grounded=(grounding in {"pack", "teaching"}), + subject_named=subj_named, + expects_connective=bool(case.get("expects_connective", False)), + ) + + +def run_lane(cases: list[dict[str, Any]], config: Any = None) -> LaneReport: # noqa: ARG001 + if not cases: + return LaneReport(metrics={}, case_details=[]) + + results = [_run_case(c) for c in cases] + total = len(results) + + multi = sum(1 for r in results if r.sentence_count >= 2) + non_frag = sum(1 for r in results if r.each_sentence_long_enough) + grounded = sum(1 for r in results if r.grounded) + named = sum(1 for r in results if r.subject_named) + + conn_expected = [r for r in results if r.expects_connective] + conn_rate = ( + round(sum(1 for r in conn_expected if r.connective_present) / len(conn_expected), 4) + if conn_expected else 1.0 + ) + + metrics: dict[str, Any] = { + "cases": total, + "multi_sentence_rate": round(multi / total, 4) if total else 0.0, + "non_fragment_rate": round(non_frag / total, 4) if total else 0.0, + "grounded_rate": round(grounded / total, 4) if total else 0.0, + "subject_named_rate": round(named / total, 4) if total else 0.0, + "connective_present_rate": conn_rate, + } + + case_details = [ + { + "case_id": r.case_id, + "category": r.category, + "prompt": r.prompt, + "surface": r.surface, + "grounding_source": r.grounding_source, + "sentence_count": r.sentence_count, + "each_sentence_long_enough": r.each_sentence_long_enough, + "connective_present": r.connective_present, + "grounded": r.grounded, + "subject_named": r.subject_named, + "expects_connective": r.expects_connective, + } + for r in results + ] + + return LaneReport(metrics=metrics, case_details=case_details) diff --git a/evals/self_consistency_over_time/contract.md b/evals/self_consistency_over_time/contract.md new file mode 100644 index 00000000..06b28936 --- /dev/null +++ b/evals/self_consistency_over_time/contract.md @@ -0,0 +1,46 @@ +# Self-Consistency Over Time Eval Lane — Contract + +**Lane:** `self_consistency_over_time` +**Version:** v1 +**Created:** 2026-05-19 +**Status:** Red on creation — measurement substrate for thread-level +truthfulness. + +## What this lane measures + +Whether `ChatRuntime` produces the same answer to a factual prompt +at turn 1, turn N, and turn 2N — with arbitrary unrelated turns +interleaved. This is the strongest test of identity-truthfulness +under accumulated state. + +Distinct from `warmed_session_consistency` (which only replays the +*same* prompt back-to-back, where vault state has barely diverged). +Here we measure stability across **drift**. + +## Per-case predicates + +| Predicate | Definition | +|---|---| +| `byte_identical` | every probe response is byte-for-byte identical | +| `key_terms_stable` | the prompt's `expected_key_terms` all appear in every probe response | +| `grounding_source_stable`| every probe response has the same `grounding_source` | +| `no_walk_fragment` | no probe response degrades to a < 4-token surface | + +## Scoring rubric + +```text +byte_identical_rate = cases_byte_identical / total_cases +terms_stable_rate = cases_terms_stable / total_cases +grounding_stable_rate = cases_grounding_stable / total_cases +no_drift_to_fragment = cases_no_fragment / total_cases +``` + +## Doctrine constraints + +- Byte-identical is the gold standard but the lane also tracks the + weaker `key_terms_stable` predicate so we can distinguish *exact* + determinism from *semantic* stability. +- No LLM judge. +- Red-on-creation is acceptable for `byte_identical` — the warm path + may inject thread anaphora prefixes that legitimately change bytes; + the `key_terms_stable` predicate is the load-bearing one. diff --git a/evals/self_consistency_over_time/dev/cases.jsonl b/evals/self_consistency_over_time/dev/cases.jsonl new file mode 100644 index 00000000..4e3c369a --- /dev/null +++ b/evals/self_consistency_over_time/dev/cases.jsonl @@ -0,0 +1,2 @@ +{"id":"dev_consistency_long_thread_001","category":"long_thread","probe_prompt":"What is truth?","expected_key_terms":["truth"],"probe_at_turns":[0,8,16,24],"filler_prompts":["What is light?","Define memory.","What is knowledge?","Tell me about wisdom.","What is parent?","What is family?","Define evidence."]} +{"id":"dev_consistency_with_anaphora_002","category":"anaphora_interleaved","probe_prompt":"What is light?","expected_key_terms":["light"],"probe_at_turns":[0,5,10],"filler_prompts":["Why does it exist?","What is memory?","Why does it matter?","What is truth?"]} diff --git a/evals/self_consistency_over_time/public/v1/cases.jsonl b/evals/self_consistency_over_time/public/v1/cases.jsonl new file mode 100644 index 00000000..46ef2ab9 --- /dev/null +++ b/evals/self_consistency_over_time/public/v1/cases.jsonl @@ -0,0 +1,7 @@ +{"id":"consistency_truth_001","category":"cognition","probe_prompt":"What is truth?","expected_key_terms":["truth"],"probe_at_turns":[0,4,9],"filler_prompts":["What is light?","Define memory.","What is knowledge?","Tell me about wisdom."]} +{"id":"consistency_knowledge_002","category":"cognition","probe_prompt":"What is knowledge?","expected_key_terms":["knowledge"],"probe_at_turns":[0,5,11],"filler_prompts":["What is parent?","Define light.","What is family?","Tell me about memory.","What is recall?"]} +{"id":"consistency_light_003","category":"cognition","probe_prompt":"What is light?","expected_key_terms":["light"],"probe_at_turns":[0,3,7,12],"filler_prompts":["What is truth?","Define wisdom.","What is meaning."]} +{"id":"consistency_parent_004","category":"relations","probe_prompt":"What is parent?","expected_key_terms":["parent"],"probe_at_turns":[0,4,8],"filler_prompts":["What is truth?","Define knowledge.","What is light?"]} +{"id":"consistency_memory_005","category":"cognition","probe_prompt":"What is memory?","expected_key_terms":["memory"],"probe_at_turns":[0,6,12],"filler_prompts":["What is family?","Define wisdom.","Tell me about recall.","What is judgment?"]} +{"id":"consistency_define_evidence_006","category":"definition","probe_prompt":"Define evidence.","expected_key_terms":["evidence"],"probe_at_turns":[0,5,10],"filler_prompts":["What is truth?","What is light?","What is knowledge?"]} +{"id":"consistency_cause_oov_007","category":"cause_no_chain","probe_prompt":"How does memory work?","expected_key_terms":[],"probe_at_turns":[0,4,9],"filler_prompts":["What is truth?","What is light?","What is knowledge?"]} diff --git a/evals/self_consistency_over_time/runner.py b/evals/self_consistency_over_time/runner.py new file mode 100644 index 00000000..b55962e4 --- /dev/null +++ b/evals/self_consistency_over_time/runner.py @@ -0,0 +1,143 @@ +"""Self-consistency over time eval lane runner. + +Same factual prompt is asked at multiple turn indices with unrelated +turns interleaved. Measures whether accumulated state causes the +answer to drift. + +Framework contract: ``run_lane(cases, config=None) -> LaneReport``. + +Case schema: + { + "id": "...", + "category": "...", + "probe_prompt": "What is truth?", + "expected_key_terms": ["truth", "evidence"], + "probe_at_turns": [0, 4, 9], + "filler_prompts": ["What is light?", "Define memory.", ...] + } + +The lane interleaves probe_prompt at the requested turn indices and +fills remaining indices with filler_prompts (cycled if needed). +""" +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any + +from chat.runtime import ChatRuntime + + +def _alpha_tokens(text: str) -> int: + return len(re.findall(r"[A-Za-z]+", text)) + + +@dataclass(frozen=True, slots=True) +class ProbeResult: + turn_index: int + surface: str + grounding_source: str + + +@dataclass(frozen=True, slots=True) +class CaseResult: + case_id: str + category: str + probe_prompt: str + probes: tuple[ProbeResult, ...] + byte_identical: bool + key_terms_stable: bool + grounding_source_stable: bool + no_walk_fragment: bool + + +@dataclass +class LaneReport: + metrics: dict[str, Any] = field(default_factory=dict) + case_details: list[dict[str, Any]] = field(default_factory=list) + + +def _run_case(case: dict[str, Any]) -> CaseResult: + rt = ChatRuntime() + probe_prompt: str = case["probe_prompt"] + probe_turns = sorted(int(i) for i in case["probe_at_turns"]) + fillers: list[str] = list(case.get("filler_prompts", [])) + key_terms = [t.lower() for t in case.get("expected_key_terms", [])] + max_turn = max(probe_turns) + probes: list[ProbeResult] = [] + filler_idx = 0 + + for turn in range(max_turn + 1): + if turn in probe_turns: + resp = rt.chat(probe_prompt) + probes.append(ProbeResult( + turn_index=turn, + surface=resp.surface, + grounding_source=resp.grounding_source or "none", + )) + else: + prompt = fillers[filler_idx % len(fillers)] if fillers else "What is light?" + filler_idx += 1 + rt.chat(prompt) + + surfaces = [p.surface for p in probes] + groundings = [p.grounding_source for p in probes] + + byte_id = len(set(surfaces)) == 1 + grounding_stable = len(set(groundings)) == 1 + terms_stable = all( + all(term in s.lower() for term in key_terms) for s in surfaces + ) if key_terms else True + no_fragment = all(_alpha_tokens(s) >= 4 for s in surfaces) + + return CaseResult( + case_id=case["id"], + category=case.get("category", "uncategorised"), + probe_prompt=probe_prompt, + probes=tuple(probes), + byte_identical=byte_id, + key_terms_stable=terms_stable, + grounding_source_stable=grounding_stable, + no_walk_fragment=no_fragment, + ) + + +def run_lane(cases: list[dict[str, Any]], config: Any = None) -> LaneReport: # noqa: ARG001 + if not cases: + return LaneReport(metrics={}, case_details=[]) + + results = [_run_case(c) for c in cases] + total = len(results) + + metrics: dict[str, Any] = { + "cases": total, + "byte_identical_rate": round(sum(1 for r in results if r.byte_identical) / total, 4), + "key_terms_stable_rate": round(sum(1 for r in results if r.key_terms_stable) / total, 4), + "grounding_source_stable_rate": round( + sum(1 for r in results if r.grounding_source_stable) / total, 4 + ), + "no_walk_fragment_rate": round(sum(1 for r in results if r.no_walk_fragment) / total, 4), + } + + case_details = [ + { + "case_id": r.case_id, + "category": r.category, + "probe_prompt": r.probe_prompt, + "byte_identical": r.byte_identical, + "key_terms_stable": r.key_terms_stable, + "grounding_source_stable": r.grounding_source_stable, + "no_walk_fragment": r.no_walk_fragment, + "probes": [ + { + "turn_index": p.turn_index, + "surface": p.surface, + "grounding_source": p.grounding_source, + } + for p in r.probes + ], + } + for r in results + ] + + return LaneReport(metrics=metrics, case_details=case_details) diff --git a/notes/long_span_fluency_baseline_2026-05-19.md b/notes/long_span_fluency_baseline_2026-05-19.md new file mode 100644 index 00000000..07d9f802 --- /dev/null +++ b/notes/long_span_fluency_baseline_2026-05-19.md @@ -0,0 +1,103 @@ +# Long-Span Fluency Baseline — 2026-05-19 + +Numbers-only baseline at the moment the warm-path pack-grounding +patch lands and three new eval lanes go red. Each lane is a +deterministic, lexical-predicate measurement substrate; no LLM judge, +no embedding similarity. + +## Step 1 fix — warm_grounding_stability targeted patch + +`chat/runtime.py:_maybe_pack_grounded_surface` now accepts +`allow_warm=True`; warm path invokes it after articulation, overrides +`response_surface` / `articulation` / `grounding_source` when a pack +or teaching surface is available. CAUSE / VERIFICATION without a +teaching chain emit the unknown-domain disclosure on warm just as on +cold (preserves the discovery signal — no fabricated vault content). + +### warmed_session_consistency (public/v1, 8 cases / 18 turns) + +| metric | before | after | +|---|---|---| +| no_placeholder_rate | 1.0 | 1.0 | +| telemetry_consistency_rate | 1.0 | 1.0 | +| grounding_match_rate | n/a | 1.0 | +| warm_grounding_stability | 0.0 | 1.0 | + +### Cognition lane (regression check) + +| split | intent | term capture | surface groundedness | versor closure | +|---|---|---|---|---| +| public | 100% | 91.7% | 100% | 100% | +| holdout | 100% | 83.3% | 100% | 100% | + +Cognition lane byte-identical to pre-patch baseline. Full test +suite: **2294 passed, 3 skipped** in ~5 min. + +## Step 2 — three new red lanes + +### conversational_thread_coherence (public/v1, 6 cases / 45 turns) + +| metric | value | +|---|---| +| no_placeholder_rate | 1.0 | +| not_walk_fragment_rate | 1.0 | +| length_adequate_rate | 1.0 | +| is_grounded_rate | 0.9333 | +| topic_anchor_rate | 0.5 | +| no_topic_drift_rate | 0.8333 | + +**Read:** placeholder / fragment / length predicates are clean (Step 1 +fix carries through). `topic_anchor_rate=0.5` is the next gap — +when a follow-up uses anaphora ("Why does it exist?") the system fails +to anchor on the prior subject half the time. `no_topic_drift_rate=0.8333` +— 1 of 6 cases drops a grounded subject to `none` on later replay. + +### multi_sentence_response (public/v1, 15 cases) + +| metric | value | +|---|---| +| multi_sentence_rate | 0.5333 | +| non_fragment_rate | 1.0 | +| grounded_rate | 0.4667 | +| subject_named_rate | 0.5333 | +| connective_present_rate | 0.1 | + +**Read:** half of the elaboration prompts ("Explain X", "Describe X") +still get a single-sentence response; only 10% of cases that *should* +contain a discourse connective do. This is the **single biggest +architectural gap** — there is no paragraph-level composer. + +### self_consistency_over_time (public/v1, 7 cases) + +| metric | value | +|---|---| +| byte_identical_rate | 0.8571 | +| key_terms_stable_rate | 0.8571 | +| grounding_source_stable_rate | 0.8571 | +| no_walk_fragment_rate | 1.0 | + +**Read:** 6 of 7 probes are byte-identical across long interleaved +threads. The one drifting case is CAUSE-no-chain (`How does memory +work?`) — vault-content accumulation across unrelated fillers changes +the disclosure. Worth a follow-up: disclosure should also be stable. + +## Architectural reading + +The Step 1 patch closes the *replay* dimension. The three new lanes +quantify what remains for *robust long-span fluency*: + +1. **Multi-clause composition** (multi_sentence + connective_present) — + no paragraph-level composer exists today; every composer in + `chat/pack_grounding.py` and `chat/teaching_grounding.py` returns + one terminal-`.` string. This is the biggest gap. +2. **Anaphora resolution** (topic_anchor_rate=0.5) — the existing + `thread_anaphora_prefix` is structural ("Recalling turn 0:") not + in-clause. Real coreference would lift this to ~1.0. +3. **No-drift across interleaved turns** (consistency byte_identical + not 1.0) — drift only affects ungrounded CAUSE; disclosure + stability is a smaller follow-up. + +The natural next step is a paragraph-level composer that orchestrates +the existing single-sentence composers. Doctrine-respecting because +deterministic. Forward-compatible with the deferred SurfaceSelector +RFC.