From 07fefb923c30e2450d73959878bd3dc7c21cdc7d Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 19 May 2026 12:13:44 -0700 Subject: [PATCH] feat(evals): articulate/disclosure/unarticulate partition Tightens the multi_sentence_response lane predicates so OOV invitations and refusal disclosures can no longer be counted as articulate capability. Three new metrics partition the case space: articulate_sentence_rate - >=2 sentences AND grounded in {pack, teaching}. Real capability. disclosure_sentence_rate - >=2 sentences AND grounded in {oov, refusal, none}. Structural multi-sentence from disclosure templates. unarticulate_rate - <2 sentences regardless of source. The three sum to 1.0 (modulo rounding) by construction. The doctrine-correct headline is now ``articulate_sentence_rate``; ``multi_sentence_rate`` is kept as a continuity metric only. 2 new tests pin: (a) the three-way partition is total and disjoint (articulate + disclosure + unarticulate == 1.0); (b) OOV/refusal disclosure surfaces contribute to disclosure_sentence_rate but never to articulate_sentence_rate. Live A/B on 21 cases under the new partition: flag off: articulate=0.0952, disclosure=0.0476, unarticulate=0.8571 flag on : articulate=0.8571, disclosure=0.0476, unarticulate=0.0952 Planner lift is +76pp on articulate. Disclosure stays flat across the flag (the planner gate correctly leaves disclosure surfaces alone). The remaining 9.5pp unarticulate flag-on is the genuine miss list (walkthrough + compound prompts) that the next two landings will target. contract.md updated to make articulate_sentence_rate the headline and to document the partition explicitly. cognition eval byte-identical: public 100/100/91.7/100. smoke suite 67/67. --- evals/multi_sentence_response/contract.md | 25 ++++-- evals/multi_sentence_response/runner.py | 42 ++++++++-- tests/test_multi_sentence_response_eval.py | 96 ++++++++++++++++++++++ 3 files changed, 153 insertions(+), 10 deletions(-) diff --git a/evals/multi_sentence_response/contract.md b/evals/multi_sentence_response/contract.md index ac502a7a..61e07720 100644 --- a/evals/multi_sentence_response/contract.md +++ b/evals/multi_sentence_response/contract.md @@ -30,13 +30,28 @@ as the *only* multi-sentence-capable code path. ## Scoring rubric ```text -multi_sentence_rate = cases_with_>=2_sentences / total_cases -non_fragment_rate = cases_where_every_sentence_>=4_tokens / total_cases -connective_present_rate = cases_with_connective / cases_expecting_connective -primed_cases = cases_where_priming_prompts_engaged -primed_multi_sentence_rate = primed_cases_with_>=2_sentences / primed_cases +articulate_sentence_rate = cases with >=2 sentences AND grounded in {pack, teaching} / total +disclosure_sentence_rate = cases with >=2 sentences AND grounded in {oov, refusal, none} / total +unarticulate_rate = cases with <2 sentences / total +multi_sentence_rate = cases_with_>=2_sentences / total_cases # continuity metric +non_fragment_rate = cases_where_every_sentence_>=4_tokens / total_cases +connective_present_rate = cases_with_connective / cases_expecting_connective +primed_cases = cases_where_priming_prompts_engaged +primed_multi_sentence_rate = primed_cases_with_>=2_sentences / primed_cases ``` +**Doctrine-correct headline:** `articulate_sentence_rate`. + +`multi_sentence_rate` is kept for continuity but is misleading on its own: +OOV teaching-invitation surfaces ("I don't know that yet — can you teach +me?") and refusal disclosures ("I don't know — insufficient grounding +for that yet.") are categorically multi-sentence by template, not by +articulation. They count toward `disclosure_sentence_rate`, never +`articulate_sentence_rate`. + +The decomposition is total: +`articulate + disclosure + unarticulate = 1.0` (modulo rounding). + ## Priming (warm-path measurement) A case may carry an optional `priming_prompts: [str, ...]` array. The diff --git a/evals/multi_sentence_response/runner.py b/evals/multi_sentence_response/runner.py index be0f0fe7..d1cd1030 100644 --- a/evals/multi_sentence_response/runner.py +++ b/evals/multi_sentence_response/runner.py @@ -163,13 +163,45 @@ def run_lane(cases: list[dict[str, Any]], config: Any = None) -> LaneReport: if conn_expected else 1.0 ) + # ``multi_sentence_rate`` historically counted any case with ≥ 2 + # sentences regardless of grounding source. That admitted OOV + # teaching invitations and refusal disclosures into the headline + # capability number — fixed here by splitting into three honest + # buckets: + # + # articulate_sentence_rate — ≥2 sentences AND grounded in pack + # or teaching (real capability). + # disclosure_sentence_rate — ≥2 sentences but grounded in oov, + # refusal, none, etc. (structural + # multi-sentence from disclosure + # templates, not articulation). + # unarticulate_rate — <2 sentences regardless of source. + # + # ``multi_sentence_rate`` is retained as a continuity metric. The + # doctrine-correct headline is ``articulate_sentence_rate``. + _DISCLOSURE_SOURCES = {"oov", "refusal", "none"} + articulate = sum( + 1 for r in results + if r.sentence_count >= 2 + and r.grounding_source in {"pack", "teaching"} + ) + disclosure = sum( + 1 for r in results + if r.sentence_count >= 2 + and r.grounding_source in _DISCLOSURE_SOURCES + ) + unarticulate = sum(1 for r in results if r.sentence_count < 2) + metrics: dict[str, Any] = { "cases": total, - "multi_sentence_rate": round(multi / total, 4) if total else 0.0, - "non_fragment_rate": round(non_frag / total, 4) if total else 0.0, - "grounded_rate": round(grounded / total, 4) if total else 0.0, - "subject_named_rate": round(named / total, 4) if total else 0.0, - "connective_present_rate": conn_rate, + "multi_sentence_rate": round(multi / total, 4) if total else 0.0, + "articulate_sentence_rate": round(articulate / total, 4) if total else 0.0, + "disclosure_sentence_rate": round(disclosure / total, 4) if total else 0.0, + "unarticulate_rate": round(unarticulate / total, 4) if total else 0.0, + "non_fragment_rate": round(non_frag / total, 4) if total else 0.0, + "grounded_rate": round(grounded / total, 4) if total else 0.0, + "subject_named_rate": round(named / total, 4) if total else 0.0, + "connective_present_rate": conn_rate, } primed_results = [r for r in results if r.primed] diff --git a/tests/test_multi_sentence_response_eval.py b/tests/test_multi_sentence_response_eval.py index ba931b41..92cded21 100644 --- a/tests/test_multi_sentence_response_eval.py +++ b/tests/test_multi_sentence_response_eval.py @@ -173,6 +173,102 @@ def test_priming_default_is_cold_start(monkeypatch) -> None: assert detail["primed"] is False +def test_articulate_disclosure_unarticulate_partition(monkeypatch) -> None: + """``articulate + disclosure + unarticulate`` must equal 1.0 modulo + rounding. No case can contribute to more than one bucket. + + Articulate: ≥2 sentences AND grounding in {pack, teaching}. + Disclosure: ≥2 sentences AND grounding in {oov, refusal, none}. + Unarticulate: <2 sentences (regardless of source). + """ + + class _FakeResponse: + def __init__(self, surface: str, source: str) -> None: + self.surface = surface + self.grounding_source = source + + plan = iter([ + # case 0: articulate (pack + ≥2 sentences) + _FakeResponse( + "Truth is X. Furthermore, truth belongs to cognition.truth.", + "pack", + ), + # case 1: articulate (teaching + ≥2 sentences) + _FakeResponse( + "Light reveals truth. In turn, truth grounds knowledge.", + "teaching", + ), + # case 2: disclosure (oov + ≥2 sentences) + _FakeResponse( + "I don't know that yet. Can you teach me?", + "oov", + ), + # case 3: disclosure (none + ≥2 sentences) + _FakeResponse( + "I don't know. Insufficient grounding for that yet.", + "none", + ), + # case 4: unarticulate (single sentence, regardless of source) + _FakeResponse("Truth.", "vault"), + ]) + + class _FakeRuntime: + def __init__(self, config=None): # noqa: ARG002 + pass + + def chat(self, prompt: str) -> _FakeResponse: # noqa: ARG002 + return next(plan) + + monkeypatch.setattr(runner, "ChatRuntime", _FakeRuntime) + cases = [ + {"id": f"c{i}", "category": "x", "prompt": "p", + "subject_lemma": "", "expects_connective": False} + for i in range(5) + ] + metrics = run_lane(cases).metrics + + assert metrics["articulate_sentence_rate"] == 0.4 # 2/5 + assert metrics["disclosure_sentence_rate"] == 0.4 # 2/5 + assert metrics["unarticulate_rate"] == 0.2 # 1/5 + # Partition is total — must sum to 1.0 modulo rounding. + total = ( + metrics["articulate_sentence_rate"] + + metrics["disclosure_sentence_rate"] + + metrics["unarticulate_rate"] + ) + assert abs(total - 1.0) < 1e-9 + + +def test_disclosure_never_inflates_articulate(monkeypatch) -> None: + """OOV invitations and refusal disclosures must never contribute to + ``articulate_sentence_rate`` even when they are multi-sentence by + template. + """ + + class _FakeResponse: + surface = "I don't know that yet. Can you teach me?" + grounding_source = "oov" + + class _FakeRuntime: + def __init__(self, config=None): # noqa: ARG002 + pass + + def chat(self, prompt: str) -> _FakeResponse: # noqa: ARG002 + return _FakeResponse() + + monkeypatch.setattr(runner, "ChatRuntime", _FakeRuntime) + cases = [ + {"id": "oov_case", "category": "x", "prompt": "p", + "subject_lemma": "", "expects_connective": False} + ] + metrics = run_lane(cases).metrics + + # Multi-sentence is True (continuity metric), but articulate is False. + assert metrics["multi_sentence_rate"] == 1.0 + assert metrics["articulate_sentence_rate"] == 0.0 + assert metrics["disclosure_sentence_rate"] == 1.0 + + def test_primed_multi_sentence_rate_separates_from_aggregate(monkeypatch) -> None: """The ``primed_multi_sentence_rate`` metric reports only on cases that actually exercised priming, so cold-start cases never inflate