From 6b0d7239879d15a1edf1aeb8dc197387e9b0ad01 Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 20 May 2026 15:43:01 -0700 Subject: [PATCH] =?UTF-8?q?fix(evals):=20prompt=5Fdiversity=20gloss-quote?= =?UTF-8?q?=20heuristic=20=E2=80=94=204-token=20window=20=E2=86=92=20subst?= =?UTF-8?q?ring=20(#69)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v1 gloss-quote detector used a 4-token contiguous window of ≥4-char tokens. That heuristic was too strict for the actual ADR-0084 brief gloss style, which is deliberately short and primitive-only: light "visible medium that reveal truth" 5 tokens ≥4 chars parent "person with a child" 3 tokens ≥4 chars ← can't window recall "get memory from before" 3 tokens ≥4 chars ← can't window wisdom "good use of knowledge" 2 tokens ≥4 chars ← can't window Result: post-PR #65 baseline showed gloss_quote_rate=0.0% even though the pack-grounded composer was visibly emitting glosses verbatim: surface: "Parent is person with a child. pack-grounded (en_core_relations_v1)." gloss: "person with a child" window: could not even form Replace with substring match against the gloss text. The composer emits the gloss verbatim (no paraphrasing — that's the no-LLM discipline), so substring is exact, high-confidence, and trivially correct: gloss_quoted ⟺ gloss.lower().strip() in surface.lower() Re-baselined v1/public (26 cases): gloss_quote_rate: 7.7% (false-positive 4-token window noise) → 0.0% (post-#65, broken metric) → 11.5% (this PR, real signal) The other four metrics unchanged. 3/26 cases (DEFINITION on ``evidence``/``recall``/``parent``) are detected as gloss-quoted now, which matches reality — the pack-grounded composer at chat/pack_grounding.py:398 has been gloss-aware all along; it just had no glosses to quote pre-#65. Why this is just a heuristic refinement, not a contract change: The contract.md still says v1 has NO pass thresholds beyond versor_closure_rate==1.00. The lane's job is to establish baseline distribution. The heuristic was *measuring the wrong thing* — fixing the measurement is a contract clarification, not a contract change. Tests added (TestGlossQuote, 4 cases): - short brief-style gloss detected via substring - chain-walk surface for same lemma NOT counted as gloss-quoted - unknown term returns False - empty terms returns False Updated the function docstring with the post-#65 context so future readers understand why v1's contract predicted 0% but reality is ~12%. --- evals/prompt_diversity/runner.py | 30 +++++++++------- tests/test_prompt_diversity_runner.py | 49 +++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 13 deletions(-) diff --git a/evals/prompt_diversity/runner.py b/evals/prompt_diversity/runner.py index 9d2f8b12..f17f212b 100644 --- a/evals/prompt_diversity/runner.py +++ b/evals/prompt_diversity/runner.py @@ -189,12 +189,21 @@ def _surface_has_audit_leak(surface: str) -> bool: def _surface_quotes_gloss(surface: str, expected_terms: tuple[str, ...]) -> bool: """Return True iff the surface visibly draws from a pack gloss. - Uses :func:`chat.pack_resolver.resolve_gloss` to fetch any gloss - bound to each expected term, then checks for a substantive - substring (4-token window) of the gloss in the surface. + Resolves each expected term via + :func:`chat.pack_resolver.resolve_gloss`, then asks: does the + surface contain the gloss text verbatim? The pack-grounded + composer emits the gloss without paraphrasing + (``"{Lemma} is {gloss}."``), so substring match is an exact and + high-confidence "gloss actually quoted" signal — no fuzzy windows, + no false-positives from one shared content word. - v1 ≈ 0% by design — the composer does not consume glosses yet. - The metric exists so ADR-0085's lift is quantifiable. + Note on the v1 prediction: the contract predicted ``≈ 0%`` here, + on the assumption that the composer would not consume glosses + until ADR-0085 landed. In fact the pack-grounded composer at + ``chat/pack_grounding.py:398-434`` was *already* gloss-aware + pre-ADR-0084 but had no glosses to consume. Once PR #65's content + landed, the composer immediately started emitting glosses on + DEFINITION/RECALL. This metric now reflects that reality. """ if not expected_terms: return False @@ -207,15 +216,10 @@ def _surface_quotes_gloss(surface: str, expected_terms: tuple[str, ...]) -> bool if resolved is None: continue _pack_id, _pos, gloss = resolved - gloss_tokens = [t for t in re.findall(r"[a-z]+", gloss.lower()) if len(t) >= 4] - if not gloss_tokens: + if not gloss.strip(): continue - # 4-token contiguous window match — high-confidence "gloss - # actually quoted", not "shared one common word". - for i in range(len(gloss_tokens) - 3): - window = " ".join(gloss_tokens[i : i + 4]) - if window in surface_lower: - return True + if gloss.lower().strip() in surface_lower: + return True return False diff --git a/tests/test_prompt_diversity_runner.py b/tests/test_prompt_diversity_runner.py index ae094af8..3df6bd22 100644 --- a/tests/test_prompt_diversity_runner.py +++ b/tests/test_prompt_diversity_runner.py @@ -82,6 +82,55 @@ class TestAuditLeak: assert _surface_has_audit_leak(surface) is is_leak +# --------------------------------------------------------------------------- # +# Gloss-quote detector +# --------------------------------------------------------------------------- # + + +class TestGlossQuote: + """The detector is exact-substring against the pack's gloss text, + not a fuzzy window. The pack-grounded composer emits gloss text + verbatim, so substring match is the right signal: zero false + positives, zero false negatives on brief-style short glosses where + a 4-token window would be impossible (e.g. ``person`` → ``"person + with a child"`` has only 3 tokens ≥4 chars). + """ + + def _make(self, surface: str, terms: tuple[str, ...]) -> bool: + from evals.prompt_diversity.runner import _surface_quotes_gloss + return _surface_quotes_gloss(surface, terms) + + def test_quoted_short_gloss_detected(self) -> None: + # ``light`` gloss is ``"visible medium that reveal truth"`` — + # 5 tokens, but only 5 are ≥4 chars; the old 4-token window + # would barely fit. ``parent`` gloss is ``"person with a child"`` + # — 4 tokens, 3 are ≥4 chars; the old window could never match. + # Substring match handles both natively. + assert self._make( + "Parent is person with a child. pack-grounded (en_core_relations_v1).", + ("parent",), + ) is True + assert self._make( + "Light is visible medium that reveal truth. pack-grounded (en_core_cognition_v1).", + ("light",), + ) is True + + def test_unquoted_surface_returns_false(self) -> None: + # Chain-walk surface for the same lemma must NOT count as + # gloss-quoted — it shares vocabulary but doesn't quote the + # gloss itself. + assert self._make( + "light — teaching-grounded (cognition_chains_v1): cognition.illumination; logos.core.", + ("light",), + ) is False + + def test_unknown_term_returns_false(self) -> None: + assert self._make("anything", ("nonsense_lemma_42",)) is False + + def test_empty_terms_returns_false(self) -> None: + assert self._make("anything", ()) is False + + # --------------------------------------------------------------------------- # # End-to-end run on the v1 public split # --------------------------------------------------------------------------- #