diff --git a/evals/prompt_diversity/runner.py b/evals/prompt_diversity/runner.py index 9d2f8b12..f17f212b 100644 --- a/evals/prompt_diversity/runner.py +++ b/evals/prompt_diversity/runner.py @@ -189,12 +189,21 @@ def _surface_has_audit_leak(surface: str) -> bool: def _surface_quotes_gloss(surface: str, expected_terms: tuple[str, ...]) -> bool: """Return True iff the surface visibly draws from a pack gloss. - Uses :func:`chat.pack_resolver.resolve_gloss` to fetch any gloss - bound to each expected term, then checks for a substantive - substring (4-token window) of the gloss in the surface. + Resolves each expected term via + :func:`chat.pack_resolver.resolve_gloss`, then asks: does the + surface contain the gloss text verbatim? The pack-grounded + composer emits the gloss without paraphrasing + (``"{Lemma} is {gloss}."``), so substring match is an exact and + high-confidence "gloss actually quoted" signal — no fuzzy windows, + no false-positives from one shared content word. - v1 ≈ 0% by design — the composer does not consume glosses yet. - The metric exists so ADR-0085's lift is quantifiable. + Note on the v1 prediction: the contract predicted ``≈ 0%`` here, + on the assumption that the composer would not consume glosses + until ADR-0085 landed. In fact the pack-grounded composer at + ``chat/pack_grounding.py:398-434`` was *already* gloss-aware + pre-ADR-0084 but had no glosses to consume. Once PR #65's content + landed, the composer immediately started emitting glosses on + DEFINITION/RECALL. This metric now reflects that reality. """ if not expected_terms: return False @@ -207,15 +216,10 @@ def _surface_quotes_gloss(surface: str, expected_terms: tuple[str, ...]) -> bool if resolved is None: continue _pack_id, _pos, gloss = resolved - gloss_tokens = [t for t in re.findall(r"[a-z]+", gloss.lower()) if len(t) >= 4] - if not gloss_tokens: + if not gloss.strip(): continue - # 4-token contiguous window match — high-confidence "gloss - # actually quoted", not "shared one common word". - for i in range(len(gloss_tokens) - 3): - window = " ".join(gloss_tokens[i : i + 4]) - if window in surface_lower: - return True + if gloss.lower().strip() in surface_lower: + return True return False diff --git a/tests/test_prompt_diversity_runner.py b/tests/test_prompt_diversity_runner.py index ae094af8..3df6bd22 100644 --- a/tests/test_prompt_diversity_runner.py +++ b/tests/test_prompt_diversity_runner.py @@ -82,6 +82,55 @@ class TestAuditLeak: assert _surface_has_audit_leak(surface) is is_leak +# --------------------------------------------------------------------------- # +# Gloss-quote detector +# --------------------------------------------------------------------------- # + + +class TestGlossQuote: + """The detector is exact-substring against the pack's gloss text, + not a fuzzy window. The pack-grounded composer emits gloss text + verbatim, so substring match is the right signal: zero false + positives, zero false negatives on brief-style short glosses where + a 4-token window would be impossible (e.g. ``person`` → ``"person + with a child"`` has only 3 tokens ≥4 chars). + """ + + def _make(self, surface: str, terms: tuple[str, ...]) -> bool: + from evals.prompt_diversity.runner import _surface_quotes_gloss + return _surface_quotes_gloss(surface, terms) + + def test_quoted_short_gloss_detected(self) -> None: + # ``light`` gloss is ``"visible medium that reveal truth"`` — + # 5 tokens, but only 5 are ≥4 chars; the old 4-token window + # would barely fit. ``parent`` gloss is ``"person with a child"`` + # — 4 tokens, 3 are ≥4 chars; the old window could never match. + # Substring match handles both natively. + assert self._make( + "Parent is person with a child. pack-grounded (en_core_relations_v1).", + ("parent",), + ) is True + assert self._make( + "Light is visible medium that reveal truth. pack-grounded (en_core_cognition_v1).", + ("light",), + ) is True + + def test_unquoted_surface_returns_false(self) -> None: + # Chain-walk surface for the same lemma must NOT count as + # gloss-quoted — it shares vocabulary but doesn't quote the + # gloss itself. + assert self._make( + "light — teaching-grounded (cognition_chains_v1): cognition.illumination; logos.core.", + ("light",), + ) is False + + def test_unknown_term_returns_false(self) -> None: + assert self._make("anything", ("nonsense_lemma_42",)) is False + + def test_empty_terms_returns_false(self) -> None: + assert self._make("anything", ()) is False + + # --------------------------------------------------------------------------- # # End-to-end run on the v1 public split # --------------------------------------------------------------------------- #