From ef7d59287b5f262a3ebd060ed0fb67122a565f82 Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 20 May 2026 20:44:19 -0700 Subject: [PATCH] rigor(intent): consistent subject normalization across all classifier paths (#93) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comb pass 2026-05-21 (item 16). Pre-fix ``classify_intent`` applied ``_normalize_subject`` only to DEFINITION / CAUSE / VERIFICATION paths. COMPARISON, FRAME_TRANSFER, TRANSITIVE_QUERY (non-"means" branch), and BELONG_QUERY returned bare ``.strip()`` subjects. A probe like *"Compare the parent and a child"* would carry the articles ("the parent", "a child") into the subject slot, breaking downstream pack-resolver lookups that key on bare lemmas. Fix: apply ``_normalize_subject(..., IntentTag.DEFINITION)`` at every classifier return site that was previously bare ``.strip()``. DEFINITION mode preserves multi-word noun phrases (only strips leading articles + trailing punctuation + infinitive markers); the aux-verb stripping that's only meaningful for CAUSE/VERIFICATION stays scoped to those paths. Sites fixed (5): * COMPARISON subject + secondary_subject * FRAME_TRANSFER subject + frame * TRANSITIVE_QUERY subject (both the regular and "means" → DEFINITION redirect branches now share one normalized binding) * BELONG_QUERY subject Behavior: * Eval cases without articles (the entirety of cognition v1) are byte-identical: ``"memory"`` and ``"recall"`` survive ``_normalize_subject`` unchanged. * Multi-word noun phrases survive intact: ``"artificial intelligence"`` is preserved (no aux-verb-strip wrongly trimming to head-noun). * Article-prefixed subjects ("the parent") now strip consistently with the DEFINITION path that's done so since ADR-0049. Validation: * 7 new tests in ``tests/test_intent_subject_normalization_consistency.py`` pin the consistency contract across COMPARISON, FRAME_TRANSFER, TRANSITIVE_QUERY, BELONG_QUERY, DEFINITION (regression guard on the pre-existing path), and CAUSE (regression guard on the aux-verb-strip behavior). * ``core eval cognition`` byte-identical across all three splits: public 100/100/91.7/100, dev 100/100/78.6/100, holdout 100/100/83.3/100. * ``core test --suite cognition`` 120/0/1, ``smoke`` 67/0. * ``pytest -k intent`` 229/0. --- generate/intent.py | 34 +++++++-- ...ntent_subject_normalization_consistency.py | 75 +++++++++++++++++++ 2 files changed, 101 insertions(+), 8 deletions(-) create mode 100644 tests/test_intent_subject_normalization_consistency.py diff --git a/generate/intent.py b/generate/intent.py index 04aedcd1..af9d7131 100644 --- a/generate/intent.py +++ b/generate/intent.py @@ -306,10 +306,18 @@ def classify_intent(prompt: str) -> DialogueIntent: compare_match = _COMPARE_RE.match(text) if compare_match: + # Comb pass 2026-05-21 — apply ``_normalize_subject`` so leading + # articles ("the parent", "a question") are stripped consistently + # with DEFINITION / CAUSE / VERIFICATION paths. DEFINITION mode + # preserves multi-word noun phrases (only strips articles + + # punctuation + infinitive markers); aux-verb stripping is + # CAUSE/VERIFICATION-only and would be wrong here. return DialogueIntent( tag=IntentTag.COMPARISON, - subject=compare_match.group(1).strip(), - secondary_subject=compare_match.group(2).strip(), + subject=_normalize_subject(compare_match.group(1), IntentTag.DEFINITION), + secondary_subject=_normalize_subject( + compare_match.group(2), IntentTag.DEFINITION + ), ) frame_match = _FRAME_TRANSFER_RE.match(text) @@ -322,18 +330,27 @@ def classify_intent(prompt: str) -> DialogueIntent: relation = "belongs_to" else: relation = _RELATION_NORMALIZE.get(raw_relation, raw_relation) + # Comb pass 2026-05-21 — consistent article-strip; see COMPARISON + # branch above for rationale. return DialogueIntent( tag=IntentTag.FRAME_TRANSFER, - subject=frame_match.group("subject").strip(), + subject=_normalize_subject(frame_match.group("subject"), IntentTag.DEFINITION), relation=relation, - frame=frame_match.group("frame").strip(), + frame=_normalize_subject(frame_match.group("frame"), IntentTag.DEFINITION), ) transitive_match = _TRANSITIVE_QUERY_RE.match(text) if transitive_match: raw_relation = transitive_match.group("relation").lower().strip() relation = _RELATION_NORMALIZE.get(raw_relation, raw_relation) - raw_subject = transitive_match.group("subject").strip() + # Comb pass 2026-05-21 — apply ``_normalize_subject`` consistently + # across both the "means" → DEFINITION redirect and the regular + # TRANSITIVE_QUERY path. Pre-fix the redirect normalized but the + # regular path returned ``raw_subject`` with only a ``.strip()``, + # so "Where does the parent live?" left the article in. + normalized_subject = _normalize_subject( + transitive_match.group("subject"), IntentTag.DEFINITION + ) # "What does X mean?" is a definitional probe, not a transitive # relation query — there is no edge ``X --means--> Y`` to walk; # the user wants the definition of X. Route to DEFINITION so @@ -341,19 +358,20 @@ def classify_intent(prompt: str) -> DialogueIntent: if raw_relation in {"mean", "means"}: return DialogueIntent( tag=IntentTag.DEFINITION, - subject=_normalize_subject(raw_subject, IntentTag.DEFINITION), + subject=normalized_subject, ) return DialogueIntent( tag=IntentTag.TRANSITIVE_QUERY, - subject=raw_subject, + subject=normalized_subject, relation=relation, ) belong_match = _BELONG_QUERY_RE.match(text) if belong_match: + # Comb pass 2026-05-21 — consistent article-strip. return DialogueIntent( tag=IntentTag.TRANSITIVE_QUERY, - subject=belong_match.group("subject").strip(), + subject=_normalize_subject(belong_match.group("subject"), IntentTag.DEFINITION), relation="belongs_to", ) diff --git a/tests/test_intent_subject_normalization_consistency.py b/tests/test_intent_subject_normalization_consistency.py new file mode 100644 index 00000000..86ff8caa --- /dev/null +++ b/tests/test_intent_subject_normalization_consistency.py @@ -0,0 +1,75 @@ +"""Subject normalization consistency across intent paths (comb pass 2026-05-21). + +Pre-fix ``classify_intent`` applied ``_normalize_subject`` only to +DEFINITION / CAUSE / VERIFICATION paths. COMPARISON, FRAME_TRANSFER, +TRANSITIVE_QUERY (non-"means" branch), and BELONG_QUERY returned bare +``.strip()`` subjects. A probe like *"Compare the parent and a child"* +would carry the articles into the subject slot, breaking downstream +pack-resolver lookups that key on bare lemmas. + +These tests pin the post-fix consistency: every intent path now +strips leading articles via ``_normalize_subject(..., IntentTag.DEFINITION)`` +(article-strip + multi-word preservation; aux-verb stripping stays +CAUSE/VERIFICATION-only). +""" + +from __future__ import annotations + +from generate.intent import IntentTag, classify_intent + + +def test_comparison_strips_articles_from_both_subjects() -> None: + intent = classify_intent("Compare the parent and a child") + assert intent.tag is IntentTag.COMPARISON + assert intent.subject == "parent" + assert intent.secondary_subject == "child" + + +def test_comparison_preserves_multi_word_subjects() -> None: + intent = classify_intent("Compare artificial intelligence and natural intelligence") + assert intent.tag is IntentTag.COMPARISON + # Multi-word noun phrases survive intact; only leading articles strip. + assert intent.subject == "artificial intelligence" + assert intent.secondary_subject == "natural intelligence" + + +def test_comparison_no_articles_byte_identical_to_pre_fix() -> None: + """The cognition eval cases (no articles) must remain byte-identical.""" + intent = classify_intent("Compare memory and recall") + assert intent.subject == "memory" + assert intent.secondary_subject == "recall" + + +def test_transitive_query_strips_articles() -> None: + """The non-"means" TRANSITIVE_QUERY branch now normalizes consistently + with the "means" → DEFINITION redirect.""" + # TRANSITIVE_QUERY shape is "what does X R …"; the subject capture + # is what we're testing for article-strip. + intent = classify_intent("What does the parent require") + assert intent.tag is IntentTag.TRANSITIVE_QUERY + assert intent.subject == "parent" + + +def test_belong_query_strips_articles() -> None: + # BELONG_QUERY shape is "where does X belong"; test that the + # captured subject loses its article. + intent = classify_intent("Where does the dog belong") + assert intent.tag is IntentTag.TRANSITIVE_QUERY + assert intent.relation == "belongs_to" + assert intent.subject == "dog" + + +def test_definition_path_unchanged() -> None: + """The DEFINITION path was already normalizing; this pins it.""" + intent = classify_intent("What is the parent?") + assert intent.tag is IntentTag.DEFINITION + assert intent.subject == "parent" + + +def test_cause_path_keeps_aux_verb_strip() -> None: + """CAUSE / VERIFICATION normalization still strips aux verbs and + returns the head noun — the comb-pass fix did not regress this.""" + intent = classify_intent("Why does light exist?") + assert intent.tag is IntentTag.CAUSE + # Aux-verb-strip + head-noun extraction → "light" + assert intent.subject == "light"