rigor(intent): consistent subject normalization across all classifier paths (#93)

Comb pass 2026-05-21 (item 16). Pre-fix ``classify_intent`` applied ``_normalize_subject`` only to DEFINITION / CAUSE / VERIFICATION paths. COMPARISON, FRAME_TRANSFER, TRANSITIVE_QUERY (non-"means" branch), and BELONG_QUERY returned bare ``.strip()`` subjects. A probe like *"Compare the parent and a child"* would carry the articles ("the parent", "a child") into the subject slot, breaking downstream pack-resolver lookups that key on bare lemmas. Fix: apply ``_normalize_subject(..., IntentTag.DEFINITION)`` at every classifier return site that was previously bare ``.strip()``. DEFINITION mode preserves multi-word noun phrases (only strips leading articles + trailing punctuation + infinitive markers); the aux-verb stripping that's only meaningful for CAUSE/VERIFICATION stays scoped to those paths. Sites fixed (5): * COMPARISON subject + secondary_subject * FRAME_TRANSFER subject + frame * TRANSITIVE_QUERY subject (both the regular and "means" → DEFINITION redirect branches now share one normalized binding) * BELONG_QUERY subject Behavior: * Eval cases without articles (the entirety of cognition v1) are byte-identical: ``"memory"`` and ``"recall"`` survive ``_normalize_subject`` unchanged. * Multi-word noun phrases survive intact: ``"artificial intelligence"`` is preserved (no aux-verb-strip wrongly trimming to head-noun). * Article-prefixed subjects ("the parent") now strip consistently with the DEFINITION path that's done so since ADR-0049. Validation: * 7 new tests in ``tests/test_intent_subject_normalization_consistency.py`` pin the consistency contract across COMPARISON, FRAME_TRANSFER, TRANSITIVE_QUERY, BELONG_QUERY, DEFINITION (regression guard on the pre-existing path), and CAUSE (regression guard on the aux-verb-strip behavior). * ``core eval cognition`` byte-identical across all three splits: public 100/100/91.7/100, dev 100/100/78.6/100, holdout 100/100/83.3/100. * ``core test --suite cognition`` 120/0/1, ``smoke`` 67/0. * ``pytest -k intent`` 229/0.
2026-05-20 20:44:19 -07:00 · 2026-05-20 20:44:19 -07:00 · ef7d59287b
commit ef7d59287b
parent 548282fadc
2 changed files with 101 additions and 8 deletions
--- a/generate/intent.py
+++ b/generate/intent.py
@ -306,10 +306,18 @@ def classify_intent(prompt: str) -> DialogueIntent:

    compare_match = _COMPARE_RE.match(text)
    if compare_match:
+        # Comb pass 2026-05-21 — apply ``_normalize_subject`` so leading
+        # articles ("the parent", "a question") are stripped consistently
+        # with DEFINITION / CAUSE / VERIFICATION paths.  DEFINITION mode
+        # preserves multi-word noun phrases (only strips articles +
+        # punctuation + infinitive markers); aux-verb stripping is
+        # CAUSE/VERIFICATION-only and would be wrong here.
        return DialogueIntent(
            tag=IntentTag.COMPARISON,
-            subject=compare_match.group(1).strip(),
-            secondary_subject=compare_match.group(2).strip(),
+            subject=_normalize_subject(compare_match.group(1), IntentTag.DEFINITION),
+            secondary_subject=_normalize_subject(
+                compare_match.group(2), IntentTag.DEFINITION
+            ),
        )

    frame_match = _FRAME_TRANSFER_RE.match(text)
@ -322,18 +330,27 @@ def classify_intent(prompt: str) -> DialogueIntent:
            relation = "belongs_to"
        else:
            relation = _RELATION_NORMALIZE.get(raw_relation, raw_relation)
+        # Comb pass 2026-05-21 — consistent article-strip; see COMPARISON
+        # branch above for rationale.
        return DialogueIntent(
            tag=IntentTag.FRAME_TRANSFER,
-            subject=frame_match.group("subject").strip(),
+            subject=_normalize_subject(frame_match.group("subject"), IntentTag.DEFINITION),
            relation=relation,
-            frame=frame_match.group("frame").strip(),
+            frame=_normalize_subject(frame_match.group("frame"), IntentTag.DEFINITION),
        )

    transitive_match = _TRANSITIVE_QUERY_RE.match(text)
    if transitive_match:
        raw_relation = transitive_match.group("relation").lower().strip()
        relation = _RELATION_NORMALIZE.get(raw_relation, raw_relation)
-        raw_subject = transitive_match.group("subject").strip()
+        # Comb pass 2026-05-21 — apply ``_normalize_subject`` consistently
+        # across both the "means" → DEFINITION redirect and the regular
+        # TRANSITIVE_QUERY path.  Pre-fix the redirect normalized but the
+        # regular path returned ``raw_subject`` with only a ``.strip()``,
+        # so "Where does the parent live?" left the article in.
+        normalized_subject = _normalize_subject(
+            transitive_match.group("subject"), IntentTag.DEFINITION
+        )
        # "What does X mean?" is a definitional probe, not a transitive
        # relation query — there is no edge ``X --means--> Y`` to walk;
        # the user wants the definition of X.  Route to DEFINITION so
@ -341,19 +358,20 @@ def classify_intent(prompt: str) -> DialogueIntent:
        if raw_relation in {"mean", "means"}:
            return DialogueIntent(
                tag=IntentTag.DEFINITION,
-                subject=_normalize_subject(raw_subject, IntentTag.DEFINITION),
+                subject=normalized_subject,
            )
        return DialogueIntent(
            tag=IntentTag.TRANSITIVE_QUERY,
-            subject=raw_subject,
+            subject=normalized_subject,
            relation=relation,
        )

    belong_match = _BELONG_QUERY_RE.match(text)
    if belong_match:
+        # Comb pass 2026-05-21 — consistent article-strip.
        return DialogueIntent(
            tag=IntentTag.TRANSITIVE_QUERY,
-            subject=belong_match.group("subject").strip(),
+            subject=_normalize_subject(belong_match.group("subject"), IntentTag.DEFINITION),
            relation="belongs_to",
        )

--- a/tests/test_intent_subject_normalization_consistency.py
+++ b/tests/test_intent_subject_normalization_consistency.py
@ -0,0 +1,75 @@
+"""Subject normalization consistency across intent paths (comb pass 2026-05-21).
+
+Pre-fix ``classify_intent`` applied ``_normalize_subject`` only to
+DEFINITION / CAUSE / VERIFICATION paths.  COMPARISON, FRAME_TRANSFER,
+TRANSITIVE_QUERY (non-"means" branch), and BELONG_QUERY returned bare
+``.strip()`` subjects.  A probe like *"Compare the parent and a child"*
+would carry the articles into the subject slot, breaking downstream
+pack-resolver lookups that key on bare lemmas.
+
+These tests pin the post-fix consistency: every intent path now
+strips leading articles via ``_normalize_subject(..., IntentTag.DEFINITION)``
+(article-strip + multi-word preservation; aux-verb stripping stays
+CAUSE/VERIFICATION-only).
+"""
+
+from __future__ import annotations
+
+from generate.intent import IntentTag, classify_intent
+
+
+def test_comparison_strips_articles_from_both_subjects() -> None:
+    intent = classify_intent("Compare the parent and a child")
+    assert intent.tag is IntentTag.COMPARISON
+    assert intent.subject == "parent"
+    assert intent.secondary_subject == "child"
+
+
+def test_comparison_preserves_multi_word_subjects() -> None:
+    intent = classify_intent("Compare artificial intelligence and natural intelligence")
+    assert intent.tag is IntentTag.COMPARISON
+    # Multi-word noun phrases survive intact; only leading articles strip.
+    assert intent.subject == "artificial intelligence"
+    assert intent.secondary_subject == "natural intelligence"
+
+
+def test_comparison_no_articles_byte_identical_to_pre_fix() -> None:
+    """The cognition eval cases (no articles) must remain byte-identical."""
+    intent = classify_intent("Compare memory and recall")
+    assert intent.subject == "memory"
+    assert intent.secondary_subject == "recall"
+
+
+def test_transitive_query_strips_articles() -> None:
+    """The non-"means" TRANSITIVE_QUERY branch now normalizes consistently
+    with the "means" → DEFINITION redirect."""
+    # TRANSITIVE_QUERY shape is "what does X R …"; the subject capture
+    # is what we're testing for article-strip.
+    intent = classify_intent("What does the parent require")
+    assert intent.tag is IntentTag.TRANSITIVE_QUERY
+    assert intent.subject == "parent"
+
+
+def test_belong_query_strips_articles() -> None:
+    # BELONG_QUERY shape is "where does X belong"; test that the
+    # captured subject loses its article.
+    intent = classify_intent("Where does the dog belong")
+    assert intent.tag is IntentTag.TRANSITIVE_QUERY
+    assert intent.relation == "belongs_to"
+    assert intent.subject == "dog"
+
+
+def test_definition_path_unchanged() -> None:
+    """The DEFINITION path was already normalizing; this pins it."""
+    intent = classify_intent("What is the parent?")
+    assert intent.tag is IntentTag.DEFINITION
+    assert intent.subject == "parent"
+
+
+def test_cause_path_keeps_aux_verb_strip() -> None:
+    """CAUSE / VERIFICATION normalization still strips aux verbs and
+    returns the head noun — the comb-pass fix did not regress this."""
+    intent = classify_intent("Why does light exist?")
+    assert intent.tag is IntentTag.CAUSE
+    # Aux-verb-strip + head-noun extraction → "light"
+    assert intent.subject == "light"