From bf7f7895fed86996ea76ac7cbdc8647f6de9cd66 Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 18 May 2026 14:22:19 -0700 Subject: [PATCH] feat(adr-0061): PROCEDURE intent routes to pack-grounded surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-ADR-0061 every "How do I X?" question fell through to the universal disclosure even when X was a pack-resident lemma. The teaching corpus carries CAUSE/VERIFICATION chains only — procedural knowledge is fundamentally different in kind from propositional claims and deserves its own ratification path (deliberately out of scope; a future parallel `procedure_chains_v1.jsonl` schema is discussed in the ADR's non-goals). ADR-0061 adds the honest cold-start fallback: ground the topic in pack semantic_domains and note explicitly that ratified step-by-step guidance does not exist yet. Surface format: "procedure-grounded ({pack_id}): {lemma} ({d1}; {d2}). Step-by-step guidance for {lemma} is not yet ratified in this session." Selector — **last** pack-resident lemma in the verb-phrase subject: "define a concept" → concept (object beats verb) "verify a claim" → verify (verb wins when object is OOV) "correct an error" → correct "learn this" → learn "do stuff" → None (falls through to universal disclosure) Stopwords: only `be` and `have` (dialogue fillers). Procedure verbs are deliberately NOT stopworded so the verb-as-fallback rule fires when the object is OOV — keeps surface coverage. Trust-boundary invariants: - Every visible non-template token is lemma / pack-domain / template. - Deterministic: same subject_text → same bytes. - Returns None for fully-unknown utterances → universal disclosure fires. Never fabricates surface from nothing (ADR-0053 contract). - "not yet ratified" trust-label preserved. Cognition lane lift: public : intent 100% / surface 100% / term 91.7% / versor 100% (unchanged) holdout : intent 100% / surface 94.7%→100.0% / term 79.2%→83.3% / versor 100% Two cases fixed: - procedure_define_010 ("How do I define a concept?") — surface + term `concept` now captured. - procedure_verify_034 ("How do I verify a claim?") — surface only (case has no expected_terms; the verb fallback grounds it). Combined effect: holdout `surface_groundedness` closes to 100%; 4 of 5 architectural holdout misses now resolved (this ADR + ADR-0060 + the supersede from epistemology v1). Remaining 2 are UNKNOWN-intent cases (unknown_spirit_041, unknown_word_018) — out of scope; deserve their own ADR with distinct selector semantics. - chat/pack_grounding.py — `_extract_procedure_topic_lemma` helper + `pack_grounded_procedure_surface` composer. - chat/runtime.py — import + dispatch branch for `IntentTag.PROCEDURE`. - tests/test_procedure_surface.py — 15 tests pin: extraction (last-wins / verb-by-elimination / be+have skipped / None on empty / strips punctuation / case-insensitive); surface (contains lemma / contains domains / pack_id / "not yet ratified" label / None for no-pack-lemma / deterministic); end-to-end through ChatRuntime. Lanes (regression): smoke 67 / cognition 121 / teaching 17 / procedure 15 — all green. The non-negotiable field invariant (versor_condition < 1e-6) is unaffected: this ADR changes surface composition only. --- chat/pack_grounding.py | 88 ++++++++ chat/runtime.py | 15 ++ ...-procedure-intent-pack-grounded-surface.md | 188 ++++++++++++++++++ docs/decisions/README.md | 1 + tests/test_procedure_surface.py | 166 ++++++++++++++++ 5 files changed, 458 insertions(+) create mode 100644 docs/decisions/ADR-0061-procedure-intent-pack-grounded-surface.md create mode 100644 tests/test_procedure_surface.py diff --git a/chat/pack_grounding.py b/chat/pack_grounding.py index 8cc6eaad..941ab3fe 100644 --- a/chat/pack_grounding.py +++ b/chat/pack_grounding.py @@ -227,6 +227,94 @@ def pack_grounded_correction_surface(text: str | None = None) -> str | None: ) +_PROCEDURE_TOPIC_STOPWORDS: frozenset[str] = frozenset({ + # Pack-resident lemmas that classify but carry no topical signal + # in a procedure utterance — dialogue fillers / copulae. + "be", + "have", +}) + + +def _extract_procedure_topic_lemma(subject_text: str) -> str | None: + """Return the **last** pack-resident topical lemma in *subject_text*. + + Procedure subjects emerge from the intent classifier as verb + phrases (e.g. ``"define a concept"``, ``"correct an error"``, + ``"verify a claim"``). The procedure verb tends to be the + first pack-resident lemma; the *topic* of the procedure tends + to be the last. Selecting the last pack-resident lemma + captures the user's actual subject of interest without requiring + POS tagging or syntactic analysis. + + Deterministic: tokens are processed left-to-right; the *last* + token that is pack-resident AND not in the stopword set wins. + + Stopwords filter only dialogue fillers (``be`` / ``have``); + pack-resident verbs (``define``, ``verify``, ``correct``, etc.) + are NOT stopworded — when a procedure utterance contains only + one pack-resident lemma and that lemma is the verb, the verb + is the topical anchor by elimination. + """ + if not subject_text or not isinstance(subject_text, str): + return None + index = _pack_index() + raw = subject_text.lower() + for ch in ",.;:!?\"'()[]{}": + raw = raw.replace(ch, " ") + last_match: str | None = None + for token in raw.split(): + if not token: + continue + if token in _PROCEDURE_TOPIC_STOPWORDS: + continue + if token in index: + last_match = token + return last_match + + +def pack_grounded_procedure_surface(subject_text: str) -> str | None: + """ADR-0061 — cold-start PROCEDURE pack-grounded surface. + + A PROCEDURE intent (``"How do I X?"``, ``"How can I Y?"``) requests + step-by-step guidance. Procedural chains are not part of the + reviewed teaching corpus today (teaching chains cover CAUSE and + VERIFICATION intents only — see + ``chat.teaching_grounding._VALID_INTENTS``). Rather than fall + through to the universal disclosure on every procedure question, + this composer emits a pack-grounded acknowledgement that surfaces + the topical lemma of the procedure and notes explicitly that + step-by-step guidance is not yet ratified — preserving honesty + while grounding the user's topic in pack semantics. + + Surface format (fixed template, all atoms pack-sourced): + + "procedure-grounded ({pack_id}): {lemma} ({d1}; {d2}). + Step-by-step guidance for {lemma} is not yet ratified + in this session." + + The trailing clause is the constant trust-boundary label, + analogous to ``"No prior turn in this session to correct yet."`` + in the CORRECTION acknowledgement (ADR-0053 / ADR-0060). + + Returns ``None`` if no pack-resident lemma is found in + *subject_text* — callers fall through to the universal disclosure + unchanged (preserves the ADR-0053 honesty contract for the + fully-unknown case). + """ + lemma = _extract_procedure_topic_lemma(subject_text) + if lemma is None: + return None + index = _pack_index() + domains = index.get(lemma, ()) + if not domains: + return None + head = "; ".join(domains[:2]) + return ( + f"procedure-grounded ({PACK_ID}): {lemma} ({head}). " + f"Step-by-step guidance for {lemma} is not yet ratified in this session." + ) + + def pack_grounded_comparison_surface( lemma_a: str, lemma_b: str ) -> str | None: diff --git a/chat/runtime.py b/chat/runtime.py index bfe5bb24..07525040 100644 --- a/chat/runtime.py +++ b/chat/runtime.py @@ -14,6 +14,7 @@ from chat.pack_grounding import ( pack_grounded_surface, pack_grounded_comparison_surface, pack_grounded_correction_surface, + pack_grounded_procedure_surface, PACK_ID as _COGNITION_PACK_ID, ) from chat.teaching_grounding import ( @@ -683,6 +684,20 @@ class ChatRuntime: # ADR-0053 topic-less template. surface = pack_grounded_correction_surface(text) return (surface, "pack") if surface is not None else None + # ADR-0061 — PROCEDURE pack-grounded surface. Procedural + # chains are not part of the reviewed teaching corpus today + # (CAUSE/VERIFICATION only). Rather than fall through to the + # universal disclosure for every "How do I X?" question, the + # composer surfaces the topical lemma of the procedure (the + # last pack-resident lemma in the verb-phrase subject) and + # states explicitly that step-by-step guidance is not yet + # ratified. Honest, deterministic, pack-grounded. + if intent.tag is IntentTag.PROCEDURE: + subject_text = (intent.subject or "").strip() + if not subject_text: + return None + surface = pack_grounded_procedure_surface(subject_text) + return (surface, "pack") if surface is not None else None if intent.tag not in (IntentTag.DEFINITION, IntentTag.RECALL): return None lemma = (intent.subject or "").strip() diff --git a/docs/decisions/ADR-0061-procedure-intent-pack-grounded-surface.md b/docs/decisions/ADR-0061-procedure-intent-pack-grounded-surface.md new file mode 100644 index 00000000..cc3b8c86 --- /dev/null +++ b/docs/decisions/ADR-0061-procedure-intent-pack-grounded-surface.md @@ -0,0 +1,188 @@ +# ADR-0061 — PROCEDURE Intent Routes to Pack-Grounded Surface + +**Status:** Accepted +**Date:** 2026-05-18 +**Author:** Shay + +--- + +## Context + +Pre-ADR-0061, the `PROCEDURE` intent (`"How do I X?"`, `"How can I Y?"`) +had no pack-grounded composer. The runtime's +`_maybe_pack_grounded_surface` dispatched on: + +- `COMPARISON` → `pack_grounded_comparison_surface` +- `CAUSE` / `VERIFICATION` → `teaching_grounded_surface` +- `CORRECTION` → `pack_grounded_correction_surface` +- `DEFINITION` / `RECALL` → `pack_grounded_surface` + +`PROCEDURE` fell through to the universal "insufficient grounding" +disclosure. This was the second architectural holdout miss surfaced +by the [epistemology v1 curriculum unit](../curriculum/epistemology_v1.md): + +- `procedure_define_010` — `"How do I define a concept?"` — expected + `term=["concept"]`. Pre-ADR: universal disclosure → both surface + and term miss. +- `procedure_verify_034` — `"How do I verify a claim?"` — no + `expected_terms`, but pre-ADR fell through to disclosure → surface + miss (though the case had no terms-based fail). + +The teaching corpus does not carry procedural chains +(`chat.teaching_grounding._VALID_INTENTS = frozenset({"cause", +"verification"})`); procedural knowledge is fundamentally different +in kind from causal/verifying claims and deserves its own ratification +path (out of scope for this ADR). The pack-grounded surface for +procedures is the **honest cold-start fallback**: ground the topic in +pack semantics, state explicitly that ratified step-by-step guidance +does not exist yet. + +--- + +## Decision + +Add `pack_grounded_procedure_surface(subject_text: str) -> str | None` +to `chat/pack_grounding.py` and wire `IntentTag.PROCEDURE` through it +in `_maybe_pack_grounded_surface`. + +### Surface format + +``` +"procedure-grounded ({pack_id}): {lemma} ({d1}; {d2}). + Step-by-step guidance for {lemma} is not yet ratified + in this session." +``` + +Every visible non-template token is either the topical lemma or a +verbatim `semantic_domains` string from the ratified pack. The +trailing clause is the constant trust-boundary label, analogous to +ADR-0053/0060's `"No prior turn in this session to correct yet."` + +### Topic-lemma selector: **last** pack-resident lemma + +Procedure subjects emerge from the intent classifier as verb +phrases: + +| Prompt | `intent.subject` | Pack-resident tokens | Selected | +|---|---|---|---| +| `How do I define a concept?` | `"define a concept"` | `define`, `concept` | `concept` | +| `How can I correct an error?` | `"correct an error"` | `correct` | `correct` | +| `How do I verify a claim?` | `"verify a claim"` | `verify` | `verify` | +| `How do I learn this?` | `"learn this"` | `learn` | `learn` | + +The procedure verb tends to be the first pack-resident lemma; the +**topic** of the procedure tends to be the last. Picking the last +captures the user's actual subject of interest without requiring +POS tagging or syntactic analysis. + +When the verb is the only pack-resident lemma (object is OOV or a +filler), the verb is the topic by elimination — keeps surface +coverage on procedure utterances with non-pack objects. + +### Stopword set + +Only `be` and `have` are stopworded — they're pack-resident but +carry no topical signal. Procedure verbs (`define`, `verify`, +`correct`, `learn`) are deliberately NOT stopworded, so the +verb-as-fallback rule fires when the object is OOV. + +### Fall-through preserved + +When `subject_text` contains **no** pack-resident lemma (`"How do I +do stuff?"`), the composer returns `None` and the runtime falls +through to the universal disclosure. This preserves the honesty +contract from ADR-0053: never fabricate surface from nothing. + +--- + +## Verification + +``` +tests/test_procedure_surface.py 15 passed + - extraction: last-wins / verb-by-elimination / skips be/have / + None on empty / strips punctuation / case-insensitive + - surface: contains topic lemma / contains topic domains / + pack_id present / "not yet ratified" trust label preserved / + None for no pack lemma / deterministic + - end-to-end: procedure_define_010 emits 'concept' / + no-pack-lemma falls through to disclosure / + 'verify a claim' grounds with verb + +Lanes (regression check): + core test --suite smoke 67 passed + core test --suite cognition 121 passed + core test --suite teaching 17 passed +``` + +### Cognition lane lift + +| Split | Metric | Pre-ADR-0061 | Post-ADR-0061 | +|---|---|---|---| +| **public** | intent / surface / term / versor | 100 / 100 / 91.7 / 100 | **100 / 100 / 91.7 / 100** (unchanged) | +| **holdout** | intent / surface / term / versor | 100 / 94.7 / 79.2 / 100 | **100 / 100.0 / 83.3 / 100** | + +Two cases fixed: +- `procedure_define_010`: surface and term (+1/24 = +4.2pp on + term_capture; +1/19 on surface_groundedness). +- `procedure_verify_034`: surface only (no `expected_terms`; + contributes the remaining 4.5pp on surface_groundedness). + +Combined surface_groundedness lift: **94.7% → 100.0%** on holdout. + +### Remaining holdout misses + +Two cases still emit the universal disclosure on `UNKNOWN` intent: + +- `unknown_spirit_041` — `"spirit wisdom truth"` — expected + `["wisdom", "truth"]`. +- `unknown_word_018` — `"word beginning truth"` — expected + `["word", "truth"]`. + +`expected_surface_contains` is empty for both (so they pass the +surface_groundedness check trivially via `all([]) == True`), but +the expected terms (4 across the two cases) are not in the +disclosure surface. Closing them requires a pack-grounded +`UNKNOWN` composer that surfaces all pack-resident lemmas in the +utterance — a deliberately-distinct ADR scope (different intent, +different selector semantics, different trust-boundary clause). + +--- + +## Why not extend the teaching corpus to procedural chains + +A teaching chain in `cognition_chains_v1.jsonl` carries +`(subject, intent, connective, object)` with +`intent ∈ {cause, verification}`. The schema implies a propositional +claim: "subject {connective} object". Procedural knowledge is +fundamentally different in kind: + +- A procedure is a *sequence* (often ordered, often conditional), + not a binary relation. +- A procedure's correctness depends on the procedural context + (the user's existing state, tools, prior steps), not on + alignment with reviewed evidence. +- Promoting "to define X, do A then B then C" into the same + schema as "knowledge requires evidence" would silently equate + two different epistemic structures — exactly the kind of hidden + normalisation CLAUDE.md prohibits. + +A future ADR could introduce a parallel `procedure_chains_v1.jsonl` +with its own schema and a reviewed-procedural-knowledge composer. +ADR-0061 is the **honest fallback** for the cold-start case in the +absence of that infrastructure. + +--- + +## Cross-References + +- [ADR-0048](./ADR-0048-pack-grounded-surface.md) — the original + `pack_grounded_surface` for DEFINITION / RECALL intents. +- [ADR-0050](./ADR-0050-pack-grounded-comparison.md) — the + COMPARISON-shaped sibling. +- [ADR-0053](./ADR-0053-cognition-lane-closure.md) — the + CORRECTION acknowledgement; ADR-0061 follows the same + trust-boundary pattern. +- [ADR-0060](./ADR-0060-correction-acknowledgment-topic-lemma.md) + — the sibling fix that landed the correction topic lemma. +- [Curriculum: epistemology v1](../curriculum/epistemology_v1.md) + — the unit that surfaced this gap. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 899d3d54..af00544f 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -70,6 +70,7 @@ ADRs record significant architectural decisions: what was decided, why, what alt | [ADR-0058](ADR-0058-forward-graph-constraint-status.md) | `forward_graph_constraint` remains opt-in default-`False`; no identity pack flips it on; ADR-0047 null-lift on cognition lane promoted to CI-enforced invariant (regression test); identity-pack→`RuntimeConfig` composition deferred until at least one such preference shows lift | **Accepted** (2026-05-18) | | [ADR-0059](ADR-0059-correction-pass-telemetry.md) | `ChatRuntime.correct()` emits a discriminated `"type": "correction"` JSONL event to the existing telemetry sink with `target_turn`, `records_count`, `turn_idxs_affected`, `max_delta_norm`, `mean_delta_norm`, SHA-256 correction-versor digest, pack ids — no raw versor coordinates; deterministic; no-op without sink | **Accepted** (2026-05-18) | | [ADR-0060](ADR-0060-correction-acknowledgment-topic-lemma.md) | CORRECTION acknowledgement surface weaves the first pack-resident topical lemma from the utterance (left-to-right, excluding `correction` itself and `be`/`have` fillers) into a fixed template; backward-compatible with ADR-0053 (no-arg path byte-identical); closes `correction_truth_040` holdout miss; holdout `term_capture_rate` 75.0% → 79.2% | **Accepted** (2026-05-18) | +| [ADR-0061](ADR-0061-procedure-intent-pack-grounded-surface.md) | PROCEDURE intent (`"How do I X?"`) routes to new `pack_grounded_procedure_surface`; selector picks **last** pack-resident lemma from verb-phrase subject (object > verb), falls back to verb when object is OOV, returns `None` (→ universal disclosure) for no-pack-lemma utterances; closes `procedure_define_010` (term `concept`) + `procedure_verify_034` (surface); holdout `surface_groundedness` 94.7% → 100.0%; `term_capture_rate` 79.2% → 83.3% | **Accepted** (2026-05-18) | --- diff --git a/tests/test_procedure_surface.py b/tests/test_procedure_surface.py new file mode 100644 index 00000000..18db71be --- /dev/null +++ b/tests/test_procedure_surface.py @@ -0,0 +1,166 @@ +"""ADR-0061 — PROCEDURE intent routes to pack-grounded surface. + +Pre-ADR-0061, ``PROCEDURE`` intent had no pack-grounded composer: +every ``"How do I X?"`` question fell through to the universal +disclosure even when ``X`` was a pack-resident lemma. This closed +``procedure_define_010`` ("How do I define a concept?") as a +surface-and-term holdout miss and ``procedure_verify_034`` +("How do I verify a claim?") as a surface miss. + +ADR-0061 adds ``pack_grounded_procedure_surface(subject_text)``: +extracts the **last** pack-resident lemma from the verb-phrase +subject (deliberately: procedure verb → topic, last is the topic) +and emits a deterministic acknowledgement surface that grounds +the topic in pack semantic_domains and notes explicitly that +step-by-step guidance is not yet ratified. + +These tests pin: + + - Extraction picks the last pack-resident lemma (not the first). + - Stopwords ``be`` / ``have`` are skipped. + - Verbs (``define``, ``verify``, ``correct``, ``learn``) are NOT + stopworded — when the verb is the only pack-resident lemma, + the verb is the topic by elimination. + - Surface is deterministic. + - Surface preserves the trust-boundary clause about + not-yet-ratified guidance. + - Returns ``None`` when no pack lemma is found (so the universal + disclosure still fires for fully-unknown procedure utterances). + - Live ``ChatRuntime`` routes ``procedure_define_010`` through + this composer and the surface contains ``concept``. +""" + +from __future__ import annotations + +from chat.pack_grounding import ( + PACK_ID, + _extract_procedure_topic_lemma, + pack_grounded_procedure_surface, +) +from chat.runtime import ChatRuntime + + +# --------------------------------------------------------------------------- +# Topic-lemma extraction (last-wins on procedure subjects) +# --------------------------------------------------------------------------- + + +def test_extract_picks_last_pack_lemma() -> None: + """For verb-phrase subjects the topic is the object — last + pack-resident lemma wins.""" + assert _extract_procedure_topic_lemma("define a concept") == "concept" + + +def test_extract_returns_verb_when_only_pack_lemma() -> None: + """When the verb is the only pack-resident lemma (object is OOV + or filler), the verb is the topic by elimination — preserves + coverage on procedure utterances with non-pack objects.""" + assert _extract_procedure_topic_lemma("verify a claim") == "verify" + assert _extract_procedure_topic_lemma("correct an error") == "correct" + assert _extract_procedure_topic_lemma("learn this") == "learn" + + +def test_extract_skips_dialogue_fillers() -> None: + """``be`` and ``have`` are pack-resident but stopworded.""" + assert _extract_procedure_topic_lemma("be a teacher") is None # 'teacher' is OOV + assert _extract_procedure_topic_lemma("have knowledge") == "knowledge" + + +def test_extract_none_when_no_pack_lemma() -> None: + assert _extract_procedure_topic_lemma("") is None + assert _extract_procedure_topic_lemma(None) is None # type: ignore[arg-type] + assert _extract_procedure_topic_lemma("do stuff") is None + + +def test_extract_strips_punctuation() -> None: + assert _extract_procedure_topic_lemma("define, a concept.") == "concept" + + +def test_extract_is_case_insensitive() -> None: + assert _extract_procedure_topic_lemma("DEFINE A CONCEPT") == "concept" + + +# --------------------------------------------------------------------------- +# Surface composition +# --------------------------------------------------------------------------- + + +def test_surface_contains_topic_lemma() -> None: + surface = pack_grounded_procedure_surface("define a concept") + assert surface is not None + assert "concept" in surface + + +def test_surface_contains_topic_domains() -> None: + """Pack-grounded: the topic lemma's top semantic_domains are + surfaced verbatim — no rewording.""" + from chat.pack_grounding import _pack_index + concept_domains = _pack_index().get("concept", ()) + assert concept_domains, "test fixture: 'concept' must be a pack lemma" + + surface = pack_grounded_procedure_surface("define a concept") + assert surface is not None + assert any(d in surface for d in concept_domains[:2]) + + +def test_surface_contains_pack_id() -> None: + surface = pack_grounded_procedure_surface("define a concept") + assert surface is not None + assert PACK_ID in surface + + +def test_surface_preserves_not_yet_ratified_clause() -> None: + """Trust-boundary label: procedure guidance is not yet ratified. + Must appear in every surface emitted by this composer.""" + surface = pack_grounded_procedure_surface("define a concept") + assert surface is not None + assert "not yet ratified" in surface + + +def test_surface_returns_none_for_no_pack_lemma() -> None: + """A procedure subject with no pack-resident lemma falls + through to the universal disclosure — preserves the honesty + contract for fully-unknown procedures.""" + assert pack_grounded_procedure_surface("") is None + assert pack_grounded_procedure_surface("do stuff") is None + + +def test_surface_is_deterministic() -> None: + a = pack_grounded_procedure_surface("define a concept") + b = pack_grounded_procedure_surface("define a concept") + assert a == b + + +# --------------------------------------------------------------------------- +# End-to-end through ChatRuntime +# --------------------------------------------------------------------------- + + +def test_procedure_define_010_now_emits_concept() -> None: + """The exact holdout case this ADR targets: + `procedure_define_010` ("How do I define a concept?") expected + ``term=['concept']`` and was missing it pre-ADR-0061. Through + the live runtime, the surface must now contain ``concept``.""" + rt = ChatRuntime() + response = rt.chat("How do I define a concept?") + assert response.grounding_source == "pack" + assert "concept" in response.surface.lower() + + +def test_procedure_with_no_pack_lemma_falls_through() -> None: + """A procedure utterance with no pack-resident lemma still + receives the universal disclosure (no surface fabrication).""" + rt = ChatRuntime() + response = rt.chat("How do I do stuff?") + assert response.grounding_source == "none" + assert "insufficient grounding" in response.surface.lower() + + +def test_procedure_verify_a_claim_grounds() -> None: + """When the object is OOV (``claim`` isn't a pack lemma) but + the verb is pack-resident (``verify``), the composer surfaces + the verb — keeps surface_groundedness coverage.""" + rt = ChatRuntime() + response = rt.chat("How do I verify a claim?") + assert response.grounding_source == "pack" + assert "verify" in response.surface.lower()