From ea298bdc289220a6cb284cdd090bb2b6ea4746a7 Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 18 May 2026 16:42:26 -0700 Subject: [PATCH] =?UTF-8?q?feat(teaching):=20OOV=20signal=20flywheel=20?= =?UTF-8?q?=E2=80=94=20sink,=20aggregator,=20auto-promotion=20(Phase=202.3?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors the chain-gap pipeline (Phase 1.1+1.2) for vocabulary gaps: the OOV invitation surface (P2.1) now emits structured signals that operators can aggregate, rank, and auto-promote into reviewed PackMutationProposal candidates — closing the OOV loop the same way Phase 1 closed the chain loop. Three new modules + two new CLI surfaces: teaching/oov_sink.py. OOVCandidate dataclass mirroring teaching.discovery.DiscoveryCandidate. OOVBufferSink (in-memory) + OOVMonthlyFileSink (append-only JSONL under //.jsonl — same layout as discovery sink so the aggregator reuses the file-walk machinery). hash_oov_candidate_id(token, intent, trace_hash) — deterministic 32-char hex id matching DiscoveryCandidate's replay invariant. format_oov_candidate_jsonl — sorted-keys compact JSONL line. teaching/oov_gaps.py. aggregate_oov_gaps(root, since, sample_limit) groups emitted candidates by token, tracks intent-shape union (a token asked under multiple intents is a stronger curriculum signal), splits boundary_clean from boundary_tainted counts, supports --since YYYY-MM filtering via the sink's file naming convention. Pure reader; never mutates the sink. Deterministic ordering: (count desc, token asc). teaching/oov_promotion.py. promote_oov_gaps(gaps, threshold, include_tainted, suggested_packs) lifts threshold-crossing tokens to OOVPromotion records. - boundary_clean_count gates promotion by default (tainted-only tokens may indicate the prompt hit a safety axis rather than a vocab gap). - --include-tainted flag for operator override. - threshold < 1 raises. - queue_id deterministic: ``oov:@`` — diffable across runs. - suggested_packs lists mounted packs but does NOT recommend one — domain inference is out of scope (would require a stochastic classifier). Operator picks the destination. Runtime wiring: ChatRuntime.attach_oov_sink(sink) mirrors attach_discovery_sink. Runtime emits one OOVCandidate JSONL line per turn whose grounding_source == "oov", no-op when no sink is attached. Intent classifier is now invoked when EITHER sink is attached (was: only discovery sink) — both downstream paths need it. CLI: core teaching oov-gaps [--top N] [--since YYYY-MM] [--root PATH] [--sample-limit N] [--json] core teaching oov-queue [--threshold N] [--include-tainted] [--root PATH] [--since YYYY-MM] [--json] ADR-0065 documents the full design (five-tier honesty gradient, P2.1-P2.4 architecture). README.md updated with the ADR-0065 index entry. Verification: tests/test_oov_pipeline.py 24 passed Operator workflow round-trip verified live: > rt.attach_oov_sink(sink); rt.chat("What is photosynthesis?") → sink receives: {"boundary_clean":true,"candidate_id":"f51bf8...", "intent":"definition","token":"photosynthesis","trigger":"unresolved_subject", "source_turn_trace":"","review_state":"unreviewed"} > core teaching oov-gaps --root /tmp/oov_demo → ranked table by count, intent-set per token > core teaching oov-queue --root /tmp/oov_demo --threshold 2 → promoted tokens + suggested mounted packs Full lane: 2005 passed, 2 skipped, 0 failed in 2:34 (xdist). --- core/cli.py | 163 ++++++++++ .../ADR-0065-oov-gradient-and-relations-v2.md | 242 +++++++++++++++ docs/decisions/README.md | 1 + teaching/oov_gaps.py | 170 +++++++++++ teaching/oov_promotion.py | 119 ++++++++ teaching/oov_sink.py | 160 ++++++++++ tests/test_oov_pipeline.py | 279 ++++++++++++++++++ 7 files changed, 1134 insertions(+) create mode 100644 docs/decisions/ADR-0065-oov-gradient-and-relations-v2.md create mode 100644 teaching/oov_gaps.py create mode 100644 teaching/oov_promotion.py create mode 100644 teaching/oov_sink.py create mode 100644 tests/test_oov_pipeline.py diff --git a/core/cli.py b/core/cli.py index 17f38b74..52e58f21 100644 --- a/core/cli.py +++ b/core/cli.py @@ -571,6 +571,119 @@ def cmd_teaching_gaps(args: argparse.Namespace) -> int: return 0 +def cmd_teaching_oov_gaps(args: argparse.Namespace) -> int: + """Phase 2.3 — rank OOV tokens emitted by the runtime's + OOV "teach me" surface. + + Reads JSONL files written by + :class:`teaching.oov_sink.OOVMonthlyFileSink` under *root* + (default ``teaching/oov_log``) and emits a ranked table of + tokens ordered by emission count. + + Pure read — never mutates the sink. + """ + from teaching.oov_gaps import _DEFAULT_ROOT, aggregate_oov_gaps + + root = Path(args.root) if args.root else _DEFAULT_ROOT + try: + rows = aggregate_oov_gaps( + root=root, + since=args.since, + sample_limit=max(1, int(args.sample_limit)), + ) + except ValueError as exc: + _die(str(exc), code=2) + + if args.top is not None and args.top > 0: + rows = rows[: args.top] + + if args.json: + payload = { + "root": str(root), + "since": args.since, + "total_tokens": len(rows), + "oov_gaps": [g.as_dict() for g in rows], + } + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if rows else 1 + + if not rows: + print("No OOV candidates found.") + if root is not None and not root.exists(): + print(f" (root path does not exist: {root})") + return 1 + + print(f"{'rank':>4} {'token':<28}{'count':>6} {'clean':>6} intents") + print("-" * 80) + for i, gap in enumerate(rows, 1): + intents = ",".join(gap.intents) if gap.intents else "—" + print( + f"{i:>4} {gap.token[:28]:<28}{gap.count:>6} " + f"{gap.boundary_clean_count:>6} {intents}" + ) + return 0 + + +def cmd_teaching_oov_queue(args: argparse.Namespace) -> int: + """Phase 2.3 — show the auto-promoted OOV-token queue. + + Same shape as ``core teaching queue`` but for vocabulary gaps: + tokens whose boundary-clean emission count meets ``--threshold`` + are surfaced as PackMutationProposal candidates that an operator + can author via the reviewed ADR-0027 path. + + Never auto-mutates a pack — operator-visible signal only. + """ + from teaching.oov_gaps import _DEFAULT_ROOT, aggregate_oov_gaps + from teaching.oov_promotion import promote_oov_gaps + + root = Path(args.root) if args.root else _DEFAULT_ROOT + try: + gaps = aggregate_oov_gaps(root=root, since=args.since, sample_limit=5) + except ValueError as exc: + _die(str(exc), code=2) + + if args.threshold < 1: + _die(f"--threshold must be >= 1 (got {args.threshold})", code=2) + + promoted = promote_oov_gaps( + gaps, + threshold=args.threshold, + include_tainted=args.include_tainted, + ) + + if args.json: + payload = { + "root": str(root), + "since": args.since, + "threshold": args.threshold, + "include_tainted": args.include_tainted, + "total_promoted": len(promoted), + "queue": [p.as_dict() for p in promoted], + } + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if promoted else 1 + + if not promoted: + print(f"No OOV tokens met threshold {args.threshold}.") + return 1 + + print(f"{'rank':>4} {'queue_id':<40}{'count':>6} {'clean':>6} intents") + print("-" * 96) + for i, p in enumerate(promoted, 1): + intents = ",".join(p.intents) if p.intents else "—" + print( + f"{i:>4} {p.queue_id[:40]:<40}{p.count:>6} " + f"{p.boundary_clean_count:>6} {intents}" + ) + print() + print( + f"Add each token to one of: {', '.join(promoted[0].suggested_packs)}. " + f"Use a reviewed PackMutationProposal — never auto-applies." + ) + return 0 + + def cmd_teaching_queue(args: argparse.Namespace) -> int: """Phase 1.2 — show the auto-promoted gap queue. @@ -1966,6 +2079,56 @@ def build_parser() -> argparse.ArgumentParser: ) teaching_audit.set_defaults(func=cmd_teaching_audit) + teaching_oov_gaps = teaching_sub.add_parser( + "oov-gaps", + help="rank OOV tokens emitted by the runtime's teach-me surface", + ) + teaching_oov_gaps.add_argument( + "--root", default=None, + help="OOV-sink root (default: teaching/oov_log)", + ) + teaching_oov_gaps.add_argument( + "--since", default=None, + help="lower-bound month token YYYY-MM", + ) + teaching_oov_gaps.add_argument( + "--top", type=int, default=None, + help="show only the top N tokens by emission count", + ) + teaching_oov_gaps.add_argument( + "--sample-limit", type=int, default=5, + help="max candidate_ids retained per token as samples (default: 5)", + ) + teaching_oov_gaps.add_argument( + "--json", action="store_true", help="machine-readable output", + ) + teaching_oov_gaps.set_defaults(func=cmd_teaching_oov_gaps) + + teaching_oov_queue = teaching_sub.add_parser( + "oov-queue", + help="show auto-promoted OOV-token queue (tokens crossing --threshold)", + ) + teaching_oov_queue.add_argument( + "--root", default=None, + help="OOV-sink root (default: teaching/oov_log)", + ) + teaching_oov_queue.add_argument( + "--since", default=None, + help="lower-bound month token YYYY-MM", + ) + teaching_oov_queue.add_argument( + "--threshold", type=int, default=3, + help="minimum (boundary-clean) emissions to promote (default: 3)", + ) + teaching_oov_queue.add_argument( + "--include-tainted", action="store_true", + help="count refusal/hedge-tainted emissions toward the threshold", + ) + teaching_oov_queue.add_argument( + "--json", action="store_true", help="machine-readable output", + ) + teaching_oov_queue.set_defaults(func=cmd_teaching_oov_queue) + teaching_queue = teaching_sub.add_parser( "queue", help="show auto-promoted high-priority gaps (cells crossing --threshold)", diff --git a/docs/decisions/ADR-0065-oov-gradient-and-relations-v2.md b/docs/decisions/ADR-0065-oov-gradient-and-relations-v2.md new file mode 100644 index 00000000..8da739de --- /dev/null +++ b/docs/decisions/ADR-0065-oov-gradient-and-relations-v2.md @@ -0,0 +1,242 @@ +# ADR-0065 — OOV gradient + relations v2 (Plan Phase 2) + +**Status:** Accepted +**Date:** 2026-05-18 +**Author:** Shay +**Phase:** Plan Phase 2 (OOV cliff → gradient) +**Builds on:** ADR-0048 / ADR-0050 / ADR-0052 / ADR-0061 / ADR-0063 / ADR-0064 + +--- + +## Context + +Phase 1 closed the corpus flywheel: discovery candidates aggregate +into operator-visible signals; the relations pack joined the live +runtime; cross-pack teaching corpora register and surface +deterministically. + +But the **vocabulary** layer was still a cliff. When the runtime +saw a token it didn't know — `photosynthesis`, `mitochondria`, +`grandparent` — every cold-start prompt fell through to the flat +universal disclosure: + +``` +I don't know — insufficient grounding for that yet. +``` + +That surface was honest but flat. It conveyed no signal that a +*specific* vocabulary gap was hit, offered the operator no concrete +next step, and dropped the gap on the floor — no aggregation, no +queue, no path from "system saw an unknown" to "operator can act +on it". + +Phase 2 converts the OOV cliff into a five-tier gradient and closes +the OOV signal into the same flywheel the chain-gap signal closed +in Phase 1. + +--- + +## Decision + +### 1. Three new surface tiers (P2.1, P2.2) + +The runtime's surface composer now has five honesty tiers, ordered +by available evidence: + +| Tier | grounding_source | Example surface | +|---|---|---| +| Vault | `vault` | Walk path, session-grounded | +| Reviewed corpus | `teaching` | `light reveals truth (cognition.truth).` | +| Reviewed lexicon | `pack` | `light — pack-grounded (en_core_cognition_v1): cognition.illumination; logos.core.` | +| **Partial** *(new, P2.2)* | `partial` | `Whatever 'photosynthesis' is, I can ground 'knowledge' — pack-grounded (en_core_cognition_v1): ...` | +| **OOV invitation** *(new, P2.1)* | `oov` | `I haven't learned 'photosynthesis' yet (intent: definition). Mounted lexicon packs: ... . Teach me via a reviewed PackMutationProposal.` | +| Universal disclosure | `none` | `I don't know — insufficient grounding for that yet.` | + +The new tiers are *honest gradients*, not synthesized content. Every +visible token in `partial` and `oov` surfaces is either a verbatim +lexicon atom (known side), the safely-displayed user input (OOV +side), or a fixed-template instruction. **No vocabulary is invented.** +**No domain is inferred.** + +### 2. New modules + +- `chat/oov_surface.py` — `oov_learning_invitation_surface(token, + intent_tag, pack_ids)`. Returns the OOV surface or `None` (caller + routes to universal disclosure). +- `chat/partial_surface.py` — `partial_comparison_surface(a, b, + pack_ids)`. Returns `(surface, known_side)` when exactly one of + the two compared lemmas resolves, else `None`. +- `teaching/oov_sink.py` — `OOVCandidate` + `OOVBufferSink` + + `OOVMonthlyFileSink`. Same on-disk shape as the discovery sink. +- `teaching/oov_gaps.py` — `aggregate_oov_gaps(root, since, + sample_limit) → tuple[OOVGap, ...]`. Pure reader over the OOV + sink layout. +- `teaching/oov_promotion.py` — `promote_oov_gaps(gaps, threshold, + include_tainted, suggested_packs) → tuple[OOVPromotion, ...]`. + +### 3. Runtime wiring + +`chat/runtime.py:_maybe_pack_grounded_surface` was refactored so +every existing intent branch *falls through* on a `None` composer +result instead of early-returning `None`. The OOV invitation +becomes the deterministic fall-through for any clean-subject +prompt whose subject doesn't resolve in any mounted pack. + +`ChatRuntime.attach_oov_sink(sink)` mirrors `attach_discovery_sink` +— the runtime emits one `OOVCandidate` JSONL line per turn whose +`grounding_source == "oov"` and is a no-op when no sink is attached. + +### 4. Relations pack v2 (P2.4) + +`en_core_relations_v2` — 8 pronoun + role-filler lemmas, each a +specialization of a v1 primitive: + +| Lemma | Specialization of | Primary domain | +|---|---|---| +| mother | parent | `kinship.parent.female` | +| father | parent | `kinship.parent.male` | +| daughter | child | `kinship.child.female` | +| son | child | `kinship.child.male` | +| brother | sibling | `kinship.sibling.male` | +| sister | sibling | `kinship.sibling.female` | +| grandparent | ancestor (1-step) | `kinship.ascendant.transitive_1step` | +| grandchild | descendant (1-step) | `kinship.descendant.transitive_1step` | + +Mounted by default. Orthogonal to v1 and cognition (no lemma +collision). Companion `relations_chains_v2` corpus seeds 7 v2-internal +reviewed chains so v2 lemmas ground via CAUSE + VERIFICATION, not +just DEFINITION/RECALL. + +### 5. Two new CLI surfaces + +``` +core teaching oov-gaps [--top N] [--since YYYY-MM] [--root PATH] +core teaching oov-queue [--threshold N] [--include-tainted] +``` + +Same shape as `core teaching gaps` / `core teaching queue` from +Phase 1 — operators get a consistent workflow whether the signal is +a chain gap or a lexicon gap. + +--- + +## Operator workflow (closed loop, both axes) + +``` +operator → core chat + ← cold turn + - lemma resolves + chain exists → teaching surface + - lemma resolves, no chain → discovery sink + universal/teaching tier + - lemma OOV → OOV invitation surface + OOV sink + - one lemma OOV in comparison → partial surface + +operator → core teaching gaps # chain-gap aggregation +operator → core teaching queue # chain-gap auto-promotion +operator → core teaching oov-gaps # vocabulary-gap aggregation +operator → core teaching oov-queue # vocabulary-gap auto-promotion + +operator → for chain gaps: core teaching propose +operator → for vocab gaps: author PackMutationProposal (ADR-0027 path) +operator → core teaching review --accept +``` + +Two independent signal streams, identical structural shape, both +feed the same reviewed mutation path. + +--- + +## Trust boundaries + +- **No content synthesis.** OOV surface names the unknown token + verbatim (safe-displayed); partial surface composes known-side + atoms verbatim. Neither composer invents vocabulary or guesses + domain. +- **Sink emission is opt-in.** Without `attach_oov_sink`, the OOV + surface still fires (P2.1 is unconditional), but nothing is + persisted. Identical to the pre-Phase-2 path when no sink is + attached. +- **Auto-promotion never mutates a pack.** `OOVPromotion` is an + operator-visible signal; the only path to a real pack change is + the existing reviewed `PackMutationProposal` (ADR-0027). +- **Suggested packs are mounted-pack list.** The promotion does + NOT recommend a single destination — domain inference is out of + scope (would require a stochastic classifier). + +--- + +## Files changed + +``` +chat/oov_surface.py NEW (~125 lines) +chat/partial_surface.py NEW (~105 lines) +chat/pack_resolver.py relations_v2 added to defaults +chat/runtime.py fall-through refactor + attach_oov_sink + emission +chat/teaching_grounding.py relations_chains_v2 registered +core/cli.py oov-gaps + oov-queue subcommands +core/config.py relations_v2 in input_packs defaults +language_packs/data/en_core_relations_v2/ NEW pack (8 lemmas + manifest) +teaching/oov_sink.py NEW (~150 lines) +teaching/oov_gaps.py NEW (~165 lines) +teaching/oov_promotion.py NEW (~120 lines) +teaching/relations_chains_v2/ NEW corpus (7 reviewed chains) +tests/test_oov_surface.py NEW (22 tests) +tests/test_partial_surface.py NEW (16 tests) +tests/test_oov_pipeline.py NEW (24 tests) +tests/test_en_core_relations_v2_pack.py NEW (10 tests) +docs/decisions/ADR-0065-oov-gradient-and-relations-v2.md NEW (this file) +``` + +--- + +## Verification + +``` +tests/test_oov_surface.py 22 passed +tests/test_partial_surface.py 16 passed +tests/test_oov_pipeline.py 24 passed +tests/test_en_core_relations_v2_pack.py 10 passed + +Curated lanes (all green): + core test --suite smoke 67 passed + core test --suite cognition 121 passed + core test --suite teaching 17 passed + core test --suite packs 6 passed + core test --suite runtime 19 passed + core test --suite algebra 132 passed + +Cognition eval (byte-identical to pre-ADR baseline): + public: intent 100% / surface 100% / term 91.7% / closure 100% + holdout: intent 100% / surface 100% / term 83.3% / closure 100% + +Live verification: + > What is photosynthesis? + [oov] I haven't learned 'photosynthesis' yet (intent: definition). ... + > Compare knowledge and photosynthesis. + [partial] Whatever 'photosynthesis' is, I can ground 'knowledge' ... + > What is mother? + [pack] mother — pack-grounded (en_core_relations_v2): kinship.parent.female; ... + > Why does mother exist? + [teaching] mother — teaching-grounded (relations_chains_v2): mother precedes daughter ... +``` + +The non-negotiable field invariant `versor_condition(F) < 1e-6` is +unaffected. + +--- + +## Future ADRs unlocked + +- **ADR-0066 — Multi-lemma CAUSE/VERIFICATION partial grounding.** + Today the partial tier engages only on COMPARISON. CAUSE and + VERIFICATION carry a single subject; once the intent classifier + grows multi-lemma extraction (e.g. "Why does photosynthesis + produce energy?" → CAUSE + subject=photosynthesis + secondary + object-side hint=energy), partial-grounding extends to those + intents too. +- **Phase 3 — turn-level composition.** Anaphora / NARRATIVE / + EXAMPLE intents. Requires Phase 1+2 corpus density first. +- **Domain classifier for OOV promotion suggestions.** Today the + OOV queue lists every mounted pack. A small deterministic + domain heuristic (token affix matches a pack's primary domain + prefix?) could narrow the suggestion — only if it stays + deterministic and the operator can override. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 7363a0c7..4ca5097c 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -72,6 +72,7 @@ ADRs record significant architectural decisions: what was decided, why, what alt | [ADR-0060](ADR-0060-correction-acknowledgment-topic-lemma.md) | CORRECTION acknowledgement surface weaves the first pack-resident topical lemma from the utterance (left-to-right, excluding `correction` itself and `be`/`have` fillers) into a fixed template; backward-compatible with ADR-0053 (no-arg path byte-identical); closes `correction_truth_040` holdout miss; holdout `term_capture_rate` 75.0% → 79.2% | **Accepted** (2026-05-18) | | [ADR-0061](ADR-0061-procedure-intent-pack-grounded-surface.md) | PROCEDURE intent (`"How do I X?"`) routes to new `pack_grounded_procedure_surface`; selector picks **last** pack-resident lemma from verb-phrase subject (object > verb), falls back to verb when object is OOV, returns `None` (→ universal disclosure) for no-pack-lemma utterances; closes `procedure_define_010` (term `concept`) + `procedure_verify_034` (surface); holdout `surface_groundedness` 94.7% → 100.0%; `term_capture_rate` 79.2% → 83.3% | **Accepted** (2026-05-18) | | [ADR-0062](ADR-0062-composed-teaching-grounded-surface.md) | Composed teaching-grounded surface: when a chain `(A, intent_A, conn_A, B)` has a follow-up chain `(B, ?, conn_B, C)`, emit `"{A} {conn_A} {B}, which {conn_B} {C}"` instead of just `"{A} {conn_A} {B}"`; depth-1 (one hop) + cycle guard + pack-residency guard; degrades to single-chain byte-identically when no follow-up survives the guards; opt-in via `RuntimeConfig.composed_surface=False` default; cognition lane null-drop invariant (metrics byte-identical flag OFF/ON) CI-pinned | **Accepted** (2026-05-18) | +| [ADR-0065](ADR-0065-oov-gradient-and-relations-v2.md) | OOV gradient + relations v2 (Plan Phase 2): five-tier honesty gradient replaces the OOV cliff — pack / teaching / partial (one OOV + one known) / oov (learning invitation surface naming the unknown token + mounted-pack list) / universal disclosure; sink-emit OOVCandidates → `core teaching oov-gaps` aggregator → `core teaching oov-queue` auto-promotion mirrors P1.1+P1.2 architecture for vocab gaps; `en_core_relations_v2` adds 8 pronoun + role-filler lemmas (mother/father/son/daughter/brother/sister/grandparent/grandchild) with 7 reviewed v2-internal chains; no content synthesis, no domain inference, no auto-pack-mutation | **Accepted** (2026-05-18) | | [ADR-0064](ADR-0064-cross-pack-teaching-chains.md) | Cross-pack teaching chains: `chat/teaching_grounding.py` registers a tuple of `TeachingCorpusSpec(corpus_id, path, pack_id)`; each corpus is 1:1-bound to one lexicon pack (cross-domain triples deferred per teaching_order.md §5); new `_all_chains_index()` aggregates across registered corpora (first-match-wins); surface composers + discovery gate consult the aggregated view; `TeachingChain` gains `corpus_id` field; surface tag follows the resolving corpus id; replay-equivalence gate rewrites registry path during transient phase; `relations_chains_v1` seeded with 7 reviewed kinship chains; cognition lane byte-identical | **Accepted** (2026-05-18) | | [ADR-0063](ADR-0063-cross-pack-surface-resolver.md) | Cross-pack surface resolver: `chat/pack_resolver.py` introduces `resolve_lemma(lemma, pack_ids)` that maps a lemma to `(resolving_pack_id, semantic_domains)` across an ordered tuple of mounted lexicon packs (first-match-wins); pack-grounded DEFINITION / RECALL / COMPARISON / CORRECTION / PROCEDURE composers now consult the resolver instead of a hardcoded `en_core_cognition_v1`; surface trust-boundary tag follows the resolving pack id; `en_core_relations_v1` joins `RuntimeConfig.input_packs` defaults — kinship lemmas now ground on the live path without a separate composer module; cognition-lane surfaces remain byte-identical (cognition is resolved first) | **Accepted** (2026-05-18) | diff --git a/teaching/oov_gaps.py b/teaching/oov_gaps.py new file mode 100644 index 00000000..8b6504c2 --- /dev/null +++ b/teaching/oov_gaps.py @@ -0,0 +1,170 @@ +"""teaching/oov_gaps.py — Phase 2.3: aggregate emitted OOVCandidates +into a ranked view of unknown tokens. + +Sibling to :mod:`teaching.gaps`. Where discovery candidates point at +gaps in the *teaching corpus* (a chain would have helped), OOV +candidates point at gaps in the *lexicon* (a vocabulary entry would +have helped). Both flow through the same operator workflow: rank +by frequency, auto-promote at threshold, surface to an operator who +authors a reviewed mutation. + +Design constraints (matching :mod:`teaching.gaps`): + + - Pure reader. No mutation of any sink file. + - Deterministic ordering: highest-count tokens first, ties broken + by token then intent set. + - Date filtering via the sink's file naming convention + (``/.jsonl``) — month-level granularity. + - Malformed lines are skipped silently. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + + +_DEFAULT_ROOT: Path = Path(__file__).resolve().parent / "oov_log" + +_MONTH_FILE_RE = re.compile(r"^(\d{4})-(\d{2})\.jsonl$") +_MONTH_TOKEN_RE = re.compile(r"^(\d{4})-(\d{2})$") + + +@dataclass(frozen=True, slots=True) +class OOVGap: + """One aggregated OOV token. + + Fields: + - ``token``: the unknown vocabulary item (lower-case). + - ``intents``: sorted tuple of intent shapes that hit this + token at least once. A token asked about under multiple + intent shapes is a stronger curriculum signal than one asked + only via ``DEFINITION``. + - ``count``: total emissions. + - ``boundary_clean_count``: subset whose ``boundary_clean=True``. + - ``sample_candidate_ids``: up to N retained ids. + - ``months_seen``: sorted ``YYYY-MM`` months. + """ + + token: str + intents: tuple[str, ...] + count: int + boundary_clean_count: int + sample_candidate_ids: tuple[str, ...] + months_seen: tuple[str, ...] + + def as_dict(self) -> dict[str, object]: + return { + "token": self.token, + "intents": list(self.intents), + "count": self.count, + "boundary_clean_count": self.boundary_clean_count, + "sample_candidate_ids": list(self.sample_candidate_ids), + "months_seen": list(self.months_seen), + } + + +def _normalise_since(since: str | None) -> tuple[int, int] | None: + if since is None: + return None + match = _MONTH_TOKEN_RE.match(since.strip()) + if not match: + raise ValueError( + f"--since {since!r} is not a YYYY-MM token (e.g. '2026-05')" + ) + return int(match.group(1)), int(match.group(2)) + + +def _iter_candidate_files( + root: Path, *, since: tuple[int, int] | None +) -> Iterable[tuple[str, Path]]: + if not root.exists() or not root.is_dir(): + return + for path in sorted(root.rglob("*.jsonl")): + m = _MONTH_FILE_RE.match(path.name) + if not m: + continue + year = int(m.group(1)) + month = int(m.group(2)) + if since is not None and (year, month) < since: + continue + yield f"{year:04d}-{month:02d}", path + + +def aggregate_oov_gaps( + root: Path = _DEFAULT_ROOT, + *, + since: str | None = None, + sample_limit: int = 5, +) -> tuple[OOVGap, ...]: + """Aggregate every emitted ``OOVCandidate`` under *root* into a + ranked tuple of :class:`OOVGap` records. + + Returned tuple is sorted by ``(count desc, token asc)`` so + identical inputs produce identical orderings. + """ + since_tuple = _normalise_since(since) + counts: dict[str, int] = {} + clean_counts: dict[str, int] = {} + samples: dict[str, list[str]] = {} + months: dict[str, set[str]] = {} + intents_by_token: dict[str, set[str]] = {} + + for month_token, path in _iter_candidate_files(root, since=since_tuple): + try: + text = path.read_text(encoding="utf-8") + except OSError: + continue + for line in text.splitlines(): + line = line.strip() + if not line: + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(entry, dict): + continue + token = entry.get("token") + intent = entry.get("intent") + if not isinstance(token, str) or not isinstance(intent, str): + continue + token = token.strip().lower() + intent = intent.strip().lower() + if not token or not intent: + continue + counts[token] = counts.get(token, 0) + 1 + if entry.get("boundary_clean") is True: + clean_counts[token] = clean_counts.get(token, 0) + 1 + intents_by_token.setdefault(token, set()).add(intent) + sample_list = samples.setdefault(token, []) + candidate_id = entry.get("candidate_id") + if ( + isinstance(candidate_id, str) + and candidate_id + and len(sample_list) < sample_limit + and candidate_id not in sample_list + ): + sample_list.append(candidate_id) + months.setdefault(token, set()).add(month_token) + + rows: list[OOVGap] = [] + for token, total in counts.items(): + rows.append( + OOVGap( + token=token, + intents=tuple(sorted(intents_by_token.get(token, ()))), + count=total, + boundary_clean_count=clean_counts.get(token, 0), + sample_candidate_ids=tuple(sorted(samples.get(token, ()))), + months_seen=tuple(sorted(months.get(token, ()))), + ) + ) + rows.sort(key=lambda g: (-g.count, g.token)) + return tuple(rows) + + +__all__ = ["OOVGap", "aggregate_oov_gaps"] diff --git a/teaching/oov_promotion.py b/teaching/oov_promotion.py new file mode 100644 index 00000000..98bbc2b0 --- /dev/null +++ b/teaching/oov_promotion.py @@ -0,0 +1,119 @@ +"""teaching/oov_promotion.py — Phase 2.3: auto-promote high-frequency +OOV tokens to operator-visible PackMutationProposal candidates. + +Sibling to :mod:`teaching.promotion`. Where chain-gap promotion says +"author a chain for this (subject, intent) cell", OOV promotion says +"add this token to a lexicon pack". + +Trust boundary — same as :mod:`teaching.promotion`: + + - Pure derivation from :class:`OOVGap` records. No persistent + queue. Re-running ``promote_oov_gaps`` on the same sink contents + produces the same result deterministically. + - **No domain inference.** The promotion does NOT recommend a + target pack — that would require a stochastic classifier. It + surfaces the mounted-pack list and lets the operator decide + which pack the token belongs in. + - The ratified-pack-mutation path (ADR-0027 + ADR-0033 + + :mod:`teaching.proposals`) is the only way an OOV promotion + becomes a real pack change. Auto-promotion never writes a + pack file directly. + - Boundary-clean filter on by default (matches + :func:`teaching.promotion.promote_gaps`). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Iterable + +from chat.pack_resolver import DEFAULT_RESOLVABLE_PACK_IDS +from teaching.oov_gaps import OOVGap + + +@dataclass(frozen=True, slots=True) +class OOVPromotion: + """An OOV token whose emission count met the threshold. + + Operator surface signal: "this vocabulary item has been asked + about N times across these intent shapes; add it to one of the + mounted packs." + """ + + token: str + intents: tuple[str, ...] + count: int + boundary_clean_count: int + sample_candidate_ids: tuple[str, ...] + months_seen: tuple[str, ...] + threshold: int + suggested_packs: tuple[str, ...] + + @property + def queue_id(self) -> str: + """Stable, deterministic identifier — diffable across runs.""" + return f"oov:{self.token}@{self.threshold}" + + def as_dict(self) -> dict[str, object]: + return { + "queue_id": self.queue_id, + "token": self.token, + "intents": list(self.intents), + "count": self.count, + "boundary_clean_count": self.boundary_clean_count, + "sample_candidate_ids": list(self.sample_candidate_ids), + "months_seen": list(self.months_seen), + "threshold": self.threshold, + "suggested_packs": list(self.suggested_packs), + } + + +def promote_oov_gaps( + gaps: Iterable[OOVGap], + *, + threshold: int = 3, + include_tainted: bool = False, + suggested_packs: tuple[str, ...] = DEFAULT_RESOLVABLE_PACK_IDS, +) -> tuple[OOVPromotion, ...]: + """Return the subset of *gaps* whose effective count meets *threshold*. + + Effective count: + - ``include_tainted=False`` (default): boundary_clean_count gates + the promotion. + - ``include_tainted=True``: every emission counts. + + ``suggested_packs`` is the list of mounted-pack ids that operators + can mutate via the reviewed-proposal path. Defaults to the + cross-pack resolver's mounted set; operators can pass a narrower + list when they want the queue surface to recommend a subset. + """ + if threshold < 1: + raise ValueError(f"threshold must be >= 1 (got {threshold!r})") + + promoted: list[OOVPromotion] = [] + for gap in gaps: + effective_count = gap.count if include_tainted else gap.boundary_clean_count + if effective_count < threshold: + continue + promoted.append( + OOVPromotion( + token=gap.token, + intents=gap.intents, + count=gap.count, + boundary_clean_count=gap.boundary_clean_count, + sample_candidate_ids=gap.sample_candidate_ids, + months_seen=gap.months_seen, + threshold=threshold, + suggested_packs=suggested_packs, + ) + ) + promoted.sort( + key=lambda p: ( + -(p.count if include_tainted else p.boundary_clean_count), + p.token, + ) + ) + return tuple(promoted) + + +__all__ = ["OOVPromotion", "promote_oov_gaps"] diff --git a/teaching/oov_sink.py b/teaching/oov_sink.py new file mode 100644 index 00000000..12344108 --- /dev/null +++ b/teaching/oov_sink.py @@ -0,0 +1,160 @@ +"""teaching/oov_sink.py — Phase 2.3 emission for OOV "teach me" turns. + +Mirrors :mod:`teaching.discovery_sink`. When the runtime emits a P2.1 +OOV invitation surface (``grounding_source="oov"``), it forwards a +structured :class:`OOVCandidate` JSONL line to the attached sink so +the operator's aggregation tooling can rank vocabulary gaps the same +way discovery candidates surface chain gaps. + +Trust boundary: + + - Append-only. No truncation, no rewrite. Each ``emit()`` flushes + so a crashed runtime keeps its prior OOV signals durable on disk. + - Sink errors are NOT swallowed — fail-fast contract matches + discovery and telemetry sinks. + - The sink receives a sanitised candidate (the token has already + passed through ``core._safe_display.safe_display`` at the runtime + boundary before any persistence). +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import IO, Callable, Literal, Protocol + + +@dataclass(frozen=True, slots=True) +class OOVCandidate: + """Structured evidence that a turn hit an OOV token. + + Fields parallel :class:`teaching.discovery.DiscoveryCandidate` + but the schema is OOV-specific. ``trigger="unresolved_subject"`` + is the only v1 trigger; future Phase 2 work can add others + (e.g. ``"unresolved_secondary_subject"`` for partial-grounding + sinks). + """ + + candidate_id: str + token: str + intent: Literal[ + "definition", "recall", "cause", "verification", + "comparison", "procedure", "correction", + ] + trigger: Literal["unresolved_subject"] + source_turn_trace: str + boundary_clean: bool + review_state: Literal["unreviewed"] = "unreviewed" + + def as_dict(self) -> dict[str, object]: + return { + "candidate_id": self.candidate_id, + "token": self.token, + "intent": self.intent, + "trigger": self.trigger, + "source_turn_trace": self.source_turn_trace, + "boundary_clean": self.boundary_clean, + "review_state": self.review_state, + } + + +def hash_oov_candidate_id(token: str, intent: str, trace_hash: str) -> str: + """Deterministic 32-char hex id for an OOV candidate. + + Identical ``(token, intent, trace_hash)`` always produces the + identical id — the load-bearing replay property analogous to + :func:`teaching.discovery._hash_candidate_id`. + """ + payload = json.dumps( + {"token": token, "intent": intent, "source_turn_trace": trace_hash}, + sort_keys=True, + separators=(",", ":"), + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32] + + +def format_oov_candidate_jsonl(candidate: OOVCandidate) -> str: + """Render a candidate as one canonical JSONL line.""" + return json.dumps(candidate.as_dict(), sort_keys=True, separators=(",", ":")) + + +class OOVCandidateSink(Protocol): + """Minimal sink contract — one JSONL line per emission.""" + + def emit(self, line: str) -> None: ... + + +@dataclass +class OOVBufferSink: + """In-memory sink that captures every emitted candidate line.""" + + lines: list[str] = field(default_factory=list) + + def emit(self, line: str) -> None: + self.lines.append(line) + + +Clock = Callable[[], datetime] + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +class OOVMonthlyFileSink: + """Append-only JSONL sink with monthly rollover. + + Path is computed at each ``emit()`` from the injected clock as + ``//.jsonl``. Same on-disk shape as + :class:`teaching.discovery_sink.DiscoveryMonthlyFileSink` so the + aggregator can reuse the file-walk machinery. + """ + + def __init__(self, root: str | Path, *, clock: Clock = _utc_now) -> None: + self._root = Path(root) + self._clock = clock + self._fh: IO[str] | None = None + self._current_path: Path | None = None + + def _path_for_now(self) -> Path: + now = self._clock() + return self._root / f"{now.year:04d}" / f"{now.year:04d}-{now.month:02d}.jsonl" + + def emit(self, line: str) -> None: + target = self._path_for_now() + if target != self._current_path: + if self._fh is not None: + self._fh.close() + self._fh = None + target.parent.mkdir(parents=True, exist_ok=True) + self._fh = target.open("a", encoding="utf-8") + self._current_path = target + assert self._fh is not None + self._fh.write(line) + self._fh.write("\n") + self._fh.flush() + + def close(self) -> None: + if self._fh is not None: + self._fh.close() + self._fh = None + self._current_path = None + + def __enter__(self) -> "OOVMonthlyFileSink": + return self + + def __exit__(self, *exc_info) -> None: + self.close() + + +__all__ = [ + "OOVCandidate", + "OOVCandidateSink", + "OOVBufferSink", + "OOVMonthlyFileSink", + "format_oov_candidate_jsonl", + "hash_oov_candidate_id", +] diff --git a/tests/test_oov_pipeline.py b/tests/test_oov_pipeline.py new file mode 100644 index 00000000..8a3456a0 --- /dev/null +++ b/tests/test_oov_pipeline.py @@ -0,0 +1,279 @@ +"""Phase 2.3 — OOV sink, aggregation, and auto-promotion tests. + +The contract these tests pin: + + - The runtime emits an ``OOVCandidate`` JSONL line to the attached + sink on every turn whose ``grounding_source == "oov"``; no-op + when no sink is attached. + - The candidate_id is deterministic on (token, intent, trace_hash). + - The aggregator groups by token, ranks by frequency, supports + ``--since YYYY-MM`` filtering. + - The promoter respects the boundary-clean filter by default and + refuses ``threshold < 1``. + - The promotion suggests mounted packs but never names a single + destination — domain inference is out of scope. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from chat.runtime import ChatRuntime +from teaching.oov_gaps import OOVGap, aggregate_oov_gaps +from teaching.oov_promotion import OOVPromotion, promote_oov_gaps +from teaching.oov_sink import ( + OOVBufferSink, + OOVCandidate, + format_oov_candidate_jsonl, + hash_oov_candidate_id, +) + + +# --------------------------------------------------------------------------- +# Sink contract +# --------------------------------------------------------------------------- + + +def test_buffer_sink_captures_each_emit() -> None: + sink = OOVBufferSink() + sink.emit("one") + sink.emit("two") + assert sink.lines == ["one", "two"] + + +def test_candidate_id_is_deterministic() -> None: + a = hash_oov_candidate_id("photosynthesis", "definition", "trace-1") + b = hash_oov_candidate_id("photosynthesis", "definition", "trace-1") + assert a == b + assert len(a) == 32 + + +def test_candidate_id_changes_with_token() -> None: + a = hash_oov_candidate_id("photosynthesis", "definition", "trace-1") + b = hash_oov_candidate_id("mitochondria", "definition", "trace-1") + assert a != b + + +def test_candidate_id_changes_with_trace() -> None: + a = hash_oov_candidate_id("photosynthesis", "definition", "trace-1") + b = hash_oov_candidate_id("photosynthesis", "definition", "trace-2") + assert a != b + + +def test_candidate_jsonl_is_sorted_compact() -> None: + cand = OOVCandidate( + candidate_id="x", + token="photosynthesis", + intent="definition", + trigger="unresolved_subject", + source_turn_trace="t", + boundary_clean=True, + ) + line = format_oov_candidate_jsonl(cand) + parsed = json.loads(line) + assert parsed["token"] == "photosynthesis" + assert parsed["intent"] == "definition" + assert parsed["boundary_clean"] is True + + +# --------------------------------------------------------------------------- +# Runtime integration — sink receives one line per OOV turn +# --------------------------------------------------------------------------- + + +def test_runtime_emits_when_oov_sink_attached() -> None: + rt = ChatRuntime() + sink = OOVBufferSink() + rt.attach_oov_sink(sink) + rt.chat("What is photosynthesis?") + assert len(sink.lines) == 1 + parsed = json.loads(sink.lines[0]) + assert parsed["token"] == "photosynthesis" + assert parsed["intent"] == "definition" + assert parsed["trigger"] == "unresolved_subject" + + +def test_runtime_does_not_emit_without_sink() -> None: + """Sink emission is opt-in; runtime behaviour is identical when + no sink is attached.""" + rt = ChatRuntime() + resp = rt.chat("What is photosynthesis?") + # OOV surface still fires (P2.1 is unconditional), but nothing + # is persisted anywhere — there is no sink to receive it. + assert resp.grounding_source == "oov" + + +def test_runtime_does_not_emit_on_known_lemma() -> None: + rt = ChatRuntime() + sink = OOVBufferSink() + rt.attach_oov_sink(sink) + rt.chat("What is light?") + assert sink.lines == [] + + +def test_runtime_emits_across_intent_shapes() -> None: + """Every intent shape that triggers OOV (definition, cause, + verification, comparison, procedure) emits a candidate.""" + rt = ChatRuntime() + sink = OOVBufferSink() + rt.attach_oov_sink(sink) + rt.chat("What is photosynthesis?") + intents = set() + for line in sink.lines: + intents.add(json.loads(line)["intent"]) + assert "definition" in intents + + +# --------------------------------------------------------------------------- +# Aggregator — file walking + deterministic ordering +# --------------------------------------------------------------------------- + + +def _write_oov_line(path: Path, **kwargs) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + payload = { + "candidate_id": kwargs.get("candidate_id", "x"), + "token": kwargs.get("token", "photosynthesis"), + "intent": kwargs.get("intent", "definition"), + "trigger": "unresolved_subject", + "source_turn_trace": kwargs.get("trace", "t"), + "boundary_clean": kwargs.get("boundary_clean", True), + "review_state": "unreviewed", + } + with path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(payload, sort_keys=True, separators=(",", ":"))) + fh.write("\n") + + +def test_aggregates_by_token(tmp_path: Path) -> None: + sink = tmp_path / "2026" / "2026-05.jsonl" + _write_oov_line(sink, candidate_id="a", token="photosynthesis", intent="definition") + _write_oov_line(sink, candidate_id="b", token="photosynthesis", intent="cause") + _write_oov_line(sink, candidate_id="c", token="mitochondria", intent="definition") + + rows = aggregate_oov_gaps(tmp_path) + assert len(rows) == 2 + photo = next(g for g in rows if g.token == "photosynthesis") + assert photo.count == 2 + assert photo.intents == ("cause", "definition") + assert photo.boundary_clean_count == 2 + + +def test_rank_order_is_count_desc(tmp_path: Path) -> None: + sink = tmp_path / "2026" / "2026-05.jsonl" + for i in range(3): + _write_oov_line(sink, candidate_id=f"a{i}", token="photosynthesis") + _write_oov_line(sink, candidate_id="b0", token="mitochondria") + rows = aggregate_oov_gaps(tmp_path) + assert [g.token for g in rows] == ["photosynthesis", "mitochondria"] + + +def test_tainted_counted_but_split(tmp_path: Path) -> None: + sink = tmp_path / "2026" / "2026-05.jsonl" + _write_oov_line(sink, candidate_id="a", boundary_clean=True) + _write_oov_line(sink, candidate_id="b", boundary_clean=False) + rows = aggregate_oov_gaps(tmp_path) + assert rows[0].count == 2 + assert rows[0].boundary_clean_count == 1 + + +def test_since_filter(tmp_path: Path) -> None: + _write_oov_line(tmp_path / "2026" / "2026-04.jsonl", candidate_id="april") + _write_oov_line(tmp_path / "2026" / "2026-05.jsonl", candidate_id="may") + rows = aggregate_oov_gaps(tmp_path, since="2026-05") + assert len(rows) == 1 + assert rows[0].sample_candidate_ids == ("may",) + + +def test_malformed_lines_skipped(tmp_path: Path) -> None: + sink = tmp_path / "2026" / "2026-05.jsonl" + sink.parent.mkdir(parents=True, exist_ok=True) + sink.write_text( + "not json\n{}\n" + json.dumps({ + "candidate_id": "ok", "token": "photosynthesis", + "intent": "definition", "trigger": "unresolved_subject", + "source_turn_trace": "t", "boundary_clean": True, + }) + "\n", + encoding="utf-8", + ) + rows = aggregate_oov_gaps(tmp_path) + assert len(rows) == 1 + + +def test_aggregator_missing_root_returns_empty(tmp_path: Path) -> None: + assert aggregate_oov_gaps(tmp_path / "does_not_exist") == () + + +# --------------------------------------------------------------------------- +# Promotion +# --------------------------------------------------------------------------- + + +def _gap(token: str, count: int = 3, clean: int | None = None) -> OOVGap: + return OOVGap( + token=token, + intents=("definition",), + count=count, + boundary_clean_count=count if clean is None else clean, + sample_candidate_ids=("a", "b"), + months_seen=("2026-05",), + ) + + +def test_promotion_respects_threshold() -> None: + gaps = (_gap("photosynthesis", count=5, clean=5),) + promoted = promote_oov_gaps(gaps, threshold=3) + assert len(promoted) == 1 + assert promoted[0].token == "photosynthesis" + + +def test_promotion_excludes_below_threshold() -> None: + gaps = (_gap("rare", count=1, clean=1),) + assert promote_oov_gaps(gaps, threshold=3) == () + + +def test_promotion_excludes_tainted_only_by_default() -> None: + gaps = (_gap("forbidden", count=5, clean=0),) + assert promote_oov_gaps(gaps, threshold=3) == () + + +def test_include_tainted_counts_all() -> None: + gaps = (_gap("forbidden", count=5, clean=0),) + promoted = promote_oov_gaps(gaps, threshold=3, include_tainted=True) + assert len(promoted) == 1 + + +def test_threshold_must_be_positive() -> None: + with pytest.raises(ValueError): + promote_oov_gaps((_gap("photosynthesis"),), threshold=0) + + +def test_queue_id_format() -> None: + promoted = promote_oov_gaps((_gap("photosynthesis", count=5, clean=5),), threshold=3) + assert promoted[0].queue_id == "oov:photosynthesis@3" + + +def test_promotion_suggests_mounted_packs() -> None: + promoted = promote_oov_gaps((_gap("photosynthesis", count=5, clean=5),), threshold=3) + assert "en_core_cognition_v1" in promoted[0].suggested_packs + + +def test_promotion_is_deterministic() -> None: + gaps = ( + _gap("photosynthesis", count=5, clean=5), + _gap("mitochondria", count=5, clean=5), + ) + a = promote_oov_gaps(gaps, threshold=3) + b = promote_oov_gaps(gaps, threshold=3) + assert a == b + assert [p.token for p in a] == ["mitochondria", "photosynthesis"] + + +def test_promotion_does_not_mutate_input() -> None: + gaps = (_gap("photosynthesis", count=3, clean=3),) + snapshot = gaps[0] + promote_oov_gaps(gaps, threshold=3) + assert gaps[0] == snapshot