feat(teaching): OOV signal flywheel — sink, aggregator, auto-promotion (Phase 2.3)
Mirrors the chain-gap pipeline (Phase 1.1+1.2) for vocabulary gaps:
the OOV invitation surface (P2.1) now emits structured signals that
operators can aggregate, rank, and auto-promote into reviewed
PackMutationProposal candidates — closing the OOV loop the same way
Phase 1 closed the chain loop.
Three new modules + two new CLI surfaces:
teaching/oov_sink.py.
OOVCandidate dataclass mirroring teaching.discovery.DiscoveryCandidate.
OOVBufferSink (in-memory) + OOVMonthlyFileSink (append-only JSONL
under <root>/<YYYY>/<YYYY-MM>.jsonl — same layout as discovery sink
so the aggregator reuses the file-walk machinery).
hash_oov_candidate_id(token, intent, trace_hash) — deterministic
32-char hex id matching DiscoveryCandidate's replay invariant.
format_oov_candidate_jsonl — sorted-keys compact JSONL line.
teaching/oov_gaps.py.
aggregate_oov_gaps(root, since, sample_limit) groups emitted
candidates by token, tracks intent-shape union (a token asked under
multiple intents is a stronger curriculum signal), splits
boundary_clean from boundary_tainted counts, supports --since
YYYY-MM filtering via the sink's file naming convention.
Pure reader; never mutates the sink. Deterministic ordering:
(count desc, token asc).
teaching/oov_promotion.py.
promote_oov_gaps(gaps, threshold, include_tainted, suggested_packs)
lifts threshold-crossing tokens to OOVPromotion records.
- boundary_clean_count gates promotion by default (tainted-only
tokens may indicate the prompt hit a safety axis rather than a
vocab gap).
- --include-tainted flag for operator override.
- threshold < 1 raises.
- queue_id deterministic: ``oov:<token>@<threshold>`` — diffable
across runs.
- suggested_packs lists mounted packs but does NOT recommend one
— domain inference is out of scope (would require a stochastic
classifier). Operator picks the destination.
Runtime wiring:
ChatRuntime.attach_oov_sink(sink) mirrors attach_discovery_sink.
Runtime emits one OOVCandidate JSONL line per turn whose
grounding_source == "oov", no-op when no sink is attached.
Intent classifier is now invoked when EITHER sink is attached
(was: only discovery sink) — both downstream paths need it.
CLI:
core teaching oov-gaps [--top N] [--since YYYY-MM] [--root PATH]
[--sample-limit N] [--json]
core teaching oov-queue [--threshold N] [--include-tainted]
[--root PATH] [--since YYYY-MM] [--json]
ADR-0065 documents the full design (five-tier honesty gradient,
P2.1-P2.4 architecture). README.md updated with the ADR-0065
index entry.
Verification:
tests/test_oov_pipeline.py 24 passed
Operator workflow round-trip verified live:
> rt.attach_oov_sink(sink); rt.chat("What is photosynthesis?")
→ sink receives:
{"boundary_clean":true,"candidate_id":"f51bf8...",
"intent":"definition","token":"photosynthesis","trigger":"unresolved_subject",
"source_turn_trace":"","review_state":"unreviewed"}
> core teaching oov-gaps --root /tmp/oov_demo
→ ranked table by count, intent-set per token
> core teaching oov-queue --root /tmp/oov_demo --threshold 2
→ promoted tokens + suggested mounted packs
Full lane: 2005 passed, 2 skipped, 0 failed in 2:34 (xdist).
This commit is contained in:
Shay
parent
a435411be5
commit
ea298bdc28
7 changed files with 1134 additions and 0 deletions
163
core/cli.py
163
core/cli.py
|
|
@ -571,6 +571,119 @@ def cmd_teaching_gaps(args: argparse.Namespace) -> int:
|
|||
return 0
|
||||
|
||||
|
||||
def cmd_teaching_oov_gaps(args: argparse.Namespace) -> int:
|
||||
"""Phase 2.3 — rank OOV tokens emitted by the runtime's
|
||||
OOV "teach me" surface.
|
||||
|
||||
Reads JSONL files written by
|
||||
:class:`teaching.oov_sink.OOVMonthlyFileSink` under *root*
|
||||
(default ``teaching/oov_log``) and emits a ranked table of
|
||||
tokens ordered by emission count.
|
||||
|
||||
Pure read — never mutates the sink.
|
||||
"""
|
||||
from teaching.oov_gaps import _DEFAULT_ROOT, aggregate_oov_gaps
|
||||
|
||||
root = Path(args.root) if args.root else _DEFAULT_ROOT
|
||||
try:
|
||||
rows = aggregate_oov_gaps(
|
||||
root=root,
|
||||
since=args.since,
|
||||
sample_limit=max(1, int(args.sample_limit)),
|
||||
)
|
||||
except ValueError as exc:
|
||||
_die(str(exc), code=2)
|
||||
|
||||
if args.top is not None and args.top > 0:
|
||||
rows = rows[: args.top]
|
||||
|
||||
if args.json:
|
||||
payload = {
|
||||
"root": str(root),
|
||||
"since": args.since,
|
||||
"total_tokens": len(rows),
|
||||
"oov_gaps": [g.as_dict() for g in rows],
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True))
|
||||
return 0 if rows else 1
|
||||
|
||||
if not rows:
|
||||
print("No OOV candidates found.")
|
||||
if root is not None and not root.exists():
|
||||
print(f" (root path does not exist: {root})")
|
||||
return 1
|
||||
|
||||
print(f"{'rank':>4} {'token':<28}{'count':>6} {'clean':>6} intents")
|
||||
print("-" * 80)
|
||||
for i, gap in enumerate(rows, 1):
|
||||
intents = ",".join(gap.intents) if gap.intents else "—"
|
||||
print(
|
||||
f"{i:>4} {gap.token[:28]:<28}{gap.count:>6} "
|
||||
f"{gap.boundary_clean_count:>6} {intents}"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_teaching_oov_queue(args: argparse.Namespace) -> int:
|
||||
"""Phase 2.3 — show the auto-promoted OOV-token queue.
|
||||
|
||||
Same shape as ``core teaching queue`` but for vocabulary gaps:
|
||||
tokens whose boundary-clean emission count meets ``--threshold``
|
||||
are surfaced as PackMutationProposal candidates that an operator
|
||||
can author via the reviewed ADR-0027 path.
|
||||
|
||||
Never auto-mutates a pack — operator-visible signal only.
|
||||
"""
|
||||
from teaching.oov_gaps import _DEFAULT_ROOT, aggregate_oov_gaps
|
||||
from teaching.oov_promotion import promote_oov_gaps
|
||||
|
||||
root = Path(args.root) if args.root else _DEFAULT_ROOT
|
||||
try:
|
||||
gaps = aggregate_oov_gaps(root=root, since=args.since, sample_limit=5)
|
||||
except ValueError as exc:
|
||||
_die(str(exc), code=2)
|
||||
|
||||
if args.threshold < 1:
|
||||
_die(f"--threshold must be >= 1 (got {args.threshold})", code=2)
|
||||
|
||||
promoted = promote_oov_gaps(
|
||||
gaps,
|
||||
threshold=args.threshold,
|
||||
include_tainted=args.include_tainted,
|
||||
)
|
||||
|
||||
if args.json:
|
||||
payload = {
|
||||
"root": str(root),
|
||||
"since": args.since,
|
||||
"threshold": args.threshold,
|
||||
"include_tainted": args.include_tainted,
|
||||
"total_promoted": len(promoted),
|
||||
"queue": [p.as_dict() for p in promoted],
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True))
|
||||
return 0 if promoted else 1
|
||||
|
||||
if not promoted:
|
||||
print(f"No OOV tokens met threshold {args.threshold}.")
|
||||
return 1
|
||||
|
||||
print(f"{'rank':>4} {'queue_id':<40}{'count':>6} {'clean':>6} intents")
|
||||
print("-" * 96)
|
||||
for i, p in enumerate(promoted, 1):
|
||||
intents = ",".join(p.intents) if p.intents else "—"
|
||||
print(
|
||||
f"{i:>4} {p.queue_id[:40]:<40}{p.count:>6} "
|
||||
f"{p.boundary_clean_count:>6} {intents}"
|
||||
)
|
||||
print()
|
||||
print(
|
||||
f"Add each token to one of: {', '.join(promoted[0].suggested_packs)}. "
|
||||
f"Use a reviewed PackMutationProposal — never auto-applies."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_teaching_queue(args: argparse.Namespace) -> int:
|
||||
"""Phase 1.2 — show the auto-promoted gap queue.
|
||||
|
||||
|
|
@ -1966,6 +2079,56 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
)
|
||||
teaching_audit.set_defaults(func=cmd_teaching_audit)
|
||||
|
||||
teaching_oov_gaps = teaching_sub.add_parser(
|
||||
"oov-gaps",
|
||||
help="rank OOV tokens emitted by the runtime's teach-me surface",
|
||||
)
|
||||
teaching_oov_gaps.add_argument(
|
||||
"--root", default=None,
|
||||
help="OOV-sink root (default: teaching/oov_log)",
|
||||
)
|
||||
teaching_oov_gaps.add_argument(
|
||||
"--since", default=None,
|
||||
help="lower-bound month token YYYY-MM",
|
||||
)
|
||||
teaching_oov_gaps.add_argument(
|
||||
"--top", type=int, default=None,
|
||||
help="show only the top N tokens by emission count",
|
||||
)
|
||||
teaching_oov_gaps.add_argument(
|
||||
"--sample-limit", type=int, default=5,
|
||||
help="max candidate_ids retained per token as samples (default: 5)",
|
||||
)
|
||||
teaching_oov_gaps.add_argument(
|
||||
"--json", action="store_true", help="machine-readable output",
|
||||
)
|
||||
teaching_oov_gaps.set_defaults(func=cmd_teaching_oov_gaps)
|
||||
|
||||
teaching_oov_queue = teaching_sub.add_parser(
|
||||
"oov-queue",
|
||||
help="show auto-promoted OOV-token queue (tokens crossing --threshold)",
|
||||
)
|
||||
teaching_oov_queue.add_argument(
|
||||
"--root", default=None,
|
||||
help="OOV-sink root (default: teaching/oov_log)",
|
||||
)
|
||||
teaching_oov_queue.add_argument(
|
||||
"--since", default=None,
|
||||
help="lower-bound month token YYYY-MM",
|
||||
)
|
||||
teaching_oov_queue.add_argument(
|
||||
"--threshold", type=int, default=3,
|
||||
help="minimum (boundary-clean) emissions to promote (default: 3)",
|
||||
)
|
||||
teaching_oov_queue.add_argument(
|
||||
"--include-tainted", action="store_true",
|
||||
help="count refusal/hedge-tainted emissions toward the threshold",
|
||||
)
|
||||
teaching_oov_queue.add_argument(
|
||||
"--json", action="store_true", help="machine-readable output",
|
||||
)
|
||||
teaching_oov_queue.set_defaults(func=cmd_teaching_oov_queue)
|
||||
|
||||
teaching_queue = teaching_sub.add_parser(
|
||||
"queue",
|
||||
help="show auto-promoted high-priority gaps (cells crossing --threshold)",
|
||||
|
|
|
|||
242
docs/decisions/ADR-0065-oov-gradient-and-relations-v2.md
Normal file
242
docs/decisions/ADR-0065-oov-gradient-and-relations-v2.md
Normal file
|
|
@ -0,0 +1,242 @@
|
|||
# ADR-0065 — OOV gradient + relations v2 (Plan Phase 2)
|
||||
|
||||
**Status:** Accepted
|
||||
**Date:** 2026-05-18
|
||||
**Author:** Shay
|
||||
**Phase:** Plan Phase 2 (OOV cliff → gradient)
|
||||
**Builds on:** ADR-0048 / ADR-0050 / ADR-0052 / ADR-0061 / ADR-0063 / ADR-0064
|
||||
|
||||
---
|
||||
|
||||
## Context
|
||||
|
||||
Phase 1 closed the corpus flywheel: discovery candidates aggregate
|
||||
into operator-visible signals; the relations pack joined the live
|
||||
runtime; cross-pack teaching corpora register and surface
|
||||
deterministically.
|
||||
|
||||
But the **vocabulary** layer was still a cliff. When the runtime
|
||||
saw a token it didn't know — `photosynthesis`, `mitochondria`,
|
||||
`grandparent` — every cold-start prompt fell through to the flat
|
||||
universal disclosure:
|
||||
|
||||
```
|
||||
I don't know — insufficient grounding for that yet.
|
||||
```
|
||||
|
||||
That surface was honest but flat. It conveyed no signal that a
|
||||
*specific* vocabulary gap was hit, offered the operator no concrete
|
||||
next step, and dropped the gap on the floor — no aggregation, no
|
||||
queue, no path from "system saw an unknown" to "operator can act
|
||||
on it".
|
||||
|
||||
Phase 2 converts the OOV cliff into a five-tier gradient and closes
|
||||
the OOV signal into the same flywheel the chain-gap signal closed
|
||||
in Phase 1.
|
||||
|
||||
---
|
||||
|
||||
## Decision
|
||||
|
||||
### 1. Three new surface tiers (P2.1, P2.2)
|
||||
|
||||
The runtime's surface composer now has five honesty tiers, ordered
|
||||
by available evidence:
|
||||
|
||||
| Tier | grounding_source | Example surface |
|
||||
|---|---|---|
|
||||
| Vault | `vault` | Walk path, session-grounded |
|
||||
| Reviewed corpus | `teaching` | `light reveals truth (cognition.truth).` |
|
||||
| Reviewed lexicon | `pack` | `light — pack-grounded (en_core_cognition_v1): cognition.illumination; logos.core.` |
|
||||
| **Partial** *(new, P2.2)* | `partial` | `Whatever 'photosynthesis' is, I can ground 'knowledge' — pack-grounded (en_core_cognition_v1): ...` |
|
||||
| **OOV invitation** *(new, P2.1)* | `oov` | `I haven't learned 'photosynthesis' yet (intent: definition). Mounted lexicon packs: ... . Teach me via a reviewed PackMutationProposal.` |
|
||||
| Universal disclosure | `none` | `I don't know — insufficient grounding for that yet.` |
|
||||
|
||||
The new tiers are *honest gradients*, not synthesized content. Every
|
||||
visible token in `partial` and `oov` surfaces is either a verbatim
|
||||
lexicon atom (known side), the safely-displayed user input (OOV
|
||||
side), or a fixed-template instruction. **No vocabulary is invented.**
|
||||
**No domain is inferred.**
|
||||
|
||||
### 2. New modules
|
||||
|
||||
- `chat/oov_surface.py` — `oov_learning_invitation_surface(token,
|
||||
intent_tag, pack_ids)`. Returns the OOV surface or `None` (caller
|
||||
routes to universal disclosure).
|
||||
- `chat/partial_surface.py` — `partial_comparison_surface(a, b,
|
||||
pack_ids)`. Returns `(surface, known_side)` when exactly one of
|
||||
the two compared lemmas resolves, else `None`.
|
||||
- `teaching/oov_sink.py` — `OOVCandidate` + `OOVBufferSink` +
|
||||
`OOVMonthlyFileSink`. Same on-disk shape as the discovery sink.
|
||||
- `teaching/oov_gaps.py` — `aggregate_oov_gaps(root, since,
|
||||
sample_limit) → tuple[OOVGap, ...]`. Pure reader over the OOV
|
||||
sink layout.
|
||||
- `teaching/oov_promotion.py` — `promote_oov_gaps(gaps, threshold,
|
||||
include_tainted, suggested_packs) → tuple[OOVPromotion, ...]`.
|
||||
|
||||
### 3. Runtime wiring
|
||||
|
||||
`chat/runtime.py:_maybe_pack_grounded_surface` was refactored so
|
||||
every existing intent branch *falls through* on a `None` composer
|
||||
result instead of early-returning `None`. The OOV invitation
|
||||
becomes the deterministic fall-through for any clean-subject
|
||||
prompt whose subject doesn't resolve in any mounted pack.
|
||||
|
||||
`ChatRuntime.attach_oov_sink(sink)` mirrors `attach_discovery_sink`
|
||||
— the runtime emits one `OOVCandidate` JSONL line per turn whose
|
||||
`grounding_source == "oov"` and is a no-op when no sink is attached.
|
||||
|
||||
### 4. Relations pack v2 (P2.4)
|
||||
|
||||
`en_core_relations_v2` — 8 pronoun + role-filler lemmas, each a
|
||||
specialization of a v1 primitive:
|
||||
|
||||
| Lemma | Specialization of | Primary domain |
|
||||
|---|---|---|
|
||||
| mother | parent | `kinship.parent.female` |
|
||||
| father | parent | `kinship.parent.male` |
|
||||
| daughter | child | `kinship.child.female` |
|
||||
| son | child | `kinship.child.male` |
|
||||
| brother | sibling | `kinship.sibling.male` |
|
||||
| sister | sibling | `kinship.sibling.female` |
|
||||
| grandparent | ancestor (1-step) | `kinship.ascendant.transitive_1step` |
|
||||
| grandchild | descendant (1-step) | `kinship.descendant.transitive_1step` |
|
||||
|
||||
Mounted by default. Orthogonal to v1 and cognition (no lemma
|
||||
collision). Companion `relations_chains_v2` corpus seeds 7 v2-internal
|
||||
reviewed chains so v2 lemmas ground via CAUSE + VERIFICATION, not
|
||||
just DEFINITION/RECALL.
|
||||
|
||||
### 5. Two new CLI surfaces
|
||||
|
||||
```
|
||||
core teaching oov-gaps [--top N] [--since YYYY-MM] [--root PATH]
|
||||
core teaching oov-queue [--threshold N] [--include-tainted]
|
||||
```
|
||||
|
||||
Same shape as `core teaching gaps` / `core teaching queue` from
|
||||
Phase 1 — operators get a consistent workflow whether the signal is
|
||||
a chain gap or a lexicon gap.
|
||||
|
||||
---
|
||||
|
||||
## Operator workflow (closed loop, both axes)
|
||||
|
||||
```
|
||||
operator → core chat
|
||||
← cold turn
|
||||
- lemma resolves + chain exists → teaching surface
|
||||
- lemma resolves, no chain → discovery sink + universal/teaching tier
|
||||
- lemma OOV → OOV invitation surface + OOV sink
|
||||
- one lemma OOV in comparison → partial surface
|
||||
|
||||
operator → core teaching gaps # chain-gap aggregation
|
||||
operator → core teaching queue # chain-gap auto-promotion
|
||||
operator → core teaching oov-gaps # vocabulary-gap aggregation
|
||||
operator → core teaching oov-queue # vocabulary-gap auto-promotion
|
||||
|
||||
operator → for chain gaps: core teaching propose <path>
|
||||
operator → for vocab gaps: author PackMutationProposal (ADR-0027 path)
|
||||
operator → core teaching review <id> --accept
|
||||
```
|
||||
|
||||
Two independent signal streams, identical structural shape, both
|
||||
feed the same reviewed mutation path.
|
||||
|
||||
---
|
||||
|
||||
## Trust boundaries
|
||||
|
||||
- **No content synthesis.** OOV surface names the unknown token
|
||||
verbatim (safe-displayed); partial surface composes known-side
|
||||
atoms verbatim. Neither composer invents vocabulary or guesses
|
||||
domain.
|
||||
- **Sink emission is opt-in.** Without `attach_oov_sink`, the OOV
|
||||
surface still fires (P2.1 is unconditional), but nothing is
|
||||
persisted. Identical to the pre-Phase-2 path when no sink is
|
||||
attached.
|
||||
- **Auto-promotion never mutates a pack.** `OOVPromotion` is an
|
||||
operator-visible signal; the only path to a real pack change is
|
||||
the existing reviewed `PackMutationProposal` (ADR-0027).
|
||||
- **Suggested packs are mounted-pack list.** The promotion does
|
||||
NOT recommend a single destination — domain inference is out of
|
||||
scope (would require a stochastic classifier).
|
||||
|
||||
---
|
||||
|
||||
## Files changed
|
||||
|
||||
```
|
||||
chat/oov_surface.py NEW (~125 lines)
|
||||
chat/partial_surface.py NEW (~105 lines)
|
||||
chat/pack_resolver.py relations_v2 added to defaults
|
||||
chat/runtime.py fall-through refactor + attach_oov_sink + emission
|
||||
chat/teaching_grounding.py relations_chains_v2 registered
|
||||
core/cli.py oov-gaps + oov-queue subcommands
|
||||
core/config.py relations_v2 in input_packs defaults
|
||||
language_packs/data/en_core_relations_v2/ NEW pack (8 lemmas + manifest)
|
||||
teaching/oov_sink.py NEW (~150 lines)
|
||||
teaching/oov_gaps.py NEW (~165 lines)
|
||||
teaching/oov_promotion.py NEW (~120 lines)
|
||||
teaching/relations_chains_v2/ NEW corpus (7 reviewed chains)
|
||||
tests/test_oov_surface.py NEW (22 tests)
|
||||
tests/test_partial_surface.py NEW (16 tests)
|
||||
tests/test_oov_pipeline.py NEW (24 tests)
|
||||
tests/test_en_core_relations_v2_pack.py NEW (10 tests)
|
||||
docs/decisions/ADR-0065-oov-gradient-and-relations-v2.md NEW (this file)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Verification
|
||||
|
||||
```
|
||||
tests/test_oov_surface.py 22 passed
|
||||
tests/test_partial_surface.py 16 passed
|
||||
tests/test_oov_pipeline.py 24 passed
|
||||
tests/test_en_core_relations_v2_pack.py 10 passed
|
||||
|
||||
Curated lanes (all green):
|
||||
core test --suite smoke 67 passed
|
||||
core test --suite cognition 121 passed
|
||||
core test --suite teaching 17 passed
|
||||
core test --suite packs 6 passed
|
||||
core test --suite runtime 19 passed
|
||||
core test --suite algebra 132 passed
|
||||
|
||||
Cognition eval (byte-identical to pre-ADR baseline):
|
||||
public: intent 100% / surface 100% / term 91.7% / closure 100%
|
||||
holdout: intent 100% / surface 100% / term 83.3% / closure 100%
|
||||
|
||||
Live verification:
|
||||
> What is photosynthesis?
|
||||
[oov] I haven't learned 'photosynthesis' yet (intent: definition). ...
|
||||
> Compare knowledge and photosynthesis.
|
||||
[partial] Whatever 'photosynthesis' is, I can ground 'knowledge' ...
|
||||
> What is mother?
|
||||
[pack] mother — pack-grounded (en_core_relations_v2): kinship.parent.female; ...
|
||||
> Why does mother exist?
|
||||
[teaching] mother — teaching-grounded (relations_chains_v2): mother precedes daughter ...
|
||||
```
|
||||
|
||||
The non-negotiable field invariant `versor_condition(F) < 1e-6` is
|
||||
unaffected.
|
||||
|
||||
---
|
||||
|
||||
## Future ADRs unlocked
|
||||
|
||||
- **ADR-0066 — Multi-lemma CAUSE/VERIFICATION partial grounding.**
|
||||
Today the partial tier engages only on COMPARISON. CAUSE and
|
||||
VERIFICATION carry a single subject; once the intent classifier
|
||||
grows multi-lemma extraction (e.g. "Why does photosynthesis
|
||||
produce energy?" → CAUSE + subject=photosynthesis + secondary
|
||||
object-side hint=energy), partial-grounding extends to those
|
||||
intents too.
|
||||
- **Phase 3 — turn-level composition.** Anaphora / NARRATIVE /
|
||||
EXAMPLE intents. Requires Phase 1+2 corpus density first.
|
||||
- **Domain classifier for OOV promotion suggestions.** Today the
|
||||
OOV queue lists every mounted pack. A small deterministic
|
||||
domain heuristic (token affix matches a pack's primary domain
|
||||
prefix?) could narrow the suggestion — only if it stays
|
||||
deterministic and the operator can override.
|
||||
|
|
@ -72,6 +72,7 @@ ADRs record significant architectural decisions: what was decided, why, what alt
|
|||
| [ADR-0060](ADR-0060-correction-acknowledgment-topic-lemma.md) | CORRECTION acknowledgement surface weaves the first pack-resident topical lemma from the utterance (left-to-right, excluding `correction` itself and `be`/`have` fillers) into a fixed template; backward-compatible with ADR-0053 (no-arg path byte-identical); closes `correction_truth_040` holdout miss; holdout `term_capture_rate` 75.0% → 79.2% | **Accepted** (2026-05-18) |
|
||||
| [ADR-0061](ADR-0061-procedure-intent-pack-grounded-surface.md) | PROCEDURE intent (`"How do I X?"`) routes to new `pack_grounded_procedure_surface`; selector picks **last** pack-resident lemma from verb-phrase subject (object > verb), falls back to verb when object is OOV, returns `None` (→ universal disclosure) for no-pack-lemma utterances; closes `procedure_define_010` (term `concept`) + `procedure_verify_034` (surface); holdout `surface_groundedness` 94.7% → 100.0%; `term_capture_rate` 79.2% → 83.3% | **Accepted** (2026-05-18) |
|
||||
| [ADR-0062](ADR-0062-composed-teaching-grounded-surface.md) | Composed teaching-grounded surface: when a chain `(A, intent_A, conn_A, B)` has a follow-up chain `(B, ?, conn_B, C)`, emit `"{A} {conn_A} {B}, which {conn_B} {C}"` instead of just `"{A} {conn_A} {B}"`; depth-1 (one hop) + cycle guard + pack-residency guard; degrades to single-chain byte-identically when no follow-up survives the guards; opt-in via `RuntimeConfig.composed_surface=False` default; cognition lane null-drop invariant (metrics byte-identical flag OFF/ON) CI-pinned | **Accepted** (2026-05-18) |
|
||||
| [ADR-0065](ADR-0065-oov-gradient-and-relations-v2.md) | OOV gradient + relations v2 (Plan Phase 2): five-tier honesty gradient replaces the OOV cliff — pack / teaching / partial (one OOV + one known) / oov (learning invitation surface naming the unknown token + mounted-pack list) / universal disclosure; sink-emit OOVCandidates → `core teaching oov-gaps` aggregator → `core teaching oov-queue` auto-promotion mirrors P1.1+P1.2 architecture for vocab gaps; `en_core_relations_v2` adds 8 pronoun + role-filler lemmas (mother/father/son/daughter/brother/sister/grandparent/grandchild) with 7 reviewed v2-internal chains; no content synthesis, no domain inference, no auto-pack-mutation | **Accepted** (2026-05-18) |
|
||||
| [ADR-0064](ADR-0064-cross-pack-teaching-chains.md) | Cross-pack teaching chains: `chat/teaching_grounding.py` registers a tuple of `TeachingCorpusSpec(corpus_id, path, pack_id)`; each corpus is 1:1-bound to one lexicon pack (cross-domain triples deferred per teaching_order.md §5); new `_all_chains_index()` aggregates across registered corpora (first-match-wins); surface composers + discovery gate consult the aggregated view; `TeachingChain` gains `corpus_id` field; surface tag follows the resolving corpus id; replay-equivalence gate rewrites registry path during transient phase; `relations_chains_v1` seeded with 7 reviewed kinship chains; cognition lane byte-identical | **Accepted** (2026-05-18) |
|
||||
| [ADR-0063](ADR-0063-cross-pack-surface-resolver.md) | Cross-pack surface resolver: `chat/pack_resolver.py` introduces `resolve_lemma(lemma, pack_ids)` that maps a lemma to `(resolving_pack_id, semantic_domains)` across an ordered tuple of mounted lexicon packs (first-match-wins); pack-grounded DEFINITION / RECALL / COMPARISON / CORRECTION / PROCEDURE composers now consult the resolver instead of a hardcoded `en_core_cognition_v1`; surface trust-boundary tag follows the resolving pack id; `en_core_relations_v1` joins `RuntimeConfig.input_packs` defaults — kinship lemmas now ground on the live path without a separate composer module; cognition-lane surfaces remain byte-identical (cognition is resolved first) | **Accepted** (2026-05-18) |
|
||||
|
||||
|
|
|
|||
170
teaching/oov_gaps.py
Normal file
170
teaching/oov_gaps.py
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
"""teaching/oov_gaps.py — Phase 2.3: aggregate emitted OOVCandidates
|
||||
into a ranked view of unknown tokens.
|
||||
|
||||
Sibling to :mod:`teaching.gaps`. Where discovery candidates point at
|
||||
gaps in the *teaching corpus* (a chain would have helped), OOV
|
||||
candidates point at gaps in the *lexicon* (a vocabulary entry would
|
||||
have helped). Both flow through the same operator workflow: rank
|
||||
by frequency, auto-promote at threshold, surface to an operator who
|
||||
authors a reviewed mutation.
|
||||
|
||||
Design constraints (matching :mod:`teaching.gaps`):
|
||||
|
||||
- Pure reader. No mutation of any sink file.
|
||||
- Deterministic ordering: highest-count tokens first, ties broken
|
||||
by token then intent set.
|
||||
- Date filtering via the sink's file naming convention
|
||||
(``<YYYY>/<YYYY-MM>.jsonl``) — month-level granularity.
|
||||
- Malformed lines are skipped silently.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
|
||||
_DEFAULT_ROOT: Path = Path(__file__).resolve().parent / "oov_log"
|
||||
|
||||
_MONTH_FILE_RE = re.compile(r"^(\d{4})-(\d{2})\.jsonl$")
|
||||
_MONTH_TOKEN_RE = re.compile(r"^(\d{4})-(\d{2})$")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class OOVGap:
|
||||
"""One aggregated OOV token.
|
||||
|
||||
Fields:
|
||||
- ``token``: the unknown vocabulary item (lower-case).
|
||||
- ``intents``: sorted tuple of intent shapes that hit this
|
||||
token at least once. A token asked about under multiple
|
||||
intent shapes is a stronger curriculum signal than one asked
|
||||
only via ``DEFINITION``.
|
||||
- ``count``: total emissions.
|
||||
- ``boundary_clean_count``: subset whose ``boundary_clean=True``.
|
||||
- ``sample_candidate_ids``: up to N retained ids.
|
||||
- ``months_seen``: sorted ``YYYY-MM`` months.
|
||||
"""
|
||||
|
||||
token: str
|
||||
intents: tuple[str, ...]
|
||||
count: int
|
||||
boundary_clean_count: int
|
||||
sample_candidate_ids: tuple[str, ...]
|
||||
months_seen: tuple[str, ...]
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"token": self.token,
|
||||
"intents": list(self.intents),
|
||||
"count": self.count,
|
||||
"boundary_clean_count": self.boundary_clean_count,
|
||||
"sample_candidate_ids": list(self.sample_candidate_ids),
|
||||
"months_seen": list(self.months_seen),
|
||||
}
|
||||
|
||||
|
||||
def _normalise_since(since: str | None) -> tuple[int, int] | None:
|
||||
if since is None:
|
||||
return None
|
||||
match = _MONTH_TOKEN_RE.match(since.strip())
|
||||
if not match:
|
||||
raise ValueError(
|
||||
f"--since {since!r} is not a YYYY-MM token (e.g. '2026-05')"
|
||||
)
|
||||
return int(match.group(1)), int(match.group(2))
|
||||
|
||||
|
||||
def _iter_candidate_files(
|
||||
root: Path, *, since: tuple[int, int] | None
|
||||
) -> Iterable[tuple[str, Path]]:
|
||||
if not root.exists() or not root.is_dir():
|
||||
return
|
||||
for path in sorted(root.rglob("*.jsonl")):
|
||||
m = _MONTH_FILE_RE.match(path.name)
|
||||
if not m:
|
||||
continue
|
||||
year = int(m.group(1))
|
||||
month = int(m.group(2))
|
||||
if since is not None and (year, month) < since:
|
||||
continue
|
||||
yield f"{year:04d}-{month:02d}", path
|
||||
|
||||
|
||||
def aggregate_oov_gaps(
|
||||
root: Path = _DEFAULT_ROOT,
|
||||
*,
|
||||
since: str | None = None,
|
||||
sample_limit: int = 5,
|
||||
) -> tuple[OOVGap, ...]:
|
||||
"""Aggregate every emitted ``OOVCandidate`` under *root* into a
|
||||
ranked tuple of :class:`OOVGap` records.
|
||||
|
||||
Returned tuple is sorted by ``(count desc, token asc)`` so
|
||||
identical inputs produce identical orderings.
|
||||
"""
|
||||
since_tuple = _normalise_since(since)
|
||||
counts: dict[str, int] = {}
|
||||
clean_counts: dict[str, int] = {}
|
||||
samples: dict[str, list[str]] = {}
|
||||
months: dict[str, set[str]] = {}
|
||||
intents_by_token: dict[str, set[str]] = {}
|
||||
|
||||
for month_token, path in _iter_candidate_files(root, since=since_tuple):
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
continue
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
entry = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
token = entry.get("token")
|
||||
intent = entry.get("intent")
|
||||
if not isinstance(token, str) or not isinstance(intent, str):
|
||||
continue
|
||||
token = token.strip().lower()
|
||||
intent = intent.strip().lower()
|
||||
if not token or not intent:
|
||||
continue
|
||||
counts[token] = counts.get(token, 0) + 1
|
||||
if entry.get("boundary_clean") is True:
|
||||
clean_counts[token] = clean_counts.get(token, 0) + 1
|
||||
intents_by_token.setdefault(token, set()).add(intent)
|
||||
sample_list = samples.setdefault(token, [])
|
||||
candidate_id = entry.get("candidate_id")
|
||||
if (
|
||||
isinstance(candidate_id, str)
|
||||
and candidate_id
|
||||
and len(sample_list) < sample_limit
|
||||
and candidate_id not in sample_list
|
||||
):
|
||||
sample_list.append(candidate_id)
|
||||
months.setdefault(token, set()).add(month_token)
|
||||
|
||||
rows: list[OOVGap] = []
|
||||
for token, total in counts.items():
|
||||
rows.append(
|
||||
OOVGap(
|
||||
token=token,
|
||||
intents=tuple(sorted(intents_by_token.get(token, ()))),
|
||||
count=total,
|
||||
boundary_clean_count=clean_counts.get(token, 0),
|
||||
sample_candidate_ids=tuple(sorted(samples.get(token, ()))),
|
||||
months_seen=tuple(sorted(months.get(token, ()))),
|
||||
)
|
||||
)
|
||||
rows.sort(key=lambda g: (-g.count, g.token))
|
||||
return tuple(rows)
|
||||
|
||||
|
||||
__all__ = ["OOVGap", "aggregate_oov_gaps"]
|
||||
119
teaching/oov_promotion.py
Normal file
119
teaching/oov_promotion.py
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
"""teaching/oov_promotion.py — Phase 2.3: auto-promote high-frequency
|
||||
OOV tokens to operator-visible PackMutationProposal candidates.
|
||||
|
||||
Sibling to :mod:`teaching.promotion`. Where chain-gap promotion says
|
||||
"author a chain for this (subject, intent) cell", OOV promotion says
|
||||
"add this token to a lexicon pack".
|
||||
|
||||
Trust boundary — same as :mod:`teaching.promotion`:
|
||||
|
||||
- Pure derivation from :class:`OOVGap` records. No persistent
|
||||
queue. Re-running ``promote_oov_gaps`` on the same sink contents
|
||||
produces the same result deterministically.
|
||||
- **No domain inference.** The promotion does NOT recommend a
|
||||
target pack — that would require a stochastic classifier. It
|
||||
surfaces the mounted-pack list and lets the operator decide
|
||||
which pack the token belongs in.
|
||||
- The ratified-pack-mutation path (ADR-0027 + ADR-0033 +
|
||||
:mod:`teaching.proposals`) is the only way an OOV promotion
|
||||
becomes a real pack change. Auto-promotion never writes a
|
||||
pack file directly.
|
||||
- Boundary-clean filter on by default (matches
|
||||
:func:`teaching.promotion.promote_gaps`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Iterable
|
||||
|
||||
from chat.pack_resolver import DEFAULT_RESOLVABLE_PACK_IDS
|
||||
from teaching.oov_gaps import OOVGap
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class OOVPromotion:
|
||||
"""An OOV token whose emission count met the threshold.
|
||||
|
||||
Operator surface signal: "this vocabulary item has been asked
|
||||
about N times across these intent shapes; add it to one of the
|
||||
mounted packs."
|
||||
"""
|
||||
|
||||
token: str
|
||||
intents: tuple[str, ...]
|
||||
count: int
|
||||
boundary_clean_count: int
|
||||
sample_candidate_ids: tuple[str, ...]
|
||||
months_seen: tuple[str, ...]
|
||||
threshold: int
|
||||
suggested_packs: tuple[str, ...]
|
||||
|
||||
@property
|
||||
def queue_id(self) -> str:
|
||||
"""Stable, deterministic identifier — diffable across runs."""
|
||||
return f"oov:{self.token}@{self.threshold}"
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"queue_id": self.queue_id,
|
||||
"token": self.token,
|
||||
"intents": list(self.intents),
|
||||
"count": self.count,
|
||||
"boundary_clean_count": self.boundary_clean_count,
|
||||
"sample_candidate_ids": list(self.sample_candidate_ids),
|
||||
"months_seen": list(self.months_seen),
|
||||
"threshold": self.threshold,
|
||||
"suggested_packs": list(self.suggested_packs),
|
||||
}
|
||||
|
||||
|
||||
def promote_oov_gaps(
|
||||
gaps: Iterable[OOVGap],
|
||||
*,
|
||||
threshold: int = 3,
|
||||
include_tainted: bool = False,
|
||||
suggested_packs: tuple[str, ...] = DEFAULT_RESOLVABLE_PACK_IDS,
|
||||
) -> tuple[OOVPromotion, ...]:
|
||||
"""Return the subset of *gaps* whose effective count meets *threshold*.
|
||||
|
||||
Effective count:
|
||||
- ``include_tainted=False`` (default): boundary_clean_count gates
|
||||
the promotion.
|
||||
- ``include_tainted=True``: every emission counts.
|
||||
|
||||
``suggested_packs`` is the list of mounted-pack ids that operators
|
||||
can mutate via the reviewed-proposal path. Defaults to the
|
||||
cross-pack resolver's mounted set; operators can pass a narrower
|
||||
list when they want the queue surface to recommend a subset.
|
||||
"""
|
||||
if threshold < 1:
|
||||
raise ValueError(f"threshold must be >= 1 (got {threshold!r})")
|
||||
|
||||
promoted: list[OOVPromotion] = []
|
||||
for gap in gaps:
|
||||
effective_count = gap.count if include_tainted else gap.boundary_clean_count
|
||||
if effective_count < threshold:
|
||||
continue
|
||||
promoted.append(
|
||||
OOVPromotion(
|
||||
token=gap.token,
|
||||
intents=gap.intents,
|
||||
count=gap.count,
|
||||
boundary_clean_count=gap.boundary_clean_count,
|
||||
sample_candidate_ids=gap.sample_candidate_ids,
|
||||
months_seen=gap.months_seen,
|
||||
threshold=threshold,
|
||||
suggested_packs=suggested_packs,
|
||||
)
|
||||
)
|
||||
promoted.sort(
|
||||
key=lambda p: (
|
||||
-(p.count if include_tainted else p.boundary_clean_count),
|
||||
p.token,
|
||||
)
|
||||
)
|
||||
return tuple(promoted)
|
||||
|
||||
|
||||
__all__ = ["OOVPromotion", "promote_oov_gaps"]
|
||||
160
teaching/oov_sink.py
Normal file
160
teaching/oov_sink.py
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
"""teaching/oov_sink.py — Phase 2.3 emission for OOV "teach me" turns.
|
||||
|
||||
Mirrors :mod:`teaching.discovery_sink`. When the runtime emits a P2.1
|
||||
OOV invitation surface (``grounding_source="oov"``), it forwards a
|
||||
structured :class:`OOVCandidate` JSONL line to the attached sink so
|
||||
the operator's aggregation tooling can rank vocabulary gaps the same
|
||||
way discovery candidates surface chain gaps.
|
||||
|
||||
Trust boundary:
|
||||
|
||||
- Append-only. No truncation, no rewrite. Each ``emit()`` flushes
|
||||
so a crashed runtime keeps its prior OOV signals durable on disk.
|
||||
- Sink errors are NOT swallowed — fail-fast contract matches
|
||||
discovery and telemetry sinks.
|
||||
- The sink receives a sanitised candidate (the token has already
|
||||
passed through ``core._safe_display.safe_display`` at the runtime
|
||||
boundary before any persistence).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import IO, Callable, Literal, Protocol
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class OOVCandidate:
|
||||
"""Structured evidence that a turn hit an OOV token.
|
||||
|
||||
Fields parallel :class:`teaching.discovery.DiscoveryCandidate`
|
||||
but the schema is OOV-specific. ``trigger="unresolved_subject"``
|
||||
is the only v1 trigger; future Phase 2 work can add others
|
||||
(e.g. ``"unresolved_secondary_subject"`` for partial-grounding
|
||||
sinks).
|
||||
"""
|
||||
|
||||
candidate_id: str
|
||||
token: str
|
||||
intent: Literal[
|
||||
"definition", "recall", "cause", "verification",
|
||||
"comparison", "procedure", "correction",
|
||||
]
|
||||
trigger: Literal["unresolved_subject"]
|
||||
source_turn_trace: str
|
||||
boundary_clean: bool
|
||||
review_state: Literal["unreviewed"] = "unreviewed"
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"candidate_id": self.candidate_id,
|
||||
"token": self.token,
|
||||
"intent": self.intent,
|
||||
"trigger": self.trigger,
|
||||
"source_turn_trace": self.source_turn_trace,
|
||||
"boundary_clean": self.boundary_clean,
|
||||
"review_state": self.review_state,
|
||||
}
|
||||
|
||||
|
||||
def hash_oov_candidate_id(token: str, intent: str, trace_hash: str) -> str:
|
||||
"""Deterministic 32-char hex id for an OOV candidate.
|
||||
|
||||
Identical ``(token, intent, trace_hash)`` always produces the
|
||||
identical id — the load-bearing replay property analogous to
|
||||
:func:`teaching.discovery._hash_candidate_id`.
|
||||
"""
|
||||
payload = json.dumps(
|
||||
{"token": token, "intent": intent, "source_turn_trace": trace_hash},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32]
|
||||
|
||||
|
||||
def format_oov_candidate_jsonl(candidate: OOVCandidate) -> str:
|
||||
"""Render a candidate as one canonical JSONL line."""
|
||||
return json.dumps(candidate.as_dict(), sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
class OOVCandidateSink(Protocol):
|
||||
"""Minimal sink contract — one JSONL line per emission."""
|
||||
|
||||
def emit(self, line: str) -> None: ...
|
||||
|
||||
|
||||
@dataclass
|
||||
class OOVBufferSink:
|
||||
"""In-memory sink that captures every emitted candidate line."""
|
||||
|
||||
lines: list[str] = field(default_factory=list)
|
||||
|
||||
def emit(self, line: str) -> None:
|
||||
self.lines.append(line)
|
||||
|
||||
|
||||
Clock = Callable[[], datetime]
|
||||
|
||||
|
||||
def _utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class OOVMonthlyFileSink:
|
||||
"""Append-only JSONL sink with monthly rollover.
|
||||
|
||||
Path is computed at each ``emit()`` from the injected clock as
|
||||
``<root>/<YYYY>/<YYYY-MM>.jsonl``. Same on-disk shape as
|
||||
:class:`teaching.discovery_sink.DiscoveryMonthlyFileSink` so the
|
||||
aggregator can reuse the file-walk machinery.
|
||||
"""
|
||||
|
||||
def __init__(self, root: str | Path, *, clock: Clock = _utc_now) -> None:
|
||||
self._root = Path(root)
|
||||
self._clock = clock
|
||||
self._fh: IO[str] | None = None
|
||||
self._current_path: Path | None = None
|
||||
|
||||
def _path_for_now(self) -> Path:
|
||||
now = self._clock()
|
||||
return self._root / f"{now.year:04d}" / f"{now.year:04d}-{now.month:02d}.jsonl"
|
||||
|
||||
def emit(self, line: str) -> None:
|
||||
target = self._path_for_now()
|
||||
if target != self._current_path:
|
||||
if self._fh is not None:
|
||||
self._fh.close()
|
||||
self._fh = None
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._fh = target.open("a", encoding="utf-8")
|
||||
self._current_path = target
|
||||
assert self._fh is not None
|
||||
self._fh.write(line)
|
||||
self._fh.write("\n")
|
||||
self._fh.flush()
|
||||
|
||||
def close(self) -> None:
|
||||
if self._fh is not None:
|
||||
self._fh.close()
|
||||
self._fh = None
|
||||
self._current_path = None
|
||||
|
||||
def __enter__(self) -> "OOVMonthlyFileSink":
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc_info) -> None:
|
||||
self.close()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"OOVCandidate",
|
||||
"OOVCandidateSink",
|
||||
"OOVBufferSink",
|
||||
"OOVMonthlyFileSink",
|
||||
"format_oov_candidate_jsonl",
|
||||
"hash_oov_candidate_id",
|
||||
]
|
||||
279
tests/test_oov_pipeline.py
Normal file
279
tests/test_oov_pipeline.py
Normal file
|
|
@ -0,0 +1,279 @@
|
|||
"""Phase 2.3 — OOV sink, aggregation, and auto-promotion tests.
|
||||
|
||||
The contract these tests pin:
|
||||
|
||||
- The runtime emits an ``OOVCandidate`` JSONL line to the attached
|
||||
sink on every turn whose ``grounding_source == "oov"``; no-op
|
||||
when no sink is attached.
|
||||
- The candidate_id is deterministic on (token, intent, trace_hash).
|
||||
- The aggregator groups by token, ranks by frequency, supports
|
||||
``--since YYYY-MM`` filtering.
|
||||
- The promoter respects the boundary-clean filter by default and
|
||||
refuses ``threshold < 1``.
|
||||
- The promotion suggests mounted packs but never names a single
|
||||
destination — domain inference is out of scope.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from chat.runtime import ChatRuntime
|
||||
from teaching.oov_gaps import OOVGap, aggregate_oov_gaps
|
||||
from teaching.oov_promotion import OOVPromotion, promote_oov_gaps
|
||||
from teaching.oov_sink import (
|
||||
OOVBufferSink,
|
||||
OOVCandidate,
|
||||
format_oov_candidate_jsonl,
|
||||
hash_oov_candidate_id,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sink contract
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_buffer_sink_captures_each_emit() -> None:
|
||||
sink = OOVBufferSink()
|
||||
sink.emit("one")
|
||||
sink.emit("two")
|
||||
assert sink.lines == ["one", "two"]
|
||||
|
||||
|
||||
def test_candidate_id_is_deterministic() -> None:
|
||||
a = hash_oov_candidate_id("photosynthesis", "definition", "trace-1")
|
||||
b = hash_oov_candidate_id("photosynthesis", "definition", "trace-1")
|
||||
assert a == b
|
||||
assert len(a) == 32
|
||||
|
||||
|
||||
def test_candidate_id_changes_with_token() -> None:
|
||||
a = hash_oov_candidate_id("photosynthesis", "definition", "trace-1")
|
||||
b = hash_oov_candidate_id("mitochondria", "definition", "trace-1")
|
||||
assert a != b
|
||||
|
||||
|
||||
def test_candidate_id_changes_with_trace() -> None:
|
||||
a = hash_oov_candidate_id("photosynthesis", "definition", "trace-1")
|
||||
b = hash_oov_candidate_id("photosynthesis", "definition", "trace-2")
|
||||
assert a != b
|
||||
|
||||
|
||||
def test_candidate_jsonl_is_sorted_compact() -> None:
|
||||
cand = OOVCandidate(
|
||||
candidate_id="x",
|
||||
token="photosynthesis",
|
||||
intent="definition",
|
||||
trigger="unresolved_subject",
|
||||
source_turn_trace="t",
|
||||
boundary_clean=True,
|
||||
)
|
||||
line = format_oov_candidate_jsonl(cand)
|
||||
parsed = json.loads(line)
|
||||
assert parsed["token"] == "photosynthesis"
|
||||
assert parsed["intent"] == "definition"
|
||||
assert parsed["boundary_clean"] is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Runtime integration — sink receives one line per OOV turn
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_runtime_emits_when_oov_sink_attached() -> None:
|
||||
rt = ChatRuntime()
|
||||
sink = OOVBufferSink()
|
||||
rt.attach_oov_sink(sink)
|
||||
rt.chat("What is photosynthesis?")
|
||||
assert len(sink.lines) == 1
|
||||
parsed = json.loads(sink.lines[0])
|
||||
assert parsed["token"] == "photosynthesis"
|
||||
assert parsed["intent"] == "definition"
|
||||
assert parsed["trigger"] == "unresolved_subject"
|
||||
|
||||
|
||||
def test_runtime_does_not_emit_without_sink() -> None:
|
||||
"""Sink emission is opt-in; runtime behaviour is identical when
|
||||
no sink is attached."""
|
||||
rt = ChatRuntime()
|
||||
resp = rt.chat("What is photosynthesis?")
|
||||
# OOV surface still fires (P2.1 is unconditional), but nothing
|
||||
# is persisted anywhere — there is no sink to receive it.
|
||||
assert resp.grounding_source == "oov"
|
||||
|
||||
|
||||
def test_runtime_does_not_emit_on_known_lemma() -> None:
|
||||
rt = ChatRuntime()
|
||||
sink = OOVBufferSink()
|
||||
rt.attach_oov_sink(sink)
|
||||
rt.chat("What is light?")
|
||||
assert sink.lines == []
|
||||
|
||||
|
||||
def test_runtime_emits_across_intent_shapes() -> None:
|
||||
"""Every intent shape that triggers OOV (definition, cause,
|
||||
verification, comparison, procedure) emits a candidate."""
|
||||
rt = ChatRuntime()
|
||||
sink = OOVBufferSink()
|
||||
rt.attach_oov_sink(sink)
|
||||
rt.chat("What is photosynthesis?")
|
||||
intents = set()
|
||||
for line in sink.lines:
|
||||
intents.add(json.loads(line)["intent"])
|
||||
assert "definition" in intents
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Aggregator — file walking + deterministic ordering
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _write_oov_line(path: Path, **kwargs) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {
|
||||
"candidate_id": kwargs.get("candidate_id", "x"),
|
||||
"token": kwargs.get("token", "photosynthesis"),
|
||||
"intent": kwargs.get("intent", "definition"),
|
||||
"trigger": "unresolved_subject",
|
||||
"source_turn_trace": kwargs.get("trace", "t"),
|
||||
"boundary_clean": kwargs.get("boundary_clean", True),
|
||||
"review_state": "unreviewed",
|
||||
}
|
||||
with path.open("a", encoding="utf-8") as fh:
|
||||
fh.write(json.dumps(payload, sort_keys=True, separators=(",", ":")))
|
||||
fh.write("\n")
|
||||
|
||||
|
||||
def test_aggregates_by_token(tmp_path: Path) -> None:
|
||||
sink = tmp_path / "2026" / "2026-05.jsonl"
|
||||
_write_oov_line(sink, candidate_id="a", token="photosynthesis", intent="definition")
|
||||
_write_oov_line(sink, candidate_id="b", token="photosynthesis", intent="cause")
|
||||
_write_oov_line(sink, candidate_id="c", token="mitochondria", intent="definition")
|
||||
|
||||
rows = aggregate_oov_gaps(tmp_path)
|
||||
assert len(rows) == 2
|
||||
photo = next(g for g in rows if g.token == "photosynthesis")
|
||||
assert photo.count == 2
|
||||
assert photo.intents == ("cause", "definition")
|
||||
assert photo.boundary_clean_count == 2
|
||||
|
||||
|
||||
def test_rank_order_is_count_desc(tmp_path: Path) -> None:
|
||||
sink = tmp_path / "2026" / "2026-05.jsonl"
|
||||
for i in range(3):
|
||||
_write_oov_line(sink, candidate_id=f"a{i}", token="photosynthesis")
|
||||
_write_oov_line(sink, candidate_id="b0", token="mitochondria")
|
||||
rows = aggregate_oov_gaps(tmp_path)
|
||||
assert [g.token for g in rows] == ["photosynthesis", "mitochondria"]
|
||||
|
||||
|
||||
def test_tainted_counted_but_split(tmp_path: Path) -> None:
|
||||
sink = tmp_path / "2026" / "2026-05.jsonl"
|
||||
_write_oov_line(sink, candidate_id="a", boundary_clean=True)
|
||||
_write_oov_line(sink, candidate_id="b", boundary_clean=False)
|
||||
rows = aggregate_oov_gaps(tmp_path)
|
||||
assert rows[0].count == 2
|
||||
assert rows[0].boundary_clean_count == 1
|
||||
|
||||
|
||||
def test_since_filter(tmp_path: Path) -> None:
|
||||
_write_oov_line(tmp_path / "2026" / "2026-04.jsonl", candidate_id="april")
|
||||
_write_oov_line(tmp_path / "2026" / "2026-05.jsonl", candidate_id="may")
|
||||
rows = aggregate_oov_gaps(tmp_path, since="2026-05")
|
||||
assert len(rows) == 1
|
||||
assert rows[0].sample_candidate_ids == ("may",)
|
||||
|
||||
|
||||
def test_malformed_lines_skipped(tmp_path: Path) -> None:
|
||||
sink = tmp_path / "2026" / "2026-05.jsonl"
|
||||
sink.parent.mkdir(parents=True, exist_ok=True)
|
||||
sink.write_text(
|
||||
"not json\n{}\n" + json.dumps({
|
||||
"candidate_id": "ok", "token": "photosynthesis",
|
||||
"intent": "definition", "trigger": "unresolved_subject",
|
||||
"source_turn_trace": "t", "boundary_clean": True,
|
||||
}) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
rows = aggregate_oov_gaps(tmp_path)
|
||||
assert len(rows) == 1
|
||||
|
||||
|
||||
def test_aggregator_missing_root_returns_empty(tmp_path: Path) -> None:
|
||||
assert aggregate_oov_gaps(tmp_path / "does_not_exist") == ()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Promotion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _gap(token: str, count: int = 3, clean: int | None = None) -> OOVGap:
|
||||
return OOVGap(
|
||||
token=token,
|
||||
intents=("definition",),
|
||||
count=count,
|
||||
boundary_clean_count=count if clean is None else clean,
|
||||
sample_candidate_ids=("a", "b"),
|
||||
months_seen=("2026-05",),
|
||||
)
|
||||
|
||||
|
||||
def test_promotion_respects_threshold() -> None:
|
||||
gaps = (_gap("photosynthesis", count=5, clean=5),)
|
||||
promoted = promote_oov_gaps(gaps, threshold=3)
|
||||
assert len(promoted) == 1
|
||||
assert promoted[0].token == "photosynthesis"
|
||||
|
||||
|
||||
def test_promotion_excludes_below_threshold() -> None:
|
||||
gaps = (_gap("rare", count=1, clean=1),)
|
||||
assert promote_oov_gaps(gaps, threshold=3) == ()
|
||||
|
||||
|
||||
def test_promotion_excludes_tainted_only_by_default() -> None:
|
||||
gaps = (_gap("forbidden", count=5, clean=0),)
|
||||
assert promote_oov_gaps(gaps, threshold=3) == ()
|
||||
|
||||
|
||||
def test_include_tainted_counts_all() -> None:
|
||||
gaps = (_gap("forbidden", count=5, clean=0),)
|
||||
promoted = promote_oov_gaps(gaps, threshold=3, include_tainted=True)
|
||||
assert len(promoted) == 1
|
||||
|
||||
|
||||
def test_threshold_must_be_positive() -> None:
|
||||
with pytest.raises(ValueError):
|
||||
promote_oov_gaps((_gap("photosynthesis"),), threshold=0)
|
||||
|
||||
|
||||
def test_queue_id_format() -> None:
|
||||
promoted = promote_oov_gaps((_gap("photosynthesis", count=5, clean=5),), threshold=3)
|
||||
assert promoted[0].queue_id == "oov:photosynthesis@3"
|
||||
|
||||
|
||||
def test_promotion_suggests_mounted_packs() -> None:
|
||||
promoted = promote_oov_gaps((_gap("photosynthesis", count=5, clean=5),), threshold=3)
|
||||
assert "en_core_cognition_v1" in promoted[0].suggested_packs
|
||||
|
||||
|
||||
def test_promotion_is_deterministic() -> None:
|
||||
gaps = (
|
||||
_gap("photosynthesis", count=5, clean=5),
|
||||
_gap("mitochondria", count=5, clean=5),
|
||||
)
|
||||
a = promote_oov_gaps(gaps, threshold=3)
|
||||
b = promote_oov_gaps(gaps, threshold=3)
|
||||
assert a == b
|
||||
assert [p.token for p in a] == ["mitochondria", "photosynthesis"]
|
||||
|
||||
|
||||
def test_promotion_does_not_mutate_input() -> None:
|
||||
gaps = (_gap("photosynthesis", count=3, clean=3),)
|
||||
snapshot = gaps[0]
|
||||
promote_oov_gaps(gaps, threshold=3)
|
||||
assert gaps[0] == snapshot
|
||||
Loading…
Reference in a new issue