From 7cc2888ed23e11567034e35d76bb965c1fb31fd3 Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 19 May 2026 22:35:09 -0700 Subject: [PATCH] =?UTF-8?q?feat(coherence):=20ADR-0075=20=E2=80=94=20reali?= =?UTF-8?q?zer=20slot-type=20guard=20(C1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit C1 coherence floor: a deterministic verifier that runs on every candidate surface produced by the truth path, before assignment to ChatResponse.surface. Rejects illegal articulations and routes them to a bounded disclosure string — admission control with a deterministic fallback, not normalization. Active rules (R1 deferred during ratification — see ADR): R2_aux_neg_requires_verb — " not " rejected R3_be_neg_requires_predicate — " not " rejected Fail-open on unknown POS, fail-closed on explicit wrong POS. Cognition eval byte-identical (100/91.7/100/100). Original bug class — "Light reveals truth, right?" → "Right does not thought." — now routes to "I do not have a reviewed articulation for that yet." with grounding_source=none, walk_surface preserving the rejected candidate, and telemetry carrying R2_aux_neg_requires_verb. Files: generate/realizer_guard.py NEW — pure verifier chat/runtime.py hook on stub + main paths chat/telemetry.py serialize guard fields core/physics/identity.py TurnEvent +2 fields evals/realizer_guard/run_holdout.py NEW — 6-prompt cluster tests/test_realizer_guard_*.py NEW — 46 tests (unit/seam/holdout) docs/decisions/ADR-0075-*.md NEW — ratified Invariants pinned: invariant_realizer_no_illegal_articulation invariant_realizer_guard_byte_identity_on_currently_passing_cases Lanes (excluding 1 pre-existing TestDemoPreambles failure unrelated to C1, already present at 4426f38): smoke 67/67 cognition 120/120(+1s) teaching 17/17 packs 6/6 runtime 19/19 algebra 132/132 full 2792/2793 --- chat/runtime.py | 63 ++- chat/telemetry.py | 4 + core/physics/identity.py | 12 + .../ADR-0075-realizer-slot-type-guard.md | 384 ++++++++++++++++++ evals/realizer_guard/__init__.py | 6 + evals/realizer_guard/run_holdout.py | 151 +++++++ generate/realizer_guard.py | 244 +++++++++++ tests/test_realizer_guard_holdout.py | 155 +++++++ tests/test_realizer_guard_runtime_seam.py | 200 +++++++++ tests/test_realizer_guard_unit.py | 211 ++++++++++ 10 files changed, 1428 insertions(+), 2 deletions(-) create mode 100644 docs/decisions/ADR-0075-realizer-slot-type-guard.md create mode 100644 evals/realizer_guard/__init__.py create mode 100644 evals/realizer_guard/run_holdout.py create mode 100644 generate/realizer_guard.py create mode 100644 tests/test_realizer_guard_holdout.py create mode 100644 tests/test_realizer_guard_runtime_seam.py create mode 100644 tests/test_realizer_guard_unit.py diff --git a/chat/runtime.py b/chat/runtime.py index 689091a4..77f4185f 100644 --- a/chat/runtime.py +++ b/chat/runtime.py @@ -57,6 +57,10 @@ from packs.ethics.loader import ( ) from packs.identity.loader import load_identity_manifold from chat.register_variation import decorate_surface +from generate.realizer_guard import ( + DISCLOSURE_SURFACE as _GUARD_DISCLOSURE_SURFACE, + check_surface as _check_realizer_surface, +) from packs.anchor_lens.loader import AnchorLens, load_anchor_lens from packs.register.loader import RegisterPack, load_register_pack from packs.safety.check import SafetyCheck, SafetyContext @@ -343,6 +347,12 @@ class ChatResponse: # telemetry JSONL. ``""`` defaults preserve pre-L1.4 byte-identity. anchor_lens_id: str = "" anchor_lens_mode_label: str = "" + # ADR-0075 (C1) — realizer slot-type guard verdict. Mirrors the + # TurnEvent fields so callers (CLI, demos, tests) can read the + # guard state from ChatResponse without re-parsing the telemetry + # JSONL. ``""`` defaults preserve pre-C1 byte-identity. + realizer_guard_status: str = "" + realizer_guard_rule: str = "" class ChatRuntime: @@ -983,6 +993,28 @@ class ChatRuntime: grounding_source = grounded_source_tag else: grounding_source = "none" + # ADR-0075 (C1) — realizer slot-type guard. Runs BEFORE + # register decoration so a register cannot accidentally heal + # an illegal articulation by wrapping it, and BEFORE anchor- + # lens annotation extraction so the lens annotation never + # rides on a guard-rejected surface. On rejection, route to + # the bounded disclosure string and force grounding_source to + # ``"none"`` (an illegal surface is ungrounded by construction). + # The pre-guard candidate is preserved on walk_surface_stub + # for telemetry — the stub path normally leaves walk_surface as + # _UNKNOWN_DOMAIN_SURFACE, so this swap strictly increases + # observability under rejection. + guard_verdict_stub = _check_realizer_surface( + response_surface, + pos_lookup=self._pos_by_surface.get, + ) + realizer_guard_status_stub = guard_verdict_stub.status + realizer_guard_rule_stub = guard_verdict_stub.rule_id + walk_surface_stub = _UNKNOWN_DOMAIN_SURFACE + if guard_verdict_stub.status == "rejected": + walk_surface_stub = response_surface + response_surface = _GUARD_DISCLOSURE_SURFACE + grounding_source = "none" # ADR-0071 (R4) — apply seeded discourse-marker decoration to # the realized surface AFTER grounding source is decided. # Empty marker buckets ⇒ no-op (UNREGISTERED / neutral / terse). @@ -1024,7 +1056,7 @@ class ChatRuntime: turn=max(self._context.turn - 1, 0), input_tokens=tokens, surface=response_surface, - walk_surface=_UNKNOWN_DOMAIN_SURFACE, + walk_surface=walk_surface_stub, articulation_surface=_UNKNOWN_DOMAIN_SURFACE, dialogue_role="assert", identity_score=None, @@ -1041,6 +1073,8 @@ class ChatRuntime: register_variant_id=decoration_stub.variant_id, anchor_lens_id=anchor_lens_id_stub, anchor_lens_mode_label=anchor_lens_mode_label_stub, + realizer_guard_status=realizer_guard_status_stub, + realizer_guard_rule=realizer_guard_rule_stub, ) self.turn_log.append(stub_event) self._emit_turn_event(stub_event) @@ -1073,7 +1107,7 @@ class ChatRuntime: versor_condition=versor_condition(field_state.F), output_language=self.config.output_language, frame_pack=self.config.frame_pack, - walk_surface=_UNKNOWN_DOMAIN_SURFACE, + walk_surface=walk_surface_stub, salience_top_k=None, candidates_used=None, vault_hits=0, @@ -1089,6 +1123,8 @@ class ChatRuntime: register_variant_id=decoration_stub.variant_id, anchor_lens_id=anchor_lens_id_stub, anchor_lens_mode_label=anchor_lens_mode_label_stub, + realizer_guard_status=realizer_guard_status_stub, + realizer_guard_rule=realizer_guard_rule_stub, ) def chat(self, text: str, max_tokens: int | None = None) -> ChatResponse: @@ -1350,6 +1386,25 @@ class ChatRuntime: before = response_surface response_surface = inject_hedge(response_surface, hedge_prefix) hedge_injected = response_surface != before + # ADR-0075 (C1) — realizer slot-type guard (main path). Runs + # AFTER all composer / planner / hedge transformations and + # BEFORE register decoration so a single seam covers every + # articulation path. On rejection: surface is replaced with + # the bounded disclosure string, grounding_source forced to + # ``"none"``, and walk_surface preserves the rejected + # candidate so the manifold-walk evidence is overwritten only + # in the rejection branch (the contract says illegal + # articulation evidence is the relevant telemetry). + guard_verdict_main = _check_realizer_surface( + response_surface, + pos_lookup=self._pos_by_surface.get, + ) + realizer_guard_status_main = guard_verdict_main.status + realizer_guard_rule_main = guard_verdict_main.rule_id + if guard_verdict_main.status == "rejected": + walk_surface = response_surface + response_surface = _GUARD_DISCLOSURE_SURFACE + warm_grounding_source = "none" # ADR-0071 (R4) — seeded discourse-marker decoration is the # last step before TurnEvent is sealed. Applies uniformly to # every grounding path (vault / pack / teaching / planner / @@ -1406,6 +1461,8 @@ class ChatRuntime: register_variant_id=decoration_main.variant_id, anchor_lens_id=anchor_lens_id_main, anchor_lens_mode_label=anchor_lens_mode_label_main, + realizer_guard_status=realizer_guard_status_main, + realizer_guard_rule=realizer_guard_rule_main, ) self.turn_log.append(turn_event) self._emit_turn_event(turn_event) @@ -1443,6 +1500,8 @@ class ChatRuntime: register_variant_id=decoration_main.variant_id, anchor_lens_id=anchor_lens_id_main, anchor_lens_mode_label=anchor_lens_mode_label_main, + realizer_guard_status=realizer_guard_status_main, + realizer_guard_rule=realizer_guard_rule_main, ) def _unknown_domain_response(self, field_state: FieldState, filtered: list[str]) -> ChatResponse: diff --git a/chat/telemetry.py b/chat/telemetry.py index 32804b0f..68515c7e 100644 --- a/chat/telemetry.py +++ b/chat/telemetry.py @@ -79,6 +79,10 @@ def serialize_turn_event( # turns where the lens did not engage. "anchor_lens_id": str(getattr(event, "anchor_lens_id", "") or ""), "anchor_lens_mode_label": str(getattr(event, "anchor_lens_mode_label", "") or ""), + # ADR-0075 (C1) — realizer slot-type guard verdict per turn. + # Empty strings on pre-C1 events; closed enums otherwise. + "realizer_guard_status": str(getattr(event, "realizer_guard_status", "") or ""), + "realizer_guard_rule": str(getattr(event, "realizer_guard_rule", "") or ""), } safety = getattr(event, "safety_verdict", None) if safety is not None: diff --git a/core/physics/identity.py b/core/physics/identity.py index 57efe939..ec10a5cb 100644 --- a/core/physics/identity.py +++ b/core/physics/identity.py @@ -312,3 +312,15 @@ class TurnEvent: # ``""`` so pre-L1.4 callers stay byte-identical. anchor_lens_id: str = "" anchor_lens_mode_label: str = "" + # ADR-0075 (C1) — realizer slot-type guard verdict per turn. + # ``realizer_guard_status`` is ``"ok"`` when the guard accepted + # the candidate surface, ``"rejected"`` when one of R1/R2/R3 + # fired and the runtime routed the surface to the bounded + # disclosure string, or ``""`` on pre-C1 events that pre-date + # the guard hook. ``realizer_guard_rule`` carries the rule_id + # (one of ``"R1_no_finite_verb"``, ``"R2_aux_neg_requires_verb"``, + # ``"R3_be_neg_requires_predicate"``) when status is + # ``"rejected"``, otherwise ``""``. Both default to ``""`` so + # pre-C1 callers stay byte-identical. + realizer_guard_status: str = "" + realizer_guard_rule: str = "" diff --git a/docs/decisions/ADR-0075-realizer-slot-type-guard.md b/docs/decisions/ADR-0075-realizer-slot-type-guard.md new file mode 100644 index 00000000..63b7445b --- /dev/null +++ b/docs/decisions/ADR-0075-realizer-slot-type-guard.md @@ -0,0 +1,384 @@ +# ADR-0075 — Realizer slot-type guard (C1: coherence floor) + +**Status:** Accepted +**Date:** 2026-05-19 +**Ratified:** 2026-05-19 +**Author:** Shay +**Builds on:** [ADR-0048](./ADR-0048-pack-grounded-surface.md) +(pack-grounded surface), [ADR-0049](./ADR-0049-intent-subject-extraction.md) +(intent subject extraction), [ADR-0053](./ADR-0053-cognition-lane-closure.md) +(cognition lane closure) +**Series:** C1 — Coherence floor (precedes C2 confirmation-tag normalization, +R6 substantive register knobs, T1 curated teaching depth) + +--- + +## Context + +The orthogonality tour landed `4426f38` with `all_claims_supported=true` +across the full 3 × 3 × 2 grid. But inspection of the grid surfaced a +real coherence regression on a fourth prompt under live demo: + +```text +"Light reveals truth, right?" → "Right does not thought." +``` + +That surface is grammatically illegal: the verb slot of a +` does not ` construction is occupied by a bare noun +(`thought`, which is a noun in `en_core_cognition_v1`). All three +registers reproduced this illegal form byte-identically, which is the +correct behavior for the register axis (it preserves whatever the +truth path produced) but reveals that the truth path itself is +licensing surfaces that no realizer template should be able to emit. + +The doctrine in CLAUDE.md is explicit: + +> Code and tests should make illegal states difficult to represent. +> Prefer inspectable state, provenance, and deterministic replay over +> impressive-looking but ungrounded outputs. + +C1 addresses this at the construction boundary — a deterministic +slot-type verifier that rejects illegal articulations before they can +escape to the user surface, regardless of which composer or intent +path produced them. This is the coherence floor that C2 +(confirmation-tag normalization), R6 (substantive register knobs), +and T1 (curated teaching) all rest on. + +### Why this comes first + +If C2 is fixed before C1, the specific bug above goes away but the +underlying property — "no illegal articulation ever escapes" — is +not established. The next intent/extraction bug will produce a new +word-salad surface and the cycle repeats. C1 establishes the +invariant; C2 fixes the specific input class that exposed it. + +If R6 ships before C1, terse and convivial registers will start +emitting substantively distinct surfaces on top of a path that can +still produce word salad. Worse cosmetics on the same broken floor. + +If T1 ships before C1, the reviewed corpus inherits the illegal +articulation pattern wherever it touches a path the guard would have +caught — the corpus becomes a vector for the bug, not a fix for it. + +--- + +## Decision + +### Mechanism + +A pure verifier `generate/realizer_guard.py` that runs on every +candidate surface produced by `pack_grounded_surface`, +`pack_grounded_comparison_surface`, +`pack_grounded_correction_surface`, +`pack_grounded_procedure_surface`, the cross-pack chain composer, +NARRATIVE / EXAMPLE composers, and any vault-grounded realizer +output, before that surface is assigned to `ChatResponse.surface` or +emitted on `walk_surface`. + +The guard is deterministic, reads pack POS data already loaded at +runtime, performs a single linear pass, and emits a verdict: + +```python +@dataclass(frozen=True) +class RealizerGuardVerdict: + status: Literal["ok", "rejected"] + rule_id: str # e.g. "R1_no_finite_verb"; "" when ok + detail: str # surface fragment that violated the rule +``` + +### Slot-type rules (C1 active scope — two rules) + +``` +R2_aux_neg_requires_verb: + If the surface contains a do-support negation + ("does not" / "do not" / "did not"), the immediately following + content token must have POS = VERB per pack lookup. + Adverbs between the negation and the verb are allowed + ("does not always reveal" — "always" is skipped, then "reveal" + must be VERB). + +R3_be_neg_requires_predicate: + If the surface contains a be-negation + ("is not" / "are not" / "was not" / "were not"), the immediately + following content token must have POS ∈ {NOUN, ADJ, DET, ADV, + PRON} per pack lookup. Bare verbs after be-negation are + illegal ("is not reveal" is rejected). +``` + +Both rules fire on **presence of an illegal pattern**, not on +**absence of an expected token** — high precision, low false-positive +rate. They cover the observed failure mode and the symmetric form. + +### Deferred during ratification + +``` +R1_no_finite_verb: + Every emitted clause must contain at least one finite verb token. + Deferred from active C1 scope: the cognition pack's POS coverage + does not include every English finite verb the teaching-chain + realizer emits (notably "requires" and "makes"), and switching + R1 on regresses currently-passing cases — a violation of the + byte-identity invariant the ADR explicitly flags as the canary. + R1's intent is preserved for a follow-up coherence ADR that + either broadens pack POS coverage or adds a closed English- + vocabulary POS table. + +Subject / object / predicate pack-residency: + Further rules (subject-pack-residency, object-pack-residency, + proposition-graph-residency) are deliberately deferred to a + follow-up coherence ADR — C1 is a narrow floor, not a parser. +``` + +### Failure routing — fail closed, not silent + +When the guard returns `status == "rejected"` on a candidate surface, +the truth path: + +1. **Does not** silently drop the surface. +2. **Does not** retry the realizer with different parameters. +3. **Does** replace the surface with a deterministic bounded + disclosure string: + +```text +"I do not have a reviewed articulation for that yet." +``` + +4. **Does** preserve the pre-guard surface on `walk_surface` for + telemetry / debugging (consistent with the existing + `surface` ≠ `walk_surface` runtime contract). +5. **Does** flag `grounding_source` as `"none"` regardless of what + the rejected candidate had claimed (an illegal surface is + ungrounded by construction). + +### Telemetry surface + +`TurnEvent` gains two fields, default empty: + +``` +realizer_guard_status : "" | "ok" | "rejected" +realizer_guard_rule : "" | "R1_no_finite_verb" | "R2_aux_neg_requires_verb" + | "R3_be_neg_requires_predicate" +``` + +`serialize_turn_event` emits both fields alphabetically, deterministic. + +`ChatResponse` gains the same two fields, propagated identically on +stub and main paths (mirrors `register_id` / `anchor_lens_id` +threading pattern). + +### Hook point + +Single seam in `chat/runtime.py`, immediately after the composer +returns a candidate surface and before the surface is assigned to +`ChatResponse.surface` or to the `pre_decoration_surface_*` slot that +feeds the register decoration step. This placement matters: + +* Guard runs **before** register decoration, so the + register-decorated form is never the verifier's responsibility. +* Guard runs **before** anchor-lens annotation extraction, so the + `[lens(...):...]` annotation is never on a guard-rejected surface. +* Guard runs **after** all composers, so a single seam catches + every articulation path. + +### Files + +``` +generate/realizer_guard.py NEW + - RealizerGuardVerdict frozen dataclass + - check_surface(surface: str, *, pack_lexicons: Mapping) + -> RealizerGuardVerdict + - Three pure rule helpers; no I/O; no mutation. + +chat/runtime.py EDIT + - Hook check_surface() between composer return and surface assignment + - Failure routing: surface = bounded disclosure; walk_surface = raw + rejected candidate; grounding_source = "none"; populate guard + telemetry fields on TurnEvent + ChatResponse. + +core/physics/identity.py EDIT + - TurnEvent.realizer_guard_status: str = "" + - TurnEvent.realizer_guard_rule: str = "" + +chat/telemetry.py EDIT + - serialize_turn_event: emit both fields alphabetically. + +evals/realizer_guard/__init__.py NEW +evals/realizer_guard/run_holdout.py NEW + - Six confirmation-tag / illegal-articulation prompts that + currently produce illegal surfaces. Under C1 alone (without + C2), every one routes to bounded disclosure with + realizer_guard_status="rejected" and a recorded rule_id. + - Exit code 0 iff invariant_realizer_no_illegal_articulation holds + across the cluster. + +tests/test_realizer_guard_unit.py NEW + - 22 cases: R2 / R3 (each negation form, copula handling, adverb + skipping, end-of-surface, unknown lemma fail-closed); R1 + deferred (the verb-less cases pin current pass-through behavior + so re-enablement is intentional). + +tests/test_realizer_guard_runtime_seam.py NEW + - Stub + main path: rejected candidate → bounded disclosure on + surface, raw candidate on walk_surface, grounding_source="none", + telemetry fields populated. + - AST seam: realizer_guard imports no truth-path modules. + +tests/test_realizer_guard_holdout.py NEW + - Holdout cluster: six confirmation-tag / illegal-articulation + prompts; every one rejected with expected rule_id. + - Cognition eval cases: zero rejections (byte-identical baseline). + +docs/decisions/ADR-0075-realizer-slot-type-guard.md NEW (this file) +``` + +No pack mutation. No composer rewrites. No new dynamic imports. +No new filesystem writes. No CLI surface changes (telemetry only). + +### Invariants pinned + +``` +invariant_realizer_no_illegal_articulation (NEW): + For every prompt in the cognition eval lane + every prompt in the + C1 holdout cluster + every demo grid cell in register-tour / + anchor-lens-tour / orthogonality-tour, the surface emitted on + ChatResponse.surface satisfies R2 and R3. + Holdout cluster surfaces are bounded disclosure strings (rejection + is the correct behavior for that input class under C1 alone). + +invariant_realizer_guard_byte_identity_on_currently_passing_cases (NEW): + For every cognition eval case that currently produces a legal + articulation, the post-C1 surface is byte-identical to the pre-C1 + surface. The guard must not regress any legal articulation. + +register-tour invariant — preserved +anchor-lens-tour invariant — preserved +orthogonality-tour invariant — preserved +anchor_lens_byte_identity_null_lift — preserved +register_invariant_grounding — preserved +``` + +The first invariant is the load-bearing one — it is the formal +statement that illegal articulations cannot escape, regardless of +which composer or intent path produced them. + +--- + +## Consequences + +### Capability unlocked + +Word-salad surfaces (`"Right does not thought."` and its symmetric +forms) cannot escape to the user, even when intent classification or +subject extraction is wrong. Bugs in upstream stages now degrade +gracefully to a bounded disclosure surface rather than to ungrounded +articulation. This is the precondition that lets C2 / R6 / T1 ship +safely. + +### Cognition lane + +Expected byte-identical on currently-passing cases (invariant +`invariant_realizer_guard_byte_identity_on_currently_passing_cases` +is part of the CI gate). Any case that the guard rejects under C1 +is, by definition, a case where the prior surface was illegal — +those move to bounded disclosure and lose their `term_capture` / +`surface_groundedness` credit until C2 + T1 land the proper fix. + +If any currently-passing case regresses, the rule is too aggressive +and must be narrowed before merge. The byte-identity invariant is +the canary. + +### Performance + +Single linear pass over the candidate surface tokens with a +dictionary lookup per content token. O(n) on surface length, runs +once per turn. Well below the per-turn latency floor measured in +the teaching-loop bench (mean 1.85s). No new caches. + +### Trust boundaries + +* **Construction-boundary verifier, not hot-path repair.** The + guard rejects and reroutes; it never edits the candidate surface. + This keeps it firmly outside the "forbidden normalization sites" + list in CLAUDE.md (it is not normalization — it is admission + control with a deterministic fallback). +* **No filesystem, no dynamic import, no shell.** The guard reads + pack lexicons already loaded at runtime. +* **No user-text mutation path.** Guard output is structural + (status + rule_id + detail); the bounded disclosure string is a + module-level constant, never composed from user input. +* **AST seam test** in `tests/test_realizer_guard_runtime_seam.py` + refuses truth-path imports of `generate.realizer_guard` from any + module other than `chat/runtime.py` — the hook is centralized, + not sprinkled. +* **Telemetry redaction unchanged.** `realizer_guard_rule` is a + rule_id from a closed set; `realizer_guard_status` is a closed + enum. Neither field can leak user content. + +### Replay determinism + +Guard is a pure function of `(surface_string, pack_lexicons)`. +Pack lexicons are deterministic from manifest checksums. Therefore +guard verdicts are deterministic and replay-equivalent across runs. +This composes with the existing trace_hash discipline: a rejected +turn's trace_hash is computed on the pre-decoration **disclosure** +surface (the user-facing surface), which keeps replay equivalence +intact. + +### What this explicitly does *not* do + +* Does not fix the confirmation-tag intent bug. That is C2. +* Does not change register substantive consumption. That is R6. +* Does not add new teaching chains. That is T1. +* Does not extend the rule set beyond R1/R2/R3. Subject-pack- + residency, object-pack-residency, and proposition-graph residency + are deferred to a follow-up ADR after C1 + C2 land and the failure + modes are re-surveyed. +* Does not retry or refine on rejection. Fail closed, route to + disclosure, surface the verdict to telemetry, move on. + +--- + +## Verification + +``` +python -m pytest tests/test_realizer_guard_unit.py -q N passed +python -m pytest tests/test_realizer_guard_runtime_seam.py -q N passed +python -m pytest tests/test_realizer_guard_holdout.py -q N passed + +python -m evals.realizer_guard.run_holdout exit 0 +core eval cognition byte-identical + on currently + passing cases + +Curated lanes (must remain green): + smoke / cognition / teaching / packs / runtime / algebra / full + +Existing demos continue to pass: + core demo register-tour exit 0 + core demo anchor-lens-tour exit 0 + core demo orthogonality-tour exit 0 +``` + +The C1 holdout cluster's exit code is the canonical coherence-floor +gate — if any illegal articulation can escape, it exits non-zero. + +--- + +## Follow-ups (not in C1 scope) + +* **C2** — Confirmation-tag normalization. Make + `"X reveals Y, right?"` preserve the propositional content + (`subject=X, relation=reveals, object=Y`, intent=VERIFICATION) + so the holdout cluster moves from "correctly rejected" to + "correctly articulated". Eval cases re-pin under C2. +* **R6** — Substantive register knobs on pack-grounded composers, + plus replacement of the weak `surfaces_vary_at_least_once` + register-tour gate with per-register substantive distinctness. +* **T1** — Curated teaching depth for the core epistemic vocabulary + appearing in the live demos (light / truth / knowledge / evidence + / wisdom / recall), through the existing + `core teaching propose → review → accept` flywheel. +* **Coherence rule extensions** — subject-pack-residency, + object-pack-residency, proposition-graph residency. Deferred + until the C1 / C2 failure surface is observed in CI for at least + one full eval cycle. diff --git a/evals/realizer_guard/__init__.py b/evals/realizer_guard/__init__.py new file mode 100644 index 00000000..4879e138 --- /dev/null +++ b/evals/realizer_guard/__init__.py @@ -0,0 +1,6 @@ +"""ADR-0075 (C1) holdout cluster — illegal articulations that +should be rejected by the realizer slot-type guard. + +Exit code 0 iff every illegal candidate is rejected with the +expected rule_id. See ``run_holdout.py``. +""" diff --git a/evals/realizer_guard/run_holdout.py b/evals/realizer_guard/run_holdout.py new file mode 100644 index 00000000..4376bdbd --- /dev/null +++ b/evals/realizer_guard/run_holdout.py @@ -0,0 +1,151 @@ +"""C1 holdout cluster — illegal-articulation prompts that the +ADR-0075 realizer slot-type guard must reject. + +Each prompt is run through ``CognitiveTurnPipeline`` and the +recorded ``TurnEvent`` is checked for: + +* ``realizer_guard_status == "rejected"`` +* ``realizer_guard_rule == `` +* ``surface == DISCLOSURE_SURFACE`` +* ``walk_surface`` carries the pre-guard candidate (non-empty, + not the disclosure string itself) +* ``grounding_source == "none"`` + +The cluster is reached by priming the vault with three pack-known +DEFINITION prompts first, which is the same sequence that exposed +the original bug in the orthogonality tour. This is necessary +because the bug surfaces only when the truth path reaches the +vault-grounded realizer with cross-turn evidence — a fresh runtime +on the bug prompt alone routes to the stub path and never produces +the illegal articulation. + +Exit code 0 iff ``all_claims_supported`` is true. +""" + +from __future__ import annotations + +import json +import sys +from typing import Any + +from chat.runtime import ChatRuntime +from core.cognition.pipeline import CognitiveTurnPipeline +from core.config import RuntimeConfig +from generate.realizer_guard import DISCLOSURE_SURFACE + + +_PRIMING_PROMPTS: tuple[str, ...] = ( + "What is light?", + "Define knowledge.", + "What is truth?", +) + + +# Each cluster entry is (prompt, expected_rule_id). +# +# The cluster covers the observed bug class: confirmation-tag +# discourse particles (``right`` / ``no`` / ``yes``) that steer the +# realizer to emit `` does not .`` — an illegal +# do-support negation with a noun in the verb slot. +# +# All six prompts run on freshly-primed isolated runtimes (no shared +# vault state across prompts) so each cell is order-independent. +_HOLDOUT_PROMPTS: tuple[tuple[str, str], ...] = ( + ("Light reveals truth, right?", "R2_aux_neg_requires_verb"), + ("Light reveals truth, no?", "R2_aux_neg_requires_verb"), + ("Light reveals truth, yes?", "R2_aux_neg_requires_verb"), + ("Knowledge supports truth, right?", "R2_aux_neg_requires_verb"), + ("Light grounds truth, right?", "R2_aux_neg_requires_verb"), + ("Light supports truth, right?", "R2_aux_neg_requires_verb"), +) + + +def _build_runtime() -> ChatRuntime: + """Fresh runtime with neutral register and unanchored lens — + the bug reproduces on the truth path regardless of register + or lens, so we pick the simplest combination.""" + return ChatRuntime(config=RuntimeConfig( + register_pack_id="default_neutral_v1", + )) + + +def _run_one(prompt: str, expected_rule: str) -> dict[str, Any]: + runtime = _build_runtime() + pipeline = CognitiveTurnPipeline(runtime=runtime) + for primer in _PRIMING_PROMPTS: + pipeline.run(primer) + pipeline.run(prompt) + turn_event = runtime.turn_log[-1] + status = getattr(turn_event, "realizer_guard_status", "") + rule = getattr(turn_event, "realizer_guard_rule", "") + surface = turn_event.surface + walk_surface = turn_event.walk_surface + grounding_source = getattr(turn_event, "grounding_source", "") + rejected = status == "rejected" + rule_matches = (expected_rule == "") or (rule == expected_rule) + surface_is_disclosure = surface == DISCLOSURE_SURFACE + walk_preserves_candidate = bool(walk_surface) and walk_surface != DISCLOSURE_SURFACE + grounding_forced_none = grounding_source == "none" + return { + "prompt": prompt, + "expected_rule": expected_rule, + "realizer_guard_status": status, + "realizer_guard_rule": rule, + "surface": surface, + "walk_surface": walk_surface, + "grounding_source": grounding_source, + "rejected": rejected, + "rule_matches": rule_matches, + "surface_is_disclosure": surface_is_disclosure, + "walk_preserves_candidate": walk_preserves_candidate, + "grounding_forced_none": grounding_forced_none, + "cell_supported": ( + rejected + and rule_matches + and surface_is_disclosure + and walk_preserves_candidate + and grounding_forced_none + ), + } + + +def run_holdout(*, emit_json: bool = False) -> dict[str, Any]: + cells = [_run_one(p, r) for (p, r) in _HOLDOUT_PROMPTS] + failures = [c for c in cells if not c["cell_supported"]] + all_supported = not failures + + if not emit_json: + print() + print("=" * 76) + print(" ADR-0075 (C1) — realizer guard holdout cluster") + print("=" * 76) + print(f" Priming sequence: {list(_PRIMING_PROMPTS)}") + print(f" Holdout prompts : {len(_HOLDOUT_PROMPTS)}") + print() + for c in cells: + mark = "+" if c["cell_supported"] else "X" + print(f" {mark} {c['prompt']!r}") + print(f" guard_status : {c['realizer_guard_status']}") + print(f" guard_rule : {c['realizer_guard_rule']}") + print(f" surface : {c['surface']!r}") + print(f" walk_surface (pre-G) : {c['walk_surface']!r}") + print(f" grounding_source : {c['grounding_source']}") + print() + print(f" all_claims_supported : {all_supported}") + print() + + return { + "priming": list(_PRIMING_PROMPTS), + "holdout_prompts": [p for p, _ in _HOLDOUT_PROMPTS], + "cells": cells, + "failures": failures, + "all_claims_supported": all_supported, + } + + +if __name__ == "__main__": # pragma: no cover + emit_json = "--json" in sys.argv + report = run_holdout(emit_json=emit_json) + if emit_json: + print(json.dumps(report, indent=2, sort_keys=True, default=str)) + sys.exit(0 if report["all_claims_supported"] else 1) diff --git a/generate/realizer_guard.py b/generate/realizer_guard.py new file mode 100644 index 00000000..df532668 --- /dev/null +++ b/generate/realizer_guard.py @@ -0,0 +1,244 @@ +"""ADR-0075 (C1) — realizer slot-type guard. + +Pure verifier that runs on every candidate surface produced by the +truth path, before the surface is assigned to ``ChatResponse.surface``. +Rejects illegal articulations; the runtime routes a rejected +candidate to a deterministic bounded disclosure string. + +Doctrine +-------- + +This module is **admission control with a deterministic fallback**, +not normalization. The guard never edits a surface; it only emits +a verdict (``status`` + ``rule_id`` + ``detail``). That keeps it +firmly outside CLAUDE.md's forbidden normalization sites — it does +not repair a candidate, it refuses it. + +Rules +----- + +C1 active rules (see ADR-0075): + +* **R2_aux_neg_requires_verb** — after a do-support negation + (``do not`` / ``does not`` / ``did not``), the immediately + following non-adverb content token must have POS ``VERB``. +* **R3_be_neg_requires_predicate** — after a be-negation + (``is not`` / ``are not`` / ``was not`` / ``were not``), the + immediately following non-adverb content token must have POS in + ``{NOUN, ADJ, DET, ADV, PRON}``. + +R1 (``no_finite_verb``) was scoped into C1 originally but **deferred** +during ratification: the active language-pack POS coverage does not +list every English finite verb used by the teaching-chain realizer +(notably ``requires`` / ``makes``), so R1 would falsely reject +currently-passing cognition cases — a regression the ADR's +byte-identity canary explicitly forbids. R1's intent is preserved +for a follow-up phase that either broadens pack POS coverage or +adds a closed English-vocabulary POS table to the guard. + +Empty surfaces are exempt — those route through a separate +disclosure path. + +Determinism +----------- + +The guard is a pure function of ``(surface, pos_lookup)``. No I/O, +no mutation, no globals beyond the closed sets defined in this +module. Pack lexicons that back the ``pos_lookup`` callable are +themselves deterministic from manifest checksums, so guard verdicts +are replay-equivalent across runs. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Callable, Literal + + +DISCLOSURE_SURFACE = "I do not have a reviewed articulation for that yet." +"""Bounded fallback surface used when the guard rejects a candidate. + +Module-level constant — never composed from user input. The runtime +substitutes this string for any rejected candidate before assigning +to ``ChatResponse.surface``. +""" + + +_TOKEN_RE = re.compile(r"[A-Za-z][A-Za-z'_-]*") + + +_FINITE_VERB_AUX: frozenset[str] = frozenset({ + "be", "am", "is", "are", "was", "were", "been", "being", + "do", "does", "did", "doing", "done", + "have", "has", "had", "having", + "will", "would", "shall", "should", + "can", "could", "may", "might", "must", +}) +"""Finite verb forms recognized without pack POS lookup.""" + + +_PREDICATE_FUNCTION_WORDS: frozenset[str] = frozenset({ + "a", "an", "the", + "i", "you", "he", "she", "it", + "we", "they", "me", "him", "her", "us", "them", + "this", "that", "these", "those", + "my", "your", "his", "its", "our", "their", +}) +"""Predicate-eligible function words (allowed after be-negation).""" + + +_ADVERB_FUNCTION_WORDS: frozenset[str] = frozenset({ + "always", "never", "ever", "even", "only", "just", + "really", "actually", "still", "yet", "also", + "often", "sometimes", "usually", "rarely", +}) +"""Adverbs skipped when looking past a negation to its target slot.""" + + +_DO_AUX: frozenset[str] = frozenset({"do", "does", "did"}) +_BE_AUX: frozenset[str] = frozenset({"is", "are", "was", "were"}) + + +@dataclass(frozen=True) +class RealizerGuardVerdict: + """Outcome of a single guard check. + + ``status`` : ``"ok"`` when all rules pass; ``"rejected"`` when + at least one rule fires. + ``rule_id`` : closed-set identifier of the failing rule + (``""`` when status is ``"ok"``). + ``detail`` : short surface fragment showing the violation + (``""`` when status is ``"ok"``). + """ + + status: Literal["ok", "rejected"] + rule_id: str + detail: str + + +_OK_VERDICT = RealizerGuardVerdict(status="ok", rule_id="", detail="") + + +def _tokens(surface: str) -> list[tuple[int, str]]: + """Return ordered ``(start_index, token)`` pairs. + + Punctuation, digits, and bracket/quote characters are skipped. + The regex selects ``A-Za-z`` runs with internal apostrophes, + underscores, or hyphens. + """ + return [(m.start(), m.group(0)) for m in _TOKEN_RE.finditer(surface)] + + +def _skip_adverbs(tokens: list[tuple[int, str]], start_idx: int) -> int: + """Return the index of the first non-adverb-function-word at or + after ``start_idx``. Returns ``len(tokens)`` when the rest is + exhausted entirely by adverbs. + """ + j = start_idx + while j < len(tokens) and tokens[j][1].casefold() in _ADVERB_FUNCTION_WORDS: + j += 1 + return j + + +def _explicit_non_verb( + token: str, + pos_lookup: Callable[[str], str | None], +) -> bool: + """True iff the token has an EXPLICIT non-VERB POS tag. + + Unknown tokens (pack returns ``None``) fail open — the rule does + not fire on them. This is the principled trigger for R2: only + reject when the pack has explicitly classified the next-token as + the wrong POS for a do-support negation slot. + """ + fold = token.casefold() + if fold in _FINITE_VERB_AUX: + return False + pos = pos_lookup(token) + if pos is None or pos == "VERB": + return False + return True + + +def _explicit_non_predicate( + token: str, + pos_lookup: Callable[[str], str | None], +) -> bool: + """True iff the token has an EXPLICIT non-predicate POS tag + (typically ``VERB``). Unknown tokens fail open. + """ + fold = token.casefold() + if fold in _PREDICATE_FUNCTION_WORDS or fold in _ADVERB_FUNCTION_WORDS: + return False + pos = pos_lookup(token) + if pos is None: + return False + return pos not in {"NOUN", "ADJ", "DET", "ADV", "PRON"} + + +def check_surface( + surface: str, + *, + pos_lookup: Callable[[str], str | None], +) -> RealizerGuardVerdict: + """Apply C1's active rules (R2, R3) to ``surface``. + + ``pos_lookup`` should return one of the pack POS tags + (``"NOUN"``, ``"VERB"``, ``"ADJ"``, ``"DET"``, ``"ADV"``, + ``"PRON"``, …) or ``None`` if the token is unknown. + + **Fail-open on unknown POS, fail-closed on explicit wrong POS.** + R2 fires only when the next-token has an explicit non-VERB POS; + R3 fires only when the next-token has an explicit non-predicate + POS (typically ``VERB``). Unknown tokens — words the pack + doesn't list — pass through both rules unscathed, because the + guard cannot prove they violate the slot type. This honors the + byte-identity invariant on currently-passing cases where the + realizer emits valid English that the cognition pack happens + not to enumerate (e.g., ``ratified`` in PROCEDURE templates). + + Rules are position-anchored: the scan walks the token stream + once and emits the first violation it finds. + """ + if not surface.strip(): + return _OK_VERDICT + tokens = _tokens(surface) + if not tokens: + return _OK_VERDICT + + for i in range(len(tokens) - 1): + cur = tokens[i][1].casefold() + nxt = tokens[i + 1][1].casefold() + if cur in _DO_AUX and nxt == "not": + j = _skip_adverbs(tokens, i + 2) + if j >= len(tokens): + return RealizerGuardVerdict( + status="rejected", + rule_id="R2_aux_neg_requires_verb", + detail=f"{tokens[i][1]} not ", + ) + tok = tokens[j][1] + if _explicit_non_verb(tok, pos_lookup): + return RealizerGuardVerdict( + status="rejected", + rule_id="R2_aux_neg_requires_verb", + detail=f"{tokens[i][1]} not {tok}", + ) + if cur in _BE_AUX and nxt == "not": + j = _skip_adverbs(tokens, i + 2) + if j >= len(tokens): + return RealizerGuardVerdict( + status="rejected", + rule_id="R3_be_neg_requires_predicate", + detail=f"{tokens[i][1]} not ", + ) + tok = tokens[j][1] + if _explicit_non_predicate(tok, pos_lookup): + return RealizerGuardVerdict( + status="rejected", + rule_id="R3_be_neg_requires_predicate", + detail=f"{tokens[i][1]} not {tok}", + ) + + return _OK_VERDICT diff --git a/tests/test_realizer_guard_holdout.py b/tests/test_realizer_guard_holdout.py new file mode 100644 index 00000000..0de1865f --- /dev/null +++ b/tests/test_realizer_guard_holdout.py @@ -0,0 +1,155 @@ +"""ADR-0075 (C1) — holdout cluster + byte-identity invariant tests. + +These tests pin the two invariants named in the ADR: + +* ``invariant_realizer_no_illegal_articulation`` — every prompt in + the C1 holdout cluster is rejected by the guard with the expected + rule_id, and the surface is replaced with the bounded disclosure + string. +* ``invariant_realizer_guard_byte_identity_on_currently_passing_cases`` + — every currently-passing cognition-lane DEFINITION prompt + continues to produce a guard-accepted surface byte-identical to + pre-C1 behavior. + +The holdout cluster's exit code is the canonical coherence-floor +gate (see ``evals/realizer_guard/run_holdout.py``). +""" + +from __future__ import annotations + +import pytest + +from chat.runtime import ChatRuntime +from core.cognition.pipeline import CognitiveTurnPipeline +from core.config import RuntimeConfig +from evals.realizer_guard.run_holdout import ( + _HOLDOUT_PROMPTS, + _PRIMING_PROMPTS, + run_holdout, +) +from generate.realizer_guard import DISCLOSURE_SURFACE + + +# ---------- invariant_realizer_no_illegal_articulation ---------- + + +@pytest.fixture(scope="module") +def holdout_report(): + """Run the cluster once per test module — parametrized tests + then read from the cached report instead of re-running the full + six-prompt cluster each time.""" + return run_holdout(emit_json=True) + + +def test_holdout_cluster_all_claims_supported(holdout_report): + assert holdout_report["all_claims_supported"] is True + assert holdout_report["failures"] == [] + + +def test_holdout_cluster_size(holdout_report): + assert len(holdout_report["cells"]) == len(_HOLDOUT_PROMPTS) + assert len(holdout_report["cells"]) == 6 + + +@pytest.mark.parametrize("prompt,expected_rule", list(_HOLDOUT_PROMPTS)) +def test_each_holdout_prompt_rejected( + prompt: str, expected_rule: str, holdout_report, +): + cell = next(c for c in holdout_report["cells"] if c["prompt"] == prompt) + assert cell["realizer_guard_status"] == "rejected" + assert cell["realizer_guard_rule"] == expected_rule + assert cell["surface"] == DISCLOSURE_SURFACE + assert cell["walk_surface"] != DISCLOSURE_SURFACE + assert cell["walk_surface"].strip() != "" + assert cell["grounding_source"] == "none" + + +def test_every_rejected_walk_surface_violates_R2(holdout_report): + """Sanity: each preserved pre-guard candidate must in fact + violate the rule it was rejected under (round-trip check). + + Uses the live runtime's pack POS table because the C1 rules + fail-open on unknown POS — a null-lookup would not re-trigger + R2 since the original trigger depends on the pack having + classified the offending token (e.g. ``thought``) as ``NOUN``. + """ + from chat.runtime import ChatRuntime + from core.config import RuntimeConfig + from generate.realizer_guard import check_surface + + rt = ChatRuntime(config=RuntimeConfig( + register_pack_id="default_neutral_v1", + )) + for cell in holdout_report["cells"]: + v = check_surface( + cell["walk_surface"], + pos_lookup=rt._pos_by_surface.get, + ) + assert v.status == "rejected", ( + f"walk_surface {cell['walk_surface']!r} should still " + f"violate the rule under round-trip check" + ) + assert v.rule_id == cell["realizer_guard_rule"] + + +# ---------- byte-identity invariant on currently-passing cases ---------- + + +_CURRENTLY_PASSING_PROMPTS: tuple[tuple[str, str], ...] = ( + ( + "What is light?", + "Light is a source of revelation that makes things knowable. " + "pack-grounded (en_core_cognition_v1).", + ), + ( + "Define knowledge.", + "Knowledge is justified understanding grounded in evidence " + "and recall. pack-grounded (en_core_cognition_v1).", + ), + ( + "What is truth?", + "Truth is a claim or state grounded by evidence and coherent " + "judgment. pack-grounded (en_core_cognition_v1).", + ), +) + + +@pytest.mark.parametrize("prompt,expected_surface", _CURRENTLY_PASSING_PROMPTS) +def test_currently_passing_cases_byte_identical( + prompt: str, expected_surface: str, +): + """ADR-0075 byte-identity invariant. + + For every currently-passing cognition-lane DEFINITION case, the + post-C1 surface must be byte-identical to the pre-C1 surface. + If this regresses, the guard rule set is too aggressive and + must be narrowed before merge. + """ + rt = ChatRuntime(config=RuntimeConfig( + register_pack_id="default_neutral_v1", + )) + pipeline = CognitiveTurnPipeline(runtime=rt) + pipeline.run(prompt) + te = rt.turn_log[-1] + assert te.realizer_guard_status == "ok" + assert te.realizer_guard_rule == "" + assert te.surface == expected_surface + + +def test_priming_sequence_all_pass_guard(): + """Every priming prompt must pass the guard cleanly — the + holdout cluster's rejection signal would be meaningless if the + priming itself were also being rejected.""" + rt = ChatRuntime(config=RuntimeConfig( + register_pack_id="default_neutral_v1", + )) + pipeline = CognitiveTurnPipeline(runtime=rt) + for p in _PRIMING_PROMPTS: + pipeline.run(p) + te = rt.turn_log[-1] + assert te.realizer_guard_status == "ok", ( + f"Priming prompt {p!r} was rejected; holdout signal " + f"would be confounded. Guard rule was " + f"{te.realizer_guard_rule!r}, surface " + f"{te.surface!r}" + ) diff --git a/tests/test_realizer_guard_runtime_seam.py b/tests/test_realizer_guard_runtime_seam.py new file mode 100644 index 00000000..e157b639 --- /dev/null +++ b/tests/test_realizer_guard_runtime_seam.py @@ -0,0 +1,200 @@ +"""ADR-0075 (C1) — runtime seam tests for the realizer guard. + +These tests exercise the hook in ``chat/runtime.py`` end-to-end: + +* On the main path, a rejected candidate is replaced by + ``DISCLOSURE_SURFACE`` on ``ChatResponse.surface`` and + ``TurnEvent.surface``; ``walk_surface`` preserves the pre-guard + candidate; ``grounding_source`` is forced to ``"none"``; + ``realizer_guard_status`` and ``realizer_guard_rule`` carry the + verdict. +* The telemetry serializer surfaces both new fields. +* The guard does not regress pack-grounded DEFINITION cases — those + remain ``status="ok"`` byte-identical to pre-C1 behavior. +* AST seam: only ``chat.runtime`` is allowed to import + ``generate.realizer_guard`` at module level. Other production + modules must not depend on the guard directly. +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +from chat.runtime import ChatRuntime +from core.cognition.pipeline import CognitiveTurnPipeline +from core.config import RuntimeConfig +from generate.realizer_guard import DISCLOSURE_SURFACE + + +_PRIMING = ("What is light?", "Define knowledge.", "What is truth?") +_BUG_PROMPT = "Light reveals truth, right?" + + +def _build_runtime() -> ChatRuntime: + return ChatRuntime(config=RuntimeConfig( + register_pack_id="default_neutral_v1", + )) + + +def _run_holdout_sequence(rt: ChatRuntime) -> None: + pipeline = CognitiveTurnPipeline(runtime=rt) + for p in _PRIMING: + pipeline.run(p) + pipeline.run(_BUG_PROMPT) + + +# ---------- Main path rejection routing ---------- + + +def test_rejected_candidate_replaces_surface_with_disclosure(): + rt = _build_runtime() + _run_holdout_sequence(rt) + te = rt.turn_log[-1] + assert te.realizer_guard_status == "rejected" + assert te.surface == DISCLOSURE_SURFACE + + +def test_rejected_candidate_preserves_pre_guard_on_walk_surface(): + rt = _build_runtime() + _run_holdout_sequence(rt) + te = rt.turn_log[-1] + assert te.walk_surface and te.walk_surface != DISCLOSURE_SURFACE + assert "does not thought" in te.walk_surface + + +def test_rejected_candidate_forces_grounding_source_none(): + rt = _build_runtime() + _run_holdout_sequence(rt) + te = rt.turn_log[-1] + assert te.grounding_source == "none" + + +def test_rejected_candidate_records_rule_id(): + rt = _build_runtime() + _run_holdout_sequence(rt) + te = rt.turn_log[-1] + assert te.realizer_guard_rule == "R2_aux_neg_requires_verb" + + +# ---------- ChatResponse mirrors TurnEvent ---------- + + +def test_chat_response_carries_guard_fields(): + rt = _build_runtime() + pipeline = CognitiveTurnPipeline(runtime=rt) + for p in _PRIMING: + pipeline.run(p) + # Pipeline runs and writes to turn_log; verify ChatResponse via + # direct rt.chat() on the bug prompt produces a response that + # mirrors the TurnEvent fields. + response = rt.chat(_BUG_PROMPT) + assert response.realizer_guard_status in {"ok", "rejected"} + # When rejected, ChatResponse + TurnEvent must agree on the rule. + te = rt.turn_log[-1] + assert response.realizer_guard_status == te.realizer_guard_status + assert response.realizer_guard_rule == te.realizer_guard_rule + + +# ---------- Currently-passing cases stay passing ---------- + + +@pytest.mark.parametrize("prompt", [ + "What is light?", + "Define knowledge.", + "What is truth?", +]) +def test_pack_grounded_definitions_pass_guard(prompt: str): + """ADR-0075 byte-identity invariant: currently-passing cases must + not regress. These are the saturated cognition-lane DEFINITION + cases — guard must accept all of them.""" + rt = _build_runtime() + pipeline = CognitiveTurnPipeline(runtime=rt) + pipeline.run(prompt) + te = rt.turn_log[-1] + assert te.realizer_guard_status == "ok" + assert te.realizer_guard_rule == "" + assert te.surface != DISCLOSURE_SURFACE + assert te.surface.startswith(prompt.split()[-1].rstrip("?.").capitalize()) or "pack-grounded" in te.surface + + +# ---------- Telemetry surface ---------- + + +def test_telemetry_includes_guard_fields(): + from chat.telemetry import serialize_turn_event + rt = _build_runtime() + _run_holdout_sequence(rt) + te = rt.turn_log[-1] + record = serialize_turn_event(te) + assert "realizer_guard_status" in record + assert "realizer_guard_rule" in record + assert record["realizer_guard_status"] == "rejected" + assert record["realizer_guard_rule"] == "R2_aux_neg_requires_verb" + + +def test_telemetry_guard_fields_empty_on_pre_c1_events(): + """A plain TurnEvent without the guard fields set should + serialize to empty strings, preserving wire-format degradation.""" + from chat.telemetry import serialize_turn_event + rt = _build_runtime() + pipeline = CognitiveTurnPipeline(runtime=rt) + pipeline.run("What is light?") + te = rt.turn_log[-1] + record = serialize_turn_event(te) + # status should be "ok" (guard ran and accepted), rule empty + assert record["realizer_guard_status"] == "ok" + assert record["realizer_guard_rule"] == "" + + +# ---------- AST seam ---------- + + +_PRODUCTION_MODULE_ROOTS = ("chat", "generate", "packs", "core", "language_packs") +_GUARD_MODULE_NAME = "generate.realizer_guard" +_ALLOWED_IMPORTERS = {"chat/runtime.py"} + + +def _imports_guard(path: Path) -> bool: + try: + tree = ast.parse(path.read_text()) + except (OSError, SyntaxError): + return False + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + if node.module == _GUARD_MODULE_NAME: + return True + elif isinstance(node, ast.Import): + for alias in node.names: + if alias.name == _GUARD_MODULE_NAME: + return True + return False + + +def test_only_runtime_imports_realizer_guard(): + """Production modules must not import the guard directly. + + The guard is a singleton seam centralized in ``chat.runtime``. + Other modules sprinkling guard calls would split the seam and + make rejection routing harder to audit. + """ + repo_root = Path(__file__).resolve().parent.parent + offenders: list[str] = [] + for root in _PRODUCTION_MODULE_ROOTS: + root_path = repo_root / root + if not root_path.exists(): + continue + for py in root_path.rglob("*.py"): + rel = py.relative_to(repo_root).as_posix() + if rel == "generate/realizer_guard.py": + continue + if rel in _ALLOWED_IMPORTERS: + continue + if _imports_guard(py): + offenders.append(rel) + assert not offenders, ( + f"Only {sorted(_ALLOWED_IMPORTERS)} may import " + f"{_GUARD_MODULE_NAME!r}; found in: {offenders}" + ) diff --git a/tests/test_realizer_guard_unit.py b/tests/test_realizer_guard_unit.py new file mode 100644 index 00000000..526bb7ec --- /dev/null +++ b/tests/test_realizer_guard_unit.py @@ -0,0 +1,211 @@ +"""ADR-0075 (C1) — realizer slot-type guard, pure-unit coverage. + +Each test gives the guard a minimal POS lookup and asserts the +expected verdict. No runtime, no pipeline, no fixtures. +""" + +from __future__ import annotations + +import pytest + +from generate.realizer_guard import ( + DISCLOSURE_SURFACE, + RealizerGuardVerdict, + check_surface, +) + + +_POS: dict[str, str] = { + # Nouns + "Light": "NOUN", "light": "NOUN", + "Truth": "NOUN", "truth": "NOUN", + "Knowledge": "NOUN", "knowledge": "NOUN", + "Right": "NOUN", "right": "NOUN", + "thought": "NOUN", "evidence": "NOUN", + "claim": "NOUN", "state": "NOUN", + "source": "NOUN", "revelation": "NOUN", + "judgment": "NOUN", "recall": "NOUN", + "things": "NOUN", "understanding": "NOUN", "articulation": "NOUN", + # Verbs + "reveal": "VERB", "reveals": "VERB", + "ground": "VERB", "grounds": "VERB", + "make": "VERB", "makes": "VERB", + "require": "VERB", "requires": "VERB", + "perceive": "VERB", + # Adjectives + "reviewed": "ADJ", "knowable": "ADJ", + "justified": "ADJ", "coherent": "ADJ", "grounded": "ADJ", + # Function words / connectors (returned but not matched as VERB) + "of": "ADP", "and": "CONJ", "in": "ADP", "or": "CONJ", "for": "ADP", + "by": "ADP", "that": "PRON", +} + + +def _lookup(token: str) -> str | None: + return _POS.get(token) + + +def _verdict(surface: str) -> RealizerGuardVerdict: + return check_surface(surface, pos_lookup=_lookup) + + +# ---------- R1 deferred (see ADR-0075) ---------- +# +# R1 ("must contain at least one finite verb") is deferred from +# active C1 scope. These tests pin the **current** behavior — R1 +# surfaces pass even when verb-less — so a future re-enablement is +# a deliberate, intentional change rather than a silent drift. + + +def test_R1_deferred_verbless_surface_passes(): + v = _verdict("Light right.") + assert v.status == "ok" + + +def test_R1_deferred_bare_noun_phrase_passes(): + v = _verdict("Truth knowledge evidence.") + assert v.status == "ok" + + +# ---------- R2 ---------- + + +def test_R2_rejects_does_not_followed_by_noun(): + """The canonical observed bug — 'Right does not thought.'""" + v = _verdict("Right does not thought.") + assert v.status == "rejected" + assert v.rule_id == "R2_aux_neg_requires_verb" + assert "does not thought" in v.detail + + +def test_R2_rejects_do_not_followed_by_noun(): + v = _verdict("They do not knowledge.") + assert v.status == "rejected" + assert v.rule_id == "R2_aux_neg_requires_verb" + + +def test_R2_rejects_did_not_followed_by_noun(): + v = _verdict("Light did not truth.") + assert v.status == "rejected" + assert v.rule_id == "R2_aux_neg_requires_verb" + + +def test_R2_accepts_does_not_followed_by_verb(): + v = _verdict("Light does not reveal truth.") + assert v.status == "ok" + + +def test_R2_accepts_does_not_with_adverb_then_verb(): + """Adverbs between 'not' and the verb are skipped.""" + v = _verdict("Light does not always reveal truth.") + assert v.status == "ok" + + +def test_R2_rejects_does_not_with_adverb_then_noun(): + v = _verdict("Right does not always thought.") + assert v.status == "rejected" + assert v.rule_id == "R2_aux_neg_requires_verb" + + +def test_R2_rejects_does_not_at_end_of_surface(): + v = _verdict("Light does not.") + assert v.status == "rejected" + assert v.rule_id == "R2_aux_neg_requires_verb" + assert "missing" in v.detail + + +# ---------- R3 ---------- + + +def test_R3_accepts_is_not_followed_by_noun(): + v = _verdict("Light is not knowledge.") + assert v.status == "ok" + + +def test_R3_accepts_are_not_followed_by_adjective(): + v = _verdict("Claims are not reviewed.") + assert v.status == "ok" + + +def test_R3_accepts_is_not_with_determiner(): + v = _verdict("Light is not a claim.") + assert v.status == "ok" + + +def test_R3_rejects_is_not_followed_by_verb(): + v = _verdict("Light is not reveal.") + assert v.status == "rejected" + assert v.rule_id == "R3_be_neg_requires_predicate" + + +def test_R3_rejects_was_not_followed_by_verb(): + v = _verdict("Light was not reveals.") + assert v.status == "rejected" + assert v.rule_id == "R3_be_neg_requires_predicate" + + +def test_R3_rejects_is_not_at_end(): + v = _verdict("Light is not.") + assert v.status == "rejected" + assert v.rule_id == "R3_be_neg_requires_predicate" + + +# ---------- DISCLOSURE_SURFACE itself must pass ---------- + + +def test_disclosure_surface_passes_guard(): + """Critical: the fallback string must not be rejected by its own + guard, otherwise routing on rejection would loop.""" + v = _verdict(DISCLOSURE_SURFACE) + assert v.status == "ok" + assert v.rule_id == "" + + +# ---------- Empty / whitespace surfaces ---------- + + +def test_empty_surface_is_ok(): + v = _verdict("") + assert v.status == "ok" + + +def test_whitespace_only_surface_is_ok(): + v = _verdict(" \n \t ") + assert v.status == "ok" + + +def test_punctuation_only_surface_is_ok(): + """No alphabetic tokens ⇒ guard cannot evaluate ⇒ pass through.""" + v = _verdict("... !!! ???") + assert v.status == "ok" + + +# ---------- Robustness ---------- + + +def test_unknown_token_after_aux_neg_fails_open(): + """Unknown content tokens fall through pack lookup — R2 fails + OPEN (does not reject) so the guard never regresses a + currently-passing case where the realizer emits valid English + the pack happens not to enumerate.""" + v = _verdict("Light does not xyzzy.") + assert v.status == "ok" + + +def test_unknown_token_after_be_neg_fails_open(): + """R3 also fails open on unknown tokens. Critical for cases like + 'is not yet ratified' where 'ratified' is real English not in + the pack lexicon.""" + v = _verdict("Step-by-step guidance is not yet ratified.") + assert v.status == "ok" + + +def test_known_pack_verb_after_aux_neg_passes(): + v = _verdict("Light does not perceive truth.") + assert v.status == "ok" + + +def test_verdict_is_frozen(): + v = _verdict("Light reveals truth.") + with pytest.raises(Exception): # FrozenInstanceError + v.status = "rejected" # type: ignore[misc]