feat(coherence): ADR-0075 — realizer slot-type guard (C1)
C1 coherence floor: a deterministic verifier that runs on every
candidate surface produced by the truth path, before assignment to
ChatResponse.surface. Rejects illegal articulations and routes them
to a bounded disclosure string — admission control with a
deterministic fallback, not normalization.
Active rules (R1 deferred during ratification — see ADR):
R2_aux_neg_requires_verb — "<aux> not <wrong-POS>" rejected
R3_be_neg_requires_predicate — "<be> not <verb>" rejected
Fail-open on unknown POS, fail-closed on explicit wrong POS.
Cognition eval byte-identical (100/91.7/100/100).
Original bug class — "Light reveals truth, right?" → "Right does not
thought." — now routes to "I do not have a reviewed articulation for
that yet." with grounding_source=none, walk_surface preserving the
rejected candidate, and telemetry carrying R2_aux_neg_requires_verb.
Files:
generate/realizer_guard.py NEW — pure verifier
chat/runtime.py hook on stub + main paths
chat/telemetry.py serialize guard fields
core/physics/identity.py TurnEvent +2 fields
evals/realizer_guard/run_holdout.py NEW — 6-prompt cluster
tests/test_realizer_guard_*.py NEW — 46 tests (unit/seam/holdout)
docs/decisions/ADR-0075-*.md NEW — ratified
Invariants pinned:
invariant_realizer_no_illegal_articulation
invariant_realizer_guard_byte_identity_on_currently_passing_cases
Lanes (excluding 1 pre-existing TestDemoPreambles failure unrelated
to C1, already present at 4426f38):
smoke 67/67 cognition 120/120(+1s) teaching 17/17
packs 6/6 runtime 19/19 algebra 132/132 full 2792/2793
This commit is contained in:
Shay
parent
4426f387d1
commit
7cc2888ed2
10 changed files with 1428 additions and 2 deletions
|
|
@ -57,6 +57,10 @@ from packs.ethics.loader import (
|
|||
)
|
||||
from packs.identity.loader import load_identity_manifold
|
||||
from chat.register_variation import decorate_surface
|
||||
from generate.realizer_guard import (
|
||||
DISCLOSURE_SURFACE as _GUARD_DISCLOSURE_SURFACE,
|
||||
check_surface as _check_realizer_surface,
|
||||
)
|
||||
from packs.anchor_lens.loader import AnchorLens, load_anchor_lens
|
||||
from packs.register.loader import RegisterPack, load_register_pack
|
||||
from packs.safety.check import SafetyCheck, SafetyContext
|
||||
|
|
@ -343,6 +347,12 @@ class ChatResponse:
|
|||
# telemetry JSONL. ``""`` defaults preserve pre-L1.4 byte-identity.
|
||||
anchor_lens_id: str = ""
|
||||
anchor_lens_mode_label: str = ""
|
||||
# ADR-0075 (C1) — realizer slot-type guard verdict. Mirrors the
|
||||
# TurnEvent fields so callers (CLI, demos, tests) can read the
|
||||
# guard state from ChatResponse without re-parsing the telemetry
|
||||
# JSONL. ``""`` defaults preserve pre-C1 byte-identity.
|
||||
realizer_guard_status: str = ""
|
||||
realizer_guard_rule: str = ""
|
||||
|
||||
|
||||
class ChatRuntime:
|
||||
|
|
@ -983,6 +993,28 @@ class ChatRuntime:
|
|||
grounding_source = grounded_source_tag
|
||||
else:
|
||||
grounding_source = "none"
|
||||
# ADR-0075 (C1) — realizer slot-type guard. Runs BEFORE
|
||||
# register decoration so a register cannot accidentally heal
|
||||
# an illegal articulation by wrapping it, and BEFORE anchor-
|
||||
# lens annotation extraction so the lens annotation never
|
||||
# rides on a guard-rejected surface. On rejection, route to
|
||||
# the bounded disclosure string and force grounding_source to
|
||||
# ``"none"`` (an illegal surface is ungrounded by construction).
|
||||
# The pre-guard candidate is preserved on walk_surface_stub
|
||||
# for telemetry — the stub path normally leaves walk_surface as
|
||||
# _UNKNOWN_DOMAIN_SURFACE, so this swap strictly increases
|
||||
# observability under rejection.
|
||||
guard_verdict_stub = _check_realizer_surface(
|
||||
response_surface,
|
||||
pos_lookup=self._pos_by_surface.get,
|
||||
)
|
||||
realizer_guard_status_stub = guard_verdict_stub.status
|
||||
realizer_guard_rule_stub = guard_verdict_stub.rule_id
|
||||
walk_surface_stub = _UNKNOWN_DOMAIN_SURFACE
|
||||
if guard_verdict_stub.status == "rejected":
|
||||
walk_surface_stub = response_surface
|
||||
response_surface = _GUARD_DISCLOSURE_SURFACE
|
||||
grounding_source = "none"
|
||||
# ADR-0071 (R4) — apply seeded discourse-marker decoration to
|
||||
# the realized surface AFTER grounding source is decided.
|
||||
# Empty marker buckets ⇒ no-op (UNREGISTERED / neutral / terse).
|
||||
|
|
@ -1024,7 +1056,7 @@ class ChatRuntime:
|
|||
turn=max(self._context.turn - 1, 0),
|
||||
input_tokens=tokens,
|
||||
surface=response_surface,
|
||||
walk_surface=_UNKNOWN_DOMAIN_SURFACE,
|
||||
walk_surface=walk_surface_stub,
|
||||
articulation_surface=_UNKNOWN_DOMAIN_SURFACE,
|
||||
dialogue_role="assert",
|
||||
identity_score=None,
|
||||
|
|
@ -1041,6 +1073,8 @@ class ChatRuntime:
|
|||
register_variant_id=decoration_stub.variant_id,
|
||||
anchor_lens_id=anchor_lens_id_stub,
|
||||
anchor_lens_mode_label=anchor_lens_mode_label_stub,
|
||||
realizer_guard_status=realizer_guard_status_stub,
|
||||
realizer_guard_rule=realizer_guard_rule_stub,
|
||||
)
|
||||
self.turn_log.append(stub_event)
|
||||
self._emit_turn_event(stub_event)
|
||||
|
|
@ -1073,7 +1107,7 @@ class ChatRuntime:
|
|||
versor_condition=versor_condition(field_state.F),
|
||||
output_language=self.config.output_language,
|
||||
frame_pack=self.config.frame_pack,
|
||||
walk_surface=_UNKNOWN_DOMAIN_SURFACE,
|
||||
walk_surface=walk_surface_stub,
|
||||
salience_top_k=None,
|
||||
candidates_used=None,
|
||||
vault_hits=0,
|
||||
|
|
@ -1089,6 +1123,8 @@ class ChatRuntime:
|
|||
register_variant_id=decoration_stub.variant_id,
|
||||
anchor_lens_id=anchor_lens_id_stub,
|
||||
anchor_lens_mode_label=anchor_lens_mode_label_stub,
|
||||
realizer_guard_status=realizer_guard_status_stub,
|
||||
realizer_guard_rule=realizer_guard_rule_stub,
|
||||
)
|
||||
|
||||
def chat(self, text: str, max_tokens: int | None = None) -> ChatResponse:
|
||||
|
|
@ -1350,6 +1386,25 @@ class ChatRuntime:
|
|||
before = response_surface
|
||||
response_surface = inject_hedge(response_surface, hedge_prefix)
|
||||
hedge_injected = response_surface != before
|
||||
# ADR-0075 (C1) — realizer slot-type guard (main path). Runs
|
||||
# AFTER all composer / planner / hedge transformations and
|
||||
# BEFORE register decoration so a single seam covers every
|
||||
# articulation path. On rejection: surface is replaced with
|
||||
# the bounded disclosure string, grounding_source forced to
|
||||
# ``"none"``, and walk_surface preserves the rejected
|
||||
# candidate so the manifold-walk evidence is overwritten only
|
||||
# in the rejection branch (the contract says illegal
|
||||
# articulation evidence is the relevant telemetry).
|
||||
guard_verdict_main = _check_realizer_surface(
|
||||
response_surface,
|
||||
pos_lookup=self._pos_by_surface.get,
|
||||
)
|
||||
realizer_guard_status_main = guard_verdict_main.status
|
||||
realizer_guard_rule_main = guard_verdict_main.rule_id
|
||||
if guard_verdict_main.status == "rejected":
|
||||
walk_surface = response_surface
|
||||
response_surface = _GUARD_DISCLOSURE_SURFACE
|
||||
warm_grounding_source = "none"
|
||||
# ADR-0071 (R4) — seeded discourse-marker decoration is the
|
||||
# last step before TurnEvent is sealed. Applies uniformly to
|
||||
# every grounding path (vault / pack / teaching / planner /
|
||||
|
|
@ -1406,6 +1461,8 @@ class ChatRuntime:
|
|||
register_variant_id=decoration_main.variant_id,
|
||||
anchor_lens_id=anchor_lens_id_main,
|
||||
anchor_lens_mode_label=anchor_lens_mode_label_main,
|
||||
realizer_guard_status=realizer_guard_status_main,
|
||||
realizer_guard_rule=realizer_guard_rule_main,
|
||||
)
|
||||
self.turn_log.append(turn_event)
|
||||
self._emit_turn_event(turn_event)
|
||||
|
|
@ -1443,6 +1500,8 @@ class ChatRuntime:
|
|||
register_variant_id=decoration_main.variant_id,
|
||||
anchor_lens_id=anchor_lens_id_main,
|
||||
anchor_lens_mode_label=anchor_lens_mode_label_main,
|
||||
realizer_guard_status=realizer_guard_status_main,
|
||||
realizer_guard_rule=realizer_guard_rule_main,
|
||||
)
|
||||
|
||||
def _unknown_domain_response(self, field_state: FieldState, filtered: list[str]) -> ChatResponse:
|
||||
|
|
|
|||
|
|
@ -79,6 +79,10 @@ def serialize_turn_event(
|
|||
# turns where the lens did not engage.
|
||||
"anchor_lens_id": str(getattr(event, "anchor_lens_id", "") or ""),
|
||||
"anchor_lens_mode_label": str(getattr(event, "anchor_lens_mode_label", "") or ""),
|
||||
# ADR-0075 (C1) — realizer slot-type guard verdict per turn.
|
||||
# Empty strings on pre-C1 events; closed enums otherwise.
|
||||
"realizer_guard_status": str(getattr(event, "realizer_guard_status", "") or ""),
|
||||
"realizer_guard_rule": str(getattr(event, "realizer_guard_rule", "") or ""),
|
||||
}
|
||||
safety = getattr(event, "safety_verdict", None)
|
||||
if safety is not None:
|
||||
|
|
|
|||
|
|
@ -312,3 +312,15 @@ class TurnEvent:
|
|||
# ``""`` so pre-L1.4 callers stay byte-identical.
|
||||
anchor_lens_id: str = ""
|
||||
anchor_lens_mode_label: str = ""
|
||||
# ADR-0075 (C1) — realizer slot-type guard verdict per turn.
|
||||
# ``realizer_guard_status`` is ``"ok"`` when the guard accepted
|
||||
# the candidate surface, ``"rejected"`` when one of R1/R2/R3
|
||||
# fired and the runtime routed the surface to the bounded
|
||||
# disclosure string, or ``""`` on pre-C1 events that pre-date
|
||||
# the guard hook. ``realizer_guard_rule`` carries the rule_id
|
||||
# (one of ``"R1_no_finite_verb"``, ``"R2_aux_neg_requires_verb"``,
|
||||
# ``"R3_be_neg_requires_predicate"``) when status is
|
||||
# ``"rejected"``, otherwise ``""``. Both default to ``""`` so
|
||||
# pre-C1 callers stay byte-identical.
|
||||
realizer_guard_status: str = ""
|
||||
realizer_guard_rule: str = ""
|
||||
|
|
|
|||
384
docs/decisions/ADR-0075-realizer-slot-type-guard.md
Normal file
384
docs/decisions/ADR-0075-realizer-slot-type-guard.md
Normal file
|
|
@ -0,0 +1,384 @@
|
|||
# ADR-0075 — Realizer slot-type guard (C1: coherence floor)
|
||||
|
||||
**Status:** Accepted
|
||||
**Date:** 2026-05-19
|
||||
**Ratified:** 2026-05-19
|
||||
**Author:** Shay
|
||||
**Builds on:** [ADR-0048](./ADR-0048-pack-grounded-surface.md)
|
||||
(pack-grounded surface), [ADR-0049](./ADR-0049-intent-subject-extraction.md)
|
||||
(intent subject extraction), [ADR-0053](./ADR-0053-cognition-lane-closure.md)
|
||||
(cognition lane closure)
|
||||
**Series:** C1 — Coherence floor (precedes C2 confirmation-tag normalization,
|
||||
R6 substantive register knobs, T1 curated teaching depth)
|
||||
|
||||
---
|
||||
|
||||
## Context
|
||||
|
||||
The orthogonality tour landed `4426f38` with `all_claims_supported=true`
|
||||
across the full 3 × 3 × 2 grid. But inspection of the grid surfaced a
|
||||
real coherence regression on a fourth prompt under live demo:
|
||||
|
||||
```text
|
||||
"Light reveals truth, right?" → "Right does not thought."
|
||||
```
|
||||
|
||||
That surface is grammatically illegal: the verb slot of a
|
||||
`<subject> does not <verb>` construction is occupied by a bare noun
|
||||
(`thought`, which is a noun in `en_core_cognition_v1`). All three
|
||||
registers reproduced this illegal form byte-identically, which is the
|
||||
correct behavior for the register axis (it preserves whatever the
|
||||
truth path produced) but reveals that the truth path itself is
|
||||
licensing surfaces that no realizer template should be able to emit.
|
||||
|
||||
The doctrine in CLAUDE.md is explicit:
|
||||
|
||||
> Code and tests should make illegal states difficult to represent.
|
||||
> Prefer inspectable state, provenance, and deterministic replay over
|
||||
> impressive-looking but ungrounded outputs.
|
||||
|
||||
C1 addresses this at the construction boundary — a deterministic
|
||||
slot-type verifier that rejects illegal articulations before they can
|
||||
escape to the user surface, regardless of which composer or intent
|
||||
path produced them. This is the coherence floor that C2
|
||||
(confirmation-tag normalization), R6 (substantive register knobs),
|
||||
and T1 (curated teaching) all rest on.
|
||||
|
||||
### Why this comes first
|
||||
|
||||
If C2 is fixed before C1, the specific bug above goes away but the
|
||||
underlying property — "no illegal articulation ever escapes" — is
|
||||
not established. The next intent/extraction bug will produce a new
|
||||
word-salad surface and the cycle repeats. C1 establishes the
|
||||
invariant; C2 fixes the specific input class that exposed it.
|
||||
|
||||
If R6 ships before C1, terse and convivial registers will start
|
||||
emitting substantively distinct surfaces on top of a path that can
|
||||
still produce word salad. Worse cosmetics on the same broken floor.
|
||||
|
||||
If T1 ships before C1, the reviewed corpus inherits the illegal
|
||||
articulation pattern wherever it touches a path the guard would have
|
||||
caught — the corpus becomes a vector for the bug, not a fix for it.
|
||||
|
||||
---
|
||||
|
||||
## Decision
|
||||
|
||||
### Mechanism
|
||||
|
||||
A pure verifier `generate/realizer_guard.py` that runs on every
|
||||
candidate surface produced by `pack_grounded_surface`,
|
||||
`pack_grounded_comparison_surface`,
|
||||
`pack_grounded_correction_surface`,
|
||||
`pack_grounded_procedure_surface`, the cross-pack chain composer,
|
||||
NARRATIVE / EXAMPLE composers, and any vault-grounded realizer
|
||||
output, before that surface is assigned to `ChatResponse.surface` or
|
||||
emitted on `walk_surface`.
|
||||
|
||||
The guard is deterministic, reads pack POS data already loaded at
|
||||
runtime, performs a single linear pass, and emits a verdict:
|
||||
|
||||
```python
|
||||
@dataclass(frozen=True)
|
||||
class RealizerGuardVerdict:
|
||||
status: Literal["ok", "rejected"]
|
||||
rule_id: str # e.g. "R1_no_finite_verb"; "" when ok
|
||||
detail: str # surface fragment that violated the rule
|
||||
```
|
||||
|
||||
### Slot-type rules (C1 active scope — two rules)
|
||||
|
||||
```
|
||||
R2_aux_neg_requires_verb:
|
||||
If the surface contains a do-support negation
|
||||
("does not" / "do not" / "did not"), the immediately following
|
||||
content token must have POS = VERB per pack lookup.
|
||||
Adverbs between the negation and the verb are allowed
|
||||
("does not always reveal" — "always" is skipped, then "reveal"
|
||||
must be VERB).
|
||||
|
||||
R3_be_neg_requires_predicate:
|
||||
If the surface contains a be-negation
|
||||
("is not" / "are not" / "was not" / "were not"), the immediately
|
||||
following content token must have POS ∈ {NOUN, ADJ, DET, ADV,
|
||||
PRON} per pack lookup. Bare verbs after be-negation are
|
||||
illegal ("is not reveal" is rejected).
|
||||
```
|
||||
|
||||
Both rules fire on **presence of an illegal pattern**, not on
|
||||
**absence of an expected token** — high precision, low false-positive
|
||||
rate. They cover the observed failure mode and the symmetric form.
|
||||
|
||||
### Deferred during ratification
|
||||
|
||||
```
|
||||
R1_no_finite_verb:
|
||||
Every emitted clause must contain at least one finite verb token.
|
||||
Deferred from active C1 scope: the cognition pack's POS coverage
|
||||
does not include every English finite verb the teaching-chain
|
||||
realizer emits (notably "requires" and "makes"), and switching
|
||||
R1 on regresses currently-passing cases — a violation of the
|
||||
byte-identity invariant the ADR explicitly flags as the canary.
|
||||
R1's intent is preserved for a follow-up coherence ADR that
|
||||
either broadens pack POS coverage or adds a closed English-
|
||||
vocabulary POS table.
|
||||
|
||||
Subject / object / predicate pack-residency:
|
||||
Further rules (subject-pack-residency, object-pack-residency,
|
||||
proposition-graph-residency) are deliberately deferred to a
|
||||
follow-up coherence ADR — C1 is a narrow floor, not a parser.
|
||||
```
|
||||
|
||||
### Failure routing — fail closed, not silent
|
||||
|
||||
When the guard returns `status == "rejected"` on a candidate surface,
|
||||
the truth path:
|
||||
|
||||
1. **Does not** silently drop the surface.
|
||||
2. **Does not** retry the realizer with different parameters.
|
||||
3. **Does** replace the surface with a deterministic bounded
|
||||
disclosure string:
|
||||
|
||||
```text
|
||||
"I do not have a reviewed articulation for that yet."
|
||||
```
|
||||
|
||||
4. **Does** preserve the pre-guard surface on `walk_surface` for
|
||||
telemetry / debugging (consistent with the existing
|
||||
`surface` ≠ `walk_surface` runtime contract).
|
||||
5. **Does** flag `grounding_source` as `"none"` regardless of what
|
||||
the rejected candidate had claimed (an illegal surface is
|
||||
ungrounded by construction).
|
||||
|
||||
### Telemetry surface
|
||||
|
||||
`TurnEvent` gains two fields, default empty:
|
||||
|
||||
```
|
||||
realizer_guard_status : "" | "ok" | "rejected"
|
||||
realizer_guard_rule : "" | "R1_no_finite_verb" | "R2_aux_neg_requires_verb"
|
||||
| "R3_be_neg_requires_predicate"
|
||||
```
|
||||
|
||||
`serialize_turn_event` emits both fields alphabetically, deterministic.
|
||||
|
||||
`ChatResponse` gains the same two fields, propagated identically on
|
||||
stub and main paths (mirrors `register_id` / `anchor_lens_id`
|
||||
threading pattern).
|
||||
|
||||
### Hook point
|
||||
|
||||
Single seam in `chat/runtime.py`, immediately after the composer
|
||||
returns a candidate surface and before the surface is assigned to
|
||||
`ChatResponse.surface` or to the `pre_decoration_surface_*` slot that
|
||||
feeds the register decoration step. This placement matters:
|
||||
|
||||
* Guard runs **before** register decoration, so the
|
||||
register-decorated form is never the verifier's responsibility.
|
||||
* Guard runs **before** anchor-lens annotation extraction, so the
|
||||
`[lens(...):...]` annotation is never on a guard-rejected surface.
|
||||
* Guard runs **after** all composers, so a single seam catches
|
||||
every articulation path.
|
||||
|
||||
### Files
|
||||
|
||||
```
|
||||
generate/realizer_guard.py NEW
|
||||
- RealizerGuardVerdict frozen dataclass
|
||||
- check_surface(surface: str, *, pack_lexicons: Mapping)
|
||||
-> RealizerGuardVerdict
|
||||
- Three pure rule helpers; no I/O; no mutation.
|
||||
|
||||
chat/runtime.py EDIT
|
||||
- Hook check_surface() between composer return and surface assignment
|
||||
- Failure routing: surface = bounded disclosure; walk_surface = raw
|
||||
rejected candidate; grounding_source = "none"; populate guard
|
||||
telemetry fields on TurnEvent + ChatResponse.
|
||||
|
||||
core/physics/identity.py EDIT
|
||||
- TurnEvent.realizer_guard_status: str = ""
|
||||
- TurnEvent.realizer_guard_rule: str = ""
|
||||
|
||||
chat/telemetry.py EDIT
|
||||
- serialize_turn_event: emit both fields alphabetically.
|
||||
|
||||
evals/realizer_guard/__init__.py NEW
|
||||
evals/realizer_guard/run_holdout.py NEW
|
||||
- Six confirmation-tag / illegal-articulation prompts that
|
||||
currently produce illegal surfaces. Under C1 alone (without
|
||||
C2), every one routes to bounded disclosure with
|
||||
realizer_guard_status="rejected" and a recorded rule_id.
|
||||
- Exit code 0 iff invariant_realizer_no_illegal_articulation holds
|
||||
across the cluster.
|
||||
|
||||
tests/test_realizer_guard_unit.py NEW
|
||||
- 22 cases: R2 / R3 (each negation form, copula handling, adverb
|
||||
skipping, end-of-surface, unknown lemma fail-closed); R1
|
||||
deferred (the verb-less cases pin current pass-through behavior
|
||||
so re-enablement is intentional).
|
||||
|
||||
tests/test_realizer_guard_runtime_seam.py NEW
|
||||
- Stub + main path: rejected candidate → bounded disclosure on
|
||||
surface, raw candidate on walk_surface, grounding_source="none",
|
||||
telemetry fields populated.
|
||||
- AST seam: realizer_guard imports no truth-path modules.
|
||||
|
||||
tests/test_realizer_guard_holdout.py NEW
|
||||
- Holdout cluster: six confirmation-tag / illegal-articulation
|
||||
prompts; every one rejected with expected rule_id.
|
||||
- Cognition eval cases: zero rejections (byte-identical baseline).
|
||||
|
||||
docs/decisions/ADR-0075-realizer-slot-type-guard.md NEW (this file)
|
||||
```
|
||||
|
||||
No pack mutation. No composer rewrites. No new dynamic imports.
|
||||
No new filesystem writes. No CLI surface changes (telemetry only).
|
||||
|
||||
### Invariants pinned
|
||||
|
||||
```
|
||||
invariant_realizer_no_illegal_articulation (NEW):
|
||||
For every prompt in the cognition eval lane + every prompt in the
|
||||
C1 holdout cluster + every demo grid cell in register-tour /
|
||||
anchor-lens-tour / orthogonality-tour, the surface emitted on
|
||||
ChatResponse.surface satisfies R2 and R3.
|
||||
Holdout cluster surfaces are bounded disclosure strings (rejection
|
||||
is the correct behavior for that input class under C1 alone).
|
||||
|
||||
invariant_realizer_guard_byte_identity_on_currently_passing_cases (NEW):
|
||||
For every cognition eval case that currently produces a legal
|
||||
articulation, the post-C1 surface is byte-identical to the pre-C1
|
||||
surface. The guard must not regress any legal articulation.
|
||||
|
||||
register-tour invariant — preserved
|
||||
anchor-lens-tour invariant — preserved
|
||||
orthogonality-tour invariant — preserved
|
||||
anchor_lens_byte_identity_null_lift — preserved
|
||||
register_invariant_grounding — preserved
|
||||
```
|
||||
|
||||
The first invariant is the load-bearing one — it is the formal
|
||||
statement that illegal articulations cannot escape, regardless of
|
||||
which composer or intent path produced them.
|
||||
|
||||
---
|
||||
|
||||
## Consequences
|
||||
|
||||
### Capability unlocked
|
||||
|
||||
Word-salad surfaces (`"Right does not thought."` and its symmetric
|
||||
forms) cannot escape to the user, even when intent classification or
|
||||
subject extraction is wrong. Bugs in upstream stages now degrade
|
||||
gracefully to a bounded disclosure surface rather than to ungrounded
|
||||
articulation. This is the precondition that lets C2 / R6 / T1 ship
|
||||
safely.
|
||||
|
||||
### Cognition lane
|
||||
|
||||
Expected byte-identical on currently-passing cases (invariant
|
||||
`invariant_realizer_guard_byte_identity_on_currently_passing_cases`
|
||||
is part of the CI gate). Any case that the guard rejects under C1
|
||||
is, by definition, a case where the prior surface was illegal —
|
||||
those move to bounded disclosure and lose their `term_capture` /
|
||||
`surface_groundedness` credit until C2 + T1 land the proper fix.
|
||||
|
||||
If any currently-passing case regresses, the rule is too aggressive
|
||||
and must be narrowed before merge. The byte-identity invariant is
|
||||
the canary.
|
||||
|
||||
### Performance
|
||||
|
||||
Single linear pass over the candidate surface tokens with a
|
||||
dictionary lookup per content token. O(n) on surface length, runs
|
||||
once per turn. Well below the per-turn latency floor measured in
|
||||
the teaching-loop bench (mean 1.85s). No new caches.
|
||||
|
||||
### Trust boundaries
|
||||
|
||||
* **Construction-boundary verifier, not hot-path repair.** The
|
||||
guard rejects and reroutes; it never edits the candidate surface.
|
||||
This keeps it firmly outside the "forbidden normalization sites"
|
||||
list in CLAUDE.md (it is not normalization — it is admission
|
||||
control with a deterministic fallback).
|
||||
* **No filesystem, no dynamic import, no shell.** The guard reads
|
||||
pack lexicons already loaded at runtime.
|
||||
* **No user-text mutation path.** Guard output is structural
|
||||
(status + rule_id + detail); the bounded disclosure string is a
|
||||
module-level constant, never composed from user input.
|
||||
* **AST seam test** in `tests/test_realizer_guard_runtime_seam.py`
|
||||
refuses truth-path imports of `generate.realizer_guard` from any
|
||||
module other than `chat/runtime.py` — the hook is centralized,
|
||||
not sprinkled.
|
||||
* **Telemetry redaction unchanged.** `realizer_guard_rule` is a
|
||||
rule_id from a closed set; `realizer_guard_status` is a closed
|
||||
enum. Neither field can leak user content.
|
||||
|
||||
### Replay determinism
|
||||
|
||||
Guard is a pure function of `(surface_string, pack_lexicons)`.
|
||||
Pack lexicons are deterministic from manifest checksums. Therefore
|
||||
guard verdicts are deterministic and replay-equivalent across runs.
|
||||
This composes with the existing trace_hash discipline: a rejected
|
||||
turn's trace_hash is computed on the pre-decoration **disclosure**
|
||||
surface (the user-facing surface), which keeps replay equivalence
|
||||
intact.
|
||||
|
||||
### What this explicitly does *not* do
|
||||
|
||||
* Does not fix the confirmation-tag intent bug. That is C2.
|
||||
* Does not change register substantive consumption. That is R6.
|
||||
* Does not add new teaching chains. That is T1.
|
||||
* Does not extend the rule set beyond R1/R2/R3. Subject-pack-
|
||||
residency, object-pack-residency, and proposition-graph residency
|
||||
are deferred to a follow-up ADR after C1 + C2 land and the failure
|
||||
modes are re-surveyed.
|
||||
* Does not retry or refine on rejection. Fail closed, route to
|
||||
disclosure, surface the verdict to telemetry, move on.
|
||||
|
||||
---
|
||||
|
||||
## Verification
|
||||
|
||||
```
|
||||
python -m pytest tests/test_realizer_guard_unit.py -q N passed
|
||||
python -m pytest tests/test_realizer_guard_runtime_seam.py -q N passed
|
||||
python -m pytest tests/test_realizer_guard_holdout.py -q N passed
|
||||
|
||||
python -m evals.realizer_guard.run_holdout exit 0
|
||||
core eval cognition byte-identical
|
||||
on currently
|
||||
passing cases
|
||||
|
||||
Curated lanes (must remain green):
|
||||
smoke / cognition / teaching / packs / runtime / algebra / full
|
||||
|
||||
Existing demos continue to pass:
|
||||
core demo register-tour exit 0
|
||||
core demo anchor-lens-tour exit 0
|
||||
core demo orthogonality-tour exit 0
|
||||
```
|
||||
|
||||
The C1 holdout cluster's exit code is the canonical coherence-floor
|
||||
gate — if any illegal articulation can escape, it exits non-zero.
|
||||
|
||||
---
|
||||
|
||||
## Follow-ups (not in C1 scope)
|
||||
|
||||
* **C2** — Confirmation-tag normalization. Make
|
||||
`"X reveals Y, right?"` preserve the propositional content
|
||||
(`subject=X, relation=reveals, object=Y`, intent=VERIFICATION)
|
||||
so the holdout cluster moves from "correctly rejected" to
|
||||
"correctly articulated". Eval cases re-pin under C2.
|
||||
* **R6** — Substantive register knobs on pack-grounded composers,
|
||||
plus replacement of the weak `surfaces_vary_at_least_once`
|
||||
register-tour gate with per-register substantive distinctness.
|
||||
* **T1** — Curated teaching depth for the core epistemic vocabulary
|
||||
appearing in the live demos (light / truth / knowledge / evidence
|
||||
/ wisdom / recall), through the existing
|
||||
`core teaching propose → review → accept` flywheel.
|
||||
* **Coherence rule extensions** — subject-pack-residency,
|
||||
object-pack-residency, proposition-graph residency. Deferred
|
||||
until the C1 / C2 failure surface is observed in CI for at least
|
||||
one full eval cycle.
|
||||
6
evals/realizer_guard/__init__.py
Normal file
6
evals/realizer_guard/__init__.py
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
"""ADR-0075 (C1) holdout cluster — illegal articulations that
|
||||
should be rejected by the realizer slot-type guard.
|
||||
|
||||
Exit code 0 iff every illegal candidate is rejected with the
|
||||
expected rule_id. See ``run_holdout.py``.
|
||||
"""
|
||||
151
evals/realizer_guard/run_holdout.py
Normal file
151
evals/realizer_guard/run_holdout.py
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
"""C1 holdout cluster — illegal-articulation prompts that the
|
||||
ADR-0075 realizer slot-type guard must reject.
|
||||
|
||||
Each prompt is run through ``CognitiveTurnPipeline`` and the
|
||||
recorded ``TurnEvent`` is checked for:
|
||||
|
||||
* ``realizer_guard_status == "rejected"``
|
||||
* ``realizer_guard_rule == <expected rule id>``
|
||||
* ``surface == DISCLOSURE_SURFACE``
|
||||
* ``walk_surface`` carries the pre-guard candidate (non-empty,
|
||||
not the disclosure string itself)
|
||||
* ``grounding_source == "none"``
|
||||
|
||||
The cluster is reached by priming the vault with three pack-known
|
||||
DEFINITION prompts first, which is the same sequence that exposed
|
||||
the original bug in the orthogonality tour. This is necessary
|
||||
because the bug surfaces only when the truth path reaches the
|
||||
vault-grounded realizer with cross-turn evidence — a fresh runtime
|
||||
on the bug prompt alone routes to the stub path and never produces
|
||||
the illegal articulation.
|
||||
|
||||
Exit code 0 iff ``all_claims_supported`` is true.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from chat.runtime import ChatRuntime
|
||||
from core.cognition.pipeline import CognitiveTurnPipeline
|
||||
from core.config import RuntimeConfig
|
||||
from generate.realizer_guard import DISCLOSURE_SURFACE
|
||||
|
||||
|
||||
_PRIMING_PROMPTS: tuple[str, ...] = (
|
||||
"What is light?",
|
||||
"Define knowledge.",
|
||||
"What is truth?",
|
||||
)
|
||||
|
||||
|
||||
# Each cluster entry is (prompt, expected_rule_id).
|
||||
#
|
||||
# The cluster covers the observed bug class: confirmation-tag
|
||||
# discourse particles (``right`` / ``no`` / ``yes``) that steer the
|
||||
# realizer to emit ``<particle> does not <noun>.`` — an illegal
|
||||
# do-support negation with a noun in the verb slot.
|
||||
#
|
||||
# All six prompts run on freshly-primed isolated runtimes (no shared
|
||||
# vault state across prompts) so each cell is order-independent.
|
||||
_HOLDOUT_PROMPTS: tuple[tuple[str, str], ...] = (
|
||||
("Light reveals truth, right?", "R2_aux_neg_requires_verb"),
|
||||
("Light reveals truth, no?", "R2_aux_neg_requires_verb"),
|
||||
("Light reveals truth, yes?", "R2_aux_neg_requires_verb"),
|
||||
("Knowledge supports truth, right?", "R2_aux_neg_requires_verb"),
|
||||
("Light grounds truth, right?", "R2_aux_neg_requires_verb"),
|
||||
("Light supports truth, right?", "R2_aux_neg_requires_verb"),
|
||||
)
|
||||
|
||||
|
||||
def _build_runtime() -> ChatRuntime:
|
||||
"""Fresh runtime with neutral register and unanchored lens —
|
||||
the bug reproduces on the truth path regardless of register
|
||||
or lens, so we pick the simplest combination."""
|
||||
return ChatRuntime(config=RuntimeConfig(
|
||||
register_pack_id="default_neutral_v1",
|
||||
))
|
||||
|
||||
|
||||
def _run_one(prompt: str, expected_rule: str) -> dict[str, Any]:
|
||||
runtime = _build_runtime()
|
||||
pipeline = CognitiveTurnPipeline(runtime=runtime)
|
||||
for primer in _PRIMING_PROMPTS:
|
||||
pipeline.run(primer)
|
||||
pipeline.run(prompt)
|
||||
turn_event = runtime.turn_log[-1]
|
||||
status = getattr(turn_event, "realizer_guard_status", "")
|
||||
rule = getattr(turn_event, "realizer_guard_rule", "")
|
||||
surface = turn_event.surface
|
||||
walk_surface = turn_event.walk_surface
|
||||
grounding_source = getattr(turn_event, "grounding_source", "")
|
||||
rejected = status == "rejected"
|
||||
rule_matches = (expected_rule == "") or (rule == expected_rule)
|
||||
surface_is_disclosure = surface == DISCLOSURE_SURFACE
|
||||
walk_preserves_candidate = bool(walk_surface) and walk_surface != DISCLOSURE_SURFACE
|
||||
grounding_forced_none = grounding_source == "none"
|
||||
return {
|
||||
"prompt": prompt,
|
||||
"expected_rule": expected_rule,
|
||||
"realizer_guard_status": status,
|
||||
"realizer_guard_rule": rule,
|
||||
"surface": surface,
|
||||
"walk_surface": walk_surface,
|
||||
"grounding_source": grounding_source,
|
||||
"rejected": rejected,
|
||||
"rule_matches": rule_matches,
|
||||
"surface_is_disclosure": surface_is_disclosure,
|
||||
"walk_preserves_candidate": walk_preserves_candidate,
|
||||
"grounding_forced_none": grounding_forced_none,
|
||||
"cell_supported": (
|
||||
rejected
|
||||
and rule_matches
|
||||
and surface_is_disclosure
|
||||
and walk_preserves_candidate
|
||||
and grounding_forced_none
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def run_holdout(*, emit_json: bool = False) -> dict[str, Any]:
|
||||
cells = [_run_one(p, r) for (p, r) in _HOLDOUT_PROMPTS]
|
||||
failures = [c for c in cells if not c["cell_supported"]]
|
||||
all_supported = not failures
|
||||
|
||||
if not emit_json:
|
||||
print()
|
||||
print("=" * 76)
|
||||
print(" ADR-0075 (C1) — realizer guard holdout cluster")
|
||||
print("=" * 76)
|
||||
print(f" Priming sequence: {list(_PRIMING_PROMPTS)}")
|
||||
print(f" Holdout prompts : {len(_HOLDOUT_PROMPTS)}")
|
||||
print()
|
||||
for c in cells:
|
||||
mark = "+" if c["cell_supported"] else "X"
|
||||
print(f" {mark} {c['prompt']!r}")
|
||||
print(f" guard_status : {c['realizer_guard_status']}")
|
||||
print(f" guard_rule : {c['realizer_guard_rule']}")
|
||||
print(f" surface : {c['surface']!r}")
|
||||
print(f" walk_surface (pre-G) : {c['walk_surface']!r}")
|
||||
print(f" grounding_source : {c['grounding_source']}")
|
||||
print()
|
||||
print(f" all_claims_supported : {all_supported}")
|
||||
print()
|
||||
|
||||
return {
|
||||
"priming": list(_PRIMING_PROMPTS),
|
||||
"holdout_prompts": [p for p, _ in _HOLDOUT_PROMPTS],
|
||||
"cells": cells,
|
||||
"failures": failures,
|
||||
"all_claims_supported": all_supported,
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
emit_json = "--json" in sys.argv
|
||||
report = run_holdout(emit_json=emit_json)
|
||||
if emit_json:
|
||||
print(json.dumps(report, indent=2, sort_keys=True, default=str))
|
||||
sys.exit(0 if report["all_claims_supported"] else 1)
|
||||
244
generate/realizer_guard.py
Normal file
244
generate/realizer_guard.py
Normal file
|
|
@ -0,0 +1,244 @@
|
|||
"""ADR-0075 (C1) — realizer slot-type guard.
|
||||
|
||||
Pure verifier that runs on every candidate surface produced by the
|
||||
truth path, before the surface is assigned to ``ChatResponse.surface``.
|
||||
Rejects illegal articulations; the runtime routes a rejected
|
||||
candidate to a deterministic bounded disclosure string.
|
||||
|
||||
Doctrine
|
||||
--------
|
||||
|
||||
This module is **admission control with a deterministic fallback**,
|
||||
not normalization. The guard never edits a surface; it only emits
|
||||
a verdict (``status`` + ``rule_id`` + ``detail``). That keeps it
|
||||
firmly outside CLAUDE.md's forbidden normalization sites — it does
|
||||
not repair a candidate, it refuses it.
|
||||
|
||||
Rules
|
||||
-----
|
||||
|
||||
C1 active rules (see ADR-0075):
|
||||
|
||||
* **R2_aux_neg_requires_verb** — after a do-support negation
|
||||
(``do not`` / ``does not`` / ``did not``), the immediately
|
||||
following non-adverb content token must have POS ``VERB``.
|
||||
* **R3_be_neg_requires_predicate** — after a be-negation
|
||||
(``is not`` / ``are not`` / ``was not`` / ``were not``), the
|
||||
immediately following non-adverb content token must have POS in
|
||||
``{NOUN, ADJ, DET, ADV, PRON}``.
|
||||
|
||||
R1 (``no_finite_verb``) was scoped into C1 originally but **deferred**
|
||||
during ratification: the active language-pack POS coverage does not
|
||||
list every English finite verb used by the teaching-chain realizer
|
||||
(notably ``requires`` / ``makes``), so R1 would falsely reject
|
||||
currently-passing cognition cases — a regression the ADR's
|
||||
byte-identity canary explicitly forbids. R1's intent is preserved
|
||||
for a follow-up phase that either broadens pack POS coverage or
|
||||
adds a closed English-vocabulary POS table to the guard.
|
||||
|
||||
Empty surfaces are exempt — those route through a separate
|
||||
disclosure path.
|
||||
|
||||
Determinism
|
||||
-----------
|
||||
|
||||
The guard is a pure function of ``(surface, pos_lookup)``. No I/O,
|
||||
no mutation, no globals beyond the closed sets defined in this
|
||||
module. Pack lexicons that back the ``pos_lookup`` callable are
|
||||
themselves deterministic from manifest checksums, so guard verdicts
|
||||
are replay-equivalent across runs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Literal
|
||||
|
||||
|
||||
DISCLOSURE_SURFACE = "I do not have a reviewed articulation for that yet."
|
||||
"""Bounded fallback surface used when the guard rejects a candidate.
|
||||
|
||||
Module-level constant — never composed from user input. The runtime
|
||||
substitutes this string for any rejected candidate before assigning
|
||||
to ``ChatResponse.surface``.
|
||||
"""
|
||||
|
||||
|
||||
_TOKEN_RE = re.compile(r"[A-Za-z][A-Za-z'_-]*")
|
||||
|
||||
|
||||
_FINITE_VERB_AUX: frozenset[str] = frozenset({
|
||||
"be", "am", "is", "are", "was", "were", "been", "being",
|
||||
"do", "does", "did", "doing", "done",
|
||||
"have", "has", "had", "having",
|
||||
"will", "would", "shall", "should",
|
||||
"can", "could", "may", "might", "must",
|
||||
})
|
||||
"""Finite verb forms recognized without pack POS lookup."""
|
||||
|
||||
|
||||
_PREDICATE_FUNCTION_WORDS: frozenset[str] = frozenset({
|
||||
"a", "an", "the",
|
||||
"i", "you", "he", "she", "it",
|
||||
"we", "they", "me", "him", "her", "us", "them",
|
||||
"this", "that", "these", "those",
|
||||
"my", "your", "his", "its", "our", "their",
|
||||
})
|
||||
"""Predicate-eligible function words (allowed after be-negation)."""
|
||||
|
||||
|
||||
_ADVERB_FUNCTION_WORDS: frozenset[str] = frozenset({
|
||||
"always", "never", "ever", "even", "only", "just",
|
||||
"really", "actually", "still", "yet", "also",
|
||||
"often", "sometimes", "usually", "rarely",
|
||||
})
|
||||
"""Adverbs skipped when looking past a negation to its target slot."""
|
||||
|
||||
|
||||
_DO_AUX: frozenset[str] = frozenset({"do", "does", "did"})
|
||||
_BE_AUX: frozenset[str] = frozenset({"is", "are", "was", "were"})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RealizerGuardVerdict:
|
||||
"""Outcome of a single guard check.
|
||||
|
||||
``status`` : ``"ok"`` when all rules pass; ``"rejected"`` when
|
||||
at least one rule fires.
|
||||
``rule_id`` : closed-set identifier of the failing rule
|
||||
(``""`` when status is ``"ok"``).
|
||||
``detail`` : short surface fragment showing the violation
|
||||
(``""`` when status is ``"ok"``).
|
||||
"""
|
||||
|
||||
status: Literal["ok", "rejected"]
|
||||
rule_id: str
|
||||
detail: str
|
||||
|
||||
|
||||
_OK_VERDICT = RealizerGuardVerdict(status="ok", rule_id="", detail="")
|
||||
|
||||
|
||||
def _tokens(surface: str) -> list[tuple[int, str]]:
|
||||
"""Return ordered ``(start_index, token)`` pairs.
|
||||
|
||||
Punctuation, digits, and bracket/quote characters are skipped.
|
||||
The regex selects ``A-Za-z`` runs with internal apostrophes,
|
||||
underscores, or hyphens.
|
||||
"""
|
||||
return [(m.start(), m.group(0)) for m in _TOKEN_RE.finditer(surface)]
|
||||
|
||||
|
||||
def _skip_adverbs(tokens: list[tuple[int, str]], start_idx: int) -> int:
|
||||
"""Return the index of the first non-adverb-function-word at or
|
||||
after ``start_idx``. Returns ``len(tokens)`` when the rest is
|
||||
exhausted entirely by adverbs.
|
||||
"""
|
||||
j = start_idx
|
||||
while j < len(tokens) and tokens[j][1].casefold() in _ADVERB_FUNCTION_WORDS:
|
||||
j += 1
|
||||
return j
|
||||
|
||||
|
||||
def _explicit_non_verb(
|
||||
token: str,
|
||||
pos_lookup: Callable[[str], str | None],
|
||||
) -> bool:
|
||||
"""True iff the token has an EXPLICIT non-VERB POS tag.
|
||||
|
||||
Unknown tokens (pack returns ``None``) fail open — the rule does
|
||||
not fire on them. This is the principled trigger for R2: only
|
||||
reject when the pack has explicitly classified the next-token as
|
||||
the wrong POS for a do-support negation slot.
|
||||
"""
|
||||
fold = token.casefold()
|
||||
if fold in _FINITE_VERB_AUX:
|
||||
return False
|
||||
pos = pos_lookup(token)
|
||||
if pos is None or pos == "VERB":
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _explicit_non_predicate(
|
||||
token: str,
|
||||
pos_lookup: Callable[[str], str | None],
|
||||
) -> bool:
|
||||
"""True iff the token has an EXPLICIT non-predicate POS tag
|
||||
(typically ``VERB``). Unknown tokens fail open.
|
||||
"""
|
||||
fold = token.casefold()
|
||||
if fold in _PREDICATE_FUNCTION_WORDS or fold in _ADVERB_FUNCTION_WORDS:
|
||||
return False
|
||||
pos = pos_lookup(token)
|
||||
if pos is None:
|
||||
return False
|
||||
return pos not in {"NOUN", "ADJ", "DET", "ADV", "PRON"}
|
||||
|
||||
|
||||
def check_surface(
|
||||
surface: str,
|
||||
*,
|
||||
pos_lookup: Callable[[str], str | None],
|
||||
) -> RealizerGuardVerdict:
|
||||
"""Apply C1's active rules (R2, R3) to ``surface``.
|
||||
|
||||
``pos_lookup`` should return one of the pack POS tags
|
||||
(``"NOUN"``, ``"VERB"``, ``"ADJ"``, ``"DET"``, ``"ADV"``,
|
||||
``"PRON"``, …) or ``None`` if the token is unknown.
|
||||
|
||||
**Fail-open on unknown POS, fail-closed on explicit wrong POS.**
|
||||
R2 fires only when the next-token has an explicit non-VERB POS;
|
||||
R3 fires only when the next-token has an explicit non-predicate
|
||||
POS (typically ``VERB``). Unknown tokens — words the pack
|
||||
doesn't list — pass through both rules unscathed, because the
|
||||
guard cannot prove they violate the slot type. This honors the
|
||||
byte-identity invariant on currently-passing cases where the
|
||||
realizer emits valid English that the cognition pack happens
|
||||
not to enumerate (e.g., ``ratified`` in PROCEDURE templates).
|
||||
|
||||
Rules are position-anchored: the scan walks the token stream
|
||||
once and emits the first violation it finds.
|
||||
"""
|
||||
if not surface.strip():
|
||||
return _OK_VERDICT
|
||||
tokens = _tokens(surface)
|
||||
if not tokens:
|
||||
return _OK_VERDICT
|
||||
|
||||
for i in range(len(tokens) - 1):
|
||||
cur = tokens[i][1].casefold()
|
||||
nxt = tokens[i + 1][1].casefold()
|
||||
if cur in _DO_AUX and nxt == "not":
|
||||
j = _skip_adverbs(tokens, i + 2)
|
||||
if j >= len(tokens):
|
||||
return RealizerGuardVerdict(
|
||||
status="rejected",
|
||||
rule_id="R2_aux_neg_requires_verb",
|
||||
detail=f"{tokens[i][1]} not <missing>",
|
||||
)
|
||||
tok = tokens[j][1]
|
||||
if _explicit_non_verb(tok, pos_lookup):
|
||||
return RealizerGuardVerdict(
|
||||
status="rejected",
|
||||
rule_id="R2_aux_neg_requires_verb",
|
||||
detail=f"{tokens[i][1]} not {tok}",
|
||||
)
|
||||
if cur in _BE_AUX and nxt == "not":
|
||||
j = _skip_adverbs(tokens, i + 2)
|
||||
if j >= len(tokens):
|
||||
return RealizerGuardVerdict(
|
||||
status="rejected",
|
||||
rule_id="R3_be_neg_requires_predicate",
|
||||
detail=f"{tokens[i][1]} not <missing>",
|
||||
)
|
||||
tok = tokens[j][1]
|
||||
if _explicit_non_predicate(tok, pos_lookup):
|
||||
return RealizerGuardVerdict(
|
||||
status="rejected",
|
||||
rule_id="R3_be_neg_requires_predicate",
|
||||
detail=f"{tokens[i][1]} not {tok}",
|
||||
)
|
||||
|
||||
return _OK_VERDICT
|
||||
155
tests/test_realizer_guard_holdout.py
Normal file
155
tests/test_realizer_guard_holdout.py
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
"""ADR-0075 (C1) — holdout cluster + byte-identity invariant tests.
|
||||
|
||||
These tests pin the two invariants named in the ADR:
|
||||
|
||||
* ``invariant_realizer_no_illegal_articulation`` — every prompt in
|
||||
the C1 holdout cluster is rejected by the guard with the expected
|
||||
rule_id, and the surface is replaced with the bounded disclosure
|
||||
string.
|
||||
* ``invariant_realizer_guard_byte_identity_on_currently_passing_cases``
|
||||
— every currently-passing cognition-lane DEFINITION prompt
|
||||
continues to produce a guard-accepted surface byte-identical to
|
||||
pre-C1 behavior.
|
||||
|
||||
The holdout cluster's exit code is the canonical coherence-floor
|
||||
gate (see ``evals/realizer_guard/run_holdout.py``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from chat.runtime import ChatRuntime
|
||||
from core.cognition.pipeline import CognitiveTurnPipeline
|
||||
from core.config import RuntimeConfig
|
||||
from evals.realizer_guard.run_holdout import (
|
||||
_HOLDOUT_PROMPTS,
|
||||
_PRIMING_PROMPTS,
|
||||
run_holdout,
|
||||
)
|
||||
from generate.realizer_guard import DISCLOSURE_SURFACE
|
||||
|
||||
|
||||
# ---------- invariant_realizer_no_illegal_articulation ----------
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def holdout_report():
|
||||
"""Run the cluster once per test module — parametrized tests
|
||||
then read from the cached report instead of re-running the full
|
||||
six-prompt cluster each time."""
|
||||
return run_holdout(emit_json=True)
|
||||
|
||||
|
||||
def test_holdout_cluster_all_claims_supported(holdout_report):
|
||||
assert holdout_report["all_claims_supported"] is True
|
||||
assert holdout_report["failures"] == []
|
||||
|
||||
|
||||
def test_holdout_cluster_size(holdout_report):
|
||||
assert len(holdout_report["cells"]) == len(_HOLDOUT_PROMPTS)
|
||||
assert len(holdout_report["cells"]) == 6
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prompt,expected_rule", list(_HOLDOUT_PROMPTS))
|
||||
def test_each_holdout_prompt_rejected(
|
||||
prompt: str, expected_rule: str, holdout_report,
|
||||
):
|
||||
cell = next(c for c in holdout_report["cells"] if c["prompt"] == prompt)
|
||||
assert cell["realizer_guard_status"] == "rejected"
|
||||
assert cell["realizer_guard_rule"] == expected_rule
|
||||
assert cell["surface"] == DISCLOSURE_SURFACE
|
||||
assert cell["walk_surface"] != DISCLOSURE_SURFACE
|
||||
assert cell["walk_surface"].strip() != ""
|
||||
assert cell["grounding_source"] == "none"
|
||||
|
||||
|
||||
def test_every_rejected_walk_surface_violates_R2(holdout_report):
|
||||
"""Sanity: each preserved pre-guard candidate must in fact
|
||||
violate the rule it was rejected under (round-trip check).
|
||||
|
||||
Uses the live runtime's pack POS table because the C1 rules
|
||||
fail-open on unknown POS — a null-lookup would not re-trigger
|
||||
R2 since the original trigger depends on the pack having
|
||||
classified the offending token (e.g. ``thought``) as ``NOUN``.
|
||||
"""
|
||||
from chat.runtime import ChatRuntime
|
||||
from core.config import RuntimeConfig
|
||||
from generate.realizer_guard import check_surface
|
||||
|
||||
rt = ChatRuntime(config=RuntimeConfig(
|
||||
register_pack_id="default_neutral_v1",
|
||||
))
|
||||
for cell in holdout_report["cells"]:
|
||||
v = check_surface(
|
||||
cell["walk_surface"],
|
||||
pos_lookup=rt._pos_by_surface.get,
|
||||
)
|
||||
assert v.status == "rejected", (
|
||||
f"walk_surface {cell['walk_surface']!r} should still "
|
||||
f"violate the rule under round-trip check"
|
||||
)
|
||||
assert v.rule_id == cell["realizer_guard_rule"]
|
||||
|
||||
|
||||
# ---------- byte-identity invariant on currently-passing cases ----------
|
||||
|
||||
|
||||
_CURRENTLY_PASSING_PROMPTS: tuple[tuple[str, str], ...] = (
|
||||
(
|
||||
"What is light?",
|
||||
"Light is a source of revelation that makes things knowable. "
|
||||
"pack-grounded (en_core_cognition_v1).",
|
||||
),
|
||||
(
|
||||
"Define knowledge.",
|
||||
"Knowledge is justified understanding grounded in evidence "
|
||||
"and recall. pack-grounded (en_core_cognition_v1).",
|
||||
),
|
||||
(
|
||||
"What is truth?",
|
||||
"Truth is a claim or state grounded by evidence and coherent "
|
||||
"judgment. pack-grounded (en_core_cognition_v1).",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prompt,expected_surface", _CURRENTLY_PASSING_PROMPTS)
|
||||
def test_currently_passing_cases_byte_identical(
|
||||
prompt: str, expected_surface: str,
|
||||
):
|
||||
"""ADR-0075 byte-identity invariant.
|
||||
|
||||
For every currently-passing cognition-lane DEFINITION case, the
|
||||
post-C1 surface must be byte-identical to the pre-C1 surface.
|
||||
If this regresses, the guard rule set is too aggressive and
|
||||
must be narrowed before merge.
|
||||
"""
|
||||
rt = ChatRuntime(config=RuntimeConfig(
|
||||
register_pack_id="default_neutral_v1",
|
||||
))
|
||||
pipeline = CognitiveTurnPipeline(runtime=rt)
|
||||
pipeline.run(prompt)
|
||||
te = rt.turn_log[-1]
|
||||
assert te.realizer_guard_status == "ok"
|
||||
assert te.realizer_guard_rule == ""
|
||||
assert te.surface == expected_surface
|
||||
|
||||
|
||||
def test_priming_sequence_all_pass_guard():
|
||||
"""Every priming prompt must pass the guard cleanly — the
|
||||
holdout cluster's rejection signal would be meaningless if the
|
||||
priming itself were also being rejected."""
|
||||
rt = ChatRuntime(config=RuntimeConfig(
|
||||
register_pack_id="default_neutral_v1",
|
||||
))
|
||||
pipeline = CognitiveTurnPipeline(runtime=rt)
|
||||
for p in _PRIMING_PROMPTS:
|
||||
pipeline.run(p)
|
||||
te = rt.turn_log[-1]
|
||||
assert te.realizer_guard_status == "ok", (
|
||||
f"Priming prompt {p!r} was rejected; holdout signal "
|
||||
f"would be confounded. Guard rule was "
|
||||
f"{te.realizer_guard_rule!r}, surface "
|
||||
f"{te.surface!r}"
|
||||
)
|
||||
200
tests/test_realizer_guard_runtime_seam.py
Normal file
200
tests/test_realizer_guard_runtime_seam.py
Normal file
|
|
@ -0,0 +1,200 @@
|
|||
"""ADR-0075 (C1) — runtime seam tests for the realizer guard.
|
||||
|
||||
These tests exercise the hook in ``chat/runtime.py`` end-to-end:
|
||||
|
||||
* On the main path, a rejected candidate is replaced by
|
||||
``DISCLOSURE_SURFACE`` on ``ChatResponse.surface`` and
|
||||
``TurnEvent.surface``; ``walk_surface`` preserves the pre-guard
|
||||
candidate; ``grounding_source`` is forced to ``"none"``;
|
||||
``realizer_guard_status`` and ``realizer_guard_rule`` carry the
|
||||
verdict.
|
||||
* The telemetry serializer surfaces both new fields.
|
||||
* The guard does not regress pack-grounded DEFINITION cases — those
|
||||
remain ``status="ok"`` byte-identical to pre-C1 behavior.
|
||||
* AST seam: only ``chat.runtime`` is allowed to import
|
||||
``generate.realizer_guard`` at module level. Other production
|
||||
modules must not depend on the guard directly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from chat.runtime import ChatRuntime
|
||||
from core.cognition.pipeline import CognitiveTurnPipeline
|
||||
from core.config import RuntimeConfig
|
||||
from generate.realizer_guard import DISCLOSURE_SURFACE
|
||||
|
||||
|
||||
_PRIMING = ("What is light?", "Define knowledge.", "What is truth?")
|
||||
_BUG_PROMPT = "Light reveals truth, right?"
|
||||
|
||||
|
||||
def _build_runtime() -> ChatRuntime:
|
||||
return ChatRuntime(config=RuntimeConfig(
|
||||
register_pack_id="default_neutral_v1",
|
||||
))
|
||||
|
||||
|
||||
def _run_holdout_sequence(rt: ChatRuntime) -> None:
|
||||
pipeline = CognitiveTurnPipeline(runtime=rt)
|
||||
for p in _PRIMING:
|
||||
pipeline.run(p)
|
||||
pipeline.run(_BUG_PROMPT)
|
||||
|
||||
|
||||
# ---------- Main path rejection routing ----------
|
||||
|
||||
|
||||
def test_rejected_candidate_replaces_surface_with_disclosure():
|
||||
rt = _build_runtime()
|
||||
_run_holdout_sequence(rt)
|
||||
te = rt.turn_log[-1]
|
||||
assert te.realizer_guard_status == "rejected"
|
||||
assert te.surface == DISCLOSURE_SURFACE
|
||||
|
||||
|
||||
def test_rejected_candidate_preserves_pre_guard_on_walk_surface():
|
||||
rt = _build_runtime()
|
||||
_run_holdout_sequence(rt)
|
||||
te = rt.turn_log[-1]
|
||||
assert te.walk_surface and te.walk_surface != DISCLOSURE_SURFACE
|
||||
assert "does not thought" in te.walk_surface
|
||||
|
||||
|
||||
def test_rejected_candidate_forces_grounding_source_none():
|
||||
rt = _build_runtime()
|
||||
_run_holdout_sequence(rt)
|
||||
te = rt.turn_log[-1]
|
||||
assert te.grounding_source == "none"
|
||||
|
||||
|
||||
def test_rejected_candidate_records_rule_id():
|
||||
rt = _build_runtime()
|
||||
_run_holdout_sequence(rt)
|
||||
te = rt.turn_log[-1]
|
||||
assert te.realizer_guard_rule == "R2_aux_neg_requires_verb"
|
||||
|
||||
|
||||
# ---------- ChatResponse mirrors TurnEvent ----------
|
||||
|
||||
|
||||
def test_chat_response_carries_guard_fields():
|
||||
rt = _build_runtime()
|
||||
pipeline = CognitiveTurnPipeline(runtime=rt)
|
||||
for p in _PRIMING:
|
||||
pipeline.run(p)
|
||||
# Pipeline runs and writes to turn_log; verify ChatResponse via
|
||||
# direct rt.chat() on the bug prompt produces a response that
|
||||
# mirrors the TurnEvent fields.
|
||||
response = rt.chat(_BUG_PROMPT)
|
||||
assert response.realizer_guard_status in {"ok", "rejected"}
|
||||
# When rejected, ChatResponse + TurnEvent must agree on the rule.
|
||||
te = rt.turn_log[-1]
|
||||
assert response.realizer_guard_status == te.realizer_guard_status
|
||||
assert response.realizer_guard_rule == te.realizer_guard_rule
|
||||
|
||||
|
||||
# ---------- Currently-passing cases stay passing ----------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prompt", [
|
||||
"What is light?",
|
||||
"Define knowledge.",
|
||||
"What is truth?",
|
||||
])
|
||||
def test_pack_grounded_definitions_pass_guard(prompt: str):
|
||||
"""ADR-0075 byte-identity invariant: currently-passing cases must
|
||||
not regress. These are the saturated cognition-lane DEFINITION
|
||||
cases — guard must accept all of them."""
|
||||
rt = _build_runtime()
|
||||
pipeline = CognitiveTurnPipeline(runtime=rt)
|
||||
pipeline.run(prompt)
|
||||
te = rt.turn_log[-1]
|
||||
assert te.realizer_guard_status == "ok"
|
||||
assert te.realizer_guard_rule == ""
|
||||
assert te.surface != DISCLOSURE_SURFACE
|
||||
assert te.surface.startswith(prompt.split()[-1].rstrip("?.").capitalize()) or "pack-grounded" in te.surface
|
||||
|
||||
|
||||
# ---------- Telemetry surface ----------
|
||||
|
||||
|
||||
def test_telemetry_includes_guard_fields():
|
||||
from chat.telemetry import serialize_turn_event
|
||||
rt = _build_runtime()
|
||||
_run_holdout_sequence(rt)
|
||||
te = rt.turn_log[-1]
|
||||
record = serialize_turn_event(te)
|
||||
assert "realizer_guard_status" in record
|
||||
assert "realizer_guard_rule" in record
|
||||
assert record["realizer_guard_status"] == "rejected"
|
||||
assert record["realizer_guard_rule"] == "R2_aux_neg_requires_verb"
|
||||
|
||||
|
||||
def test_telemetry_guard_fields_empty_on_pre_c1_events():
|
||||
"""A plain TurnEvent without the guard fields set should
|
||||
serialize to empty strings, preserving wire-format degradation."""
|
||||
from chat.telemetry import serialize_turn_event
|
||||
rt = _build_runtime()
|
||||
pipeline = CognitiveTurnPipeline(runtime=rt)
|
||||
pipeline.run("What is light?")
|
||||
te = rt.turn_log[-1]
|
||||
record = serialize_turn_event(te)
|
||||
# status should be "ok" (guard ran and accepted), rule empty
|
||||
assert record["realizer_guard_status"] == "ok"
|
||||
assert record["realizer_guard_rule"] == ""
|
||||
|
||||
|
||||
# ---------- AST seam ----------
|
||||
|
||||
|
||||
_PRODUCTION_MODULE_ROOTS = ("chat", "generate", "packs", "core", "language_packs")
|
||||
_GUARD_MODULE_NAME = "generate.realizer_guard"
|
||||
_ALLOWED_IMPORTERS = {"chat/runtime.py"}
|
||||
|
||||
|
||||
def _imports_guard(path: Path) -> bool:
|
||||
try:
|
||||
tree = ast.parse(path.read_text())
|
||||
except (OSError, SyntaxError):
|
||||
return False
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom):
|
||||
if node.module == _GUARD_MODULE_NAME:
|
||||
return True
|
||||
elif isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
if alias.name == _GUARD_MODULE_NAME:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def test_only_runtime_imports_realizer_guard():
|
||||
"""Production modules must not import the guard directly.
|
||||
|
||||
The guard is a singleton seam centralized in ``chat.runtime``.
|
||||
Other modules sprinkling guard calls would split the seam and
|
||||
make rejection routing harder to audit.
|
||||
"""
|
||||
repo_root = Path(__file__).resolve().parent.parent
|
||||
offenders: list[str] = []
|
||||
for root in _PRODUCTION_MODULE_ROOTS:
|
||||
root_path = repo_root / root
|
||||
if not root_path.exists():
|
||||
continue
|
||||
for py in root_path.rglob("*.py"):
|
||||
rel = py.relative_to(repo_root).as_posix()
|
||||
if rel == "generate/realizer_guard.py":
|
||||
continue
|
||||
if rel in _ALLOWED_IMPORTERS:
|
||||
continue
|
||||
if _imports_guard(py):
|
||||
offenders.append(rel)
|
||||
assert not offenders, (
|
||||
f"Only {sorted(_ALLOWED_IMPORTERS)} may import "
|
||||
f"{_GUARD_MODULE_NAME!r}; found in: {offenders}"
|
||||
)
|
||||
211
tests/test_realizer_guard_unit.py
Normal file
211
tests/test_realizer_guard_unit.py
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
"""ADR-0075 (C1) — realizer slot-type guard, pure-unit coverage.
|
||||
|
||||
Each test gives the guard a minimal POS lookup and asserts the
|
||||
expected verdict. No runtime, no pipeline, no fixtures.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from generate.realizer_guard import (
|
||||
DISCLOSURE_SURFACE,
|
||||
RealizerGuardVerdict,
|
||||
check_surface,
|
||||
)
|
||||
|
||||
|
||||
_POS: dict[str, str] = {
|
||||
# Nouns
|
||||
"Light": "NOUN", "light": "NOUN",
|
||||
"Truth": "NOUN", "truth": "NOUN",
|
||||
"Knowledge": "NOUN", "knowledge": "NOUN",
|
||||
"Right": "NOUN", "right": "NOUN",
|
||||
"thought": "NOUN", "evidence": "NOUN",
|
||||
"claim": "NOUN", "state": "NOUN",
|
||||
"source": "NOUN", "revelation": "NOUN",
|
||||
"judgment": "NOUN", "recall": "NOUN",
|
||||
"things": "NOUN", "understanding": "NOUN", "articulation": "NOUN",
|
||||
# Verbs
|
||||
"reveal": "VERB", "reveals": "VERB",
|
||||
"ground": "VERB", "grounds": "VERB",
|
||||
"make": "VERB", "makes": "VERB",
|
||||
"require": "VERB", "requires": "VERB",
|
||||
"perceive": "VERB",
|
||||
# Adjectives
|
||||
"reviewed": "ADJ", "knowable": "ADJ",
|
||||
"justified": "ADJ", "coherent": "ADJ", "grounded": "ADJ",
|
||||
# Function words / connectors (returned but not matched as VERB)
|
||||
"of": "ADP", "and": "CONJ", "in": "ADP", "or": "CONJ", "for": "ADP",
|
||||
"by": "ADP", "that": "PRON",
|
||||
}
|
||||
|
||||
|
||||
def _lookup(token: str) -> str | None:
|
||||
return _POS.get(token)
|
||||
|
||||
|
||||
def _verdict(surface: str) -> RealizerGuardVerdict:
|
||||
return check_surface(surface, pos_lookup=_lookup)
|
||||
|
||||
|
||||
# ---------- R1 deferred (see ADR-0075) ----------
|
||||
#
|
||||
# R1 ("must contain at least one finite verb") is deferred from
|
||||
# active C1 scope. These tests pin the **current** behavior — R1
|
||||
# surfaces pass even when verb-less — so a future re-enablement is
|
||||
# a deliberate, intentional change rather than a silent drift.
|
||||
|
||||
|
||||
def test_R1_deferred_verbless_surface_passes():
|
||||
v = _verdict("Light right.")
|
||||
assert v.status == "ok"
|
||||
|
||||
|
||||
def test_R1_deferred_bare_noun_phrase_passes():
|
||||
v = _verdict("Truth knowledge evidence.")
|
||||
assert v.status == "ok"
|
||||
|
||||
|
||||
# ---------- R2 ----------
|
||||
|
||||
|
||||
def test_R2_rejects_does_not_followed_by_noun():
|
||||
"""The canonical observed bug — 'Right does not thought.'"""
|
||||
v = _verdict("Right does not thought.")
|
||||
assert v.status == "rejected"
|
||||
assert v.rule_id == "R2_aux_neg_requires_verb"
|
||||
assert "does not thought" in v.detail
|
||||
|
||||
|
||||
def test_R2_rejects_do_not_followed_by_noun():
|
||||
v = _verdict("They do not knowledge.")
|
||||
assert v.status == "rejected"
|
||||
assert v.rule_id == "R2_aux_neg_requires_verb"
|
||||
|
||||
|
||||
def test_R2_rejects_did_not_followed_by_noun():
|
||||
v = _verdict("Light did not truth.")
|
||||
assert v.status == "rejected"
|
||||
assert v.rule_id == "R2_aux_neg_requires_verb"
|
||||
|
||||
|
||||
def test_R2_accepts_does_not_followed_by_verb():
|
||||
v = _verdict("Light does not reveal truth.")
|
||||
assert v.status == "ok"
|
||||
|
||||
|
||||
def test_R2_accepts_does_not_with_adverb_then_verb():
|
||||
"""Adverbs between 'not' and the verb are skipped."""
|
||||
v = _verdict("Light does not always reveal truth.")
|
||||
assert v.status == "ok"
|
||||
|
||||
|
||||
def test_R2_rejects_does_not_with_adverb_then_noun():
|
||||
v = _verdict("Right does not always thought.")
|
||||
assert v.status == "rejected"
|
||||
assert v.rule_id == "R2_aux_neg_requires_verb"
|
||||
|
||||
|
||||
def test_R2_rejects_does_not_at_end_of_surface():
|
||||
v = _verdict("Light does not.")
|
||||
assert v.status == "rejected"
|
||||
assert v.rule_id == "R2_aux_neg_requires_verb"
|
||||
assert "missing" in v.detail
|
||||
|
||||
|
||||
# ---------- R3 ----------
|
||||
|
||||
|
||||
def test_R3_accepts_is_not_followed_by_noun():
|
||||
v = _verdict("Light is not knowledge.")
|
||||
assert v.status == "ok"
|
||||
|
||||
|
||||
def test_R3_accepts_are_not_followed_by_adjective():
|
||||
v = _verdict("Claims are not reviewed.")
|
||||
assert v.status == "ok"
|
||||
|
||||
|
||||
def test_R3_accepts_is_not_with_determiner():
|
||||
v = _verdict("Light is not a claim.")
|
||||
assert v.status == "ok"
|
||||
|
||||
|
||||
def test_R3_rejects_is_not_followed_by_verb():
|
||||
v = _verdict("Light is not reveal.")
|
||||
assert v.status == "rejected"
|
||||
assert v.rule_id == "R3_be_neg_requires_predicate"
|
||||
|
||||
|
||||
def test_R3_rejects_was_not_followed_by_verb():
|
||||
v = _verdict("Light was not reveals.")
|
||||
assert v.status == "rejected"
|
||||
assert v.rule_id == "R3_be_neg_requires_predicate"
|
||||
|
||||
|
||||
def test_R3_rejects_is_not_at_end():
|
||||
v = _verdict("Light is not.")
|
||||
assert v.status == "rejected"
|
||||
assert v.rule_id == "R3_be_neg_requires_predicate"
|
||||
|
||||
|
||||
# ---------- DISCLOSURE_SURFACE itself must pass ----------
|
||||
|
||||
|
||||
def test_disclosure_surface_passes_guard():
|
||||
"""Critical: the fallback string must not be rejected by its own
|
||||
guard, otherwise routing on rejection would loop."""
|
||||
v = _verdict(DISCLOSURE_SURFACE)
|
||||
assert v.status == "ok"
|
||||
assert v.rule_id == ""
|
||||
|
||||
|
||||
# ---------- Empty / whitespace surfaces ----------
|
||||
|
||||
|
||||
def test_empty_surface_is_ok():
|
||||
v = _verdict("")
|
||||
assert v.status == "ok"
|
||||
|
||||
|
||||
def test_whitespace_only_surface_is_ok():
|
||||
v = _verdict(" \n \t ")
|
||||
assert v.status == "ok"
|
||||
|
||||
|
||||
def test_punctuation_only_surface_is_ok():
|
||||
"""No alphabetic tokens ⇒ guard cannot evaluate ⇒ pass through."""
|
||||
v = _verdict("... !!! ???")
|
||||
assert v.status == "ok"
|
||||
|
||||
|
||||
# ---------- Robustness ----------
|
||||
|
||||
|
||||
def test_unknown_token_after_aux_neg_fails_open():
|
||||
"""Unknown content tokens fall through pack lookup — R2 fails
|
||||
OPEN (does not reject) so the guard never regresses a
|
||||
currently-passing case where the realizer emits valid English
|
||||
the pack happens not to enumerate."""
|
||||
v = _verdict("Light does not xyzzy.")
|
||||
assert v.status == "ok"
|
||||
|
||||
|
||||
def test_unknown_token_after_be_neg_fails_open():
|
||||
"""R3 also fails open on unknown tokens. Critical for cases like
|
||||
'is not yet ratified' where 'ratified' is real English not in
|
||||
the pack lexicon."""
|
||||
v = _verdict("Step-by-step guidance is not yet ratified.")
|
||||
assert v.status == "ok"
|
||||
|
||||
|
||||
def test_known_pack_verb_after_aux_neg_passes():
|
||||
v = _verdict("Light does not perceive truth.")
|
||||
assert v.status == "ok"
|
||||
|
||||
|
||||
def test_verdict_is_frozen():
|
||||
v = _verdict("Light reveals truth.")
|
||||
with pytest.raises(Exception): # FrozenInstanceError
|
||||
v.status = "rejected" # type: ignore[misc]
|
||||
Loading…
Reference in a new issue