core/tests/test_realizer_guard_unit.py
Shay 7cc2888ed2 feat(coherence): ADR-0075 — realizer slot-type guard (C1)
C1 coherence floor: a deterministic verifier that runs on every
candidate surface produced by the truth path, before assignment to
ChatResponse.surface.  Rejects illegal articulations and routes them
to a bounded disclosure string — admission control with a
deterministic fallback, not normalization.

Active rules (R1 deferred during ratification — see ADR):
  R2_aux_neg_requires_verb     — "<aux> not <wrong-POS>"  rejected
  R3_be_neg_requires_predicate — "<be>  not <verb>"       rejected

Fail-open on unknown POS, fail-closed on explicit wrong POS.
Cognition eval byte-identical (100/91.7/100/100).

Original bug class — "Light reveals truth, right?" → "Right does not
thought." — now routes to "I do not have a reviewed articulation for
that yet." with grounding_source=none, walk_surface preserving the
rejected candidate, and telemetry carrying R2_aux_neg_requires_verb.

Files:
  generate/realizer_guard.py            NEW — pure verifier
  chat/runtime.py                       hook on stub + main paths
  chat/telemetry.py                     serialize guard fields
  core/physics/identity.py              TurnEvent +2 fields
  evals/realizer_guard/run_holdout.py   NEW — 6-prompt cluster
  tests/test_realizer_guard_*.py        NEW — 46 tests (unit/seam/holdout)
  docs/decisions/ADR-0075-*.md          NEW — ratified

Invariants pinned:
  invariant_realizer_no_illegal_articulation
  invariant_realizer_guard_byte_identity_on_currently_passing_cases

Lanes (excluding 1 pre-existing TestDemoPreambles failure unrelated
to C1, already present at 4426f38):
  smoke 67/67  cognition 120/120(+1s)  teaching 17/17
  packs 6/6   runtime 19/19   algebra 132/132   full 2792/2793
2026-05-19 22:35:09 -07:00

211 lines
5.9 KiB
Python

"""ADR-0075 (C1) — realizer slot-type guard, pure-unit coverage.
Each test gives the guard a minimal POS lookup and asserts the
expected verdict. No runtime, no pipeline, no fixtures.
"""
from __future__ import annotations
import pytest
from generate.realizer_guard import (
DISCLOSURE_SURFACE,
RealizerGuardVerdict,
check_surface,
)
_POS: dict[str, str] = {
# Nouns
"Light": "NOUN", "light": "NOUN",
"Truth": "NOUN", "truth": "NOUN",
"Knowledge": "NOUN", "knowledge": "NOUN",
"Right": "NOUN", "right": "NOUN",
"thought": "NOUN", "evidence": "NOUN",
"claim": "NOUN", "state": "NOUN",
"source": "NOUN", "revelation": "NOUN",
"judgment": "NOUN", "recall": "NOUN",
"things": "NOUN", "understanding": "NOUN", "articulation": "NOUN",
# Verbs
"reveal": "VERB", "reveals": "VERB",
"ground": "VERB", "grounds": "VERB",
"make": "VERB", "makes": "VERB",
"require": "VERB", "requires": "VERB",
"perceive": "VERB",
# Adjectives
"reviewed": "ADJ", "knowable": "ADJ",
"justified": "ADJ", "coherent": "ADJ", "grounded": "ADJ",
# Function words / connectors (returned but not matched as VERB)
"of": "ADP", "and": "CONJ", "in": "ADP", "or": "CONJ", "for": "ADP",
"by": "ADP", "that": "PRON",
}
def _lookup(token: str) -> str | None:
return _POS.get(token)
def _verdict(surface: str) -> RealizerGuardVerdict:
return check_surface(surface, pos_lookup=_lookup)
# ---------- R1 deferred (see ADR-0075) ----------
#
# R1 ("must contain at least one finite verb") is deferred from
# active C1 scope. These tests pin the **current** behavior — R1
# surfaces pass even when verb-less — so a future re-enablement is
# a deliberate, intentional change rather than a silent drift.
def test_R1_deferred_verbless_surface_passes():
v = _verdict("Light right.")
assert v.status == "ok"
def test_R1_deferred_bare_noun_phrase_passes():
v = _verdict("Truth knowledge evidence.")
assert v.status == "ok"
# ---------- R2 ----------
def test_R2_rejects_does_not_followed_by_noun():
"""The canonical observed bug — 'Right does not thought.'"""
v = _verdict("Right does not thought.")
assert v.status == "rejected"
assert v.rule_id == "R2_aux_neg_requires_verb"
assert "does not thought" in v.detail
def test_R2_rejects_do_not_followed_by_noun():
v = _verdict("They do not knowledge.")
assert v.status == "rejected"
assert v.rule_id == "R2_aux_neg_requires_verb"
def test_R2_rejects_did_not_followed_by_noun():
v = _verdict("Light did not truth.")
assert v.status == "rejected"
assert v.rule_id == "R2_aux_neg_requires_verb"
def test_R2_accepts_does_not_followed_by_verb():
v = _verdict("Light does not reveal truth.")
assert v.status == "ok"
def test_R2_accepts_does_not_with_adverb_then_verb():
"""Adverbs between 'not' and the verb are skipped."""
v = _verdict("Light does not always reveal truth.")
assert v.status == "ok"
def test_R2_rejects_does_not_with_adverb_then_noun():
v = _verdict("Right does not always thought.")
assert v.status == "rejected"
assert v.rule_id == "R2_aux_neg_requires_verb"
def test_R2_rejects_does_not_at_end_of_surface():
v = _verdict("Light does not.")
assert v.status == "rejected"
assert v.rule_id == "R2_aux_neg_requires_verb"
assert "missing" in v.detail
# ---------- R3 ----------
def test_R3_accepts_is_not_followed_by_noun():
v = _verdict("Light is not knowledge.")
assert v.status == "ok"
def test_R3_accepts_are_not_followed_by_adjective():
v = _verdict("Claims are not reviewed.")
assert v.status == "ok"
def test_R3_accepts_is_not_with_determiner():
v = _verdict("Light is not a claim.")
assert v.status == "ok"
def test_R3_rejects_is_not_followed_by_verb():
v = _verdict("Light is not reveal.")
assert v.status == "rejected"
assert v.rule_id == "R3_be_neg_requires_predicate"
def test_R3_rejects_was_not_followed_by_verb():
v = _verdict("Light was not reveals.")
assert v.status == "rejected"
assert v.rule_id == "R3_be_neg_requires_predicate"
def test_R3_rejects_is_not_at_end():
v = _verdict("Light is not.")
assert v.status == "rejected"
assert v.rule_id == "R3_be_neg_requires_predicate"
# ---------- DISCLOSURE_SURFACE itself must pass ----------
def test_disclosure_surface_passes_guard():
"""Critical: the fallback string must not be rejected by its own
guard, otherwise routing on rejection would loop."""
v = _verdict(DISCLOSURE_SURFACE)
assert v.status == "ok"
assert v.rule_id == ""
# ---------- Empty / whitespace surfaces ----------
def test_empty_surface_is_ok():
v = _verdict("")
assert v.status == "ok"
def test_whitespace_only_surface_is_ok():
v = _verdict(" \n \t ")
assert v.status == "ok"
def test_punctuation_only_surface_is_ok():
"""No alphabetic tokens ⇒ guard cannot evaluate ⇒ pass through."""
v = _verdict("... !!! ???")
assert v.status == "ok"
# ---------- Robustness ----------
def test_unknown_token_after_aux_neg_fails_open():
"""Unknown content tokens fall through pack lookup — R2 fails
OPEN (does not reject) so the guard never regresses a
currently-passing case where the realizer emits valid English
the pack happens not to enumerate."""
v = _verdict("Light does not xyzzy.")
assert v.status == "ok"
def test_unknown_token_after_be_neg_fails_open():
"""R3 also fails open on unknown tokens. Critical for cases like
'is not yet ratified' where 'ratified' is real English not in
the pack lexicon."""
v = _verdict("Step-by-step guidance is not yet ratified.")
assert v.status == "ok"
def test_known_pack_verb_after_aux_neg_passes():
v = _verdict("Light does not perceive truth.")
assert v.status == "ok"
def test_verdict_is_frozen():
v = _verdict("Light reveals truth.")
with pytest.raises(Exception): # FrozenInstanceError
v.status = "rejected" # type: ignore[misc]