core/tests/test_realizer_guard_holdout.py

"""ADR-0075 (C1) — holdout cluster + byte-identity invariant tests.

These tests pin the two invariants named in the ADR:

* ``invariant_realizer_no_illegal_articulation`` — every prompt in
  the C1 holdout cluster is rejected by the guard with the expected
  rule_id, and the surface is replaced with the bounded disclosure
  string.
* ``invariant_realizer_guard_byte_identity_on_currently_passing_cases``
  — every currently-passing cognition-lane DEFINITION prompt
  continues to produce a guard-accepted surface byte-identical to
  pre-C1 behavior.

The holdout cluster's exit code is the canonical coherence-floor
gate (see ``evals/realizer_guard/run_holdout.py``).
"""

from __future__ import annotations

import pytest

from chat.runtime import ChatRuntime
from core.cognition.pipeline import CognitiveTurnPipeline
from core.config import RuntimeConfig
from evals.realizer_guard.run_holdout import (
    _HOLDOUT_PROMPTS,
    _PRIMING_PROMPTS,
    run_holdout,
)
from generate.realizer_guard import DISCLOSURE_SURFACE


# ---------- invariant_realizer_no_illegal_articulation ----------


@pytest.fixture(scope="module")
def holdout_report():
    """Run the cluster once per test module — parametrized tests
    then read from the cached report instead of re-running the full
    six-prompt cluster each time."""
    return run_holdout(emit_json=True)


def test_holdout_cluster_all_claims_supported(holdout_report):
    assert holdout_report["all_claims_supported"] is True
    assert holdout_report["failures"] == []


def test_holdout_cluster_size(holdout_report):
    assert len(holdout_report["cells"]) == len(_HOLDOUT_PROMPTS)
    assert len(holdout_report["cells"]) == 6


@pytest.mark.parametrize("prompt,expected_rule", list(_HOLDOUT_PROMPTS))
def test_each_holdout_prompt_rejected(
    prompt: str, expected_rule: str, holdout_report,
):
    cell = next(c for c in holdout_report["cells"] if c["prompt"] == prompt)
    assert cell["realizer_guard_status"] == "rejected"
    assert cell["realizer_guard_rule"] == expected_rule
    assert cell["surface"] == DISCLOSURE_SURFACE
    assert cell["walk_surface"] != DISCLOSURE_SURFACE
    assert cell["walk_surface"].strip() != ""
    assert cell["grounding_source"] == "none"


def test_every_rejected_walk_surface_violates_R2(holdout_report):
    """Sanity: each preserved pre-guard candidate must in fact
    violate the rule it was rejected under (round-trip check).

    Uses the live runtime's pack POS table because the C1 rules
    fail-open on unknown POS — a null-lookup would not re-trigger
    R2 since the original trigger depends on the pack having
    classified the offending token (e.g. ``thought``) as ``NOUN``.
    """
    from chat.runtime import ChatRuntime
    from core.config import RuntimeConfig
    from generate.realizer_guard import check_surface

    rt = ChatRuntime(config=RuntimeConfig(
        register_pack_id="default_neutral_v1",
    ))
    for cell in holdout_report["cells"]:
        v = check_surface(
            cell["walk_surface"],
            pos_lookup=rt._pos_by_surface.get,
        )
        assert v.status == "rejected", (
            f"walk_surface {cell['walk_surface']!r} should still "
            f"violate the rule under round-trip check"
        )
        assert v.rule_id == cell["realizer_guard_rule"]


# ---------- byte-identity invariant on currently-passing cases ----------


_CURRENTLY_PASSING_PROMPTS: tuple[tuple[str, str], ...] = (
    (
        "What is light?",
        "Light is a source of revelation that makes things knowable. "
        "pack-grounded (en_core_cognition_v1).",
    ),
    (
        "Define knowledge.",
        "Knowledge is justified understanding grounded in evidence "
        "and recall. pack-grounded (en_core_cognition_v1).",
    ),
    (
        "What is truth?",
        "Truth is a claim or state grounded by evidence and coherent "
        "judgment. pack-grounded (en_core_cognition_v1).",
    ),
)


@pytest.mark.parametrize("prompt,expected_surface", _CURRENTLY_PASSING_PROMPTS)
def test_currently_passing_cases_byte_identical(
    prompt: str, expected_surface: str,
):
    """ADR-0075 byte-identity invariant.

    For every currently-passing cognition-lane DEFINITION case, the
    post-C1 surface must be byte-identical to the pre-C1 surface.
    If this regresses, the guard rule set is too aggressive and
    must be narrowed before merge.
    """
    rt = ChatRuntime(config=RuntimeConfig(
        register_pack_id="default_neutral_v1",
    ))
    pipeline = CognitiveTurnPipeline(runtime=rt)
    pipeline.run(prompt)
    te = rt.turn_log[-1]
    assert te.realizer_guard_status == "ok"
    assert te.realizer_guard_rule == ""
    assert te.surface == expected_surface


def test_priming_sequence_all_pass_guard():
    """Every priming prompt must pass the guard cleanly — the
    holdout cluster's rejection signal would be meaningless if the
    priming itself were also being rejected."""
    rt = ChatRuntime(config=RuntimeConfig(
        register_pack_id="default_neutral_v1",
    ))
    pipeline = CognitiveTurnPipeline(runtime=rt)
    for p in _PRIMING_PROMPTS:
        pipeline.run(p)
        te = rt.turn_log[-1]
        assert te.realizer_guard_status == "ok", (
            f"Priming prompt {p!r} was rejected; holdout signal "
            f"would be confounded.  Guard rule was "
            f"{te.realizer_guard_rule!r}, surface "
            f"{te.surface!r}"
        )