"""ADR-0075/0076 (C1/C2) — hybrid holdout + byte-identity tests. These tests pin the two invariants named in the ADR: * ``invariant_realizer_no_illegal_articulation`` — synthetic illegal candidates are rejected by the guard with the expected rule_id. * C2 confirmation prompts that used to reach the guard as illegal candidates now produce accepted propositional surfaces. * ``invariant_realizer_guard_byte_identity_on_currently_passing_cases`` — every currently-passing cognition-lane DEFINITION prompt continues to produce a guard-accepted surface byte-identical to pre-C1 behavior. The holdout cluster's exit code is the canonical coherence-floor gate (see ``evals/realizer_guard/run_holdout.py``). """ from __future__ import annotations import pytest from chat.runtime import ChatRuntime from core.cognition.pipeline import CognitiveTurnPipeline from core.config import RuntimeConfig from evals.realizer_guard.run_holdout import ( _HOLDOUT_PROMPTS, _PRIMING_PROMPTS, _SYNTHETIC_ILLEGAL_CANDIDATES, run_holdout, ) from generate.realizer_guard import DISCLOSURE_SURFACE # ---------- invariant_realizer_no_illegal_articulation ---------- @pytest.fixture(scope="module") def holdout_report(): """Run the cluster once per test module — parametrized tests then read from the cached report instead of re-running the full six-prompt cluster each time.""" return run_holdout(emit_json=True) def test_holdout_cluster_all_claims_supported(holdout_report): assert holdout_report["all_claims_supported"] is True assert holdout_report["failures"] == [] def test_holdout_cluster_size(holdout_report): assert len(holdout_report["synthetic_cells"]) == len(_SYNTHETIC_ILLEGAL_CANDIDATES) assert len(holdout_report["runtime_cells"]) == len(_HOLDOUT_PROMPTS) assert len(holdout_report["runtime_cells"]) == 6 @pytest.mark.parametrize("candidate,expected_rule", list(_SYNTHETIC_ILLEGAL_CANDIDATES)) def test_each_synthetic_illegal_candidate_rejected( candidate: str, expected_rule: str, holdout_report, ): cell = next(c for c in holdout_report["synthetic_cells"] if c["candidate"] == candidate) assert cell["realizer_guard_status"] == "rejected" assert cell["realizer_guard_rule"] == expected_rule @pytest.mark.parametrize("prompt", list(_HOLDOUT_PROMPTS)) def test_each_confirmation_prompt_now_articulates( prompt: str, holdout_report, ): cell = next(c for c in holdout_report["runtime_cells"] if c["prompt"] == prompt) assert cell["realizer_guard_status"] == "ok" assert cell["realizer_guard_rule"] == "" assert cell["grounding_source"] == "pack" assert "pack-grounded" in cell["surface"] assert cell["surface"] != DISCLOSURE_SURFACE # ---------- byte-identity invariant on currently-passing cases ---------- # Surfaces re-pinned after the gloss-aware composer landed; the prior # strings (e.g. "Light is a source of revelation that makes things knowable.") # were superseded by gloss-driven phrasings ("Light is a visible medium # that reveals truth.") that are visibly more grounded. The byte-identity # guard role is preserved by pinning the current outputs going forward. _CURRENTLY_PASSING_PROMPTS: tuple[tuple[str, str], ...] = ( ( "What is light?", "Light is a visible medium that reveals truth. " "pack-grounded (en_core_cognition_v1).", ), ( "Define knowledge.", "Knowledge is what a person knows from truth and evidence. " "pack-grounded (en_core_cognition_v1).", ), ( "What is truth?", "Truth is what is true. pack-grounded (en_core_cognition_v1).", ), ) @pytest.mark.parametrize("prompt,expected_surface", _CURRENTLY_PASSING_PROMPTS) def test_currently_passing_cases_byte_identical( prompt: str, expected_surface: str, ): """ADR-0075 byte-identity invariant. For every currently-passing cognition-lane DEFINITION case, the post-C1 surface must be byte-identical to the pre-C1 surface. If this regresses, the guard rule set is too aggressive and must be narrowed before merge. """ rt = ChatRuntime(config=RuntimeConfig( register_pack_id="default_neutral_v1", )) pipeline = CognitiveTurnPipeline(runtime=rt) pipeline.run(prompt) te = rt.turn_log[-1] assert te.realizer_guard_status == "ok" assert te.realizer_guard_rule == "" assert te.surface == expected_surface def test_priming_sequence_all_pass_guard(): """Every priming prompt must pass the guard cleanly — the holdout cluster's rejection signal would be meaningless if the priming itself were also being rejected.""" rt = ChatRuntime(config=RuntimeConfig( register_pack_id="default_neutral_v1", )) pipeline = CognitiveTurnPipeline(runtime=rt) for p in _PRIMING_PROMPTS: pipeline.run(p) te = rt.turn_log[-1] assert te.realizer_guard_status == "ok", ( f"Priming prompt {p!r} was rejected; holdout signal " f"would be confounded. Guard rule was " f"{te.realizer_guard_rule!r}, surface " f"{te.surface!r}" )