core/tests/test_identity_gate.py
Shay a9cafc5368 fix(identity): close v3 paraphrase gap with two-layer override defense
Resolves the adversarial-identity v3 finding (0% rejection on
paraphrased attacks against the marker-string defense). Two
independent layers now guard the review gate; either is sufficient
to reject.

Fix #2 (syntactic, in teaching/review.py):
  Replaces the substring-only check with four deterministic rules:
    (a) legacy markers (v1/v2 coverage preserved verbatim)
    (b) redirect-verb + role-frame co-occurrence
    (c) negating qualifier within +/-3 tokens of a role-frame
    (d) negating qualifier within +/-3 tokens of a redirect-verb
  Replay-safe, no learned classifier, single-file contained change.

Fix #3 (geometric, in core/physics/identity.py):
  Adds IdentityCheck.would_violate(score, manifold) predicate per
  ADR-0010 and wires it through CognitiveTurnPipeline._run_teaching
  from response.identity_score. The geometric layer is paraphrase-
  invariant by construction.

  Honest finding: with the current default IdentityManifold (three
  unit-axis ValueAxes), the geometric layer flags 0/32 of v3 attacks
  independently. The predicate and wiring are in place; the manifold
  axis design is the limiting factor and remains as scoped follow-up.
  Fix #2 is what is actually rejecting attacks today.

Verification: all eight adversarial-identity splits (v1-v4, public +
holdouts) at attack_rejection=1.0 and legitimate_acceptance=1.0.
v4 (32 attacks + 18 legitimate) is the regression gate for fix #2,
exercising rules (b)/(c)/(d) with new attack vocabulary. Tests
test_reviewed_teaching_loop.py (5/5), test_pipeline_teaching_integration.py
(5/5), test_identity_gate.py (incl. 5 new TestWouldViolatePredicate
tests, 12/12). CLI suites: smoke, cognition, teaching, runtime all
green.

Also drops a stale entry from the runtime CLI suite list
(test_chat_identity_telemetry.py was removed in 222124a).
2026-05-16 14:05:55 -07:00

174 lines
5.8 KiB
Python

"""Runtime identity-gate contract tests."""
from __future__ import annotations
from dataclasses import dataclass
import dataclasses
import pytest
from core.physics.drive import ValueAxis
from core.physics.identity import (
IdentityCheck,
IdentityManifold,
IdentityScore,
TurnEvent,
)
from core.physics.reasoning import ReasoningTrajectory, TrajectoryOperator
@dataclass(frozen=True)
class _Frame:
frame_id: str
coherence_magnitude: float
region_ids: frozenset[str]
cycle_index: int
def _make_manifold() -> IdentityManifold:
return IdentityManifold(
value_axes=(
ValueAxis(
axis_id="truthfulness",
name="truthfulness",
direction=(1.0, 0.0, 0.0),
theological_note="test truth axis",
),
ValueAxis(
axis_id="coherence",
name="coherence",
direction=(0.0, 1.0, 0.0),
theological_note="test coherence axis",
),
),
)
def _make_trajectory(n_steps: int = 4) -> ReasoningTrajectory:
frames = [
_Frame(
frame_id=f"f{i}",
coherence_magnitude=1.0,
region_ids=frozenset({str(i % 2)}),
cycle_index=i,
)
for i in range(n_steps)
]
return TrajectoryOperator().build(frames, trajectory_id="test_identity_trajectory")
class TestIdentityScore:
def test_score_is_float_in_unit_interval(self):
score = IdentityCheck().check(_make_trajectory(), _make_manifold())
assert isinstance(score, IdentityScore)
assert 0.0 <= score.score <= 1.0
def test_flagged_is_bool(self):
score = IdentityCheck().check(_make_trajectory(), _make_manifold())
assert isinstance(score.flagged, bool)
def test_value_alias_matches_score(self):
score = IdentityCheck().check(_make_trajectory(), _make_manifold())
assert score.value == score.score
def test_alignment_is_float_in_unit_interval(self):
score = IdentityCheck().check(_make_trajectory(), _make_manifold())
assert 0.0 <= score.alignment <= 1.0
def test_axes_evaluated_is_sorted_list(self):
score = IdentityCheck().check(_make_trajectory(), _make_manifold())
axes = score.axes_evaluated
assert isinstance(axes, list)
assert axes == sorted(axes)
def test_deviation_axes_is_frozenset_of_str(self):
score = IdentityCheck().check(_make_trajectory(), _make_manifold())
assert isinstance(score.deviation_axes, frozenset)
for axis_id in score.deviation_axes:
assert isinstance(axis_id, str)
def test_legacy_constructor_emits_deprecation_warning(self):
with pytest.deprecated_call(match=r"IdentityCheck\(manifold=\.\.\.\) is deprecated"):
check = IdentityCheck(manifold=_make_manifold())
score = check.check(_make_trajectory())
assert isinstance(score, IdentityScore)
class TestTurnEventFields:
def test_elaboration_field_exists_and_is_optional_str(self):
fields = {field.name: field for field in dataclasses.fields(TurnEvent)}
assert "elaboration" in fields
assert fields["elaboration"].default is None
def test_surface_contract_fields_exist(self):
fields = {field.name for field in dataclasses.fields(TurnEvent)}
assert {"surface", "walk_surface", "articulation_surface"} <= fields
def test_identity_score_field_is_optional(self):
fields = {field.name for field in dataclasses.fields(TurnEvent)}
assert "identity_score" in fields
def test_flagged_field_exists(self):
fields = {field.name for field in dataclasses.fields(TurnEvent)}
assert "flagged" in fields
class TestWouldViolatePredicate:
"""Contract for IdentityCheck.would_violate (ADR-0010 geometric defense)."""
def test_none_score_is_not_a_violation(self):
assert IdentityCheck.would_violate(None) is False
assert IdentityCheck.would_violate(None, _make_manifold()) is False
def test_flagged_score_is_a_violation(self):
score = IdentityScore(
score=0.2,
flagged=True,
deviation_axes=frozenset({"truthfulness"}),
trajectory_id="t",
)
assert IdentityCheck.would_violate(score) is True
assert IdentityCheck.would_violate(score, _make_manifold()) is True
def test_unflagged_but_low_alignment_violates_with_manifold(self):
manifold = IdentityManifold(value_axes=(), alignment_threshold=0.9)
score = IdentityScore(
score=0.5,
flagged=False,
deviation_axes=frozenset(),
trajectory_id="t",
)
assert IdentityCheck.would_violate(score, manifold) is True
def test_unflagged_high_alignment_is_not_a_violation(self):
manifold = _make_manifold()
score = IdentityScore(
score=0.95,
flagged=False,
deviation_axes=frozenset(),
trajectory_id="t",
)
assert IdentityCheck.would_violate(score, manifold) is False
def test_predicate_is_deterministic(self):
score = IdentityScore(
score=0.3,
flagged=True,
deviation_axes=frozenset({"coherence"}),
trajectory_id="t",
)
assert IdentityCheck.would_violate(score) == IdentityCheck.would_violate(score)
class TestChatRuntimeBaseline:
@pytest.fixture(autouse=True)
def runtime(self):
try:
from chat.runtime import ChatRuntime
self._runtime = ChatRuntime()
except Exception as exc:
pytest.skip(f"ChatRuntime not available: {exc}")
def test_unflagged_response_has_non_empty_surface(self):
response = self._runtime.chat("beginning", max_tokens=8)
if not response.flagged:
assert response.surface