Resolves the adversarial-identity v3 finding (0% rejection on
paraphrased attacks against the marker-string defense). Two
independent layers now guard the review gate; either is sufficient
to reject.
Fix #2 (syntactic, in teaching/review.py):
Replaces the substring-only check with four deterministic rules:
(a) legacy markers (v1/v2 coverage preserved verbatim)
(b) redirect-verb + role-frame co-occurrence
(c) negating qualifier within +/-3 tokens of a role-frame
(d) negating qualifier within +/-3 tokens of a redirect-verb
Replay-safe, no learned classifier, single-file contained change.
Fix #3 (geometric, in core/physics/identity.py):
Adds IdentityCheck.would_violate(score, manifold) predicate per
ADR-0010 and wires it through CognitiveTurnPipeline._run_teaching
from response.identity_score. The geometric layer is paraphrase-
invariant by construction.
Honest finding: with the current default IdentityManifold (three
unit-axis ValueAxes), the geometric layer flags 0/32 of v3 attacks
independently. The predicate and wiring are in place; the manifold
axis design is the limiting factor and remains as scoped follow-up.
Fix #2 is what is actually rejecting attacks today.
Verification: all eight adversarial-identity splits (v1-v4, public +
holdouts) at attack_rejection=1.0 and legitimate_acceptance=1.0.
v4 (32 attacks + 18 legitimate) is the regression gate for fix #2,
exercising rules (b)/(c)/(d) with new attack vocabulary. Tests
test_reviewed_teaching_loop.py (5/5), test_pipeline_teaching_integration.py
(5/5), test_identity_gate.py (incl. 5 new TestWouldViolatePredicate
tests, 12/12). CLI suites: smoke, cognition, teaching, runtime all
green.
Also drops a stale entry from the runtime CLI suite list
(test_chat_identity_telemetry.py was removed in 222124a).
174 lines
5.8 KiB
Python
174 lines
5.8 KiB
Python
"""Runtime identity-gate contract tests."""
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
import dataclasses
|
|
|
|
import pytest
|
|
|
|
from core.physics.drive import ValueAxis
|
|
from core.physics.identity import (
|
|
IdentityCheck,
|
|
IdentityManifold,
|
|
IdentityScore,
|
|
TurnEvent,
|
|
)
|
|
from core.physics.reasoning import ReasoningTrajectory, TrajectoryOperator
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _Frame:
|
|
frame_id: str
|
|
coherence_magnitude: float
|
|
region_ids: frozenset[str]
|
|
cycle_index: int
|
|
|
|
|
|
def _make_manifold() -> IdentityManifold:
|
|
return IdentityManifold(
|
|
value_axes=(
|
|
ValueAxis(
|
|
axis_id="truthfulness",
|
|
name="truthfulness",
|
|
direction=(1.0, 0.0, 0.0),
|
|
theological_note="test truth axis",
|
|
),
|
|
ValueAxis(
|
|
axis_id="coherence",
|
|
name="coherence",
|
|
direction=(0.0, 1.0, 0.0),
|
|
theological_note="test coherence axis",
|
|
),
|
|
),
|
|
)
|
|
|
|
|
|
def _make_trajectory(n_steps: int = 4) -> ReasoningTrajectory:
|
|
frames = [
|
|
_Frame(
|
|
frame_id=f"f{i}",
|
|
coherence_magnitude=1.0,
|
|
region_ids=frozenset({str(i % 2)}),
|
|
cycle_index=i,
|
|
)
|
|
for i in range(n_steps)
|
|
]
|
|
return TrajectoryOperator().build(frames, trajectory_id="test_identity_trajectory")
|
|
|
|
|
|
class TestIdentityScore:
|
|
def test_score_is_float_in_unit_interval(self):
|
|
score = IdentityCheck().check(_make_trajectory(), _make_manifold())
|
|
assert isinstance(score, IdentityScore)
|
|
assert 0.0 <= score.score <= 1.0
|
|
|
|
def test_flagged_is_bool(self):
|
|
score = IdentityCheck().check(_make_trajectory(), _make_manifold())
|
|
assert isinstance(score.flagged, bool)
|
|
|
|
def test_value_alias_matches_score(self):
|
|
score = IdentityCheck().check(_make_trajectory(), _make_manifold())
|
|
assert score.value == score.score
|
|
|
|
def test_alignment_is_float_in_unit_interval(self):
|
|
score = IdentityCheck().check(_make_trajectory(), _make_manifold())
|
|
assert 0.0 <= score.alignment <= 1.0
|
|
|
|
def test_axes_evaluated_is_sorted_list(self):
|
|
score = IdentityCheck().check(_make_trajectory(), _make_manifold())
|
|
axes = score.axes_evaluated
|
|
assert isinstance(axes, list)
|
|
assert axes == sorted(axes)
|
|
|
|
def test_deviation_axes_is_frozenset_of_str(self):
|
|
score = IdentityCheck().check(_make_trajectory(), _make_manifold())
|
|
assert isinstance(score.deviation_axes, frozenset)
|
|
for axis_id in score.deviation_axes:
|
|
assert isinstance(axis_id, str)
|
|
|
|
def test_legacy_constructor_emits_deprecation_warning(self):
|
|
with pytest.deprecated_call(match=r"IdentityCheck\(manifold=\.\.\.\) is deprecated"):
|
|
check = IdentityCheck(manifold=_make_manifold())
|
|
score = check.check(_make_trajectory())
|
|
assert isinstance(score, IdentityScore)
|
|
|
|
|
|
class TestTurnEventFields:
|
|
def test_elaboration_field_exists_and_is_optional_str(self):
|
|
fields = {field.name: field for field in dataclasses.fields(TurnEvent)}
|
|
assert "elaboration" in fields
|
|
assert fields["elaboration"].default is None
|
|
|
|
def test_surface_contract_fields_exist(self):
|
|
fields = {field.name for field in dataclasses.fields(TurnEvent)}
|
|
assert {"surface", "walk_surface", "articulation_surface"} <= fields
|
|
|
|
def test_identity_score_field_is_optional(self):
|
|
fields = {field.name for field in dataclasses.fields(TurnEvent)}
|
|
assert "identity_score" in fields
|
|
|
|
def test_flagged_field_exists(self):
|
|
fields = {field.name for field in dataclasses.fields(TurnEvent)}
|
|
assert "flagged" in fields
|
|
|
|
|
|
class TestWouldViolatePredicate:
|
|
"""Contract for IdentityCheck.would_violate (ADR-0010 geometric defense)."""
|
|
|
|
def test_none_score_is_not_a_violation(self):
|
|
assert IdentityCheck.would_violate(None) is False
|
|
assert IdentityCheck.would_violate(None, _make_manifold()) is False
|
|
|
|
def test_flagged_score_is_a_violation(self):
|
|
score = IdentityScore(
|
|
score=0.2,
|
|
flagged=True,
|
|
deviation_axes=frozenset({"truthfulness"}),
|
|
trajectory_id="t",
|
|
)
|
|
assert IdentityCheck.would_violate(score) is True
|
|
assert IdentityCheck.would_violate(score, _make_manifold()) is True
|
|
|
|
def test_unflagged_but_low_alignment_violates_with_manifold(self):
|
|
manifold = IdentityManifold(value_axes=(), alignment_threshold=0.9)
|
|
score = IdentityScore(
|
|
score=0.5,
|
|
flagged=False,
|
|
deviation_axes=frozenset(),
|
|
trajectory_id="t",
|
|
)
|
|
assert IdentityCheck.would_violate(score, manifold) is True
|
|
|
|
def test_unflagged_high_alignment_is_not_a_violation(self):
|
|
manifold = _make_manifold()
|
|
score = IdentityScore(
|
|
score=0.95,
|
|
flagged=False,
|
|
deviation_axes=frozenset(),
|
|
trajectory_id="t",
|
|
)
|
|
assert IdentityCheck.would_violate(score, manifold) is False
|
|
|
|
def test_predicate_is_deterministic(self):
|
|
score = IdentityScore(
|
|
score=0.3,
|
|
flagged=True,
|
|
deviation_axes=frozenset({"coherence"}),
|
|
trajectory_id="t",
|
|
)
|
|
assert IdentityCheck.would_violate(score) == IdentityCheck.would_violate(score)
|
|
|
|
|
|
class TestChatRuntimeBaseline:
|
|
@pytest.fixture(autouse=True)
|
|
def runtime(self):
|
|
try:
|
|
from chat.runtime import ChatRuntime
|
|
self._runtime = ChatRuntime()
|
|
except Exception as exc:
|
|
pytest.skip(f"ChatRuntime not available: {exc}")
|
|
|
|
def test_unflagged_response_has_non_empty_surface(self):
|
|
response = self._runtime.chat("beginning", max_tokens=8)
|
|
if not response.flagged:
|
|
assert response.surface
|