feat(adr-0039): audit completeness — TurnVerdicts bundle, stub TurnEvent, hedge_injected
Closes three audit gaps left by the ADR-0035→ADR-0038 pack-layer surface: 1. TurnVerdicts bundle (chat/verdicts.py) — frozen dataclass aggregating identity_score + safety_verdict + ethics_verdict + refusal_emitted + hedge_injected. Attached to both ChatResponse.verdicts and TurnEvent.verdicts. Fields typed as object for the same module-coupling reason as TurnEvent.safety_verdict. 2. Stub-path TurnEvent emission — _stub_response accepts optional tokens kwarg and appends a TurnEvent to turn_log when invoked from a real turn. Audit consumers can now iterate turn_log end-to-end without missing stub paths. Defensive call sites (correct() fallback) bypass the append by omitting tokens. 3. refusal_emitted / hedge_injected flags — runtime tracks whether it actually mutated the surface this turn. hedge_injected uses idempotent-on-prefix semantics (True iff the runtime ADDED a hedge, not iff a hedge happens to be present). Test-pattern note: previous "gate on rt.turn_log to detect main vs stub" pattern is now broken; updated to gate on walk_surface == _UNKNOWN_DOMAIN_SURFACE. One existing hedge-injection test gate updated accordingly. Back-compat: ADR-0035→0038 per-field accessors (response.safety_verdict, etc.) still work. New consumers should read response.verdicts. Files: - chat/verdicts.py (new) — TurnVerdicts dataclass - chat/runtime.py — _stub_response tokens kwarg + stub TurnEvent append + hedge_injected tracking + bundle construction - core/physics/identity.py — TurnEvent.verdicts: object = None - tests/test_turn_verdicts_bundle.py (new) — 16 tests - tests/test_hedge_injection.py — gate fix for stub detection - docs/decisions/ADR-0039-audit-completeness.md (new) Verification: - Combined pack-layer suite: 170 green (was 154 after ADR-0038) - CLI suites unchanged: smoke 67, runtime 19, cognition 121 - core eval cognition: intent 100%, versor_closure 100% (baseline)
This commit is contained in:
Shay
parent
ad8495d777
commit
f3cc408f82
6 changed files with 554 additions and 7 deletions
|
|
@ -16,6 +16,7 @@ from chat.refusal import (
|
|||
inject_hedge,
|
||||
should_inject_hedge,
|
||||
)
|
||||
from chat.verdicts import TurnVerdicts
|
||||
from core.config import DEFAULT_CONFIG, DEFAULT_IDENTITY_PACK, RuntimeConfig
|
||||
from core.physics.drive import DriveGradientMap, GradientField
|
||||
from core.physics.energy import EnergyProfile
|
||||
|
|
@ -253,6 +254,11 @@ class ChatResponse:
|
|||
# ``None`` only on stub/refusal paths that bypass the turn loop.
|
||||
safety_verdict: object = None
|
||||
ethics_verdict: object = None
|
||||
# ADR-0039 — unified TurnVerdicts bundle carrying identity / safety
|
||||
# / ethics verdicts and the two remediation flags
|
||||
# (refusal_emitted, hedge_injected). Typed as ``object`` to avoid
|
||||
# coupling at module-resolution time; downcast at use site.
|
||||
verdicts: object = None
|
||||
|
||||
|
||||
class ChatRuntime:
|
||||
|
|
@ -459,7 +465,12 @@ class ChatRuntime:
|
|||
axis_hedges=axis_hedges,
|
||||
)
|
||||
|
||||
def _stub_response(self, field_state: FieldState) -> ChatResponse:
|
||||
def _stub_response(
|
||||
self,
|
||||
field_state: FieldState,
|
||||
*,
|
||||
tokens: tuple[str, ...] = (),
|
||||
) -> ChatResponse:
|
||||
zero = np.zeros(field_state.F.shape, dtype=np.float32)
|
||||
prop = Proposition(
|
||||
subject="",
|
||||
|
|
@ -510,11 +521,47 @@ class ChatRuntime:
|
|||
refusal_surface = build_refusal_surface(
|
||||
safety_verdict, ethics_verdict, self.ethics_pack,
|
||||
)
|
||||
if refusal_surface is not None:
|
||||
refusal_emitted = refusal_surface is not None
|
||||
if refusal_emitted:
|
||||
response_surface = refusal_surface
|
||||
self._last_refusal_was_typed = True
|
||||
else:
|
||||
response_surface = _UNKNOWN_DOMAIN_SURFACE
|
||||
# ADR-0038 — hedge injection does NOT run on the stub path
|
||||
# (the unknown-domain marker is already a disclosure surface;
|
||||
# prepending a hedge would be a confused double-disclosure).
|
||||
# ``hedge_injected`` is therefore always False on stub paths.
|
||||
verdicts_bundle = TurnVerdicts(
|
||||
identity_score=None,
|
||||
safety_verdict=safety_verdict,
|
||||
ethics_verdict=ethics_verdict,
|
||||
refusal_emitted=refusal_emitted,
|
||||
hedge_injected=False,
|
||||
)
|
||||
# ADR-0039 — emit a TurnEvent on stub paths too so ``turn_log``
|
||||
# covers the entire turn stream for audit consumers. Only
|
||||
# append when invoked from a real turn (``tokens`` is
|
||||
# non-empty); defensive call sites that pass no tokens
|
||||
# preserve the prior bypass-turn_log behavior.
|
||||
if tokens:
|
||||
stub_event = TurnEvent(
|
||||
turn=max(self._context.turn - 1, 0),
|
||||
input_tokens=tokens,
|
||||
surface=response_surface,
|
||||
walk_surface=_UNKNOWN_DOMAIN_SURFACE,
|
||||
articulation_surface=_UNKNOWN_DOMAIN_SURFACE,
|
||||
dialogue_role="assert",
|
||||
identity_score=None,
|
||||
cycle_cost_total=0.0,
|
||||
vault_hits=0,
|
||||
versor_condition=float(versor_condition(field_state.F)),
|
||||
flagged=False,
|
||||
elaboration=None,
|
||||
safety_verdict=safety_verdict,
|
||||
ethics_verdict=ethics_verdict,
|
||||
verdicts=verdicts_bundle,
|
||||
)
|
||||
self.turn_log.append(stub_event)
|
||||
return ChatResponse(
|
||||
surface=response_surface,
|
||||
proposition=prop,
|
||||
|
|
@ -533,6 +580,7 @@ class ChatRuntime:
|
|||
flagged=False,
|
||||
safety_verdict=safety_verdict,
|
||||
ethics_verdict=ethics_verdict,
|
||||
verdicts=verdicts_bundle,
|
||||
)
|
||||
|
||||
def chat(self, text: str, max_tokens: int | None = None) -> ChatResponse:
|
||||
|
|
@ -564,7 +612,7 @@ class ChatRuntime:
|
|||
dialogue_role="assert",
|
||||
metadata={"unknown": True, "unknown_source": gate_decision.source},
|
||||
)
|
||||
return self._stub_response(committed)
|
||||
return self._stub_response(committed, tokens=tuple(filtered))
|
||||
|
||||
field_state = self._context.commit_ingest(filtered)
|
||||
field_state = self._apply_drive_bias(field_state)
|
||||
|
|
@ -697,7 +745,9 @@ class ChatRuntime:
|
|||
refusal_surface = build_refusal_surface(
|
||||
safety_verdict, ethics_verdict, self.ethics_pack,
|
||||
)
|
||||
if refusal_surface is not None:
|
||||
refusal_emitted = refusal_surface is not None
|
||||
hedge_injected = False
|
||||
if refusal_emitted:
|
||||
response_surface = refusal_surface
|
||||
self._last_refusal_was_typed = True
|
||||
else:
|
||||
|
|
@ -712,7 +762,19 @@ class ChatRuntime:
|
|||
# audit (same discipline as ADR-0036).
|
||||
if should_inject_hedge(ethics_verdict, self.ethics_pack):
|
||||
hedge_prefix = build_hedge_prefix(self.identity_manifold)
|
||||
before = response_surface
|
||||
response_surface = inject_hedge(response_surface, hedge_prefix)
|
||||
hedge_injected = response_surface != before
|
||||
# ADR-0039 — unified TurnVerdicts bundle attached to both
|
||||
# ChatResponse and TurnEvent. Audit consumers read the bundle
|
||||
# instead of correlating individual fields.
|
||||
verdicts_bundle = TurnVerdicts(
|
||||
identity_score=identity_score,
|
||||
safety_verdict=safety_verdict,
|
||||
ethics_verdict=ethics_verdict,
|
||||
refusal_emitted=refusal_emitted,
|
||||
hedge_injected=hedge_injected,
|
||||
)
|
||||
turn_event = TurnEvent(
|
||||
turn=self._context.turn - 1,
|
||||
input_tokens=tuple(filtered),
|
||||
|
|
@ -728,6 +790,7 @@ class ChatRuntime:
|
|||
elaboration=sentence_plan.elaboration,
|
||||
safety_verdict=safety_verdict,
|
||||
ethics_verdict=ethics_verdict,
|
||||
verdicts=verdicts_bundle,
|
||||
)
|
||||
self.turn_log.append(turn_event)
|
||||
return ChatResponse(
|
||||
|
|
@ -750,6 +813,7 @@ class ChatRuntime:
|
|||
region_was_unconstrained=result.region_was_unconstrained,
|
||||
safety_verdict=safety_verdict,
|
||||
ethics_verdict=ethics_verdict,
|
||||
verdicts=verdicts_bundle,
|
||||
)
|
||||
|
||||
def _unknown_domain_response(self, field_state: FieldState, filtered: list[str]) -> ChatResponse:
|
||||
|
|
|
|||
49
chat/verdicts.py
Normal file
49
chat/verdicts.py
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
"""ADR-0039 — unified per-turn verdict bundle.
|
||||
|
||||
Three orthogonal verdict surfaces fire on every turn:
|
||||
|
||||
* ``IdentityScore`` — geometric alignment against the manifold.
|
||||
* ``SafetyVerdict`` — universal-floor boundary predicates.
|
||||
* ``EthicsVerdict`` — deployment-commitment predicates.
|
||||
|
||||
Plus two remediation flags captured by the runtime:
|
||||
|
||||
* ``refusal_emitted`` — typed refusal replaced the surface this turn.
|
||||
* ``hedge_injected`` — manifold's hedge phrase was prepended.
|
||||
|
||||
The bundle is a convenience aggregate. The individual fields remain
|
||||
on ``ChatResponse`` and ``TurnEvent`` for back-compat with callers
|
||||
written against ADR-0035 / ADR-0036; the bundle is the new single
|
||||
accessor for downstream telemetry and audit consumers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TurnVerdicts:
|
||||
"""Single-turn verdict bundle.
|
||||
|
||||
Fields are typed as ``object`` to avoid coupling this module's
|
||||
consumers to the concrete packs.* and core.physics.identity
|
||||
types at import time. Callers that need the concrete shape
|
||||
downcast at use site — same discipline ``TurnEvent`` uses.
|
||||
|
||||
The two remediation flags are derived from the surface that was
|
||||
actually emitted:
|
||||
|
||||
* ``refusal_emitted=True`` iff the response surface is a typed
|
||||
refusal (ADR-0036 / ADR-0037).
|
||||
* ``hedge_injected=True`` iff a hedge phrase was prepended this
|
||||
turn (ADR-0038).
|
||||
|
||||
The two are mutually exclusive: refusal supersedes hedge.
|
||||
"""
|
||||
|
||||
identity_score: object
|
||||
safety_verdict: object
|
||||
ethics_verdict: object
|
||||
refusal_emitted: bool
|
||||
hedge_injected: bool
|
||||
|
|
@ -280,3 +280,8 @@ class TurnEvent:
|
|||
# Typed as ``object`` to avoid coupling identity.py to packs.*.
|
||||
safety_verdict: object = None
|
||||
ethics_verdict: object = None
|
||||
# ADR-0039 — unified verdict bundle (TurnVerdicts). Typed as
|
||||
# ``object`` to avoid coupling identity.py to chat.verdicts.
|
||||
# Carries refusal_emitted / hedge_injected remediation flags
|
||||
# alongside the three verdict surfaces.
|
||||
verdicts: object = None
|
||||
|
|
|
|||
186
docs/decisions/ADR-0039-audit-completeness.md
Normal file
186
docs/decisions/ADR-0039-audit-completeness.md
Normal file
|
|
@ -0,0 +1,186 @@
|
|||
# ADR-0039: Audit Completeness — `TurnVerdicts` Bundle, Stub-Path `TurnEvent`, `hedge_injected` Signal
|
||||
|
||||
**Status:** Accepted (2026-05-17)
|
||||
**Author:** Joshua Shay + planner pass
|
||||
**Companion docs:** [`ADR-0035-turn-loop-verdict-surfacing.md`](ADR-0035-turn-loop-verdict-surfacing.md), [`ADR-0036-safety-refusal-policy.md`](ADR-0036-safety-refusal-policy.md), [`ADR-0037-per-predicate-ethics-refusal.md`](ADR-0037-per-predicate-ethics-refusal.md), [`ADR-0038-hedge-injection.md`](ADR-0038-hedge-injection.md)
|
||||
|
||||
## Context
|
||||
|
||||
After ADR-0035 → ADR-0038, the runtime emits three verdicts per turn
|
||||
(identity / safety / ethics) and two possible remediations (typed
|
||||
refusal, hedge injection). But auditing it had three rough edges:
|
||||
|
||||
1. **Per-field correlation.** An audit consumer had to read three
|
||||
separate fields (`identity_score`, `safety_verdict`,
|
||||
`ethics_verdict`) and *infer* the remediation by inspecting the
|
||||
surface text (does it start with the refusal prefix? with the
|
||||
hedge phrase?). Surface-text inference is brittle.
|
||||
2. **Stub-path TurnEvent gap.** Stub turns (cold start, unknown
|
||||
domain) bypassed `turn_log.append()`. ADR-0035 already noted
|
||||
this as a known limit. The result: audit consumers reading the
|
||||
turn stream couldn't see stub turns at all, even though stub turns
|
||||
*did* carry a `ChatResponse.safety_verdict` and
|
||||
`ChatResponse.ethics_verdict`.
|
||||
3. **`hedge_injected` invisibility.** Whether the runtime actually
|
||||
prepended a hedge this turn could only be detected by surface
|
||||
inspection (does it start with `preferred_hedge_soft`?). An
|
||||
audit consumer wanting "count hedged turns this hour" had to
|
||||
re-implement runtime decision logic.
|
||||
|
||||
The three rough edges share a root cause: the runtime knew the
|
||||
answers but didn't *surface* them in a form callers could read.
|
||||
|
||||
## Decision
|
||||
|
||||
Three changes land together. Each is small; together they close the
|
||||
audit gap.
|
||||
|
||||
### 1. `TurnVerdicts` bundle type
|
||||
|
||||
A new frozen dataclass in `chat/verdicts.py`:
|
||||
|
||||
```python
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TurnVerdicts:
|
||||
identity_score: object # IdentityScore | None
|
||||
safety_verdict: object # SafetyVerdict | None
|
||||
ethics_verdict: object # EthicsVerdict | None
|
||||
refusal_emitted: bool # ADR-0036 / ADR-0037
|
||||
hedge_injected: bool # ADR-0038
|
||||
```
|
||||
|
||||
Fields are typed `object` for the same reason `TurnEvent.safety_verdict`
|
||||
was: avoid coupling `chat/verdicts.py` to packs.* at module-resolution
|
||||
time. Audit consumers downcast at use site.
|
||||
|
||||
The bundle is attached to both `ChatResponse.verdicts` and
|
||||
`TurnEvent.verdicts`. The pre-existing individual fields
|
||||
(`safety_verdict`, `ethics_verdict`) remain — back-compat with
|
||||
ADR-0035 callers — but new consumers should read the bundle.
|
||||
|
||||
### 2. Stub-path `TurnEvent` emission
|
||||
|
||||
`_stub_response` now accepts an optional `tokens` kwarg. When invoked
|
||||
from a real turn (with `tokens` non-empty), it constructs and appends
|
||||
a `TurnEvent` to `turn_log` before returning the `ChatResponse`.
|
||||
|
||||
The stub event records:
|
||||
|
||||
| Field | Value |
|
||||
|---|---|
|
||||
| `turn` | `self._context.turn - 1` (after `finalize_turn` already ran) |
|
||||
| `input_tokens` | tokens passed in |
|
||||
| `surface` | typed refusal if it fired, else the unknown-domain marker |
|
||||
| `walk_surface` | unknown-domain marker (preserved) |
|
||||
| `articulation_surface` | unknown-domain marker (preserved) |
|
||||
| `identity_score` | `None` (no trajectory ran) |
|
||||
| `cycle_cost_total` | `0.0` |
|
||||
| `vault_hits` | `0` |
|
||||
| `versor_condition` | `versor_condition(field_state.F)` |
|
||||
| `flagged` | `False` |
|
||||
| `safety_verdict` / `ethics_verdict` / `verdicts` | computed verdicts |
|
||||
|
||||
The `correct()` fallback path still calls `_stub_response` without
|
||||
tokens — that's a defensive call where no real "turn" happened, and
|
||||
appending a `TurnEvent` would mis-record the audit stream.
|
||||
|
||||
### 3. `hedge_injected` signal
|
||||
|
||||
The main turn path tracks whether the runtime actually mutated the
|
||||
surface during hedge injection:
|
||||
|
||||
```python
|
||||
before = response_surface
|
||||
response_surface = inject_hedge(response_surface, hedge_prefix)
|
||||
hedge_injected = response_surface != before
|
||||
```
|
||||
|
||||
`inject_hedge()` is idempotent on prefix (ADR-0038) — if the surface
|
||||
already begins with the hedge phrase, the function returns it
|
||||
unchanged and `hedge_injected` stays `False`. This is the correct
|
||||
audit semantic: "did the runtime ADD a hedge this turn?", not "is
|
||||
there a hedge somewhere in the surface?"
|
||||
|
||||
Stub paths always report `hedge_injected=False` (ADR-0038 prohibits
|
||||
hedge on stub).
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
* **One field, full picture.** An audit consumer reads
|
||||
`response.verdicts` (or `event.verdicts`) and gets identity +
|
||||
safety + ethics + remediation flags. No correlation across fields.
|
||||
* **Surface-inspection no longer needed.** `refusal_emitted` and
|
||||
`hedge_injected` answer the runtime-decision questions directly.
|
||||
* **Stub turns are now first-class audit events.** `turn_log` now
|
||||
covers the entire turn stream; downstream telemetry and replay
|
||||
systems can iterate over `turn_log` without missing stub paths.
|
||||
* **Mutual exclusion is verifiable.** Existing test
|
||||
(`test_refusal_and_hedge_never_both_true`) confirms the runtime
|
||||
contract holds at the bundle level.
|
||||
* **No back-compat breakage.** ADR-0035 / ADR-0036 / ADR-0037 /
|
||||
ADR-0038 individual fields and helpers still work.
|
||||
|
||||
### Negative / risks
|
||||
|
||||
* **Stub turns now have `identity_score=None` in the audit stream.**
|
||||
Downstream consumers that assumed every TurnEvent carried an
|
||||
IdentityScore need to handle `None`. Mitigated: this was already
|
||||
true on `ChatResponse.identity_score` for stub paths; consumers
|
||||
reading the stream just had no stream entries at all before, and
|
||||
now they have entries with `identity_score=None`. The change is
|
||||
additive.
|
||||
* **The bundle duplicates per-field state.** `safety_verdict` is
|
||||
reachable as both `response.safety_verdict` and
|
||||
`response.verdicts.safety_verdict`. Acceptable cost for
|
||||
back-compat; a future ADR could remove the per-field accessors if
|
||||
no in-tree consumer still uses them.
|
||||
* **`hedge_injected` doesn't distinguish "would have fired but
|
||||
idempotent" from "didn't fire at all."** Two distinct cases
|
||||
collapse to `hedge_injected=False`. Audit consumers wanting that
|
||||
distinction can read the ethics_verdict directly.
|
||||
* **Hedge-injection test gate updated.** The pre-ADR-0039 gate
|
||||
("skip if `turn_log` empty") no longer discriminates stub vs main
|
||||
path; the test updated to gate on `walk_surface ==
|
||||
_UNKNOWN_DOMAIN_SURFACE` instead. No semantic change.
|
||||
|
||||
## Verification
|
||||
|
||||
* `tests/test_turn_verdicts_bundle.py` — 16 tests covering: bundle
|
||||
shape and frozen contract; `ChatResponse` and `TurnEvent` carry
|
||||
the bundle; stub path appends a `TurnEvent` with input tokens,
|
||||
unknown walk/articulation surfaces, `identity_score=None`,
|
||||
recorded versor condition; `refusal_emitted` flag toggles on
|
||||
forced safety violation, appears symmetrically on
|
||||
`ChatResponse.verdicts` and `TurnEvent.verdicts`; `hedge_injected`
|
||||
flag defaults False, stays False on stub paths even with opt-in;
|
||||
mutual exclusion (refusal supersedes hedge); response and event
|
||||
bundles agree on remediation flags and reference the same
|
||||
underlying verdicts.
|
||||
* Combined pack-layer suite: **170 tests, all green** (was 154 after
|
||||
ADR-0038).
|
||||
* CLI suites unchanged: smoke 67, runtime 19, cognition 121.
|
||||
* `core eval cognition`: intent 100%, versor_closure 100% — baseline
|
||||
preserved.
|
||||
|
||||
## Open questions deferred to a future ADR
|
||||
|
||||
1. **Structured-logging sink that consumes `turn_log`.** Now that
|
||||
stub turns participate in the stream, a structured emitter could
|
||||
produce one log line per turn with `pack_id`, `refusal_emitted`,
|
||||
`hedge_injected`, and violated boundary ids.
|
||||
2. **`core chat --show-verdicts` CLI flag.** Print per-turn verdict
|
||||
bundle summaries to stdout/stderr for manual audit.
|
||||
3. **Drop per-field accessors after a deprecation cycle.**
|
||||
`response.safety_verdict` could become `response.verdicts.safety_verdict`-only
|
||||
once internal callers migrate.
|
||||
4. **`identity_score` on stub turns.** Today stub paths skip the
|
||||
trajectory operator entirely and report `None`. A future ADR
|
||||
could compute a degenerate-but-valid IdentityScore for stubs so
|
||||
the audit stream is fully populated.
|
||||
5. **Bundle versioning.** As more remediation tiers land (per-domain
|
||||
default policies, score-decomposition surfaces), the bundle may
|
||||
grow. Frozen-dataclass + optional defaults should scale fine,
|
||||
but a `schema_version` field could be added if downstream consumers
|
||||
need explicit versioning.
|
||||
|
|
@ -203,9 +203,10 @@ class TestRuntimeHedgeInjection:
|
|||
return # nothing to inject in this pack
|
||||
# Hedge injection is a main-path-only affordance — the stub
|
||||
# path's unknown-domain marker is already a disclosure surface
|
||||
# and is intentionally not hedge-prefixed. Only assert when
|
||||
# the main path was taken (turn_log populated).
|
||||
if not rt.turn_log:
|
||||
# and is intentionally not hedge-prefixed. Detect stub path
|
||||
# by walk_surface (ADR-0039 now emits stub TurnEvents too, so
|
||||
# ``rt.turn_log`` is no longer a stub/main discriminator).
|
||||
if resp.walk_surface == "I don't know — insufficient grounding for that yet.":
|
||||
return
|
||||
assert resp.surface.startswith(prefix)
|
||||
|
||||
|
|
|
|||
242
tests/test_turn_verdicts_bundle.py
Normal file
242
tests/test_turn_verdicts_bundle.py
Normal file
|
|
@ -0,0 +1,242 @@
|
|||
"""ADR-0039 — audit completeness: TurnVerdicts bundle, stub-path
|
||||
TurnEvent emission, hedge_injected signal.
|
||||
|
||||
The bundle replaces the per-field correlation pattern downstream
|
||||
audit consumers had to do (read identity_score, safety_verdict,
|
||||
ethics_verdict, then infer remediation from surface text). The
|
||||
remediation flags ``refusal_emitted`` and ``hedge_injected`` make
|
||||
the runtime's decisions auditable without text inspection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import replace
|
||||
|
||||
from chat.runtime import ChatRuntime, _UNKNOWN_DOMAIN_SURFACE
|
||||
from chat.verdicts import TurnVerdicts
|
||||
from core.config import RuntimeConfig
|
||||
from packs.ethics.check import EthicsCheckResult, EthicsVerdict
|
||||
from packs.safety.check import SafetyCheckResult, SafetyVerdict
|
||||
|
||||
|
||||
# ---------- TurnVerdicts shape ----------
|
||||
|
||||
|
||||
class TestTurnVerdictsShape:
|
||||
def test_construct_with_required_fields(self) -> None:
|
||||
v = TurnVerdicts(
|
||||
identity_score=None,
|
||||
safety_verdict=None,
|
||||
ethics_verdict=None,
|
||||
refusal_emitted=False,
|
||||
hedge_injected=False,
|
||||
)
|
||||
assert v.refusal_emitted is False
|
||||
assert v.hedge_injected is False
|
||||
|
||||
def test_is_frozen(self) -> None:
|
||||
v = TurnVerdicts(
|
||||
identity_score=None,
|
||||
safety_verdict=None,
|
||||
ethics_verdict=None,
|
||||
refusal_emitted=False,
|
||||
hedge_injected=False,
|
||||
)
|
||||
try:
|
||||
v.refusal_emitted = True # type: ignore[misc]
|
||||
except (AttributeError, Exception):
|
||||
return
|
||||
else:
|
||||
raise AssertionError("TurnVerdicts should be frozen")
|
||||
|
||||
|
||||
# ---------- ChatResponse and TurnEvent carry the bundle ----------
|
||||
|
||||
|
||||
class TestBundleAttachment:
|
||||
def test_chat_response_carries_verdicts_bundle(self) -> None:
|
||||
rt = ChatRuntime(config=RuntimeConfig())
|
||||
resp = rt.chat("light is")
|
||||
assert isinstance(resp.verdicts, TurnVerdicts)
|
||||
# Bundle's individual verdicts match the per-field versions.
|
||||
assert resp.verdicts.safety_verdict is resp.safety_verdict
|
||||
assert resp.verdicts.ethics_verdict is resp.ethics_verdict
|
||||
|
||||
def test_turn_event_carries_verdicts_bundle(self) -> None:
|
||||
rt = ChatRuntime(config=RuntimeConfig())
|
||||
rt.chat("light is")
|
||||
assert rt.turn_log, "stub path should now append TurnEvent (ADR-0039)"
|
||||
event = rt.turn_log[-1]
|
||||
assert isinstance(event.verdicts, TurnVerdicts)
|
||||
|
||||
|
||||
# ---------- stub-path TurnEvent emission (ADR-0039) ----------
|
||||
|
||||
|
||||
class TestStubPathTurnEvent:
|
||||
def test_stub_path_appends_turn_event(self) -> None:
|
||||
rt = ChatRuntime(config=RuntimeConfig())
|
||||
before = len(rt.turn_log)
|
||||
resp = rt.chat("light is")
|
||||
# Cold start with no vault hits → stub path fires.
|
||||
assert resp.walk_surface == _UNKNOWN_DOMAIN_SURFACE
|
||||
assert len(rt.turn_log) == before + 1
|
||||
|
||||
def test_stub_event_records_unknown_walk_surface(self) -> None:
|
||||
rt = ChatRuntime(config=RuntimeConfig())
|
||||
rt.chat("light is")
|
||||
event = rt.turn_log[-1]
|
||||
assert event.walk_surface == _UNKNOWN_DOMAIN_SURFACE
|
||||
assert event.articulation_surface == _UNKNOWN_DOMAIN_SURFACE
|
||||
|
||||
def test_stub_event_carries_input_tokens(self) -> None:
|
||||
rt = ChatRuntime(config=RuntimeConfig())
|
||||
rt.chat("light is")
|
||||
event = rt.turn_log[-1]
|
||||
assert event.input_tokens, "stub TurnEvent must record input tokens"
|
||||
|
||||
def test_stub_event_identity_score_is_none(self) -> None:
|
||||
"""No reasoning trajectory ran on stub path, so no
|
||||
IdentityScore is computed."""
|
||||
rt = ChatRuntime(config=RuntimeConfig())
|
||||
rt.chat("light is")
|
||||
event = rt.turn_log[-1]
|
||||
assert event.identity_score is None
|
||||
|
||||
def test_stub_event_versor_condition_recorded(self) -> None:
|
||||
rt = ChatRuntime(config=RuntimeConfig())
|
||||
rt.chat("light is")
|
||||
event = rt.turn_log[-1]
|
||||
assert isinstance(event.versor_condition, float)
|
||||
|
||||
|
||||
# ---------- refusal_emitted flag ----------
|
||||
|
||||
|
||||
class TestRefusalEmittedFlag:
|
||||
def test_no_violation_flag_is_false(self) -> None:
|
||||
rt = ChatRuntime(config=RuntimeConfig())
|
||||
resp = rt.chat("light is")
|
||||
assert resp.verdicts.refusal_emitted is False
|
||||
|
||||
def test_safety_violation_sets_flag(self) -> None:
|
||||
rt = ChatRuntime(config=RuntimeConfig())
|
||||
|
||||
def _failing(ctx): # noqa: ANN001
|
||||
return SafetyCheckResult(
|
||||
boundary_id="preserve_versor_closure",
|
||||
upheld=False,
|
||||
reason="forced",
|
||||
runtime_checkable=True,
|
||||
)
|
||||
|
||||
rt.safety_check.register("preserve_versor_closure", _failing)
|
||||
resp = rt.chat("light is")
|
||||
assert resp.verdicts.refusal_emitted is True
|
||||
# And the flag matches the surface state.
|
||||
assert resp.verdicts.refusal_emitted == (resp.surface != resp.walk_surface)
|
||||
|
||||
def test_flag_appears_on_turn_event_too(self) -> None:
|
||||
rt = ChatRuntime(config=RuntimeConfig())
|
||||
|
||||
def _failing(ctx): # noqa: ANN001
|
||||
return SafetyCheckResult(
|
||||
boundary_id="preserve_versor_closure",
|
||||
upheld=False,
|
||||
reason="forced",
|
||||
runtime_checkable=True,
|
||||
)
|
||||
|
||||
rt.safety_check.register("preserve_versor_closure", _failing)
|
||||
rt.chat("light is")
|
||||
event = rt.turn_log[-1]
|
||||
assert event.verdicts.refusal_emitted is True
|
||||
|
||||
|
||||
# ---------- hedge_injected flag ----------
|
||||
|
||||
|
||||
class TestHedgeInjectedFlag:
|
||||
def test_no_opt_in_flag_is_false(self) -> None:
|
||||
rt = ChatRuntime(config=RuntimeConfig())
|
||||
resp = rt.chat("light is")
|
||||
assert resp.verdicts.hedge_injected is False
|
||||
|
||||
def test_stub_path_never_sets_hedge_injected(self) -> None:
|
||||
"""Even with opted-in hedge_commitments and a forced
|
||||
violation, the stub path must report hedge_injected=False
|
||||
(stub never hedges — the unknown-domain marker is already
|
||||
a disclosure)."""
|
||||
rt = ChatRuntime(config=RuntimeConfig())
|
||||
rt.ethics_pack = replace(
|
||||
rt.ethics_pack,
|
||||
hedge_commitments=frozenset({"acknowledge_uncertainty"}),
|
||||
)
|
||||
|
||||
def _failing_ethics(ctx): # noqa: ANN001
|
||||
return EthicsCheckResult(
|
||||
commitment_id="acknowledge_uncertainty",
|
||||
upheld=False,
|
||||
reason="forced",
|
||||
runtime_checkable=True,
|
||||
)
|
||||
|
||||
rt.ethics_check.register("acknowledge_uncertainty", _failing_ethics)
|
||||
resp = rt.chat("light is")
|
||||
if resp.walk_surface == _UNKNOWN_DOMAIN_SURFACE:
|
||||
assert resp.verdicts.hedge_injected is False
|
||||
|
||||
|
||||
# ---------- mutual exclusion refusal vs hedge in the bundle ----------
|
||||
|
||||
|
||||
class TestRemediationMutualExclusion:
|
||||
def test_refusal_and_hedge_never_both_true(self) -> None:
|
||||
"""The runtime contract: refusal supersedes hedge. When both
|
||||
a runtime-checkable safety violation AND an opted-in ethics
|
||||
hedge would fire, refusal wins and hedge_injected stays
|
||||
False."""
|
||||
rt = ChatRuntime(config=RuntimeConfig())
|
||||
rt.ethics_pack = replace(
|
||||
rt.ethics_pack,
|
||||
hedge_commitments=frozenset({"acknowledge_uncertainty"}),
|
||||
)
|
||||
|
||||
def _failing_safety(ctx): # noqa: ANN001
|
||||
return SafetyCheckResult(
|
||||
boundary_id="preserve_versor_closure",
|
||||
upheld=False,
|
||||
reason="forced",
|
||||
runtime_checkable=True,
|
||||
)
|
||||
|
||||
def _failing_ethics(ctx): # noqa: ANN001
|
||||
return EthicsCheckResult(
|
||||
commitment_id="acknowledge_uncertainty",
|
||||
upheld=False,
|
||||
reason="forced",
|
||||
runtime_checkable=True,
|
||||
)
|
||||
|
||||
rt.safety_check.register("preserve_versor_closure", _failing_safety)
|
||||
rt.ethics_check.register("acknowledge_uncertainty", _failing_ethics)
|
||||
resp = rt.chat("light is")
|
||||
assert resp.verdicts.refusal_emitted is True
|
||||
assert resp.verdicts.hedge_injected is False
|
||||
|
||||
|
||||
# ---------- response and event bundles agree ----------
|
||||
|
||||
|
||||
class TestBundleConsistency:
|
||||
def test_response_and_event_bundle_are_consistent(self) -> None:
|
||||
"""The TurnVerdicts on ChatResponse and TurnEvent for the same
|
||||
turn carry the same remediation flags and reference the same
|
||||
underlying verdicts."""
|
||||
rt = ChatRuntime(config=RuntimeConfig())
|
||||
resp = rt.chat("light is")
|
||||
event = rt.turn_log[-1]
|
||||
assert resp.verdicts.refusal_emitted == event.verdicts.refusal_emitted
|
||||
assert resp.verdicts.hedge_injected == event.verdicts.hedge_injected
|
||||
assert resp.verdicts.safety_verdict is event.verdicts.safety_verdict
|
||||
assert resp.verdicts.ethics_verdict is event.verdicts.ethics_verdict
|
||||
Loading…
Reference in a new issue