4ba1ef2da3
Converts the load-bearing claims of the ADR-0027→0042 pack-layer chain into CI-enforced numbers across the three ratified identity packs (default_general_v1, precision_first_v1, generosity_first_v1). Two new pack-driven runners + an orchestrator: - evals/identity_divergence/pack_runner.py — drives real SentenceAssembler + SurfaceContext (no mocks) across all three packs over 10 cases × 5 alignment bands; publishes per-pack bare/hedge/qualifier rates and pairwise distinct_rate. - evals/refusal_calibration/pack_runner.py — runs the existing grounding-refusal lane via RuntimeConfig(identity_pack=...); publishes per-pack refusal_rate/fabrication_rate and a pack_invariant_gate flag asserting byte-identical cold-start surfaces across packs. - scripts/publish_pack_measurements.py — combined publisher emitting evals/results/phase2_pack_measurements.json. Baseline numbers (2026-05-17): - precision_first hedge_rate=0.60, qualifier_rate=0.20 - generosity_first hedge_rate=0.20, qualifier_rate=0.00 - default_general hedge_rate=0.40, qualifier_rate=0.00 - pairwise distinct_rate ∈ [0.40, 0.80] - refusal_rate=1.00, fabrication_rate=0.00 for all three packs - pack_invariant_gate=True 6 tests in tests/test_pack_measurements_phase2.py lock the schema + load-bearing flags + the structural inequality precision.hedge_rate > generosity.hedge_rate. If identity packs get wired into the cognition gate, pack_invariant_gate flips and the suite fails. ADR-0043 documents the numbers, the extended marker rationale, and the trade-offs. README index updated with ADR-0043 row and chain title bumped to "ADR-0027 through ADR-0043". Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
92 lines
3.8 KiB
Python
92 lines
3.8 KiB
Python
"""Phase-2 pack-measurement tests — ADR-0043.
|
|
|
|
Locks in the load-bearing claim of the pack-layer chain (ADR-0027 →
|
|
ADR-0042) as numbers, not assertions:
|
|
|
|
1. The three ratified identity packs produce *distinct* articulation
|
|
distributions (`pack_runner` in `evals/identity_divergence/`).
|
|
2. The grounding gate is *invariant* across the three packs
|
|
(`pack_runner` in `evals/refusal_calibration/`).
|
|
|
|
Both runners must emit a schema-stable report and the load-bearing
|
|
flags must hold. If a regression flips either claim to False this
|
|
suite fails, catching the issue before it lands in main.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from evals.identity_divergence.pack_runner import (
|
|
PACK_IDS as IDENTITY_PACK_IDS,
|
|
run_pack_divergence_eval,
|
|
)
|
|
from evals.refusal_calibration.pack_runner import (
|
|
PACK_IDS as REFUSAL_PACK_IDS,
|
|
run_pack_refusal_eval,
|
|
)
|
|
|
|
|
|
class TestIdentityDivergencePackRunner:
|
|
def test_report_schema_is_stable(self) -> None:
|
|
report = run_pack_divergence_eval()
|
|
assert report["schema_version"] == 1
|
|
assert report["case_count"] > 0
|
|
assert {p["pack_id"] for p in report["packs"]} == set(IDENTITY_PACK_IDS)
|
|
assert len(report["pairwise_divergence"]) == 3
|
|
for entry in report["packs"]:
|
|
assert 0.0 <= entry["bare_rate"] <= 1.0
|
|
assert 0.0 <= entry["hedge_rate"] <= 1.0
|
|
assert 0.0 <= entry["qualifier_rate"] <= 1.0
|
|
# rates must cover the surface population without overflow
|
|
assert (
|
|
entry["bare_rate"] + entry["hedge_rate"] + entry["qualifier_rate"]
|
|
<= 1.0001
|
|
)
|
|
|
|
def test_load_bearing_claim_holds(self) -> None:
|
|
report = run_pack_divergence_eval()
|
|
assert report["load_bearing"] is True, (
|
|
"every ratified pack pair must produce at least one distinct "
|
|
"surface across the alignment grid"
|
|
)
|
|
for pair in report["pairwise_divergence"]:
|
|
assert pair["distinct_rate"] > 0.0, pair
|
|
|
|
def test_precision_hedges_more_than_generosity(self) -> None:
|
|
"""The two specialization packs must be measurably different.
|
|
|
|
Precision-first is configured to hedge sooner and to qualify in
|
|
the marginal band; generosity-first is configured to stay bare.
|
|
If these inequalities flip, the surface_preferences fields
|
|
have drifted out of alignment with the pack identities.
|
|
"""
|
|
report = run_pack_divergence_eval()
|
|
by_id = {p["pack_id"]: p for p in report["packs"]}
|
|
prec = by_id["precision_first_v1"]
|
|
gen = by_id["generosity_first_v1"]
|
|
assert prec["hedge_rate"] > gen["hedge_rate"]
|
|
assert prec["qualifier_rate"] >= gen["qualifier_rate"]
|
|
assert gen["bare_rate"] > prec["bare_rate"]
|
|
|
|
|
|
class TestRefusalCalibrationPackRunner:
|
|
def test_report_schema_is_stable(self) -> None:
|
|
report = run_pack_refusal_eval()
|
|
assert report["schema_version"] == 1
|
|
assert report["case_count"] > 0
|
|
assert {p["pack_id"] for p in report["packs"]} == set(REFUSAL_PACK_IDS)
|
|
for entry in report["packs"]:
|
|
assert 0.0 <= entry["refusal_rate"] <= 1.0
|
|
assert 0.0 <= entry["fabrication_rate"] <= 1.0
|
|
|
|
def test_grounding_gate_is_pack_invariant(self) -> None:
|
|
"""ADR-0043 — Identity packs must NOT change grounding refusal.
|
|
|
|
Cold-start out-of-grounding probes hit the gate upstream of
|
|
articulation; surface should be byte-identical across packs.
|
|
"""
|
|
report = run_pack_refusal_eval()
|
|
assert report["pack_invariant_gate"] is True
|
|
|
|
def test_no_fabrication_under_any_pack(self) -> None:
|
|
report = run_pack_refusal_eval()
|
|
for entry in report["packs"]:
|
|
assert entry["fabrication_rate"] == 0.0, entry
|