4ba1ef2da3
Converts the load-bearing claims of the ADR-0027→0042 pack-layer chain into CI-enforced numbers across the three ratified identity packs (default_general_v1, precision_first_v1, generosity_first_v1). Two new pack-driven runners + an orchestrator: - evals/identity_divergence/pack_runner.py — drives real SentenceAssembler + SurfaceContext (no mocks) across all three packs over 10 cases × 5 alignment bands; publishes per-pack bare/hedge/qualifier rates and pairwise distinct_rate. - evals/refusal_calibration/pack_runner.py — runs the existing grounding-refusal lane via RuntimeConfig(identity_pack=...); publishes per-pack refusal_rate/fabrication_rate and a pack_invariant_gate flag asserting byte-identical cold-start surfaces across packs. - scripts/publish_pack_measurements.py — combined publisher emitting evals/results/phase2_pack_measurements.json. Baseline numbers (2026-05-17): - precision_first hedge_rate=0.60, qualifier_rate=0.20 - generosity_first hedge_rate=0.20, qualifier_rate=0.00 - default_general hedge_rate=0.40, qualifier_rate=0.00 - pairwise distinct_rate ∈ [0.40, 0.80] - refusal_rate=1.00, fabrication_rate=0.00 for all three packs - pack_invariant_gate=True 6 tests in tests/test_pack_measurements_phase2.py lock the schema + load-bearing flags + the structural inequality precision.hedge_rate > generosity.hedge_rate. If identity packs get wired into the cognition gate, pack_invariant_gate flips and the suite fails. ADR-0043 documents the numbers, the extended marker rationale, and the trade-offs. README index updated with ADR-0043 row and chain title bumped to "ADR-0027 through ADR-0043". Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
104 lines
3.3 KiB
Python
104 lines
3.3 KiB
Python
"""Publish combined Phase-2 pack-measurement report (ADR-0043).
|
||
|
||
Runs both pack-driven runners (identity-divergence + refusal-calibration)
|
||
and writes a unified report to
|
||
`evals/results/phase2_pack_measurements.json` plus a per-runner copy
|
||
under each lane's own `results/packs_v1/` directory.
|
||
|
||
Usage:
|
||
PYTHONPATH=. python3 scripts/publish_pack_measurements.py [--json]
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from evals.identity_divergence.pack_runner import run_pack_divergence_eval
|
||
from evals.refusal_calibration.pack_runner import run_pack_refusal_eval
|
||
|
||
|
||
def build_combined_report() -> dict[str, Any]:
|
||
identity = run_pack_divergence_eval()
|
||
refusal = run_pack_refusal_eval()
|
||
return {
|
||
"schema_version": 1,
|
||
"identity_divergence": identity,
|
||
"refusal_calibration": refusal,
|
||
"claims_supported": {
|
||
"identity_load_bearing": identity["load_bearing"],
|
||
"grounding_gate_pack_invariant": refusal["pack_invariant_gate"],
|
||
"no_fabrication_under_any_pack": all(
|
||
p["fabrication_rate"] == 0.0 for p in refusal["packs"]
|
||
),
|
||
},
|
||
}
|
||
|
||
|
||
def write_report(report: dict[str, Any], out_path: Path) -> Path:
|
||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||
with out_path.open("w") as fh:
|
||
json.dump(report, fh, indent=2, sort_keys=True)
|
||
fh.write("\n")
|
||
return out_path
|
||
|
||
|
||
def _print_human(report: dict[str, Any]) -> None:
|
||
identity = report["identity_divergence"]
|
||
refusal = report["refusal_calibration"]
|
||
claims = report["claims_supported"]
|
||
|
||
print("Phase-2 pack measurements (ADR-0043)")
|
||
print("=" * 72)
|
||
print(
|
||
f"identity-divergence: {identity['case_count']} cases × "
|
||
f"{len(identity['alignment_bands'])} alignment bands"
|
||
)
|
||
for entry in identity["packs"]:
|
||
print(
|
||
f" {entry['pack_id']:<24} bare={entry['bare_rate']:.2f} "
|
||
f"hedge={entry['hedge_rate']:.2f} qualifier={entry['qualifier_rate']:.2f}"
|
||
)
|
||
print()
|
||
for pair in identity["pairwise_divergence"]:
|
||
print(
|
||
f" {pair['pack_a']} ⇆ {pair['pack_b']:<22} "
|
||
f"distinct_rate={pair['distinct_rate']:.2f}"
|
||
)
|
||
print()
|
||
print(
|
||
f"refusal-calibration: {refusal['case_count']} cases "
|
||
f"(out_of_grounding={refusal['out_of_grounding_count']})"
|
||
)
|
||
for entry in refusal["packs"]:
|
||
print(
|
||
f" {entry['pack_id']:<24} refusal_rate={entry['refusal_rate']:.2f} "
|
||
f"fabrication_rate={entry['fabrication_rate']:.2f}"
|
||
)
|
||
print()
|
||
print("Claims:")
|
||
for k, v in claims.items():
|
||
print(f" {k:<40} {v}")
|
||
print("=" * 72)
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument("--json", action="store_true", help="emit only JSON to stdout")
|
||
args = parser.parse_args()
|
||
|
||
report = build_combined_report()
|
||
out_path = Path("evals/results/phase2_pack_measurements.json")
|
||
write_report(report, out_path)
|
||
|
||
if args.json:
|
||
print(json.dumps(report, indent=2, sort_keys=True))
|
||
else:
|
||
_print_human(report)
|
||
print(f"wrote → {out_path}")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|