core/scripts/publish_pack_measurements.py

"""Publish combined Phase-2 pack-measurement report (ADR-0043).

Runs both pack-driven runners (identity-divergence + refusal-calibration)
and writes a unified report to
`evals/results/phase2_pack_measurements.json` plus a per-runner copy
under each lane's own `results/packs_v1/` directory.

Usage:
    PYTHONPATH=. python3 scripts/publish_pack_measurements.py [--json]
"""
from __future__ import annotations

import argparse
import json
from pathlib import Path
from typing import Any

from evals.identity_divergence.pack_runner import run_pack_divergence_eval
from evals.refusal_calibration.pack_runner import run_pack_refusal_eval


def build_combined_report() -> dict[str, Any]:
    identity = run_pack_divergence_eval()
    refusal = run_pack_refusal_eval()
    claims = {
        "identity_load_bearing": identity["load_bearing"],
        "grounding_gate_pack_invariant": refusal["pack_invariant_gate"],
        "no_fabrication_under_any_pack": all(
            p["fabrication_rate"] == 0.0 for p in refusal["packs"]
        ),
    }
    return {
        "schema_version": 1,
        "identity_divergence": identity,
        "refusal_calibration": refusal,
        "claims_supported": claims,
        # ``all_claims_supported`` is the canonical cross-demo success
        # field — AND of every entry in the nested claims_supported dict.
        # Operator tooling can consume this without knowing the claim list.
        "all_claims_supported": all(claims.values()),
    }


def write_report(report: dict[str, Any], out_path: Path) -> Path:
    out_path.parent.mkdir(parents=True, exist_ok=True)
    with out_path.open("w") as fh:
        json.dump(report, fh, indent=2, sort_keys=True)
        fh.write("\n")
    return out_path


def _print_human(report: dict[str, Any]) -> None:
    identity = report["identity_divergence"]
    refusal = report["refusal_calibration"]
    claims = report["claims_supported"]

    print("Phase-2 pack measurements (ADR-0043)")
    print("=" * 72)
    print(
        f"identity-divergence: {identity['case_count']} cases × "
        f"{len(identity['alignment_bands'])} alignment bands"
    )
    for entry in identity["packs"]:
        print(
            f"  {entry['pack_id']:<24}  bare={entry['bare_rate']:.2f}  "
            f"hedge={entry['hedge_rate']:.2f}  qualifier={entry['qualifier_rate']:.2f}"
        )
    print()
    for pair in identity["pairwise_divergence"]:
        print(
            f"  {pair['pack_a']} ⇆ {pair['pack_b']:<22}  "
            f"distinct_rate={pair['distinct_rate']:.2f}"
        )
    print()
    print(
        f"refusal-calibration: {refusal['case_count']} cases "
        f"(out_of_grounding={refusal['out_of_grounding_count']})"
    )
    for entry in refusal["packs"]:
        print(
            f"  {entry['pack_id']:<24}  refusal_rate={entry['refusal_rate']:.2f}  "
            f"fabrication_rate={entry['fabrication_rate']:.2f}"
        )
    print()
    print("Claims:")
    for k, v in claims.items():
        print(f"  {k:<40}  {v}")
    print("=" * 72)


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--json", action="store_true", help="emit only JSON to stdout")
    args = parser.parse_args()

    report = build_combined_report()
    out_path = Path("evals/results/phase2_pack_measurements.json")
    write_report(report, out_path)

    if args.json:
        print(json.dumps(report, indent=2, sort_keys=True))
    else:
        _print_human(report)
        print(f"wrote → {out_path}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())