277 lines
8.5 KiB
Python
277 lines
8.5 KiB
Python
"""Tests for the generalization audit runner skeleton."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import pytest
|
|
|
|
from evals.generalization.audit_runner import run_generalization_audit
|
|
from evals.generalization.item_schema import (
|
|
GENERALIZATION_AUDIT_RUNNER_POLICY_VERSION,
|
|
GeneralizationAuditItem,
|
|
GeneralizationAuditOutcome,
|
|
)
|
|
|
|
|
|
def test_aggregates_counts_and_metrics() -> None:
|
|
"""Ensure run_generalization_audit aggregates dispositions, counts, and reasons correctly."""
|
|
items = (
|
|
GeneralizationAuditItem(
|
|
dataset="TEST_DATA",
|
|
split="test",
|
|
item_id="id_1",
|
|
prompt_ref="test:test:id_1",
|
|
answer_kind="numeric",
|
|
metadata=(("tag", "a"),),
|
|
),
|
|
GeneralizationAuditItem(
|
|
dataset="TEST_DATA",
|
|
split="test",
|
|
item_id="id_2",
|
|
prompt_ref="test:test:id_2",
|
|
answer_kind="numeric",
|
|
metadata=(("tag", "b"),),
|
|
),
|
|
GeneralizationAuditItem(
|
|
dataset="TEST_DATA",
|
|
split="test",
|
|
item_id="id_3",
|
|
prompt_ref="test:test:id_3",
|
|
answer_kind="numeric",
|
|
metadata=(("tag", "c"),),
|
|
),
|
|
GeneralizationAuditItem(
|
|
dataset="TEST_DATA",
|
|
split="test",
|
|
item_id="id_4",
|
|
prompt_ref="test:test:id_4",
|
|
answer_kind="numeric",
|
|
metadata=(("tag", "d"),),
|
|
),
|
|
)
|
|
|
|
def mock_evaluator(
|
|
item: GeneralizationAuditItem,
|
|
) -> GeneralizationAuditOutcome:
|
|
if item.item_id == "id_1":
|
|
return GeneralizationAuditOutcome(
|
|
item_id=item.item_id,
|
|
disposition="correct",
|
|
residual_kinds=("none",),
|
|
candidate_attempt_count=1,
|
|
binding_failure_count=0,
|
|
replay_refusal_count=0,
|
|
sealed_trace_dispositions=("success",),
|
|
reason_codes=(),
|
|
)
|
|
elif item.item_id == "id_2":
|
|
return GeneralizationAuditOutcome(
|
|
item_id=item.item_id,
|
|
disposition="wrong",
|
|
residual_kinds=("precision",),
|
|
candidate_attempt_count=3,
|
|
binding_failure_count=0,
|
|
replay_refusal_count=0,
|
|
sealed_trace_dispositions=("fail", "fail", "success"),
|
|
reason_codes=("precision_error",),
|
|
)
|
|
elif item.item_id == "id_3":
|
|
return GeneralizationAuditOutcome(
|
|
item_id=item.item_id,
|
|
disposition="refused",
|
|
residual_kinds=(),
|
|
candidate_attempt_count=1,
|
|
binding_failure_count=1,
|
|
replay_refusal_count=1,
|
|
sealed_trace_dispositions=("refused",),
|
|
reason_codes=("safety_policy",),
|
|
)
|
|
else:
|
|
return GeneralizationAuditOutcome(
|
|
item_id=item.item_id,
|
|
disposition="unsupported",
|
|
residual_kinds=(),
|
|
candidate_attempt_count=0,
|
|
binding_failure_count=0,
|
|
replay_refusal_count=0,
|
|
sealed_trace_dispositions=(),
|
|
reason_codes=("unsupported_format",),
|
|
)
|
|
|
|
report = run_generalization_audit(
|
|
dataset="TEST_DATA",
|
|
split="test",
|
|
items=items,
|
|
evaluator=mock_evaluator,
|
|
)
|
|
|
|
# Core disposition counts
|
|
assert report.correct == 1
|
|
assert report.wrong == 1
|
|
assert report.refused == 1
|
|
assert report.unsupported == 1
|
|
assert report.n_items == 4
|
|
|
|
# Direct additions of metrics
|
|
assert report.candidate_attempts == 5 # 1 + 3 + 1 + 0
|
|
assert report.binding_failures == 1 # 0 + 0 + 1 + 0
|
|
assert report.replay_refusals == 1 # 0 + 0 + 1 + 0
|
|
|
|
# Histograms
|
|
# "success" -> 2 times, "fail" -> 2 times, "refused" -> 1 time
|
|
# Sorted by count descending, then by key ascending:
|
|
# (('fail', 2), ('success', 2), ('refused', 1))
|
|
assert report.sealed_trace_dispositions == (
|
|
("fail", 2),
|
|
("success", 2),
|
|
("refused", 1),
|
|
)
|
|
|
|
# residual kinds: "none" -> 1, "precision" -> 1
|
|
# Sorted by count descending, then key:
|
|
# (('none', 1), ('precision', 1))
|
|
assert report.dominant_residual_kinds == (
|
|
("none", 1),
|
|
("precision", 1),
|
|
)
|
|
|
|
# Reason codes: sorted union of all codes
|
|
assert report.reason_codes == (
|
|
"precision_error",
|
|
"safety_policy",
|
|
"unsupported_format",
|
|
)
|
|
|
|
|
|
def test_empty_item_set_refuses() -> None:
|
|
"""Ensure run_generalization_audit refuses (raises ValueError) if items tuple is empty."""
|
|
with pytest.raises(ValueError, match="requires a non-empty sequence"):
|
|
run_generalization_audit(
|
|
dataset="TEST_DATA",
|
|
split="test",
|
|
items=(),
|
|
evaluator=lambda x: None,
|
|
)
|
|
|
|
|
|
def test_evaluator_exception_fail_closed() -> None:
|
|
"""Ensure run_generalization_audit catches evaluator exceptions and fails closed."""
|
|
items = (
|
|
GeneralizationAuditItem(
|
|
dataset="TEST_DATA",
|
|
split="test",
|
|
item_id="id_1",
|
|
prompt_ref="test:test:id_1",
|
|
answer_kind="numeric",
|
|
metadata=(),
|
|
),
|
|
)
|
|
|
|
def exploding_evaluator(
|
|
item: GeneralizationAuditItem,
|
|
) -> GeneralizationAuditOutcome:
|
|
raise RuntimeError("Something exploded!")
|
|
|
|
report = run_generalization_audit(
|
|
dataset="TEST_DATA",
|
|
split="test",
|
|
items=items,
|
|
evaluator=exploding_evaluator,
|
|
)
|
|
|
|
assert report.n_items == 1
|
|
assert report.refused == 1
|
|
assert report.correct == 0
|
|
assert report.wrong == 0
|
|
assert "evaluator_exception" in report.reason_codes
|
|
assert "RuntimeError" in report.reason_codes
|
|
|
|
|
|
def test_json_output_deterministic_and_no_raw_prompts() -> None:
|
|
"""Ensure serialized report is deterministic and does not contain raw prompt/answer fields."""
|
|
items = (
|
|
GeneralizationAuditItem(
|
|
dataset="TEST_DATA",
|
|
split="test",
|
|
item_id="id_1",
|
|
prompt_ref="test:test:id_1",
|
|
answer_kind="numeric",
|
|
metadata=(),
|
|
),
|
|
)
|
|
report = run_generalization_audit(
|
|
dataset="TEST_DATA",
|
|
split="test",
|
|
items=items,
|
|
evaluator=lambda x: GeneralizationAuditOutcome(
|
|
item_id=x.item_id,
|
|
disposition="correct",
|
|
residual_kinds=(),
|
|
candidate_attempt_count=1,
|
|
binding_failure_count=0,
|
|
replay_refusal_count=0,
|
|
sealed_trace_dispositions=(),
|
|
reason_codes=(),
|
|
),
|
|
)
|
|
|
|
from dataclasses import asdict
|
|
|
|
report_dict = asdict(report)
|
|
|
|
# 1. Deterministic check
|
|
json1 = json.dumps(report_dict, indent=2, sort_keys=True)
|
|
json2 = json.dumps(report_dict, indent=2, sort_keys=True)
|
|
assert json1 == json2
|
|
|
|
# 2. Check no raw prompt/answer leakages in the report
|
|
serialized_str = json1.lower()
|
|
assert "prompt" not in report_dict
|
|
assert "answer" not in report_dict
|
|
|
|
# Check that common prompt/answer related strings are absent
|
|
assert "raw_prompt" not in serialized_str
|
|
assert "raw_answer" not in serialized_str
|
|
assert "chain_of_thought" not in serialized_str
|
|
assert "example_text" not in serialized_str
|
|
|
|
|
|
def test_cli_synthetic_smoke() -> None:
|
|
"""Verify that --synthetic-smoke runs and produces expected report structure."""
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"scripts/benchmarks/run_generalization_audit.py",
|
|
"--synthetic-smoke",
|
|
"--json",
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
)
|
|
assert result.returncode == 0
|
|
report = json.loads(result.stdout)
|
|
assert report["dataset"] == "SYNTHETIC_SMOKE"
|
|
assert report["policy_version"] == GENERALIZATION_AUDIT_RUNNER_POLICY_VERSION
|
|
assert report["n_items"] == 3
|
|
assert report["correct"] == 1
|
|
assert report["wrong"] == 1
|
|
assert report["refused"] == 1
|
|
|
|
|
|
def test_cli_real_dataset_refuses() -> None:
|
|
"""Verify that requesting a real dataset fails with dataset_adapter_unavailable."""
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"scripts/benchmarks/run_generalization_audit.py",
|
|
"--dataset",
|
|
"gsm1k",
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert result.returncode != 0
|
|
assert "dataset_adapter_unavailable" in result.stderr
|