472 lines
16 KiB
Python
472 lines
16 KiB
Python
"""ADR-0168 — FrameClaim ratification handler tests.
|
|
|
|
Mirrors :mod:`tests.test_math_lexical_ratification` (W2-D) and pins:
|
|
|
|
1. SAFE_FRAME_CATEGORIES is exactly the ADR-0168 allowlist (4 entries).
|
|
2. apply_frame_claim writes a frame entry for a safe category.
|
|
3. receipt records before/after sha + evidence_hash.
|
|
4. idempotent same-evidence → AlreadyRatified.
|
|
5. rejects non-frame sub_type.
|
|
6. rejects categories outside SAFE_FRAME_CATEGORIES (wrong=0 hazard).
|
|
7. rejects invalid polarity (must be affirms|falsifies).
|
|
8. rejects evidence tampering.
|
|
9. rejects evidence laundering (source='corpus' is forbidden).
|
|
10. case 0050 hazard pin — after ratification, case 0050 still refuses.
|
|
11. polarity=falsifies branch records non-opener without admitting.
|
|
12. duplicate evidence on second apply appends evidence_hash, not new row.
|
|
13. manifest.json checksum is unchanged by frame ratification.
|
|
14. alphabetical sort by surface_form preserved across writes.
|
|
|
|
ADR-0168 §"Decision" + ADR-0168.1 §"Evidence floor" + hazard pins.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import functools
|
|
import hashlib
|
|
import json
|
|
import shutil
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from generate.comprehension import lexicon as comprehension_lexicon
|
|
from generate.comprehension import lifecycle
|
|
from generate.comprehension.audit import AuditRow, audit_problem
|
|
from generate.comprehension.state import ReaderRefusal
|
|
from teaching.math_evidence import from_audit_row
|
|
from teaching.math_frame_ratification import (
|
|
AlreadyRatified,
|
|
EvidenceLaundering,
|
|
EvidenceTampering,
|
|
InvalidPolarity,
|
|
SAFE_FRAME_CATEGORIES,
|
|
WrongClaimSubType,
|
|
WrongZeroViolationCandidate,
|
|
apply_frame_claim,
|
|
)
|
|
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
PACK_ROOT = REPO_ROOT / "packs" / "data" / "en_core_math_v1"
|
|
CASES_PATH = REPO_ROOT / "evals" / "gsm8k_math" / "train_sample" / "v1" / "cases.jsonl"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixtures
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.fixture()
|
|
def pack_copy(tmp_path: Path) -> Path:
|
|
target = tmp_path / "en_core_math_v1"
|
|
shutil.copytree(PACK_ROOT, target)
|
|
# Ensure the frames/ scaffold exists in the copy.
|
|
(target / "frames").mkdir(exist_ok=True)
|
|
comprehension_lexicon._CACHE.clear()
|
|
lifecycle._get_lexicon.cache_clear()
|
|
yield target
|
|
comprehension_lexicon._CACHE.clear()
|
|
lifecycle._get_lexicon.cache_clear()
|
|
|
|
|
|
def _row(
|
|
surface: str,
|
|
*,
|
|
missing_operator: str = "pre_frame_filler_sentence",
|
|
refusal_reason: str = "unexpected_category",
|
|
) -> AuditRow:
|
|
return AuditRow(
|
|
case_id=f"case-{surface}",
|
|
sentence_index=0,
|
|
token_index=2,
|
|
token_text=surface,
|
|
recognized_terms=("Mark", "does"),
|
|
skipped_frame=None,
|
|
missing_operator=missing_operator,
|
|
refusal_reason=refusal_reason,
|
|
refusal_detail=f"unexpected category for '{surface}'",
|
|
)
|
|
|
|
|
|
def _claim(surface: str, *, sub_type: str = "frame"):
|
|
return from_audit_row(_row(surface), sub_type) # type: ignore[arg-type]
|
|
|
|
|
|
def _entries(path: Path) -> list[dict]:
|
|
if not path.exists():
|
|
return []
|
|
return [
|
|
json.loads(line)
|
|
for line in path.read_text(encoding="utf-8").splitlines()
|
|
if line.strip()
|
|
]
|
|
|
|
|
|
def _manifest_sha(pack_root: Path) -> str:
|
|
return hashlib.sha256((pack_root / "manifest.json").read_bytes()).hexdigest()
|
|
|
|
|
|
def _use_pack_for_reader(monkeypatch: pytest.MonkeyPatch, pack_root: Path) -> None:
|
|
comprehension_lexicon._CACHE.clear()
|
|
|
|
@functools.cache
|
|
def _tmp_lexicon():
|
|
return comprehension_lexicon.load_lexicon(pack_root)
|
|
|
|
monkeypatch.setattr(lifecycle, "_get_lexicon", _tmp_lexicon)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 1 — SAFE_FRAME_CATEGORIES is the ADR-0168 allowlist
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_safe_frame_categories_is_adr_0168_allowlist() -> None:
|
|
"""The four ADR-0168 categories, no more, no less."""
|
|
assert SAFE_FRAME_CATEGORIES == frozenset(
|
|
{"increment_frame", "decrement_frame", "transfer_frame", "remainder_frame"}
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 2 — apply writes a frame entry for a safe category
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_apply_frame_claim_writes_entry(pack_copy: Path) -> None:
|
|
receipt = apply_frame_claim(
|
|
claim=_claim("gives"),
|
|
frame_category="transfer_frame",
|
|
polarity="affirms",
|
|
reviewer="Ada",
|
|
pack_root=pack_copy,
|
|
)
|
|
|
|
assert receipt.surface_form == "gives"
|
|
assert receipt.frame_category == "transfer_frame"
|
|
assert receipt.polarity == "affirms"
|
|
assert receipt.is_duplicate_evidence is False
|
|
entry = next(
|
|
e
|
|
for e in _entries(pack_copy / "frames" / "transfer_frame.jsonl")
|
|
if e["surface_form"] == "gives"
|
|
)
|
|
assert entry["frame_category"] == "transfer_frame"
|
|
assert entry["polarity"] == "affirms"
|
|
assert receipt.evidence_hash in entry["evidence_hashes"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 3 — receipt records before/after sha
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_receipt_records_before_after_sha(pack_copy: Path) -> None:
|
|
receipt = apply_frame_claim(
|
|
claim=_claim("gives"),
|
|
frame_category="transfer_frame",
|
|
polarity="affirms",
|
|
reviewer="Ada",
|
|
pack_root=pack_copy,
|
|
)
|
|
|
|
assert len(receipt.file_sha256_before) == 64
|
|
assert len(receipt.file_sha256_after) == 64
|
|
assert receipt.file_sha256_before != receipt.file_sha256_after
|
|
assert receipt.evidence_hash == _claim("gives").evidence_hash
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 4 — idempotent same-evidence raises AlreadyRatified
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_idempotent_same_evidence_raises_already_ratified(pack_copy: Path) -> None:
|
|
claim = _claim("gives")
|
|
apply_frame_claim(
|
|
claim=claim,
|
|
frame_category="transfer_frame",
|
|
polarity="affirms",
|
|
reviewer="Ada",
|
|
pack_root=pack_copy,
|
|
)
|
|
|
|
with pytest.raises(AlreadyRatified, match="already ratified"):
|
|
apply_frame_claim(
|
|
claim=claim,
|
|
frame_category="transfer_frame",
|
|
polarity="affirms",
|
|
reviewer="Ada",
|
|
pack_root=pack_copy,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 5 — rejects non-frame sub_type
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_rejects_non_frame_sub_type(pack_copy: Path) -> None:
|
|
with pytest.raises(WrongClaimSubType):
|
|
apply_frame_claim(
|
|
claim=_claim("gives", sub_type="lexical"),
|
|
frame_category="transfer_frame",
|
|
polarity="affirms",
|
|
reviewer="Ada",
|
|
pack_root=pack_copy,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 6 — rejects categories outside SAFE_FRAME_CATEGORIES (wrong=0 hazard)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_rejects_unsafe_frame_category(pack_copy: Path) -> None:
|
|
with pytest.raises(
|
|
WrongZeroViolationCandidate, match="SAFE_FRAME_CATEGORIES"
|
|
):
|
|
apply_frame_claim(
|
|
claim=_claim("gives"),
|
|
frame_category="comparison_frame", # not in allowlist
|
|
polarity="affirms",
|
|
reviewer="Ada",
|
|
pack_root=pack_copy,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 7 — rejects invalid polarity
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_rejects_invalid_polarity(pack_copy: Path) -> None:
|
|
with pytest.raises(InvalidPolarity, match="polarity must be one of"):
|
|
apply_frame_claim(
|
|
claim=_claim("gives"),
|
|
frame_category="transfer_frame",
|
|
polarity="maybe", # not affirms/falsifies
|
|
reviewer="Ada",
|
|
pack_root=pack_copy,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 8 — rejects evidence tampering
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_rejects_evidence_tampering(pack_copy: Path) -> None:
|
|
claim = _claim("gives")
|
|
object.__setattr__(claim, "evidence_hash", "0" * 64)
|
|
|
|
with pytest.raises(EvidenceTampering):
|
|
apply_frame_claim(
|
|
claim=claim,
|
|
frame_category="transfer_frame",
|
|
polarity="affirms",
|
|
reviewer="Ada",
|
|
pack_root=pack_copy,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 9 — rejects evidence laundering as source='corpus'
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_rejects_evidence_laundered_as_corpus(pack_copy: Path) -> None:
|
|
"""ADR-0168.1 §'Evidence floor': source='corpus' MUST be rejected."""
|
|
with pytest.raises(EvidenceLaundering, match="MUST NOT be laundered"):
|
|
apply_frame_claim(
|
|
claim=_claim("gives"),
|
|
frame_category="transfer_frame",
|
|
polarity="affirms",
|
|
reviewer="Ada",
|
|
pack_root=pack_copy,
|
|
evidence_source="corpus", # forbidden
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 10 — case 0050 hazard pin: still refused after ratification
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_case_0050_remains_refused_after_frame_ratification(
|
|
pack_copy: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
"""After ratifying a safe transfer-frame verb, case 0050 must still refuse.
|
|
|
|
Case 0050 ("Mark does a gig every other day for 2 weeks. ..." — the
|
|
period token at sentence_index=0 with missing_operator=
|
|
pre_frame_filler_sentence) is the prototype wrong=0 hazard: a
|
|
speculative frame opener here would admit a partial graph. The new
|
|
frames/ entries must not open that admission path.
|
|
"""
|
|
apply_frame_claim(
|
|
claim=_claim("gives"),
|
|
frame_category="transfer_frame",
|
|
polarity="affirms",
|
|
reviewer="Ada",
|
|
pack_root=pack_copy,
|
|
)
|
|
_use_pack_for_reader(monkeypatch, pack_copy)
|
|
cases = [json.loads(line) for line in CASES_PATH.read_text().splitlines()]
|
|
case = next(c for c in cases if c["case_id"] == "gsm8k-train-sample-v1-0050")
|
|
|
|
result, _rows = audit_problem(case["question"], case_id=case["case_id"])
|
|
|
|
assert isinstance(result, ReaderRefusal), (
|
|
"case 0050 must remain refused after FrameClaim ratification"
|
|
)
|
|
assert result.sentence_index == 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 11 — polarity=falsifies branch records non-opener
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_polarity_falsifies_records_non_opener(pack_copy: Path) -> None:
|
|
"""A 'falsifies' ratification records the surface as NOT opening the frame.
|
|
|
|
This is the negative-evidence path: an operator reviewed the audit row
|
|
and decided the surface does not in fact open the frame category. The
|
|
entry is written to the same source file but with polarity=falsifies
|
|
so downstream registry builds can mark the surface as a known non-opener.
|
|
"""
|
|
receipt = apply_frame_claim(
|
|
claim=_claim("along"), # not actually a transfer verb
|
|
frame_category="transfer_frame",
|
|
polarity="falsifies",
|
|
reviewer="Ada",
|
|
pack_root=pack_copy,
|
|
)
|
|
|
|
assert receipt.polarity == "falsifies"
|
|
entry = next(
|
|
e
|
|
for e in _entries(pack_copy / "frames" / "transfer_frame.jsonl")
|
|
if e["surface_form"] == "along"
|
|
)
|
|
assert entry["polarity"] == "falsifies"
|
|
|
|
# affirms + falsifies of the same surface produce distinct entries
|
|
receipt2 = apply_frame_claim(
|
|
claim=_claim("gave"),
|
|
frame_category="transfer_frame",
|
|
polarity="affirms",
|
|
reviewer="Ada",
|
|
pack_root=pack_copy,
|
|
)
|
|
assert receipt2.polarity == "affirms"
|
|
polarities = {
|
|
e["polarity"]
|
|
for e in _entries(pack_copy / "frames" / "transfer_frame.jsonl")
|
|
}
|
|
assert {"affirms", "falsifies"} <= polarities
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 12 — duplicate evidence appends to existing row
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_duplicate_surface_polarity_with_new_evidence_appends_hash(
|
|
pack_copy: Path,
|
|
) -> None:
|
|
"""ADR-0168.1 §'Idempotency': same claim + new evidence appends hash, not row."""
|
|
# First ratification
|
|
apply_frame_claim(
|
|
claim=_claim("gives"),
|
|
frame_category="transfer_frame",
|
|
polarity="affirms",
|
|
reviewer="Ada",
|
|
pack_root=pack_copy,
|
|
)
|
|
|
|
# Second piece of evidence for the same claim — different case_id, so
|
|
# the evidence_hash differs even though surface+polarity+category match.
|
|
second_row = AuditRow(
|
|
case_id="case-gives-2",
|
|
sentence_index=1,
|
|
token_index=4,
|
|
token_text="gives",
|
|
recognized_terms=("Bob", "buys"),
|
|
skipped_frame=None,
|
|
missing_operator="pre_frame_filler_sentence",
|
|
refusal_reason="unexpected_category",
|
|
refusal_detail="duplicate surface evidence",
|
|
)
|
|
second_claim = from_audit_row(second_row, "frame")
|
|
receipt = apply_frame_claim(
|
|
claim=second_claim,
|
|
frame_category="transfer_frame",
|
|
polarity="affirms",
|
|
reviewer="Ada",
|
|
pack_root=pack_copy,
|
|
)
|
|
|
|
assert receipt.is_duplicate_evidence is True
|
|
rows = [
|
|
e
|
|
for e in _entries(pack_copy / "frames" / "transfer_frame.jsonl")
|
|
if e["surface_form"] == "gives"
|
|
]
|
|
assert len(rows) == 1, "duplicate evidence must not create a second row"
|
|
assert len(rows[0]["evidence_hashes"]) == 2
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 13 — manifest.json checksum unchanged
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_lexicon_checksum_preserved_by_frame_ratification(pack_copy: Path) -> None:
|
|
"""RAT-1 — the lexicon ``checksum`` field is preserved across frame
|
|
ratification. The manifest may gain a ``frame_checksum`` field
|
|
(auto-compile per RAT-1), but the pre-existing lexicon checksum
|
|
bytes are untouched.
|
|
"""
|
|
manifest_before = json.loads((pack_copy / "manifest.json").read_bytes())
|
|
declared_before = manifest_before["checksum"]
|
|
|
|
apply_frame_claim(
|
|
claim=_claim("gives"),
|
|
frame_category="transfer_frame",
|
|
polarity="affirms",
|
|
reviewer="Ada",
|
|
pack_root=pack_copy,
|
|
)
|
|
|
|
manifest_after = json.loads((pack_copy / "manifest.json").read_bytes())
|
|
assert manifest_after["checksum"] == declared_before, (
|
|
"lexicon checksum must not change during frame ratification"
|
|
)
|
|
# RAT-1: frame_checksum may now be present (auto-compile).
|
|
if "frame_checksum" in manifest_after:
|
|
assert isinstance(manifest_after["frame_checksum"], str)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test 14 — alphabetical sort preserved across multiple writes
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_alphabetical_sort_preserved(pack_copy: Path) -> None:
|
|
for surface in ("transfers", "gives", "ceded"):
|
|
apply_frame_claim(
|
|
claim=_claim(surface),
|
|
frame_category="transfer_frame",
|
|
polarity="affirms",
|
|
reviewer="Ada",
|
|
pack_root=pack_copy,
|
|
)
|
|
|
|
surfaces = [
|
|
e["surface_form"]
|
|
for e in _entries(pack_copy / "frames" / "transfer_frame.jsonl")
|
|
]
|
|
assert surfaces == sorted(surfaces)
|