193764e3fd
Adds surface_pattern, composition_category, and polarity to the proposed_change_payload for composition_reclassification proposals so operators can call apply_composition_claim() without field synthesis. Dispatch by missing_operator: - quantity_extraction → multiplicative_composition + bound(count) × bound(unit_cost) - multi_quantity_composition → additive_composition + bound(qty_a) + bound(qty_b) All other change kinds (matcher_extension, injector_sub_shape, frame_reclassification) keep the existing evidence-aggregation payload. Legacy fields (evidence_count, group_key, modal_sub_type) preserved. Adds tests/test_contemplation_ratifiable_payload.py with 11 tests including a round-trip from decompose_audit → apply_composition_claim.
414 lines
14 KiB
Python
414 lines
14 KiB
Python
"""Brief B — Ratifiable payload tests for contemplation proposals.
|
||
|
||
Pins that ``decompose_audit`` emits ``proposed_change_payload`` fields that
|
||
satisfy ``apply_composition_claim()`` preconditions directly, with no
|
||
operator-side field synthesis.
|
||
|
||
Coverage:
|
||
1. quantity_extraction group → payload carries composition_category,
|
||
surface_pattern, polarity.
|
||
2. multi_quantity_composition group → payload carries the additive fields.
|
||
3. Legacy fields (evidence_count, group_key, modal_sub_type) still present
|
||
(backward-compat schema regression).
|
||
4. Non-composition change kinds (matcher_extension, injector_sub_shape,
|
||
frame_reclassification) do NOT get the enriched fields.
|
||
5. Enriched payload fields satisfy apply_composition_claim preconditions.
|
||
6. Round-trip: real audit composition proposal → apply_composition_claim
|
||
without any field synthesis.
|
||
7. polarity is always "affirms" in the enriched payload.
|
||
8. Determinism: payload is identical across reruns.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import shutil
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
|
||
from generate.comprehension import lexicon as comprehension_lexicon
|
||
from generate.comprehension import lifecycle
|
||
from teaching.math_composition_ratification import (
|
||
SAFE_COMPOSITION_CATEGORIES,
|
||
apply_composition_claim,
|
||
)
|
||
from teaching.math_contemplation import decompose_audit
|
||
|
||
|
||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||
PACK_ROOT = REPO_ROOT / "language_packs" / "data" / "en_core_math_v1"
|
||
REAL_AUDIT_PATH = (
|
||
REPO_ROOT / "evals" / "gsm8k_math" / "train_sample" / "v1" / "audit_brief_11.json"
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _case(
|
||
*,
|
||
case_id: str,
|
||
refusal_reason: str,
|
||
missing_operator: str,
|
||
token_text: str = "x",
|
||
) -> dict:
|
||
return {
|
||
"case_id": case_id,
|
||
"outcome": "refused",
|
||
"refusal_reason": refusal_reason,
|
||
"missing_operator": missing_operator,
|
||
"refusal_detail": "",
|
||
"sentence_index": 0,
|
||
"token_index": 0,
|
||
"token_text": token_text,
|
||
"recognized_terms": [],
|
||
"skipped_frame": None,
|
||
}
|
||
|
||
|
||
def _write_audit(tmp_path: Path, per_case: list[dict]) -> Path:
|
||
audit = {
|
||
"schema_version": 1,
|
||
"brief": "synthetic-ratifiable-payload-test",
|
||
"case_count": len(per_case),
|
||
"per_case": per_case,
|
||
}
|
||
target = tmp_path / "audit.json"
|
||
target.write_text(json.dumps(audit), encoding="utf-8")
|
||
return target
|
||
|
||
|
||
@pytest.fixture()
|
||
def pack_copy(tmp_path: Path) -> Path:
|
||
target = tmp_path / "en_core_math_v1"
|
||
shutil.copytree(PACK_ROOT, target)
|
||
(target / "compositions").mkdir(exist_ok=True)
|
||
# ensure additive_composition.jsonl exists for round-trip
|
||
(target / "compositions" / "additive_composition.jsonl").write_text(
|
||
"", encoding="utf-8"
|
||
)
|
||
comprehension_lexicon._CACHE.clear()
|
||
lifecycle._get_lexicon.cache_clear()
|
||
yield target
|
||
comprehension_lexicon._CACHE.clear()
|
||
lifecycle._get_lexicon.cache_clear()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 1. quantity_extraction payload
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_quantity_extraction_payload_has_ratifiable_fields(tmp_path: Path) -> None:
|
||
"""quantity_extraction group payload carries composition_category, surface_pattern, polarity."""
|
||
audit = _write_audit(
|
||
tmp_path,
|
||
[
|
||
_case(
|
||
case_id="qe-001",
|
||
refusal_reason="incomplete_operation",
|
||
missing_operator="quantity_extraction",
|
||
),
|
||
_case(
|
||
case_id="qe-002",
|
||
refusal_reason="incomplete_operation",
|
||
missing_operator="quantity_extraction",
|
||
),
|
||
],
|
||
)
|
||
proposals = decompose_audit(audit)
|
||
assert len(proposals) == 1
|
||
payload = proposals[0].proposed_change_payload
|
||
assert isinstance(payload, dict)
|
||
assert payload["composition_category"] == "multiplicative_composition"
|
||
assert payload["surface_pattern"] == "bound(count) × bound(unit_cost)"
|
||
assert payload["polarity"] == "affirms"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 2. multi_quantity_composition payload
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_multi_quantity_composition_payload_has_ratifiable_fields(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
"""multi_quantity_composition group payload carries the additive fields."""
|
||
audit = _write_audit(
|
||
tmp_path,
|
||
[
|
||
_case(
|
||
case_id="mqc-001",
|
||
refusal_reason="incomplete_operation",
|
||
missing_operator="multi_quantity_composition",
|
||
),
|
||
_case(
|
||
case_id="mqc-002",
|
||
refusal_reason="incomplete_operation",
|
||
missing_operator="multi_quantity_composition",
|
||
),
|
||
],
|
||
)
|
||
proposals = decompose_audit(audit)
|
||
assert len(proposals) == 1
|
||
payload = proposals[0].proposed_change_payload
|
||
assert isinstance(payload, dict)
|
||
assert payload["composition_category"] == "additive_composition"
|
||
assert payload["surface_pattern"] == "bound(qty_a) + bound(qty_b)"
|
||
assert payload["polarity"] == "affirms"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 3. Legacy fields preserved (backward-compat schema regression)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_composition_payload_preserves_legacy_fields(tmp_path: Path) -> None:
|
||
"""evidence_count, group_key, modal_sub_type must still be present."""
|
||
audit = _write_audit(
|
||
tmp_path,
|
||
[
|
||
_case(
|
||
case_id="leg-001",
|
||
refusal_reason="incomplete_operation",
|
||
missing_operator="quantity_extraction",
|
||
),
|
||
_case(
|
||
case_id="leg-002",
|
||
refusal_reason="incomplete_operation",
|
||
missing_operator="quantity_extraction",
|
||
),
|
||
_case(
|
||
case_id="leg-003",
|
||
refusal_reason="incomplete_operation",
|
||
missing_operator="quantity_extraction",
|
||
),
|
||
],
|
||
)
|
||
proposals = decompose_audit(audit)
|
||
assert len(proposals) == 1
|
||
payload = proposals[0].proposed_change_payload
|
||
assert isinstance(payload, dict)
|
||
assert payload["evidence_count"] == 3
|
||
assert payload["group_key"] == {
|
||
"missing_operator": "quantity_extraction",
|
||
"refusal_reason": "incomplete_operation",
|
||
}
|
||
assert "modal_sub_type" in payload
|
||
|
||
|
||
def test_multi_qty_payload_preserves_legacy_fields(tmp_path: Path) -> None:
|
||
"""multi_quantity_composition also preserves legacy fields."""
|
||
audit = _write_audit(
|
||
tmp_path,
|
||
[
|
||
_case(
|
||
case_id="mql-001",
|
||
refusal_reason="incomplete_operation",
|
||
missing_operator="multi_quantity_composition",
|
||
),
|
||
_case(
|
||
case_id="mql-002",
|
||
refusal_reason="incomplete_operation",
|
||
missing_operator="multi_quantity_composition",
|
||
),
|
||
],
|
||
)
|
||
proposals = decompose_audit(audit)
|
||
assert len(proposals) == 1
|
||
payload = proposals[0].proposed_change_payload
|
||
assert "evidence_count" in payload
|
||
assert "group_key" in payload
|
||
assert "modal_sub_type" in payload
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 4. Non-composition kinds do NOT get enriched fields
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"refusal_reason,missing_operator",
|
||
[
|
||
("unexpected_category", "pre_frame_filler_sentence"),
|
||
("unexpected_category", "multi_subject_sentence"),
|
||
("narrowness_violation", "multi_quantity_composition"),
|
||
],
|
||
)
|
||
def test_non_composition_payload_not_enriched(
|
||
tmp_path: Path, refusal_reason: str, missing_operator: str
|
||
) -> None:
|
||
"""Non-composition change kinds must not carry composition_category etc."""
|
||
audit = _write_audit(
|
||
tmp_path,
|
||
[
|
||
_case(
|
||
case_id="nc-001",
|
||
refusal_reason=refusal_reason,
|
||
missing_operator=missing_operator,
|
||
),
|
||
_case(
|
||
case_id="nc-002",
|
||
refusal_reason=refusal_reason,
|
||
missing_operator=missing_operator,
|
||
),
|
||
],
|
||
)
|
||
proposals = decompose_audit(audit)
|
||
assert len(proposals) == 1
|
||
payload = proposals[0].proposed_change_payload
|
||
assert isinstance(payload, dict)
|
||
assert "composition_category" not in payload
|
||
assert "surface_pattern" not in payload
|
||
assert "polarity" not in payload
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 5. Enriched fields satisfy apply_composition_claim preconditions
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_enriched_fields_satisfy_apply_composition_claim_preconditions(
|
||
tmp_path: Path,
|
||
) -> None:
|
||
"""Each enriched composition_category is in SAFE_COMPOSITION_CATEGORIES."""
|
||
for missing_operator, refusal_reason in [
|
||
("quantity_extraction", "incomplete_operation"),
|
||
("multi_quantity_composition", "incomplete_operation"),
|
||
]:
|
||
sub = tmp_path / missing_operator
|
||
sub.mkdir()
|
||
audit = _write_audit(
|
||
sub,
|
||
[
|
||
_case(
|
||
case_id=f"{missing_operator}-p-001",
|
||
refusal_reason=refusal_reason,
|
||
missing_operator=missing_operator,
|
||
),
|
||
_case(
|
||
case_id=f"{missing_operator}-p-002",
|
||
refusal_reason=refusal_reason,
|
||
missing_operator=missing_operator,
|
||
),
|
||
],
|
||
)
|
||
proposals = decompose_audit(audit)
|
||
assert len(proposals) == 1
|
||
payload = proposals[0].proposed_change_payload
|
||
assert isinstance(payload, dict)
|
||
assert payload["composition_category"] in SAFE_COMPOSITION_CATEGORIES
|
||
assert payload["polarity"] in {"affirms", "falsifies"}
|
||
assert payload["surface_pattern"].strip()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 6. Round-trip: real audit composition proposal → apply_composition_claim
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_round_trip_composition_proposal_into_apply_composition_claim(
|
||
pack_copy: Path,
|
||
) -> None:
|
||
"""First composition_reclassification proposal from real audit → apply_composition_claim
|
||
without operator-side field synthesis."""
|
||
assert REAL_AUDIT_PATH.exists(), REAL_AUDIT_PATH
|
||
|
||
proposals = decompose_audit(REAL_AUDIT_PATH)
|
||
comp_proposals = [
|
||
p
|
||
for p in proposals
|
||
if p.proposed_change_kind == "composition_reclassification"
|
||
]
|
||
assert comp_proposals, "real audit must produce at least one composition_reclassification proposal"
|
||
|
||
proposal = comp_proposals[0]
|
||
payload = proposal.proposed_change_payload
|
||
assert isinstance(payload, dict)
|
||
|
||
# All fields needed by apply_composition_claim come straight from the payload
|
||
composition_category: str = payload["composition_category"]
|
||
surface_pattern: str = payload["surface_pattern"]
|
||
polarity: str = payload["polarity"]
|
||
|
||
# Use the first evidence pointer's backing record
|
||
evidence_record = proposal.evidence_pointers[0]
|
||
|
||
# apply_composition_claim must succeed without exception
|
||
receipt = apply_composition_claim(
|
||
claim=evidence_record,
|
||
composition_category=composition_category,
|
||
polarity=polarity,
|
||
reviewer="test_round_trip",
|
||
surface_pattern=surface_pattern,
|
||
pack_root=pack_copy,
|
||
)
|
||
assert receipt.composition_category == composition_category
|
||
assert receipt.polarity == polarity
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 7. polarity is always "affirms" in enriched payload
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_composition_payload_polarity_is_always_affirms(tmp_path: Path) -> None:
|
||
"""polarity must be 'affirms' in all auto-generated composition payloads."""
|
||
for missing_operator in ("quantity_extraction", "multi_quantity_composition"):
|
||
sub = tmp_path / missing_operator
|
||
sub.mkdir(exist_ok=True)
|
||
audit = _write_audit(
|
||
sub,
|
||
[
|
||
_case(
|
||
case_id=f"pol-{missing_operator}-001",
|
||
refusal_reason="incomplete_operation",
|
||
missing_operator=missing_operator,
|
||
),
|
||
_case(
|
||
case_id=f"pol-{missing_operator}-002",
|
||
refusal_reason="incomplete_operation",
|
||
missing_operator=missing_operator,
|
||
),
|
||
],
|
||
)
|
||
proposals = decompose_audit(audit)
|
||
assert len(proposals) == 1
|
||
assert proposals[0].proposed_change_payload["polarity"] == "affirms"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 8. Determinism: payload is identical across reruns
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_enriched_payload_deterministic_across_reruns(tmp_path: Path) -> None:
|
||
"""Same audit → byte-identical payload across 5 reruns."""
|
||
audit = _write_audit(
|
||
tmp_path,
|
||
[
|
||
_case(
|
||
case_id="det-001",
|
||
refusal_reason="incomplete_operation",
|
||
missing_operator="quantity_extraction",
|
||
),
|
||
_case(
|
||
case_id="det-002",
|
||
refusal_reason="incomplete_operation",
|
||
missing_operator="quantity_extraction",
|
||
),
|
||
],
|
||
)
|
||
first = decompose_audit(audit)
|
||
first_payload = json.dumps(
|
||
first[0].proposed_change_payload, sort_keys=True, separators=(",", ":")
|
||
)
|
||
for _ in range(4):
|
||
rerun = decompose_audit(audit)
|
||
rerun_payload = json.dumps(
|
||
rerun[0].proposed_change_payload, sort_keys=True, separators=(",", ":")
|
||
)
|
||
assert rerun_payload == first_payload
|