core/tests/test_math_claim_signature.py

"""Tests for ADR-0167 W2-B: lexical claim signature normalisation.

Verifies:
- Determinism (same input → same hex)
- Punctuation stripping
- Case insensitivity
- Format invariant (64-char lowercase hex)
- Fallback when refusal_detail doesn't match the canonical regex
- Real-data sanity: no false collisions on audit_brief_11.json
- Real-data dedup: duplicate tokens collapse to one signature
- Non-lexical evidence pins the W2-A invariant (claim_signature stays "")
"""

from __future__ import annotations

import hashlib
import json
from pathlib import Path

import pytest

from teaching.math_claim_signature import lexical_claim_signature
from teaching.math_contemplation import audit_to_evidence
from teaching.math_evidence import SUB_TYPE_FOR_OPERATOR

# ---------------------------------------------------------------------------
# Fixture: load audit_brief_11.json once
# ---------------------------------------------------------------------------

_ARTIFACT_PATH = (
    Path(__file__).parent.parent
    / "evals/gsm8k_math/train_sample/v1/audit_brief_11.json"
)

_LEXICAL_OPS: frozenset[str] = frozenset(
    op
    for op, sub in SUB_TYPE_FOR_OPERATOR.items()
    if sub == "lexical"
)


@pytest.fixture(scope="module")
def artifact() -> dict:
    return json.loads(_ARTIFACT_PATH.read_text(encoding="utf-8"))


@pytest.fixture(scope="module")
def lexical_cases(artifact) -> list[dict]:
    return [
        c
        for c in artifact["per_case"]
        if c.get("missing_operator") in _LEXICAL_OPS
    ]


# ---------------------------------------------------------------------------
# Core invariant tests
# ---------------------------------------------------------------------------


def test_identical_surface_identical_signature():
    sig1 = lexical_claim_signature(
        surface="crayons",
        refusal_detail="no primitive or lexicon match for 'crayons'",
    )
    sig2 = lexical_claim_signature(
        surface="crayons",
        refusal_detail="no primitive or lexicon match for 'crayons'",
    )
    assert sig1 == sig2


def test_different_surface_different_signature():
    sig_a = lexical_claim_signature(
        surface="crayons",
        refusal_detail="no primitive or lexicon match for 'crayons'",
    )
    sig_b = lexical_claim_signature(
        surface="oysters",
        refusal_detail="no primitive or lexicon match for 'oysters'",
    )
    assert sig_a != sig_b


def test_punctuation_strip():
    """crayons, crayons. and crayons should all collapse to the same signature."""
    base_detail = "no primitive or lexicon match for 'crayons'"
    sig_plain = lexical_claim_signature(surface="crayons", refusal_detail=base_detail)
    sig_comma = lexical_claim_signature(surface="crayons,", refusal_detail=base_detail)
    sig_period = lexical_claim_signature(surface="crayons.", refusal_detail=base_detail)
    # Note: when refusal_detail matches the regex, the extracted token wins
    # (which is always un-punctuated).  All three should produce the same sig.
    assert sig_plain == sig_comma == sig_period


def test_case_insensitive():
    detail = "no primitive or lexicon match for 'crayons'"
    sig_lower = lexical_claim_signature(surface="crayons", refusal_detail=detail)
    sig_upper = lexical_claim_signature(surface="Crayons", refusal_detail=detail)
    assert sig_lower == sig_upper


def test_signature_is_64_char_lowercase_hex():
    sig = lexical_claim_signature(
        surface="widgets",
        refusal_detail="no primitive or lexicon match for 'widgets'",
    )
    assert len(sig) == 64
    assert sig == sig.lower()
    assert all(c in "0123456789abcdef" for c in sig)


def test_extraction_falls_back_to_surface():
    """When refusal_detail doesn't match the regex, surface is used verbatim."""
    sig_fallback = lexical_claim_signature(
        surface="10%",
        refusal_detail="fraction/percentage literal at position 7 is out-of-scope (eval only)",
    )
    # Manually compute expected: strip punctuation from "10%", lowercase → "10"
    # Wait — "%" is in string.punctuation, so strip yields "10"
    # canonical = "lexical:10"
    expected = hashlib.sha256("lexical:10".encode("utf-8")).hexdigest()
    assert sig_fallback == expected


def test_extraction_fallback_is_still_deterministic():
    sig1 = lexical_claim_signature(
        surface="1/4",
        refusal_detail="fraction/percentage literal at position 11 is out-of-scope (eval only)",
    )
    sig2 = lexical_claim_signature(
        surface="1/4",
        refusal_detail="fraction/percentage literal at position 11 is out-of-scope (eval only)",
    )
    assert sig1 == sig2


# ---------------------------------------------------------------------------
# Real-data tests
# ---------------------------------------------------------------------------


def test_real_data_no_false_collisions(lexical_cases):
    """Distinct token surfaces must produce distinct signatures."""
    # Build {token_text → signature} using the canonical regex path where applicable
    token_to_sig: dict[str, str] = {}
    for case in lexical_cases:
        token = case["token_text"]
        sig = lexical_claim_signature(
            surface=token,
            refusal_detail=case.get("refusal_detail", ""),
        )
        if token in token_to_sig:
            assert token_to_sig[token] == sig, (
                f"Same token '{token}' produced different signatures"
            )
        else:
            token_to_sig[token] = sig
    # All distinct tokens should map to distinct signatures
    sigs = list(token_to_sig.values())
    assert len(sigs) == len(set(sigs)), (
        "False collision: distinct tokens collapsed to same signature"
    )


def test_real_data_collapses_duplicates(lexical_cases):
    """Two cases with the same extracted token collapse to one signature.

    In audit_brief_11.json each token is unique, so we simulate by taking
    a single token and verifying two invocations with different case context
    produce the same signature (dedup works across case boundaries).
    """
    # Pick the first lexical case and duplicate it with a different "caller"
    case = lexical_cases[0]
    sig_a = lexical_claim_signature(
        surface=case["token_text"],
        refusal_detail=case.get("refusal_detail", ""),
    )
    sig_b = lexical_claim_signature(
        surface=case["token_text"],
        refusal_detail=case.get("refusal_detail", ""),
    )
    assert sig_a == sig_b, "Same token should collapse to the same signature"


# ---------------------------------------------------------------------------
# W2-A invariant pin: non-lexical evidence has empty claim_signature
# ---------------------------------------------------------------------------


def test_non_lexical_evidence_has_empty_signature(artifact):
    """audit_to_evidence must leave claim_signature == '' for non-lexical rows."""
    from generate.comprehension.audit import AuditRow

    non_lexical_cases = [
        c
        for c in artifact["per_case"]
        if c.get("missing_operator") is not None
        and c.get("missing_operator") not in _LEXICAL_OPS
    ]
    assert non_lexical_cases, "Expected at least one non-lexical case in artifact"

    # Build AuditRow instances for non-lexical cases
    rows = [
        AuditRow(
            case_id=c["case_id"],
            sentence_index=c["sentence_index"],
            token_index=c.get("token_index", 0),
            token_text=c.get("token_text", ""),
            recognized_terms=tuple(c.get("recognized_terms", [])),
            skipped_frame=c.get("skipped_frame"),
            missing_operator=c["missing_operator"],
            refusal_reason=c["refusal_reason"],
            refusal_detail=c.get("refusal_detail", ""),
        )
        for c in non_lexical_cases
    ]
    evidence_records = audit_to_evidence(rows)
    for ev in evidence_records:
        assert ev.claim_signature == "", (
            f"Non-lexical sub_type '{ev.sub_type}' should have empty claim_signature, "
            f"got {ev.claim_signature!r}"
        )


# ---------------------------------------------------------------------------
# Lexical evidence in contemplation gets non-empty signature
# ---------------------------------------------------------------------------


def test_lexical_evidence_gets_non_empty_signature(lexical_cases):
    """audit_to_evidence fills claim_signature for lexical rows."""
    from generate.comprehension.audit import AuditRow

    rows = [
        AuditRow(
            case_id=c["case_id"],
            sentence_index=c["sentence_index"],
            token_index=c.get("token_index", 0),
            token_text=c.get("token_text", ""),
            recognized_terms=tuple(c.get("recognized_terms", [])),
            skipped_frame=c.get("skipped_frame"),
            missing_operator=c["missing_operator"],
            refusal_reason=c["refusal_reason"],
            refusal_detail=c.get("refusal_detail", ""),
        )
        for c in lexical_cases
    ]
    evidence_records = audit_to_evidence(rows)
    assert evidence_records, "Expected at least one lexical evidence record"
    for ev in evidence_records:
        assert ev.sub_type == "lexical"
        assert ev.claim_signature != "", (
            f"Lexical evidence for case {ev.case_id} must have non-empty claim_signature"
        )
        assert len(ev.claim_signature) == 64