core/tests/test_comprehension_structure_preserving.py

"""Structure preservation — the reader recovers the EXACT structure, not just a
verdict-equivalent one (closes the coincidental-correctness gap).

`test_comprehension_wrong_zero_property.py` proves ANSWER preservation: the
comprehension path yields the same oracle verdict as the ground-truth structure. But
a *misread* graph can coincidentally yield the same verdict and pass that test. This
module proves the stronger property: over randomly generated structures rendered to
prose, the projected structure (and query) the reader recovers equals the
ground-truth structure exactly — or the reader refuses. Verdict agreement then
follows for free; here we assert STRUCTURE, separately.

Each generator renders prose that FULLY determines its structure (every term /
class / item / atom it claims is actually stated), so projected == ground truth is
the honest bar. A reader that drops, adds, swaps, or mis-roles any element fails
here even when the final verdict happens to match.
"""

from __future__ import annotations

import random

from generate.meaning_graph.projectors import (
    to_deductive_logic,
    to_set_membership,
    to_syllogism,
    to_total_ordering,
)
from generate.meaning_graph.reader import Refusal, comprehend
from generate.quantitative_comprehension import comprehend_quantitative, to_relational_metric

_TERMS = [f"t{i}" for i in range(8)]


def _canon(value):
    """Order-insensitive canonicalization (lists/tuples are sets here, since the
    reasoning structures are order-free) for exact structure comparison."""
    if isinstance(value, dict):
        return tuple(sorted((k, _canon(v)) for k, v in value.items()))
    if isinstance(value, (list, tuple)):
        return tuple(sorted((_canon(v) for v in value), key=repr))
    return value


def _project(comp_or_refusal, projector):
    if isinstance(comp_or_refusal, Refusal):
        return None
    return projector(comp_or_refusal)


# --------------------------------------------------------------------------- #
# Syllogism
# --------------------------------------------------------------------------- #

_PREM = {
    "A": lambda s, p: f"All {s}s are {p}s.",
    "E": lambda s, p: f"No {s}s are {p}s.",
    "I": lambda s, p: f"Some {s}s are {p}s.",
    "O": lambda s, p: f"Some {s}s are not {p}s.",
}
_CONCL = {
    "A": lambda s, p: f"Therefore all {s}s are {p}s.",
    "E": lambda s, p: f"Therefore no {s}s are {p}s.",
    "I": lambda s, p: f"Therefore some {s}s are {p}s.",
    "O": lambda s, p: f"Therefore some {s}s are not {p}s.",
}


def test_syllogism_structure_is_preserved_exactly() -> None:
    rng = random.Random(11)
    committed = 0
    for _ in range(400):
        pool = rng.sample(_TERMS, 3)
        prem = [(rng.choice("AEIO"), *rng.sample(pool, 2)) for _ in range(2)]
        cc = (rng.choice("AEIO"), *rng.sample(pool, 2))
        used = sorted({t for _, s, p in prem for t in (s, p)} | {cc[1], cc[2]})
        prose = " ".join([_PREM[f](s, p) for f, s, p in prem] + [_CONCL[cc[0]](cc[1], cc[2])])
        structure = {
            "terms": used,
            "domain_size": 3,
            "premises": [{"form": f, "subject": s, "predicate": p} for f, s, p in prem],
        }
        query = {"kind": "validity", "conclusion": {"form": cc[0], "subject": cc[1], "predicate": cc[2]}}
        proj = _project(comprehend(prose), to_syllogism)
        if proj is None:
            continue
        committed += 1
        pstruct, pquery = proj
        assert _canon(pstruct) == _canon(structure), (prose, pstruct, structure)
        assert pquery == query, (prose, pquery, query)
    assert committed > 50


# --------------------------------------------------------------------------- #
# Total ordering
# --------------------------------------------------------------------------- #


def test_total_ordering_structure_is_preserved_exactly() -> None:
    rng = random.Random(22)
    committed = 0
    for _ in range(300):
        n = rng.randint(2, 5)
        chain = rng.sample(_TERMS, n)
        rels = [{"less": chain[i], "greater": chain[i + 1]} for i in range(n - 1)]
        facts = ", and ".join(f"{lo} is below {hi}" for lo, hi in zip(chain, chain[1:])) + "."
        if rng.random() < 0.5:
            order = rng.choice(["ascending", "descending"])
            prose = f"{facts} Sort {order}."
            query = {"kind": "sort", "order": order}
        else:
            x, y = rng.sample(chain, 2)
            prose = f"{facts} Compare {x} with {y}."
            query = {"kind": "compare", "left": x, "right": y}
        structure = {"items": sorted(chain), "relations": rels}
        proj = _project(comprehend(prose), to_total_ordering)
        if proj is None:
            continue
        committed += 1
        pstruct, pquery = proj
        assert _canon(pstruct) == _canon(structure), (prose, pstruct, structure)
        assert pquery == query, (prose, pquery, query)
    assert committed > 50


# --------------------------------------------------------------------------- #
# Set membership — classes derived from stated facts (so prose fully determines it)
# --------------------------------------------------------------------------- #


def test_set_membership_structure_is_preserved_exactly() -> None:
    rng = random.Random(33)
    committed = 0
    for _ in range(300):
        pool = rng.sample(_TERMS, rng.randint(2, 4))
        individuals = [f"e{i}" for i in range(rng.randint(1, 3))]
        member_facts = [(ind, rng.choice(pool)) for ind in individuals]
        subset_facts = [
            (pool[i], pool[i + 1]) for i in range(len(pool) - 1) if rng.random() < 0.7
        ]
        # Classes that are actually STATED (member class or either side of a subset).
        used_classes = sorted(
            {c for _, c in member_facts} | {a for a, _ in subset_facts} | {b for _, b in subset_facts}
        )
        member_lines = [f"{ind} is a {cls}." for ind, cls in member_facts]
        subset_lines = [f"All {a}s are {b}s." for a, b in subset_facts]
        sets = [
            {"id": c, "members": sorted({i for i, cl in member_facts if cl == c})}
            for c in used_classes
        ]
        structure = {
            "elements": sorted(individuals),
            "sets": sets,
            "subsets": [{"subset": a, "superset": b} for a, b in subset_facts],
        }
        # Query over stated entities only.
        if rng.random() < 0.5 and individuals:
            ind = rng.choice(individuals)
            target = rng.choice(used_classes)
            prose = " ".join(member_lines + subset_lines + [f"Is {ind} a {target}?"])
            query = {"kind": "member", "element": ind, "set": target}
        elif len(used_classes) >= 2:
            a, b = rng.sample(used_classes, 2)
            prose = " ".join(member_lines + subset_lines + [f"Are all {a}s {b}s?"])
            query = {"kind": "subset", "subset": a, "superset": b}
        else:
            continue
        proj = _project(comprehend(prose), to_set_membership)
        if proj is None:
            continue
        committed += 1
        pstruct, pquery = proj
        assert _canon(pstruct) == _canon(structure), (prose, pstruct, structure)
        assert pquery == query, (prose, pquery, query)
    assert committed > 50


# --------------------------------------------------------------------------- #
# Propositional logic — premises (as a set of formula strings) + query string
# --------------------------------------------------------------------------- #

_ATOMS = [f"p{i}" for i in range(5)]


def _prop_fact(rng):
    kind = rng.choice(["implies", "or", "atom", "not_atom"])
    if kind == "implies":
        a, b = rng.sample(_ATOMS, 2)
        return f"If {a} then {b}.", f"{a} implies {b}"
    if kind == "or":
        a, b = rng.sample(_ATOMS, 2)
        return f"{a} or {b}.", f"{a} or {b}"
    if kind == "not_atom":
        a = rng.choice(_ATOMS)
        return f"Not {a}.", f"not {a}"
    a = rng.choice(_ATOMS)
    return f"{a}.", a


def _prop_query(rng):
    kind = rng.choice(["atom", "not_atom", "implies"])
    if kind == "implies":
        a, b = rng.sample(_ATOMS, 2)
        return f"Therefore if {a} then {b}.", f"{a} implies {b}"
    if kind == "not_atom":
        a = rng.choice(_ATOMS)
        return f"Therefore not {a}.", f"not {a}"
    a = rng.choice(_ATOMS)
    return f"Therefore {a}.", a


def test_propositional_structure_is_preserved_exactly() -> None:
    rng = random.Random(44)
    committed = 0
    for _ in range(400):
        facts = [_prop_fact(rng) for _ in range(rng.randint(1, 3))]
        concl_prose, query_formula = _prop_query(rng)
        prose = " ".join(p for p, _ in facts) + " " + concl_prose
        premises = frozenset(f for _, f in facts)
        proj = _project(comprehend(prose), to_deductive_logic)
        if proj is None:
            continue
        committed += 1
        pprem, pquery = proj
        assert frozenset(pprem) == premises, (prose, pprem, premises)
        assert pquery == query_formula, (prose, pquery, query_formula)
    assert committed > 50


# --------------------------------------------------------------------------- #
# Perturbation invariance — meaning-preserving surface changes (premise/clause
# reordering, capitalization, extra whitespace) must yield the SAME structure.
# --------------------------------------------------------------------------- #


def _struct(prose, projector):
    proj = _project(comprehend(prose), projector)
    return None if proj is None else (_canon(proj[0]), proj[1])


def test_syllogism_invariant_to_premise_reorder_and_caps() -> None:
    # Same two premises in either order + capitalized variant -> identical structure.
    base = "All mammals are animals. All whales are mammals. Therefore all whales are animals."
    swapped = "All whales are mammals. All mammals are animals. Therefore all whales are animals."
    caps = "ALL MAMMALS ARE ANIMALS. All Whales Are Mammals. Therefore all whales are animals."
    s_base = _struct(base, to_syllogism)
    assert s_base is not None
    assert _struct(swapped, to_syllogism) == s_base
    assert _struct(caps, to_syllogism) == s_base


def test_total_ordering_invariant_to_clause_reorder_and_whitespace() -> None:
    base = "a is below b, and b is below c. Sort ascending."
    reordered = "b is below c, and a is below b. Sort ascending."
    spaced = "a   is below b,  and b is below   c.  Sort ascending."
    s_base = _struct(base, to_total_ordering)
    assert s_base is not None
    assert _struct(reordered, to_total_ordering) == s_base
    assert _struct(spaced, to_total_ordering) == s_base


def test_propositional_invariant_to_premise_reorder() -> None:
    base = "If p then q. p. Therefore q."
    swapped = "p. If p then q. Therefore q."
    s_base = _struct(base, to_deductive_logic)
    assert s_base is not None
    assert _struct(swapped, to_deductive_logic) == s_base


# --------------------------------------------------------------------------- #
# Arithmetic (binding_graph) — projected relations + query preserved exactly.
# --------------------------------------------------------------------------- #


def test_arithmetic_structure_is_preserved_exactly() -> None:
    rng = random.Random(55)
    committed = 0
    for _ in range(300):
        ents = rng.sample([f"e{i}" for i in range(6)], rng.randint(2, 4))
        base, base_val = ents[0], rng.randint(1, 20)
        relations = [{"kind": "fact", "entity": base, "value": base_val}]
        lines = [f"{base} has {base_val} things."]
        prev = base
        for e in ents[1:]:
            delta = rng.randint(1, 15)
            kind = rng.choice(["more_than", "fewer_than"])
            word = "more" if kind == "more_than" else "fewer"
            relations.append({"kind": kind, "entity": e, "ref": prev, "delta": delta})
            lines.append(f"{e} has {delta} {word} things than {prev}.")
            prev = e
        ask = rng.choice(ents)
        lines.append(f"How many things does {ask} have?")
        prose = " ".join(lines)
        expected_query = {"entity": ask, "unit": "item"}  # "things" -> item dimension

        comp = comprehend_quantitative(prose)
        if isinstance(comp, Refusal):
            continue
        proj = to_relational_metric(comp)
        if proj is None:
            continue
        committed += 1
        prelations, pquery = proj
        assert _canon(prelations) == _canon(relations), (prose, prelations, relations)
        assert pquery == expected_query, (prose, pquery, expected_query)
    assert committed > 50