core/tests/test_relational_transitive_lane.py

"""Lane test — transitive strict-order relational inference (capability-index lane +
wrong=0 bite + independent-oracle cross-check).

The positive lane (``v1/cases.jsonl``) is the capability-index coverage (breadth 10→11);
the confuser lane (``v1/refusals.jsonl``) is the wrong=0 bite — every confuser MUST refuse,
or the transitive rule over-fired. The independent transitive-closure oracle
(``evals.relational_transitive.oracle``, disjoint from the engine per INV-25/27) confirms
every positive IS entailed and every confuser is NOT — so the gold is non-vacuous. Fixture
SHAs are pinned so the gold cannot drift silently.
"""

from __future__ import annotations

import hashlib
from pathlib import Path

import pytest

from chat.runtime import ChatRuntime
from evals.relational_transitive.oracle import transitively_entails
from evals.relational_transitive.runner import _load_cases, run
from generate.determine import Determined, Undetermined, determine
from generate.meaning_graph.relational import (
    comprehend_relational,
    load_relational_pack_lemmas,
)
from generate.realize import realize_comprehension
from session.context import SessionContext

_V1 = Path(__file__).resolve().parent.parent / "evals" / "relational_transitive" / "v1"
_CASES_SHA = "18f1c8fad6030b4d141f80a4e6547bec6ae384c5d1a098372d4d16cedda50150"
_REFUSALS_SHA = "984e0df23e286c857f72049a399d46d2a8e72660ded9cf59aa1f7a09d9b11673"
_HIGH = 10**9


def _sha(p: Path) -> str:
    return hashlib.sha256(p.read_bytes()).hexdigest()


@pytest.fixture(scope="module")
def vocab_persona():
    rt = ChatRuntime(no_load_state=True)
    return rt._context.vocab, rt._context.persona


@pytest.fixture(scope="module")
def pack():
    return load_relational_pack_lemmas()


def _determine_case(case, vocab_persona, pack):
    vocab, persona = vocab_persona
    ctx = SessionContext(vocab=vocab, persona=persona, vault_reproject_interval=_HIGH)
    for fact in case["facts"]:
        realize_comprehension(comprehend_relational(fact, pack), ctx)
    return determine(comprehend_relational(case["query"], pack), ctx)


def test_fixture_shas_pinned() -> None:
    assert _sha(_V1 / "cases.jsonl") == _CASES_SHA
    assert _sha(_V1 / "refusals.jsonl") == _REFUSALS_SHA


def test_positive_lane_wrong_zero_and_covers() -> None:
    r = run()
    assert r["wrong"] == 0, r["wrongs"]
    assert r["correct"] > 0  # coverage > 0 — else the index geomean zeroes the score
    assert r["correct"] + r["wrong"] + r["refused"] == r["total"]


def test_positives_determine_transitive(vocab_persona, pack) -> None:
    # non-vacuous coverage: every positive determines True via the TRANSITIVE rule.
    for case in _load_cases(_V1 / "cases.jsonl"):
        res = _determine_case(case, vocab_persona, pack)
        assert isinstance(res, Determined), case["id"]
        assert res.answer is True and res.rule == "transitive", case["id"]
        assert [res.predicate, res.subject, res.object] == case["query_edge"], case["id"]
        assert len(res.grounds) >= 2  # a real chain, not a single direct edge


def test_confusers_all_refuse(vocab_persona, pack) -> None:
    # wrong=0 BITE: a Determined on ANY confuser means the transitive rule over-fired
    # (a non-transitive predicate, a non-admitted spatial predicate, a mixed/disjoint
    # chain, a reflexive cycle, or inverse+transitive composition). All must refuse.
    asserted = []
    for case in _load_cases(_V1 / "refusals.jsonl"):
        res = _determine_case(case, vocab_persona, pack)
        if isinstance(res, Determined):
            asserted.append(
                {
                    "id": case["id"],
                    "why": case["why"],
                    "got": [res.answer, res.predicate, res.subject, res.object, res.rule],
                }
            )
        else:
            assert isinstance(res, Undetermined)
    assert not asserted, f"confuser(s) asserted (transitive rule over-fired): {asserted}"


def test_oracle_confirms_positives() -> None:
    # INV-25/27 independence + non-vacuity: the disjoint BFS oracle agrees every positive
    # IS in the same-predicate transitive closure (so the gold is real, not the engine's echo).
    for case in _load_cases(_V1 / "cases.jsonl"):
        p, s, o = case["query_edge"]
        assert transitively_entails(case["edges"], p, s, o) is True, case["id"]


def test_oracle_refutes_confusers() -> None:
    # the disjoint oracle agrees every confuser is NOT entailed — so each engine refusal is
    # the CORRECT verdict, not a coverage gap masking a real entailment.
    for case in _load_cases(_V1 / "refusals.jsonl"):
        p, s, o = case["query_edge"]
        assert transitively_entails(case["edges"], p, s, o) is False, case["id"]