core/tests/test_reader_coexistence.py

"""ADR-0164 Phase 1 — reader/regex coexistence integration tests.

Verifies:
  1. Flag-OFF byte-identity: parse_and_solve without config == parse_and_solve
     with comprehension_reader_questions=False on the 3 currently-correct cases.
  2. Flag-ON determinism: identical input + flag ON → identical reader_trace,
     answer, and graph canonical bytes.
  3. wrong=0 invariant: flag ON never produces a wrong outcome on the
     50-case train_sample.
  4. Trace shape: every reader_trace element is valid JSON with the expected
     layer/phase/outcome keys.
  5. Brief-8 targets: reader is consulted (non-empty reader_trace) for the
     5 GSM8K question sentences referenced in ADR-0164.3 §Worked example.
  6. Fallthrough preserved: flag ON on an unknown-word question produces a
     fallthrough trace event and the same answer as flag OFF.
"""

from __future__ import annotations

import json
from pathlib import Path
from typing import Any

import pytest

from core.config import RuntimeConfig
from generate.math_candidate_graph import parse_and_solve

_CASES_PATH = Path(__file__).resolve().parents[1] / "evals/gsm8k_math/train_sample/v1/cases.jsonl"


def _load_cases() -> list[dict[str, Any]]:
    return [json.loads(l) for l in _CASES_PATH.open(encoding="utf-8") if l.strip()]


def _adapt(case: dict[str, Any]) -> dict[str, Any]:
    return {
        "id": case["case_id"],
        "problem": case["question"],
        "expected_answer": case["answer_numeric"],
        "expected_unit": "",
    }


# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------

_CONFIG_OFF = RuntimeConfig(comprehension_reader_questions=False)
_CONFIG_ON = RuntimeConfig(comprehension_reader_questions=True)

# 3 currently-correct cases (fast-path, reader does not affect them)
_CORRECT_IDS = frozenset({
    "gsm8k-train-sample-v1-0014",
    "gsm8k-train-sample-v1-0018",
    "gsm8k-train-sample-v1-0042",
})

# 5 Brief-8 target question sentences (ADR-0164.3 §Worked example follow-on)
_BRIEF8_IDS = frozenset({
    "gsm8k-train-sample-v1-0007",  # How many more boxes...
    "gsm8k-train-sample-v1-0017",  # How much will it cost him?
    "gsm8k-train-sample-v1-0027",  # How many followers does Malcolm have...
    "gsm8k-train-sample-v1-0036",  # How much time did she spend studying...
    "gsm8k-train-sample-v1-0043",  # How much money will she be left with...
})


@pytest.fixture(scope="module")
def all_cases() -> list[dict[str, Any]]:
    return _load_cases()


@pytest.fixture(scope="module")
def correct_cases(all_cases: list[dict[str, Any]]) -> list[dict[str, Any]]:
    return [c for c in all_cases if c["case_id"] in _CORRECT_IDS]


@pytest.fixture(scope="module")
def brief8_cases(all_cases: list[dict[str, Any]]) -> list[dict[str, Any]]:
    return [c for c in all_cases if c["case_id"] in _BRIEF8_IDS]


# ---------------------------------------------------------------------------
# 1. Flag-OFF byte-identity
# ---------------------------------------------------------------------------


class TestFlagOffByteIdentity:
    """parse_and_solve(config=None) and parse_and_solve(config=CONFIG_OFF)
    must produce the same answer on the 3 currently-correct cases."""

    def test_correct_cases_no_config_vs_off(self, correct_cases: list[dict[str, Any]]) -> None:
        assert correct_cases, "fixture must contain at least one correct case"
        for case in correct_cases:
            text = case["question"]
            r_none = parse_and_solve(text, config=None)
            r_off = parse_and_solve(text, config=_CONFIG_OFF)
            assert r_none.answer == r_off.answer, (
                f"{case['case_id']}: answer differs between config=None and flag-OFF"
            )
            assert r_none.is_admitted == r_off.is_admitted, (
                f"{case['case_id']}: is_admitted differs"
            )
            # Flag OFF must produce empty reader_trace (reader never consulted).
            assert r_off.reader_trace == (), (
                f"{case['case_id']}: reader_trace must be empty with flag OFF"
            )

    def test_correct_cases_off_vs_on_same_answer(self, correct_cases: list[dict[str, Any]]) -> None:
        """Fast-path cases are unaffected by the reader — answer must be identical."""
        for case in correct_cases:
            text = case["question"]
            r_off = parse_and_solve(text, config=_CONFIG_OFF)
            r_on = parse_and_solve(text, config=_CONFIG_ON)
            assert r_off.answer == r_on.answer, (
                f"{case['case_id']}: reader flag changed a fast-path answer"
            )
            assert r_off.is_admitted == r_on.is_admitted, (
                f"{case['case_id']}: reader flag changed admission status"
            )


# ---------------------------------------------------------------------------
# 2. Determinism (flag ON)
# ---------------------------------------------------------------------------


class TestDeterminism:
    def test_flag_on_reader_trace_deterministic(self, all_cases: list[dict[str, Any]]) -> None:
        """Two calls with the same input and flag ON must produce identical reader_trace."""
        sample = all_cases[:10]
        for case in sample:
            text = case["question"]
            r1 = parse_and_solve(text, config=_CONFIG_ON)
            r2 = parse_and_solve(text, config=_CONFIG_ON)
            assert r1.reader_trace == r2.reader_trace, (
                f"{case['case_id']}: reader_trace not deterministic"
            )
            assert r1.answer == r2.answer, (
                f"{case['case_id']}: answer not deterministic"
            )

    def test_flag_off_reader_trace_empty(self, all_cases: list[dict[str, Any]]) -> None:
        """Flag OFF must never populate reader_trace."""
        for case in all_cases:
            r = parse_and_solve(case["question"], config=_CONFIG_OFF)
            assert r.reader_trace == (), (
                f"{case['case_id']}: reader_trace must be empty with flag OFF"
            )


# ---------------------------------------------------------------------------
# 3. wrong=0 invariant
# ---------------------------------------------------------------------------


class TestWrongIsZero:
    def test_flag_on_wrong_is_zero(self, all_cases: list[dict[str, Any]]) -> None:
        """Flag ON must never produce wrong > 0 on the 50-case train sample.

        Wrong outcome requires: admitted=True AND answer != expected. Since
        the reader is additive (refusal falls through to regex), and the
        underlying regex path is already wrong=0, this invariant must hold.
        """
        wrong_cases: list[str] = []
        for raw in all_cases:
            result = parse_and_solve(raw["question"], config=_CONFIG_ON)
            if result.is_admitted and result.answer != raw["answer_numeric"]:
                wrong_cases.append(
                    f"{raw['case_id']}: got {result.answer}, expected {raw['answer_numeric']}"
                )
        assert not wrong_cases, f"wrong > 0 with flag ON:\n" + "\n".join(wrong_cases)


# ---------------------------------------------------------------------------
# 4. Trace event shape
# ---------------------------------------------------------------------------

_REQUIRED_TRACE_KEYS = frozenset({"layer", "phase", "outcome"})
_VALID_OUTCOMES = frozenset({"admit", "fallthrough_to_regex"})


class TestTraceShape:
    def test_trace_events_are_valid_json(self, all_cases: list[dict[str, Any]]) -> None:
        for case in all_cases:
            r = parse_and_solve(case["question"], config=_CONFIG_ON)
            for event_str in r.reader_trace:
                try:
                    event = json.loads(event_str)
                except json.JSONDecodeError:
                    pytest.fail(f"{case['case_id']}: reader_trace event is not valid JSON: {event_str!r}")
                missing = _REQUIRED_TRACE_KEYS - set(event.keys())
                assert not missing, (
                    f"{case['case_id']}: trace event missing keys {missing}: {event}"
                )
                assert event["layer"] == "comprehension_reader", (
                    f"{case['case_id']}: unexpected layer: {event}"
                )
                assert event["phase"] == 1, (
                    f"{case['case_id']}: unexpected phase: {event}"
                )
                assert event["outcome"] in _VALID_OUTCOMES, (
                    f"{case['case_id']}: unexpected outcome: {event}"
                )

    def test_admit_event_has_entity_and_unit(self, all_cases: list[dict[str, Any]]) -> None:
        """Every 'admit' trace event must carry entity and unit keys."""
        for case in all_cases:
            r = parse_and_solve(case["question"], config=_CONFIG_ON)
            for event_str in r.reader_trace:
                event = json.loads(event_str)
                if event["outcome"] == "admit":
                    assert "entity" in event, (
                        f"{case['case_id']}: admit event missing 'entity': {event}"
                    )
                    assert "unit" in event, (
                        f"{case['case_id']}: admit event missing 'unit': {event}"
                    )

    def test_fallthrough_event_has_refusal_reason(self, all_cases: list[dict[str, Any]]) -> None:
        """Every 'fallthrough_to_regex' trace event must carry refusal_reason."""
        for case in all_cases:
            r = parse_and_solve(case["question"], config=_CONFIG_ON)
            for event_str in r.reader_trace:
                event = json.loads(event_str)
                if event["outcome"] == "fallthrough_to_regex":
                    assert "refusal_reason" in event, (
                        f"{case['case_id']}: fallthrough event missing 'refusal_reason': {event}"
                    )


# ---------------------------------------------------------------------------
# 5. Brief-8 targets: reader is consulted
# ---------------------------------------------------------------------------


class TestBrief8Targets:
    def test_reader_consulted_for_brief8_cases(self, brief8_cases: list[dict[str, Any]]) -> None:
        """When flag ON, the reader is consulted for each of the 5 Brief-8 target
        question sentences — reader_trace is non-empty."""
        assert len(brief8_cases) == 5, (
            f"Expected 5 Brief-8 cases, found {len(brief8_cases)}"
        )
        for case in brief8_cases:
            r = parse_and_solve(case["question"], config=_CONFIG_ON)
            assert r.reader_trace, (
                f"{case['case_id']}: reader_trace is empty — reader was not consulted"
            )

    def test_brief8_cases_wrong_stays_zero(self, brief8_cases: list[dict[str, Any]]) -> None:
        """Brief-8 cases must not produce wrong outcomes with flag ON."""
        for case in brief8_cases:
            r = parse_and_solve(case["question"], config=_CONFIG_ON)
            if r.is_admitted:
                assert r.answer == case["answer_numeric"], (
                    f"{case['case_id']}: wrong answer with flag ON: "
                    f"got {r.answer}, expected {case['answer_numeric']}"
                )

    def test_case_0027_malcolm_admits(self, brief8_cases: list[dict[str, Any]]) -> None:
        """Case 0027 (Malcolm/followers) has no pronoun ambiguity — reader admits it."""
        case = next(c for c in brief8_cases if "0027" in c["case_id"])
        r = parse_and_solve(case["question"], config=_CONFIG_ON)
        assert r.reader_trace, "reader must produce a trace for case 0027"
        events = [json.loads(e) for e in r.reader_trace]
        admit_events = [e for e in events if e["outcome"] == "admit"]
        assert admit_events, (
            f"case 0027 must produce at least one admit event; got: {events}"
        )
        admit = admit_events[0]
        assert admit["entity"] == "malcolm"
        assert admit["unit"] == "followers"


# ---------------------------------------------------------------------------
# 6. Fallthrough preserved for unknown words
# ---------------------------------------------------------------------------


class TestFallthroughPreserved:
    def test_unknown_unit_falls_through_to_regex(self) -> None:
        """A question with an unknown unit noun falls through to regex — result is correct."""
        problem = "Martha has 5 apples. How many apples does Martha have?"
        r_off = parse_and_solve(problem, config=_CONFIG_OFF)
        r_on = parse_and_solve(problem, config=_CONFIG_ON)

        # answer must be identical between flag OFF and flag ON
        assert r_off.answer == r_on.answer
        assert r_off.is_admitted == r_on.is_admitted

        # flag ON must record a fallthrough trace event
        assert r_on.reader_trace, "fallthrough case must produce a trace event"
        event = json.loads(r_on.reader_trace[0])
        assert event["outcome"] == "fallthrough_to_regex"
        assert event["refusal_reason"] == "unknown_word"

    def test_flag_off_no_trace_for_fallthrough_case(self) -> None:
        """Flag OFF must never produce any trace events, even for fallthrough-prone inputs."""
        problem = "Martha has 5 apples. How many apples does Martha have?"
        r = parse_and_solve(problem, config=_CONFIG_OFF)
        assert r.reader_trace == ()