core/tests/test_math_realizer.py

"""ADR-0118 — stepped realizer invariants.

Pins five load-bearing invariants:

1. **Every dev-set case realizes.** All 50 cases produce a
   :class:`RealizedTrace` without raising.

2. **Determinism.** Same trace produces byte-equal RealizedTrace bytes
   across two calls.

3. **Setup sentences cover every entity with initial state.** One
   setup sentence per :class:`InitialPossession`.

4. **Step sentence count equals operation count.** Exactly one step
   sentence per :class:`SolutionStep`.

5. **Answer sentence contains the resolved answer value.** Both
   numeric value and unit appear in the answer sentence.

Round-trip parseability of the rendered prose is **out of scope for
ADR-0118**; the trace is the verifiable artifact (ADR-0117), the
realizer's prose is human-readable documentation.
"""

from __future__ import annotations

import json
from pathlib import Path

import pytest

from generate.math_parser import parse_problem
from generate.math_realizer import RealizedTrace, RealizerError, realize
from generate.math_solver import REQUIRED_PACK_ID, solve


_REPO_ROOT = Path(__file__).resolve().parent.parent
_CASES_PATH = _REPO_ROOT / "evals" / "gsm8k_parser_dev" / "cases.jsonl"


def _load_cases() -> list[dict]:
    return [
        json.loads(line) for line in _CASES_PATH.read_text().splitlines() if line.strip()
    ]


def _build(case: dict) -> tuple:
    graph = parse_problem(case["problem"])
    trace = solve(graph)
    return graph, trace


class TestAllDevSetCasesRealize:
    @pytest.mark.parametrize("case", _load_cases(), ids=lambda c: c["id"])
    def test_realizes_without_error(self, case: dict) -> None:
        graph, trace = _build(case)
        result = realize(graph.initial_state, trace)
        assert isinstance(result, RealizedTrace)
        assert result.pack_id == REQUIRED_PACK_ID


class TestDeterminism:
    @pytest.mark.parametrize("case", _load_cases(), ids=lambda c: c["id"])
    def test_two_realizations_produce_byte_equal_output(self, case: dict) -> None:
        graph, trace = _build(case)
        a = realize(graph.initial_state, trace)
        b = realize(graph.initial_state, trace)
        assert a.canonical_bytes() == b.canonical_bytes()


class TestSetupSentenceCoverage:
    @pytest.mark.parametrize("case", _load_cases(), ids=lambda c: c["id"])
    def test_one_setup_sentence_per_initial_possession(self, case: dict) -> None:
        graph, trace = _build(case)
        result = realize(graph.initial_state, trace)
        assert len(result.setup_sentences) == len(graph.initial_state)


class TestStepSentenceCoverage:
    @pytest.mark.parametrize("case", _load_cases(), ids=lambda c: c["id"])
    def test_one_step_sentence_per_operation(self, case: dict) -> None:
        graph, trace = _build(case)
        result = realize(graph.initial_state, trace)
        assert len(result.step_sentences) == len(trace.steps)


class TestAnswerSentenceContainsResolvedAnswer:
    @pytest.mark.parametrize("case", _load_cases(), ids=lambda c: c["id"])
    def test_answer_value_and_unit_appear_in_answer_sentence(
        self, case: dict
    ) -> None:
        graph, trace = _build(case)
        result = realize(graph.initial_state, trace)
        # numeric value (allow integer form when value is integral)
        if isinstance(trace.answer_value, float) and trace.answer_value.is_integer():
            expected_num = str(int(trace.answer_value))
        else:
            expected_num = str(trace.answer_value)
        assert expected_num in result.answer_sentence, (
            f"answer value {expected_num!r} not in {result.answer_sentence!r}"
        )
        # unit (allow singular surface when value == 1)
        if trace.answer_value == 1:
            # singular check is approximate; the unit-stem must be in there
            stem = trace.answer_unit.rstrip("s")
            assert stem in result.answer_sentence
        else:
            assert trace.answer_unit in result.answer_sentence


class TestRealizerRefuses:
    def test_non_solution_trace_input_raises(self) -> None:
        with pytest.raises(RealizerError):
            realize((), object())  # type: ignore[arg-type]


class TestProseSurface:
    def test_simple_case_prose_is_readable(self) -> None:
        graph, trace = _build(
            {"problem": "Sam has 5 apples. He buys 3 more. How many apples does Sam have?"}
        )
        result = realize(graph.initial_state, trace)
        prose = result.as_prose()
        # Should mention the actor, the operand quantity, and the answer
        assert "Sam" in prose
        assert "5 apples" in prose
        assert "3" in prose
        assert "8" in prose

    def test_transfer_case_mentions_target(self) -> None:
        graph, trace = _build(
            {
                "problem": (
                    "Anna has 8 marbles. She gives 3 to Ben. "
                    "How many marbles does Anna have now?"
                )
            }
        )
        result = realize(graph.initial_state, trace)
        prose = result.as_prose()
        assert "Anna" in prose
        assert "Ben" in prose
        assert "3" in prose
        assert "5" in prose

    def test_total_across_question_uses_collective_phrasing(self) -> None:
        graph, trace = _build(
            {
                "problem": (
                    "Tom has 4 stickers. Sara has 7 stickers. "
                    "How many stickers do they have in total?"
                )
            }
        )
        result = realize(graph.initial_state, trace)
        assert "they have" in result.answer_sentence.lower() or "in total" in result.answer_sentence.lower()
        assert "11" in result.answer_sentence


class TestPackIdPropagates:
    @pytest.mark.parametrize("case", _load_cases()[:5], ids=lambda c: c["id"])
    def test_realized_trace_carries_same_pack_id_as_solution_trace(
        self, case: dict
    ) -> None:
        graph, trace = _build(case)
        result = realize(graph.initial_state, trace)
        assert result.pack_id == trace.pack_id == REQUIRED_PACK_ID