core/tests/test_grammar_roundtrip.py

"""Tests for the grammar round-trip lane.

The load-bearing tests here are the *mutation* tests.  This lane exists because
``evals/deterministic_fluency`` reports 1.00 on all six predicates while
accepting ``"banana does the."`` — a pin that cannot fail is worthless.  So
every guarantee this lane makes is paired with a test proving the guarantee can
be broken.
"""

from __future__ import annotations

import pytest

from evals.grammar_roundtrip import runner as rt_runner
from evals.grammar_roundtrip.corpora import (
    negative_surface_cases,
    positive_graph_cases,
    positive_surface_cases,
)
from evals.grammar_roundtrip.projection import (
    CanonicalProposition,
    compare,
    from_meaning_graph,
    from_proposition_graph,
    quantifier_predicate,
)
from evals.grammar_roundtrip.runner import run_lane
from generate.graph_planner import (
    ArticulationStep,
    ArticulationTarget,
    GraphNode,
    PropositionGraph,
    RhetoricalMove,
)
from generate.intent import IntentTag
from generate.meaning_graph.model import (
    Entity,
    MeaningGraph,
    MeaningSpan,
    Relation,
)
from generate.meaning_graph.reader import Comprehension, Refusal, comprehend


# The three strings that pass every content predicate of
# evals/deterministic_fluency.  This lane's rejection of them is the concrete,
# recorded improvement over that lane.
_FLUENCY_LANE_FALSE_POSITIVES = (
    "banana does the.",
    "wet ground rains the is.",
    "is is is is.",
)


@pytest.fixture(scope="module")
def report():
    return run_lane()


# --------------------------------------------------------------------------- #
# The negative corpus: the guarantee, and proof it can fail
# --------------------------------------------------------------------------- #


def test_negative_corpus_is_fully_rejected(report):
    """Word salad must never be comprehended into propositions."""
    assert report.metrics["reject_rate"] == 1.0
    assert report.metrics["negative_cases"] >= 16


def test_the_fluency_lanes_false_positives_are_rejected():
    """The exact strings deterministic_fluency passes must be refused here."""
    for surface in _FLUENCY_LANE_FALSE_POSITIVES:
        result = comprehend(surface, source_id="t")
        if isinstance(result, Refusal):
            continue
        assert not from_meaning_graph(result.meaning_graph), (
            f"{surface!r} was comprehended into propositions; the negative "
            "corpus guarantee is broken"
        )


def test_reject_rate_goes_red_when_the_reader_accepts_everything(monkeypatch):
    """MUTATION: an accept-everything reader must drive reject_rate to 0.

    Without this test ``reject_rate == 1.0`` would be unfalsifiable — exactly
    the defect that makes the existing fluency lane decoration.
    """
    span = MeaningSpan(source_id="t", start=0, end=1, text="x")
    graph = MeaningGraph(
        entities=(
            Entity(entity_id="a", name="a", span=span),
            Entity(entity_id="b", name="b", span=span),
        ),
        relations=(Relation(predicate="subset", arguments=("a", "b"), span=span),),
    )
    monkeypatch.setattr(
        rt_runner, "comprehend", lambda text, source_id="input": Comprehension(meaning_graph=graph)
    )
    mutated = run_lane()
    assert mutated.metrics["reject_rate"] == 0.0, (
        "an accept-everything reader still scored a perfect reject_rate — the "
        "negative corpus is not actually gating"
    )


def test_shuffled_negatives_reuse_positive_vocabulary():
    """Shuffles must be lexically identical to a positive, order destroyed.

    A lane that rejects only hand-authored salad could be checking vocabulary
    rather than grammar.  The shuffles remove that escape.
    """
    positives = {
        frozenset(c.surface.rstrip(".").lower().split()) for c in positive_surface_cases()
    }
    shuffles = [c for c in negative_surface_cases() if c.case_id.startswith("neg-shuf-")]
    assert shuffles, "no shuffled negatives were generated"
    for case in shuffles:
        tokens = frozenset(case.surface.rstrip(".").lower().split())
        assert tokens in positives, (
            f"{case.case_id} does not reuse a positive case's exact vocabulary"
        )


def test_shuffles_are_byte_stable_across_calls():
    """The negative corpus must be reproducible — no PRNG, no clock."""
    first = tuple(c.surface for c in negative_surface_cases())
    second = tuple(c.surface for c in negative_surface_cases())
    assert first == second


# --------------------------------------------------------------------------- #
# Baselines — these pin DEFECTS and must be revised upward when fixed
# --------------------------------------------------------------------------- #


def test_g_roundtrip_baseline_is_one_case_of_293(report):
    """RATCHET PIN. Was 0/293 on main @ 9696443a — CORE read *nothing* it wrote.

    Phase 4 made it **1**. This must only ever be revised upward; never
    downward to accommodate a regression.

    The one case is ``gram_C14_p01``, and what unblocked it is worth recording
    because it is the opposite of what §6 of the plan predicted. The writer was
    emitting ``all molecules are defined as compound`` — a predicate nominal
    that does not agree with its subject. The reader accepts
    ``...as compounds`` and refuses ``...as compound``, so the blocker was a
    one-line **writer** defect, not the MeaningGraph/PropositionGraph type
    mismatch of §1.8.

    The remaining 292 decompose cleanly, and none of them is a model mismatch
    either — see ``test_the_remaining_blockers_are_reader_construction_coverage``.
    """
    # 293, not the original 280: Phase 3 added 13 quantified-copular cases
    # (construction C14) to grammatical_coverage/public/v1, and this lane
    # harvests its graph corpus from the committed case files rather than
    # holding its own copy. An exact count is the point — a corpus that grows
    # or shrinks should require a deliberate edit here, not pass silently.
    assert report.metrics["graph_cases"] == 293
    assert report.metrics["g_write_rate"] == 1.0
    assert report.metrics["g_read_rate"] >= 0.003413, "the ratchet may not go down"
    assert report.metrics["g_read_rate"] == 0.003413


def test_the_remaining_blockers_are_reader_construction_coverage(report):
    """WHY g_read_rate is 1/293 and not 293/293 — the measurement §6 turns on.

    The plan pre-committed to reading a near-zero rate as evidence for §1.8:
    two incompatible graph models, next step an ADR. The refusal reasons say
    otherwise. Every one of them is the reader declining a CONSTRUCTION it has
    no template for — not a projection disagreeing about a graph it parsed:

        no_template_match     289   reader has no SUBJ-VERB-OBJ template at all
        unknown_morphology      2   prepositional objects (reserved_word_in_np)
        unsupported_negation    1   reader has no negated-categorical template

    Where a construction IS in both inventories, the round trip closes exactly
    (``s_surface_match_rate == s_renderable_rate``). So the barrier is the
    *overlap* of the two construction inventories, which is currently one
    construction wide — and that is Phase 5's item 1, not an ADR.
    """
    reasons: dict[str, int] = {}
    for row in report.case_details:
        if "wrote" not in row:
            continue
        reasons[row.get("refusal_reason") or "READ"] = (
            reasons.get(row.get("refusal_reason") or "READ", 0) + 1
        )
    assert reasons == {
        "no_template_match": 289,
        "unknown_morphology": 2,
        "unsupported_negation": 1,
        "READ": 1,
    }
    # The load-bearing claim: not one refusal is a graph-model disagreement.
    assert "projection_mismatch" not in reasons


def test_s_roundtrip_closes_for_every_renderable_surface(report):
    """Phase 2A pinned this at **0.0** — nothing CORE read rendered back to its
    input, because the serving renderer interpolated singularized entity ids
    into plural templates ("all dog are animal").

    Phase 2B re-inflects at render time, and every surface the renderer can
    express now returns its input exactly. The two rates are equal on purpose:
    ``s_surface_match_rate == s_renderable_rate`` says the *only* remaining
    round-trip losses are surfaces the categorical renderer cannot express at
    all, which is a coverage gap in a later phase, not a grammar defect.

    Asserting equality rather than a literal keeps this honest if the corpus
    grows: adding an unrenderable positive lowers both numbers together, while
    reintroducing the plural defect lowers only the match rate.
    """
    assert report.metrics["s_read_rate"] == 1.0
    assert report.metrics["s_surface_match_rate"] == report.metrics["s_renderable_rate"]
    assert report.metrics["s_surface_match_rate"] > 0.0


def test_the_categorical_render_defect_is_fixed_concretely():
    """The 2A defect stated as an example rather than a rate, now inverted.

    Both sides matter: the graph still carries the SINGULAR canonical id
    (``dog``), so the fix is in the renderer's inflection and not in the
    comprehension it was built to preserve.
    """
    result = comprehend("All dogs are animals.", source_id="t")
    assert isinstance(result, Comprehension)
    props = sorted(from_meaning_graph(result.meaning_graph))
    assert props == [CanonicalProposition("subset", "dog", "animal", False)]
    rendered = rt_runner._render_categorical("subset", "dog", "animal")
    assert rendered == "all dogs are animals"


def test_irregular_plurals_survive_the_full_round_trip():
    """The reader and renderer now share one number table, so an irregular
    plural returns as itself. Before 2B this produced ``all wolve are
    mammals`` — the reader's bare ``-s`` strip leaking a corrupted id straight
    into served text."""
    cases = (
        ("All wolves are mammals.", "wolf", "all wolves are mammals"),
        ("All children are mammals.", "child", "all children are mammals"),
        ("All men are mammals.", "man", "all men are mammals"),
        ("All knives are tools.", "knife", "all knives are tools"),
    )
    for surface, singular, expected_clause in cases:
        result = comprehend(surface, source_id="t")
        assert isinstance(result, Comprehension), surface
        props = sorted(from_meaning_graph(result.meaning_graph))
        assert props[0].subject == singular, f"{surface} -> {props}"
        rendered = rt_runner._render_categorical("subset", singular, props[0].obj)
        assert rendered == expected_clause


# --------------------------------------------------------------------------- #
# The projection itself
# --------------------------------------------------------------------------- #


def test_quantifier_map_matches_reader():
    """The lane's local quantifier map must track the reader's.

    The copy is deliberate — the lane must not import the thing it measures —
    so this test is what keeps the yardstick honest if the reader changes.
    """
    from generate.meaning_graph.reader import _QUANTIFIER_PREDICATE as reader_map

    for quantifier, predicate in reader_map.items():
        assert quantifier_predicate(quantifier) == predicate
    assert quantifier_predicate(None) is None
    assert quantifier_predicate("nonsense") is None


def test_proposition_graph_projection_folds_quantifier():
    """A writer graph's ``quantifier`` slot folds into the reading predicate."""
    node = GraphNode(
        node_id="n1", subject="dog", predicate="is_a", obj="animal",
        source_intent=IntentTag.UNKNOWN,
    )
    step = ArticulationStep(
        node_id="n1", move=RhetoricalMove.ASSERT, predicate="is_a",
        subject="dog", quantifier="all",
    )
    projected = from_proposition_graph(
        PropositionGraph(nodes=(node,)),
        ArticulationTarget(steps=(step,), source_intent=IntentTag.UNKNOWN),
    )
    assert projected == frozenset({CanonicalProposition("subset", "dog", "animal", False)})


def test_projection_drops_non_binary_relations():
    """Arity != 2 is dropped — the writing side cannot express it."""
    span = MeaningSpan(source_id="t", start=0, end=1, text="x")
    graph = MeaningGraph(
        entities=(
            Entity(entity_id="a", name="a", span=span),
            Entity(entity_id="b", name="b", span=span),
            Entity(entity_id="c", name="c", span=span),
        ),
        relations=(
            Relation(predicate="between", arguments=("a", "b", "c"), span=span),
            Relation(predicate="subset", arguments=("a", "b"), span=span),
        ),
    )
    assert from_meaning_graph(graph) == frozenset(
        {CanonicalProposition("subset", "a", "b", False)}
    )


def test_compare_decomposes_argument_and_predicate_agreement():
    """A predicate name only counts when it agrees on the SAME arguments."""
    expected = frozenset({CanonicalProposition("subset", "dog", "animal")})
    # right arguments, wrong predicate name
    args_only = frozenset({CanonicalProposition("member", "dog", "animal")})
    agreement = compare(expected, args_only)
    assert agreement.args_match == 1
    assert agreement.predicates_match == 0
    assert agreement.exact_match == 0

    # right predicate name, wrong arguments — must NOT count
    pred_elsewhere = frozenset({CanonicalProposition("subset", "cat", "plant")})
    agreement = compare(expected, pred_elsewhere)
    assert agreement.args_match == 0
    assert agreement.predicates_match == 0


def test_compare_exact_agreement():
    prop = CanonicalProposition("subset", "dog", "animal")
    agreement = compare(frozenset({prop}), frozenset({prop}))
    assert agreement.exact_rate == 1.0
    assert agreement.args_rate == 1.0
    assert agreement.predicates_rate == 1.0


# --------------------------------------------------------------------------- #
# Corpus integrity
# --------------------------------------------------------------------------- #


def test_positive_surfaces_are_all_inside_the_reader_envelope():
    """Every positive surface must actually be comprehended.

    A refused positive would silently measure the reader's coverage instead of
    the round-trip, and would make s_surface_match_rate uninterpretable.
    """
    for case in positive_surface_cases():
        result = comprehend(case.surface, source_id="t")
        assert isinstance(result, Comprehension), (
            f"{case.case_id} ({case.surface!r}) is refused; it does not belong "
            "in the positive corpus"
        )


def test_graph_corpus_is_harvested_from_committed_case_files():
    cases = positive_graph_cases()
    assert len(cases) == 293  # +13 C14 quantified-copular (Phase 3)
    assert all(c.nodes for c in cases)