"""Deductive-logic lane runner — scores the entailment engine against the oracle gold.

The honest capability metric for CORE's deterministic deduction. For each committed
case it runs :func:`generate.proof_chain.entail.evaluate_entailment` and compares the
outcome to the gold (computed by the independent truth-table oracle at generation).

Counts:
* ``correct``  — engine outcome == gold (a right deduction, including a right
  ``unknown``).
* ``wrong``    — engine outcome != gold and engine did not refuse (a confabulated
  deduction — this MUST stay 0).
* ``refused``  — engine returned ``refused`` on a committed in-regime case. This
  is a capability failure for this lane, not a safety success.

A breakdown by gold class (entailed / refuted / unknown) is reported so the
"sizeable numbers" are visible: how many non-trivial entailments/refutations the
engine decides correctly, not just how many ``unknown``s it passes through.

Exits non-zero unless every committed in-regime case is correct. Refusal-boundary
cases live in unit tests, not in these dev/holdout formula splits.
"""

from __future__ import annotations

import json
import sys
from collections import Counter
from pathlib import Path

from generate.proof_chain.entail import Entailment, evaluate_entailment

_ROOT = Path(__file__).resolve().parent


def _load(path: Path) -> list[dict]:
    with path.open(encoding="utf-8") as fh:
        return [json.loads(line) for line in fh if line.strip()]


def build_report(cases: list[dict]) -> dict:
    counts = Counter({"correct": 0, "wrong": 0, "refused": 0})
    by_gold: Counter[str] = Counter()
    correct_by_gold: Counter[str] = Counter()
    wrong_examples: list[dict] = []
    refused_examples: list[dict] = []

    for case in cases:
        gold = case["gold"]
        by_gold[gold] += 1
        verdict = evaluate_entailment(tuple(case["premises"]), case["query"])
        got = verdict.outcome.value
        if got == gold:
            counts["correct"] += 1
            correct_by_gold[gold] += 1
        elif verdict.outcome is Entailment.REFUSED:
            counts["refused"] += 1
            if len(refused_examples) < 10:
                refused_examples.append(
                    {"id": case["id"], "gold": gold, "reason": verdict.reason,
                     "premises": case["premises"], "query": case["query"]}
                )
        else:
            counts["wrong"] += 1
            if len(wrong_examples) < 10:
                wrong_examples.append(
                    {"id": case["id"], "gold": gold, "got": got,
                     "premises": case["premises"], "query": case["query"]}
                )

    all_cases_correct = counts["correct"] == len(cases)
    return {
        "n": len(cases),
        "counts": dict(counts),
        "by_gold": dict(by_gold),
        "correct_by_gold": dict(correct_by_gold),
        "all_cases_correct": all_cases_correct,
        "wrong_examples": wrong_examples,
        "refused_examples": refused_examples,
    }


def _run(name: str, path: Path) -> dict:
    report = build_report(_load(path))
    c = report["counts"]
    bg = report["by_gold"]
    cbg = report["correct_by_gold"]
    print(f"[{name}] n={report['n']} "
          f"correct={c['correct']} wrong={c['wrong']} refused={c['refused']}")
    print(f"    gold mix: entailed={bg.get('entailed', 0)} "
          f"refuted={bg.get('refuted', 0)} unknown={bg.get('unknown', 0)}")
    print(f"    correct by class: entailed={cbg.get('entailed', 0)} "
          f"refuted={cbg.get('refuted', 0)} unknown={cbg.get('unknown', 0)}")
    if report["wrong_examples"]:
        print("    WRONG examples:")
        for w in report["wrong_examples"]:
            print(f"      {w['id']}: gold={w['gold']} got={w['got']} "
                  f"premises={w['premises']} query={w['query']}")
    if report["refused_examples"]:
        print("    REFUSED examples:")
        for r in report["refused_examples"]:
            print(f"      {r['id']}: gold={r['gold']} reason={r['reason']} "
                  f"premises={r['premises']} query={r['query']}")
    return report


if __name__ == "__main__":
    reports = {
        "dev": _run("dev", _ROOT / "dev" / "cases.jsonl"),
        "holdout-v1": _run("holdout-v1", _ROOT / "holdout" / "v1" / "cases.jsonl"),
        "external-v1": _run("external-v1", _ROOT / "external" / "v1" / "cases.jsonl"),
    }
    total_wrong = sum(r["counts"]["wrong"] for r in reports.values())
    total_refused = sum(r["counts"]["refused"] for r in reports.values())
    all_correct = all(r["all_cases_correct"] for r in reports.values())
    sys.exit(0 if total_wrong == 0 and total_refused == 0 and all_correct else 1)