From 92886886405a090ac5243e947da100d799a32012 Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 22 May 2026 17:37:54 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20ADR-0119.3=20=E2=80=94=20gsm8k=5Fmath?= =?UTF-8?q?=20lane=20runner=20(Phase=205.3)=20(#145)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Composes the Phases 1-4 pipeline (parser → solver → verifier → realizer) into a per-case scoring decision: correct / wrong / refused. Outcome categorization (ADR-0114a Obligation #4): parser ParseError → refused solver SolveError → refused verifier verdict failed → wrong realizer error → wrong answer/unit mismatch → wrong all match → correct `wrong == 0` is the load-bearing gate. The lane's overall_pass holds only if wrong == 0 AND correct + refused == total. Initial measurement on the Phase 5.2 corpus: dev (50) : 50 correct, 0 wrong, 0 refused, overall_pass=True public (150) : 150 correct, 0 wrong, 0 refused, overall_pass=True Every correct case carries a trace_hash (64-char SHA-256) and realized prose — full audit trail per case, consumable by ADR-0119.4 (frontier comparison), ADR-0119.6 (depth curve), and ADR-0120 (eventual expert-tier gate). Tests: 13/13 green; 443 total green across runner + realizer + solver + verifier; 67/67 smoke green. Co-authored-by: Claude Opus 4.7 --- docs/decisions/ADR-0119.3-lane-runner.md | 167 +++++++++++++++ docs/decisions/README.md | 3 +- evals/gsm8k_math/runner.py | 249 +++++++++++++++++++++++ tests/test_gsm8k_math_runner.py | 180 ++++++++++++++++ 4 files changed, 598 insertions(+), 1 deletion(-) create mode 100644 docs/decisions/ADR-0119.3-lane-runner.md create mode 100644 evals/gsm8k_math/runner.py create mode 100644 tests/test_gsm8k_math_runner.py diff --git a/docs/decisions/ADR-0119.3-lane-runner.md b/docs/decisions/ADR-0119.3-lane-runner.md new file mode 100644 index 00000000..7dbe18de --- /dev/null +++ b/docs/decisions/ADR-0119.3-lane-runner.md @@ -0,0 +1,167 @@ +# ADR-0119.3 — gsm8k_math Lane Runner (Phase 5.3) + +**Status:** Accepted +**Date:** 2026-05-22 +**Author:** CORE agents + reviewers +**Depends on:** ADR-0114, ADR-0114a, ADR-0115, ADR-0116, ADR-0117, ADR-0118, ADR-0119, ADR-0119.1, ADR-0119.2 + +--- + +## Context + +Phase 5.3 of [ADR-0119](ADR-0119-gsm8k-eval-lane-roadmap.md). Composes +the Phases 1–4 pipeline (parser → solver → verifier → realizer) into a +per-case scoring decision that the GSM8K eval lane consumes. + +Per ADR-0114a Obligation #4, every case must land in exactly one of +three outcomes: `correct`, `wrong`, or `refused`. The lane gates on +**`wrong == 0`** — CORE must refuse rather than confabulate. A +nonzero `wrong` count invalidates the lane regardless of `correct` +rate. + +--- + +## Decision + +### `evals/gsm8k_math/runner.py` + +Exposes `run_lane(cases, *, config=None) → LaneReport` per the +framework runner interface (`evals/framework.py`). Pure function; +same input → byte-equal `LaneReport.canonical_bytes()`. + +Per-case pipeline: + +```text +parse_problem(text) → graph # ParseError → refused +solve(graph) → trace # SolveError → refused +verify(graph, trace) # passed=False → wrong +realize(initial_state, trace) # RealizerError → wrong +compare answer/unit to expected # mismatch → wrong; match → correct +``` + +`CaseOutcome` dataclass records: `case_id`, `outcome`, `reason`, +`expected_answer/unit`, `actual_answer/unit`, `trace_hash`, +`realized_prose`. The trace hash + prose are present on every +non-refused outcome, so a downstream depth-curve harness (ADR-0119.6) +or frontier-comparison report (ADR-0119.4) has a full per-case +audit trail to consume. + +### Aggregate metrics + +| Key | Type | Meaning | +|---|---|---| +| `cases_total` | int | input size | +| `correct` | int | answer matches AND verifier passes AND realizer succeeds | +| `wrong` | int | verifier failed OR realizer failed OR answer mismatched | +| `refused` | int | parser or solver raised typed refusal | +| `correct_rate` / `wrong_rate` / `refused_rate` | float | counts ÷ total | +| `wrong_count_is_zero` | bool | the load-bearing gate | +| `overall_pass` | bool | `wrong == 0 AND correct + refused == total` | + +`overall_pass` is the single line that consumers (ADR-0120 eventual +expert-tier gate; ADR-0119.8 lane-gate composer) check. + +### Initial measurement (Phase 5.2 corpus, dev + public) + +Run against the on-main `evals/gsm8k_math/` corpus (200 CORE-original +cases authored under ADR-0119.2): + +| Split | Total | Correct | Wrong | Refused | Overall pass | +|---|---|---|---|---|---| +| dev | 50 | 50 | 0 | 0 | True | +| public | 150 | 150 | 0 | 0 | True | + +**200/200 correct; zero wrong; zero refused.** Establishes the lane +baseline. ADR-0119.4 (frontier comparison) and ADR-0119.6 (depth +curve) will pair this measurement with the relevant context. ADR-0119.7 +(sealed GSM8K test holdout) will eventually exercise the same runner +against the real GSM8K test set. + +--- + +## ADR-0114a obligation discharge update + +ADR-0119.3 hardens **#4 (typed refusal + wrong==0)** at the lane +layer. Previously the discipline lived in the solver (ADR-0116); now +the lane runner enforces and reports it. + +| # | Obligation | Status | +|---|---|---| +| 1 | Sealed-holdout discipline | ADR-0119.1 (fab_control); 5.7 for GSM8K test | +| 2 | OOD surface variation | ADR-0118a | +| 3 | Replay-equal trace | ADR-0117 | +| 4 | Typed refusal + wrong==0 | **hardened at lane layer (this ADR)** | +| 5 | Reasoning-isolation perturbation suite | ADR-0125 | +| 6 | Compositional-depth curve | ADR-0119.6 (in flight) | +| 7 | Frontier-baseline comparison | ADR-0119.4 (in flight) | +| 8 | Adversarial generation | ADR-0119.5 (pending) | +| 9 | Determinism | discharged at solver + verifier + realizer + lane layers | +| 10 | Operation provenance via pack | ADR-0116 | + +--- + +## Invariants + +### `adr_0119_3_determinism` + +Two `run_lane(cases)` calls produce byte-equal `LaneReport.canonical_bytes()`. + +### `adr_0119_3_outcome_exhaustive` + +Every case lands in exactly one of `correct` / `wrong` / `refused`. + +### `adr_0119_3_zero_wrong_on_phase_5_2_corpus` + +On the gsm8k_math/dev + public splits (200 cases authored under +ADR-0119.2), `wrong == 0`. Today: dev 50/50, public 150/150. + +### `adr_0119_3_refusal_is_typed` + +A graph that triggers ParseError or SolveError produces +`outcome == "refused"` with a reason that names the failing stage +(`parser: ...` or `solver: ...`). + +### `adr_0119_3_correct_outcomes_carry_audit_trail` + +Every `correct` case has a `trace_hash` (64-char hex) and a +`realized_prose` field set. The audit trail enables ADR-0119.4 +(frontier-comparison report) and ADR-0119.6 (depth curve) to +inspect per-case reasoning without re-running the pipeline. + +--- + +## Acceptance evidence + +- `evals/gsm8k_math/runner.py` exports `run_lane`, `LaneReport`, + `CaseOutcome` +- `tests/test_gsm8k_math_runner.py` (13 cases) green +- 200/200 correct, 0 wrong, 0 refused on the Phase 5.2 corpus +- Smoke suite green +- ADR linked from `docs/decisions/README.md` index + frontier + +--- + +## Consequences + +- Phase 5.3 of [ADR-0119](ADR-0119-gsm8k-eval-lane-roadmap.md) lands. + Phases 5.4, 5.5, 5.6 can now run against this runner's outputs. +- The "every correct answer ships with replay-equal trace + readable + prose" promise (ADR-0114a #3 + #10) is now mechanically inspectable + per-case via `report.case_details[i]["trace_hash"]` and + `["realized_prose"]`. +- The lane runner is the substrate the eventual ADR-0120 expert-tier + gate consumes. ADR-0120 will require `overall_pass == True` on both + the public split AND the sealed holdout. + +--- + +## Out of scope + +- Adversarial case authoring + scoring (ADR-0119.5) +- Depth-curve bucketing harness (ADR-0119.6) +- Frontier-baseline comparison report (ADR-0119.4) +- Lane registry shape + ADR-0120 numeric thresholds (ADR-0119.8 + + ADR-0120) +- Parallel execution / case batching (current runner is single-threaded; + small dev/public corpora don't need it. Future ADR may add a + parallel mode that produces the same byte-equal report.) diff --git a/docs/decisions/README.md b/docs/decisions/README.md index a221b63f..6e3c3856 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -49,6 +49,7 @@ ADRs record significant architectural decisions: what was decided, why, what alt | [ADR-0119](ADR-0119-gsm8k-eval-lane-roadmap.md) | GSM8K Eval Lane Roadmap (Phase 5; decomposes into 5.1..5.8) | Proposed (2026-05-22) | | [ADR-0119.1](ADR-0119.1-sealed-holdout-fabrication-control.md) | Seal `fabrication_control` Holdout (ADR-0105 Amendment) | Accepted (2026-05-23) | | [ADR-0119.2](ADR-0119.2-gsm8k-eval-corpus-dev-public.md) | GSM8K Eval Corpus Dev/Public Splits | Accepted (2026-05-22) | +| [ADR-0119.3](ADR-0119.3-lane-runner.md) | gsm8k_math Lane Runner (Phase 5.3) | Accepted (2026-05-22) | | [ADR-0119.6](ADR-0119.6-depth-curve-harness.md) | GSM8K Math Depth-Curve Measurement Harness | Accepted (2026-05-23) | --- @@ -90,11 +91,11 @@ The ADR-0091..0114 slate is fully accepted (0091..0113) plus one proposed-roadma - `symbolic_logic` Lane-Shape Remap (ADR-0109 amendment) — ADR-0123 - `systems_software` Audit-Passed Promotion (third successful) — ADR-0124 - `all_three_pass_rate` Synonym in `inference_shape` (ADR-0109 Amendment) — ADR-0123a -<<<<<<< HEAD - Reasoning-Isolation Perturbation Suite (224 deterministic applicable semantic perturbations; discharges ADR-0114a obligation #5 for the GSM8K-style parser dev lane) — ADR-0125 - GSM8K Eval Lane Roadmap (Phase 5; decomposes into 5.1..5.8; proposed) — ADR-0119 - Seal `fabrication_control` Holdout (ADR-0105 Amendment) — ADR-0119.1 - GSM8K Eval Corpus Dev/Public Splits (200 CORE-original cases; verify.py 200/200; sealed holdout placeholder reserved for ADR-0119.7) — ADR-0119.2 +- gsm8k_math Lane Runner (Phase 5.3; correct/wrong/refused triple; wrong==0 gate; current: 200/200 correct on dev+public) — ADR-0119.3 - GSM8K Math Depth-Curve Measurement Harness (discharges ADR-0114a Obligation #6 measurement-side) — ADR-0119.6 ADR-0080 has also landed: Contemplation Loop Phase 1 adds a read-only frontier-compare miner that emits `SPECULATIVE` findings only. diff --git a/evals/gsm8k_math/runner.py b/evals/gsm8k_math/runner.py new file mode 100644 index 00000000..f485c48e --- /dev/null +++ b/evals/gsm8k_math/runner.py @@ -0,0 +1,249 @@ +"""ADR-0119.3 — GSM8K math eval lane runner. + +Composes the Phases 1-4 pipeline (parser → solver → verifier → realizer) +into a per-case scoring decision: ``correct`` / ``wrong`` / ``refused``. + +Outcome categorization (ADR-0114a Obligation #4 — the load-bearing +"refusal is first-class; misparse rate zero" discipline): + +| Stage that raised | Outcome | Reason recorded | +|---|---|---| +| ``parse_problem(text)`` raised ``ParseError`` | refused | typed parser error | +| ``solve(graph)`` raised ``SolveError`` | refused | typed solver error | +| ``verify(graph, trace)`` returned ``passed=False`` | wrong | verifier reason | +| Everything succeeds AND ``trace.answer_value == expected_answer`` AND ``trace.answer_unit == expected_unit`` | correct | empty | +| Everything succeeds BUT answer or unit differs | wrong | "answer/unit mismatch" | + +**`wrong == 0` is the gate** — ADR-0114a Obligation #4 requires CORE +to refuse rather than confabulate. A nonzero ``wrong`` count +invalidates the lane regardless of ``correct`` rate. + +The runner is pure / deterministic: same case set → same +:class:`LaneReport.canonical_bytes()`. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from typing import Any + +from generate.math_parser import ParseError, parse_problem +from generate.math_problem_graph import MathProblemGraph +from generate.math_realizer import RealizerError, realize +from generate.math_solver import SolveError, solve +from generate.math_verifier import verify + + +@dataclass(frozen=True, slots=True) +class CaseOutcome: + """Per-case scoring decision with full audit trail.""" + + case_id: str + outcome: str # "correct" | "wrong" | "refused" + reason: str + expected_answer: float + expected_unit: str + actual_answer: float | None + actual_unit: str | None + trace_hash: str | None + realized_prose: str | None + + def as_json(self) -> dict[str, Any]: + return { + "case_id": self.case_id, + "outcome": self.outcome, + "reason": self.reason, + "expected_answer": self.expected_answer, + "expected_unit": self.expected_unit, + "actual_answer": self.actual_answer, + "actual_unit": self.actual_unit, + "trace_hash": self.trace_hash, + "realized_prose": self.realized_prose, + } + + +@dataclass(slots=True) +class LaneReport: + """Aggregate lane scoring report. + + Conforms to the framework runner interface (``metrics`` dict + + ``case_details`` list). + """ + + metrics: dict[str, Any] = field(default_factory=dict) + case_details: list[dict[str, Any]] = field(default_factory=list) + + def canonical_bytes(self) -> bytes: + """Deterministic JSON for hashing/byte-equality comparison.""" + payload = {"metrics": self.metrics, "case_details": self.case_details} + return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def _score_one(case: dict[str, Any]) -> CaseOutcome: + """Run the full pipeline against one case and classify the outcome.""" + case_id = case["id"] + expected_answer = case["expected_answer"] + expected_unit = case["expected_unit"] + + # Stage 1 — parse + try: + graph: MathProblemGraph = parse_problem(case["problem"]) + except ParseError as exc: + return CaseOutcome( + case_id=case_id, + outcome="refused", + reason=f"parser: {exc}", + expected_answer=expected_answer, + expected_unit=expected_unit, + actual_answer=None, + actual_unit=None, + trace_hash=None, + realized_prose=None, + ) + + # Stage 2 — solve + try: + trace = solve(graph) + except SolveError as exc: + return CaseOutcome( + case_id=case_id, + outcome="refused", + reason=f"solver: {exc}", + expected_answer=expected_answer, + expected_unit=expected_unit, + actual_answer=None, + actual_unit=None, + trace_hash=None, + realized_prose=None, + ) + + # Stage 3 — verify (independent re-derivation) + verdict = verify(graph, trace) + trace_hash = hashlib.sha256(trace.canonical_bytes()).hexdigest() + + if not verdict.passed: + return CaseOutcome( + case_id=case_id, + outcome="wrong", + reason=f"verifier: {verdict.reason}", + expected_answer=expected_answer, + expected_unit=expected_unit, + actual_answer=trace.answer_value, + actual_unit=trace.answer_unit, + trace_hash=trace_hash, + realized_prose=None, + ) + + # Stage 4 — realize (failures here are treated as wrong, not refused, + # because the trace already verified) + try: + realized = realize(graph.initial_state, trace) + prose = realized.as_prose() + except RealizerError as exc: + return CaseOutcome( + case_id=case_id, + outcome="wrong", + reason=f"realizer: {exc}", + expected_answer=expected_answer, + expected_unit=expected_unit, + actual_answer=trace.answer_value, + actual_unit=trace.answer_unit, + trace_hash=trace_hash, + realized_prose=None, + ) + + # Stage 5 — compare against expected + if trace.answer_unit != expected_unit: + return CaseOutcome( + case_id=case_id, + outcome="wrong", + reason=( + f"unit mismatch: got {trace.answer_unit!r}, " + f"expected {expected_unit!r}" + ), + expected_answer=expected_answer, + expected_unit=expected_unit, + actual_answer=trace.answer_value, + actual_unit=trace.answer_unit, + trace_hash=trace_hash, + realized_prose=prose, + ) + if trace.answer_value != expected_answer: + return CaseOutcome( + case_id=case_id, + outcome="wrong", + reason=( + f"answer mismatch: got {trace.answer_value!r}, " + f"expected {expected_answer!r}" + ), + expected_answer=expected_answer, + expected_unit=expected_unit, + actual_answer=trace.answer_value, + actual_unit=trace.answer_unit, + trace_hash=trace_hash, + realized_prose=prose, + ) + + return CaseOutcome( + case_id=case_id, + outcome="correct", + reason="", + expected_answer=expected_answer, + expected_unit=expected_unit, + actual_answer=trace.answer_value, + actual_unit=trace.answer_unit, + trace_hash=trace_hash, + realized_prose=prose, + ) + + +def run_lane( + cases: list[dict[str, Any]], + *, + config: Any = None, # noqa: ARG001 — framework interface compat +) -> LaneReport: + """Score every case and emit aggregate metrics + per-case details. + + The runner is pure: no globals, no I/O. Returns a + :class:`LaneReport` whose ``canonical_bytes()`` is byte-equal across + two calls with the same input list. + + Aggregate metrics: + cases_total int + correct int + wrong int (gate: must == 0) + refused int + correct_rate float = correct / total + wrong_rate float = wrong / total + refused_rate float = refused / total + wrong_count_is_zero bool = wrong == 0 + overall_pass bool = wrong == 0 AND correct + refused == total + """ + outcomes = [_score_one(c) for c in cases] + + total = len(outcomes) + correct = sum(1 for o in outcomes if o.outcome == "correct") + wrong = sum(1 for o in outcomes if o.outcome == "wrong") + refused = sum(1 for o in outcomes if o.outcome == "refused") + + wrong_count_is_zero = wrong == 0 + overall_pass = wrong_count_is_zero and (correct + refused == total) + + metrics = { + "cases_total": total, + "correct": correct, + "wrong": wrong, + "refused": refused, + "correct_rate": (correct / total) if total else 0.0, + "wrong_rate": (wrong / total) if total else 0.0, + "refused_rate": (refused / total) if total else 0.0, + "wrong_count_is_zero": wrong_count_is_zero, + "overall_pass": overall_pass, + } + + report = LaneReport() + report.metrics = metrics + report.case_details = [o.as_json() for o in outcomes] + return report diff --git a/tests/test_gsm8k_math_runner.py b/tests/test_gsm8k_math_runner.py new file mode 100644 index 00000000..64887593 --- /dev/null +++ b/tests/test_gsm8k_math_runner.py @@ -0,0 +1,180 @@ +"""ADR-0119.3 — gsm8k_math lane runner invariants. + +Pins five load-bearing invariants: + +1. **Determinism (ADR-0114a Obligation #9).** Same case list → + byte-equal :class:`LaneReport.canonical_bytes()`. + +2. **Outcome categorization is exhaustive.** Every case lands in + exactly one of ``correct`` / ``wrong`` / ``refused``. + +3. **Zero-wrong gate (ADR-0114a Obligation #4).** On the existing + parser dev set (gpd-001..050, where all 50 cases verify with + the on-main pipeline), the runner produces ``wrong == 0``. + +4. **Refusal is first-class.** A graph that triggers SolveError + (e.g. division by zero) produces ``outcome == "refused"`` with a + reason that names the solver. + +5. **Correct outcomes carry full audit trail.** Every ``correct`` + case has trace_hash + realized_prose set; every ``refused`` case + has trace_hash / realized_prose either None or set per spec. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from evals.gsm8k_math.runner import CaseOutcome, LaneReport, run_lane + + +_REPO_ROOT = Path(__file__).resolve().parent.parent + + +def _load_jsonl(path: Path) -> list[dict]: + return [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + + +# Use gpd-001..050 as a known-good case set (already on main; 50/50 verify) +_GPD_CASES = _load_jsonl(_REPO_ROOT / "evals" / "gsm8k_parser_dev" / "cases.jsonl") + + +class TestDeterminism: + def test_two_runs_produce_byte_equal_report(self) -> None: + r1 = run_lane(_GPD_CASES) + r2 = run_lane(_GPD_CASES) + assert r1.canonical_bytes() == r2.canonical_bytes() + + +class TestOutcomeCategorizationIsExhaustive: + def test_every_case_lands_in_one_outcome(self) -> None: + report = run_lane(_GPD_CASES) + total = report.metrics["cases_total"] + correct = report.metrics["correct"] + wrong = report.metrics["wrong"] + refused = report.metrics["refused"] + assert correct + wrong + refused == total + assert total == len(_GPD_CASES) + + def test_each_case_detail_has_recognized_outcome(self) -> None: + report = run_lane(_GPD_CASES) + for detail in report.case_details: + assert detail["outcome"] in {"correct", "wrong", "refused"} + + +class TestZeroWrongOnKnownGoodCases: + """The gpd-001..050 dev set is fully exercised on main today. + + ADR-0114a Obligation #4: ``wrong == 0`` is the gate. The runner + must reflect that on known-good cases. + """ + + def test_wrong_count_is_zero_on_gpd_dev_set(self) -> None: + report = run_lane(_GPD_CASES) + assert report.metrics["wrong"] == 0, ( + f"wrong={report.metrics['wrong']} on gpd dev set; " + f"details: {[d for d in report.case_details if d['outcome'] == 'wrong']}" + ) + + def test_wrong_count_is_zero_gate_flag(self) -> None: + report = run_lane(_GPD_CASES) + assert report.metrics["wrong_count_is_zero"] is True + + def test_overall_pass_holds_when_no_wrong(self) -> None: + report = run_lane(_GPD_CASES) + assert report.metrics["overall_pass"] is True + + +class TestRefusalIsFirstClass: + def test_unsupported_grammar_produces_refused(self) -> None: + # "If" clause is out of scope per ADR-0115 §Phase 1.1 boundary; + # parser raises ParseError → runner classifies as refused. + case = { + "id": "synthetic-refuse-01", + "problem": "If Sam had 5 apples, how many would he have?", + "expected_answer": 5, + "expected_unit": "apples", + } + report = run_lane([case]) + assert report.metrics["refused"] == 1 + assert report.metrics["wrong"] == 0 + outcome = report.case_details[0] + assert outcome["outcome"] == "refused" + assert "parser" in outcome["reason"] + + def test_empty_input_produces_refused(self) -> None: + case = { + "id": "synthetic-refuse-02", + "problem": "", + "expected_answer": 0, + "expected_unit": "apples", + } + report = run_lane([case]) + assert report.metrics["refused"] == 1 + assert report.metrics["wrong"] == 0 + + +class TestWrongDetectedWhenAnswerMismatches: + """If we deliberately mis-author a case (wrong expected_answer), + the runner should report ``wrong``, not ``correct`` — proving the + answer-comparison gate is actually load-bearing.""" + + def test_mismatched_expected_answer_produces_wrong(self) -> None: + # gpd-001's actual answer is 8 apples. Plant a wrong expected. + case = dict(_GPD_CASES[0]) + case["id"] = "synthetic-wrong-01" + case["expected_answer"] = 9999 # deliberately wrong + report = run_lane([case]) + assert report.metrics["wrong"] == 1 + assert report.metrics["wrong_count_is_zero"] is False + assert report.metrics["overall_pass"] is False + outcome = report.case_details[0] + assert outcome["outcome"] == "wrong" + assert "answer mismatch" in outcome["reason"] + + def test_mismatched_expected_unit_produces_wrong(self) -> None: + case = dict(_GPD_CASES[0]) + case["id"] = "synthetic-wrong-02" + case["expected_unit"] = "bananas" # wrong unit + report = run_lane([case]) + assert report.metrics["wrong"] == 1 + outcome = report.case_details[0] + assert outcome["outcome"] == "wrong" + assert "unit mismatch" in outcome["reason"] + + +class TestCorrectOutcomeCarriesAuditTrail: + def test_correct_case_has_trace_hash_and_prose(self) -> None: + report = run_lane([_GPD_CASES[0]]) + outcome = report.case_details[0] + assert outcome["outcome"] == "correct" + assert outcome["trace_hash"] is not None + assert len(outcome["trace_hash"]) == 64 # sha256 hex + assert outcome["realized_prose"] is not None + assert _GPD_CASES[0]["problem"].split(".")[0].split()[0] in outcome["realized_prose"] + + +class TestLaneReportShape: + def test_report_is_conformant_lane_report(self) -> None: + report = run_lane(_GPD_CASES[:3]) + assert isinstance(report, LaneReport) + assert hasattr(report, "metrics") + assert hasattr(report, "case_details") + + def test_metrics_keys_match_documented_schema(self) -> None: + report = run_lane(_GPD_CASES[:3]) + expected_keys = { + "cases_total", + "correct", + "wrong", + "refused", + "correct_rate", + "wrong_rate", + "refused_rate", + "wrong_count_is_zero", + "overall_pass", + } + assert set(report.metrics.keys()) == expected_keys