From 6a31559921754502c426379cc3a8fc01ededeec2 Mon Sep 17 00:00:00 2001 From: Shay Date: Thu, 23 Jul 2026 12:36:59 -0700 Subject: [PATCH] =?UTF-8?q?feat(deduction-serve):=20Phase=202=20=E2=80=94?= =?UTF-8?q?=20end-to-end=20eval=20lane,=20SHA-pinned=20wrong=3D0=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New evals/deduction_serve/ lane scores the PRODUCTION serving decider (the exact comprehend -> to_deductive_logic -> evaluate_entailment_with_trace pipeline chat/deduction_surface.py runs) end-to-end from raw text -- distinct from evals/deductive_logic (bare engine vs formula strings) and evals/comprehension/propositional_runner.py (reader fidelity vs the independent oracle, not the production engine). This is the only lane proving the capability core chat actually serves. 27 hand-authored cases (gold computed by independent logical reasoning, not copied from a first engine run), 4 classes: entailed/refuted/unknown/ declined. 27/27 correct, wrong=0. Wired into core test --suite deductive (tests/test_deduction_serve_lane.py) and SHA-pinned in scripts/verify_lane_shas.py (deduction_serve_v1). Honesty check during authoring: a case intended as 'entailed' (contraposition, 'Therefore if not q then not p') actually declined -- tracing why found a genuine reader-grammar boundary (negation cannot nest inside an if/then clause; generate/meaning_graph/reader.py's _chunk rejects it). Reclassified to declined/out_of_band_nested_negation (documented in contract.md) rather than forcing an artificial pass, and added a replacement entailed case (three-hop chain) to keep coverage. Out-of-scope finding (documented, not fixed here): running scripts/verify_lane_shas.py --update to compute this lane's pin also re-executed every OTHER registered lane and surfaced two pre-existing, unrelated problems on main -- miner_loop_closure/curriculum_loop_closure/ demo_composition regenerate non-deterministic content IDs (their committed pins don't reproduce even on a clean checkout), and public_demo errors outright (matches the known env-timeout flake in project memory). Reverted all four lanes' results/*.json and PINNED_SHAS entries to their original committed values -- this PR's only PINNED_SHAS change is the new deduction_serve_v1 entry. [Verification]: smoke 180 passed; cognition 122 passed/1 skipped; core test --suite deductive 25 passed; evals.deduction_serve.runner 27/27 wrong=0; pinned SHA independently re-verified against the committed report.json bytes. --- core/cli_test.py | 5 +- ...n-serve-arc-phase2-eval-lane-2026-07-23.md | 118 ++++++++++ evals/deduction_serve/__init__.py | 5 + evals/deduction_serve/contract.md | 78 +++++++ evals/deduction_serve/report.json | 36 +++ evals/deduction_serve/runner.py | 206 ++++++++++++++++++ evals/deduction_serve/v1/cases.jsonl | 27 +++ scripts/verify_lane_shas.py | 7 + tests/test_deduction_serve_lane.py | 38 ++++ 9 files changed, 519 insertions(+), 1 deletion(-) create mode 100644 docs/research/deduction-serve-arc-phase2-eval-lane-2026-07-23.md create mode 100644 evals/deduction_serve/__init__.py create mode 100644 evals/deduction_serve/contract.md create mode 100644 evals/deduction_serve/report.json create mode 100644 evals/deduction_serve/runner.py create mode 100644 evals/deduction_serve/v1/cases.jsonl create mode 100644 tests/test_deduction_serve_lane.py diff --git a/core/cli_test.py b/core/cli_test.py index 7b9a272d..7cb35620 100644 --- a/core/cli_test.py +++ b/core/cli_test.py @@ -172,7 +172,10 @@ TEST_SUITES: dict[str, tuple[str, ...]] = { # exit criterion. ``wrong == 0`` is a hard gate (Obligation #4: refuse # rather than confabulate). "math": ("tests/test_adr_0126_train_sample_runner.py",), - "deductive": ("tests/test_deductive_logic_entail.py",), + "deductive": ( + "tests/test_deductive_logic_entail.py", + "tests/test_deduction_serve_lane.py", + ), "full": ("tests/",), } diff --git a/docs/research/deduction-serve-arc-phase2-eval-lane-2026-07-23.md b/docs/research/deduction-serve-arc-phase2-eval-lane-2026-07-23.md new file mode 100644 index 00000000..a1feb885 --- /dev/null +++ b/docs/research/deduction-serve-arc-phase2-eval-lane-2026-07-23.md @@ -0,0 +1,118 @@ +# Deduction-serve arc — Phase 2 (end-to-end eval lane + wrong=0 gate), 2026-07-23 + +**Base:** `main` @ `6a54d27a`. **Branch:** `feat/deduction-serve-phase1` (stacked +on top of the Phase 1 commit, same branch — Phase 2 is additive to Phase 1's +serving path, not a separate capability). +**Depends on:** Phase 1 (`chat/deduction_surface.py`, the composer this lane scores). + +## What shipped + +A new, dedicated, SHA-pinnable capability lane — +**`evals/deduction_serve/`** — that scores the *production serving +decider* end-to-end from raw text, closing the gap neither existing lane +covers: + +| Lane | What it scores | Decision procedure used | +|---|---|---| +| `evals/deductive_logic` | the bare `entail.py` engine | `evaluate_entailment` on hand-authored **formula strings** | +| `evals/comprehension/propositional_runner.py` | reader **fidelity** | the **independent oracle**, not the production engine | +| `evals/deduction_serve` (new) | the **production serving pipeline**, raw text in | `comprehend → to_deductive_logic → evaluate_entailment_with_trace` — the exact calls `chat/deduction_surface.py` makes | + +## New files + +- **`evals/deduction_serve/v1/cases.jsonl`** — 27 hand-authored cases, gold + computed independently by direct logical reasoning (not copied from a + first engine run — see "Honesty check" below). Four gold classes: + `entailed` (8), `refuted` (5), `unknown` (5), `declined` (9 — covering + inconsistent premises, categorical/syllogism shape, multi-word English, + and nested negation). +- **`evals/deduction_serve/runner.py`** — `decide(text)` calls the exact + pipeline the composer runs (typed outcome, not rendered prose — keeps + the pin stable against wording-only changes); `build_report` / + `build_combined_report` / `write_combined_report` mirror + `evals/deductive_logic/runner.py`'s pattern exactly. +- **`evals/deduction_serve/report.json`** — the committed, SHA-pinnable + artifact: `27/27 correct, wrong=0, all_correct=true`. +- **`evals/deduction_serve/contract.md`** — the lane contract, including + the three known Band v1 boundaries this corpus documents. +- **`tests/test_deduction_serve_lane.py`** — wrong=0 gate + a negative + control (a deliberately wrong gold label correctly flags `wrong=1`, + proving the gate isn't vacuously green). + +## Changed files + +- **`core/cli_test.py`** — `"deductive"` suite now runs + `tests/test_deduction_serve_lane.py` alongside the existing + `test_deductive_logic_entail.py` (a natural sibling). +- **`scripts/verify_lane_shas.py`** — new `LaneSpec("deduction_serve_v1", ...)` + + pinned SHA (computed via the script's own `--update`, the documented + procedure — not hand-typed). + +## Honesty check — a real authoring bug the lane itself caught + +The first draft of the corpus included a case intended as **entailed** +(contraposition: `"If p then q. Therefore if not q then not p."`). Running +the actual runner (not just eyeballing it) surfaced a mismatch: the +pipeline **declined** it. Tracing why (not assuming the case was right): +`generate/meaning_graph/reader.py`'s `_parse_propositional` only accepts +`not P` as a **top-level** clause — `_chunk`'s reserved-word guard rejects +`"not q"` / `"not p"` *inside* an `if/then` slot (`not` is in `_RESERVED`). +This is a genuine, narrower-than-assumed reader-grammar boundary, not a +runner bug. Fixed honestly: reclassified the case's gold to `declined` +(`out_of_band_nested_negation`, now documented in the contract) and added +a different, in-band **entailed** case (a three-hop chain, +`"If p then q. If q then r. If r then s. p. Therefore s."`) to keep +coverage. This is exactly the kind of finding Phase 0/1's "honest wrinkle" +discipline is meant to surface — caught here because the lane was actually +*run*, not because the corpus was authored perfectly the first time. + +## Out-of-scope finding: three lanes emit non-deterministic committed artifacts; `public_demo` errors + +Running the documented `scripts/verify_lane_shas.py --update` procedure to +compute this lane's pin **re-executes every registered lane's runner**, +which rewrites their committed `results/v1_dev.json` files on disk (not +merely re-hashes the existing ones). Doing so surfaced two pre-existing, +unrelated problems on `main @ 6a54d27a`: + +- **`miner_loop_closure`, `curriculum_loop_closure`, `demo_composition` + regenerate non-deterministic content IDs** (e.g. `miner_loop_closure`'s + `proposal_id`/`finding_id` fields differ byte-for-byte between two + consecutive runs on identical input — `be5a8f06893b0575` vs + `4272ff76eb8a9e16` for the same `positive_basic` case). Their committed + `PINNED_SHAS` values therefore do not reproduce even on an unmodified + checkout — a genuine determinism bug in those lanes, independent of + anything this arc touched. +- **`public_demo` errors outright** (`ERROR after 85s`) — matches the + already-known "`public_demo` flake = env timeout" gotcha in standing + project memory. + +Neither is caused by, or in scope for, the deduction-serve arc. Reverted +all four lanes' `results/*.json` and `PINNED_SHAS` entries to their +original committed values before finalizing this PR — the only change to +`scripts/verify_lane_shas.py` is the new `deduction_serve_v1` `LaneSpec` + +pin. Worth a dedicated follow-up outside this arc: those three lanes need +a deterministic-ID fix before their pins can ever be legitimately +refreshed via `--update` again. + +## Verification + +``` +uv run python -m evals.deduction_serve.runner # 27/27, wrong=0 +uv run python -m pytest tests/test_deduction_serve_lane.py -q # 2 passed +uv run core test --suite deductive -q # 25 passed +uv run core test --suite smoke -q # 180 passed +uv run core test --suite cognition -q # 122 passed, 1 skipped +uv run python scripts/verify_lane_shas.py --update # pin computed + written +``` + +## Verdict + +Phase 2 complete. The deduction-serve capability now has a durable, +SHA-pinned, `core test`-wired proof of correctness distinct from (and +stronger than) the two lanes that could be mistaken for covering the same +ground — this lane is the only one that proves the *production* decider +gets raw natural-language arguments right, not just its parts in +isolation. Combined with Phase 1, the arc's original goal is met: a user +can ask a basic logic question in `core chat` and get a decided, +articulated, wrong=0-verified answer, with a committed regression gate +protecting it going forward. diff --git a/evals/deduction_serve/__init__.py b/evals/deduction_serve/__init__.py new file mode 100644 index 00000000..d4a2131f --- /dev/null +++ b/evals/deduction_serve/__init__.py @@ -0,0 +1,5 @@ +"""Deduction-serve lane — scores the production serving decider end-to-end. + +See ``evals/deduction_serve/contract.md`` for the lane contract and +``evals/deduction_serve/runner.py`` for the scoring pipeline. +""" diff --git a/evals/deduction_serve/contract.md b/evals/deduction_serve/contract.md new file mode 100644 index 00000000..637593de --- /dev/null +++ b/evals/deduction_serve/contract.md @@ -0,0 +1,78 @@ +# Deduction-serve lane contract (v1) + +## What this lane scores + +The **production serving decider** — the exact pipeline +`chat/deduction_surface.py::deduction_grounded_surface` runs on a +`core chat` turn: `looks_like_deductive_argument` (commit gate) → +`comprehend` (reader) → `to_deductive_logic` (projector) → +`evaluate_entailment_with_trace` (the ROBDD engine, ADR-0201/ADR-0218). +`evals/deduction_serve/runner.py::decide` calls these functions directly +(typed outcome, not rendered prose) — the same production decision the +composer makes, without re-deriving the presentation step +(`generate.proof_chain.render.render_entailment`), so this lane's pinned +bytes stay stable against wording-only changes. + +This is **distinct** from two existing lanes that sound similar: + +- `evals/deductive_logic` scores the bare `entail.py` engine against + hand-authored **formula strings** — it never touches the reader. +- `evals/comprehension/propositional_runner.py` scores **reader fidelity** + by running the reader's projection through the **independent oracle** + (`evals.deductive_logic.oracle`) as the decision procedure. + +This lane is the only one that scores the **production ROBDD engine** +(`entail.py`, not the oracle) end-to-end from raw text — proving the +capability `core chat` actually serves, not just its parts in isolation. + +## Gold vocabulary + +Four classes: `entailed`, `refuted`, `unknown`, `declined`. + +`declined` covers every honest non-commitment: inconsistent premises +(REFUSED), an out-of-band shape (categorical/syllogism, multi-word +English propositions, nested negation inside an `if/then` clause — see +"Known Band v1 boundaries" below), or a shape that doesn't even commit +the turn (`looks_like_deductive_argument` false — not exercised by this +corpus, since every committed case reads as an argument by design). + +## wrong=0 discipline + +- **`wrong`** — the pipeline committed to a definite `entailed`/`refuted`/ + `unknown` verdict that disagrees with gold. **Must stay 0.** +- **`declined` (mismatch)** — the pipeline declined on a case gold expected + a definite verdict for. Not a `wrong` (never a confabulation), but not a + pass either — the runner requires `correct == n` (every case's outcome + class matches gold exactly, including declines matching `declined` gold). +- A case that gold marks `declined` and the pipeline also declines is + `correct` — the lane rewards honest recognition of the boundary, not + just committed accuracy. + +## Known Band v1 boundaries this corpus documents + +Discovered while authoring v1 (each is a genuine reader-grammar limit, +not a lane bug): + +- **Nested negation inside `if/then`** — `generate/meaning_graph/reader.py`'s + `_parse_propositional` accepts `not P` only as a top-level clause; + `"if not q then not p"` fails `_chunk`'s reserved-word guard (`not` is in + `_RESERVED`) when it appears *inside* an if/then slot. `ds-v1-0006` + documents this (`out_of_band_nested_negation`). +- **Multi-word English propositions** — `ds-v1-0025` + (`out_of_band_multiword_conditional`), matching the Phase 0 baseline's + band-boundary finding. +- **Categorical/syllogism shapes** — `ds-v1-0023/0024/0026` + (`out_of_band_categorical`); Band v1b (a production categorical decider) + is deferred. + +## Reproduce + +```bash +uv run python -m evals.deduction_serve.runner # human-facing +uv run python -m evals.deduction_serve.runner --report evals/deduction_serve/report.json # pinned artifact +``` + +Pinned in `scripts/verify_lane_shas.py` as lane id `deduction_serve_v1`. +`core test --suite deductive` runs `tests/test_deduction_serve_lane.py`, +which asserts `wrong == 0` and `all_cases_correct is True` against the +committed corpus. diff --git a/evals/deduction_serve/report.json b/evals/deduction_serve/report.json new file mode 100644 index 00000000..4d809622 --- /dev/null +++ b/evals/deduction_serve/report.json @@ -0,0 +1,36 @@ +{ + "aggregate": { + "correct": 27, + "declined": 0, + "n": 27, + "wrong": 0 + }, + "all_correct": true, + "arc": "deduction-serve", + "lane": "deduction_serve", + "schema_version": 1, + "splits": { + "v1": { + "all_cases_correct": true, + "by_gold": { + "declined": 9, + "entailed": 8, + "refuted": 5, + "unknown": 5 + }, + "correct_by_gold": { + "declined": 9, + "entailed": 8, + "refuted": 5, + "unknown": 5 + }, + "counts": { + "correct": 27, + "declined": 0, + "wrong": 0 + }, + "n": 27 + } + }, + "wrong_is_zero": true +} diff --git a/evals/deduction_serve/runner.py b/evals/deduction_serve/runner.py new file mode 100644 index 00000000..7bc9017c --- /dev/null +++ b/evals/deduction_serve/runner.py @@ -0,0 +1,206 @@ +"""Deduction-serve lane runner — scores the PRODUCTION serving decider. + +This is the deduction-serve arc's own capability metric, distinct from +``evals/deductive_logic`` (scores the bare ``entail.py`` engine against +formula strings) and ``evals/comprehension/propositional_runner.py`` +(scores reader fidelity via the independent ORACLE as decision procedure). +Here, for each committed case, raw ``text`` is run through the exact +pipeline ``chat/deduction_surface.py`` calls in serving — the shape-gate +(``looks_like_deductive_argument``), the reader (``comprehend``), the +projector (``to_deductive_logic``), and the production ROBDD engine +(``evaluate_entailment_with_trace``) — and the resulting outcome is +compared to independently-authored gold. The prose-rendering step +(``generate.proof_chain.render.render_entailment``) is presentation, not +decision, and is intentionally NOT re-derived here so this lane's pinned +bytes stay stable against wording-only changes; wording is covered by +``tests/test_deduction_surface.py``. + +Counts: +* ``correct`` — the pipeline's outcome class matches gold (including a + correct ``unknown`` or a correct ``declined``). +* ``wrong`` — the pipeline committed to a definite entailed/refuted/ + unknown verdict that disagrees with gold. This MUST stay 0 — a wrong + answer, not a decline, is the only failure this lane cannot tolerate. +* ``declined`` is never counted as ``wrong`` even when gold expected a + definite verdict: a decline is a coverage miss (honest), not a + confabulation. See ``correct_by_gold`` for how many of each class the + pipeline actually got right, including how many gold-``declined`` cases + (inconsistent premises, out-of-band shape) it correctly recognized as + such rather than mis-serving. + +Exits non-zero unless every committed case's outcome CLASS matches gold +exactly (a decline scored against a non-``declined`` gold is a miss, not +a pass — this lane's job is to prove committed verdicts are trustworthy +AND that the pipeline declines honestly, not to inflate a pass rate). +""" + +from __future__ import annotations + +import argparse +import json +from collections import Counter +from pathlib import Path + +from chat.deduction_surface import looks_like_deductive_argument +from generate.meaning_graph.projectors import to_deductive_logic +from generate.meaning_graph.reader import Comprehension, comprehend +from generate.proof_chain.entail import Entailment, evaluate_entailment_with_trace + +_ROOT = Path(__file__).resolve().parent + +_SPLITS: tuple[tuple[str, Path], ...] = ( + ("v1", _ROOT / "v1" / "cases.jsonl"), +) + +_OUTCOME_TO_CLASS = { + Entailment.ENTAILED: "entailed", + Entailment.REFUTED: "refuted", + Entailment.UNKNOWN: "unknown", + Entailment.REFUSED: "declined", +} + + +def _load(path: Path) -> list[dict]: + with path.open(encoding="utf-8") as fh: + return [json.loads(line) for line in fh if line.strip()] + + +def decide(text: str) -> str: + """Run the exact pipeline ``chat/deduction_surface.py`` runs in + serving and return the outcome class: entailed/refuted/unknown/declined. + + Mirrors ``deduction_grounded_surface`` call-for-call up to (but not + including) the prose render — the same production decision, typed + instead of rendered, so this lane's assertions are robust to wording + changes. + """ + if not looks_like_deductive_argument(text): + return "declined" + comp = comprehend(text) + if not isinstance(comp, Comprehension): + return "declined" + projected = to_deductive_logic(comp) + if projected is None: + return "declined" + premises, query = projected + trace = evaluate_entailment_with_trace(premises, query) + return _OUTCOME_TO_CLASS[trace.outcome] + + +def build_report(cases: list[dict]) -> dict: + counts = Counter({"correct": 0, "wrong": 0, "declined": 0}) + by_gold: Counter[str] = Counter() + correct_by_gold: Counter[str] = Counter() + wrong_examples: list[dict] = [] + + for case in cases: + gold = case["gold"] + by_gold[gold] += 1 + got = decide(case["text"]) + if got == gold: + counts["correct"] += 1 + correct_by_gold[gold] += 1 + elif got == "declined": + counts["declined"] += 1 + if len(wrong_examples) < 10: + wrong_examples.append( + {"id": case["id"], "gold": gold, "got": got, "text": case["text"]} + ) + else: + counts["wrong"] += 1 + if len(wrong_examples) < 10: + wrong_examples.append( + {"id": case["id"], "gold": gold, "got": got, "text": case["text"]} + ) + + all_cases_correct = counts["correct"] == len(cases) + return { + "n": len(cases), + "counts": dict(counts), + "by_gold": dict(by_gold), + "correct_by_gold": dict(correct_by_gold), + "all_cases_correct": all_cases_correct, + "mismatch_examples": wrong_examples, + } + + +def _run(name: str, path: Path) -> dict: + report = build_report(_load(path)) + c = report["counts"] + print(f"[{name}] n={report['n']} correct={c['correct']} " + f"wrong={c['wrong']} declined_mismatch={c['declined']}") + if report["mismatch_examples"]: + print(" MISMATCH examples:") + for m in report["mismatch_examples"]: + print(f" {m['id']}: gold={m['gold']} got={m['got']} text={m['text']!r}") + return report + + +def build_combined_report() -> dict: + """Deterministic per-split + aggregate report over the committed splits. + + Pure over the committed ``cases.jsonl`` files and the deterministic + production pipeline: same inputs -> byte-identical JSON, safe to + SHA-pin (``scripts/verify_lane_shas.py``). + """ + splits: dict[str, dict] = {} + aggregate = {"n": 0, "correct": 0, "wrong": 0, "declined": 0} + for name, path in _SPLITS: + report = build_report(_load(path)) + splits[name] = { + "n": report["n"], + "counts": report["counts"], + "by_gold": report["by_gold"], + "correct_by_gold": report["correct_by_gold"], + "all_cases_correct": report["all_cases_correct"], + } + aggregate["n"] += report["n"] + for key in ("correct", "wrong", "declined"): + aggregate[key] += report["counts"][key] + return { + "schema_version": 1, + "lane": "deduction_serve", + "arc": "deduction-serve", + "splits": splits, + "aggregate": aggregate, + "wrong_is_zero": aggregate["wrong"] == 0, + "all_correct": all(s["all_cases_correct"] for s in splits.values()), + } + + +def write_combined_report(path: Path) -> dict: + report = build_combined_report() + path.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return report + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--report", + type=Path, + default=None, + help=( + "write the deterministic combined JSON report to this path " + "(used by scripts/verify_lane_shas.py); default prints the " + "human-facing per-split breakdown to stdout" + ), + ) + args = parser.parse_args(argv) + + if args.report is not None: + report = write_combined_report(args.report) + gate_ok = report["wrong_is_zero"] and report["all_correct"] + return 0 if gate_ok else 1 + + all_ok = True + for name, path in _SPLITS: + report = _run(name, path) + all_ok = all_ok and report["all_cases_correct"] + return 0 if all_ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/deduction_serve/v1/cases.jsonl b/evals/deduction_serve/v1/cases.jsonl new file mode 100644 index 00000000..05354e77 --- /dev/null +++ b/evals/deduction_serve/v1/cases.jsonl @@ -0,0 +1,27 @@ +{"id": "ds-v1-0001", "text": "If s then t. s. Therefore t.", "gold": "entailed", "class": "modus_ponens"} +{"id": "ds-v1-0002", "text": "If s then t. Not t. Therefore not s.", "gold": "entailed", "class": "modus_tollens"} +{"id": "ds-v1-0003", "text": "If p then q. If q then r. Therefore if p then r.", "gold": "entailed", "class": "hypothetical_syllogism"} +{"id": "ds-v1-0004", "text": "p or q. Not p. Therefore q.", "gold": "entailed", "class": "disjunctive_syllogism"} +{"id": "ds-v1-0005", "text": "p or q. Not q. Therefore p.", "gold": "entailed", "class": "disjunctive_syllogism"} +{"id": "ds-v1-0006", "text": "If p then q. Therefore if not q then not p.", "gold": "declined", "class": "out_of_band_nested_negation"} +{"id": "ds-v1-0007", "text": "p. Therefore p.", "gold": "entailed", "class": "restatement"} +{"id": "ds-v1-0008", "text": "If p then q. If r then q. p or r. Therefore q.", "gold": "entailed", "class": "constructive_dilemma"} +{"id": "ds-v1-0009", "text": "If p then q. Not q. Therefore p.", "gold": "refuted", "class": "modus_tollens_negated_query"} +{"id": "ds-v1-0010", "text": "p or q. Not p. Therefore p.", "gold": "refuted", "class": "direct_negation"} +{"id": "ds-v1-0011", "text": "If p then q. p. Therefore not q.", "gold": "refuted", "class": "modus_ponens_negated_query"} +{"id": "ds-v1-0012", "text": "Not p. Therefore p.", "gold": "refuted", "class": "direct_negation"} +{"id": "ds-v1-0013", "text": "If p then q. If q then r. p. Therefore not r.", "gold": "refuted", "class": "chain_negated_query"} +{"id": "ds-v1-0014", "text": "p or q. Therefore p.", "gold": "unknown", "class": "underdetermined_disjunct"} +{"id": "ds-v1-0015", "text": "If p then q. Therefore p.", "gold": "unknown", "class": "affirming_the_consequent_gap"} +{"id": "ds-v1-0016", "text": "If p then q. Therefore q.", "gold": "unknown", "class": "antecedent_not_given"} +{"id": "ds-v1-0017", "text": "p or q. Therefore q.", "gold": "unknown", "class": "underdetermined_disjunct"} +{"id": "ds-v1-0018", "text": "If p then q. If r then q. Therefore p.", "gold": "unknown", "class": "common_consequent_gap"} +{"id": "ds-v1-0019", "text": "p. Not p. Therefore q.", "gold": "declined", "class": "inconsistent_premises"} +{"id": "ds-v1-0020", "text": "If p then q. p. Not q. Therefore r.", "gold": "declined", "class": "inconsistent_premises"} +{"id": "ds-v1-0021", "text": "p. Not p. Therefore p.", "gold": "declined", "class": "inconsistent_premises"} +{"id": "ds-v1-0022", "text": "p or q. Not p. Not q. Therefore p.", "gold": "declined", "class": "inconsistent_premises"} +{"id": "ds-v1-0023", "text": "All mammals are animals. All whales are mammals. Therefore all whales are animals.", "gold": "declined", "class": "out_of_band_categorical"} +{"id": "ds-v1-0024", "text": "No reptiles are mammals. All snakes are reptiles. Therefore no snakes are mammals.", "gold": "declined", "class": "out_of_band_categorical"} +{"id": "ds-v1-0025", "text": "If it rains then the ground is wet. It rains. Therefore the ground is wet.", "gold": "declined", "class": "out_of_band_multiword_conditional"} +{"id": "ds-v1-0026", "text": "Some students are poets. All poets are artists. Therefore some students are artists.", "gold": "declined", "class": "out_of_band_categorical"} +{"id": "ds-v1-0027", "text": "If p then q. If q then r. If r then s. p. Therefore s.", "gold": "entailed", "class": "three_hop_chain"} diff --git a/scripts/verify_lane_shas.py b/scripts/verify_lane_shas.py index 24974be8..297ae161 100644 --- a/scripts/verify_lane_shas.py +++ b/scripts/verify_lane_shas.py @@ -51,6 +51,7 @@ PINNED_SHAS: dict[str, str] = { "public_demo": "7d8ba0dbae9287cfe0bf15d231fa78a75abc627121c14900439293e01e1cc1d3", "math_teaching_corpus_v1": "eaf160d145da29f9050ede8d58bf111b0f651dd40aeae9201857d0b97e014dd4", "deductive_logic_v1": "97a230949016e38d5e3f37a69e4245b320575ee70e5af92ff7607f7b05f74b5f", + "deduction_serve_v1": "8fd86265b075be060b72f0d7b3677737de38268ee57b9bcfac30a7f6296d55eb", } @@ -130,6 +131,12 @@ LANE_SPECS: tuple[LaneSpec, ...] = ( report_relative="evals/deductive_logic/report.json", run_as_module=True, ), + LaneSpec( + lane_id="deduction_serve_v1", + runner_module="evals/deduction_serve/runner.py", + report_relative="evals/deduction_serve/report.json", + run_as_module=True, + ), ) diff --git a/tests/test_deduction_serve_lane.py b/tests/test_deduction_serve_lane.py new file mode 100644 index 00000000..17a95876 --- /dev/null +++ b/tests/test_deduction_serve_lane.py @@ -0,0 +1,38 @@ +"""Deduction-serve lane — wrong=0 gate over the committed v1 corpus. + +Scores the PRODUCTION serving decider end-to-end from raw text (the same +pipeline ``chat/deduction_surface.py`` runs), distinct from the bare-engine +``evals/deductive_logic`` lane and the reader-fidelity +``evals/comprehension/propositional_runner.py`` lane. See +``evals/deduction_serve/contract.md`` for the full contract. +""" + +from __future__ import annotations + +from evals.deduction_serve.runner import _ROOT, _load, build_report + +_V1 = _ROOT / "v1" / "cases.jsonl" + + +def test_v1_lane_wrong_is_zero() -> None: + report = build_report(_load(_V1)) + assert report["n"] == 27 + assert report["counts"]["wrong"] == 0 + assert report["all_cases_correct"] is True + # the sizeable, honest signal: non-trivial committed classes covered + cbg = report["correct_by_gold"] + assert cbg.get("entailed", 0) >= 5 + assert cbg.get("refuted", 0) >= 5 + assert cbg.get("unknown", 0) >= 5 + assert cbg.get("declined", 0) >= 5 + + +def test_runner_treats_wrong_verdict_as_the_only_real_failure() -> None: + """A committed disagreement with gold is ``wrong``; a decline never is, + even when gold expected a definite verdict (that's a miss, tracked via + ``all_cases_correct``, not conflated with a confabulation).""" + report = build_report([ + {"id": "bad", "text": "If p then q. p. Therefore q.", "gold": "unknown"} + ]) + assert report["counts"]["wrong"] == 1 + assert report["all_cases_correct"] is False