ab6da7765e
Composes v2-EN's connective grammar (if/then, or, and, either) with v3-MEM's singular-membership sentence reading over one shared per-individual atom space, so "If Socrates is a man then Socrates is mortal. Socrates is a man. Therefore Socrates is mortal." decides — the exact gap ADR-0258 §6.1 reserved. Genuinely fuses the two mechanisms: a bare universal's instantiated atom can unify (via the same closed morphology relation) with a connective leaf's atom, deciding arguments neither band alone could. - generate/proof_chain/cond_member.py: new reader. A sentence is checked for a connective token BEFORE the universal-lead check (so a stray connective can never leak into an opaque name/class run); reuses v3-MEM's own _parse_singular/_parse_universal verbatim (already negation-aware, so no separate negation layer is needed, unlike v2-EN's opaque minter). Four new shape-bands (fused/disjunctive/chain/conditional), each earning its own SERVE license. - chat/deduction_surface.py: new fallback tier tried strictly after v3-MEM — pure widening, every previously-served argument stays byte-identical. - evals/deduction_serve/practice/gold.py: 4 new synthetic template groups (20 templates), reusing the v3-MEM case generator; intended formulas cross-checked against the independent oracle (INV-25). - evals/deduction_serve/v2_condmem/: 26 hand-authored real-English cases, content-disjoint from the synthetic corpus. - ds-mem-0024 promoted declined -> unknown (not entailed — the antecedent is never asserted in that text; this is the first promotion in the corpus that doesn't land on entailed). - Ledger re-sealed to 17 bands; deduction_serve_v1 lane SHA re-pinned (surgical single-line edit, not a blanket --update). Verification: core test --suite deductive 160 passed (31 new reader tests); practice arena 17 bands x 720 wrong=0, all SERVE-licensed; lane splits v1 28/28, v2_en 26/26, v2_member 26/26, v2_condmem 26/26; smoke 180 passed; warmed_session 10 passed. Flag deduction_serving_enabled stays default-off.
100 lines
4.3 KiB
Python
100 lines
4.3 KiB
Python
"""Deduction-serve lane — wrong=0 gate over the committed v1 corpus.
|
|
|
|
Scores the PRODUCTION serving decider end-to-end from raw text (the same
|
|
pipeline ``chat/deduction_surface.py`` runs), distinct from the bare-engine
|
|
``evals/deductive_logic`` lane and the reader-fidelity
|
|
``evals/comprehension/propositional_runner.py`` lane. See
|
|
``evals/deduction_serve/contract.md`` for the full contract.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from evals.deduction_serve.runner import _ROOT, _load, build_report
|
|
|
|
_V1 = _ROOT / "v1" / "cases.jsonl"
|
|
_V2_EN = _ROOT / "v2_en" / "cases.jsonl"
|
|
_V2_MEMBER = _ROOT / "v2_member" / "cases.jsonl"
|
|
_V2_CONDMEM = _ROOT / "v2_condmem" / "cases.jsonl"
|
|
|
|
|
|
def test_v1_lane_wrong_is_zero() -> None:
|
|
report = build_report(_load(_V1))
|
|
assert report["n"] == 28
|
|
assert report["counts"]["wrong"] == 0
|
|
assert report["all_cases_correct"] is True
|
|
# the sizeable, honest signal: non-trivial committed classes covered,
|
|
# including the categorical band (valid + invalid) added in Phase 4.
|
|
# (declined dropped from 6 to 4 when the two documented Band v1 boundary
|
|
# cases — multiword conditional, nested-negation contraposition — became
|
|
# DECIDED by Band v2-EN, ADR-0257; they are now entailed gold.)
|
|
cbg = report["correct_by_gold"]
|
|
assert cbg.get("entailed", 0) >= 7
|
|
assert cbg.get("refuted", 0) >= 5
|
|
assert cbg.get("unknown", 0) >= 5
|
|
assert cbg.get("declined", 0) >= 4
|
|
assert cbg.get("valid", 0) >= 3
|
|
assert cbg.get("invalid", 0) >= 1
|
|
|
|
|
|
def test_v2_en_lane_wrong_is_zero() -> None:
|
|
"""Band v2-EN (ADR-0257): hand-authored REAL-English arguments — content
|
|
deliberately disjoint from the synthetic practice lexicon — decided by the
|
|
production pipeline with wrong=0, across all four inference verdicts and
|
|
five distinct honest-decline shapes (ds-en-0022, the membership decline,
|
|
was promoted declined→entailed when the member band landed — ADR-0258)."""
|
|
report = build_report(_load(_V2_EN))
|
|
assert report["n"] == 26
|
|
assert report["counts"]["wrong"] == 0
|
|
assert report["all_cases_correct"] is True
|
|
cbg = report["correct_by_gold"]
|
|
assert cbg.get("entailed", 0) >= 14
|
|
assert cbg.get("refuted", 0) >= 3
|
|
assert cbg.get("unknown", 0) >= 4
|
|
assert cbg.get("declined", 0) >= 5
|
|
|
|
|
|
def test_v2_member_lane_wrong_is_zero() -> None:
|
|
"""Band v3-MEM (ADR-0258): hand-authored membership/universal arguments —
|
|
real nouns exercising every number-link row-type (irregular, invariant,
|
|
each regular suffix rule) — decided with wrong=0 across all four verdicts
|
|
and six distinct honest-decline shapes. (``ds-mem-0024``, a bare
|
|
conditional with no anchor, was promoted declined->unknown when Band
|
|
v4-CM landed — ADR-0259; it is genuinely UNKNOWN, not entailed, since the
|
|
antecedent is never asserted.)"""
|
|
report = build_report(_load(_V2_MEMBER))
|
|
assert report["n"] == 26
|
|
assert report["counts"]["wrong"] == 0
|
|
assert report["all_cases_correct"] is True
|
|
cbg = report["correct_by_gold"]
|
|
assert cbg.get("entailed", 0) >= 12
|
|
assert cbg.get("refuted", 0) >= 3
|
|
assert cbg.get("unknown", 0) >= 5
|
|
assert cbg.get("declined", 0) >= 6
|
|
|
|
|
|
def test_v2_condmem_lane_wrong_is_zero() -> None:
|
|
"""Band v4-CM (ADR-0259): hand-authored conditional-membership arguments
|
|
— v2-EN's connective grammar composed over v3-MEM's singular-membership
|
|
sentence reading, including genuine universal+connective fusion cases —
|
|
decided with wrong=0 across all four verdicts and seven distinct honest-
|
|
decline shapes."""
|
|
report = build_report(_load(_V2_CONDMEM))
|
|
assert report["n"] == 26
|
|
assert report["counts"]["wrong"] == 0
|
|
assert report["all_cases_correct"] is True
|
|
cbg = report["correct_by_gold"]
|
|
assert cbg.get("entailed", 0) >= 10
|
|
assert cbg.get("refuted", 0) >= 4
|
|
assert cbg.get("unknown", 0) >= 5
|
|
assert cbg.get("declined", 0) >= 7
|
|
|
|
|
|
def test_runner_treats_wrong_verdict_as_the_only_real_failure() -> None:
|
|
"""A committed disagreement with gold is ``wrong``; a decline never is,
|
|
even when gold expected a definite verdict (that's a miss, tracked via
|
|
``all_cases_correct``, not conflated with a confabulation)."""
|
|
report = build_report([
|
|
{"id": "bad", "text": "If p then q. p. Therefore q.", "gold": "unknown"}
|
|
])
|
|
assert report["counts"]["wrong"] == 1
|
|
assert report["all_cases_correct"] is False
|