d181ae30f9
Generalization arc Phase 1.1 (Tier F, docs/plans/generalization-arc-2026-07-24.md): "All philosophers teach. Socrates is a philosopher. Therefore Socrates teaches." now reads and decides — the ADR-0258 §6.3 verb-predicate scope-out, and the reading gate for Phase 2's subject serving. generate/proof_chain/verb.py extends v3-MEM's per-individual lowering with a second atom family in one shared space: membership atoms (v3-MEM's own parsers reused verbatim) + verb atoms (individual, verb-lemma-group, object-term). Verb universals instantiate as mem(i,C) -> [~]verb(i), so a copula-minted membership fact discharges a verb rule. The ONE new semantic identification is closed 3sg agreement (verb-specific irregular table + the three regular suffix rules — deliberately NOT the noun table, which would misroute "lives"/"leaves"). Scope-tight segmentation: single-token name/class/verb/object shapes only; everything longer refuses typed (tense_out_of_band joins the closed reason vocabulary). Rendered by render_entailment_verb (member surface, UNKNOWN scoping extended to the verb reading). Rides deduction_serving_enabled (default off); pure widening — every previously-served argument byte-identical, no existing lane case changed outcome. Earned: 4 en_verb_* bands x 720 arena wrong=0 (first seal), ledger resealed at 21 bands; corpus soundness asserted against the independent truth-table oracle over each template's intended lowering (15,120 cases, INV-25). Lane: v2_verb 28 hand-authored real cases 28/28 first run; full lane 134/134 wrong=0 across five splits. Surgical single-line pin update + CLAIMS regen. Stacked on feat/generalization-phase0 (merge that first — shared verify_lane_shas.py / CLAIMS.md / generate_claims.py lane-ADR mapping). [Verification]: verb reader tests 34/34; deduction battery 86/86 (surface +5, e2e +1, lane +1) in-worktree; arena seal + oracle cross-check green; smoke + warmed_session via pre-push gate.
117 lines
5.1 KiB
Python
117 lines
5.1 KiB
Python
"""Deduction-serve lane — wrong=0 gate over the committed v1 corpus.
|
|
|
|
Scores the PRODUCTION serving decider end-to-end from raw text (the same
|
|
pipeline ``chat/deduction_surface.py`` runs), distinct from the bare-engine
|
|
``evals/deductive_logic`` lane and the reader-fidelity
|
|
``evals/comprehension/propositional_runner.py`` lane. See
|
|
``evals/deduction_serve/contract.md`` for the full contract.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from evals.deduction_serve.runner import _ROOT, _load, build_report
|
|
|
|
_V1 = _ROOT / "v1" / "cases.jsonl"
|
|
_V2_EN = _ROOT / "v2_en" / "cases.jsonl"
|
|
_V2_MEMBER = _ROOT / "v2_member" / "cases.jsonl"
|
|
_V2_CONDMEM = _ROOT / "v2_condmem" / "cases.jsonl"
|
|
_V2_VERB = _ROOT / "v2_verb" / "cases.jsonl"
|
|
|
|
|
|
def test_v1_lane_wrong_is_zero() -> None:
|
|
report = build_report(_load(_V1))
|
|
assert report["n"] == 28
|
|
assert report["counts"]["wrong"] == 0
|
|
assert report["all_cases_correct"] is True
|
|
# the sizeable, honest signal: non-trivial committed classes covered,
|
|
# including the categorical band (valid + invalid) added in Phase 4.
|
|
# (declined dropped from 6 to 4 when the two documented Band v1 boundary
|
|
# cases — multiword conditional, nested-negation contraposition — became
|
|
# DECIDED by Band v2-EN, ADR-0257; they are now entailed gold.)
|
|
cbg = report["correct_by_gold"]
|
|
assert cbg.get("entailed", 0) >= 7
|
|
assert cbg.get("refuted", 0) >= 5
|
|
assert cbg.get("unknown", 0) >= 5
|
|
assert cbg.get("declined", 0) >= 4
|
|
assert cbg.get("valid", 0) >= 3
|
|
assert cbg.get("invalid", 0) >= 1
|
|
|
|
|
|
def test_v2_en_lane_wrong_is_zero() -> None:
|
|
"""Band v2-EN (ADR-0257): hand-authored REAL-English arguments — content
|
|
deliberately disjoint from the synthetic practice lexicon — decided by the
|
|
production pipeline with wrong=0, across all four inference verdicts and
|
|
five distinct honest-decline shapes (ds-en-0022, the membership decline,
|
|
was promoted declined→entailed when the member band landed — ADR-0258)."""
|
|
report = build_report(_load(_V2_EN))
|
|
assert report["n"] == 26
|
|
assert report["counts"]["wrong"] == 0
|
|
assert report["all_cases_correct"] is True
|
|
cbg = report["correct_by_gold"]
|
|
assert cbg.get("entailed", 0) >= 14
|
|
assert cbg.get("refuted", 0) >= 3
|
|
assert cbg.get("unknown", 0) >= 4
|
|
assert cbg.get("declined", 0) >= 5
|
|
|
|
|
|
def test_v2_member_lane_wrong_is_zero() -> None:
|
|
"""Band v3-MEM (ADR-0258): hand-authored membership/universal arguments —
|
|
real nouns exercising every number-link row-type (irregular, invariant,
|
|
each regular suffix rule) — decided with wrong=0 across all four verdicts
|
|
and six distinct honest-decline shapes. (``ds-mem-0024``, a bare
|
|
conditional with no anchor, was promoted declined->unknown when Band
|
|
v4-CM landed — ADR-0259; it is genuinely UNKNOWN, not entailed, since the
|
|
antecedent is never asserted.)"""
|
|
report = build_report(_load(_V2_MEMBER))
|
|
assert report["n"] == 26
|
|
assert report["counts"]["wrong"] == 0
|
|
assert report["all_cases_correct"] is True
|
|
cbg = report["correct_by_gold"]
|
|
assert cbg.get("entailed", 0) >= 12
|
|
assert cbg.get("refuted", 0) >= 3
|
|
assert cbg.get("unknown", 0) >= 5
|
|
assert cbg.get("declined", 0) >= 6
|
|
|
|
|
|
def test_v2_condmem_lane_wrong_is_zero() -> None:
|
|
"""Band v4-CM (ADR-0259): hand-authored conditional-membership arguments
|
|
— v2-EN's connective grammar composed over v3-MEM's singular-membership
|
|
sentence reading, including genuine universal+connective fusion cases —
|
|
decided with wrong=0 across all four verdicts and seven distinct honest-
|
|
decline shapes."""
|
|
report = build_report(_load(_V2_CONDMEM))
|
|
assert report["n"] == 26
|
|
assert report["counts"]["wrong"] == 0
|
|
assert report["all_cases_correct"] is True
|
|
cbg = report["correct_by_gold"]
|
|
assert cbg.get("entailed", 0) >= 10
|
|
assert cbg.get("refuted", 0) >= 4
|
|
assert cbg.get("unknown", 0) >= 5
|
|
assert cbg.get("declined", 0) >= 7
|
|
|
|
|
|
def test_v2_verb_lane_wrong_is_zero() -> None:
|
|
"""Band v5-VP (ADR-0260): hand-authored verb-predicate arguments —
|
|
universals discharged by membership facts across every closed agreement
|
|
rule (+s, +es, y↔ies, irregular go/goes), transitive objects at face
|
|
value, and eight distinct honest-decline shapes — decided wrong=0."""
|
|
report = build_report(_load(_V2_VERB))
|
|
assert report["n"] == 28
|
|
assert report["counts"]["wrong"] == 0
|
|
assert report["all_cases_correct"] is True
|
|
cbg = report["correct_by_gold"]
|
|
assert cbg.get("entailed", 0) >= 11
|
|
assert cbg.get("refuted", 0) >= 4
|
|
assert cbg.get("unknown", 0) >= 5
|
|
assert cbg.get("declined", 0) >= 8
|
|
|
|
|
|
def test_runner_treats_wrong_verdict_as_the_only_real_failure() -> None:
|
|
"""A committed disagreement with gold is ``wrong``; a decline never is,
|
|
even when gold expected a definite verdict (that's a miss, tracked via
|
|
``all_cases_correct``, not conflated with a confabulation)."""
|
|
report = build_report([
|
|
{"id": "bad", "text": "If p then q. p. Therefore q.", "gold": "unknown"}
|
|
])
|
|
assert report["counts"]["wrong"] == 1
|
|
assert report["all_cases_correct"] is False
|