fa72b7323c
Implements the measurable test from COMPREHENSION-READER-AUDIT.md's Q5, generalized to the deduction and curriculum composers: a deterministic refusal histogram split mechanism/coverage/engine_refused, plus an admissions-per-batch counter for measuring a future lexicon or curriculum expansion's effect as a before/after delta instead of asserting it. Found and fixed a real classification bug while building it: an argument successfully read by a band but declined by the ROBDD engine for inconsistent premises was being misattributed to whichever reader band happened to run last in the fallback cascade. That case is neither a shape gap (mechanism) nor a vocabulary gap (coverage), so it gets its own "engine_refused" bucket rather than being folded into either. Baseline measurement: deduction refusals are 100% mechanism-class today (the bands read a closed connective grammar, not open vocabulary) — curriculum refusals are a real mechanism/coverage mix, confirming the volume ceiling S6 quantifies independently from the ratified-chain side.
118 lines
4.7 KiB
Python
118 lines
4.7 KiB
Python
"""tests/test_vocab_trigger_instrument.py — mechanism-vs-coverage refusal instrument (Tier S3).
|
|
|
|
Pins the classifier's closed rule and the instrument's arithmetic against the
|
|
CURRENT committed lane corpora, so a future band/curriculum change that shifts
|
|
the split is a visible diff here rather than a silent behavior change.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from evals.vocab_trigger_instrument import (
|
|
_all_curriculum_texts,
|
|
_all_deduction_texts,
|
|
admissions_delta,
|
|
build_curriculum_histogram,
|
|
build_deduction_histogram,
|
|
build_report,
|
|
curriculum_refusal_reason,
|
|
deduction_refusal_reason,
|
|
refusal_class_curriculum,
|
|
refusal_class_deduction,
|
|
)
|
|
|
|
|
|
def test_deduction_refusals_are_never_coverage_class() -> None:
|
|
"""The deduction bands read a closed connective/quantifier grammar, not
|
|
open vocabulary — every declined deduction case classifies as mechanism
|
|
or engine_refused, never coverage (module docstring's central claim)."""
|
|
hist = build_deduction_histogram(_all_deduction_texts())
|
|
assert hist.declined > 0
|
|
assert "coverage" not in hist.by_class
|
|
|
|
|
|
def test_curriculum_has_real_coverage_refusals() -> None:
|
|
"""The curriculum composer's untaught_vocabulary/out_of_curriculum gate
|
|
is a genuine coverage axis — the committed physics split exercises it."""
|
|
hist = build_curriculum_histogram(_all_curriculum_texts())
|
|
assert hist.by_class.get("coverage", 0) > 0
|
|
|
|
|
|
def test_refusal_class_curriculum_closed_set() -> None:
|
|
assert refusal_class_curriculum("untaught_vocabulary") == "coverage"
|
|
assert refusal_class_curriculum("out_of_curriculum") == "coverage"
|
|
assert refusal_class_curriculum("question_shape_out_of_band") == "mechanism"
|
|
assert refusal_class_curriculum("ambiguous_reading") == "mechanism"
|
|
assert refusal_class_curriculum("empty_curriculum") == "mechanism"
|
|
assert refusal_class_curriculum("inconsistent_premises") == "engine_refused"
|
|
assert refusal_class_curriculum("out_of_regime_or_malformed") == "engine_refused"
|
|
|
|
|
|
def test_refusal_class_deduction_only_engine_reasons_are_engine_refused() -> None:
|
|
assert refusal_class_deduction("inconsistent_premises") == "engine_refused"
|
|
assert refusal_class_deduction("out_of_regime_or_malformed") == "engine_refused"
|
|
assert refusal_class_deduction("sentence_shape_out_of_band") == "mechanism"
|
|
assert refusal_class_deduction("quantifier_out_of_band") == "mechanism"
|
|
assert refusal_class_deduction("not_argument_shaped") == "mechanism"
|
|
|
|
|
|
def test_deduction_refusal_reason_recovers_engine_level_decline() -> None:
|
|
"""A well-formed but self-contradictory Band-v1 argument declines at the
|
|
ENGINE, not at any reader — a real bug this test pins (an earlier version
|
|
of this instrument misattributed it to whichever fallback band happened
|
|
to run last)."""
|
|
band, reason = deduction_refusal_reason("p. Not p. Therefore q.")
|
|
assert band == "v1"
|
|
assert reason == "inconsistent_premises"
|
|
|
|
|
|
def test_deduction_refusal_reason_recovers_reader_level_decline() -> None:
|
|
band, reason = deduction_refusal_reason("colorless green ideas therefore sleep furiously")
|
|
assert band in {"gate", "reader", "v2_en", "v3_mem", "v4_condmem", "v5_verb", "v6_exist"}
|
|
assert reason != ""
|
|
assert refusal_class_deduction(reason) == "mechanism"
|
|
|
|
|
|
def test_curriculum_refusal_reason_matches_decide() -> None:
|
|
from chat.curriculum_surface import decide_curriculum_question
|
|
|
|
text = "Does the build pass?"
|
|
assert curriculum_refusal_reason(text) == decide_curriculum_question(text).reason
|
|
|
|
|
|
def test_histogram_totals_partition_the_corpus() -> None:
|
|
hist = build_deduction_histogram(_all_deduction_texts())
|
|
assert hist.admitted + hist.declined == hist.n
|
|
assert sum(hist.by_reason.values()) == hist.declined
|
|
assert sum(hist.by_class.values()) == hist.declined
|
|
|
|
|
|
def test_build_report_is_deterministic() -> None:
|
|
a = build_report()
|
|
b = build_report()
|
|
assert a == b
|
|
|
|
|
|
def test_admissions_delta_is_zero_against_self() -> None:
|
|
report = build_report()
|
|
delta = admissions_delta(report, report)
|
|
assert delta == {
|
|
"deduction_admitted_delta": 0,
|
|
"curriculum_admitted_delta": 0,
|
|
"combined_admitted_delta": 0,
|
|
}
|
|
|
|
|
|
def test_admissions_delta_reflects_a_new_admission() -> None:
|
|
before = build_report()
|
|
after = {
|
|
"deduction": {**before["deduction"], "admitted": before["deduction"]["admitted"] + 3},
|
|
"curriculum": before["curriculum"],
|
|
"combined": {
|
|
**before["combined"],
|
|
"admitted": before["combined"]["admitted"] + 3,
|
|
},
|
|
}
|
|
delta = admissions_delta(before, after)
|
|
assert delta["deduction_admitted_delta"] == 3
|
|
assert delta["combined_admitted_delta"] == 3
|
|
assert delta["curriculum_admitted_delta"] == 0
|