73345e641e
classify_r3 normalizes the rate reader into a ComprehensionAttempt; route_setup now tries R1/R2/R3 (same exactly-one-setup_correct rule); the pass manager solves+verifies a routed r3_rate setup (solve_rate + reused answer-choice verifier); unsupported_temporal_state dropped from the unsupported-family set so temporal_state -> REFUSED_KNOWN_BOUNDARY. Organ gains r3_rate. Terminal matrix (verified): 6 solved->SOLVED_VERIFIED; rate_unit_mismatch+combined_rates->PROPOSAL_EMITTED(unsupported_rate_duration); non_integer/missing_time/temporal->REFUSED_KNOWN_BOUNDARY; exactly 2 proposals (the rate-like gaps only). REGRESSION CAUGHT+FIXED: R3's reader returned missing_rate (a substantive boundary) on non-rate R2 text, blocking r2-011's legitimate missing_total_count proposal via boundary-first. Fix: missing_rate only when a duration is present; else not_rate_shaped -> input_shape (not-my-domain) — same discipline as the N6 category_pair_not_found fix. r2-011 proposes again; R1/R2 unaffected. R1 7/0/3, 15-case 15/0/0, R2 10/0/0, R3 8/0/0 unchanged; off-serving; 121-test smoke green incl. invariants + idle contract suites.
105 lines
4.7 KiB
Python
105 lines
4.7 KiB
Python
"""Tests for the failure-family registry (N4).
|
|
|
|
Pins that the registry is a PARTITION (every reachable organ refusal reason maps to exactly
|
|
one family), that it covers the entire live refusal surface, that only the three R2 ``missing_*``
|
|
families are growth surfaces, and that correct wrong=0 boundaries stay refused with no proposal.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from core.comprehension_attempt import classify_r1, classify_r2
|
|
from core.comprehension_attempt.failure_family import (
|
|
ANSWER_KEY_CONTRADICTION,
|
|
REGISTRY,
|
|
enrich_family,
|
|
family_for_reason,
|
|
)
|
|
from evals.constraint_oracle.runner import _load_r2_gold
|
|
from evals.setup_oracle.runner import _load_r1_gold
|
|
|
|
#: The full live refusal surface (R1 reader/admissibility, classify, R2 reader/solver/choice).
|
|
ALL_REASONS = {
|
|
# R1
|
|
"empty", "no_quantity_template", "non_digit_quantity", "non_identifier_name",
|
|
"unreadable_quantity_query", "unreadable_quantity_clause", "no_single_quantity_query",
|
|
"admissibility_refused", "multiple_inverse_bases", "multiple_partitions",
|
|
"partition_query_mismatch", "partition_container_mismatch", "invalid_binding_graph",
|
|
"unprojectable",
|
|
# R2 reader
|
|
"too_many_categories", "missing_total_count", "missing_weighted_total",
|
|
"category_pair_not_found", "coefficient_unit_mismatch", "coefficient_conflict",
|
|
"query_target_not_a_category",
|
|
# R2 solver
|
|
"indistinguishable_weights", "non_integer_solution", "negative_solution",
|
|
"query_target_unsolved", "verification_failed",
|
|
# R2 answer-choice
|
|
"no_matching_option", "ambiguous_options", "no_options", "unknown_provided_label",
|
|
"unparseable_option",
|
|
# R3 rate reader
|
|
"rate_unit_mismatch", "combined_rates", "missing_rate", "missing_time", "missing_quantity",
|
|
"temporal_state", "query_target_unrecognized", "no_query", "not_rate_shaped",
|
|
}
|
|
|
|
|
|
def test_registry_is_a_partition() -> None:
|
|
seen: dict[str, str] = {}
|
|
for family in REGISTRY:
|
|
for reason in family.refusal_reasons:
|
|
assert reason not in seen, f"{reason} in both {seen.get(reason)} and {family.name}"
|
|
seen[reason] = family.name
|
|
|
|
|
|
def test_registry_covers_the_whole_refusal_surface() -> None:
|
|
for reason in ALL_REASONS:
|
|
assert family_for_reason(reason) is not None, f"unmapped reason: {reason}"
|
|
|
|
|
|
def test_every_gold_refusal_reason_maps_to_a_family() -> None:
|
|
attempts = [classify_r2(f["text"]) for f in _load_r2_gold()]
|
|
attempts += [classify_r1(f["text"]) for f in _load_r1_gold()]
|
|
for att in attempts:
|
|
if att.outcome == "setup_refused":
|
|
assert family_for_reason(att.refusal_reason) is not None, att.refusal_reason
|
|
|
|
|
|
def test_only_precise_missing_totals_are_reachable_growth_surfaces() -> None:
|
|
# Only the PRECISE R2 gaps are reachable growth surfaces. category_pair_not_found is too broad
|
|
# (fires on any non-R2 text), so it maps to input_shape, and missing_category_pair is reserved.
|
|
growth = {f.name for f in REGISTRY if f.proposal_allowed and f.refusal_reasons}
|
|
assert growth == {"missing_total_count", "missing_weighted_total", "unsupported_rate_duration"}
|
|
assert family_for_reason("category_pair_not_found").name == "input_shape"
|
|
for f in REGISTRY:
|
|
if f.proposal_allowed:
|
|
assert not f.must_remain_refused # a growth surface is never a hard boundary
|
|
|
|
|
|
def test_correct_boundaries_stay_refused_with_no_proposal() -> None:
|
|
for reason in ("too_many_categories", "non_integer_solution", "negative_solution",
|
|
"unreadable_quantity_clause", "indistinguishable_weights",
|
|
"coefficient_unit_mismatch", "admissibility_refused"):
|
|
fam = family_for_reason(reason)
|
|
assert fam is not None and fam.must_remain_refused and not fam.proposal_allowed, reason
|
|
|
|
|
|
def test_growth_reasons_allow_proposals() -> None:
|
|
for reason in ("missing_total_count", "missing_weighted_total"):
|
|
fam = family_for_reason(reason)
|
|
assert fam is not None and fam.proposal_allowed and not fam.must_remain_refused
|
|
assert fam.proposal_target == "r2_gold_fixture"
|
|
|
|
|
|
def test_enrich_sets_family_on_a_refused_attempt() -> None:
|
|
fx = next(f for f in _load_r2_gold() if f["expect"] == "reader_refuses")
|
|
enriched = enrich_family(classify_r2(fx["text"]))
|
|
assert enriched.family is not None
|
|
assert family_for_reason(enriched.refusal_reason).name == enriched.family
|
|
|
|
|
|
def test_contradiction_family_reports_and_never_proposes() -> None:
|
|
assert ANSWER_KEY_CONTRADICTION.name == "answer_key_contradiction"
|
|
assert not ANSWER_KEY_CONTRADICTION.proposal_allowed
|
|
assert "report" in ANSWER_KEY_CONTRADICTION.safe_next_action
|
|
|
|
|
|
def test_family_for_reason_none_is_none() -> None:
|
|
assert family_for_reason(None) is None
|