780f993c73
docs/analysis/r3-rate-inventory-ledger: R3 v1 state (reader 8/0/0 + 4 refused -> answers 6/0/6; gold 12/12), per-fixture table, the compound-unit substrate, covered families + R3.2/R3.3 deferrals. core/comprehension_attempt/failure_family.py: makes unsupported_rate_duration REACHABLE as a growth surface (proposal_allowed) mapping rate_unit_mismatch + combined_rates — both emitted ONLY after a rate clause is recognized, so always rate-like (anti-over-propose discipline, same as the N6 fix). missing_rate/time/quantity -> rate_underdetermined boundary; temporal_state -> unsupported_temporal_state boundary (clock detector can fire on non-rate text, NOT a safe growth surface); query_target_unrecognized/no_query -> input_shape. Owner gains 'r3'. 107-test smoke green incl. partition + contemplation + invariants; R1 7/0/3 + R2 10/0/3 unchanged; off-serving.
105 lines
4.7 KiB
Python
105 lines
4.7 KiB
Python
"""Tests for the failure-family registry (N4).
|
|
|
|
Pins that the registry is a PARTITION (every reachable organ refusal reason maps to exactly
|
|
one family), that it covers the entire live refusal surface, that only the three R2 ``missing_*``
|
|
families are growth surfaces, and that correct wrong=0 boundaries stay refused with no proposal.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from core.comprehension_attempt import classify_r1, classify_r2
|
|
from core.comprehension_attempt.failure_family import (
|
|
ANSWER_KEY_CONTRADICTION,
|
|
REGISTRY,
|
|
enrich_family,
|
|
family_for_reason,
|
|
)
|
|
from evals.constraint_oracle.runner import _load_r2_gold
|
|
from evals.setup_oracle.runner import _load_r1_gold
|
|
|
|
#: The full live refusal surface (R1 reader/admissibility, classify, R2 reader/solver/choice).
|
|
ALL_REASONS = {
|
|
# R1
|
|
"empty", "no_quantity_template", "non_digit_quantity", "non_identifier_name",
|
|
"unreadable_quantity_query", "unreadable_quantity_clause", "no_single_quantity_query",
|
|
"admissibility_refused", "multiple_inverse_bases", "multiple_partitions",
|
|
"partition_query_mismatch", "partition_container_mismatch", "invalid_binding_graph",
|
|
"unprojectable",
|
|
# R2 reader
|
|
"too_many_categories", "missing_total_count", "missing_weighted_total",
|
|
"category_pair_not_found", "coefficient_unit_mismatch", "coefficient_conflict",
|
|
"query_target_not_a_category",
|
|
# R2 solver
|
|
"indistinguishable_weights", "non_integer_solution", "negative_solution",
|
|
"query_target_unsolved", "verification_failed",
|
|
# R2 answer-choice
|
|
"no_matching_option", "ambiguous_options", "no_options", "unknown_provided_label",
|
|
"unparseable_option",
|
|
# R3 rate reader
|
|
"rate_unit_mismatch", "combined_rates", "missing_rate", "missing_time", "missing_quantity",
|
|
"temporal_state", "query_target_unrecognized", "no_query",
|
|
}
|
|
|
|
|
|
def test_registry_is_a_partition() -> None:
|
|
seen: dict[str, str] = {}
|
|
for family in REGISTRY:
|
|
for reason in family.refusal_reasons:
|
|
assert reason not in seen, f"{reason} in both {seen.get(reason)} and {family.name}"
|
|
seen[reason] = family.name
|
|
|
|
|
|
def test_registry_covers_the_whole_refusal_surface() -> None:
|
|
for reason in ALL_REASONS:
|
|
assert family_for_reason(reason) is not None, f"unmapped reason: {reason}"
|
|
|
|
|
|
def test_every_gold_refusal_reason_maps_to_a_family() -> None:
|
|
attempts = [classify_r2(f["text"]) for f in _load_r2_gold()]
|
|
attempts += [classify_r1(f["text"]) for f in _load_r1_gold()]
|
|
for att in attempts:
|
|
if att.outcome == "setup_refused":
|
|
assert family_for_reason(att.refusal_reason) is not None, att.refusal_reason
|
|
|
|
|
|
def test_only_precise_missing_totals_are_reachable_growth_surfaces() -> None:
|
|
# Only the PRECISE R2 gaps are reachable growth surfaces. category_pair_not_found is too broad
|
|
# (fires on any non-R2 text), so it maps to input_shape, and missing_category_pair is reserved.
|
|
growth = {f.name for f in REGISTRY if f.proposal_allowed and f.refusal_reasons}
|
|
assert growth == {"missing_total_count", "missing_weighted_total", "unsupported_rate_duration"}
|
|
assert family_for_reason("category_pair_not_found").name == "input_shape"
|
|
for f in REGISTRY:
|
|
if f.proposal_allowed:
|
|
assert not f.must_remain_refused # a growth surface is never a hard boundary
|
|
|
|
|
|
def test_correct_boundaries_stay_refused_with_no_proposal() -> None:
|
|
for reason in ("too_many_categories", "non_integer_solution", "negative_solution",
|
|
"unreadable_quantity_clause", "indistinguishable_weights",
|
|
"coefficient_unit_mismatch", "admissibility_refused"):
|
|
fam = family_for_reason(reason)
|
|
assert fam is not None and fam.must_remain_refused and not fam.proposal_allowed, reason
|
|
|
|
|
|
def test_growth_reasons_allow_proposals() -> None:
|
|
for reason in ("missing_total_count", "missing_weighted_total"):
|
|
fam = family_for_reason(reason)
|
|
assert fam is not None and fam.proposal_allowed and not fam.must_remain_refused
|
|
assert fam.proposal_target == "r2_gold_fixture"
|
|
|
|
|
|
def test_enrich_sets_family_on_a_refused_attempt() -> None:
|
|
fx = next(f for f in _load_r2_gold() if f["expect"] == "reader_refuses")
|
|
enriched = enrich_family(classify_r2(fx["text"]))
|
|
assert enriched.family is not None
|
|
assert family_for_reason(enriched.refusal_reason).name == enriched.family
|
|
|
|
|
|
def test_contradiction_family_reports_and_never_proposes() -> None:
|
|
assert ANSWER_KEY_CONTRADICTION.name == "answer_key_contradiction"
|
|
assert not ANSWER_KEY_CONTRADICTION.proposal_allowed
|
|
assert "report" in ANSWER_KEY_CONTRADICTION.safe_next_action
|
|
|
|
|
|
def test_family_for_reason_none_is_none() -> None:
|
|
assert family_for_reason(None) is None
|