"""Tests for the failure-family registry (N4). Pins that the registry is a PARTITION (every reachable organ refusal reason maps to exactly one family), that it covers the entire live refusal surface, that only the three R2 ``missing_*`` families are growth surfaces, and that correct wrong=0 boundaries stay refused with no proposal. """ from __future__ import annotations from core.comprehension_attempt import classify_r1, classify_r2 from core.comprehension_attempt.failure_family import ( ANSWER_KEY_CONTRADICTION, REGISTRY, enrich_family, family_for_reason, ) from evals.constraint_oracle.runner import _load_r2_gold from evals.setup_oracle.runner import _load_r1_gold #: The full live refusal surface (R1 reader/admissibility, classify, R2 reader/solver/choice). ALL_REASONS = { # R1 "empty", "no_quantity_template", "non_digit_quantity", "non_identifier_name", "unreadable_quantity_query", "unreadable_quantity_clause", "no_single_quantity_query", "admissibility_refused", "multiple_inverse_bases", "multiple_partitions", "partition_query_mismatch", "partition_container_mismatch", "invalid_binding_graph", "unprojectable", # R2 reader "too_many_categories", "missing_total_count", "missing_weighted_total", "category_pair_not_found", "coefficient_unit_mismatch", "coefficient_conflict", "query_target_not_a_category", # R2 solver "indistinguishable_weights", "non_integer_solution", "negative_solution", "query_target_unsolved", "verification_failed", # R2 answer-choice "no_matching_option", "ambiguous_options", "no_options", "unknown_provided_label", "unparseable_option", # R3 rate reader "rate_unit_mismatch", "combined_rates", "missing_rate", "missing_time", "missing_quantity", "temporal_state", "query_target_unrecognized", "no_query", "not_rate_shaped", # R4 combined-rate (reader + solver reasons namespaced cmb_*; not_combined_rate_shaped is the # bare step-aside reason -> the cross input_shape family). "not_combined_rate_shaped", "cmb_rate_unit_mismatch", "cmb_combine_mode_ambiguous", "cmb_missing_second_rate", "cmb_three_or_more_rates", "cmb_reciprocal_work_rate_deferred", "cmb_clock_interval_deferred", "cmb_non_positive_net_rate", "cmb_non_integer_solution", } def test_registry_is_a_partition() -> None: seen: dict[str, str] = {} for family in REGISTRY: for reason in family.refusal_reasons: assert reason not in seen, f"{reason} in both {seen.get(reason)} and {family.name}" seen[reason] = family.name def test_registry_covers_the_whole_refusal_surface() -> None: for reason in ALL_REASONS: assert family_for_reason(reason) is not None, f"unmapped reason: {reason}" def test_every_gold_refusal_reason_maps_to_a_family() -> None: attempts = [classify_r2(f["text"]) for f in _load_r2_gold()] attempts += [classify_r1(f["text"]) for f in _load_r1_gold()] for att in attempts: if att.outcome == "setup_refused": assert family_for_reason(att.refusal_reason) is not None, att.refusal_reason def test_only_precise_missing_totals_are_reachable_growth_surfaces() -> None: # Only the PRECISE R2 gaps are reachable growth surfaces. category_pair_not_found is too broad # (fires on any non-R2 text), so it maps to input_shape, and missing_category_pair is reserved. growth = {f.name for f in REGISTRY if f.proposal_allowed and f.refusal_reasons} assert growth == { "missing_total_count", "missing_weighted_total", "unsupported_rate_duration", "cmb_unsupported_rate_count", "cmb_unsupported_reciprocal", "cmb_unsupported_clock_interval", } assert family_for_reason("category_pair_not_found").name == "input_shape" for f in REGISTRY: if f.proposal_allowed: assert not f.must_remain_refused # a growth surface is never a hard boundary def test_correct_boundaries_stay_refused_with_no_proposal() -> None: for reason in ("too_many_categories", "non_integer_solution", "negative_solution", "unreadable_quantity_clause", "indistinguishable_weights", "coefficient_unit_mismatch", "admissibility_refused"): fam = family_for_reason(reason) assert fam is not None and fam.must_remain_refused and not fam.proposal_allowed, reason def test_growth_reasons_allow_proposals() -> None: for reason in ("missing_total_count", "missing_weighted_total"): fam = family_for_reason(reason) assert fam is not None and fam.proposal_allowed and not fam.must_remain_refused assert fam.proposal_target == "r2_gold_fixture" def test_enrich_sets_family_on_a_refused_attempt() -> None: fx = next(f for f in _load_r2_gold() if f["expect"] == "reader_refuses") enriched = enrich_family(classify_r2(fx["text"])) assert enriched.family is not None assert family_for_reason(enriched.refusal_reason).name == enriched.family def test_contradiction_family_reports_and_never_proposes() -> None: assert ANSWER_KEY_CONTRADICTION.name == "answer_key_contradiction" assert not ANSWER_KEY_CONTRADICTION.proposal_allowed assert "report" in ANSWER_KEY_CONTRADICTION.safe_next_action def test_family_for_reason_none_is_none() -> None: assert family_for_reason(None) is None