From c1ad5b81428498d1e816e11ee304279510dda3c0 Mon Sep 17 00:00:00 2001 From: Shay Date: Sun, 7 Jun 2026 08:46:07 -0700 Subject: [PATCH] feat(comprehension): failure-family registry (N4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit core/comprehension_attempt/failure_family.py: partitions every reachable organ refusal reason (R1 reader/admissibility, R2 reader/solver/answer-choice) into a named FailureFamily declaring owner / must_remain_refused / proposal_allowed / safe_next_action / proposal_target. family_for_reason is total + unambiguous (partition asserted by test); enrich_family resolves an attempt's family. Only THREE families are growth surfaces (proposal_allowed=True): the R2 missing_category_pair / missing_total_count / missing_weighted_total gaps. All other reasons are correct wrong=0 boundaries (refuse, no proposal) — correct refusal != missing capability. answer_key_contradiction is a report verdict (no reason). Reserved R3 families forward-declared. 9 tests incl. whole-surface coverage + partition. --- core/comprehension_attempt/__init__.py | 12 ++ core/comprehension_attempt/failure_family.py | 186 +++++++++++++++++++ tests/test_failure_family.py | 99 ++++++++++ 3 files changed, 297 insertions(+) create mode 100644 core/comprehension_attempt/failure_family.py create mode 100644 tests/test_failure_family.py diff --git a/core/comprehension_attempt/__init__.py b/core/comprehension_attempt/__init__.py index cf2ec183..f6e96b79 100644 --- a/core/comprehension_attempt/__init__.py +++ b/core/comprehension_attempt/__init__.py @@ -7,16 +7,28 @@ over — uniform across the R1 and R2 setup compilers. Off-serving; imports no ` from __future__ import annotations from core.comprehension_attempt.classify import classify_r1, classify_r2 +from core.comprehension_attempt.failure_family import ( + REGISTRY, + FailureFamily, + enrich_family, + family_by_name, + family_for_reason, +) from core.comprehension_attempt.model import ComprehensionAttempt, Organ, Outcome from core.comprehension_attempt.router import RouteResult, RouteStatus, route_setup __all__ = [ + "REGISTRY", "ComprehensionAttempt", + "FailureFamily", "Organ", "Outcome", "RouteResult", "RouteStatus", "classify_r1", "classify_r2", + "enrich_family", + "family_by_name", + "family_for_reason", "route_setup", ] diff --git a/core/comprehension_attempt/failure_family.py b/core/comprehension_attempt/failure_family.py new file mode 100644 index 00000000..b2f9fcc2 --- /dev/null +++ b/core/comprehension_attempt/failure_family.py @@ -0,0 +1,186 @@ +"""Failure-family registry (N4) — the heart of the contemplation batch. + +Partitions every typed organ refusal reason (R1 reader/admissibility, R2 reader/solver/ +answer-choice) into a named **failure family** that declares: + +- ``owner`` — which organ surfaces it (``r1`` / ``r2`` / ``cross``) +- ``must_remain_refused`` — is this a correct wrong=0 boundary that must stay refused? +- ``proposal_allowed`` — is this a genuine coverage gap a proposal may target? +- ``safe_next_action`` — the human-readable next step +- ``proposal_target`` — what artifact a proposal would suggest (e.g. ``r2_gold_fixture``) + +Only three families are growth surfaces (``proposal_allowed = True``): the R2 ``missing_*`` +gaps. Everything else is a correct boundary — `correct refusal != missing capability`. The +registry is a **partition**: every reachable reason maps to exactly one family (asserted by +test), so ``family_for_reason`` is total and unambiguous. ``answer_key_contradiction`` carries +no refusal reason — it is reached from the answer-choice ``contradiction`` *verdict* (N6). + +Some R1 reasons are coarse (``unreadable_quantity_clause`` covers both the pronoun and distractor +cases; ``admissibility_refused`` covers both ungrounded and unit-incompatible). v0 folds each to a +single conservative family — the *action* (refuse, no proposal) is identical for the folded cases, +so no wrong=0 signal is lost. The reserved families are forward-declared for R3 with no current +reason mapping. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +from typing import Literal + +from core.comprehension_attempt.model import ComprehensionAttempt + +Owner = Literal["r1", "r2", "cross"] + + +@dataclass(frozen=True, slots=True) +class FailureFamily: + """A named class of comprehension failure with its growth/refusal policy.""" + + name: str + owner: Owner + must_remain_refused: bool + proposal_allowed: bool + safe_next_action: str + proposal_target: str | None = None + refusal_reasons: tuple[str, ...] = () + + +#: The registry. Every reachable organ refusal reason appears in exactly one family. +REGISTRY: tuple[FailureFamily, ...] = ( + # --- correct wrong=0 boundaries (no proposal) ---------------------------------------- # + FailureFamily( + "input_shape", "cross", True, False, + "refuse — the text is not a readable problem shape", + refusal_reasons=( + "empty", "no_quantity_template", "non_digit_quantity", "non_identifier_name", + "unreadable_quantity_query", "invalid_binding_graph", "query_target_not_a_category", + "unprojectable", + ), + ), + FailureFamily( + "unsupported_clause_shape", "r1", True, False, + "refuse — compound/pronoun clause the template cannot isolate (subsumes " + "ambiguous_referent + unsupported_distractor_clause until a finer signal exists)", + refusal_reasons=("unreadable_quantity_clause",), + ), + FailureFamily( + "ungrounded_base", "r1", True, False, + "refuse — the asked quantity has no grounded anchor (underdetermined)", + refusal_reasons=("no_single_quantity_query",), + ), + FailureFamily( + "admissibility_incompatible", "cross", True, False, + "refuse — operands are ungrounded or unit-incompatible (cannot combine across dimensions)", + refusal_reasons=("admissibility_refused", "coefficient_unit_mismatch", "coefficient_conflict"), + ), + FailureFamily( + "over_determined", "r1", True, False, + "refuse — structurally incoherent (multiple bases / partition mismatch)", + refusal_reasons=( + "multiple_inverse_bases", "multiple_partitions", + "partition_query_mismatch", "partition_container_mismatch", + ), + ), + FailureFamily( + "unsupported_system_size", "r2", True, False, + "refuse — more than two categories; needs an n-variable solver (R3)", + refusal_reasons=("too_many_categories",), + ), + FailureFamily( + "indistinguishable_system", "r2", True, False, + "refuse — the system is singular/underdetermined; no unique solution", + refusal_reasons=("indistinguishable_weights", "query_target_unsolved", "verification_failed"), + ), + FailureFamily( + "non_integer_solution", "r2", True, False, + "refuse — no integer solution exists; never round", + refusal_reasons=("non_integer_solution",), + ), + FailureFamily( + "negative_solution", "r2", True, False, + "refuse — a solved count is negative; out of domain", + refusal_reasons=("negative_solution",), + ), + FailureFamily( + "answer_choice_unresolved", "r2", True, False, + "refuse — the proven value cannot be tied to exactly one option", + refusal_reasons=( + "no_matching_option", "ambiguous_options", "no_options", + "unknown_provided_label", "unparseable_option", + ), + ), + # --- growth surfaces (proposal allowed) ---------------------------------------------- # + FailureFamily( + "missing_category_pair", "r2", False, True, + "propose a category-pair gold fixture for review", + proposal_target="r2_gold_fixture", refusal_reasons=("category_pair_not_found",), + ), + FailureFamily( + "missing_total_count", "r2", False, True, + "propose a total-count-constraint gold fixture for review", + proposal_target="r2_gold_fixture", refusal_reasons=("missing_total_count",), + ), + FailureFamily( + "missing_weighted_total", "r2", False, True, + "propose a weighted-total-constraint gold fixture for review", + proposal_target="r2_gold_fixture", refusal_reasons=("missing_weighted_total",), + ), + # --- verdict (not a refusal) --------------------------------------------------------- # + FailureFamily( + "answer_key_contradiction", "r2", False, False, + "report the contradiction — the proven value disagrees with the supplied key", + refusal_reasons=(), + ), + # --- reserved / forward-declared for R3 (no current emitter) ------------------------- # + FailureFamily( + "missing_attribute_coefficient", "r2", False, True, + "RESERVED — propose an attribute-coefficient fixture (no emitter yet)", + proposal_target="r2_gold_fixture", + ), + FailureFamily( + "unsupported_rate_duration", "cross", True, False, + "RESERVED — rate/duration frames are R3 (no emitter yet)", + ), + FailureFamily( + "unsupported_temporal_state", "cross", True, False, + "RESERVED — temporal-state frames are R3 (no emitter yet)", + ), +) + +_BY_NAME: dict[str, FailureFamily] = {f.name: f for f in REGISTRY} +_BY_REASON: dict[str, FailureFamily] = { + reason: family for family in REGISTRY for reason in family.refusal_reasons +} + +#: The verdict-derived family (no refusal reason maps to it). +ANSWER_KEY_CONTRADICTION = _BY_NAME["answer_key_contradiction"] + + +def family_for_reason(reason: str | None) -> FailureFamily | None: + """The single failure family a typed organ refusal reason belongs to (or ``None``).""" + if reason is None: + return None + return _BY_REASON.get(reason) + + +def family_by_name(name: str) -> FailureFamily | None: + return _BY_NAME.get(name) + + +def enrich_family(attempt: ComprehensionAttempt) -> ComprehensionAttempt: + """Return *attempt* with its ``family`` resolved from its refusal reason (or unchanged).""" + family = family_for_reason(attempt.refusal_reason) + if family is None: + return attempt + return replace(attempt, family=family.name) + + +__all__ = [ + "ANSWER_KEY_CONTRADICTION", + "FailureFamily", + "Owner", + "REGISTRY", + "enrich_family", + "family_by_name", + "family_for_reason", +] diff --git a/tests/test_failure_family.py b/tests/test_failure_family.py new file mode 100644 index 00000000..874e6e4a --- /dev/null +++ b/tests/test_failure_family.py @@ -0,0 +1,99 @@ +"""Tests for the failure-family registry (N4). + +Pins that the registry is a PARTITION (every reachable organ refusal reason maps to exactly +one family), that it covers the entire live refusal surface, that only the three R2 ``missing_*`` +families are growth surfaces, and that correct wrong=0 boundaries stay refused with no proposal. +""" + +from __future__ import annotations + +from core.comprehension_attempt import classify_r1, classify_r2 +from core.comprehension_attempt.failure_family import ( + ANSWER_KEY_CONTRADICTION, + REGISTRY, + enrich_family, + family_for_reason, +) +from evals.constraint_oracle.runner import _load_r2_gold +from evals.setup_oracle.runner import _load_r1_gold + +#: The full live refusal surface (R1 reader/admissibility, classify, R2 reader/solver/choice). +ALL_REASONS = { + # R1 + "empty", "no_quantity_template", "non_digit_quantity", "non_identifier_name", + "unreadable_quantity_query", "unreadable_quantity_clause", "no_single_quantity_query", + "admissibility_refused", "multiple_inverse_bases", "multiple_partitions", + "partition_query_mismatch", "partition_container_mismatch", "invalid_binding_graph", + "unprojectable", + # R2 reader + "too_many_categories", "missing_total_count", "missing_weighted_total", + "category_pair_not_found", "coefficient_unit_mismatch", "coefficient_conflict", + "query_target_not_a_category", + # R2 solver + "indistinguishable_weights", "non_integer_solution", "negative_solution", + "query_target_unsolved", "verification_failed", + # R2 answer-choice + "no_matching_option", "ambiguous_options", "no_options", "unknown_provided_label", + "unparseable_option", +} + + +def test_registry_is_a_partition() -> None: + seen: dict[str, str] = {} + for family in REGISTRY: + for reason in family.refusal_reasons: + assert reason not in seen, f"{reason} in both {seen.get(reason)} and {family.name}" + seen[reason] = family.name + + +def test_registry_covers_the_whole_refusal_surface() -> None: + for reason in ALL_REASONS: + assert family_for_reason(reason) is not None, f"unmapped reason: {reason}" + + +def test_every_gold_refusal_reason_maps_to_a_family() -> None: + attempts = [classify_r2(f["text"]) for f in _load_r2_gold()] + attempts += [classify_r1(f["text"]) for f in _load_r1_gold()] + for att in attempts: + if att.outcome == "setup_refused": + assert family_for_reason(att.refusal_reason) is not None, att.refusal_reason + + +def test_only_three_missing_families_are_growth_surfaces() -> None: + growth = {f.name for f in REGISTRY if f.proposal_allowed and f.refusal_reasons} + assert growth == {"missing_category_pair", "missing_total_count", "missing_weighted_total"} + for f in REGISTRY: + if f.proposal_allowed: + assert not f.must_remain_refused # a growth surface is never a hard boundary + + +def test_correct_boundaries_stay_refused_with_no_proposal() -> None: + for reason in ("too_many_categories", "non_integer_solution", "negative_solution", + "unreadable_quantity_clause", "indistinguishable_weights", + "coefficient_unit_mismatch", "admissibility_refused"): + fam = family_for_reason(reason) + assert fam is not None and fam.must_remain_refused and not fam.proposal_allowed, reason + + +def test_growth_reasons_allow_proposals() -> None: + for reason in ("missing_total_count", "missing_weighted_total", "category_pair_not_found"): + fam = family_for_reason(reason) + assert fam is not None and fam.proposal_allowed and not fam.must_remain_refused + assert fam.proposal_target == "r2_gold_fixture" + + +def test_enrich_sets_family_on_a_refused_attempt() -> None: + fx = next(f for f in _load_r2_gold() if f["expect"] == "reader_refuses") + enriched = enrich_family(classify_r2(fx["text"])) + assert enriched.family is not None + assert family_for_reason(enriched.refusal_reason).name == enriched.family + + +def test_contradiction_family_reports_and_never_proposes() -> None: + assert ANSWER_KEY_CONTRADICTION.name == "answer_key_contradiction" + assert not ANSWER_KEY_CONTRADICTION.proposal_allowed + assert "report" in ANSWER_KEY_CONTRADICTION.safe_next_action + + +def test_family_for_reason_none_is_none() -> None: + assert family_for_reason(None) is None