1adf4f5de5
generate/contemplation/{findings,pass_manager}.py: a single bounded pass — route (N3) -> classify+enrich (N4) -> terminal -> maybe emit proposal-only (N5). No loops/daemon/L10. Seven terminals: SOLVED_VERIFIED / REFUSED_KNOWN_BOUNDARY / REFUSED_UNSUPPORTED_FAMILY / CONTRADICTION_DETECTED / PROPOSAL_EMITTED / AMBIGUOUS_ORGAN / NO_PROGRESS. R2 solved+verified end-to-end; R1 routed setup is SOLVED_VERIFIED (numeric answer stays the eval lane in v0).
Hazard found+fixed: N6 exposed that category_pair_not_found fires on ANY non-R2 text, so mapping it to a growth surface made gibberish falsely PROPOSAL_EMITTED. Reclassified it to input_shape and made missing_category_pair reserved — only the PRECISE missing_total_count/missing_weighted_total remain reachable growth surfaces. Classification is boundary-first (input_shape non-blocking), so a problem one organ recognizes as a substantive boundary never proposes against the other's broad refusal.
Acceptance proven: known correct refusal -> no proposal; unsupported gap -> proposal-only artifact (exactly the 2 missing_* gaps over the gold); answer-key contradiction -> CONTRADICTION_DETECTED; organ conflict -> AMBIGUOUS_ORGAN. 188 tests green incl. architectural invariants; R1 7/0/3 + R2 reader 10/0/0 unchanged; off-serving.
102 lines
4.5 KiB
Python
102 lines
4.5 KiB
Python
"""Tests for the failure-family registry (N4).
|
|
|
|
Pins that the registry is a PARTITION (every reachable organ refusal reason maps to exactly
|
|
one family), that it covers the entire live refusal surface, that only the three R2 ``missing_*``
|
|
families are growth surfaces, and that correct wrong=0 boundaries stay refused with no proposal.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from core.comprehension_attempt import classify_r1, classify_r2
|
|
from core.comprehension_attempt.failure_family import (
|
|
ANSWER_KEY_CONTRADICTION,
|
|
REGISTRY,
|
|
enrich_family,
|
|
family_for_reason,
|
|
)
|
|
from evals.constraint_oracle.runner import _load_r2_gold
|
|
from evals.setup_oracle.runner import _load_r1_gold
|
|
|
|
#: The full live refusal surface (R1 reader/admissibility, classify, R2 reader/solver/choice).
|
|
ALL_REASONS = {
|
|
# R1
|
|
"empty", "no_quantity_template", "non_digit_quantity", "non_identifier_name",
|
|
"unreadable_quantity_query", "unreadable_quantity_clause", "no_single_quantity_query",
|
|
"admissibility_refused", "multiple_inverse_bases", "multiple_partitions",
|
|
"partition_query_mismatch", "partition_container_mismatch", "invalid_binding_graph",
|
|
"unprojectable",
|
|
# R2 reader
|
|
"too_many_categories", "missing_total_count", "missing_weighted_total",
|
|
"category_pair_not_found", "coefficient_unit_mismatch", "coefficient_conflict",
|
|
"query_target_not_a_category",
|
|
# R2 solver
|
|
"indistinguishable_weights", "non_integer_solution", "negative_solution",
|
|
"query_target_unsolved", "verification_failed",
|
|
# R2 answer-choice
|
|
"no_matching_option", "ambiguous_options", "no_options", "unknown_provided_label",
|
|
"unparseable_option",
|
|
}
|
|
|
|
|
|
def test_registry_is_a_partition() -> None:
|
|
seen: dict[str, str] = {}
|
|
for family in REGISTRY:
|
|
for reason in family.refusal_reasons:
|
|
assert reason not in seen, f"{reason} in both {seen.get(reason)} and {family.name}"
|
|
seen[reason] = family.name
|
|
|
|
|
|
def test_registry_covers_the_whole_refusal_surface() -> None:
|
|
for reason in ALL_REASONS:
|
|
assert family_for_reason(reason) is not None, f"unmapped reason: {reason}"
|
|
|
|
|
|
def test_every_gold_refusal_reason_maps_to_a_family() -> None:
|
|
attempts = [classify_r2(f["text"]) for f in _load_r2_gold()]
|
|
attempts += [classify_r1(f["text"]) for f in _load_r1_gold()]
|
|
for att in attempts:
|
|
if att.outcome == "setup_refused":
|
|
assert family_for_reason(att.refusal_reason) is not None, att.refusal_reason
|
|
|
|
|
|
def test_only_precise_missing_totals_are_reachable_growth_surfaces() -> None:
|
|
# Only the PRECISE R2 gaps are reachable growth surfaces. category_pair_not_found is too broad
|
|
# (fires on any non-R2 text), so it maps to input_shape, and missing_category_pair is reserved.
|
|
growth = {f.name for f in REGISTRY if f.proposal_allowed and f.refusal_reasons}
|
|
assert growth == {"missing_total_count", "missing_weighted_total"}
|
|
assert family_for_reason("category_pair_not_found").name == "input_shape"
|
|
for f in REGISTRY:
|
|
if f.proposal_allowed:
|
|
assert not f.must_remain_refused # a growth surface is never a hard boundary
|
|
|
|
|
|
def test_correct_boundaries_stay_refused_with_no_proposal() -> None:
|
|
for reason in ("too_many_categories", "non_integer_solution", "negative_solution",
|
|
"unreadable_quantity_clause", "indistinguishable_weights",
|
|
"coefficient_unit_mismatch", "admissibility_refused"):
|
|
fam = family_for_reason(reason)
|
|
assert fam is not None and fam.must_remain_refused and not fam.proposal_allowed, reason
|
|
|
|
|
|
def test_growth_reasons_allow_proposals() -> None:
|
|
for reason in ("missing_total_count", "missing_weighted_total"):
|
|
fam = family_for_reason(reason)
|
|
assert fam is not None and fam.proposal_allowed and not fam.must_remain_refused
|
|
assert fam.proposal_target == "r2_gold_fixture"
|
|
|
|
|
|
def test_enrich_sets_family_on_a_refused_attempt() -> None:
|
|
fx = next(f for f in _load_r2_gold() if f["expect"] == "reader_refuses")
|
|
enriched = enrich_family(classify_r2(fx["text"]))
|
|
assert enriched.family is not None
|
|
assert family_for_reason(enriched.refusal_reason).name == enriched.family
|
|
|
|
|
|
def test_contradiction_family_reports_and_never_proposes() -> None:
|
|
assert ANSWER_KEY_CONTRADICTION.name == "answer_key_contradiction"
|
|
assert not ANSWER_KEY_CONTRADICTION.proposal_allowed
|
|
assert "report" in ANSWER_KEY_CONTRADICTION.safe_next_action
|
|
|
|
|
|
def test_family_for_reason_none_is_none() -> None:
|
|
assert family_for_reason(None) is None
|