feat(contemplation): wire CMB into multi-organ router and failure registry (CMB-d)
CMB (r4_combined_rate) now participates in route_setup and bounded contemplation. R1/R2/R3 readers/ solvers, the CMB reader/solver/oracle, and serving are all UNCHANGED — pure shared-infra integration, no new comprehension capability (that was CMB-c). - model.py: Organ += r4_combined_rate. - classify.py: classify_cmb + _cmb_signature (organ-tagged, cross-organ-distinct) + cmb_reason — namespaces CMB reasons cmb_* before the reason-string-keyed registry sees them, so a CMB boundary never inherits R2/R3's family for the SAME bare string (R3 rate_unit_mismatch is a GROWTH surface; R2/R3 non_integer_solution carry other owners). not_combined_rate_shaped/empty stay bare -> the cross input_shape family (router hygiene). - failure_family.py: Owner += r4; input_shape += not_combined_rate_shaped; 8 CMB families (must_remain_refused: cmb_unit_mismatch [until a dimension registry exists — NOT forever], cmb_combine_ambiguous, cmb_underdetermined, cmb_non_positive_net, cmb_non_integer; proposal_allowed -> cmb_gold_fixture: cmb_unsupported_rate_count/_reciprocal/_clock_interval). - pass_manager.py: _solve_and_verify_cmb (solver Refusal -> REFUSED_KNOWN_BOUNDARY, never a proposal). CMB-over-R3 domain-precedence rule (cmb_is_authoritative): when CMB POSITIVELY recognizes combined-rate shape (a setup, or a substantive cmb_* refusal — NOT the input_shape step-aside), R3's broader single-rate over-read of the SAME text is inadmissible — for BOTH routing (veto R3 setup_correct) and family attribution (suppress R3 so CMB's sharper diagnosis owns the terminal/proposal). This came from a pre-build cross-organ pressure-test: R3 reads 'Anna and Ben paint together; Anna paints 3 rooms/hour. How many do THEY paint in 4 hours?' as a single rate and would confidently answer a wrong 12. The rule fixes that (-> REFUSED) without touching R3. It does NOT mean 'CMB always wins': on input_shape CMB cedes (a plain single-rate car problem routes to R3 and solves 180). Narrow CMB<->R3 instance of a future general domain-specificity adjudication. Tests: new test_cmb_router_contemplation (full terminal matrix; cmb-11 veto-no-wrong-answer; cmb-15 cede-to-R3; solver-refusal-never-proposes; CMB-owned proposal-only artifacts mounted:false; no gold ambiguous; off-serving AST). Extended router-organ-hygiene to CMB x R1/R2/R3 (0 breaches). test_setup_router/test_r3_router_contemplation attempt-count 3->4; test_failure_family ALL_REASONS + growth set. 97 CMB-d tests; 597 broad-net pass (1 unrelated pre-existing red); CMB organ lanes green.
This commit is contained in:
Shay
parent
c441411c85
commit
6a75f5203a
11 changed files with 381 additions and 13 deletions
|
|
@ -6,7 +6,13 @@ over — uniform across the R1 and R2 setup compilers. Off-serving; imports no `
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from core.comprehension_attempt.classify import classify_r1, classify_r2, classify_r3
|
||||
from core.comprehension_attempt.classify import (
|
||||
classify_cmb,
|
||||
classify_r1,
|
||||
classify_r2,
|
||||
classify_r3,
|
||||
cmb_reason,
|
||||
)
|
||||
from core.comprehension_attempt.failure_family import (
|
||||
REGISTRY,
|
||||
FailureFamily,
|
||||
|
|
@ -20,7 +26,12 @@ from core.comprehension_attempt.proposal import (
|
|||
build_proposal,
|
||||
emit_proposal,
|
||||
)
|
||||
from core.comprehension_attempt.router import RouteResult, RouteStatus, route_setup
|
||||
from core.comprehension_attempt.router import (
|
||||
RouteResult,
|
||||
RouteStatus,
|
||||
cmb_is_authoritative,
|
||||
route_setup,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"REGISTRY",
|
||||
|
|
@ -33,9 +44,12 @@ __all__ = [
|
|||
"Outcome",
|
||||
"RouteResult",
|
||||
"RouteStatus",
|
||||
"classify_cmb",
|
||||
"classify_r1",
|
||||
"classify_r2",
|
||||
"classify_r3",
|
||||
"cmb_is_authoritative",
|
||||
"cmb_reason",
|
||||
"enrich_family",
|
||||
"family_by_name",
|
||||
"family_for_reason",
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ from __future__ import annotations
|
|||
from typing import Any
|
||||
|
||||
from core.comprehension_attempt.model import ComprehensionAttempt
|
||||
from generate.combined_rate_comprehension.model import CombinedRateProblem
|
||||
from generate.combined_rate_comprehension.reader import read_combined_rate_problem
|
||||
from generate.constraint_comprehension.model import ConstraintProblem
|
||||
from generate.constraint_comprehension.reader import read_constraint_problem
|
||||
from generate.meaning_graph.reader import Refusal
|
||||
|
|
@ -19,6 +21,19 @@ from generate.quantitative_comprehension import comprehend_quantitative, to_rela
|
|||
from generate.rate_comprehension.model import RateProblem
|
||||
from generate.rate_comprehension.reader import read_rate_problem
|
||||
|
||||
#: CMB reader/solver reasons that are namespaced ``cmb_*`` before the (reason-string-keyed) failure
|
||||
#: registry sees them, so a CMB boundary never inherits R2/R3's family for the SAME bare string
|
||||
#: (R3 ``rate_unit_mismatch`` is a growth surface; R2/R3 ``non_integer_solution`` carry other
|
||||
#: owners). The two ``input_shape`` reasons stay bare so they map to the cross ``input_shape``
|
||||
#: family (router hygiene). A ``cmb_``-prefixed attempt reason is also the signal that CMB
|
||||
#: *substantively* recognized the text (used by the CMB-over-R3 precedence rule in the router).
|
||||
_CMB_BARE_REASONS = frozenset({"not_combined_rate_shaped", "empty"})
|
||||
|
||||
|
||||
def cmb_reason(reason: str) -> str:
|
||||
"""Namespace a CMB refusal reason for the failure registry (see :data:`_CMB_BARE_REASONS`)."""
|
||||
return reason if reason in _CMB_BARE_REASONS else f"cmb_{reason}"
|
||||
|
||||
|
||||
def _r1_signature(relations: list[dict[str, Any]]) -> str:
|
||||
"""Deterministic, order-independent string signature of the projected R1 relations."""
|
||||
|
|
@ -109,4 +124,36 @@ def classify_r3(text: str, *, case_id: str | None = None) -> ComprehensionAttemp
|
|||
)
|
||||
|
||||
|
||||
__all__ = ["classify_r1", "classify_r2", "classify_r3"]
|
||||
def _cmb_signature(problem: CombinedRateProblem) -> str:
|
||||
"""Deterministic string signature of an R4 combined-rate setup. The leading ``"cmb"`` tag
|
||||
guarantees it never coincides with an R1/R2/R3 signature (cross-organ distinctness). ``sum`` is
|
||||
commutative, so its two rates are sorted; ``difference`` keeps order (which rate is the drain)."""
|
||||
rates = (problem.rate_a, problem.rate_b)
|
||||
if problem.combine_mode == "sum":
|
||||
rates = tuple(sorted(rates))
|
||||
return repr(
|
||||
(
|
||||
"cmb",
|
||||
problem.combine_mode,
|
||||
(problem.rate_unit.numerator, problem.rate_unit.denominator),
|
||||
rates,
|
||||
("time", problem.time),
|
||||
("quantity", problem.quantity),
|
||||
problem.query,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def classify_cmb(text: str, *, case_id: str | None = None) -> ComprehensionAttempt:
|
||||
"""Attempt the R4 combined-rate setup compiler on *text* (refusal reasons namespaced ``cmb_*``)."""
|
||||
problem = read_combined_rate_problem(text)
|
||||
if isinstance(problem, Refusal):
|
||||
return ComprehensionAttempt(
|
||||
"r4_combined_rate", "setup_refused", case_id=case_id, refusal_reason=cmb_reason(problem.reason)
|
||||
)
|
||||
return ComprehensionAttempt(
|
||||
"r4_combined_rate", "setup_correct", case_id=case_id, setup_signature=_cmb_signature(problem)
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["classify_cmb", "classify_r1", "classify_r2", "classify_r3", "cmb_reason"]
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ from typing import Literal
|
|||
|
||||
from core.comprehension_attempt.model import ComprehensionAttempt
|
||||
|
||||
Owner = Literal["r1", "r2", "r3", "cross"]
|
||||
Owner = Literal["r1", "r2", "r3", "r4", "cross"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -55,7 +55,7 @@ REGISTRY: tuple[FailureFamily, ...] = (
|
|||
"empty", "no_quantity_template", "non_digit_quantity", "non_identifier_name",
|
||||
"unreadable_quantity_query", "invalid_binding_graph", "query_target_not_a_category",
|
||||
"unprojectable", "category_pair_not_found", "query_target_unrecognized", "no_query",
|
||||
"not_rate_shaped",
|
||||
"not_rate_shaped", "not_combined_rate_shaped",
|
||||
),
|
||||
),
|
||||
FailureFamily(
|
||||
|
|
@ -161,6 +161,54 @@ REGISTRY: tuple[FailureFamily, ...] = (
|
|||
"fire on non-rate text, so temporal_state is not reliably rate-like.",
|
||||
refusal_reasons=("temporal_state",),
|
||||
),
|
||||
# --- R4 combined-rate organ (reasons namespaced cmb_*; CMB owns them over R3's same-string ---- #
|
||||
# reasons via the CMB-over-R3 precedence rule in the router) ------------------------------ #
|
||||
FailureFamily(
|
||||
"cmb_unit_mismatch", "r4", True, False,
|
||||
"refuse — the two rates' units are incompatible. must_remain_refused UNTIL a dimension "
|
||||
"registry exists: CMB v1 cannot distinguish a convertible pair (gallons/min vs gallons/hour) "
|
||||
"from a dimensionally-incompatible one (rooms/hour vs liters/minute), so a 'try conversion' "
|
||||
"proposal would be wrong=0-unsafe. NOT 'forever impossible' — a convertible split is future work.",
|
||||
refusal_reasons=("cmb_rate_unit_mismatch",),
|
||||
),
|
||||
FailureFamily(
|
||||
"cmb_combine_ambiguous", "r4", True, False,
|
||||
"refuse — two same-unit rates but no licensed sum/difference cue. An ambiguity, not a "
|
||||
"coverage gap: no fixture can teach which mode the text intends.",
|
||||
refusal_reasons=("cmb_combine_mode_ambiguous",),
|
||||
),
|
||||
FailureFamily(
|
||||
"cmb_underdetermined", "r4", True, False,
|
||||
"refuse — combined-shaped but a contributing rate is unstated (under-specified input); no "
|
||||
"reader enhancement can infer the missing rate.",
|
||||
refusal_reasons=("cmb_missing_second_rate",),
|
||||
),
|
||||
FailureFamily(
|
||||
"cmb_non_positive_net", "r4", True, False,
|
||||
"refuse — non-positive net rate; a quantity/time query cannot make progress (solver boundary).",
|
||||
refusal_reasons=("cmb_non_positive_net_rate",),
|
||||
),
|
||||
FailureFamily(
|
||||
"cmb_non_integer", "r4", True, False,
|
||||
"refuse — no exact integer answer; never round (solver boundary, exact-integer v1).",
|
||||
refusal_reasons=("cmb_non_integer_solution",),
|
||||
),
|
||||
# --- R4 growth surfaces (proposal allowed) — emitted ONLY after positive combined recognition -- #
|
||||
FailureFamily(
|
||||
"cmb_unsupported_rate_count", "r4", False, True,
|
||||
"propose a combined-rate fixture for ≥3 contributing rates (future capability)",
|
||||
proposal_target="cmb_gold_fixture", refusal_reasons=("cmb_three_or_more_rates",),
|
||||
),
|
||||
FailureFamily(
|
||||
"cmb_unsupported_reciprocal", "r4", False, True,
|
||||
"propose a reciprocal work-rate fixture (1/(1/a+1/b); reciprocal rates + rational arithmetic)",
|
||||
proposal_target="cmb_gold_fixture", refusal_reasons=("cmb_reciprocal_work_rate_deferred",),
|
||||
),
|
||||
FailureFamily(
|
||||
"cmb_unsupported_clock_interval", "r4", False, True,
|
||||
"propose a clock-interval fixture (elapsed-clock-time duration in a combined-rate problem)",
|
||||
proposal_target="cmb_gold_fixture", refusal_reasons=("cmb_clock_interval_deferred",),
|
||||
),
|
||||
)
|
||||
|
||||
_BY_NAME: dict[str, FailureFamily] = {f.name: f for f in REGISTRY}
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from typing import Literal
|
|||
|
||||
from generate.binding_graph.model import SourceSpanLink
|
||||
|
||||
Organ = Literal["r1_quantitative", "r2_constraints", "r3_rate"]
|
||||
Organ = Literal["r1_quantitative", "r2_constraints", "r3_rate", "r4_combined_rate"]
|
||||
|
||||
Outcome = Literal[
|
||||
"setup_correct", # an admissible setup was produced (produce-mode) / matches gold (eval-mode)
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from __future__ import annotations
|
|||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
|
||||
from core.comprehension_attempt.classify import classify_r1, classify_r2, classify_r3
|
||||
from core.comprehension_attempt.classify import classify_cmb, classify_r1, classify_r2, classify_r3
|
||||
from core.comprehension_attempt.model import ComprehensionAttempt
|
||||
|
||||
RouteStatus = Literal["routed", "all_refused", "ambiguous"]
|
||||
|
|
@ -38,14 +38,33 @@ class RouteResult:
|
|||
status: RouteStatus
|
||||
|
||||
|
||||
def cmb_is_authoritative(attempts: tuple[ComprehensionAttempt, ...]) -> bool:
|
||||
"""True when the R4 combined-rate organ **positively** recognized combined-rate shape — a setup,
|
||||
or a substantive ``cmb_*`` refusal (NOT the bare ``not_combined_rate_shaped`` / ``empty``
|
||||
step-aside). When it does, R3's broader single-rate read of the SAME text is inadmissible: a more
|
||||
specific positive recognition beats a broader partial one ("Anna and Ben paint together; Anna
|
||||
paints 3 rooms/hour" must not be answered as a single-rate 12). This is the narrow CMB↔R3
|
||||
instance of a future general domain-specificity adjudication; it does NOT mean "CMB always wins"
|
||||
(on ``input_shape`` CMB cedes to R3, e.g. a plain single-rate car problem)."""
|
||||
cmb = next((a for a in attempts if a.organ == "r4_combined_rate"), None)
|
||||
if cmb is None:
|
||||
return False
|
||||
return cmb.is_setup_correct or (cmb.refusal_reason or "").startswith("cmb_")
|
||||
|
||||
|
||||
def route_setup(text: str, *, case_id: str | None = None) -> RouteResult:
|
||||
"""Route *text* to the single organ that admits an honest setup, or refuse."""
|
||||
attempts = (
|
||||
classify_r1(text, case_id=case_id),
|
||||
classify_r2(text, case_id=case_id),
|
||||
classify_r3(text, case_id=case_id),
|
||||
classify_cmb(text, case_id=case_id),
|
||||
)
|
||||
# CMB-over-R3 domain precedence: a substantive CMB recognition vetoes R3's single-rate over-read.
|
||||
vetoed = cmb_is_authoritative(attempts)
|
||||
correct = tuple(
|
||||
a for a in attempts if a.is_setup_correct and not (vetoed and a.organ == "r3_rate")
|
||||
)
|
||||
correct = tuple(a for a in attempts if a.is_setup_correct)
|
||||
if len(correct) == 1:
|
||||
return RouteResult(attempts, correct[0], "routed")
|
||||
if not correct:
|
||||
|
|
@ -55,4 +74,4 @@ def route_setup(text: str, *, case_id: str | None = None) -> RouteResult:
|
|||
return RouteResult(attempts, None, "ambiguous")
|
||||
|
||||
|
||||
__all__ = ["RouteResult", "RouteStatus", "route_setup"]
|
||||
__all__ = ["RouteResult", "RouteStatus", "cmb_is_authoritative", "route_setup"]
|
||||
|
|
|
|||
|
|
@ -32,12 +32,16 @@ from typing import Any
|
|||
|
||||
from core.comprehension_attempt import (
|
||||
ComprehensionAttempt,
|
||||
cmb_is_authoritative,
|
||||
cmb_reason,
|
||||
emit_proposal,
|
||||
enrich_family,
|
||||
family_for_reason,
|
||||
route_setup,
|
||||
)
|
||||
from generate.answer_choices.verify import verify_answer_choice
|
||||
from generate.combined_rate_comprehension.reader import read_combined_rate_problem
|
||||
from generate.combined_rate_comprehension.solver import solve_combined_rate
|
||||
from generate.constraint_comprehension.reader import read_constraint_problem
|
||||
from generate.constraint_comprehension.solver import answer_constraint_problem
|
||||
from generate.rate_comprehension.reader import read_rate_problem
|
||||
|
|
@ -110,6 +114,8 @@ def contemplate(
|
|||
return _solve_and_verify_r2(text, options, answer_key, findings, attempts)
|
||||
if route.selected.organ == "r3_rate":
|
||||
return _solve_and_verify_r3(text, options, answer_key, findings, attempts)
|
||||
if route.selected.organ == "r4_combined_rate":
|
||||
return _solve_and_verify_cmb(text, options, answer_key, findings, attempts)
|
||||
findings.append(Finding("solve", "r1 admissible setup (numeric answer is the eval lane in v0)"))
|
||||
findings.append(Finding("terminal", Terminal.SOLVED_VERIFIED.value))
|
||||
return ContemplationResult(
|
||||
|
|
@ -206,13 +212,65 @@ def _solve_and_verify_r3(
|
|||
)
|
||||
|
||||
|
||||
def _solve_and_verify_cmb(
|
||||
text: str,
|
||||
options: dict[str, Any] | None,
|
||||
answer_key: str | None,
|
||||
findings: list[Finding],
|
||||
attempts: tuple[ComprehensionAttempt, ...],
|
||||
) -> ContemplationResult:
|
||||
problem = read_combined_rate_problem(text)
|
||||
assert not isinstance(problem, Refusal) # routed => the reader admitted a setup
|
||||
value = solve_combined_rate(problem)
|
||||
if isinstance(value, Refusal):
|
||||
# The prose WAS understood (reader setup_correct); the resulting math is outside v1's
|
||||
# answerable boundary. A solver refusal is a terminal boundary, never a proposal — and the
|
||||
# reason is namespaced cmb_* so it resolves to the CMB solver family, not R2/R3's.
|
||||
findings.append(Finding("solve", f"solver refused: {value.reason}"))
|
||||
findings.append(Finding("terminal", Terminal.REFUSED_KNOWN_BOUNDARY.value))
|
||||
return ContemplationResult(
|
||||
Terminal.REFUSED_KNOWN_BOUNDARY, tuple(findings), attempts,
|
||||
selected_organ="r4_combined_rate", family=_family_name(cmb_reason(value.reason)),
|
||||
)
|
||||
if options is not None:
|
||||
verdict = verify_answer_choice(value, options, answer_key)
|
||||
if isinstance(verdict, Refusal):
|
||||
findings.append(Finding("verify", f"answer-choice refused: {verdict.reason}"))
|
||||
findings.append(Finding("terminal", Terminal.REFUSED_KNOWN_BOUNDARY.value))
|
||||
return ContemplationResult(
|
||||
Terminal.REFUSED_KNOWN_BOUNDARY, tuple(findings), attempts,
|
||||
selected_organ="r4_combined_rate", answer=value,
|
||||
)
|
||||
if verdict.status == "contradiction":
|
||||
findings.append(Finding("verify", verdict.message))
|
||||
findings.append(Finding("terminal", Terminal.CONTRADICTION_DETECTED.value))
|
||||
return ContemplationResult(
|
||||
Terminal.CONTRADICTION_DETECTED, tuple(findings), attempts,
|
||||
selected_organ="r4_combined_rate", answer=value,
|
||||
family="answer_key_contradiction", message=verdict.message,
|
||||
)
|
||||
findings.append(Finding("verify", verdict.message))
|
||||
findings.append(Finding("solve", f"value={value}"))
|
||||
findings.append(Finding("terminal", Terminal.SOLVED_VERIFIED.value))
|
||||
return ContemplationResult(
|
||||
Terminal.SOLVED_VERIFIED, tuple(findings), attempts,
|
||||
selected_organ="r4_combined_rate", answer=value,
|
||||
)
|
||||
|
||||
|
||||
def _classify_all_refused(
|
||||
text: str,
|
||||
attempts: tuple[ComprehensionAttempt, ...],
|
||||
findings: list[Finding],
|
||||
proposal_root: Path | None,
|
||||
) -> ContemplationResult:
|
||||
families = [(a, family_for_reason(a.refusal_reason)) for a in attempts]
|
||||
# CMB-over-R3 precedence (family side): when CMB substantively recognized the text, R3's broader
|
||||
# partial classification is suppressed, so CMB's sharper diagnosis owns the terminal/proposal
|
||||
# (e.g. a reciprocal combined-rate text proposes a CMB fixture, not R3's rate_underdetermined).
|
||||
considered = attempts
|
||||
if cmb_is_authoritative(attempts):
|
||||
considered = tuple(a for a in attempts if a.organ != "r3_rate")
|
||||
families = [(a, family_for_reason(a.refusal_reason)) for a in considered]
|
||||
|
||||
# Boundary-first: a substantive recognized boundary blocks any proposal.
|
||||
for _attempt, family in families:
|
||||
|
|
|
|||
165
tests/test_cmb_router_contemplation.py
Normal file
165
tests/test_cmb_router_contemplation.py
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
"""Wiring CMB (R4 combined-rate) into the multi-organ router + bounded contemplation (CMB-d).
|
||||
|
||||
Pins the integration matrix and the CMB-over-R3 domain-precedence rule:
|
||||
- supported CMB setup -> SOLVED_VERIFIED (organ r4)
|
||||
- CMB solver boundary -> REFUSED_KNOWN_BOUNDARY (no proposal)
|
||||
- CMB substantive reader refusal -> REFUSED_KNOWN_BOUNDARY (cmb_* family)
|
||||
- CMB deferred-capability refusal -> PROPOSAL_EMITTED (cmb_unsupported_*, owner r4, proposal-only)
|
||||
- cmb-11 (missing_second_rate) -> R3's single-rate over-read is VETOED; REFUSED, never a wrong 12
|
||||
- cmb-15 (genuine single-rate) -> CMB cedes (input_shape); R3 SOLVES 180
|
||||
- R1/R2/R3 gold unaffected; no gold routes ambiguous.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from core.comprehension_attempt import classify_cmb, cmb_reason, route_setup
|
||||
from evals.combined_rate_oracle.runner import _load_combined_rate_gold
|
||||
from evals.constraint_oracle.runner import _load_r2_gold
|
||||
from evals.rate_oracle.runner import _load_rate_gold
|
||||
from evals.setup_oracle.runner import _load_r1_gold
|
||||
from generate.contemplation import Terminal, contemplate
|
||||
|
||||
_REFUSE_FAMILY = { # CMB substantive reader refusals -> REFUSED_KNOWN_BOUNDARY via these families
|
||||
"rate_unit_mismatch": "cmb_unit_mismatch",
|
||||
"combine_mode_ambiguous": "cmb_combine_ambiguous",
|
||||
"missing_second_rate": "cmb_underdetermined",
|
||||
}
|
||||
_PROPOSAL_FAMILY = { # CMB deferred-capability refusals -> PROPOSAL_EMITTED via these families
|
||||
"three_or_more_rates": "cmb_unsupported_rate_count",
|
||||
"reciprocal_work_rate_deferred": "cmb_unsupported_reciprocal",
|
||||
"clock_interval_deferred": "cmb_unsupported_clock_interval",
|
||||
}
|
||||
|
||||
|
||||
def _by_id(fid: str) -> dict:
|
||||
return next(f for f in _load_combined_rate_gold() if f["id"] == fid)
|
||||
|
||||
|
||||
# --- classification + routing -------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_classify_cmb_matches_gold_with_namespaced_reasons() -> None:
|
||||
for fx in _load_combined_rate_gold():
|
||||
att = classify_cmb(fx["text"], case_id=fx["id"])
|
||||
assert att.organ == "r4_combined_rate"
|
||||
if fx["expect"] in ("solved", "solver_refuses"):
|
||||
assert att.outcome == "setup_correct" and att.setup_signature is not None
|
||||
else:
|
||||
assert att.outcome == "setup_refused"
|
||||
assert att.refusal_reason == cmb_reason(fx["reader_reason"])
|
||||
|
||||
|
||||
def test_router_routes_combined_to_cmb_vetoes_r3_overread_and_cedes_single_rate() -> None:
|
||||
routed_cmb = 0
|
||||
for fx in _load_combined_rate_gold():
|
||||
r = route_setup(fx["text"])
|
||||
assert len(r.attempts) == 4 and r.status != "ambiguous"
|
||||
if fx["expect"] in ("solved", "solver_refuses"):
|
||||
assert r.selected is not None and r.selected.organ == "r4_combined_rate", fx["id"]
|
||||
routed_cmb += 1
|
||||
assert routed_cmb == 11 # 6 solved + 5 solver_refuses
|
||||
# the veto: a missing-second-rate combined problem must NOT route to R3 (which would solve it wrong)
|
||||
assert route_setup(_by_id("cmb-11-missing-second-rate")["text"]).selected is None
|
||||
# the cede: a genuine single-rate problem CMB stepped aside on routes to R3
|
||||
assert route_setup(_by_id("cmb-15-not-combined-shaped")["text"]).selected.organ == "r3_rate"
|
||||
|
||||
|
||||
def test_r1_r2_r3_gold_not_stolen_by_cmb_and_never_ambiguous() -> None:
|
||||
for fx in _load_r1_gold() + _load_r2_gold() + _load_rate_gold():
|
||||
r = route_setup(fx["text"])
|
||||
assert r.status != "ambiguous", fx.get("id")
|
||||
if r.selected is not None:
|
||||
assert r.selected.organ != "r4_combined_rate", fx.get("id")
|
||||
|
||||
|
||||
# --- contemplation terminals --------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_cmb_contemplation_terminal_matrix(tmp_path: Path) -> None:
|
||||
for fx in _load_combined_rate_gold():
|
||||
kw = {"options": fx["options"], "answer_key": fx["answer"]} if fx["expect"] == "solved" else {}
|
||||
r = contemplate(fx["text"], proposal_root=tmp_path, case_id=fx["id"], **kw)
|
||||
if fx["expect"] == "solved":
|
||||
assert r.terminal == Terminal.SOLVED_VERIFIED and r.selected_organ == "r4_combined_rate"
|
||||
assert r.answer == fx["gold"], fx["id"]
|
||||
elif fx["expect"] == "solver_refuses":
|
||||
assert r.terminal == Terminal.REFUSED_KNOWN_BOUNDARY and r.answer is None, fx["id"]
|
||||
expected = "cmb_non_positive_net" if fx["solver_reason"] == "non_positive_net_rate" else "cmb_non_integer"
|
||||
assert r.family == expected, fx["id"]
|
||||
else:
|
||||
reason = fx["reader_reason"]
|
||||
if reason in _REFUSE_FAMILY:
|
||||
assert r.terminal == Terminal.REFUSED_KNOWN_BOUNDARY and r.family == _REFUSE_FAMILY[reason], fx["id"]
|
||||
assert r.answer is None
|
||||
elif reason in _PROPOSAL_FAMILY:
|
||||
assert r.terminal == Terminal.PROPOSAL_EMITTED and r.family == _PROPOSAL_FAMILY[reason], fx["id"]
|
||||
|
||||
|
||||
def test_cmb11_veto_yields_refusal_not_a_wrong_answer(tmp_path: Path) -> None:
|
||||
r = contemplate(_by_id("cmb-11-missing-second-rate")["text"], proposal_root=tmp_path)
|
||||
assert r.terminal == Terminal.REFUSED_KNOWN_BOUNDARY
|
||||
assert r.family == "cmb_underdetermined" and r.answer is None
|
||||
assert not list(tmp_path.glob("*.json")) # an under-specified input is not a growth proposal
|
||||
|
||||
|
||||
def test_cmb15_cedes_to_r3_which_solves_the_single_rate(tmp_path: Path) -> None:
|
||||
r = contemplate(_by_id("cmb-15-not-combined-shaped")["text"], proposal_root=tmp_path)
|
||||
assert r.terminal == Terminal.SOLVED_VERIFIED and r.selected_organ == "r3_rate" and r.answer == 180
|
||||
|
||||
|
||||
def test_cmb_solver_boundaries_never_propose(tmp_path: Path) -> None:
|
||||
# non_positive_net_rate / non_integer_solution: prose understood, math outside v1 -> terminal
|
||||
# refusal, NEVER a growth proposal (protects against a future registry change).
|
||||
for fid in ("cmb-07-tank-non-positive-net", "cmb-08-paint-non-integer-time"):
|
||||
r = contemplate(_by_id(fid)["text"], proposal_root=tmp_path)
|
||||
assert r.terminal == Terminal.REFUSED_KNOWN_BOUNDARY and r.selected_organ == "r4_combined_rate"
|
||||
assert not list(tmp_path.glob("*.json"))
|
||||
|
||||
|
||||
def test_cmb_deferred_capabilities_emit_cmb_owned_proposal_only_artifacts(tmp_path: Path) -> None:
|
||||
from core.comprehension_attempt import family_by_name
|
||||
|
||||
for fid, family in (
|
||||
("cmb-12-three-rates", "cmb_unsupported_rate_count"),
|
||||
("cmb-13-reciprocal-work-rate", "cmb_unsupported_reciprocal"),
|
||||
("cmb-14-clock-interval", "cmb_unsupported_clock_interval"),
|
||||
):
|
||||
r = contemplate(_by_id(fid)["text"], proposal_root=tmp_path, case_id=fid)
|
||||
assert r.terminal == Terminal.PROPOSAL_EMITTED and r.family == family, fid
|
||||
fam = family_by_name(family)
|
||||
assert fam.owner == "r4" and fam.proposal_target == "cmb_gold_fixture" and not fam.must_remain_refused
|
||||
assert r.proposal_path is not None
|
||||
artifact = json.loads(Path(r.proposal_path).read_text())
|
||||
# proposal-only: never mounted, requires review (the N5 emitter contract).
|
||||
assert artifact.get("mounted") is False and artifact.get("requires_review") is True
|
||||
|
||||
|
||||
def test_cmb_wiring_modules_are_off_serving() -> None:
|
||||
# classify_cmb + the contemplation pass that runs it must import no serving path — substring
|
||||
# scans false-negative on docstring mentions, so check actual imports via AST.
|
||||
import ast
|
||||
|
||||
import core.comprehension_attempt.classify as classify_mod
|
||||
import generate.contemplation.pass_manager as pass_mod
|
||||
|
||||
forbidden = ("generate.derivation", "core.reliability_gate")
|
||||
for mod in (classify_mod, pass_mod):
|
||||
for node in ast.walk(ast.parse(Path(str(mod.__file__)).read_text(encoding="utf-8"))):
|
||||
names = (
|
||||
[a.name for a in node.names] if isinstance(node, ast.Import)
|
||||
else [node.module or ""] if isinstance(node, ast.ImportFrom)
|
||||
else []
|
||||
)
|
||||
for name in names:
|
||||
assert not any(name.startswith(t) for t in forbidden), f"{mod.__name__} imports {name}"
|
||||
|
||||
|
||||
def test_no_combined_rate_gold_routes_ambiguous() -> None:
|
||||
# AMBIGUOUS_ORGAN is a router invariant for two organs admitting with conflicting signatures. The
|
||||
# readers are mutually exclusive on shape (R3 refuses >=2 rates; CMB refuses <2), and signatures
|
||||
# are organ-tagged, so this never occurs naturally — proven here across the whole CMB corpus.
|
||||
for fx in _load_combined_rate_gold():
|
||||
assert route_setup(fx["text"]).status != "ambiguous", fx["id"]
|
||||
|
|
@ -38,6 +38,12 @@ ALL_REASONS = {
|
|||
# R3 rate reader
|
||||
"rate_unit_mismatch", "combined_rates", "missing_rate", "missing_time", "missing_quantity",
|
||||
"temporal_state", "query_target_unrecognized", "no_query", "not_rate_shaped",
|
||||
# R4 combined-rate (reader + solver reasons namespaced cmb_*; not_combined_rate_shaped is the
|
||||
# bare step-aside reason -> the cross input_shape family).
|
||||
"not_combined_rate_shaped",
|
||||
"cmb_rate_unit_mismatch", "cmb_combine_mode_ambiguous", "cmb_missing_second_rate",
|
||||
"cmb_three_or_more_rates", "cmb_reciprocal_work_rate_deferred", "cmb_clock_interval_deferred",
|
||||
"cmb_non_positive_net_rate", "cmb_non_integer_solution",
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -66,7 +72,10 @@ def test_only_precise_missing_totals_are_reachable_growth_surfaces() -> None:
|
|||
# Only the PRECISE R2 gaps are reachable growth surfaces. category_pair_not_found is too broad
|
||||
# (fires on any non-R2 text), so it maps to input_shape, and missing_category_pair is reserved.
|
||||
growth = {f.name for f in REGISTRY if f.proposal_allowed and f.refusal_reasons}
|
||||
assert growth == {"missing_total_count", "missing_weighted_total", "unsupported_rate_duration"}
|
||||
assert growth == {
|
||||
"missing_total_count", "missing_weighted_total", "unsupported_rate_duration",
|
||||
"cmb_unsupported_rate_count", "cmb_unsupported_reciprocal", "cmb_unsupported_clock_interval",
|
||||
}
|
||||
assert family_for_reason("category_pair_not_found").name == "input_shape"
|
||||
for f in REGISTRY:
|
||||
if f.proposal_allowed:
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ def test_router_routes_rate_to_r3_and_stays_exclusive() -> None:
|
|||
routed = 0
|
||||
for fx in _load_rate_gold():
|
||||
r = route_setup(fx["text"])
|
||||
assert len(r.attempts) == 3 and r.status != "ambiguous"
|
||||
assert len(r.attempts) == 4 and r.status != "ambiguous"
|
||||
if r.selected is not None:
|
||||
assert r.selected.organ == "r3_rate"
|
||||
routed += 1
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ to ``_GOLD`` and its classifier to ``_ORGANS``, and this test enforces the rule
|
|||
from __future__ import annotations
|
||||
|
||||
from core.comprehension_attempt import (
|
||||
classify_cmb,
|
||||
classify_r1,
|
||||
classify_r2,
|
||||
classify_r3,
|
||||
|
|
@ -27,10 +28,17 @@ from evals.constraint_oracle.runner import _load_r2_gold
|
|||
from evals.rate_oracle.runner import _load_rate_gold
|
||||
from evals.setup_oracle.runner import _load_r1_gold
|
||||
|
||||
# CMB (r4) is tested as an ORGAN against R1/R2/R3 foreign gold — it MUST step aside as input_shape.
|
||||
# It is deliberately NOT added to _GOLD: a CMB problem is not *foreign* to R3 — R3 genuinely
|
||||
# co-recognizes the rate clauses (refusing `combined_rates`, a growth surface), so the strict
|
||||
# "must be input_shape" rule does not apply in that direction. That asymmetry is governed instead by
|
||||
# the CMB-over-R3 domain-precedence rule (`cmb_is_authoritative`), verified end-to-end in
|
||||
# tests/test_cmb_router_contemplation.py.
|
||||
_ORGANS = {
|
||||
"r1_quantitative": classify_r1,
|
||||
"r2_constraints": classify_r2,
|
||||
"r3_rate": classify_r3,
|
||||
"r4_combined_rate": classify_cmb,
|
||||
}
|
||||
_GOLD = {
|
||||
"r1_quantitative": _load_r1_gold,
|
||||
|
|
|
|||
|
|
@ -49,4 +49,4 @@ def test_router_never_selects_a_refusal() -> None:
|
|||
result = route_setup(fx["text"])
|
||||
if result.selected is not None:
|
||||
assert result.selected.outcome == "setup_correct"
|
||||
assert len(result.attempts) == 3 # always one R1 + one R2 + one R3 attempt
|
||||
assert len(result.attempts) == 4 # always one R1 + R2 + R3 + R4(combined-rate) attempt
|
||||
|
|
|
|||
Loading…
Reference in a new issue