From 73345e641e061445511cc38788cee0a3da334398 Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 8 Jun 2026 02:05:34 -0700 Subject: [PATCH] feat(contemplation): wire R3 into the multi-organ router + pass manager (R3.1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit classify_r3 normalizes the rate reader into a ComprehensionAttempt; route_setup now tries R1/R2/R3 (same exactly-one-setup_correct rule); the pass manager solves+verifies a routed r3_rate setup (solve_rate + reused answer-choice verifier); unsupported_temporal_state dropped from the unsupported-family set so temporal_state -> REFUSED_KNOWN_BOUNDARY. Organ gains r3_rate. Terminal matrix (verified): 6 solved->SOLVED_VERIFIED; rate_unit_mismatch+combined_rates->PROPOSAL_EMITTED(unsupported_rate_duration); non_integer/missing_time/temporal->REFUSED_KNOWN_BOUNDARY; exactly 2 proposals (the rate-like gaps only). REGRESSION CAUGHT+FIXED: R3's reader returned missing_rate (a substantive boundary) on non-rate R2 text, blocking r2-011's legitimate missing_total_count proposal via boundary-first. Fix: missing_rate only when a duration is present; else not_rate_shaped -> input_shape (not-my-domain) — same discipline as the N6 category_pair_not_found fix. r2-011 proposes again; R1/R2 unaffected. R1 7/0/3, 15-case 15/0/0, R2 10/0/0, R3 8/0/0 unchanged; off-serving; 121-test smoke green incl. invariants + idle contract suites. --- core/comprehension_attempt/__init__.py | 3 +- core/comprehension_attempt/classify.py | 29 +++++++- core/comprehension_attempt/failure_family.py | 1 + core/comprehension_attempt/model.py | 2 +- core/comprehension_attempt/router.py | 8 +- .../r3-1-router-integration-2026-06-08.md | 54 ++++++++++++++ generate/contemplation/pass_manager.py | 50 ++++++++++++- generate/rate_comprehension/reader.py | 7 +- tests/test_failure_family.py | 2 +- tests/test_r3_router_contemplation.py | 74 +++++++++++++++++++ tests/test_setup_router.py | 2 +- 11 files changed, 223 insertions(+), 9 deletions(-) create mode 100644 docs/analysis/r3-1-router-integration-2026-06-08.md create mode 100644 tests/test_r3_router_contemplation.py diff --git a/core/comprehension_attempt/__init__.py b/core/comprehension_attempt/__init__.py index 6c0b674c..bd7ea247 100644 --- a/core/comprehension_attempt/__init__.py +++ b/core/comprehension_attempt/__init__.py @@ -6,7 +6,7 @@ over — uniform across the R1 and R2 setup compilers. Off-serving; imports no ` from __future__ import annotations -from core.comprehension_attempt.classify import classify_r1, classify_r2 +from core.comprehension_attempt.classify import classify_r1, classify_r2, classify_r3 from core.comprehension_attempt.failure_family import ( REGISTRY, FailureFamily, @@ -35,6 +35,7 @@ __all__ = [ "RouteStatus", "classify_r1", "classify_r2", + "classify_r3", "enrich_family", "family_by_name", "family_for_reason", diff --git a/core/comprehension_attempt/classify.py b/core/comprehension_attempt/classify.py index f94d3d34..f0b66159 100644 --- a/core/comprehension_attempt/classify.py +++ b/core/comprehension_attempt/classify.py @@ -16,6 +16,8 @@ from generate.constraint_comprehension.model import ConstraintProblem from generate.constraint_comprehension.reader import read_constraint_problem from generate.meaning_graph.reader import Refusal from generate.quantitative_comprehension import comprehend_quantitative, to_relational_metric +from generate.rate_comprehension.model import RateProblem +from generate.rate_comprehension.reader import read_rate_problem def _r1_signature(relations: list[dict[str, Any]]) -> str: @@ -82,4 +84,29 @@ def classify_r2(text: str, *, case_id: str | None = None) -> ComprehensionAttemp ) -__all__ = ["classify_r1", "classify_r2"] +def _r3_signature(problem: RateProblem) -> str: + """Deterministic string signature of an R3 single-rate setup.""" + return repr( + ( + (problem.rate_unit.numerator, problem.rate_unit.denominator), + ("rate", problem.rate), + ("time", problem.time), + ("quantity", problem.quantity), + problem.query, + ) + ) + + +def classify_r3(text: str, *, case_id: str | None = None) -> ComprehensionAttempt: + """Attempt the R3 single-rate setup compiler on *text*.""" + problem = read_rate_problem(text) + if isinstance(problem, Refusal): + return ComprehensionAttempt( + "r3_rate", "setup_refused", case_id=case_id, refusal_reason=problem.reason + ) + return ComprehensionAttempt( + "r3_rate", "setup_correct", case_id=case_id, setup_signature=_r3_signature(problem) + ) + + +__all__ = ["classify_r1", "classify_r2", "classify_r3"] diff --git a/core/comprehension_attempt/failure_family.py b/core/comprehension_attempt/failure_family.py index 69336374..4430951d 100644 --- a/core/comprehension_attempt/failure_family.py +++ b/core/comprehension_attempt/failure_family.py @@ -55,6 +55,7 @@ REGISTRY: tuple[FailureFamily, ...] = ( "empty", "no_quantity_template", "non_digit_quantity", "non_identifier_name", "unreadable_quantity_query", "invalid_binding_graph", "query_target_not_a_category", "unprojectable", "category_pair_not_found", "query_target_unrecognized", "no_query", + "not_rate_shaped", ), ), FailureFamily( diff --git a/core/comprehension_attempt/model.py b/core/comprehension_attempt/model.py index 62e9224d..181fd8cc 100644 --- a/core/comprehension_attempt/model.py +++ b/core/comprehension_attempt/model.py @@ -22,7 +22,7 @@ from typing import Literal from generate.binding_graph.model import SourceSpanLink -Organ = Literal["r1_quantitative", "r2_constraints"] +Organ = Literal["r1_quantitative", "r2_constraints", "r3_rate"] Outcome = Literal[ "setup_correct", # an admissible setup was produced (produce-mode) / matches gold (eval-mode) diff --git a/core/comprehension_attempt/router.py b/core/comprehension_attempt/router.py index 4a7b78e8..cd843f33 100644 --- a/core/comprehension_attempt/router.py +++ b/core/comprehension_attempt/router.py @@ -22,7 +22,7 @@ from __future__ import annotations from dataclasses import dataclass from typing import Literal -from core.comprehension_attempt.classify import classify_r1, classify_r2 +from core.comprehension_attempt.classify import classify_r1, classify_r2, classify_r3 from core.comprehension_attempt.model import ComprehensionAttempt RouteStatus = Literal["routed", "all_refused", "ambiguous"] @@ -40,7 +40,11 @@ class RouteResult: def route_setup(text: str, *, case_id: str | None = None) -> RouteResult: """Route *text* to the single organ that admits an honest setup, or refuse.""" - attempts = (classify_r1(text, case_id=case_id), classify_r2(text, case_id=case_id)) + attempts = ( + classify_r1(text, case_id=case_id), + classify_r2(text, case_id=case_id), + classify_r3(text, case_id=case_id), + ) correct = tuple(a for a in attempts if a.is_setup_correct) if len(correct) == 1: return RouteResult(attempts, correct[0], "routed") diff --git a/docs/analysis/r3-1-router-integration-2026-06-08.md b/docs/analysis/r3-1-router-integration-2026-06-08.md new file mode 100644 index 00000000..dcb9cdfb --- /dev/null +++ b/docs/analysis/r3-1-router-integration-2026-06-08.md @@ -0,0 +1,54 @@ +# R3.1 — R3 wired into the contemplation router (ledger) + +**As of:** R3.1, on `main @ a320c69a` + `feat/r3-1-router`. + +R3.1 completes the growth loop for the single-rate organ: R3 is now a first-class organ in the +multi-organ router and contemplation pass, so a rate problem the reader can't yet handle is +classified and (for genuine rate gaps) emits a proposal-only artifact — surfaced by the existing +read-only `idle_tick` review. The chain is now whole: + +```text +R1 / R2 / R3 organs → route_setup → contemplation pass → proposal-only artifacts → proposal review → idle_tick visibility +``` + +## What changed +- `core/comprehension_attempt/model.py` — `Organ` gains `r3_rate`. +- `core/comprehension_attempt/classify.py` — `classify_r3` normalizes the rate reader's output + into a `ComprehensionAttempt` (deterministic rate signature). +- `core/comprehension_attempt/router.py` — `route_setup` now tries R1, R2, **and R3**, with the + same exactly-one-`setup_correct` rule (zero → refuse, ≥2 → ambiguous). +- `generate/contemplation/pass_manager.py` — a routed `r3_rate` setup is solved + verified + (`solve_rate` + the reused answer-choice verifier); `unsupported_temporal_state` is dropped from + the unsupported-family set so `temporal_state` is a generic `REFUSED_KNOWN_BOUNDARY`. + +## Terminal matrix (verified) + +| R3 case | Terminal | Proposal? | +|---|---|---| +| supported single-rate (6 solved) | `SOLVED_VERIFIED` | — | +| `rate_unit_mismatch` (minutes vs /hour) | `PROPOSAL_EMITTED` (`unsupported_rate_duration`) | ✅ | +| `combined_rates` | `PROPOSAL_EMITTED` (`unsupported_rate_duration`) | ✅ | +| `non_integer_solution` (inverse) | `REFUSED_KNOWN_BOUNDARY` | — | +| `missing_time` (underdetermined) | `REFUSED_KNOWN_BOUNDARY` | — | +| `temporal_state` (clock time) | `REFUSED_KNOWN_BOUNDARY` | — | + +Exactly **2** proposals over the rate gold — only the rate-like unsupported features +(`rate_unit_mismatch`, `combined_rates`), never the boundaries. + +## Anti-over-broad-refusal fix (a regression caught in R3.1) +Adding R3 to the router initially **blocked an R2 proposal**: R3's reader returned `missing_rate` +(a substantive boundary) on R2's `r2-011` (a non-rate problem), and boundary-first classification +let that block R2's legitimate `missing_total_count` proposal. Fix: R3 claims `missing_rate` ONLY +when rate-like structure (a duration) is present; otherwise it refuses `not_rate_shaped` → +`input_shape` (not-my-domain). So **R3 never claims a substantive boundary on R1/R2 text** — the +same discipline as the N6 `category_pair_not_found` fix. `r2-011` proposes again; R1/R2 unaffected. + +## Invariants held +R1 **7/0/3** · 15-case **15/0/0** · R2 reader **10/0/0** · R3 reader **8/0/0** (answers 6/0/6) · +serving unchanged · off-serving · 121-test smoke green incl. architectural invariants and both +existing idle contract suites. No `AMBIGUOUS_ORGAN` on any gold (the three organs are exclusive). + +## Next +Per the agreed sequencing: **R3.2 — explicit unit conversion** (minutes↔hours, exact rational +only, no floats), which directly closes the `rate_unit_mismatch` gap this loop now surfaces as a +proposal. Combined rates / multi-agent composition come after. diff --git a/generate/contemplation/pass_manager.py b/generate/contemplation/pass_manager.py index 2b98be4e..6dcb4067 100644 --- a/generate/contemplation/pass_manager.py +++ b/generate/contemplation/pass_manager.py @@ -40,6 +40,8 @@ from core.comprehension_attempt import ( from generate.answer_choices.verify import verify_answer_choice from generate.constraint_comprehension.reader import read_constraint_problem from generate.constraint_comprehension.solver import answer_constraint_problem +from generate.rate_comprehension.reader import read_rate_problem +from generate.rate_comprehension.solver import solve_rate from generate.contemplation.findings import Finding, Terminal from generate.meaning_graph.reader import Refusal @@ -49,9 +51,10 @@ _UNSUPPORTED_FAMILIES = frozenset( "unsupported_system_size", "unsupported_clause_shape", "unsupported_rate_duration", - "unsupported_temporal_state", } ) +# Note: ``unsupported_temporal_state`` is deliberately NOT here — its clock-marker detector can +# fire on non-rate text, so it is a generic REFUSED_KNOWN_BOUNDARY, not a recognized-rate-capability. #: The non-substantive "this organ does not recognize the shape" family — never blocks a proposal. _NOT_MY_DOMAIN = "input_shape" @@ -105,6 +108,8 @@ def contemplate( assert route.selected is not None if route.selected.organ == "r2_constraints": return _solve_and_verify_r2(text, options, answer_key, findings, attempts) + if route.selected.organ == "r3_rate": + return _solve_and_verify_r3(text, options, answer_key, findings, attempts) findings.append(Finding("solve", "r1 admissible setup (numeric answer is the eval lane in v0)")) findings.append(Finding("terminal", Terminal.SOLVED_VERIFIED.value)) return ContemplationResult( @@ -158,6 +163,49 @@ def _solve_and_verify_r2( ) +def _solve_and_verify_r3( + text: str, + options: dict[str, Any] | None, + answer_key: str | None, + findings: list[Finding], + attempts: tuple[ComprehensionAttempt, ...], +) -> ContemplationResult: + problem = read_rate_problem(text) + assert not isinstance(problem, Refusal) # routed => the reader admitted a setup + value = solve_rate(problem) + if isinstance(value, Refusal): + findings.append(Finding("solve", f"solver refused: {value.reason}")) + findings.append(Finding("terminal", Terminal.REFUSED_KNOWN_BOUNDARY.value)) + return ContemplationResult( + Terminal.REFUSED_KNOWN_BOUNDARY, tuple(findings), attempts, + selected_organ="r3_rate", family=_family_name(value.reason), + ) + if options is not None: + verdict = verify_answer_choice(value, options, answer_key) + if isinstance(verdict, Refusal): + findings.append(Finding("verify", f"answer-choice refused: {verdict.reason}")) + findings.append(Finding("terminal", Terminal.REFUSED_KNOWN_BOUNDARY.value)) + return ContemplationResult( + Terminal.REFUSED_KNOWN_BOUNDARY, tuple(findings), attempts, + selected_organ="r3_rate", answer=value, + ) + if verdict.status == "contradiction": + findings.append(Finding("verify", verdict.message)) + findings.append(Finding("terminal", Terminal.CONTRADICTION_DETECTED.value)) + return ContemplationResult( + Terminal.CONTRADICTION_DETECTED, tuple(findings), attempts, + selected_organ="r3_rate", answer=value, + family="answer_key_contradiction", message=verdict.message, + ) + findings.append(Finding("verify", verdict.message)) + findings.append(Finding("solve", f"value={value}")) + findings.append(Finding("terminal", Terminal.SOLVED_VERIFIED.value)) + return ContemplationResult( + Terminal.SOLVED_VERIFIED, tuple(findings), attempts, + selected_organ="r3_rate", answer=value, + ) + + def _classify_all_refused( text: str, attempts: tuple[ComprehensionAttempt, ...], diff --git a/generate/rate_comprehension/reader.py b/generate/rate_comprehension/reader.py index 8cc056b7..455be34f 100644 --- a/generate/rate_comprehension/reader.py +++ b/generate/rate_comprehension/reader.py @@ -83,7 +83,12 @@ def read_rate_problem(text: str) -> RateProblem | Refusal: elif how_many: asked = _singular(how_many.group(1)) if rate_value is None: - return Refusal("missing_rate", "no rate given and rate is not the question") + # No rate clause. This is a rate-underdetermined problem ONLY if rate-like structure + # (a duration) is present; otherwise it simply is not a rate problem and must refuse as + # not-my-domain (so R3 never claims a substantive boundary on R1/R2 text). + if dur is not None: + return Refusal("missing_rate", "a duration but no rate clause") + return Refusal("not_rate_shaped", "no rate structure") rate_unit = RateUnit(num_unit, denom_unit) if asked == num_unit: query = "quantity" diff --git a/tests/test_failure_family.py b/tests/test_failure_family.py index ae176708..8136a0b3 100644 --- a/tests/test_failure_family.py +++ b/tests/test_failure_family.py @@ -37,7 +37,7 @@ ALL_REASONS = { "unparseable_option", # R3 rate reader "rate_unit_mismatch", "combined_rates", "missing_rate", "missing_time", "missing_quantity", - "temporal_state", "query_target_unrecognized", "no_query", + "temporal_state", "query_target_unrecognized", "no_query", "not_rate_shaped", } diff --git a/tests/test_r3_router_contemplation.py b/tests/test_r3_router_contemplation.py new file mode 100644 index 00000000..7dfd93e7 --- /dev/null +++ b/tests/test_r3_router_contemplation.py @@ -0,0 +1,74 @@ +"""Tests for wiring R3 into the contemplation router + pass manager (R3.1). + +Pins the R3.1c matrix: supported R3 → SOLVED_VERIFIED; rate_unit_mismatch/combined → proposal-only +unsupported_rate_duration; temporal_state/missing/non-integer → REFUSED_KNOWN_BOUNDARY (no proposal); +R1/R2 unaffected; non-rate text never blocks an R2 proposal. +""" + +from __future__ import annotations + +from pathlib import Path + +from core.comprehension_attempt import classify_r3, route_setup +from evals.constraint_oracle.runner import _load_r2_gold +from evals.rate_oracle.runner import _load_rate_gold +from evals.setup_oracle.runner import _load_r1_gold +from generate.contemplation import Terminal, contemplate + + +def test_classify_r3_matches_gold_expect() -> None: + for fx in _load_rate_gold(): + att = classify_r3(fx["text"], case_id=fx["id"]) + assert att.organ == "r3_rate" + if fx["expect"] in ("solved", "solver_refuses"): + assert att.outcome == "setup_correct" and att.setup_signature is not None + else: + assert att.outcome == "setup_refused" and att.refusal_reason == fx["reader_reason"] + + +def test_router_routes_rate_to_r3_and_stays_exclusive() -> None: + routed = 0 + for fx in _load_rate_gold(): + r = route_setup(fx["text"]) + assert len(r.attempts) == 3 and r.status != "ambiguous" + if r.selected is not None: + assert r.selected.organ == "r3_rate" + routed += 1 + assert routed == 8 # 6 solved + 2 solver_refuses + # adding R3 does not make any R1/R2 problem ambiguous, nor route it to r3 + for fx in _load_r1_gold() + _load_r2_gold(): + r = route_setup(fx["text"]) + assert r.status != "ambiguous" + if r.selected is not None: + assert r.selected.organ != "r3_rate" + + +def _expected_r3_terminal(fx: dict) -> Terminal: + if fx["expect"] == "solved": + return Terminal.SOLVED_VERIFIED + if fx["expect"] == "solver_refuses": + return Terminal.REFUSED_KNOWN_BOUNDARY + if fx["reader_reason"] in ("rate_unit_mismatch", "combined_rates"): + return Terminal.PROPOSAL_EMITTED + return Terminal.REFUSED_KNOWN_BOUNDARY # missing_time / temporal_state + + +def test_contemplation_r3_terminals_and_only_rate_like_propose(tmp_path: Path) -> None: + for fx in _load_rate_gold(): + kwargs = {"options": fx["options"], "answer_key": fx["answer"]} if fx["expect"] == "solved" else {} + result = contemplate(fx["text"], proposal_root=tmp_path, case_id=fx["id"], **kwargs) + assert result.terminal == _expected_r3_terminal(fx), f"{fx['id']}: {result.terminal}" + if fx["expect"] == "solved": + assert result.answer == fx["gold"] + if result.terminal == Terminal.PROPOSAL_EMITTED: + assert result.family == "unsupported_rate_duration" + # ONLY the two rate-like unsupported features proposed; temporal/missing/non-integer did not. + assert len(list(tmp_path.glob("*.json"))) == 2 + + +def test_non_rate_text_does_not_block_r2_proposal(tmp_path: Path) -> None: + # r2-011 (missing_total_count) must still PROPOSAL_EMITTED — R3's not_rate_shaped refusal on + # this non-rate text is input_shape (not-my-domain), never a substantive boundary. + fx = next(f for f in _load_r2_gold() if f["id"] == "r2-011-missing-total-count") + result = contemplate(fx["text"], proposal_root=tmp_path) + assert result.terminal == Terminal.PROPOSAL_EMITTED and result.family == "missing_total_count" diff --git a/tests/test_setup_router.py b/tests/test_setup_router.py index 1f8d57d6..f28736ff 100644 --- a/tests/test_setup_router.py +++ b/tests/test_setup_router.py @@ -49,4 +49,4 @@ def test_router_never_selects_a_refusal() -> None: result = route_setup(fx["text"]) if result.selected is not None: assert result.selected.outcome == "setup_correct" - assert len(result.attempts) == 2 # always one R1 + one R2 attempt + assert len(result.attempts) == 3 # always one R1 + one R2 + one R3 attempt