Merge pull request #645 from AssetOverflow/feat/r3-1-router-pr

feat(contemplation): R3.1 — wire R3 into the multi-organ router + pass manager
This commit is contained in:
Shay 2026-06-08 05:23:49 -07:00 committed by GitHub
commit f1e7925444
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 223 additions and 9 deletions

View file

@ -6,7 +6,7 @@ over — uniform across the R1 and R2 setup compilers. Off-serving; imports no `
from __future__ import annotations
from core.comprehension_attempt.classify import classify_r1, classify_r2
from core.comprehension_attempt.classify import classify_r1, classify_r2, classify_r3
from core.comprehension_attempt.failure_family import (
REGISTRY,
FailureFamily,
@ -35,6 +35,7 @@ __all__ = [
"RouteStatus",
"classify_r1",
"classify_r2",
"classify_r3",
"enrich_family",
"family_by_name",
"family_for_reason",

View file

@ -16,6 +16,8 @@ from generate.constraint_comprehension.model import ConstraintProblem
from generate.constraint_comprehension.reader import read_constraint_problem
from generate.meaning_graph.reader import Refusal
from generate.quantitative_comprehension import comprehend_quantitative, to_relational_metric
from generate.rate_comprehension.model import RateProblem
from generate.rate_comprehension.reader import read_rate_problem
def _r1_signature(relations: list[dict[str, Any]]) -> str:
@ -82,4 +84,29 @@ def classify_r2(text: str, *, case_id: str | None = None) -> ComprehensionAttemp
)
__all__ = ["classify_r1", "classify_r2"]
def _r3_signature(problem: RateProblem) -> str:
"""Deterministic string signature of an R3 single-rate setup."""
return repr(
(
(problem.rate_unit.numerator, problem.rate_unit.denominator),
("rate", problem.rate),
("time", problem.time),
("quantity", problem.quantity),
problem.query,
)
)
def classify_r3(text: str, *, case_id: str | None = None) -> ComprehensionAttempt:
"""Attempt the R3 single-rate setup compiler on *text*."""
problem = read_rate_problem(text)
if isinstance(problem, Refusal):
return ComprehensionAttempt(
"r3_rate", "setup_refused", case_id=case_id, refusal_reason=problem.reason
)
return ComprehensionAttempt(
"r3_rate", "setup_correct", case_id=case_id, setup_signature=_r3_signature(problem)
)
__all__ = ["classify_r1", "classify_r2", "classify_r3"]

View file

@ -55,6 +55,7 @@ REGISTRY: tuple[FailureFamily, ...] = (
"empty", "no_quantity_template", "non_digit_quantity", "non_identifier_name",
"unreadable_quantity_query", "invalid_binding_graph", "query_target_not_a_category",
"unprojectable", "category_pair_not_found", "query_target_unrecognized", "no_query",
"not_rate_shaped",
),
),
FailureFamily(

View file

@ -22,7 +22,7 @@ from typing import Literal
from generate.binding_graph.model import SourceSpanLink
Organ = Literal["r1_quantitative", "r2_constraints"]
Organ = Literal["r1_quantitative", "r2_constraints", "r3_rate"]
Outcome = Literal[
"setup_correct", # an admissible setup was produced (produce-mode) / matches gold (eval-mode)

View file

@ -22,7 +22,7 @@ from __future__ import annotations
from dataclasses import dataclass
from typing import Literal
from core.comprehension_attempt.classify import classify_r1, classify_r2
from core.comprehension_attempt.classify import classify_r1, classify_r2, classify_r3
from core.comprehension_attempt.model import ComprehensionAttempt
RouteStatus = Literal["routed", "all_refused", "ambiguous"]
@ -40,7 +40,11 @@ class RouteResult:
def route_setup(text: str, *, case_id: str | None = None) -> RouteResult:
"""Route *text* to the single organ that admits an honest setup, or refuse."""
attempts = (classify_r1(text, case_id=case_id), classify_r2(text, case_id=case_id))
attempts = (
classify_r1(text, case_id=case_id),
classify_r2(text, case_id=case_id),
classify_r3(text, case_id=case_id),
)
correct = tuple(a for a in attempts if a.is_setup_correct)
if len(correct) == 1:
return RouteResult(attempts, correct[0], "routed")

View file

@ -0,0 +1,54 @@
# R3.1 — R3 wired into the contemplation router (ledger)
**As of:** R3.1, on `main @ a320c69a` + `feat/r3-1-router`.
R3.1 completes the growth loop for the single-rate organ: R3 is now a first-class organ in the
multi-organ router and contemplation pass, so a rate problem the reader can't yet handle is
classified and (for genuine rate gaps) emits a proposal-only artifact — surfaced by the existing
read-only `idle_tick` review. The chain is now whole:
```text
R1 / R2 / R3 organs → route_setup → contemplation pass → proposal-only artifacts → proposal review → idle_tick visibility
```
## What changed
- `core/comprehension_attempt/model.py``Organ` gains `r3_rate`.
- `core/comprehension_attempt/classify.py``classify_r3` normalizes the rate reader's output
into a `ComprehensionAttempt` (deterministic rate signature).
- `core/comprehension_attempt/router.py``route_setup` now tries R1, R2, **and R3**, with the
same exactly-one-`setup_correct` rule (zero → refuse, ≥2 → ambiguous).
- `generate/contemplation/pass_manager.py` — a routed `r3_rate` setup is solved + verified
(`solve_rate` + the reused answer-choice verifier); `unsupported_temporal_state` is dropped from
the unsupported-family set so `temporal_state` is a generic `REFUSED_KNOWN_BOUNDARY`.
## Terminal matrix (verified)
| R3 case | Terminal | Proposal? |
|---|---|---|
| supported single-rate (6 solved) | `SOLVED_VERIFIED` | — |
| `rate_unit_mismatch` (minutes vs /hour) | `PROPOSAL_EMITTED` (`unsupported_rate_duration`) | ✅ |
| `combined_rates` | `PROPOSAL_EMITTED` (`unsupported_rate_duration`) | ✅ |
| `non_integer_solution` (inverse) | `REFUSED_KNOWN_BOUNDARY` | — |
| `missing_time` (underdetermined) | `REFUSED_KNOWN_BOUNDARY` | — |
| `temporal_state` (clock time) | `REFUSED_KNOWN_BOUNDARY` | — |
Exactly **2** proposals over the rate gold — only the rate-like unsupported features
(`rate_unit_mismatch`, `combined_rates`), never the boundaries.
## Anti-over-broad-refusal fix (a regression caught in R3.1)
Adding R3 to the router initially **blocked an R2 proposal**: R3's reader returned `missing_rate`
(a substantive boundary) on R2's `r2-011` (a non-rate problem), and boundary-first classification
let that block R2's legitimate `missing_total_count` proposal. Fix: R3 claims `missing_rate` ONLY
when rate-like structure (a duration) is present; otherwise it refuses `not_rate_shaped`
`input_shape` (not-my-domain). So **R3 never claims a substantive boundary on R1/R2 text** — the
same discipline as the N6 `category_pair_not_found` fix. `r2-011` proposes again; R1/R2 unaffected.
## Invariants held
R1 **7/0/3** · 15-case **15/0/0** · R2 reader **10/0/0** · R3 reader **8/0/0** (answers 6/0/6) ·
serving unchanged · off-serving · 121-test smoke green incl. architectural invariants and both
existing idle contract suites. No `AMBIGUOUS_ORGAN` on any gold (the three organs are exclusive).
## Next
Per the agreed sequencing: **R3.2 — explicit unit conversion** (minutes↔hours, exact rational
only, no floats), which directly closes the `rate_unit_mismatch` gap this loop now surfaces as a
proposal. Combined rates / multi-agent composition come after.

View file

@ -40,6 +40,8 @@ from core.comprehension_attempt import (
from generate.answer_choices.verify import verify_answer_choice
from generate.constraint_comprehension.reader import read_constraint_problem
from generate.constraint_comprehension.solver import answer_constraint_problem
from generate.rate_comprehension.reader import read_rate_problem
from generate.rate_comprehension.solver import solve_rate
from generate.contemplation.findings import Finding, Terminal
from generate.meaning_graph.reader import Refusal
@ -49,9 +51,10 @@ _UNSUPPORTED_FAMILIES = frozenset(
"unsupported_system_size",
"unsupported_clause_shape",
"unsupported_rate_duration",
"unsupported_temporal_state",
}
)
# Note: ``unsupported_temporal_state`` is deliberately NOT here — its clock-marker detector can
# fire on non-rate text, so it is a generic REFUSED_KNOWN_BOUNDARY, not a recognized-rate-capability.
#: The non-substantive "this organ does not recognize the shape" family — never blocks a proposal.
_NOT_MY_DOMAIN = "input_shape"
@ -105,6 +108,8 @@ def contemplate(
assert route.selected is not None
if route.selected.organ == "r2_constraints":
return _solve_and_verify_r2(text, options, answer_key, findings, attempts)
if route.selected.organ == "r3_rate":
return _solve_and_verify_r3(text, options, answer_key, findings, attempts)
findings.append(Finding("solve", "r1 admissible setup (numeric answer is the eval lane in v0)"))
findings.append(Finding("terminal", Terminal.SOLVED_VERIFIED.value))
return ContemplationResult(
@ -158,6 +163,49 @@ def _solve_and_verify_r2(
)
def _solve_and_verify_r3(
text: str,
options: dict[str, Any] | None,
answer_key: str | None,
findings: list[Finding],
attempts: tuple[ComprehensionAttempt, ...],
) -> ContemplationResult:
problem = read_rate_problem(text)
assert not isinstance(problem, Refusal) # routed => the reader admitted a setup
value = solve_rate(problem)
if isinstance(value, Refusal):
findings.append(Finding("solve", f"solver refused: {value.reason}"))
findings.append(Finding("terminal", Terminal.REFUSED_KNOWN_BOUNDARY.value))
return ContemplationResult(
Terminal.REFUSED_KNOWN_BOUNDARY, tuple(findings), attempts,
selected_organ="r3_rate", family=_family_name(value.reason),
)
if options is not None:
verdict = verify_answer_choice(value, options, answer_key)
if isinstance(verdict, Refusal):
findings.append(Finding("verify", f"answer-choice refused: {verdict.reason}"))
findings.append(Finding("terminal", Terminal.REFUSED_KNOWN_BOUNDARY.value))
return ContemplationResult(
Terminal.REFUSED_KNOWN_BOUNDARY, tuple(findings), attempts,
selected_organ="r3_rate", answer=value,
)
if verdict.status == "contradiction":
findings.append(Finding("verify", verdict.message))
findings.append(Finding("terminal", Terminal.CONTRADICTION_DETECTED.value))
return ContemplationResult(
Terminal.CONTRADICTION_DETECTED, tuple(findings), attempts,
selected_organ="r3_rate", answer=value,
family="answer_key_contradiction", message=verdict.message,
)
findings.append(Finding("verify", verdict.message))
findings.append(Finding("solve", f"value={value}"))
findings.append(Finding("terminal", Terminal.SOLVED_VERIFIED.value))
return ContemplationResult(
Terminal.SOLVED_VERIFIED, tuple(findings), attempts,
selected_organ="r3_rate", answer=value,
)
def _classify_all_refused(
text: str,
attempts: tuple[ComprehensionAttempt, ...],

View file

@ -83,7 +83,12 @@ def read_rate_problem(text: str) -> RateProblem | Refusal:
elif how_many:
asked = _singular(how_many.group(1))
if rate_value is None:
return Refusal("missing_rate", "no rate given and rate is not the question")
# No rate clause. This is a rate-underdetermined problem ONLY if rate-like structure
# (a duration) is present; otherwise it simply is not a rate problem and must refuse as
# not-my-domain (so R3 never claims a substantive boundary on R1/R2 text).
if dur is not None:
return Refusal("missing_rate", "a duration but no rate clause")
return Refusal("not_rate_shaped", "no rate structure")
rate_unit = RateUnit(num_unit, denom_unit)
if asked == num_unit:
query = "quantity"

View file

@ -37,7 +37,7 @@ ALL_REASONS = {
"unparseable_option",
# R3 rate reader
"rate_unit_mismatch", "combined_rates", "missing_rate", "missing_time", "missing_quantity",
"temporal_state", "query_target_unrecognized", "no_query",
"temporal_state", "query_target_unrecognized", "no_query", "not_rate_shaped",
}

View file

@ -0,0 +1,74 @@
"""Tests for wiring R3 into the contemplation router + pass manager (R3.1).
Pins the R3.1c matrix: supported R3 SOLVED_VERIFIED; rate_unit_mismatch/combined proposal-only
unsupported_rate_duration; temporal_state/missing/non-integer REFUSED_KNOWN_BOUNDARY (no proposal);
R1/R2 unaffected; non-rate text never blocks an R2 proposal.
"""
from __future__ import annotations
from pathlib import Path
from core.comprehension_attempt import classify_r3, route_setup
from evals.constraint_oracle.runner import _load_r2_gold
from evals.rate_oracle.runner import _load_rate_gold
from evals.setup_oracle.runner import _load_r1_gold
from generate.contemplation import Terminal, contemplate
def test_classify_r3_matches_gold_expect() -> None:
for fx in _load_rate_gold():
att = classify_r3(fx["text"], case_id=fx["id"])
assert att.organ == "r3_rate"
if fx["expect"] in ("solved", "solver_refuses"):
assert att.outcome == "setup_correct" and att.setup_signature is not None
else:
assert att.outcome == "setup_refused" and att.refusal_reason == fx["reader_reason"]
def test_router_routes_rate_to_r3_and_stays_exclusive() -> None:
routed = 0
for fx in _load_rate_gold():
r = route_setup(fx["text"])
assert len(r.attempts) == 3 and r.status != "ambiguous"
if r.selected is not None:
assert r.selected.organ == "r3_rate"
routed += 1
assert routed == 8 # 6 solved + 2 solver_refuses
# adding R3 does not make any R1/R2 problem ambiguous, nor route it to r3
for fx in _load_r1_gold() + _load_r2_gold():
r = route_setup(fx["text"])
assert r.status != "ambiguous"
if r.selected is not None:
assert r.selected.organ != "r3_rate"
def _expected_r3_terminal(fx: dict) -> Terminal:
if fx["expect"] == "solved":
return Terminal.SOLVED_VERIFIED
if fx["expect"] == "solver_refuses":
return Terminal.REFUSED_KNOWN_BOUNDARY
if fx["reader_reason"] in ("rate_unit_mismatch", "combined_rates"):
return Terminal.PROPOSAL_EMITTED
return Terminal.REFUSED_KNOWN_BOUNDARY # missing_time / temporal_state
def test_contemplation_r3_terminals_and_only_rate_like_propose(tmp_path: Path) -> None:
for fx in _load_rate_gold():
kwargs = {"options": fx["options"], "answer_key": fx["answer"]} if fx["expect"] == "solved" else {}
result = contemplate(fx["text"], proposal_root=tmp_path, case_id=fx["id"], **kwargs)
assert result.terminal == _expected_r3_terminal(fx), f"{fx['id']}: {result.terminal}"
if fx["expect"] == "solved":
assert result.answer == fx["gold"]
if result.terminal == Terminal.PROPOSAL_EMITTED:
assert result.family == "unsupported_rate_duration"
# ONLY the two rate-like unsupported features proposed; temporal/missing/non-integer did not.
assert len(list(tmp_path.glob("*.json"))) == 2
def test_non_rate_text_does_not_block_r2_proposal(tmp_path: Path) -> None:
# r2-011 (missing_total_count) must still PROPOSAL_EMITTED — R3's not_rate_shaped refusal on
# this non-rate text is input_shape (not-my-domain), never a substantive boundary.
fx = next(f for f in _load_r2_gold() if f["id"] == "r2-011-missing-total-count")
result = contemplate(fx["text"], proposal_root=tmp_path)
assert result.terminal == Terminal.PROPOSAL_EMITTED and result.family == "missing_total_count"

View file

@ -49,4 +49,4 @@ def test_router_never_selects_a_refusal() -> None:
result = route_setup(fx["text"])
if result.selected is not None:
assert result.selected.outcome == "setup_correct"
assert len(result.attempts) == 2 # always one R1 + one R2 attempt
assert len(result.attempts) == 3 # always one R1 + one R2 + one R3 attempt