Merge pull request #645 from AssetOverflow/feat/r3-1-router-pr
feat(contemplation): R3.1 — wire R3 into the multi-organ router + pass manager
This commit is contained in:
commit
f1e7925444
11 changed files with 223 additions and 9 deletions
|
|
@ -6,7 +6,7 @@ over — uniform across the R1 and R2 setup compilers. Off-serving; imports no `
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from core.comprehension_attempt.classify import classify_r1, classify_r2
|
||||
from core.comprehension_attempt.classify import classify_r1, classify_r2, classify_r3
|
||||
from core.comprehension_attempt.failure_family import (
|
||||
REGISTRY,
|
||||
FailureFamily,
|
||||
|
|
@ -35,6 +35,7 @@ __all__ = [
|
|||
"RouteStatus",
|
||||
"classify_r1",
|
||||
"classify_r2",
|
||||
"classify_r3",
|
||||
"enrich_family",
|
||||
"family_by_name",
|
||||
"family_for_reason",
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ from generate.constraint_comprehension.model import ConstraintProblem
|
|||
from generate.constraint_comprehension.reader import read_constraint_problem
|
||||
from generate.meaning_graph.reader import Refusal
|
||||
from generate.quantitative_comprehension import comprehend_quantitative, to_relational_metric
|
||||
from generate.rate_comprehension.model import RateProblem
|
||||
from generate.rate_comprehension.reader import read_rate_problem
|
||||
|
||||
|
||||
def _r1_signature(relations: list[dict[str, Any]]) -> str:
|
||||
|
|
@ -82,4 +84,29 @@ def classify_r2(text: str, *, case_id: str | None = None) -> ComprehensionAttemp
|
|||
)
|
||||
|
||||
|
||||
__all__ = ["classify_r1", "classify_r2"]
|
||||
def _r3_signature(problem: RateProblem) -> str:
|
||||
"""Deterministic string signature of an R3 single-rate setup."""
|
||||
return repr(
|
||||
(
|
||||
(problem.rate_unit.numerator, problem.rate_unit.denominator),
|
||||
("rate", problem.rate),
|
||||
("time", problem.time),
|
||||
("quantity", problem.quantity),
|
||||
problem.query,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def classify_r3(text: str, *, case_id: str | None = None) -> ComprehensionAttempt:
|
||||
"""Attempt the R3 single-rate setup compiler on *text*."""
|
||||
problem = read_rate_problem(text)
|
||||
if isinstance(problem, Refusal):
|
||||
return ComprehensionAttempt(
|
||||
"r3_rate", "setup_refused", case_id=case_id, refusal_reason=problem.reason
|
||||
)
|
||||
return ComprehensionAttempt(
|
||||
"r3_rate", "setup_correct", case_id=case_id, setup_signature=_r3_signature(problem)
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["classify_r1", "classify_r2", "classify_r3"]
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ REGISTRY: tuple[FailureFamily, ...] = (
|
|||
"empty", "no_quantity_template", "non_digit_quantity", "non_identifier_name",
|
||||
"unreadable_quantity_query", "invalid_binding_graph", "query_target_not_a_category",
|
||||
"unprojectable", "category_pair_not_found", "query_target_unrecognized", "no_query",
|
||||
"not_rate_shaped",
|
||||
),
|
||||
),
|
||||
FailureFamily(
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from typing import Literal
|
|||
|
||||
from generate.binding_graph.model import SourceSpanLink
|
||||
|
||||
Organ = Literal["r1_quantitative", "r2_constraints"]
|
||||
Organ = Literal["r1_quantitative", "r2_constraints", "r3_rate"]
|
||||
|
||||
Outcome = Literal[
|
||||
"setup_correct", # an admissible setup was produced (produce-mode) / matches gold (eval-mode)
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from __future__ import annotations
|
|||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
|
||||
from core.comprehension_attempt.classify import classify_r1, classify_r2
|
||||
from core.comprehension_attempt.classify import classify_r1, classify_r2, classify_r3
|
||||
from core.comprehension_attempt.model import ComprehensionAttempt
|
||||
|
||||
RouteStatus = Literal["routed", "all_refused", "ambiguous"]
|
||||
|
|
@ -40,7 +40,11 @@ class RouteResult:
|
|||
|
||||
def route_setup(text: str, *, case_id: str | None = None) -> RouteResult:
|
||||
"""Route *text* to the single organ that admits an honest setup, or refuse."""
|
||||
attempts = (classify_r1(text, case_id=case_id), classify_r2(text, case_id=case_id))
|
||||
attempts = (
|
||||
classify_r1(text, case_id=case_id),
|
||||
classify_r2(text, case_id=case_id),
|
||||
classify_r3(text, case_id=case_id),
|
||||
)
|
||||
correct = tuple(a for a in attempts if a.is_setup_correct)
|
||||
if len(correct) == 1:
|
||||
return RouteResult(attempts, correct[0], "routed")
|
||||
|
|
|
|||
54
docs/analysis/r3-1-router-integration-2026-06-08.md
Normal file
54
docs/analysis/r3-1-router-integration-2026-06-08.md
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
# R3.1 — R3 wired into the contemplation router (ledger)
|
||||
|
||||
**As of:** R3.1, on `main @ a320c69a` + `feat/r3-1-router`.
|
||||
|
||||
R3.1 completes the growth loop for the single-rate organ: R3 is now a first-class organ in the
|
||||
multi-organ router and contemplation pass, so a rate problem the reader can't yet handle is
|
||||
classified and (for genuine rate gaps) emits a proposal-only artifact — surfaced by the existing
|
||||
read-only `idle_tick` review. The chain is now whole:
|
||||
|
||||
```text
|
||||
R1 / R2 / R3 organs → route_setup → contemplation pass → proposal-only artifacts → proposal review → idle_tick visibility
|
||||
```
|
||||
|
||||
## What changed
|
||||
- `core/comprehension_attempt/model.py` — `Organ` gains `r3_rate`.
|
||||
- `core/comprehension_attempt/classify.py` — `classify_r3` normalizes the rate reader's output
|
||||
into a `ComprehensionAttempt` (deterministic rate signature).
|
||||
- `core/comprehension_attempt/router.py` — `route_setup` now tries R1, R2, **and R3**, with the
|
||||
same exactly-one-`setup_correct` rule (zero → refuse, ≥2 → ambiguous).
|
||||
- `generate/contemplation/pass_manager.py` — a routed `r3_rate` setup is solved + verified
|
||||
(`solve_rate` + the reused answer-choice verifier); `unsupported_temporal_state` is dropped from
|
||||
the unsupported-family set so `temporal_state` is a generic `REFUSED_KNOWN_BOUNDARY`.
|
||||
|
||||
## Terminal matrix (verified)
|
||||
|
||||
| R3 case | Terminal | Proposal? |
|
||||
|---|---|---|
|
||||
| supported single-rate (6 solved) | `SOLVED_VERIFIED` | — |
|
||||
| `rate_unit_mismatch` (minutes vs /hour) | `PROPOSAL_EMITTED` (`unsupported_rate_duration`) | ✅ |
|
||||
| `combined_rates` | `PROPOSAL_EMITTED` (`unsupported_rate_duration`) | ✅ |
|
||||
| `non_integer_solution` (inverse) | `REFUSED_KNOWN_BOUNDARY` | — |
|
||||
| `missing_time` (underdetermined) | `REFUSED_KNOWN_BOUNDARY` | — |
|
||||
| `temporal_state` (clock time) | `REFUSED_KNOWN_BOUNDARY` | — |
|
||||
|
||||
Exactly **2** proposals over the rate gold — only the rate-like unsupported features
|
||||
(`rate_unit_mismatch`, `combined_rates`), never the boundaries.
|
||||
|
||||
## Anti-over-broad-refusal fix (a regression caught in R3.1)
|
||||
Adding R3 to the router initially **blocked an R2 proposal**: R3's reader returned `missing_rate`
|
||||
(a substantive boundary) on R2's `r2-011` (a non-rate problem), and boundary-first classification
|
||||
let that block R2's legitimate `missing_total_count` proposal. Fix: R3 claims `missing_rate` ONLY
|
||||
when rate-like structure (a duration) is present; otherwise it refuses `not_rate_shaped` →
|
||||
`input_shape` (not-my-domain). So **R3 never claims a substantive boundary on R1/R2 text** — the
|
||||
same discipline as the N6 `category_pair_not_found` fix. `r2-011` proposes again; R1/R2 unaffected.
|
||||
|
||||
## Invariants held
|
||||
R1 **7/0/3** · 15-case **15/0/0** · R2 reader **10/0/0** · R3 reader **8/0/0** (answers 6/0/6) ·
|
||||
serving unchanged · off-serving · 121-test smoke green incl. architectural invariants and both
|
||||
existing idle contract suites. No `AMBIGUOUS_ORGAN` on any gold (the three organs are exclusive).
|
||||
|
||||
## Next
|
||||
Per the agreed sequencing: **R3.2 — explicit unit conversion** (minutes↔hours, exact rational
|
||||
only, no floats), which directly closes the `rate_unit_mismatch` gap this loop now surfaces as a
|
||||
proposal. Combined rates / multi-agent composition come after.
|
||||
|
|
@ -40,6 +40,8 @@ from core.comprehension_attempt import (
|
|||
from generate.answer_choices.verify import verify_answer_choice
|
||||
from generate.constraint_comprehension.reader import read_constraint_problem
|
||||
from generate.constraint_comprehension.solver import answer_constraint_problem
|
||||
from generate.rate_comprehension.reader import read_rate_problem
|
||||
from generate.rate_comprehension.solver import solve_rate
|
||||
from generate.contemplation.findings import Finding, Terminal
|
||||
from generate.meaning_graph.reader import Refusal
|
||||
|
||||
|
|
@ -49,9 +51,10 @@ _UNSUPPORTED_FAMILIES = frozenset(
|
|||
"unsupported_system_size",
|
||||
"unsupported_clause_shape",
|
||||
"unsupported_rate_duration",
|
||||
"unsupported_temporal_state",
|
||||
}
|
||||
)
|
||||
# Note: ``unsupported_temporal_state`` is deliberately NOT here — its clock-marker detector can
|
||||
# fire on non-rate text, so it is a generic REFUSED_KNOWN_BOUNDARY, not a recognized-rate-capability.
|
||||
#: The non-substantive "this organ does not recognize the shape" family — never blocks a proposal.
|
||||
_NOT_MY_DOMAIN = "input_shape"
|
||||
|
||||
|
|
@ -105,6 +108,8 @@ def contemplate(
|
|||
assert route.selected is not None
|
||||
if route.selected.organ == "r2_constraints":
|
||||
return _solve_and_verify_r2(text, options, answer_key, findings, attempts)
|
||||
if route.selected.organ == "r3_rate":
|
||||
return _solve_and_verify_r3(text, options, answer_key, findings, attempts)
|
||||
findings.append(Finding("solve", "r1 admissible setup (numeric answer is the eval lane in v0)"))
|
||||
findings.append(Finding("terminal", Terminal.SOLVED_VERIFIED.value))
|
||||
return ContemplationResult(
|
||||
|
|
@ -158,6 +163,49 @@ def _solve_and_verify_r2(
|
|||
)
|
||||
|
||||
|
||||
def _solve_and_verify_r3(
|
||||
text: str,
|
||||
options: dict[str, Any] | None,
|
||||
answer_key: str | None,
|
||||
findings: list[Finding],
|
||||
attempts: tuple[ComprehensionAttempt, ...],
|
||||
) -> ContemplationResult:
|
||||
problem = read_rate_problem(text)
|
||||
assert not isinstance(problem, Refusal) # routed => the reader admitted a setup
|
||||
value = solve_rate(problem)
|
||||
if isinstance(value, Refusal):
|
||||
findings.append(Finding("solve", f"solver refused: {value.reason}"))
|
||||
findings.append(Finding("terminal", Terminal.REFUSED_KNOWN_BOUNDARY.value))
|
||||
return ContemplationResult(
|
||||
Terminal.REFUSED_KNOWN_BOUNDARY, tuple(findings), attempts,
|
||||
selected_organ="r3_rate", family=_family_name(value.reason),
|
||||
)
|
||||
if options is not None:
|
||||
verdict = verify_answer_choice(value, options, answer_key)
|
||||
if isinstance(verdict, Refusal):
|
||||
findings.append(Finding("verify", f"answer-choice refused: {verdict.reason}"))
|
||||
findings.append(Finding("terminal", Terminal.REFUSED_KNOWN_BOUNDARY.value))
|
||||
return ContemplationResult(
|
||||
Terminal.REFUSED_KNOWN_BOUNDARY, tuple(findings), attempts,
|
||||
selected_organ="r3_rate", answer=value,
|
||||
)
|
||||
if verdict.status == "contradiction":
|
||||
findings.append(Finding("verify", verdict.message))
|
||||
findings.append(Finding("terminal", Terminal.CONTRADICTION_DETECTED.value))
|
||||
return ContemplationResult(
|
||||
Terminal.CONTRADICTION_DETECTED, tuple(findings), attempts,
|
||||
selected_organ="r3_rate", answer=value,
|
||||
family="answer_key_contradiction", message=verdict.message,
|
||||
)
|
||||
findings.append(Finding("verify", verdict.message))
|
||||
findings.append(Finding("solve", f"value={value}"))
|
||||
findings.append(Finding("terminal", Terminal.SOLVED_VERIFIED.value))
|
||||
return ContemplationResult(
|
||||
Terminal.SOLVED_VERIFIED, tuple(findings), attempts,
|
||||
selected_organ="r3_rate", answer=value,
|
||||
)
|
||||
|
||||
|
||||
def _classify_all_refused(
|
||||
text: str,
|
||||
attempts: tuple[ComprehensionAttempt, ...],
|
||||
|
|
|
|||
|
|
@ -83,7 +83,12 @@ def read_rate_problem(text: str) -> RateProblem | Refusal:
|
|||
elif how_many:
|
||||
asked = _singular(how_many.group(1))
|
||||
if rate_value is None:
|
||||
return Refusal("missing_rate", "no rate given and rate is not the question")
|
||||
# No rate clause. This is a rate-underdetermined problem ONLY if rate-like structure
|
||||
# (a duration) is present; otherwise it simply is not a rate problem and must refuse as
|
||||
# not-my-domain (so R3 never claims a substantive boundary on R1/R2 text).
|
||||
if dur is not None:
|
||||
return Refusal("missing_rate", "a duration but no rate clause")
|
||||
return Refusal("not_rate_shaped", "no rate structure")
|
||||
rate_unit = RateUnit(num_unit, denom_unit)
|
||||
if asked == num_unit:
|
||||
query = "quantity"
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ ALL_REASONS = {
|
|||
"unparseable_option",
|
||||
# R3 rate reader
|
||||
"rate_unit_mismatch", "combined_rates", "missing_rate", "missing_time", "missing_quantity",
|
||||
"temporal_state", "query_target_unrecognized", "no_query",
|
||||
"temporal_state", "query_target_unrecognized", "no_query", "not_rate_shaped",
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
74
tests/test_r3_router_contemplation.py
Normal file
74
tests/test_r3_router_contemplation.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
"""Tests for wiring R3 into the contemplation router + pass manager (R3.1).
|
||||
|
||||
Pins the R3.1c matrix: supported R3 → SOLVED_VERIFIED; rate_unit_mismatch/combined → proposal-only
|
||||
unsupported_rate_duration; temporal_state/missing/non-integer → REFUSED_KNOWN_BOUNDARY (no proposal);
|
||||
R1/R2 unaffected; non-rate text never blocks an R2 proposal.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from core.comprehension_attempt import classify_r3, route_setup
|
||||
from evals.constraint_oracle.runner import _load_r2_gold
|
||||
from evals.rate_oracle.runner import _load_rate_gold
|
||||
from evals.setup_oracle.runner import _load_r1_gold
|
||||
from generate.contemplation import Terminal, contemplate
|
||||
|
||||
|
||||
def test_classify_r3_matches_gold_expect() -> None:
|
||||
for fx in _load_rate_gold():
|
||||
att = classify_r3(fx["text"], case_id=fx["id"])
|
||||
assert att.organ == "r3_rate"
|
||||
if fx["expect"] in ("solved", "solver_refuses"):
|
||||
assert att.outcome == "setup_correct" and att.setup_signature is not None
|
||||
else:
|
||||
assert att.outcome == "setup_refused" and att.refusal_reason == fx["reader_reason"]
|
||||
|
||||
|
||||
def test_router_routes_rate_to_r3_and_stays_exclusive() -> None:
|
||||
routed = 0
|
||||
for fx in _load_rate_gold():
|
||||
r = route_setup(fx["text"])
|
||||
assert len(r.attempts) == 3 and r.status != "ambiguous"
|
||||
if r.selected is not None:
|
||||
assert r.selected.organ == "r3_rate"
|
||||
routed += 1
|
||||
assert routed == 8 # 6 solved + 2 solver_refuses
|
||||
# adding R3 does not make any R1/R2 problem ambiguous, nor route it to r3
|
||||
for fx in _load_r1_gold() + _load_r2_gold():
|
||||
r = route_setup(fx["text"])
|
||||
assert r.status != "ambiguous"
|
||||
if r.selected is not None:
|
||||
assert r.selected.organ != "r3_rate"
|
||||
|
||||
|
||||
def _expected_r3_terminal(fx: dict) -> Terminal:
|
||||
if fx["expect"] == "solved":
|
||||
return Terminal.SOLVED_VERIFIED
|
||||
if fx["expect"] == "solver_refuses":
|
||||
return Terminal.REFUSED_KNOWN_BOUNDARY
|
||||
if fx["reader_reason"] in ("rate_unit_mismatch", "combined_rates"):
|
||||
return Terminal.PROPOSAL_EMITTED
|
||||
return Terminal.REFUSED_KNOWN_BOUNDARY # missing_time / temporal_state
|
||||
|
||||
|
||||
def test_contemplation_r3_terminals_and_only_rate_like_propose(tmp_path: Path) -> None:
|
||||
for fx in _load_rate_gold():
|
||||
kwargs = {"options": fx["options"], "answer_key": fx["answer"]} if fx["expect"] == "solved" else {}
|
||||
result = contemplate(fx["text"], proposal_root=tmp_path, case_id=fx["id"], **kwargs)
|
||||
assert result.terminal == _expected_r3_terminal(fx), f"{fx['id']}: {result.terminal}"
|
||||
if fx["expect"] == "solved":
|
||||
assert result.answer == fx["gold"]
|
||||
if result.terminal == Terminal.PROPOSAL_EMITTED:
|
||||
assert result.family == "unsupported_rate_duration"
|
||||
# ONLY the two rate-like unsupported features proposed; temporal/missing/non-integer did not.
|
||||
assert len(list(tmp_path.glob("*.json"))) == 2
|
||||
|
||||
|
||||
def test_non_rate_text_does_not_block_r2_proposal(tmp_path: Path) -> None:
|
||||
# r2-011 (missing_total_count) must still PROPOSAL_EMITTED — R3's not_rate_shaped refusal on
|
||||
# this non-rate text is input_shape (not-my-domain), never a substantive boundary.
|
||||
fx = next(f for f in _load_r2_gold() if f["id"] == "r2-011-missing-total-count")
|
||||
result = contemplate(fx["text"], proposal_root=tmp_path)
|
||||
assert result.terminal == Terminal.PROPOSAL_EMITTED and result.family == "missing_total_count"
|
||||
|
|
@ -49,4 +49,4 @@ def test_router_never_selects_a_refusal() -> None:
|
|||
result = route_setup(fx["text"])
|
||||
if result.selected is not None:
|
||||
assert result.selected.outcome == "setup_correct"
|
||||
assert len(result.attempts) == 2 # always one R1 + one R2 attempt
|
||||
assert len(result.attempts) == 3 # always one R1 + one R2 + one R3 attempt
|
||||
|
|
|
|||
Loading…
Reference in a new issue