From 1adf4f5de53a64cf6b69cb48914168dcfa4279ae Mon Sep 17 00:00:00 2001 From: Shay Date: Sun, 7 Jun 2026 08:59:47 -0700 Subject: [PATCH] feat(contemplation): contemplation v0 pass manager (N6) + boundary-first growth fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit generate/contemplation/{findings,pass_manager}.py: a single bounded pass — route (N3) -> classify+enrich (N4) -> terminal -> maybe emit proposal-only (N5). No loops/daemon/L10. Seven terminals: SOLVED_VERIFIED / REFUSED_KNOWN_BOUNDARY / REFUSED_UNSUPPORTED_FAMILY / CONTRADICTION_DETECTED / PROPOSAL_EMITTED / AMBIGUOUS_ORGAN / NO_PROGRESS. R2 solved+verified end-to-end; R1 routed setup is SOLVED_VERIFIED (numeric answer stays the eval lane in v0). Hazard found+fixed: N6 exposed that category_pair_not_found fires on ANY non-R2 text, so mapping it to a growth surface made gibberish falsely PROPOSAL_EMITTED. Reclassified it to input_shape and made missing_category_pair reserved — only the PRECISE missing_total_count/missing_weighted_total remain reachable growth surfaces. Classification is boundary-first (input_shape non-blocking), so a problem one organ recognizes as a substantive boundary never proposes against the other's broad refusal. Acceptance proven: known correct refusal -> no proposal; unsupported gap -> proposal-only artifact (exactly the 2 missing_* gaps over the gold); answer-key contradiction -> CONTRADICTION_DETECTED; organ conflict -> AMBIGUOUS_ORGAN. 188 tests green incl. architectural invariants; R1 7/0/3 + R2 reader 10/0/0 unchanged; off-serving. --- core/comprehension_attempt/failure_family.py | 15 +- ...prehension-capability-ledger-2026-06-07.md | 19 +- generate/contemplation/__init__.py | 13 ++ generate/contemplation/findings.py | 34 +++ generate/contemplation/pass_manager.py | 200 ++++++++++++++++++ tests/test_comprehension_contemplation.py | 92 ++++++++ tests/test_failure_family.py | 9 +- 7 files changed, 368 insertions(+), 14 deletions(-) create mode 100644 generate/contemplation/__init__.py create mode 100644 generate/contemplation/findings.py create mode 100644 generate/contemplation/pass_manager.py create mode 100644 tests/test_comprehension_contemplation.py diff --git a/core/comprehension_attempt/failure_family.py b/core/comprehension_attempt/failure_family.py index b2f9fcc2..184a4daf 100644 --- a/core/comprehension_attempt/failure_family.py +++ b/core/comprehension_attempt/failure_family.py @@ -54,7 +54,7 @@ REGISTRY: tuple[FailureFamily, ...] = ( refusal_reasons=( "empty", "no_quantity_template", "non_digit_quantity", "non_identifier_name", "unreadable_quantity_query", "invalid_binding_graph", "query_target_not_a_category", - "unprojectable", + "unprojectable", "category_pair_not_found", ), ), FailureFamily( @@ -110,11 +110,6 @@ REGISTRY: tuple[FailureFamily, ...] = ( ), ), # --- growth surfaces (proposal allowed) ---------------------------------------------- # - FailureFamily( - "missing_category_pair", "r2", False, True, - "propose a category-pair gold fixture for review", - proposal_target="r2_gold_fixture", refusal_reasons=("category_pair_not_found",), - ), FailureFamily( "missing_total_count", "r2", False, True, "propose a total-count-constraint gold fixture for review", @@ -132,6 +127,14 @@ REGISTRY: tuple[FailureFamily, ...] = ( refusal_reasons=(), ), # --- reserved / forward-declared for R3 (no current emitter) ------------------------- # + FailureFamily( + "missing_category_pair", "r2", False, True, + "RESERVED — propose a category-pair fixture once the reader distinguishes a partial " + "(one-category) R2 problem from non-R2 text; the raw category_pair_not_found reason is " + "too broad to propose against safely (it fires on any non-R2 text), so it maps to " + "input_shape until that split exists", + proposal_target="r2_gold_fixture", + ), FailureFamily( "missing_attribute_coefficient", "r2", False, True, "RESERVED — propose an attribute-coefficient fixture (no emitter yet)", diff --git a/docs/analysis/comprehension-capability-ledger-2026-06-07.md b/docs/analysis/comprehension-capability-ledger-2026-06-07.md index ce4d596b..bc05bf02 100644 --- a/docs/analysis/comprehension-capability-ledger-2026-06-07.md +++ b/docs/analysis/comprehension-capability-ledger-2026-06-07.md @@ -73,7 +73,7 @@ N4 turns this into the executable registry; N5 emits proposals **only** for `pro | `unsupported_distractor_clause` | R1 `r1-10` `unreadable_quantity_clause` | r1 | **yes** | no (until same-unit target isolation) | | `unit_incompatible` | R1 `unit_mismatch`, R2 `coefficient_unit_mismatch` | cross | **yes** | no | | `unsupported_system_size` | R2 `too_many_categories` | r2 | **yes** | no (until ≥3-var solver, R3) | -| `missing_category_pair` | R2 `category_pair_not_found` | r2 | no | **yes** (propose category-pair fixture) | +| `missing_category_pair` | *(reserved — `category_pair_not_found` is too broad; see note)* | r2 | no | **yes** (reserved) | | `missing_attribute_coefficient` | *(reserved — no emitter yet)* | r2 | no | **yes** | | `missing_total_count` | R2 `missing_total_count` | r2 | no | **yes** (propose count-constraint fixture) | | `missing_weighted_total` | R2 `missing_weighted_total` | r2 | no | **yes** (propose weighted-total fixture) | @@ -81,11 +81,20 @@ N4 turns this into the executable registry; N5 emits proposals **only** for `pro | `non_integer_solution` | R2 solver | r2 | **yes** | no | | `negative_solution` | R2 solver | r2 | **yes** | no | | `answer_key_contradiction` | R2 answer-choice `contradiction` verdict | r2 | n/a | no — **action: report contradiction** | -| `input_shape` | R1 `non_digit_quantity`/`non_identifier_name`/`no_quantity_template`; R2 `query_target_not_a_category` | cross | **yes** | no | +| `input_shape` | R1 `non_digit_quantity`/`non_identifier_name`/`no_quantity_template`/`unprojectable`; R2 `query_target_not_a_category`/`category_pair_not_found` | cross | **yes** | no | -**Reserved (forward-declared, no current emitter):** `missing_attribute_coefficient`, -`unsupported_rate_duration`, `unsupported_temporal_state` — named so the registry is complete, -but not reachable until R3 introduces rate/temporal frames. +> **N6 correction (boundary-first + precise growth).** `category_pair_not_found` fires on *any* +> non-R2 text (0 or 1 categories), so it is **not** a safe growth trigger — it maps to +> `input_shape` ("R2 does not recognize this"), and `missing_category_pair` is reserved until the +> reader distinguishes a partial one-category R2 problem from non-R2 prose. Only the **precise** +> R2 gaps (`missing_total_count`, `missing_weighted_total` — reachable only after two categories + +> matching coefficients are read) remain reachable growth surfaces. The contemplation pass (N6) +> classifies **boundary-first** and treats `input_shape` as non-blocking, so a problem one organ +> recognizes as a substantive boundary never proposes against the other organ's broad refusal. + +**Reserved (forward-declared, no current emitter):** `missing_category_pair`, +`missing_attribute_coefficient`, `unsupported_rate_duration`, `unsupported_temporal_state` — named +so the registry is complete, but not reachable until the reader/R3 supplies a precise signal. ## 7. Next contemplation batch (N2–N6) diff --git a/generate/contemplation/__init__.py b/generate/contemplation/__init__.py new file mode 100644 index 00000000..503e098e --- /dev/null +++ b/generate/contemplation/__init__.py @@ -0,0 +1,13 @@ +"""Contemplation v0 (N6) — a single bounded read -> classify -> terminal -> maybe-emit pass. + +Off-serving growth organ over the R1/R2 setup compilers: it turns an unsolved problem into a +typed terminal state and, for genuine coverage gaps only, a proposal-only artifact (N5). No loops, +no daemon, no L10 runtime, no self-modification. +""" + +from __future__ import annotations + +from generate.contemplation.findings import Finding, Terminal +from generate.contemplation.pass_manager import ContemplationResult, contemplate + +__all__ = ["ContemplationResult", "Finding", "Terminal", "contemplate"] diff --git a/generate/contemplation/findings.py b/generate/contemplation/findings.py new file mode 100644 index 00000000..a773a333 --- /dev/null +++ b/generate/contemplation/findings.py @@ -0,0 +1,34 @@ +"""Typed terminal states + per-pass findings for the contemplation pass manager (N6). + +v0 is a single bounded pass chain — no loops, no daemon, no L10 runtime. Each pass appends a +``Finding`` (a traceable artifact, not hidden chain-of-thought), and the chain ends in exactly +one ``Terminal`` state. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + + +class Terminal(str, Enum): + """The terminal state of one bounded contemplation pass.""" + + SOLVED_VERIFIED = "SOLVED_VERIFIED" + REFUSED_KNOWN_BOUNDARY = "REFUSED_KNOWN_BOUNDARY" + REFUSED_UNSUPPORTED_FAMILY = "REFUSED_UNSUPPORTED_FAMILY" + CONTRADICTION_DETECTED = "CONTRADICTION_DETECTED" + PROPOSAL_EMITTED = "PROPOSAL_EMITTED" + AMBIGUOUS_ORGAN = "AMBIGUOUS_ORGAN" + NO_PROGRESS = "NO_PROGRESS" + + +@dataclass(frozen=True, slots=True) +class Finding: + """One pass's typed result — name of the pass and a short summary of what it observed.""" + + pass_name: str + summary: str + + +__all__ = ["Finding", "Terminal"] diff --git a/generate/contemplation/pass_manager.py b/generate/contemplation/pass_manager.py new file mode 100644 index 00000000..2b98be4e --- /dev/null +++ b/generate/contemplation/pass_manager.py @@ -0,0 +1,200 @@ +"""Contemplation v0 pass manager (N6) — a single bounded pass chain. + +```text +Pass 1 route the problem across the R1/R2 setup compilers (N3) +Pass 2 classify + enrich each attempt with its failure family (N4) +Pass 3 decide the terminal state +Pass 4 optionally emit a proposal-only artifact (N5) +``` + +No loops, no recursion, no background daemon, no L10 runtime — one bounded pass that ends in +exactly one `Terminal`. The recursive "reread with findings" loop is deferred until we have +proof the classifications are useful. + +The load-bearing rule: a problem that one organ recognizes as a **substantive boundary** (a +correct wrong=0 refusal) must NEVER generate a proposal merely because the *other* organ refused +with a growth reason. So classification is **boundary-first**, and `input_shape` ("this organ does +not recognize the shape") is treated as non-blocking, not as a boundary. Proposals are emitted +only for genuine growth-surface gaps with no substantive boundary in play — and even then only +proposal-only artifacts (N5), never a mounted change. + +Off-serving: imports the R1/R2 organs (`generate`) + the comprehension-attempt layer (`core`); +imports no `evals`, no `generate.derivation`, no `core.reliability_gate`. In v0 the R1 numeric +answer is the eval lane's domain — a routed R1 setup is `SOLVED_VERIFIED` (admissible, +forward-substitutable by construction); R2 is solved + verified end to end here. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from core.comprehension_attempt import ( + ComprehensionAttempt, + emit_proposal, + enrich_family, + family_for_reason, + route_setup, +) +from generate.answer_choices.verify import verify_answer_choice +from generate.constraint_comprehension.reader import read_constraint_problem +from generate.constraint_comprehension.solver import answer_constraint_problem +from generate.contemplation.findings import Finding, Terminal +from generate.meaning_graph.reader import Refusal + +#: Substantive boundaries that are *recognized-but-unsupported* capabilities (not hard errors). +_UNSUPPORTED_FAMILIES = frozenset( + { + "unsupported_system_size", + "unsupported_clause_shape", + "unsupported_rate_duration", + "unsupported_temporal_state", + } +) +#: The non-substantive "this organ does not recognize the shape" family — never blocks a proposal. +_NOT_MY_DOMAIN = "input_shape" + + +@dataclass(frozen=True, slots=True) +class ContemplationResult: + """The outcome of one bounded contemplation pass.""" + + terminal: Terminal + findings: tuple[Finding, ...] + attempts: tuple[ComprehensionAttempt, ...] + selected_organ: str | None = None + answer: int | None = None + family: str | None = None + proposal_path: str | None = None + message: str | None = None + + +def contemplate( + text: str, + *, + options: dict[str, Any] | None = None, + answer_key: str | None = None, + proposal_root: Path | None = None, + case_id: str | None = None, +) -> ContemplationResult: + """Run one bounded contemplation pass over *text*.""" + findings: list[Finding] = [] + + # Pass 1 — route. + route = route_setup(text, case_id=case_id) + findings.append(Finding("route", f"status={route.status}")) + + # Pass 2 — classify + enrich. + attempts = tuple(enrich_family(a) for a in route.attempts) + findings.append( + Finding( + "classify", + "; ".join( + f"{a.organ}:{a.outcome}:{a.family or a.refusal_reason or '-'}" for a in attempts + ), + ) + ) + + # Pass 3/4 — terminal (+ maybe emit). + if route.status == "ambiguous": + findings.append(Finding("terminal", Terminal.AMBIGUOUS_ORGAN.value)) + return ContemplationResult(Terminal.AMBIGUOUS_ORGAN, tuple(findings), attempts) + + if route.status == "routed": + assert route.selected is not None + if route.selected.organ == "r2_constraints": + return _solve_and_verify_r2(text, options, answer_key, findings, attempts) + findings.append(Finding("solve", "r1 admissible setup (numeric answer is the eval lane in v0)")) + findings.append(Finding("terminal", Terminal.SOLVED_VERIFIED.value)) + return ContemplationResult( + Terminal.SOLVED_VERIFIED, tuple(findings), attempts, selected_organ="r1_quantitative" + ) + + # route.status == "all_refused" + return _classify_all_refused(text, attempts, findings, proposal_root) + + +def _solve_and_verify_r2( + text: str, + options: dict[str, Any] | None, + answer_key: str | None, + findings: list[Finding], + attempts: tuple[ComprehensionAttempt, ...], +) -> ContemplationResult: + problem = read_constraint_problem(text) + assert not isinstance(problem, Refusal) # routed => the reader admitted a setup + value = answer_constraint_problem(problem) + if isinstance(value, Refusal): + findings.append(Finding("solve", f"solver refused: {value.reason}")) + findings.append(Finding("terminal", Terminal.REFUSED_KNOWN_BOUNDARY.value)) + return ContemplationResult( + Terminal.REFUSED_KNOWN_BOUNDARY, tuple(findings), attempts, + selected_organ="r2_constraints", family=_family_name(value.reason), + ) + if options is not None: + verdict = verify_answer_choice(value, options, answer_key, noun=problem.query.unit) + if isinstance(verdict, Refusal): + findings.append(Finding("verify", f"answer-choice refused: {verdict.reason}")) + findings.append(Finding("terminal", Terminal.REFUSED_KNOWN_BOUNDARY.value)) + return ContemplationResult( + Terminal.REFUSED_KNOWN_BOUNDARY, tuple(findings), attempts, + selected_organ="r2_constraints", answer=value, + ) + if verdict.status == "contradiction": + findings.append(Finding("verify", verdict.message)) + findings.append(Finding("terminal", Terminal.CONTRADICTION_DETECTED.value)) + return ContemplationResult( + Terminal.CONTRADICTION_DETECTED, tuple(findings), attempts, + selected_organ="r2_constraints", answer=value, + family="answer_key_contradiction", message=verdict.message, + ) + findings.append(Finding("verify", verdict.message)) + findings.append(Finding("solve", f"value={value}")) + findings.append(Finding("terminal", Terminal.SOLVED_VERIFIED.value)) + return ContemplationResult( + Terminal.SOLVED_VERIFIED, tuple(findings), attempts, + selected_organ="r2_constraints", answer=value, + ) + + +def _classify_all_refused( + text: str, + attempts: tuple[ComprehensionAttempt, ...], + findings: list[Finding], + proposal_root: Path | None, +) -> ContemplationResult: + families = [(a, family_for_reason(a.refusal_reason)) for a in attempts] + + # Boundary-first: a substantive recognized boundary blocks any proposal. + for _attempt, family in families: + if family is not None and family.must_remain_refused and family.name != _NOT_MY_DOMAIN: + terminal = ( + Terminal.REFUSED_UNSUPPORTED_FAMILY + if family.name in _UNSUPPORTED_FAMILIES + else Terminal.REFUSED_KNOWN_BOUNDARY + ) + findings.append(Finding("terminal", f"{terminal.value} via {family.name}")) + return ContemplationResult(terminal, tuple(findings), attempts, family=family.name) + + # No substantive boundary: a genuine growth surface may emit a proposal-only artifact. + for _attempt, family in families: + if family is not None and family.proposal_allowed: + path = emit_proposal(text, family, attempts, root=proposal_root) + findings.append(Finding("propose", f"emitted proposal-only {family.name}")) + findings.append(Finding("terminal", Terminal.PROPOSAL_EMITTED.value)) + return ContemplationResult( + Terminal.PROPOSAL_EMITTED, tuple(findings), attempts, + family=family.name, proposal_path=str(path) if path else None, + ) + + findings.append(Finding("terminal", Terminal.NO_PROGRESS.value)) + return ContemplationResult(Terminal.NO_PROGRESS, tuple(findings), attempts) + + +def _family_name(reason: str | None) -> str | None: + family = family_for_reason(reason) + return family.name if family is not None else None + + +__all__ = ["ContemplationResult", "contemplate"] diff --git a/tests/test_comprehension_contemplation.py b/tests/test_comprehension_contemplation.py new file mode 100644 index 00000000..a121877e --- /dev/null +++ b/tests/test_comprehension_contemplation.py @@ -0,0 +1,92 @@ +"""Tests for the comprehension contemplation v0 pass manager (N6). + +(Distinct from ADR-0056's teaching contemplation loop in ``tests/test_contemplation.py`` — this +is the comprehension-failure pass over the R1/R2 setup compilers, ``generate/contemplation/``.) + +Drives every terminal state over both gold corpora and pins the batch acceptance criteria: + + known correct refusal -> no proposal + unsupported capability gap -> proposal-only artifact + answer-key contradiction -> contradiction terminal + multiple organ conflict -> refusal (AMBIGUOUS_ORGAN) + +Proposals are written to a tmp root so the repo is never touched. +""" + +from __future__ import annotations + +from pathlib import Path + +from core.comprehension_attempt import ComprehensionAttempt, RouteResult +from evals.constraint_oracle.runner import _load_r2_gold +from evals.setup_oracle.runner import _load_r1_gold +from generate.contemplation import Terminal, contemplate +from generate.contemplation import pass_manager + + +def _expected_r2_terminal(fx: dict) -> Terminal: + if fx["expect"] == "solved": + return Terminal.SOLVED_VERIFIED + if fx["expect"] == "solver_refuses": + return Terminal.REFUSED_KNOWN_BOUNDARY + if fx["reader_reason"] == "too_many_categories": + return Terminal.REFUSED_UNSUPPORTED_FAMILY + return Terminal.PROPOSAL_EMITTED # missing_total_count / missing_weighted_total + + +def test_r2_gold_terminals_and_only_gaps_propose(tmp_path: Path) -> None: + for fx in _load_r2_gold(): + kwargs = {} + if fx["expect"] == "solved": + kwargs = {"options": fx["options"], "answer_key": fx["answer"]} + result = contemplate(fx["text"], proposal_root=tmp_path, case_id=fx["id"], **kwargs) + assert result.terminal == _expected_r2_terminal(fx), f"{fx['id']}: {result.terminal}" + if fx["expect"] == "solved": + assert result.answer == fx["gold"] + # ONLY the two missing_* gaps emitted a proposal — never a correct boundary. + proposals = list(tmp_path.glob("*.json")) + assert len(proposals) == 2, [p.name for p in proposals] + + +def test_r1_gold_terminals_emit_no_proposals(tmp_path: Path) -> None: + expected = { + "r1-08-ambiguous-referent": Terminal.REFUSED_UNSUPPORTED_FAMILY, + "r1-09-missing-base": Terminal.REFUSED_KNOWN_BOUNDARY, + "r1-10-distractor": Terminal.REFUSED_UNSUPPORTED_FAMILY, + } + for fx in _load_r1_gold(): + result = contemplate(fx["text"], proposal_root=tmp_path, case_id=fx["id"]) + assert result.terminal == expected.get(fx["id"], Terminal.SOLVED_VERIFIED), fx["id"] + assert list(tmp_path.glob("*.json")) == [] # no proposal for any correct R1 refusal + + +def test_answer_key_contradiction_is_a_terminal() -> None: + fx = next(f for f in _load_r2_gold() if f["id"] == "r2-002-chickens") # gold 11 == option A + result = contemplate(fx["text"], options=fx["options"], answer_key="D") # D = 13 (wrong) + assert result.terminal == Terminal.CONTRADICTION_DETECTED + assert result.answer == 11 and result.family == "answer_key_contradiction" + assert "contradicts" in result.message + + +def test_solved_setup_with_no_options_still_solves() -> None: + fx = next(f for f in _load_r2_gold() if f["id"] == "r2-001-buses") + result = contemplate(fx["text"]) # no options -> solve without choice verification + assert result.terminal == Terminal.SOLVED_VERIFIED and result.answer == fx["gold"] + + +def test_ambiguous_organ_terminal(monkeypatch) -> None: + # Both organs admitting a setup is incomparable -> the router refuses; the pass surfaces it. + a1 = ComprehensionAttempt("r1_quantitative", "setup_correct", setup_signature="x") + a2 = ComprehensionAttempt("r2_constraints", "setup_correct", setup_signature="y") + monkeypatch.setattr( + pass_manager, "route_setup", + lambda text, case_id=None: RouteResult((a1, a2), None, "ambiguous"), + ) + result = contemplate("anything") + assert result.terminal == Terminal.AMBIGUOUS_ORGAN + + +def test_unrecognized_text_makes_no_progress(tmp_path: Path) -> None: + result = contemplate("The weather is nice today.", proposal_root=tmp_path) + assert result.terminal == Terminal.NO_PROGRESS + assert list(tmp_path.glob("*.json")) == [] diff --git a/tests/test_failure_family.py b/tests/test_failure_family.py index 874e6e4a..22668fd9 100644 --- a/tests/test_failure_family.py +++ b/tests/test_failure_family.py @@ -59,9 +59,12 @@ def test_every_gold_refusal_reason_maps_to_a_family() -> None: assert family_for_reason(att.refusal_reason) is not None, att.refusal_reason -def test_only_three_missing_families_are_growth_surfaces() -> None: +def test_only_precise_missing_totals_are_reachable_growth_surfaces() -> None: + # Only the PRECISE R2 gaps are reachable growth surfaces. category_pair_not_found is too broad + # (fires on any non-R2 text), so it maps to input_shape, and missing_category_pair is reserved. growth = {f.name for f in REGISTRY if f.proposal_allowed and f.refusal_reasons} - assert growth == {"missing_category_pair", "missing_total_count", "missing_weighted_total"} + assert growth == {"missing_total_count", "missing_weighted_total"} + assert family_for_reason("category_pair_not_found").name == "input_shape" for f in REGISTRY: if f.proposal_allowed: assert not f.must_remain_refused # a growth surface is never a hard boundary @@ -76,7 +79,7 @@ def test_correct_boundaries_stay_refused_with_no_proposal() -> None: def test_growth_reasons_allow_proposals() -> None: - for reason in ("missing_total_count", "missing_weighted_total", "category_pair_not_found"): + for reason in ("missing_total_count", "missing_weighted_total"): fam = family_for_reason(reason) assert fam is not None and fam.proposal_allowed and not fam.must_remain_refused assert fam.proposal_target == "r2_gold_fixture"