Merge pull request #628 from AssetOverflow/feat/contemplation-pack-a

core/comprehension_attempt/__init__.py

@@ -0,0 +1,12 @@
+"""Shared typed record of a comprehension organ's attempt at a problem (N2).
+
+The normalization layer the contemplation batch (N3 router, N4 registry, N6 pass manager) reasons
+over — uniform across the R1 and R2 setup compilers. Off-serving; imports no `evals`.
+"""
+
+from __future__ import annotations
+
+from core.comprehension_attempt.classify import classify_r1, classify_r2
+from core.comprehension_attempt.model import ComprehensionAttempt, Organ, Outcome
+
+__all__ = ["ComprehensionAttempt", "Organ", "Outcome", "classify_r1", "classify_r2"]

core/comprehension_attempt/classify.py

@@ -0,0 +1,85 @@
+"""Normalize R1/R2 organ output into a typed ``ComprehensionAttempt`` (N2, setup-level).
+
+`classify_r1` / `classify_r2` run their organ and report **produce-mode** setup outcomes:
+`setup_refused` (with the organ's typed reason) or `setup_correct` (an admissible setup was
+produced, with a deterministic signature). They do NOT solve, do NOT compare to gold, and import
+nothing from `evals` (signatures are computed inline) — keeping this a thin, dependency-light
+normalizer. Answer-level outcomes are reached downstream (N6) when the solver/verifier run.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from core.comprehension_attempt.model import ComprehensionAttempt
+from generate.constraint_comprehension.model import ConstraintProblem
+from generate.constraint_comprehension.reader import read_constraint_problem
+from generate.meaning_graph.reader import Refusal
+from generate.quantitative_comprehension import comprehend_quantitative, to_relational_metric
+
+
+def _r1_signature(relations: list[dict[str, Any]]) -> str:
+    """Deterministic, order-independent string signature of the projected R1 relations."""
+    items: list[tuple] = []
+    for r in relations:
+        kind = r["kind"]
+        if kind == "fact":
+            items.append((kind, r["entity"], int(r["value"])))
+        elif kind in ("more_than", "fewer_than"):
+            items.append((kind, r["entity"], r["ref"], int(r["delta"])))
+        elif kind == "times_as_many":
+            items.append((kind, r["entity"], r["ref"], int(r["factor"])))
+        elif kind == "divide_by":
+            items.append((kind, r["entity"], r["ref"], int(r["divisor"])))
+        elif kind == "sum_of":
+            items.append((kind, r["entity"], tuple(sorted(r["parts"]))))
+        else:  # pragma: no cover - defensive
+            items.append(("unhandled", kind, r.get("entity", "")))
+    return repr(tuple(sorted(items, key=repr)))
+
+
+def _r2_signature(problem: ConstraintProblem) -> str:
+    """Deterministic, order-independent string signature of an R2 ConstraintProblem setup."""
+    unknowns = tuple(sorted((u.symbol, u.unit, u.domain) for u in problem.unknowns))
+    constraints: list[tuple] = []
+    for c in problem.constraints:
+        merged: dict[str, int] = {}
+        for symbol, coeff in c.lhs.terms:
+            merged[symbol] = merged.get(symbol, 0) + coeff
+        terms = tuple(sorted((s, v) for s, v in merged.items() if v != 0))
+        constraints.append((terms, c.relation, c.rhs - c.lhs.constant))
+    query = (problem.query.symbol, problem.query.unit)
+    return repr((unknowns, tuple(sorted(constraints, key=repr)), query))
+
+
+def classify_r1(text: str, *, case_id: str | None = None) -> ComprehensionAttempt:
+    """Attempt the R1 relational-arithmetic setup compiler on *text*."""
+    comp = comprehend_quantitative(text)
+    if isinstance(comp, Refusal):
+        return ComprehensionAttempt(
+            "r1_quantitative", "setup_refused", case_id=case_id, refusal_reason=comp.reason
+        )
+    projected = to_relational_metric(comp)
+    if projected is None:
+        return ComprehensionAttempt(
+            "r1_quantitative", "setup_refused", case_id=case_id, refusal_reason="unprojectable"
+        )
+    relations, _query = projected
+    return ComprehensionAttempt(
+        "r1_quantitative", "setup_correct", case_id=case_id, setup_signature=_r1_signature(relations)
+    )
+
+
+def classify_r2(text: str, *, case_id: str | None = None) -> ComprehensionAttempt:
+    """Attempt the R2 two-category constraint setup compiler on *text*."""
+    problem = read_constraint_problem(text)
+    if isinstance(problem, Refusal):
+        return ComprehensionAttempt(
+            "r2_constraints", "setup_refused", case_id=case_id, refusal_reason=problem.reason
+        )
+    return ComprehensionAttempt(
+        "r2_constraints", "setup_correct", case_id=case_id, setup_signature=_r2_signature(problem)
+    )
+
+
+__all__ = ["classify_r1", "classify_r2"]

core/comprehension_attempt/model.py

@@ -0,0 +1,63 @@
+"""Typed, immutable record of one comprehension organ's attempt at a problem (N2).
+
+A small normalization layer over the R1 (`generate.quantitative_comprehension`) and R2
+(`generate.constraint_comprehension`) setup compilers: it turns each organ's heterogeneous
+output (a typed setup, or a typed `Refusal`) into one uniform, frozen `ComprehensionAttempt`.
+Nothing here changes reader behavior — it only *describes* an outcome so the router (N3),
+failure-family registry (N4), and contemplation pass manager (N6) can reason over both organs
+uniformly.
+
+Outcome semantics. `classify` (N2) produces **produce-mode** outcomes — what the organ did on
+its own gates, with no gold in hand: `setup_refused` (the organ refused) or `setup_correct`
+(an admissible setup was produced). The gold-relative outcomes (`setup_wrong`, `answer_wrong`)
+are representable here but are emitted only in **eval mode** by the lanes that hold gold — never
+fabricated by `classify`. `answer_*` / `contradiction` are reached when the solver / answer-choice
+verifier run downstream (N6), not at setup classification time.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+from generate.binding_graph.model import SourceSpanLink
+
+Organ = Literal["r1_quantitative", "r2_constraints"]
+
+Outcome = Literal[
+    "setup_correct",   # an admissible setup was produced (produce-mode) / matches gold (eval-mode)
+    "setup_refused",   # the organ refused to assemble a setup
+    "setup_wrong",     # eval-mode only: produced setup diverges from gold (a wrong=0 breach)
+    "answer_correct",  # a value was produced and self-verified / matches gold
+    "answer_refused",  # setup produced but the solver/verifier refused
+    "answer_wrong",    # eval-mode only: produced value diverges from gold
+    "contradiction",   # a verified value contradicts a supplied answer key
+]
+
+
+@dataclass(frozen=True, slots=True)
+class ComprehensionAttempt:
+    """One organ's attempt at one problem. Immutable; carries the outcome, the refusal reason
+    (if any), a deterministic setup signature (for cross-organ comparison), the answer (if a
+    value was produced), and source-span evidence. ``family`` is left ``None`` by ``classify``
+    and resolved later by the N4 failure-family registry."""
+
+    organ: Organ
+    outcome: Outcome
+    case_id: str | None = None
+    refusal_reason: str | None = None
+    family: str | None = None
+    setup_signature: str | None = None
+    answer: int | None = None
+    evidence: tuple[SourceSpanLink, ...] = ()
+
+    @property
+    def is_setup_correct(self) -> bool:
+        return self.outcome == "setup_correct"
+
+    @property
+    def is_refusal(self) -> bool:
+        return self.outcome in ("setup_refused", "answer_refused")
+
+
+__all__ = ["ComprehensionAttempt", "Organ", "Outcome"]

docs/analysis/comprehension-capability-ledger-2026-06-07.md

@@ -0,0 +1,109 @@
+# Comprehension capability ledger (N1)
+
+**As of:** merged `main @ 51b4a6e3` (R2 batch #624–#627 merged).
+
+This ledger answers, for the merged baseline, exactly six questions — so the contemplation
+loop (N2–N6) never mistakes a faithful refusal for a missing capability:
+
+```text
+What can CORE currently read?      -> §2, §4 (admitted families)
+What can it solve?                 -> §4 (R2 solver) + §2 (R1 answer lane)
+What does it refuse?               -> §3, §5 (remaining refusals)
+Which refusals are correct boundaries?   -> §6 (must_remain_refused = yes)
+Which refusals are growth surfaces?      -> §6 (proposal_allowed = yes)
+Which organ owns each refusal?           -> §6 (owner)
+```
+
+> **The load-bearing distinction, repeated on purpose:** `correct refusal ≠ missing
+> capability`. A refusal that is the wrong=0 boundary working (a pronoun with no referent, a
+> system with no integer solution) must **stay refused**. The learning loop may only propose
+> against refusals that are genuine coverage gaps (§6). N4 encodes this per-family; N5 emits
+> proposals *only* for `proposal_allowed = true` rows.
+
+## 1. Merged baseline (verified on merged main)
+
+- R1 setup **7 / 0 / 3** · R1 answers **7 / 0 / 3** · 15-case **15 / 0 / 0**
+- R2 reader **10 setup_correct / 0 setup_wrong / 3 refused** · R2 gold **13 / 13 valid**
+  (7 solved + 3 solver-refused + 3 reader-refused)
+- Serving **unchanged** (every PR's pinned-lane SHA check green)
+
+Two independent, off-serving setup compilers now exist with typed success/refusal states —
+enough substrate for a disciplined "think again" loop without LLM-style handwaving.
+
+## 2. R1 admitted families (`generate/quantitative_comprehension.py`)
+
+Relational single-quantity arithmetic, graded by `evals/setup_oracle` + the independent
+`evals/relational_metric` oracle. **7 families:** fact; more/fewer-than; times-as-many;
+half-as-many (exact divide); multi-step chain; aggregate-then-divide partition; additive
+aggregate-query; narrow inverse (base of a more/fewer-than). All compute exact integer answers.
+Detail: `r1-inventory-ledger-2026-06-07.md`.
+
+## 3. R1 remaining refusals (all correct wrong=0 boundaries)
+
+`r1-08` pronoun (no grounded referent) · `r1-09` ungrounded base (no grounded fact) · `r1-10`
+distractor (compound clause the `X has N unit` template cannot parse). Each must stay refused;
+admitting any would breach wrong=0. None is a coverage gap.
+
+## 4. R2 admitted families (`generate/constraint_comprehension/`)
+
+Two-category finite-integer count/weight systems: read → exact Cramer solve → answer-choice
+verify (with contradiction flagging). **6 families:** buses/seats, animals/legs, tickets/price,
+coins/value, boxes/capacity, vehicles/wheels. Detail: `r2-inventory-ledger-2026-06-07.md`, ADR-0211.
+
+## 5. R2 remaining refusals
+
+- **Reader (gold-backed):** `too_many_categories` (>2), `missing_total_count`,
+  `missing_weighted_total`. Defensive (constructed-test): `coefficient_unit_mismatch`,
+  `coefficient_conflict`, `category_pair_not_found`, `query_target_not_a_category`.
+- **Solver:** `indistinguishable_weights`, `non_integer_solution`, `negative_solution`,
+  `verification_failed`.
+- **Answer-choice:** `no_matching_option`, `ambiguous_options`, `unknown_provided_label`,
+  `unparseable_option`, `no_options`; plus the `contradiction` **verdict** (not a refusal).
+
+## 6. Cross-organ refusal taxonomy draft (formalized by N4)
+
+First-pass mapping of the full refusal surface to the failure-family vocabulary, with
+`must_remain_refused` (correct boundary) and `proposal_allowed` (growth surface) and `owner`.
+N4 turns this into the executable registry; N5 emits proposals **only** for `proposal_allowed`.
+
+| Family | Evidenced by | owner | must_remain_refused | proposal_allowed |
+|---|---|---|---|---|
+| `ambiguous_referent` | R1 pronoun `unreadable_quantity_clause` | r1 | **yes** | no (needs entity-resolution design) |
+| `ungrounded_base` | R1 `unit_unbound`, `no_single_quantity_query` | r1 | **yes** | no |
+| `unsupported_distractor_clause` | R1 `r1-10` `unreadable_quantity_clause` | r1 | **yes** | no (until same-unit target isolation) |
+| `unit_incompatible` | R1 `unit_mismatch`, R2 `coefficient_unit_mismatch` | cross | **yes** | no |
+| `unsupported_system_size` | R2 `too_many_categories` | r2 | **yes** | no (until ≥3-var solver, R3) |
+| `missing_category_pair` | R2 `category_pair_not_found` | r2 | no | **yes** (propose category-pair fixture) |
+| `missing_attribute_coefficient` | *(reserved — no emitter yet)* | r2 | no | **yes** |
+| `missing_total_count` | R2 `missing_total_count` | r2 | no | **yes** (propose count-constraint fixture) |
+| `missing_weighted_total` | R2 `missing_weighted_total` | r2 | no | **yes** (propose weighted-total fixture) |
+| `indistinguishable_weights` | R2 solver | r2 | **yes** | no |
+| `non_integer_solution` | R2 solver | r2 | **yes** | no |
+| `negative_solution` | R2 solver | r2 | **yes** | no |
+| `answer_key_contradiction` | R2 answer-choice `contradiction` verdict | r2 | n/a | no — **action: report contradiction** |
+| `input_shape` | R1 `non_digit_quantity`/`non_identifier_name`/`no_quantity_template`; R2 `query_target_not_a_category` | cross | **yes** | no |
+
+**Reserved (forward-declared, no current emitter):** `missing_attribute_coefficient`,
+`unsupported_rate_duration`, `unsupported_temporal_state` — named so the registry is complete,
+but not reachable until R3 introduces rate/temporal frames.
+
+## 7. Next contemplation batch (N2–N6)
+
+```text
+N2  core/comprehension_attempt/   typed ComprehensionAttempt (organ + outcome + refusal_reason + family + signature + evidence)
+N3  setup router                  try R1 then R2; exactly-one setup_correct -> use; else typed refusal
+N4  failure-family registry       executable form of §6 (must_remain_refused / proposal_allowed / owner)
+N5  proposal-only emitter         teaching/proposals/comprehension_failures/<content_hash>.json — never mounted
+N6  generate/contemplation/ v0    single bounded pass chain: route -> classify -> terminal -> maybe emit
+```
+
+## 8. Non-claims / boundaries
+
+- This is **not** a new capability batch. R1/R2 capability is unchanged; N2–N6 add a *growth
+  organ*, not new math.
+- `setup_wrong` / `answer_wrong` stay **0**; serving stays unchanged.
+- **No self-modification.** The loop's terminal action is at most a *proposal-only* artifact.
+  It cannot ratify, mount, alter a reader, or modify tests except by a human-reviewed PR.
+  The alignment is `failure → classification → proposal → review → ratification`, never
+  `failure → self-patch`. (CLAUDE.md teaching-safety; ADR-0055/0056/0057.)
+- R2 generalization to real GSM8K prose remains **R3**, not claimed here.

tests/test_comprehension_attempt.py

@@ -0,0 +1,66 @@
+"""Tests for the shared ComprehensionAttempt normalizer (N2).
+
+Pins that `classify_r1` / `classify_r2` faithfully normalize each organ's output against the
+existing gold: well-formed fixtures → `setup_correct` with a deterministic signature; refused
+fixtures → `setup_refused` carrying the organ's typed reason. No gold comparison, no solving.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+
+import pytest
+
+from core.comprehension_attempt import classify_r1, classify_r2
+from evals.constraint_oracle.runner import _load_r2_gold
+from evals.setup_oracle.runner import _load_r1_gold
+
+
+def test_classify_r2_matches_gold_expect() -> None:
+    for fx in _load_r2_gold():
+        att = classify_r2(fx["text"], case_id=fx["id"])
+        assert att.organ == "r2_constraints"
+        if fx["expect"] in ("solved", "solver_refuses"):
+            assert att.outcome == "setup_correct", f"{fx['id']}: {att.refusal_reason}"
+            assert att.setup_signature is not None
+        else:  # reader_refuses
+            assert att.outcome == "setup_refused"
+            assert att.refusal_reason == fx["reader_reason"], fx["id"]
+
+
+def test_classify_r1_admits_seven_refuses_three() -> None:
+    attempts = [classify_r1(fx["text"], case_id=fx["id"]) for fx in _load_r1_gold()]
+    correct = [a for a in attempts if a.outcome == "setup_correct"]
+    refused = [a for a in attempts if a.outcome == "setup_refused"]
+    assert len(correct) == 7 and len(refused) == 3
+    assert all(a.organ == "r1_quantitative" for a in attempts)
+    assert all(a.setup_signature is not None for a in correct)
+    assert all(a.refusal_reason for a in refused)
+
+
+def test_classify_r1_specific_inverse_and_pronoun() -> None:
+    inverse = classify_r1("Nia has 9 more beads than Omar. Nia has 15 beads. How many beads does Omar have?")
+    assert inverse.outcome == "setup_correct"
+    pronoun = classify_r1("Pat has 5 marbles. He has 3 more than her. How many marbles does she have?")
+    assert pronoun.outcome == "setup_refused" and pronoun.refusal_reason is not None
+
+
+def test_signatures_are_deterministic() -> None:
+    text = next(f for f in _load_r2_gold() if f["expect"] == "solved")["text"]
+    assert classify_r2(text).setup_signature == classify_r2(text).setup_signature
+
+
+def test_attempt_is_frozen() -> None:
+    att = classify_r2("A school rents 6 buses. How many large buses are there?")
+    with pytest.raises(dataclasses.FrozenInstanceError):
+        att.outcome = "setup_correct"  # type: ignore[misc]
+
+
+def test_classify_does_not_import_evals() -> None:
+    # core/comprehension_attempt must not depend on evals (runtime must not import the harness).
+    import inspect
+
+    import core.comprehension_attempt.classify as m
+
+    source = inspect.getsource(m)
+    assert "import evals" not in source and "from evals" not in source