From 324db573739e4ce20366c3d75c7cf2051533f506 Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 18 Jul 2026 13:07:40 -0700 Subject: [PATCH 1/2] feat(adr-0249): P5 arithmetic-chain lift domain on real GSM8K holdout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires the turn-program executor into the generalized-lift instrument and measures it against the REAL GSM8K dev holdout (evals/gsm8k_math/dev, never the templated cases), using each problem's ground_truth_graph. Baseline is a symbolic fold of the SAME compiled turn program: both paths consume identical problems, the corridor relaxes each step, the baseline folds it numerically. Honest result on the sealed 50-case dev holdout: 26 problems are Tier-1 affine single-accumulator ingestible and solved wrong=0; the corridor matches the symbolic fold on every one (PARITY, delta=0) — the field matches arithmetic, it does not beat it, so this is real-holdout coverage, NOT a manufactured lift. The 24 refused (all not_single_accumulator / multi-entity) are the recorded Tier-2 frontier, never silently dropped. Corrects the now-stale scope-limitation note ('no reader-to-Hamiltonian compiler exists…') — the compiler exists as of this arc; the note now records real-data Tier-1 coverage and the multi-entity frontier. wrong_zero_guard_held extended to bind both exact-regime domains (deductive flagship + arithmetic). 10/10 pins (5 new + 5 existing instrument tests still green). [Verification]: uv run python -m pytest tests/test_adr_0249_arithmetic_lift.py tests/test_generalized_lift_instrument.py -q --- evals/generalized_lift_instrument.py | 130 +++++++++++++++++++++++-- tests/test_adr_0249_arithmetic_lift.py | 68 +++++++++++++ 2 files changed, 192 insertions(+), 6 deletions(-) create mode 100644 tests/test_adr_0249_arithmetic_lift.py diff --git a/evals/generalized_lift_instrument.py b/evals/generalized_lift_instrument.py index c1835a68..74681f1a 100644 --- a/evals/generalized_lift_instrument.py +++ b/evals/generalized_lift_instrument.py @@ -32,8 +32,10 @@ composition frontier, and this instrument is the harness waiting for it. from __future__ import annotations +import json from dataclasses import dataclass from itertools import product +from pathlib import Path from typing import Any, Sequence import numpy as np @@ -56,8 +58,15 @@ from core.physics.linguistic_readback import ( from core.physics.sensorium_wave_feed import ModalityPacket from core.physics.wave_manifold import WaveManifold from generate.logic_equivalence import Verdict, check_equivalence +from generate.math_problem_graph import MathGraphError, graph_from_dict from vocab.manifold import VocabManifold +from evals.turn_program import ( + TurnProgramError, + compile_turn_program, + execute_turn_program, +) + __all__ = [ "DomainOutcome", "LiftInstrumentReport", @@ -65,6 +74,7 @@ __all__ = [ "run_propositional_domain", "run_recognition_domain", "run_multimodal_domain", + "run_arithmetic_chain_domain", ] # Hot-band energy axes (existing E3/E4 precedent; caller-supplied, never invented). @@ -450,22 +460,130 @@ def run_multimodal_domain() -> DomainOutcome: # --- Composed instrument --------------------------------------------------------------- +# --- Domain D: multi-step arithmetic on the real GSM8K dev holdout (ADR-0249) ---------- + +# The sealed GSM8K dev holdout — real problems, never the templated cases. +_DEV_HOLDOUT = Path(__file__).parent / "gsm8k_math" / "dev" / "cases.jsonl" + + +def _load_dev_cases() -> tuple[dict[str, Any], ...]: + if not _DEV_HOLDOUT.exists(): + return () + with _DEV_HOLDOUT.open() as handle: + return tuple(json.loads(line) for line in handle if line.strip()) + + +def _symbolic_fold(seed: float, steps) -> float: + """The baseline: solve the SAME compiled turn program by plain arithmetic. + + Corridor and baseline consume the identical `TurnProgram`; the corridor + relaxes each step, the baseline folds it numerically. The honest question + is whether field relaxation matches arithmetic on the same problem. + """ + answer = float(seed) + for step in steps: + answer = step.scale * answer + step.offset + return answer + + +def run_arithmetic_chain_domain() -> DomainOutcome: + """Corridor turn-program executor vs symbolic fold on real GSM8K problems. + + Both paths consume the graph's `ground_truth_graph` compiled to a turn + program; non-Tier-1 graphs (multi-entity, transfer, rate, …) are refused by + BOTH and recorded as the composition frontier — never silently dropped. The + expected honest verdict is PARITY with wrong=0: the field does not beat + arithmetic at arithmetic; the result is real-holdout coverage, not a lift. + """ + corridor_correct = corridor_wrong = corridor_refused = 0 + baseline_correct = baseline_wrong = baseline_refused = 0 + rows: list[dict[str, Any]] = [] + cases = _load_dev_cases() + + for case in cases: + case_id = case.get("id", "") + gold_raw = case.get("expected_answer") + graph_dict = case.get("ground_truth_graph") + try: + graph = graph_from_dict(graph_dict) if graph_dict else None + except (MathGraphError, KeyError, TypeError, ValueError): + graph = None + try: + program = compile_turn_program(graph) if graph is not None else None + except TurnProgramError as refusal: + corridor_refused += 1 + baseline_refused += 1 + rows.append({"case_id": case_id, "ingested": False, "refusal": refusal.reason}) + continue + if program is None: + corridor_refused += 1 + baseline_refused += 1 + rows.append({"case_id": case_id, "ingested": False, "refusal": "no_graph"}) + continue + + gold = None if gold_raw is None else float(gold_raw) + corridor_answer = execute_turn_program(program).answer + baseline_answer = _symbolic_fold(program.seed, program.steps) + + corridor_ok = gold is not None and abs(corridor_answer - gold) < 1e-4 + baseline_ok = gold is not None and abs(baseline_answer - gold) < 1e-4 + corridor_correct += int(corridor_ok) + corridor_wrong += int(not corridor_ok) + baseline_correct += int(baseline_ok) + baseline_wrong += int(not baseline_ok) + rows.append( + { + "case_id": case_id, + "ingested": True, + "gold": gold, + "corridor_answer": corridor_answer, + "baseline_answer": baseline_answer, + "corridor_ok": corridor_ok, + } + ) + + ingested = corridor_correct + corridor_wrong + return DomainOutcome( + domain_id="arithmetic-chain", + n_cases=len(cases), + corridor_correct=corridor_correct, + corridor_wrong=corridor_wrong, + corridor_refused=corridor_refused, + baseline_correct=baseline_correct, + baseline_wrong=baseline_wrong, + baseline_refused=baseline_refused, + notes=( + f"Real GSM8K dev holdout: {ingested}/{len(cases)} problems are Tier-1 " + f"affine single-accumulator ingestible and solved wrong=0; the rest are " + f"refused (multi-entity/transfer/rate/…) — the recorded Tier-2 frontier.", + "Baseline is a symbolic fold of the SAME compiled turn program; PARITY " + "with wrong=0 is the honest outcome — the field matches arithmetic, it " + "does not beat it. The result is real-holdout coverage, not a lift.", + ), + cases=tuple(rows), + ) + + def run_generalized_lift_instrument() -> LiftInstrumentReport: outcomes = ( run_propositional_domain(), run_recognition_domain(), run_multimodal_domain(), + run_arithmetic_chain_domain(), ) propositional = outcomes[0] + arithmetic = outcomes[3] return LiftInstrumentReport( outcomes=outcomes, - wrong_zero_guard_held=(propositional.corridor_wrong == 0), + # Both exact-regime domains (deductive flagship + arithmetic) preserve wrong=0. + wrong_zero_guard_held=(propositional.corridor_wrong == 0 and arithmetic.corridor_wrong == 0), honest_null=all(o.delta_correct <= 0 for o in outcomes), scope_limitations=( - "GSM8K / natural-language arithmetic is NOT ingestible by corridor " - "v1: no reader-to-Hamiltonian compiler exists beyond the <=5-atom " - "propositional and quadratic-well domains. Recorded, not silently " - "dropped; that compiler is the composition frontier this " - "instrument is waiting to measure.", + "The reader→Hamiltonian compiler now exists for the affine " + "single-accumulator arithmetic subset (ADR-0249): on the real GSM8K " + "dev holdout it solves the Tier-1 subset wrong=0 (see the " + "arithmetic-chain domain). Non-affine / multi-entity GSM8K problems " + "(transfer, rate, comparison, fraction, partition) remain the " + "recorded Tier-2 frontier — refused, never silently dropped.", ), ) diff --git a/tests/test_adr_0249_arithmetic_lift.py b/tests/test_adr_0249_arithmetic_lift.py new file mode 100644 index 00000000..42a1c1ba --- /dev/null +++ b/tests/test_adr_0249_arithmetic_lift.py @@ -0,0 +1,68 @@ +"""ADR-0249 P5 — arithmetic-chain lift domain on the real GSM8K holdout. + +Wires the turn-program executor into the generalized-lift instrument and +measures it against the *real* GSM8K dev holdout (never the templated cases), +with a symbolic fold of the same compiled program as the honest baseline. The +expected, honest outcome is PARITY with wrong=0 on the Tier-1 affine subset, +with the multi-entity remainder recorded — not a manufactured lift. +""" +from __future__ import annotations + +import json + +import pytest + +from evals.generalized_lift_instrument import ( + run_arithmetic_chain_domain, + run_generalized_lift_instrument, +) + + +@pytest.fixture(scope="module") +def domain(): + return run_arithmetic_chain_domain() + + +def test_domain_solves_real_gsm8k_wrong_zero(domain) -> None: + assert domain.domain_id == "arithmetic-chain" + assert domain.corridor_wrong == 0 # wrong=0 on real GSM8K, not just templates + assert domain.corridor_correct > 0 # the corridor ingests and solves real problems + assert domain.corridor_refused > 0 # and honestly refuses what it can't ingest + + +def test_every_ingested_case_is_correct(domain) -> None: + # The load-bearing claim: on real problems the corridor CAN ingest, it is + # exactly right — refusals, never wrong answers. + for row in domain.cases: + if row["ingested"]: + assert row["corridor_ok"] is True + + +def test_parity_with_symbolic_fold_honest_null(domain) -> None: + # The field matches arithmetic on the same compiled program; it does not + # beat it. PARITY is the honest verdict here, never an inflated lift. + assert domain.corridor_correct == domain.baseline_correct + assert domain.verdict == "PARITY" + assert domain.delta_correct == 0 + + +def test_coverage_is_recorded_not_dropped(domain) -> None: + ingested = domain.corridor_correct + domain.corridor_wrong + assert ingested + domain.corridor_refused == domain.n_cases + assert domain.n_cases > 0 + # A coverage tracker against the sealed dev holdout (50 cases, 26 Tier-1). + assert domain.corridor_correct == 26 + assert domain.corridor_refused == 24 + assert any("Tier-1" in note for note in domain.notes) + + +def test_domain_joins_full_report_and_scope_corrected() -> None: + report = run_generalized_lift_instrument() + ids = [o.domain_id for o in report.outcomes] + assert "arithmetic-chain" in ids + assert report.wrong_zero_guard_held # deductive + arithmetic both wrong=0 + # The stale "no compiler exists" scope note must be gone; the compiler exists now. + joined = " ".join(report.scope_limitations) + assert "now exists" in joined + assert "no reader-to-Hamiltonian compiler exists" not in joined + json.dumps(report.as_dict()) # JSON-safe artifact -- 2.45.2 From d65fb8cf363e8adff5866d04f07ec2230adee904 Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 18 Jul 2026 13:10:01 -0700 Subject: [PATCH 2/2] =?UTF-8?q?docs(adr-0249):=20ADR-0249=20(Proposed)=20+?= =?UTF-8?q?=20acceptance=20evidence=20=E2=80=94=20arc=20closure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR-0249 documents the reader→Hamiltonian compiler (P1-P5): thesis (composition = certified turn sequences), the five components, anti-hollow discipline, the Ring-2 non-mutating correction, three-tier reproducibility, Tier-1 scope + recorded frontier, and the honest-measurement done-when. Status Proposed — no self-Accept; §10 ruling record awaits Shay. Acceptance evidence: phases + merge SHAs, the real GSM8K dev-holdout result (26/50 Tier-1 ingestible, wrong=0, PARITY vs symbolic fold, 24 multi-entity refused = recorded frontier), in-tree algebra + CNF-soundness verification, the design corrections applied, and the honest open register. --- ...miltonian-compiler-composition-frontier.md | 126 ++++++++++++++++++ docs/handoff/ADR-0249-Acceptance-Evidence.md | 69 ++++++++++ 2 files changed, 195 insertions(+) create mode 100644 docs/adr/ADR-0249-reader-hamiltonian-compiler-composition-frontier.md create mode 100644 docs/handoff/ADR-0249-Acceptance-Evidence.md diff --git a/docs/adr/ADR-0249-reader-hamiltonian-compiler-composition-frontier.md b/docs/adr/ADR-0249-reader-hamiltonian-compiler-composition-frontier.md new file mode 100644 index 00000000..2d408271 --- /dev/null +++ b/docs/adr/ADR-0249-reader-hamiltonian-compiler-composition-frontier.md @@ -0,0 +1,126 @@ +# ADR-0249: Reader→Hamiltonian Compiler — the Composition Frontier + +**Status**: **Proposed** +**Date**: 2026-07-18 +**Authors**: Joshua Shay + multi-model R&D (implemented Fable 5) +**Depends on**: ADR-0243 (cognitive lifecycle — `relax_to_ground`, `ProblemHamiltonian`), ADR-0244/0245 (content-addressing, f64 relaxation, `_cached_eigh`), ADR-0217 (R2 finite-integer constraint compiler — front-end precedent), ADR-0175/0191–0193 (calibrated-learning / composition-wall doctrine) +**Design record**: `docs/research/reader-hamiltonian-compiler-spike-2026-07-18.md` +**Acceptance evidence**: `docs/handoff/ADR-0249-Acceptance-Evidence.md` + +--- + +## 1. What this ADR is — and the thesis + +The generalized-lift instrument (ADR-0246 seam S4) recorded a single blocking +scope limitation: *no reader→Hamiltonian compiler existed beyond ≤5-atom +propositional problems and quadratic wells, so real problems (GSM8K arithmetic, +NL deduction) could not be ingested by the corridor at all.* This ADR builds +that compiler and nothing more — it does not touch serving, does not activate +any gate, and does not claim the corridor now beats symbolic solvers. + +**Thesis — composition = certified turn sequences.** The corridor's native +space is 32-dimensional (the Cl(4,1) blade basis; `_MAX_ATOMS = 5` is exactly +2⁵ = 32, structural not incidental). A problem is therefore *not* compiled into +one large Hamiltonian. It is compiled into a **turn program** — an ordered +sequence of small relation-wells, each solved in one certified relaxation turn, +with the working state flowing turn-to-turn on the field. Composition depth +lives in the certified chain, not in matrix size. This is the "wall is +COMPOSITION" lesson (ADR-0193) finally getting its mechanism. + +## 2. Components (five phases, each merged under its own PR) + +| Phase | Module | What it does | +|---|---|---| +| P1 | `core/physics/quantity_kernel.py` | Quantities as null points on the Cl(4,1) conformal line; add/scale by constants as translator/dilator versor sandwiches; projective scale-invariant decode. | +| P2 | `core/physics/relation_compiler.py` | `output = scale·input + offset` → quadratic-well `ProblemHamiltonian` via versor transport (reuses `compile_quadratic_well`). | +| P3 | `evals/logic_cnf_compiler.py` | Structural formula→CNF (reuses `logic_canonical` parser; eliminate iff/implies → NNF → distribute) → `PropositionalProblem` + query `Clause`. | +| P4 | `evals/turn_program.py` | `MathProblemGraph` → ordered turn program; chained-relaxation executor; content-addressed tamper-evident turn ledger. | +| P5 | `evals/generalized_lift_instrument.py` | `arithmetic-chain` domain: corridor vs symbolic fold on the real GSM8K dev holdout; honest coverage recorded. | + +## 3. Anti-hollow discipline (load-bearing) + +The compiler must never *evaluate* the arithmetic or logic it encodes — else the +corridor confirms a precomputed answer rather than solving, and the instrument +is hollow. Enforcement: + +- The relation/turn compilers apply the problem's **structure** as versor + operators (dilator for scale, translator for offset); the substrate's + geometric product performs the arithmetic. No Python `scale·input+offset`. +- Compiled wells are **bare projectors** — metadata `{curvature, target_digest}` + only; they carry no answer and no coefficients. +- In a turn chain the accumulator flows as a **field state and is decoded exactly + once, at the end**; intermediate values are never decoded to drive the next + turn. +- Ablation pins confirm the answer is produced by relaxation (the start state + decodes to the input, only the relaxed state to the answer, with byte-identical + Hamiltonians). + +## 4. The Ring-2 correction (verified in-tree) + +The design spike proposed "chain turns through the Ring-2 residual protocol." +Grounding against the code corrected this: `run_residual_protocol` +(`core/ports/residual_protocol.py:222`) is **zero-bound / non-mutating** — its +stage-5 recertification re-witnesses the subject and raises +`recertification_witness_changed` if it moved. Arithmetic turns *mutate*. The +protocol therefore certifies a *fixed* state's admissibility, not a state +*transition*, and cannot be the vehicle for the mutating turn. + +Resolution (no hollow integration): the per-turn certificate is the relaxation's +own `RelaxationCertificate`; the tamper-evident *sequence* is recorded with the +Ring-2 chain-integrity **pattern** — a content-addressed `TurnRecord`, +`GENESIS_DIGEST`-linked, with `verify_turn_chain` mirroring `verify_replay_chain`. + +## 5. Reproducibility — three tiers, honestly bounded (spike §4.6) + +- **Tier 1 (content address): solved.** SHA-256 over explicit `5-atom deduction, unknown× +unknown — is **refused with a typed error and recorded**, never silently dropped +(no-silent-caps). >5-atom deduction via ROBDD-partitioned turn programs is the +next arc, not this one (a sequencing choice, not an impossibility). + +## 7. Honest measurement (the done-when) + +On the sealed real GSM8K dev holdout (50 cases): **26 are Tier-1 ingestible and +solved wrong=0**; the corridor matches the symbolic fold on every one (PARITY, +delta = 0 — the field matches arithmetic, it does not beat it); the 24 refused +are all multi-entity, the recorded Tier-2 frontier. This is the honest-NULL +protocol working as designed: the deliverable is a *decidable measurement* and +*recorded real-data coverage*, not an inflated lift claim. The recognition +domain's genuine +9 lift stands separately; arithmetic is honestly PARITY. + +## 8. Governance + +Off-serving (A-04): every module lives in `core/physics/` or `evals/` and is +never imported by `chat/runtime.py`. No gate is activated; no manifold is +mutated (I-03). Reuses ratified contracts (`compile_quadratic_well`, +`HamiltonianCompileError`, `logic_canonical`, the Ring-2 chain pattern) rather +than forking them. Local-first CI: smoke green in-worktree at every phase. + +## 9. What this ADR does NOT claim (honest register) + +- Not a serving change; the symbolic `generate/` path remains the serve path. +- Not a lift over symbolic solvers on arithmetic (PARITY is the honest result). +- Not full GSM8K coverage — 52% of the dev holdout is Tier-1; the rest is the + recorded frontier. +- Not an activation of any admission gate (ADR-0246 §3.7 stays uncalibrated). + +## 10. Ruling record (§8 — Shay) + +_Awaiting ratification._ The four design rulings (affine-only Tier-1; >5-atom +deduction deferred; single ADR; field wedge independent) are recorded RESOLVED +in the spike; acceptance of this ADR is Shay's alone (no self-Accept). diff --git a/docs/handoff/ADR-0249-Acceptance-Evidence.md b/docs/handoff/ADR-0249-Acceptance-Evidence.md new file mode 100644 index 00000000..18f17d37 --- /dev/null +++ b/docs/handoff/ADR-0249-Acceptance-Evidence.md @@ -0,0 +1,69 @@ +# ADR-0249 — Acceptance Evidence + +**Status**: Evidence pack for Shay's ruling (no self-Accept) +**Date**: 2026-07-18 +**ADR**: `docs/adr/ADR-0249-reader-hamiltonian-compiler-composition-frontier.md` +**Design record**: `docs/research/reader-hamiltonian-compiler-spike-2026-07-18.md` + +The reader→Hamiltonian compiler closes the composition frontier the +generalized-lift instrument (ADR-0246 S4) was waiting to measure. Built in five +smoke-gated phases, each merged under its own PR off `forgejo/main`. + +## Phases merged + +| Phase | PR | Merge commit | Module | Pins | +|---|---|---|---|---| +| P1 quantity kernel | #66 | `aca98d85` | `core/physics/quantity_kernel.py` | 35/35 | +| P2 relation compiler | #67 | `2575d0d9` | `core/physics/relation_compiler.py` | 21/21 | +| P3 CNF converter | #68 | `309e0ef6` | `evals/logic_cnf_compiler.py` | 32/32 | +| P4 turn-program executor | #69 | `805752db` | `evals/turn_program.py` | 15/15 | +| P5 instrument integration | this PR | — | `evals/generalized_lift_instrument.py` | 10/10 | + +Smoke (`uv run core test --suite smoke -q`) green at every phase boundary (176 passed). + +## Load-bearing results + +**Multi-step arithmetic by chained certified relaxation** (P4): +`((5+3)*2)-4 = 12`, `(10/2+7)*3 = 36`, `(100-40)/4 = 15` — every turn +`ground_state_certified`; accumulator flows as a field state, decoded once. + +**Real GSM8K dev holdout** (P5, sealed 50 cases, `evals/gsm8k_math/dev`): + +| Metric | Value | +|---|---| +| Tier-1 affine ingestible | 26 / 50 (52%) | +| Corridor correct on ingested | 26 / 26 — **wrong = 0** | +| Corridor vs symbolic fold | PARITY (delta = 0) | +| Refused (multi-entity) | 24 / 50 — recorded Tier-2 frontier | + +The corridor solves real GSM8K problems it can ingest with zero wrong answers, +and honestly refuses the rest. It does not beat arithmetic at arithmetic +(PARITY is honest); the deliverable is real-data coverage + a decidable +measurement, per the honest-NULL protocol. + +**Algebra verified in-tree** (P1): conformal line embedding + translator/dilator +exactness checked against `algebra/cl41.py`'s own multiplication table (nullity +< 3e-10; translator exact to f64 rounding; dilator sign convention pinned). + +**CNF soundness proved** (P3): for a 14-formula panel, the compiled CNF has the +same ROBDD identity as the source (`canonicalize` oracle); end-to-end entailment +agrees with `evaluate_entailment` gold, wrong = 0 on consistent premises. + +## Design corrections applied (honest register) + +- **Ring-2 is non-mutating.** `run_residual_protocol` is zero-bound (stage-5 + recertification refuses a moved witness), so it certifies fixed-state + admissibility, not transitions. The turn ledger reuses the chain-integrity + *pattern* (`TurnRecord` / `verify_turn_chain`), not the admission protocol. +- **`cga_inner` is the wrong metric for versor states** — readback and quantity + decode use `phase_correlation/2`, pinned in the P1 module. +- **Scope note corrected** — the instrument's stale "no compiler exists" line is + replaced with measured Tier-1 coverage + the multi-entity frontier. + +## What remains open (not claimed here) + +- Multi-entity / non-affine GSM8K (transfer, rate, comparison, fraction) — + Tier-2, refused and recorded. +- >5-atom deduction via ROBDD-partitioned turn programs — next arc. +- ADR-0246 §3.7 admission-policy calibration — unchanged; no gate activated. +- `generate/` ("core_logos") serve-path articulation upgrade — still deferred. -- 2.45.2