From 299c92be68b412d011c76facc9daf0d4459caf48 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 28 Jul 2026 02:22:43 +0000 Subject: [PATCH] =?UTF-8?q?test(rnd):=20pre-register=20the=20ADR-0252=20?= =?UTF-8?q?=C2=A75=20criterion,=20before=20the=20run?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Track A, per 50-execution-plan.md §6: "state the GO/NO-GO criterion in writing BEFORE running (the ADR's §5 conformance bar, not a criterion chosen after seeing results)." This commit carries the criterion and the code and NO results. Why the protocol is load-bearing here rather than ceremonial: the §5 experiment is not unrun. It has been run twice and returned GO twice, on unmerged branches, and neither run's rule was binding. - rnd/structure-mapping-experiment @ fc9d0c14 — disowned by its own successor: the S1-S4 label entered the embedding, so Procrustes ran on identical matrices per structure. - rnd/sme-experiment-v2 @ 96e5f468 — blind, and then measured separability with `except ValueError: res = 1000.0`. Its GO test cross_mean > 2*same_mean is satisfied by that sentinel alone. A solver exception is not a distance. Instrument characterised before use, on dummy clouds: conformal_procrustes has two branches. The (5,K) Kabsch+Umeyama path never raises and is invariant to rotation, translation AND uniform scale (identical 1.8e-16, rotated 7.8e-16, translated 3.9e-15, scaled x2.5 1.4e-14) while separating non-similar deformation (5.9e-1) and unrelated clouds (1.1e0). The mixed versor/point sequence path performs field conjugacy and raises. Attempt 2 used the second; this uses the first, and treats a raise as non-convergence — excluded, reported as a rate, never counted as separation. Corpus fact that forced the design, measured at 797ebad5: the serving reader returns a selected graph for 5 of the 500 holdout_dev/v1 cases (1.0%), all with the same skeleton. §5.1's four-structure corpus is not extractable from holdout_dev/v1 today, so §5.1's own escape hatch is used and MARKED — 5 real cases, 48 synthetic from four declared templates. A verdict from this corpus speaks about the geometry, not about coverage; §5.5's compression estimate is not answerable at 1.0% coverage and is not attempted. Off-serving. Imported by no serving path; no answers emitted. --- ...e-experiment-preregistration-2026-07-28.md | 137 +++++++ .../structure_mapping/adr0252_s5/__init__.py | 17 + evals/structure_mapping/adr0252_s5/corpus.py | 374 ++++++++++++++++++ .../structure_mapping/adr0252_s5/embedding.py | 202 ++++++++++ .../adr0252_s5/experiment.py | 223 +++++++++++ 5 files changed, 953 insertions(+) create mode 100644 docs/research/sme-experiment-preregistration-2026-07-28.md create mode 100644 evals/structure_mapping/adr0252_s5/__init__.py create mode 100644 evals/structure_mapping/adr0252_s5/corpus.py create mode 100644 evals/structure_mapping/adr0252_s5/embedding.py create mode 100644 evals/structure_mapping/adr0252_s5/experiment.py diff --git a/docs/research/sme-experiment-preregistration-2026-07-28.md b/docs/research/sme-experiment-preregistration-2026-07-28.md new file mode 100644 index 00000000..2aeaa299 --- /dev/null +++ b/docs/research/sme-experiment-preregistration-2026-07-28.md @@ -0,0 +1,137 @@ +# ADR-0252 §5 — pre-registration of the structure-mapping acceptance gate + +**Date:** 2026-07-28 · **Author:** Opus 5 (Track A, `50-execution-plan.md` §6) +**Base:** `forgejo/main` @ `797ebad5` +**Status:** written and committed **before** the run. The commit that carries this +file also carries the experiment code and carries **no results**. Any later change +to a threshold below invalidates the verdict rather than adjusting it. + +--- + +## 0. Why a pre-registration exists for this experiment specifically + +Track A's protocol requires the GO/NO-GO criterion be stated in writing before +running — *"the ADR's §5 conformance bar, not a criterion chosen after seeing +results."* That requirement is not ceremonial here. The experiment has been run +twice, and both runs returned GO on a criterion that was not binding: + +- **Attempt 1** (`rnd/structure-mapping-experiment` @ `fc9d0c14`) is disowned by + its own successor: the S1–S4 label entered the embedding coordinates through + hand-typed "canonical" matrices, so every case of a structure had an identical + byte-array and Procrustes on identical matrices is trivially zero. +- **Attempt 2** (`rnd/sme-experiment-v2` @ `96e5f468`) fixed the leak and then + measured separability with `except ValueError: res = 1000.0`. Its GO test + (`cross_mean > same_mean * 2 and same_mean < 1.0`) is satisfied by that + sentinel alone. A solver exception is not a distance. + +Both defects are invisible unless the decision rule is fixed in advance. So the +rule below is committed as *code* — `experiment.py`'s module constants — and the +run imports it. + +## 1. What is being tested (ADR-0252 §5, H1) + +> There exists a principled embedding of a problem's role-predicate structure +> into Cl(4,1) point-configurations such that conformal-Procrustes alignment +> residual is (a) low within a deep structure, (b) high across deep structures, +> (c) invariant to surface-attribute change, and (d) sensitive to structure change. + +## 2. The instrument, characterised before use + +`conformal_procrustes` has two branches, and they are not interchangeable: + +| Input form | Branch | Behaviour | +|---|---|---| +| `(5,K)` grade-1 point cloud | Kabsch + Umeyama | never raises; residual invariant to rotation, translation **and uniform scale** | +| mixed versor/point sequence | field conjugacy `W F W⁻¹` | **raises** `field conjugacy versor not closed` on unlike inputs | + +Measured on dummy clouds at `797ebad5`, before any corpus was embedded: +identical `1.8e-16`, rotated `7.8e-16`, translated `3.9e-15`, uniformly scaled +×2.5 `1.4e-14`, non-similar z-stretch `5.9e-1`, unrelated cloud `1.1e0`. + +Attempt 2 used the second branch. This experiment uses the first, and treats a +raise as **non-convergence** — excluded from the statistics, reported as a rate, +never counted as evidence of separation. + +## 3. Corpus (§5.1), and the coverage fact that forced its shape + +§5.1 asks for ≥4 structures with clean labels drawn from `holdout_dev/v1`. +**Measured at `797ebad5`: the serving reader returns a selected graph for 5 of +the 500 holdout cases (1.0%), and all five carry the same skeleton** +(`compare_multiplicative` — they are exactly the organ cohort already recorded +in `evals/structure_mapping/scoring/labels.py`). The corpus §5.1 specifies is +therefore not extractable from `holdout_dev/v1` today. + +§5.1's own escape hatch is used, and used visibly: *"add controlled synthetic +surface-variants only where needed for clean labels (marked as such)."* + +| Arm | n | Provenance | +|---|---|---| +| real | 5 | `holdout_dev/v1`, via `parse_and_solve` — no hand-authoring | +| synthetic | 48 | four declared templates × 12 attribute realisations, `provenance="synthetic"` | + +Consequence, stated now so it cannot be quietly dropped later: a verdict from +this corpus speaks about **the geometry**, not about coverage. §5.5's +compression estimate ("how many holdout families collapse to how few canonical +structures") is **not answerable** at 1.0% reader coverage and is not attempted. + +## 4. Embedding (§5.2), declared + +Full scheme in `embedding.py`'s module docstring. In brief: entity → conformal +point on the role circle at angle `2πk/8` by **order of introduction** (never by +name); relation → a point at the circular mean of its arguments' angles, radius +`2.0 + 0.35·j` (story order), height a declared per-kind constant; the unknown → +one point at the asked role. Undeclared relation kind ⇒ refusal, never a default +position. Clouds padded to 16 points with the cloud's own centroid. + +**Two variants, because §5.2 and §5.3b pull against each other.** §5.2 says the +configuration must encode role-structure and "never surface words or literal +values"; §5.3b says residual must not move when numbers change. An embedding +that cannot see numbers satisfies §5.3b analytically — which is a proof about +the scheme, not a measurement. So both are run: + +| Variant | `ATTR_SCALE` | §5.3b status | +|---|---|---| +| **RS-A** | 0.0 | analytic — attributes are not in the geometry | +| **RS-B** | 0.02 | **measured** — quantities displace points along `e3` | + +Entity *names* are outside both variants by construction, so of §5.3b's three +perturbations, `rename` is analytic in RS-A **and** RS-B; only `rescale` and +`jitter` are empirical, and only in RS-B. + +## 5. The decision rule — binding + +Let τ be the single threshold chosen to maximise `min(same-below-rate, +cross-above-rate)` over all pairs. + +| # | Measurement | Passes iff | +|---|---|---| +| (a) | **Separability** | ROC AUC ≥ **0.90** and one τ puts ≥ **95%** of same-structure pairs below it and ≥ **95%** of cross-structure pairs above it. Means alone are not sufficient — §5.3a says so explicitly. | +| (b) | **Attribute-invariance** | every attribute-only variant pair lands below τ, and their median is ≤ **25%** of the cross-structure median | +| (c) | **Structure-sensitivity** | every minimal pair — identical roles, identical numbers, exactly one relation kind changed — lands above τ | +| (d) | **Systematicity** | reported, **not gating**: §5.4's verdict rule names only (a), (b), (c) | + +**VOID** if the non-convergence rate exceeds **5%**. An instrument that refuses +to answer has not answered, and that is neither GO nor NO-GO. + +**GO** iff (a) ∧ (b) ∧ (c) hold **on the same variant**. + +**NO-GO** otherwise — including the specific case worth naming in advance: if +RS-B fails (b) while RS-A passes (a) ∧ (c) with (b) analytic, that is **NO-GO +for H1 as stated**, recorded with the qualified finding. It is the outcome most +likely to tempt a re-reading of the bar, which is why the reading is fixed here. + +**Ambiguity resolves to NO-GO** (`50-execution-plan.md` §8: *"Ambiguity is then +a NO-GO by default, not a re-run with a new bar."*) + +A well-controlled NO-GO is full credit by ADR-0252 §5.4's own terms. + +## 6. Discipline (§5.6) + +Off-serving — `evals/structure_mapping/adr0252_s5/` is imported by no serving +path. Sealed test untouched. `holdout_dev/v1` is the only real-case source; +every non-holdout case is marked synthetic. No answers are emitted, so `wrong=0` +is not in play. Every number in the verdict document is produced by +`uv run python -m evals.structure_mapping.adr0252_s5.experiment`, and the report +carries a `deterministic_digest` so a regression flips it. STOP before +productionizing: this experiment authorizes nothing by itself; §6 remains Shay's +ratification. diff --git a/evals/structure_mapping/adr0252_s5/__init__.py b/evals/structure_mapping/adr0252_s5/__init__.py new file mode 100644 index 00000000..0b6c2547 --- /dev/null +++ b/evals/structure_mapping/adr0252_s5/__init__.py @@ -0,0 +1,17 @@ +"""ADR-0252 §5 — the structure-mapping acceptance-gate experiment. + +Off-serving. Nothing in this package is imported by ``chat/runtime.py`` or by +any serving path; it exists to answer §5's one load-bearing empirical claim and +to leave a reproducible artifact behind when it does. + +Package layout mirrors the §5 design so the mapping from document to code is +one-to-one: + + corpus.py §5.1 the structure-labelled corpus (real + MARKED synthetic) + embedding.py §5.2 the relational embedding scheme (blind to labels) + experiment.py §5.3 the decisive measurements, §5.4 the verdict rule + +The blindness invariant: ``embedding.py`` imports nothing from ``corpus.py``'s +label side and never receives a label. ``experiment.py`` loads labels only after +every residual has been computed. ``tests/test_adr_0252_s5_blindness.py`` pins it. +""" diff --git a/evals/structure_mapping/adr0252_s5/corpus.py b/evals/structure_mapping/adr0252_s5/corpus.py new file mode 100644 index 00000000..82bf7c8b --- /dev/null +++ b/evals/structure_mapping/adr0252_s5/corpus.py @@ -0,0 +1,374 @@ +"""§5.1 — the structure-labelled corpus, with provenance on every case. + +Two arms, and the split is the finding as much as the input: + +**REAL arm.** Cases drawn from ``evals/gsm8k_math/holdout_dev/v1``, extracted +through the serving reader (``parse_and_solve``) — no hand-authoring, no other +lane. Measured at 797ebad5: the reader returns a selected graph for **5 of the +500** holdout cases, and all five carry the *same* relational skeleton +(``compare_multiplicative``). §5.1 asks for "minimum four structures with clean +labels" drawn from this corpus; the reader cannot supply them today, and that +is why §5.1's own escape hatch is used below rather than ignored. + +**SYNTHETIC arm (MARKED).** §5.1: *"add controlled synthetic surface-variants +only where needed for clean labels (marked as such)."* Every synthetic case +carries ``provenance="synthetic"`` and the template that generated it. They are +constructed as :class:`MathProblemGraph` values directly — the geometry question +is about the graph, not about the sentence that produced it — and they are +generated from declared templates with seeded attributes so the corpus is a +pure function of this file. + +The control sets that make the measurements decisive live here too, because a +control is part of the corpus design, not of the scorer: + + INV attribute-only variants of one structure (§5.3b) + SENS minimal pairs: identical roles and numbers, one relation kind changed (§5.3c) + SYST chained higher-order structures (§5.3d) + +Labels never travel with the graphs. :func:`build_corpus` returns cases whose +``label`` field is ``None``; :func:`load_labels` is a separate call that the +embedding never makes. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Final, Iterator, Mapping, Sequence + +from generate.math_problem_graph import MathProblemGraph, graph_from_dict + +_HOLDOUT_CASES: Final[Path] = ( + Path(__file__).resolve().parents[2] / "gsm8k_math" / "holdout_dev" / "v1" / "cases.jsonl" +) + +#: Structure vocabulary. The label IS the role-predicate skeleton (§5.1), so it +#: is stated here once and referenced everywhere rather than re-derived. +STRUCTURES: Final[Mapping[str, str]] = { + "S1": "compare-multiplicative — B's quantity is k times A's", + "S2": "transfer — A moves j units to B", + "S3": "additive accumulation — A gains j, then gains i", + "S4": "additive decrement — A loses j", +} + +#: Entity name pools. Names are surface attributes and must never reach the +#: geometry; three disjoint pools exist so a rename control can be built. +_NAME_POOLS: Final[tuple[tuple[str, ...], ...]] = ( + ("Ada", "Bram", "Cleo", "Dov"), + ("Petra", "Quill", "Rosa", "Silas"), + ("Mira", "Nils", "Oona", "Pax"), +) + +_UNITS: Final[tuple[str, ...]] = ("marbles", "coins", "apples", "tickets") + + +@dataclass(frozen=True, slots=True) +class Case: + """One corpus member. ``label`` is always ``None`` on the mapper's side.""" + + case_id: str + graph: MathProblemGraph + provenance: str # "holdout_dev/v1" | "synthetic" + template: str | None = None + label: None = field(default=None, init=False) + + def as_json(self) -> dict[str, Any]: + return { + "case_id": self.case_id, + "provenance": self.provenance, + "template": self.template, + "graph": self.graph.as_json(), + } + + +# -------------------------------------------------------------------------- +# REAL arm +# -------------------------------------------------------------------------- + + +def extract_holdout_graphs(limit: int | None = None) -> list[Case]: + """Run the serving reader over ``holdout_dev/v1`` and keep what it decides. + + Deliberately uses the same entry point the eval lane uses + (``generate.math_candidate_graph.parse_and_solve``) so the extraction + inherits the reader's real coverage rather than a bespoke parser's. + """ + from generate.math_candidate_graph import parse_and_solve + + out: list[Case] = [] + with _HOLDOUT_CASES.open() as fh: + for n, line in enumerate(fh): + if limit is not None and n >= limit: + break + record = json.loads(line) + try: + result = parse_and_solve(record["problem"]) + except Exception: # noqa: BLE001 — reader refusals are data here + continue + graph = getattr(result, "selected_graph", None) + if graph is None: + continue + out.append( + Case(case_id=record["id"], graph=graph, provenance="holdout_dev/v1") + ) + return out + + +# -------------------------------------------------------------------------- +# SYNTHETIC arm — declared templates +# -------------------------------------------------------------------------- + + +def _s1(a: str, b: str, unit: str, base: float, factor: float) -> MathProblemGraph: + return graph_from_dict( + { + "entities": [a, b], + "initial_state": [{"entity": a, "quantity": {"value": base, "unit": unit}}], + "operations": [ + { + "actor": b, + "kind": "compare_multiplicative", + "operand": { + "direction": "times", + "reference_actor": a, + "factor": factor, + }, + } + ], + "unknown": {"entity": b, "unit": unit}, + } + ) + + +def _s2(a: str, b: str, unit: str, base: float, other: float, moved: float) -> MathProblemGraph: + return graph_from_dict( + { + "entities": [a, b], + "initial_state": [ + {"entity": a, "quantity": {"value": base, "unit": unit}}, + {"entity": b, "quantity": {"value": other, "unit": unit}}, + ], + "operations": [ + { + "actor": a, + "kind": "transfer", + "target": b, + "operand": {"value": moved, "unit": unit}, + } + ], + "unknown": {"entity": b, "unit": unit}, + } + ) + + +def _s3(a: str, unit: str, base: float, first: float, second: float) -> MathProblemGraph: + return graph_from_dict( + { + "entities": [a], + "initial_state": [{"entity": a, "quantity": {"value": base, "unit": unit}}], + "operations": [ + {"actor": a, "kind": "add", "operand": {"value": first, "unit": unit}}, + {"actor": a, "kind": "add", "operand": {"value": second, "unit": unit}}, + ], + "unknown": {"entity": a, "unit": unit}, + } + ) + + +def _s4(a: str, unit: str, base: float, lost: float) -> MathProblemGraph: + return graph_from_dict( + { + "entities": [a], + "initial_state": [{"entity": a, "quantity": {"value": base, "unit": unit}}], + "operations": [ + {"actor": a, "kind": "subtract", "operand": {"value": lost, "unit": unit}} + ], + "unknown": {"entity": a, "unit": unit}, + } + ) + + +def _realizations() -> Iterator[tuple[str, str, MathProblemGraph]]: + """Twelve surface realizations per structure, deterministic in this file. + + Attributes vary (names from three pools, units, magnitudes); the role + skeleton does not. No RNG: the schedule is written out so the corpus is + reproducible by reading, not by re-running. + """ + bases = (12.0, 35.0, 78.0, 140.0) + for i in range(12): + pool = _NAME_POOLS[i % len(_NAME_POOLS)] + unit = _UNITS[i % len(_UNITS)] + base = bases[i % len(bases)] + i + a, b = pool[0], pool[1] + yield "S1", f"syn-S1-{i:02d}", _s1(a, b, unit, base, 2.0 + (i % 4)) + yield "S2", f"syn-S2-{i:02d}", _s2(a, b, unit, base, base + 9.0, 3.0 + (i % 5)) + yield "S3", f"syn-S3-{i:02d}", _s3(a, unit, base, 4.0 + (i % 6), 7.0 + (i % 3)) + yield "S4", f"syn-S4-{i:02d}", _s4(a, unit, base, 5.0 + (i % 7)) + + +# -------------------------------------------------------------------------- +# Control sets +# -------------------------------------------------------------------------- + + +def invariance_pairs() -> list[tuple[str, Case, Case]]: + """§5.3b — same structure, attributes perturbed three ways. + + ``rename`` entity names swapped for a disjoint pool; numbers identical. + ``rescale`` every quantity multiplied by 3; names identical. + ``jitter`` quantities changed non-uniformly (the hard case: not a scale). + """ + pairs: list[tuple[str, Case, Case]] = [] + p0, p1 = _NAME_POOLS[0], _NAME_POOLS[1] + unit = "marbles" + + def case(cid: str, g: MathProblemGraph) -> Case: + return Case(case_id=cid, graph=g, provenance="synthetic", template="inv") + + base_s1 = _s1(p0[0], p0[1], unit, 20.0, 3.0) + pairs.append(("rename", case("inv-S1-base", base_s1), case("inv-S1-rename", _s1(p1[0], p1[1], unit, 20.0, 3.0)))) + pairs.append(("rescale", case("inv-S1-base", base_s1), case("inv-S1-rescale", _s1(p0[0], p0[1], unit, 60.0, 3.0)))) + pairs.append(("jitter", case("inv-S1-base", base_s1), case("inv-S1-jitter", _s1(p0[0], p0[1], unit, 29.0, 4.0)))) + + base_s2 = _s2(p0[0], p0[1], unit, 40.0, 25.0, 8.0) + pairs.append(("rename", case("inv-S2-base", base_s2), case("inv-S2-rename", _s2(p1[0], p1[1], unit, 40.0, 25.0, 8.0)))) + pairs.append(("rescale", case("inv-S2-base", base_s2), case("inv-S2-rescale", _s2(p0[0], p0[1], unit, 120.0, 75.0, 24.0)))) + pairs.append(("jitter", case("inv-S2-base", base_s2), case("inv-S2-jitter", _s2(p0[0], p0[1], unit, 57.0, 19.0, 13.0)))) + + base_s3 = _s3(p0[0], unit, 30.0, 6.0, 9.0) + pairs.append(("rename", case("inv-S3-base", base_s3), case("inv-S3-rename", _s3(p1[0], unit, 30.0, 6.0, 9.0)))) + pairs.append(("rescale", case("inv-S3-base", base_s3), case("inv-S3-rescale", _s3(p0[0], unit, 90.0, 18.0, 27.0)))) + pairs.append(("jitter", case("inv-S3-base", base_s3), case("inv-S3-jitter", _s3(p0[0], unit, 44.0, 11.0, 5.0)))) + + base_s4 = _s4(p0[0], unit, 50.0, 12.0) + pairs.append(("rename", case("inv-S4-base", base_s4), case("inv-S4-rename", _s4(p1[0], unit, 50.0, 12.0)))) + pairs.append(("rescale", case("inv-S4-base", base_s4), case("inv-S4-rescale", _s4(p0[0], unit, 150.0, 36.0)))) + pairs.append(("jitter", case("inv-S4-base", base_s4), case("inv-S4-jitter", _s4(p0[0], unit, 71.0, 4.0)))) + return pairs + + +def sensitivity_pairs() -> list[tuple[str, Case, Case]]: + """§5.3c — minimal pairs: identical roles, identical numbers, kind changed. + + This is the control that separates "aligns by structure" from "aligns by + anything else": the two graphs in each pair differ in exactly one field. + """ + p = _NAME_POOLS[0] + unit = "marbles" + + def case(cid: str, g: MathProblemGraph) -> Case: + return Case(case_id=cid, graph=g, provenance="synthetic", template="sens") + + out: list[tuple[str, Case, Case]] = [] + # Same two entities, same magnitudes — compare vs transfer. + out.append( + ( + "S1-vs-S2", + case("sens-S1", _s1(p[0], p[1], unit, 40.0, 3.0)), + case("sens-S2", _s2(p[0], p[1], unit, 40.0, 3.0, 3.0)), + ) + ) + # Same single entity, same magnitudes — gain vs loss. + out.append( + ( + "S3-vs-S4", + case("sens-S3", _s3(p[0], unit, 40.0, 3.0, 3.0)), + case("sens-S4b", _s4(p[0], unit, 40.0, 3.0)), + ) + ) + # Same single entity and numbers, one added operation vs one subtracted. + out.append( + ( + "add-vs-subtract", + case("sens-add", _s4(p[0], unit, 40.0, 3.0)), + case( + "sens-sub", + graph_from_dict( + { + "entities": [p[0]], + "initial_state": [ + {"entity": p[0], "quantity": {"value": 40.0, "unit": unit}} + ], + "operations": [ + { + "actor": p[0], + "kind": "add", + "operand": {"value": 3.0, "unit": unit}, + } + ], + "unknown": {"entity": p[0], "unit": unit}, + } + ), + ), + ) + ) + return out + + +def systematicity_pairs() -> list[tuple[str, Case, Case]]: + """§5.3d — chained higher-order structure (A = 2B, B = 2C).""" + p, q = _NAME_POOLS[0], _NAME_POOLS[1] + unit = "marbles" + + def chain(names: Sequence[str], base: float, k1: float, k2: float) -> MathProblemGraph: + a, b, c = names[0], names[1], names[2] + return graph_from_dict( + { + "entities": [a, b, c], + "initial_state": [{"entity": a, "quantity": {"value": base, "unit": unit}}], + "operations": [ + { + "actor": b, + "kind": "compare_multiplicative", + "operand": {"direction": "times", "reference_actor": a, "factor": k1}, + }, + { + "actor": c, + "kind": "compare_multiplicative", + "operand": {"direction": "times", "reference_actor": b, "factor": k2}, + }, + ], + "unknown": {"entity": c, "unit": unit}, + } + ) + + def case(cid: str, g: MathProblemGraph) -> Case: + return Case(case_id=cid, graph=g, provenance="synthetic", template="syst") + + chain_a = case("syst-chain-a", chain(p, 10.0, 2.0, 2.0)) + chain_b = case("syst-chain-b", chain(q, 37.0, 5.0, 3.0)) + flat = case("syst-flat", _s1(p[0], p[1], unit, 10.0, 2.0)) + return [("chain-vs-chain", chain_a, chain_b), ("chain-vs-flat", chain_a, flat)] + + +# -------------------------------------------------------------------------- +# Assembly +# -------------------------------------------------------------------------- + + +def build_corpus() -> tuple[list[Case], dict[str, str]]: + """Return ``(cases, labels)``. The caller must keep them apart until scoring.""" + cases: list[Case] = [] + labels: dict[str, str] = {} + + for real in extract_holdout_graphs(): + cases.append(real) + labels[real.case_id] = "S1" + + for label, cid, graph in _realizations(): + cases.append(Case(case_id=cid, graph=graph, provenance="synthetic", template=label)) + labels[cid] = label + + seen: set[str] = set() + for c in cases: + if c.case_id in seen: + raise ValueError(f"duplicate case_id in corpus: {c.case_id}") + seen.add(c.case_id) + return cases, labels + + +def load_labels() -> dict[str, str]: + """Labels only. Kept as a separate entry point so blindness is greppable.""" + return build_corpus()[1] diff --git a/evals/structure_mapping/adr0252_s5/embedding.py b/evals/structure_mapping/adr0252_s5/embedding.py new file mode 100644 index 00000000..be611920 --- /dev/null +++ b/evals/structure_mapping/adr0252_s5/embedding.py @@ -0,0 +1,202 @@ +"""§5.2 — the relational embedding, stated explicitly as §5.2 demands. + +§5.2 requires the scheme be *declared*, derived from the paradigm rather than +improvised, and that the configuration encode **role-structure** — who plays +which role in which relation — rather than surface words. It also requires +(§5.3b) that residual not move with surface attributes. Those two sentences +pull against each other: an embedding that cannot see attributes is invariant +to them by construction, which makes §5.3b analytic rather than measured. + +Both readings are therefore built, from one function with one constant changed: + + RS-A ATTR_SCALE = 0.0 role skeleton only; attribute-invariance analytic + RS-B ATTR_SCALE > 0.0 quantities enter the geometry; §5.3b is measured + +The scheme +---------- +Let a graph have entities ``e_0 … e_{E-1}`` in order of introduction and +operations ``o_0 … o_{J-1}`` in source order. Names and units are never read. + +*Entity points.* Entity ``k`` sits on the unit circle of the ``e1–e2`` plane at +angle ``θ_k = 2πk / SLOTS``. Its only identity is its **role index** — the +order in which the problem introduced it. Under RS-B its held quantity ``v`` +displaces it along ``e3`` by ``ATTR_SCALE · v``. + +*Relation points.* Operation ``j`` becomes one point at angle ``φ_j``, the +circular mean of its arguments' angles — so a relation literally sits between +the roles it relates. Its radius is ``R_OP + ORDER_STEP · j``, which encodes +*when* in the story the relation occurs. Its height is ``KIND_HEIGHT[kind]``, a +declared constant per relation kind; under RS-B the operand's magnitude adds +``ATTR_SCALE · w``. + +*Unknown.* The asked-for role is marked by one point at that entity's angle, +radius ``R_UNKNOWN`` — the question is part of the problem's structure. + +*Refusal.* A relation kind with no declared height raises. There is no default +geometry for an unknown relation: guessing one would silently place unrelated +structures on top of each other, which is exactly the failure this experiment +exists to detect. + +*Padding.* Clouds are padded to ``MAX_POINTS`` with copies of the cloud's own +centroid, so a shorter configuration is not dragged toward a fixed foreign +point. Two clouds of different sizes still differ; the difference lands in +centroid mass rather than in an arbitrary corner of the space. + +The cloud is handed to ``conformal_procrustes`` as a ``(5, K)`` array of +grade-1 components — the Kabsch + Umeyama path, whose residual is invariant to +rotation, translation and uniform scale (measured: ~1e-14) and which never +raises. The alternative sequence path performs field conjugacy and *does* raise +on structurally unlike inputs; residuals harvested from that exception are not +measurements, and this module does not use it. +""" + +from __future__ import annotations + +import math +from typing import Final, Mapping, Sequence + +import numpy as np + +from algebra.cga import embed_point +from generate.math_problem_graph import ( + Comparison, + FractionPortion, + MathProblemGraph, + PartitionChunk, + Quantity, + Rate, +) + +#: Angular slots on the role circle. Fixed across the corpus so role index k +#: means the same direction in every graph. +SLOTS: Final[int] = 8 +R_ENTITY: Final[float] = 1.0 +R_OP: Final[float] = 2.0 +R_UNKNOWN: Final[float] = 3.0 +ORDER_STEP: Final[float] = 0.35 +MAX_POINTS: Final[int] = 16 + +#: Declared height per relation kind. Distinct constants, no arithmetic +#: relationship between them implied — they are labels on an axis, not values. +KIND_HEIGHT: Final[Mapping[str, float]] = { + "add": 1.0, + "subtract": -1.0, + "transfer": 2.0, + "multiply": 3.0, + "divide": -3.0, + "compare_multiplicative": 4.0, + "compare_additive": -4.0, + "apply_rate": 5.0, + "unit_partition": -5.0, + "fraction_portion": 6.0, +} + +#: RS-A / RS-B selector. +ATTR_SCALE_STRUCTURE_ONLY: Final[float] = 0.0 +ATTR_SCALE_ATTRIBUTE_BEARING: Final[float] = 0.02 + + +class EmbeddingRefusal(ValueError): + """Fail closed: no guessed geometry for an undeclared relation kind.""" + + +def _angle(role_index: int) -> float: + return 2.0 * math.pi * (role_index % SLOTS) / SLOTS + + +def _circular_mean(angles: Sequence[float]) -> float: + if not angles: + raise EmbeddingRefusal("cannot place a relation with no arguments") + x = sum(math.cos(a) for a in angles) / len(angles) + y = sum(math.sin(a) for a in angles) / len(angles) + if abs(x) < 1e-12 and abs(y) < 1e-12: + # Antipodal arguments: the mean direction is undefined. Use the first + # argument's angle rather than an arbitrary one. + return angles[0] + return math.atan2(y, x) + + +def _operand_magnitude(operand: object) -> float: + """The scalar an operand carries, or 0.0 when it carries none.""" + if isinstance(operand, Quantity): + return float(operand.value) + if isinstance(operand, Comparison): + if operand.factor is not None: + return float(operand.factor) + if operand.delta is not None: + return float(operand.delta.value) + return 0.0 + if isinstance(operand, Rate): + return float(operand.value) + if isinstance(operand, PartitionChunk): + return float(getattr(operand, "chunk_size", 0.0) or 0.0) + if isinstance(operand, FractionPortion): + num = float(getattr(operand, "numerator", 0.0) or 0.0) + den = float(getattr(operand, "denominator", 1.0) or 1.0) + return num / den if den else 0.0 + return 0.0 + + +def _second_argument(operation: object) -> str | None: + target = getattr(operation, "target", None) + if target: + return str(target) + operand = getattr(operation, "operand", None) + if isinstance(operand, Comparison): + return operand.reference_actor + return None + + +def euclidean_configuration( + graph: MathProblemGraph, *, attr_scale: float +) -> list[tuple[float, float, float]]: + """The R^3 configuration, before conformal embedding. Blind to labels.""" + role: dict[str, int] = {name: i for i, name in enumerate(graph.entities)} + held: dict[str, float] = {} + for possession in graph.initial_state: + held.setdefault(possession.entity, float(possession.quantity.value)) + + points: list[tuple[float, float, float]] = [] + + for name in graph.entities: + theta = _angle(role[name]) + z = attr_scale * held.get(name, 0.0) + points.append((R_ENTITY * math.cos(theta), R_ENTITY * math.sin(theta), z)) + + for j, operation in enumerate(graph.operations): + kind = operation.kind + if kind not in KIND_HEIGHT: + raise EmbeddingRefusal(f"no declared height for relation kind {kind!r}") + angles = [_angle(role[operation.actor])] + second = _second_argument(operation) + if second is not None and second in role: + angles.append(_angle(role[second])) + phi = _circular_mean(angles) + radius = R_OP + ORDER_STEP * j + z = KIND_HEIGHT[kind] + attr_scale * _operand_magnitude(operation.operand) + points.append((radius * math.cos(phi), radius * math.sin(phi), z)) + + asked = graph.unknown.entity + theta = _angle(role[asked]) if asked in role else 0.0 + points.append((R_UNKNOWN * math.cos(theta), R_UNKNOWN * math.sin(theta), 0.0)) + + if len(points) > MAX_POINTS: + raise EmbeddingRefusal( + f"configuration has {len(points)} points, over MAX_POINTS={MAX_POINTS}" + ) + return points + + +def embed(graph: MathProblemGraph, *, attr_scale: float) -> np.ndarray: + """Return the padded ``(5, MAX_POINTS)`` grade-1 conformal point cloud.""" + points = euclidean_configuration(graph, attr_scale=attr_scale) + centroid = ( + sum(p[0] for p in points) / len(points), + sum(p[1] for p in points) / len(points), + sum(p[2] for p in points) / len(points), + ) + padded = list(points) + [centroid] * (MAX_POINTS - len(points)) + columns = [ + embed_point(np.asarray(p, dtype=np.float64), dtype=np.float64) for p in padded + ] + return np.stack(columns, axis=1)[1:6, :] diff --git a/evals/structure_mapping/adr0252_s5/experiment.py b/evals/structure_mapping/adr0252_s5/experiment.py new file mode 100644 index 00000000..23e6a04b --- /dev/null +++ b/evals/structure_mapping/adr0252_s5/experiment.py @@ -0,0 +1,223 @@ +"""§5.3 the three decisive measurements, §5.4 the verdict — as executable rule. + +The criterion is code, not prose, and it is committed **before** the run +(``docs/research/sme-experiment-preregistration-2026-07-28.md`` records the +same thresholds and the commit that carried them). A criterion chosen after +seeing results is the failure mode this experiment has already suffered twice; +putting the thresholds in a module that the run imports is the cheapest +available guard against choosing them a third time. + +Run: ``uv run python -m evals.structure_mapping.adr0252_s5.experiment`` +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import statistics +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Final, Sequence + +import numpy as np + +from core.physics.dynamic_manifold import conformal_procrustes +from evals.structure_mapping.adr0252_s5 import corpus as corpus_mod +from evals.structure_mapping.adr0252_s5 import embedding as emb + +# --------------------------------------------------------------------------- +# The pre-registered thresholds. Changing one of these invalidates the verdict. +# --------------------------------------------------------------------------- + +#: (a) separability — ROC AUC of residual as a "different structure" score. +AUC_FLOOR: Final[float] = 0.90 +#: (a) a single threshold τ must classify both sides at this rate. +CLASSIFY_FLOOR: Final[float] = 0.95 +#: (b) attribute-variant residuals must sit under τ, and their median must be +#: no more than this fraction of the cross-structure median. +INVARIANCE_RATIO_CEILING: Final[float] = 0.25 +#: Non-convergence budget. Above this the run is VOID, not NO-GO — an +#: instrument that refuses to answer has not answered. +NONCONVERGENCE_CEILING: Final[float] = 0.05 + +VARIANTS: Final[dict[str, float]] = { + "RS-A": emb.ATTR_SCALE_STRUCTURE_ONLY, + "RS-B": emb.ATTR_SCALE_ATTRIBUTE_BEARING, +} + + +@dataclass(frozen=True) +class PairResidual: + left: str + right: str + residual: float | None # None == the aligner raised + + +def residual(a: np.ndarray, b: np.ndarray) -> float | None: + try: + _, res = conformal_procrustes(a, b) + except Exception: # noqa: BLE001 — recorded as non-convergence, never as distance + return None + value = float(res) + if not np.isfinite(value): + return None + return value + + +def roc_auc(same: Sequence[float], cross: Sequence[float]) -> float: + """P(cross residual > same residual), ties counted as half.""" + if not same or not cross: + return float("nan") + wins = 0.0 + for s in same: + for c in cross: + if c > s: + wins += 1.0 + elif c == s: + wins += 0.5 + return wins / (len(same) * len(cross)) + + +def best_threshold(same: Sequence[float], cross: Sequence[float]) -> tuple[float, float, float]: + """τ maximising min(same-below-rate, cross-above-rate); returns (τ, below, above).""" + if not same or not cross: + return (float("nan"),) * 3 + candidates = sorted(set(list(same) + list(cross))) + midpoints = [ + (candidates[i] + candidates[i + 1]) / 2.0 for i in range(len(candidates) - 1) + ] or [candidates[0]] + best = (float("nan"), 0.0, 0.0) + best_score = -1.0 + for tau in midpoints: + below = sum(1 for s in same if s < tau) / len(same) + above = sum(1 for c in cross if c > tau) / len(cross) + score = min(below, above) + if score > best_score: + best_score, best = score, (tau, below, above) + return best + + +def run_variant(name: str, attr_scale: float) -> dict[str, Any]: + cases, labels = corpus_mod.build_corpus() + + # --- blind stage: labels are not read below this line until scoring --- + clouds: dict[str, np.ndarray] = {c.case_id: emb.embed(c.graph, attr_scale=attr_scale) for c in cases} + ids = [c.case_id for c in cases] + pairs: list[PairResidual] = [] + for i in range(len(ids)): + for j in range(i + 1, len(ids)): + pairs.append(PairResidual(ids[i], ids[j], residual(clouds[ids[i]], clouds[ids[j]]))) + # --- scoring stage: labels enter here and nowhere earlier --- + + nonconverged = [p for p in pairs if p.residual is None] + converged = [p for p in pairs if p.residual is not None] + noncon_rate = len(nonconverged) / len(pairs) if pairs else 0.0 + + same = [p.residual for p in converged if labels[p.left] == labels[p.right]] + cross = [p.residual for p in converged if labels[p.left] != labels[p.right]] + + auc = roc_auc(same, cross) + tau, below, above = best_threshold(same, cross) + margin = (min(cross) - max(same)) if same and cross else float("nan") + separable = ( + noncon_rate <= NONCONVERGENCE_CEILING + and auc >= AUC_FLOOR + and below >= CLASSIFY_FLOOR + and above >= CLASSIFY_FLOOR + ) + + # (b) attribute-invariance + inv_rows = [] + for kind, left, right in corpus_mod.invariance_pairs(): + r = residual(emb.embed(left.graph, attr_scale=attr_scale), emb.embed(right.graph, attr_scale=attr_scale)) + inv_rows.append({"perturbation": kind, "left": left.case_id, "right": right.case_id, "residual": r}) + inv_values = [row["residual"] for row in inv_rows if row["residual"] is not None] + cross_median = statistics.median(cross) if cross else float("nan") + inv_median = statistics.median(inv_values) if inv_values else float("nan") + invariant = ( + len(inv_values) == len(inv_rows) + and all(v < tau for v in inv_values) + and (inv_median <= INVARIANCE_RATIO_CEILING * cross_median) + ) + + # (c) structure-sensitivity + sens_rows = [] + for kind, left, right in corpus_mod.sensitivity_pairs(): + r = residual(emb.embed(left.graph, attr_scale=attr_scale), emb.embed(right.graph, attr_scale=attr_scale)) + sens_rows.append({"minimal_pair": kind, "left": left.case_id, "right": right.case_id, "residual": r}) + sens_values = [row["residual"] for row in sens_rows if row["residual"] is not None] + sensitive = len(sens_values) == len(sens_rows) and all(v > tau for v in sens_values) + + # (d) systematicity — reported, not gating (§5.4 names only a, b, c) + syst_rows = [] + for kind, left, right in corpus_mod.systematicity_pairs(): + r = residual(emb.embed(left.graph, attr_scale=attr_scale), emb.embed(right.graph, attr_scale=attr_scale)) + syst_rows.append({"comparison": kind, "left": left.case_id, "right": right.case_id, "residual": r}) + + verdict = "GO" if (separable and invariant and sensitive) else "NO-GO" + if noncon_rate > NONCONVERGENCE_CEILING: + verdict = "VOID" + + return { + "variant": name, + "attr_scale": attr_scale, + "n_cases": len(cases), + "n_pairs": len(pairs), + "nonconvergence_rate": noncon_rate, + "separability": { + "auc": auc, + "threshold": tau, + "same_below_rate": below, + "cross_above_rate": above, + "same_median": statistics.median(same) if same else None, + "cross_median": cross_median, + "margin_min_cross_minus_max_same": margin, + "n_same_pairs": len(same), + "n_cross_pairs": len(cross), + "pass": separable, + }, + "attribute_invariance": { + "rows": inv_rows, + "median": inv_median, + "ratio_to_cross_median": (inv_median / cross_median) if cross_median else None, + "pass": invariant, + }, + "structure_sensitivity": {"rows": sens_rows, "pass": sensitive}, + "systematicity": {"rows": syst_rows}, + "verdict": verdict, + } + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, default=None, help="write the JSON report here") + args = parser.parse_args(argv) + + report: dict[str, Any] = { + "experiment": "ADR-0252 §5 structure-mapping acceptance gate", + "criterion": { + "auc_floor": AUC_FLOOR, + "classify_floor": CLASSIFY_FLOOR, + "invariance_ratio_ceiling": INVARIANCE_RATIO_CEILING, + "nonconvergence_ceiling": NONCONVERGENCE_CEILING, + "verdict_rule": "GO iff separability AND attribute-invariance AND structure-sensitivity on the same variant", + }, + "variants": [run_variant(name, scale) for name, scale in VARIANTS.items()], + } + verdicts = {v["variant"]: v["verdict"] for v in report["variants"]} + report["verdict_by_variant"] = verdicts + report["overall_verdict"] = "GO" if "GO" in verdicts.values() else "NO-GO" + + payload = json.dumps(report, indent=2, sort_keys=True, default=float) + report["deterministic_digest"] = hashlib.sha256(payload.encode()).hexdigest() + text = json.dumps(report, indent=2, sort_keys=True, default=float) + if args.out is not None: + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(text + "\n") + print(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())