diff --git a/docs/research/trackb-s1-slice-2026-07-20.md b/docs/research/trackb-s1-slice-2026-07-20.md new file mode 100644 index 00000000..156a9211 --- /dev/null +++ b/docs/research/trackb-s1-slice-2026-07-20.md @@ -0,0 +1,173 @@ +# Track B, Increment 1 — Symbolic Structure-Mapping S1 vertical slice + +**Date**: 2026-07-20 +**Branch**: `feat/trackb-symbolic-sme-s1` +**Paradigm**: ADR-0252 (Expert Structure-Mapping on a Predictive-Processing Substrate) +**§5 status**: NO-GO for geometric SME (`conformal_procrustes` is Kabsch/Umeyama and presupposes role correspondence). PRs #83 and #84 closed as superseded. +**Scope**: S1 only (compare-multiplicative-then-total). Off-serving. No S2–S4. No serving cutover. + +--- + +## 1. What shipped + +| Piece | Path | Role | +| --- | --- | --- | +| Role-predicate schema | `generate/structure_mapping/role_predicate.py` | Typed `contain` / `transfer` / `compare` / `rate` / `total` | +| Converter | `generate/structure_mapping/convert.py` | Deterministic `MathProblemGraph` → `RoleGraph` | +| S1 canonical | `generate/structure_mapping/canonicals.py` | `compare(b,a,k) ∧ contain(a,…) ∧ total(a,b,sum)` — role vars only | +| Symbolic mapper | `generate/structure_mapping/mapper.py` | Relational match → binding or refuse; **no gold labels** | +| Solve corridor | `generate/structure_mapping/solve_s1.py` | Binding → graph → classical `solve`+`verify` **and** multi-register certificate; emit only if both agree | +| Scoring labels (isolated) | `evals/structure_mapping/scoring/` | Gold S1 ids for scoring only; forbidden from mapper imports | +| Measure script | `scripts/measure_trackb_s1_slice.py` | Blind holdout / FP / surface-variant / vs-organ | +| Unit tests | `tests/test_trackb_s1_structure_mapping.py` | 16 tests driving shipped code | + +Serving reader (`parse_and_solve` dispatch, `resolve_promotable_*`, `chat/runtime.py`) **untouched** (diff bytes vs main = 0). + +--- + +## 2. Discipline checks + +### Blindness +- Mapper signature: `map_to_s1(role_graph: RoleGraph)` — no label parameter. +- Static audit: `generate/structure_mapping/` does not import `evals.structure_mapping.scoring` or load `holdout_dev_v1_labels.jsonl`. +- Labels enter only after map decisions, in the measure script / scorer. + +### No crash-as-signal +- Solve path catches `MultiRegisterError` / classical failures and returns refuse outcomes; exceptions are never treated as residual scores. + +### wrong=0 +- Emit only when classical verifier passes **and** multi-register chain is certified **and** answers agree within relative 1e-6. + +--- + +## 3. Measurements (command + real output) + +### 3a. Parse→role coverage on S1 holdout cohort + +```text +$ uv run python scripts/measure_trackb_s1_slice.py --mode coverage +=== parse→role coverage on S1 holdout === +gsm8k-holdout-dev-v1-0101 parsed=True kinds=['compare', 'contain', 'total'] mapped_s1=True refuse=None +gsm8k-holdout-dev-v1-0108 parsed=True kinds=['compare', 'contain', 'total'] mapped_s1=True refuse=None +gsm8k-holdout-dev-v1-0268 parsed=True kinds=['compare', 'contain', 'total'] mapped_s1=True refuse=None +gsm8k-holdout-dev-v1-0411 parsed=True kinds=['compare', 'contain', 'total'] mapped_s1=True refuse=None +gsm8k-holdout-dev-v1-0453 parsed=True kinds=['compare', 'contain', 'total'] mapped_s1=True refuse=None +``` + +Coverage: **5/5** S1 organ cases parse → role graph with `{compare, contain, total}` and map to S1. + +### 3b. Right-for-right-reason (S1 holdout) + +```text +$ uv run python scripts/measure_trackb_s1_slice.py --mode right-reason +=== S1 right-for-right-reason (holdout_dev/v1 S1 cohort) === +CASE gsm8k-holdout-dev-v1-0101 emit=9.0 gold=9.0 match=True wrong_flag=False derivation='S1: Frankie = 2.0 × Eduardo; Eduardo = 3.0; total = Eduardo + Frankie = 3.0 + 2.0*3.0 = 9.0' mr_cert=True classical_v=True binding={'a': 'Eduardo', 'b': 'Frankie', 'k': 2.0, 'a_value': 3.0, 'unit': 'lasts', 'canonical_id': 'S1'} ... +CASE gsm8k-holdout-dev-v1-0108 emit=110.0 gold=110.0 match=True ... derivation='S1: Dana Point Beach = 4.0 × Newport Beach; ...' +CASE gsm8k-holdout-dev-v1-0268 emit=16.0 gold=16.0 match=True ... derivation='S1: Bruce Scored 4 Goals While Michael = 3.0 × Bruce; ...' +CASE gsm8k-holdout-dev-v1-0411 emit=52.0 gold=52.0 match=True ... derivation='S1: male students = 3.0 × female students; ...' +CASE gsm8k-holdout-dev-v1-0453 emit=28.0 gold=28.0 match=True ... derivation='S1: Olaf = 3.0 × dad; dad = 7.0; total = dad + Olaf = 7.0 + 3.0*7.0 = 28.0' ... +SUMMARY right-reason: correct=5 wrong=0 refused=0 n=5 +``` + +**Result**: `correct=5 wrong=0 refused=0`. Every emit carries a derivation of the form `b = k×a; total = a+b`, with multi-register certificate + classical verify both true. + +> Note on residuals: early runs emitted multi-register decoded floats (~`8.999999982`). That is geometric relaxation noise, **not** a zero residual trophy. The ship path requires MR certification then emits the classically verified scalar once corridors agree (relative 1e-6). Both corridors remain mandatory. + +### 3c. False-positive / separability + +```text +$ uv run python scripts/measure_trackb_s1_slice.py --mode false-positive +=== A: holdout_dev/v1 non-S1 cases (parse→map) === +SUMMARY holdout-non-S1: fp=0 tn=495 n_non_s1=495 parse_fail_among_non_s1=495 fp_rate=0.000000 +NOTE: holdout_dev/v1 currently yields selected_graph on only the 5 S1 organ cases; non-S1 holdout FP is vacuous at the graph layer when parse_fail=495. Layer B/C are the real separability tests. +=== B: synthetic non-S1 graphs === + transfer: mapped_s1=False refuse=non_s1_has_transfer + contain_total_no_compare: mapped_s1=False refuse=compare_count_not_one:0 + compare_no_total: mapped_s1=False refuse=missing_total_query + compare_with_b_seed: mapped_s1=False refuse=b_has_independent_contain_not_pure_s1 + apply_rate: mapped_s1=False refuse=non_s1_has_rate +SUMMARY synthetic: fp=0 tn=5 n=5 fp_rate=0.000000 +=== C: other gsm8k_math corpora non-compare graphs === +SUMMARY other-corpus non-S1 graphs: fp=0 tn=135 n=135 fp_rate=0.000000 +``` + +**Honest reading**: +- Layer A (holdout non-S1): **vacuous at graph level** — the serving reader produces `selected_graph` only for the five S1 cases on this split. Reporting `fp_rate=0` here without the NOTE would be the same class of cheat as prior geometric “GO”s. We refuse that claim. +- Layer B (synthetic): **0/5** false positives on transfer / additive-total / compare-without-total / dual-seed compare / rate. +- Layer C (other corpora, 135 non-compare graphs): **0/135** false positives (mostly transfers from `public/`). + +### 3d. Surface-variant generalization (re-parse) + +```text +$ uv run python scripts/measure_trackb_s1_slice.py --mode surface-variant +BASE_ID gsm8k-holdout-dev-v1-0108 +BASE_TEXT Dana Point beach has four times the number of sharks as Newport Beach. If Newport Beach has 22 sharks, how many sharks are there in total on the two beaches? +VARIANT_TEXT Cedar Cove beach has five times the number of dolphins as Harbor Beach. If Harbor Beach has 11 dolphins, how many dolphins are there in total on the two beaches? +BASE_MAP mapped_s1=True binding k=4.0 a_value=22.0 ... +VARIANT_MAP mapped_s1=True binding={'a': 'Harbor Beach', 'b': 'Cedar Cove Beach', 'k': 5.0, 'a_value': 11.0, 'unit': 'dolphins', ...} +VARIANT_SOLVE emitted=True answer=66.0 expected=66.0 derivation='S1: Cedar Cove Beach = 5.0 × Harbor Beach; Harbor Beach = 11.0; total = ... = 66.0' +SUMMARY surface-variant: map=True k_ok=True a_ok=True names_ok=True ans_ok=True +``` + +Re-parsed renamed/re-numbered text still maps to S1 with the **new** bindings and solves correctly. Driven by relations, not surface tokens. + +### 3e. Side-by-side vs S1 surface organ + +```text +$ uv run python scripts/measure_trackb_s1_slice.py --mode vs-organ +case_id organ trackb gold organ_ok tb_ok map +gsm8k-holdout-dev-v1-0101 9.0 9.0 9.0 True True True +gsm8k-holdout-dev-v1-0108 110.0 110.0 110.0 True True True +gsm8k-holdout-dev-v1-0268 16.0 16.0 16.0 True True True +gsm8k-holdout-dev-v1-0411 52.0 52.0 52.0 True True True +gsm8k-holdout-dev-v1-0453 28.0 28.0 28.0 True True True +``` + +Track B matches the organ on all five S1 cases (answer + map). Organ path remains the serving `parse_and_solve`; Track B is a parallel off-serving structure-map → certificate path. + +--- + +## 4. Unit tests + +```text +$ uv run pytest tests/test_trackb_s1_structure_mapping.py -v +16 passed in 0.62s +``` + +Includes real holdout texts, refuse paths, blindness import audit, and dual-corridor emit. + +--- + +## 5. Baselines (serving invariant) + +Pre-edit (worktree at main tip `1ccef491`, before Track B code): + +```text +holdout report (committed): {'correct': 5, 'refused': 495, 'wrong': 0} n=500 +uv run core test --suite smoke -q → 176 passed in 134.05s +``` + +Post-edit: same smoke suite (off-serving code only; serving modules diff empty). Holdout serving numbers unchanged by construction (no live path edits). + +--- + +## 6. Explicit non-claims / STOP gate + +- **Not claimed**: geometric SME revival; S2–S4 coverage; serving cutover; sealed-test movement. +- **Not claimed**: holdout non-S1 graph-level separability beyond the vacuous parse layer — only synthetic + other-corpus graphs. +- **Not claimed**: case 0148 (classic “twice the total…”) — reader currently refuses parse; out of converter reach until parse coverage grows. +- **STOP**: human reviewer verifies this slice against the code before any generalization. + +--- + +## 7. Inventory: proven vs assumed + +| Item | Status | +| --- | --- | +| S1 map+solve on organ cohort (5) | **Proven** (commands above) | +| wrong=0 on emitted path | **Proven** | +| Surface-variant re-parse | **Proven** | +| FP on synthetic + 135 non-compare graphs | **Proven** | +| Holdout non-S1 graph FP | **Unclaimed** (vacuous parse layer) | +| Generalization to S2–S4 | **Not in scope** | +| Serving supersede of organ | **Not in scope** | diff --git a/evals/structure_mapping/__init__.py b/evals/structure_mapping/__init__.py new file mode 100644 index 00000000..8f0e70ce --- /dev/null +++ b/evals/structure_mapping/__init__.py @@ -0,0 +1 @@ +"""Structure-mapping eval package (scoring + measure scripts). Off-serving.""" diff --git a/evals/structure_mapping/scoring/__init__.py b/evals/structure_mapping/scoring/__init__.py new file mode 100644 index 00000000..28a2fc63 --- /dev/null +++ b/evals/structure_mapping/scoring/__init__.py @@ -0,0 +1,18 @@ +"""Scoring-only helpers for structure-mapping evals. + +**CRITICAL:** These modules load gold structure labels. They must never be +imported by ``generate/structure_mapping`` (the mapper). Tests that audit +blindness check this import boundary. +""" + +from evals.structure_mapping.scoring.labels import ( + S1_HOLDOUT_CASE_IDS, + load_structure_labels, + score_label, +) + +__all__ = [ + "S1_HOLDOUT_CASE_IDS", + "load_structure_labels", + "score_label", +] diff --git a/evals/structure_mapping/scoring/holdout_dev_v1_labels.jsonl b/evals/structure_mapping/scoring/holdout_dev_v1_labels.jsonl new file mode 100644 index 00000000..ee8f8a20 --- /dev/null +++ b/evals/structure_mapping/scoring/holdout_dev_v1_labels.jsonl @@ -0,0 +1,5 @@ +{"case_id": "gsm8k-holdout-dev-v1-0101", "label": "S1", "note": "compare_mult then total; organ cohort"} +{"case_id": "gsm8k-holdout-dev-v1-0108", "label": "S1", "note": "compare_mult then total; organ cohort"} +{"case_id": "gsm8k-holdout-dev-v1-0268", "label": "S1", "note": "compare_mult then total; organ cohort"} +{"case_id": "gsm8k-holdout-dev-v1-0411", "label": "S1", "note": "compare_mult then total; organ cohort"} +{"case_id": "gsm8k-holdout-dev-v1-0453", "label": "S1", "note": "compare_mult then total; organ cohort"} diff --git a/evals/structure_mapping/scoring/labels.py b/evals/structure_mapping/scoring/labels.py new file mode 100644 index 00000000..c343cb7f --- /dev/null +++ b/evals/structure_mapping/scoring/labels.py @@ -0,0 +1,79 @@ +"""Isolated gold structure labels for scoring — never mapper input. + +Source of S1 membership for holdout_dev/v1: the five cases that the serving +reader already solves via ``compare_multiplicative`` + total (documented in +``docs/research/compare-multiplicative-increment-plan-2026-07-18.md`` §2). +These ids are scoring-only; the symbolic mapper never reads this module. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Final, Mapping + +# Absolute family membership for Increment 1 scoring on holdout_dev/v1. +# Labels are NOT derived at map time from parse kind alone in the scorer's +# primary list — they are the documented S1 organ cohort. Additional cases +# may be labeled S1 in extended JSONL if human-reviewed later. +S1_HOLDOUT_CASE_IDS: Final[frozenset[str]] = frozenset( + { + "gsm8k-holdout-dev-v1-0101", + "gsm8k-holdout-dev-v1-0108", + "gsm8k-holdout-dev-v1-0268", + "gsm8k-holdout-dev-v1-0411", + "gsm8k-holdout-dev-v1-0453", + } +) + +_LABELS_PATH: Final[Path] = Path(__file__).resolve().parent / "holdout_dev_v1_labels.jsonl" + + +def load_structure_labels(path: Path | None = None) -> dict[str, str]: + """Load ``case_id → structure_label`` from the isolated scoring file.""" + p = path if path is not None else _LABELS_PATH + out: dict[str, str] = {} + text = p.read_text(encoding="utf-8") + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + row = json.loads(line) + cid = row["case_id"] + lab = row["label"] + if not isinstance(cid, str) or not isinstance(lab, str): + raise ValueError(f"malformed label row: {row!r}") + out[cid] = lab + return out + + +def score_label( + case_id: str, + predicted_s1: bool, + labels: Mapping[str, str] | None = None, +) -> dict[str, object]: + """Score a binary S1-map decision against gold. Scoring-only API.""" + lab_map = labels if labels is not None else load_structure_labels() + gold = lab_map.get(case_id) + if gold is None: + return { + "case_id": case_id, + "gold": None, + "predicted_s1": predicted_s1, + "tp": False, + "fp": False, + "fn": False, + "tn": False, + "note": "unlabeled", + } + is_s1 = gold == "S1" + return { + "case_id": case_id, + "gold": gold, + "predicted_s1": predicted_s1, + "tp": bool(predicted_s1 and is_s1), + "fp": bool(predicted_s1 and not is_s1), + "fn": bool((not predicted_s1) and is_s1), + "tn": bool((not predicted_s1) and not is_s1), + "note": None, + } diff --git a/generate/structure_mapping/__init__.py b/generate/structure_mapping/__init__.py new file mode 100644 index 00000000..fb23045d --- /dev/null +++ b/generate/structure_mapping/__init__.py @@ -0,0 +1,47 @@ +"""Track B symbolic structure-mapping (ADR-0252 §6 / Increment 1). + +Off-serving. Discovers role correspondence via Gentner-style symbolic +relational graph match — not geometric Procrustes (ruled NO-GO: Kabsch +presupposes the correspondence structure-mapping must discover). + +Public surface for the S1 vertical slice only. Gold structure labels +(S1–S4) never enter the mapper; scoring helpers live in +``evals/structure_mapping/scoring`` and must not be imported here. +""" + +from __future__ import annotations + +from generate.structure_mapping.canonicals import S1_CANONICAL, CanonicalStructure +from generate.structure_mapping.convert import graph_to_role_graph +from generate.structure_mapping.mapper import ( + StructureMapRefuse, + StructureMapResult, + map_to_s1, +) +from generate.structure_mapping.role_predicate import ( + PredicateKind, + RoleGraph, + RolePredicate, + RoleTerm, +) +from generate.structure_mapping.solve_s1 import ( + S1SolveOutcome, + solve_s1_binding, + try_s1_structure_map_and_solve, +) + +__all__ = [ + "CanonicalStructure", + "PredicateKind", + "RoleGraph", + "RolePredicate", + "RoleTerm", + "S1_CANONICAL", + "S1SolveOutcome", + "StructureMapRefuse", + "StructureMapResult", + "graph_to_role_graph", + "map_to_s1", + "solve_s1_binding", + "try_s1_structure_map_and_solve", +] diff --git a/generate/structure_mapping/canonicals.py b/generate/structure_mapping/canonicals.py new file mode 100644 index 00000000..8be6d1e9 --- /dev/null +++ b/generate/structure_mapping/canonicals.py @@ -0,0 +1,54 @@ +"""Canonical role-predicate skeletons (expert schema library). + +Increment 1 ships **S1 only**: compare-multiplicative-then-total +(``compare(b, a, k) ∧ total(a, b, sum)``). Typed roles; no surface literals +as schema identity. S2–S4 deliberately absent until the S1 slice is reviewed. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from generate.structure_mapping.role_predicate import RolePredicate, RoleTerm + + +@dataclass(frozen=True, slots=True) +class CanonicalStructure: + """A named schema with abstract role variables (not surface attributes).""" + + structure_id: str + """Stable id for the schema (e.g. ``S1``). Used only as prior-knowledge + library key — never read from gold labels at match time.""" + + description: str + predicates: tuple[RolePredicate, ...] + """Abstract skeleton; terms are ``var`` roles only.""" + + +def _var(name: str) -> RoleTerm: + return RoleTerm(kind="var", name=name) + + +# S1: "b is k× a; find a+b" +# compare(b, a, k) ∧ contain(a, a_qty) ∧ total(a, b, sum) +S1_CANONICAL: CanonicalStructure = CanonicalStructure( + structure_id="S1", + description=( + "compare-multiplicative-then-total: b = k×a; query a+b. " + "Roles: a=reference quantity entity, b=scaled entity, k=factor." + ), + predicates=( + RolePredicate( + kind="contain", + args=(_var("a"), _var("a_qty")), + ), + RolePredicate( + kind="compare", + args=(_var("b"), _var("a"), _var("k")), + ), + RolePredicate( + kind="total", + args=(_var("a"), _var("b"), _var("sum")), + ), + ), +) diff --git a/generate/structure_mapping/convert.py b/generate/structure_mapping/convert.py new file mode 100644 index 00000000..079df64e --- /dev/null +++ b/generate/structure_mapping/convert.py @@ -0,0 +1,170 @@ +"""Deterministic MathProblemGraph → role-predicate graph converter. + +Pure function of the typed surface graph. No structure labels. No geometry. +""" + +from __future__ import annotations + +import hashlib +from typing import Iterable + +from generate.math_problem_graph import ( + Comparison, + MathProblemGraph, + Quantity, + Rate, +) +from generate.structure_mapping.role_predicate import ( + RoleGraph, + RolePredicate, + RoleTerm, +) + + +def graph_to_role_graph(graph: MathProblemGraph) -> RoleGraph: + """Convert a parsed :class:`MathProblemGraph` into a :class:`RoleGraph`. + + Mapping rules (load-bearing): + - ``initial_state`` → ``contain(entity, qty)`` + - ``compare_multiplicative`` → ``compare(actor, reference, factor)`` + (actor is k× reference — S1 role order ``compare(b, a, k)``) + - ``transfer`` → ``transfer(src, dst, qty)`` + - ``apply_rate`` → ``rate`` with rate value / denominator-as-count placeholder + - ``unknown.entity is None`` → ``total`` over entities that hold ``unknown.unit`` + (or all entities when unit carriers are incomplete) + - other ops (add/subtract/multiply/…) emit no role predicates in this slice + (they are attribute-level or deferred families) + + Returns a sorted, frozen role graph. Raises nothing on partial coverage — + empty predicate lists are valid (mapper will refuse). + """ + if not isinstance(graph, MathProblemGraph): + raise TypeError( + f"graph_to_role_graph expects MathProblemGraph, got {type(graph).__name__}" + ) + + preds: list[RolePredicate] = [] + + for poss in graph.initial_state: + preds.append( + RolePredicate( + kind="contain", + args=( + RoleTerm(kind="entity", name=poss.entity), + RoleTerm( + kind="quantity", + name=f"q:{poss.entity}:{poss.quantity.unit}", + value=float(poss.quantity.value), + unit=poss.quantity.unit, + ), + ), + ) + ) + + for op in graph.operations: + if op.kind == "compare_multiplicative": + assert isinstance(op.operand, Comparison) + factor = float(op.operand.factor) if op.operand.factor is not None else None + if factor is None: + continue + preds.append( + RolePredicate( + kind="compare", + args=( + RoleTerm(kind="entity", name=op.actor), # b + RoleTerm(kind="entity", name=op.operand.reference_actor), # a + RoleTerm( + kind="quantity", + name="factor", + value=factor, + unit=None, + ), # k + ), + ) + ) + elif op.kind == "transfer": + if not isinstance(op.operand, Quantity) or op.target is None: + continue + preds.append( + RolePredicate( + kind="transfer", + args=( + RoleTerm(kind="entity", name=op.actor), + RoleTerm(kind="entity", name=op.target), + RoleTerm( + kind="quantity", + name=f"xfer:{op.actor}:{op.target}", + value=float(op.operand.value), + unit=op.operand.unit, + ), + ), + ) + ) + elif op.kind == "apply_rate": + if not isinstance(op.operand, Rate): + continue + preds.append( + RolePredicate( + kind="rate", + args=( + RoleTerm( + kind="quantity", + name="rate_value", + value=float(op.operand.value), + unit=op.operand.numerator_unit, + ), + RoleTerm(kind="entity", name=op.actor), + RoleTerm( + kind="quantity", + name="rate_total_slot", + value=0.0, + unit=op.operand.numerator_unit, + ), + ), + ) + ) + + # Total query: unknown over all entities (sum) — S1 family signature. + if graph.unknown.entity is None: + unit = graph.unknown.unit + holders = _entities_with_unit(graph, unit) + if not holders: + holders = list(graph.entities) + part_terms = tuple(RoleTerm(kind="entity", name=e) for e in holders) + sum_term = RoleTerm( + kind="quantity", + name="sum_query", + value=0.0, # unknown; placeholder attribute + unit=unit, + ) + preds.append(RolePredicate(kind="total", args=part_terms + (sum_term,))) + + ordered = tuple(_sort_predicates(preds)) + digest = hashlib.sha256(graph.canonical_bytes()).hexdigest() + return RoleGraph(predicates=ordered, source_graph_hash=digest) + + +def _entities_with_unit(graph: MathProblemGraph, unit: str) -> list[str]: + seen: list[str] = [] + for poss in graph.initial_state: + if poss.quantity.unit == unit and poss.entity not in seen: + seen.append(poss.entity) + for op in graph.operations: + if op.kind == "compare_multiplicative" and isinstance(op.operand, Comparison): + # compare-defined entity inherits the reference unit; include actor + if op.actor not in seen: + seen.append(op.actor) + if op.operand.reference_actor not in seen: + seen.append(op.operand.reference_actor) + return seen + + +def _sort_predicates(preds: Iterable[RolePredicate]) -> list[RolePredicate]: + def key(p: RolePredicate) -> tuple: + arg_key = tuple( + (a.kind, a.name, a.value if a.value is not None else 0.0, a.unit or "") + for a in p.args + ) + return (p.kind, arg_key) + + return sorted(preds, key=key) diff --git a/generate/structure_mapping/mapper.py b/generate/structure_mapping/mapper.py new file mode 100644 index 00000000..bcf93ca8 --- /dev/null +++ b/generate/structure_mapping/mapper.py @@ -0,0 +1,153 @@ +"""Symbolic relational structure-mapper — S1 vertical slice (Track B). + +Aligns a problem's role-predicate graph to the S1 canonical by **relational +role matching** (predicate kinds + systematicity). Surface attributes +(entity names, numbers) become the binding; they do not drive the match. + +Blind: never reads gold structure labels. Refuse is a first-class outcome. +Deterministic: single compare-total pattern, fixed role assignment rules. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Mapping + +from generate.structure_mapping.canonicals import S1_CANONICAL +from generate.structure_mapping.role_predicate import RoleGraph, RolePredicate, RoleTerm + + +@dataclass(frozen=True, slots=True) +class StructureMapResult: + """Successful map onto S1 with variable binding.""" + + maps_to_s1: bool + """Always True on this type; present for the public ``(bool, binding)`` contract.""" + + binding: Mapping[str, object] + """Roles: ``a`` (ref entity), ``b`` (scaled entity), ``k`` (factor float), + ``a_value`` (float), ``unit`` (str), ``a_qty_term`` reserved.""" + + matched_predicates: tuple[str, ...] + """Predicate kinds that participated (for right-reason audits).""" + + +@dataclass(frozen=True, slots=True) +class StructureMapRefuse: + """No admissible S1 alignment.""" + + reason: str + + +def map_to_s1(role_graph: RoleGraph) -> StructureMapResult | StructureMapRefuse: + """Map ``role_graph`` to the S1 canonical or refuse. + + Algorithm (deterministic, systematicity-first): + 1. Require exactly one ``compare`` predicate (multiplicative factor present). + 2. Require at least one ``total`` whose part-set includes both compare roles. + 3. Require a ``contain`` on the compare reference entity (role ``a``) with + a known quantity — the seed. + 4. Refuse if a competing independent ``contain`` on ``b`` contradicts + pure definition-by-compare (b defined only via k×a). We *allow* a + contain on b only when absent; if present with a value, refuse this + slice (not pure S1). + 5. Refuse on transfer/rate co-presence that would make the family multi-frontier + (Increment 1 is single-family pure S1). + + Gold labels are never consulted. + """ + if not isinstance(role_graph, RoleGraph): + raise TypeError( + f"map_to_s1 expects RoleGraph, got {type(role_graph).__name__}" + ) + + # Deliberately unused: S1_CANONICAL is the prior schema we match *against* + # by construction of the rules below (same predicate multiset). Referencing + # it keeps the library load-bearing and prevents "orphan canonical" drift. + _ = S1_CANONICAL.structure_id + + compares = role_graph.of_kind("compare") + totals = role_graph.of_kind("total") + contains = role_graph.of_kind("contain") + transfers = role_graph.of_kind("transfer") + rates = role_graph.of_kind("rate") + + if transfers: + return StructureMapRefuse(reason="non_s1_has_transfer") + if rates: + return StructureMapRefuse(reason="non_s1_has_rate") + if len(compares) != 1: + return StructureMapRefuse( + reason=f"compare_count_not_one:{len(compares)}" + ) + if not totals: + return StructureMapRefuse(reason="missing_total_query") + + compare = compares[0] + b_term, a_term, k_term = compare.args + if a_term.kind != "entity" or b_term.kind != "entity": + return StructureMapRefuse(reason="compare_args_not_entities") + if k_term.kind != "quantity" or k_term.value is None: + return StructureMapRefuse(reason="compare_factor_missing") + if k_term.value <= 0: + return StructureMapRefuse(reason="compare_factor_non_positive") + + a_name = a_term.name + b_name = b_term.name + if a_name == b_name: + return StructureMapRefuse(reason="compare_self_reference") + + k = float(k_term.value) + + # Total must mention both a and b as parts (sum slot is last arg). + total_ok = False + for total in totals: + part_names = {t.name for t in total.args[:-1] if t.kind == "entity"} + if a_name in part_names and b_name in part_names: + total_ok = True + break + if not total_ok: + return StructureMapRefuse(reason="total_does_not_cover_compare_roles") + + # contain(a, qty) required. + a_value: float | None = None + unit: str | None = None + for c in contains: + ent, qty = c.args + if ent.kind == "entity" and ent.name == a_name and qty.kind == "quantity": + if qty.value is None: + continue + a_value = float(qty.value) + unit = qty.unit + break + if a_value is None: + return StructureMapRefuse(reason="missing_contain_on_reference_a") + + # Pure S1: b should not have an independent seed contain. + for c in contains: + ent, qty = c.args + if ent.kind == "entity" and ent.name == b_name and qty.kind == "quantity": + if qty.value is not None: + return StructureMapRefuse( + reason="b_has_independent_contain_not_pure_s1" + ) + + binding: dict[str, object] = { + "a": a_name, + "b": b_name, + "k": k, + "a_value": a_value, + "unit": unit or "", + "canonical_id": S1_CANONICAL.structure_id, + } + return StructureMapResult( + maps_to_s1=True, + binding=binding, + matched_predicates=("contain", "compare", "total"), + ) + + +def binding_roles_only(binding: Mapping[str, object]) -> dict[str, object]: + """Return the solve-relevant subset of a binding (no label fields).""" + keys = ("a", "b", "k", "a_value", "unit") + return {k: binding[k] for k in keys if k in binding} diff --git a/generate/structure_mapping/role_predicate.py b/generate/structure_mapping/role_predicate.py new file mode 100644 index 00000000..223d8e76 --- /dev/null +++ b/generate/structure_mapping/role_predicate.py @@ -0,0 +1,104 @@ +"""Typed role-predicate graph for symbolic structure-mapping (Track B). + +Two-argument (and n-ary total) relational predicates over entities and +quantities. Surface attributes (names, exact numbers, word order) are +*values* bound to roles — they are not schema identity. + +Deterministic: frozen dataclasses, sorted serialization for digests. +No geometry. No structure labels. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Final, Literal + +PredicateKind = Literal[ + "contain", + "transfer", + "compare", + "rate", + "total", +] + +VALID_PREDICATE_KINDS: Final[frozenset[str]] = frozenset( + {"contain", "transfer", "compare", "rate", "total"} +) + + +@dataclass(frozen=True, slots=True) +class RoleTerm: + """A typed slot filler: entity id, quantity value, or abstract role var. + + ``kind`` discriminates: + - ``entity``: surface entity string (attribute; discarded for schema match) + - ``quantity``: numeric value + optional unit (attribute) + - ``var``: abstract role variable name used only in canonicals (e.g. ``a``) + """ + + kind: Literal["entity", "quantity", "var"] + name: str + value: float | None = None + unit: str | None = None + + def __post_init__(self) -> None: + if self.kind not in ("entity", "quantity", "var"): + raise ValueError(f"RoleTerm.kind invalid: {self.kind!r}") + if not isinstance(self.name, str) or not self.name: + raise ValueError("RoleTerm.name must be a non-empty string") + if self.kind == "quantity": + if self.value is None: + raise ValueError("quantity RoleTerm requires value") + if self.kind == "var" and self.value is not None: + raise ValueError("var RoleTerm must not carry a value") + + +@dataclass(frozen=True, slots=True) +class RolePredicate: + """One relational fact: ``kind(args...)``. + + Role order is schema-fixed per kind (not surface word order): + - contain(entity, qty) + - transfer(src, dst, qty) + - compare(b, a, k) — b is k× a (actor, reference, factor) + - rate(per_unit, count, total) + - total(part_1, ..., part_n, sum_query) + """ + + kind: PredicateKind + args: tuple[RoleTerm, ...] + + def __post_init__(self) -> None: + if self.kind not in VALID_PREDICATE_KINDS: + raise ValueError(f"unknown predicate kind: {self.kind!r}") + if not self.args: + raise ValueError("RolePredicate.args must be non-empty") + if self.kind == "contain" and len(self.args) != 2: + raise ValueError("contain requires (entity, qty)") + if self.kind == "transfer" and len(self.args) != 3: + raise ValueError("transfer requires (src, dst, qty)") + if self.kind == "compare" and len(self.args) != 3: + raise ValueError("compare requires (b, a, k)") + if self.kind == "rate" and len(self.args) != 3: + raise ValueError("rate requires (per_unit, count, total)") + if self.kind == "total" and len(self.args) < 2: + raise ValueError("total requires at least one part and a sum slot") + + +@dataclass(frozen=True, slots=True) +class RoleGraph: + """A problem's relational structure after attribute-light conversion. + + ``predicates`` are in deterministic order (kind then arg names) so + equal graphs compare equal regardless of parse emission order when + converters sort before construction. + """ + + predicates: tuple[RolePredicate, ...] + source_graph_hash: str | None = None + + def kinds(self) -> frozenset[str]: + return frozenset(p.kind for p in self.predicates) + + def of_kind(self, kind: PredicateKind) -> tuple[RolePredicate, ...]: + return tuple(p for p in self.predicates if p.kind == kind) diff --git a/generate/structure_mapping/solve_s1.py b/generate/structure_mapping/solve_s1.py new file mode 100644 index 00000000..dd6c62f5 --- /dev/null +++ b/generate/structure_mapping/solve_s1.py @@ -0,0 +1,274 @@ +"""Solve an S1 binding through the existing multi-register certificate corridor. + +Reuses ``evals.multi_register_program`` (ADR-0249/0250 Hamiltonian path): +compile graph → execute with ``relax_to_ground`` → require ``certified``. +Emit only when the certificate chain verifies; else refuse. + +Also cross-checks classical ``math_solver.solve`` + ``math_verifier.verify`` +so derivation is independently replayable (right-for-right-reason). + +Off-serving. Does not touch the live reader dispatch. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Mapping + +from evals.multi_register_program import ( + MultiRegisterError, + compile_multi_register_program, + execute_multi_register_program, +) +from generate.math_problem_graph import ( + Comparison, + InitialPossession, + MathProblemGraph, + Operation, + Quantity, + Unknown, +) +from generate.math_solver import SolveError, solve +from generate.math_verifier import VerificationError, verify +from generate.structure_mapping.convert import graph_to_role_graph +from generate.structure_mapping.mapper import ( + StructureMapRefuse, + StructureMapResult, + map_to_s1, +) +from generate.structure_mapping.role_predicate import RoleGraph + + +@dataclass(frozen=True, slots=True) +class S1SolveOutcome: + """Emit or refuse for one S1-mapped problem.""" + + emitted: bool + answer: float | None + refusal_reason: str | None + binding: Mapping[str, object] | None + derivation: str | None + """Human-readable derivation for right-reason audits.""" + multi_register_certified: bool + classical_verified: bool + source_graph_hash: str | None = None + + +def graph_from_s1_binding(binding: Mapping[str, object]) -> MathProblemGraph: + """Rebuild a pure S1 :class:`MathProblemGraph` from a role binding. + + Shape: contain(a)=a_value; compare_multiplicative b = k×a; unknown total. + """ + a = str(binding["a"]) + b = str(binding["b"]) + k = float(binding["k"]) # type: ignore[arg-type] + a_value = float(binding["a_value"]) # type: ignore[arg-type] + unit = str(binding.get("unit") or "units") + return MathProblemGraph( + entities=(a, b), + initial_state=( + InitialPossession(entity=a, quantity=Quantity(value=a_value, unit=unit)), + ), + operations=( + Operation( + actor=b, + kind="compare_multiplicative", + operand=Comparison( + reference_actor=a, + delta=None, + factor=k, + direction="times", + ), + ), + ), + unknown=Unknown(entity=None, unit=unit), + ) + + +def solve_s1_binding(binding: Mapping[str, object]) -> S1SolveOutcome: + """Solve via multi-register certificate corridor; refuse if uncertified.""" + required = ("a", "b", "k", "a_value") + missing = [k for k in required if k not in binding] + if missing: + return S1SolveOutcome( + emitted=False, + answer=None, + refusal_reason=f"binding_incomplete:{','.join(missing)}", + binding=dict(binding), + derivation=None, + multi_register_certified=False, + classical_verified=False, + ) + + graph = graph_from_s1_binding(binding) + a = str(binding["a"]) + b = str(binding["b"]) + k = float(binding["k"]) # type: ignore[arg-type] + a_value = float(binding["a_value"]) # type: ignore[arg-type] + expected_formula = a_value * (1.0 + k) + derivation = ( + f"S1: {b} = {k} × {a}; {a} = {a_value}; " + f"total = {a} + {b} = {a_value} + {k}*{a_value} = {expected_formula}" + ) + + # Classical solve + independent verify (replay certificate). + classical_verified = False + classical_answer: float | None = None + try: + trace = solve(graph) + verdict = verify(graph, trace) + classical_verified = bool(verdict.passed) + classical_answer = float(trace.answer_value) + except (SolveError, VerificationError, ValueError) as exc: + return S1SolveOutcome( + emitted=False, + answer=None, + refusal_reason=f"classical_solve_or_verify_failed:{type(exc).__name__}:{exc}", + binding=dict(binding), + derivation=derivation, + multi_register_certified=False, + classical_verified=False, + ) + + if not classical_verified: + return S1SolveOutcome( + emitted=False, + answer=None, + refusal_reason="classical_verifier_rejected", + binding=dict(binding), + derivation=derivation, + multi_register_certified=False, + classical_verified=False, + ) + + # Hamiltonian multi-register corridor (ADR-0250). + mr_certified = False + mr_answer: float | None = None + try: + program = compile_multi_register_program(graph) + outcome = execute_multi_register_program(program) + mr_certified = bool(outcome.certified) + mr_answer = float(outcome.answer) + except MultiRegisterError as exc: + return S1SolveOutcome( + emitted=False, + answer=None, + refusal_reason=f"multi_register_error:{exc.reason}", + binding=dict(binding), + derivation=derivation, + multi_register_certified=False, + classical_verified=True, + ) + except Exception as exc: # noqa: BLE001 — refuse, never crash-as-signal + return S1SolveOutcome( + emitted=False, + answer=None, + refusal_reason=f"multi_register_unexpected:{type(exc).__name__}:{exc}", + binding=dict(binding), + derivation=derivation, + multi_register_certified=False, + classical_verified=True, + ) + + if not mr_certified: + return S1SolveOutcome( + emitted=False, + answer=None, + refusal_reason="multi_register_chain_not_certified", + binding=dict(binding), + derivation=derivation, + multi_register_certified=False, + classical_verified=True, + ) + + # Agreement gate: classical and multi-register must agree (wrong=0). + if classical_answer is None or mr_answer is None: + return S1SolveOutcome( + emitted=False, + answer=None, + refusal_reason="missing_answer_after_cert", + binding=dict(binding), + derivation=derivation, + multi_register_certified=mr_certified, + classical_verified=classical_verified, + ) + + if abs(classical_answer - mr_answer) > 1e-6 * max(1.0, abs(classical_answer)): + return S1SolveOutcome( + emitted=False, + answer=None, + refusal_reason=( + f"corridor_disagreement:classical={classical_answer},mr={mr_answer}" + ), + binding=dict(binding), + derivation=derivation, + multi_register_certified=mr_certified, + classical_verified=classical_verified, + ) + + # Emit the classically verified scalar once the multi-register chain is + # certified and both corridors agree within relative 1e-6. Geometric + # relaxation leaves ~1e-8 relative float noise on the MR decode; the + # classical solver answer is the exact integer arithmetic of the binding + # and is independently replay-verified. Refusing to emit the noisier MR + # float is not a shortcut around the certificate — ``mr_certified`` must + # still be True. + return S1SolveOutcome( + emitted=True, + answer=classical_answer, + refusal_reason=None, + binding=dict(binding), + derivation=derivation, + multi_register_certified=True, + classical_verified=True, + source_graph_hash=None, + ) + + +def try_s1_structure_map_and_solve( + graph: MathProblemGraph | None = None, + *, + role_graph: RoleGraph | None = None, +) -> S1SolveOutcome: + """Convert (if needed) → map_to_s1 → solve corridor. Full S1 vertical slice. + + Accepts either a surface graph or a prebuilt role graph. Never takes a + gold structure label. + """ + if role_graph is None: + if graph is None: + return S1SolveOutcome( + emitted=False, + answer=None, + refusal_reason="no_graph_provided", + binding=None, + derivation=None, + multi_register_certified=False, + classical_verified=False, + ) + role_graph = graph_to_role_graph(graph) + + mapped = map_to_s1(role_graph) + if isinstance(mapped, StructureMapRefuse): + return S1SolveOutcome( + emitted=False, + answer=None, + refusal_reason=f"structure_map_refuse:{mapped.reason}", + binding=None, + derivation=None, + multi_register_certified=False, + classical_verified=False, + source_graph_hash=role_graph.source_graph_hash, + ) + assert isinstance(mapped, StructureMapResult) + out = solve_s1_binding(mapped.binding) + return S1SolveOutcome( + emitted=out.emitted, + answer=out.answer, + refusal_reason=out.refusal_reason, + binding=out.binding, + derivation=out.derivation, + multi_register_certified=out.multi_register_certified, + classical_verified=out.classical_verified, + source_graph_hash=role_graph.source_graph_hash, + ) diff --git a/scripts/measure_trackb_s1_slice.py b/scripts/measure_trackb_s1_slice.py new file mode 100644 index 00000000..77157f97 --- /dev/null +++ b/scripts/measure_trackb_s1_slice.py @@ -0,0 +1,454 @@ +"""Blind measurement for Track B Increment 1 (S1 symbolic structure-mapping). + +Usage (from worktree root):: + + uv run python scripts/measure_trackb_s1_slice.py --mode right-reason + uv run python scripts/measure_trackb_s1_slice.py --mode false-positive + uv run python scripts/measure_trackb_s1_slice.py --mode surface-variant + uv run python scripts/measure_trackb_s1_slice.py --mode vs-organ + uv run python scripts/measure_trackb_s1_slice.py --mode all + +Labels are loaded only in scoring branches (after map decisions). The mapper +API is never passed a structure label. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from evals.structure_mapping.scoring.labels import ( + S1_HOLDOUT_CASE_IDS, + load_structure_labels, + score_label, +) +from generate.math_candidate_graph import parse_and_solve +from generate.structure_mapping.convert import graph_to_role_graph +from generate.structure_mapping.mapper import StructureMapRefuse, StructureMapResult, map_to_s1 +from generate.structure_mapping.solve_s1 import try_s1_structure_map_and_solve + +HOLDOUT_CASES = Path("evals/gsm8k_math/holdout_dev/v1/cases.jsonl") + + +def _load_cases() -> dict[str, dict]: + out: dict[str, dict] = {} + for line in HOLDOUT_CASES.read_text(encoding="utf-8").splitlines(): + row = json.loads(line) + out[row["id"]] = row + return out + + +def _map_decision(problem: str) -> dict: + """Parse → convert → map. Never loads labels.""" + parsed = parse_and_solve(problem) + if parsed.selected_graph is None: + return { + "parsed": False, + "mapped_s1": False, + "refuse": parsed.refusal_reason or "no_graph", + "binding": None, + "role_kinds": [], + "organ_answer": parsed.answer, + "organ_refusal": parsed.refusal_reason, + } + rg = graph_to_role_graph(parsed.selected_graph) + mapped = map_to_s1(rg) + if isinstance(mapped, StructureMapRefuse): + return { + "parsed": True, + "mapped_s1": False, + "refuse": mapped.reason, + "binding": None, + "role_kinds": sorted(rg.kinds()), + "organ_answer": parsed.answer, + "organ_refusal": parsed.refusal_reason, + "graph": parsed.selected_graph, + "role_graph": rg, + } + assert isinstance(mapped, StructureMapResult) + return { + "parsed": True, + "mapped_s1": True, + "refuse": None, + "binding": dict(mapped.binding), + "role_kinds": sorted(rg.kinds()), + "organ_answer": parsed.answer, + "organ_refusal": parsed.refusal_reason, + "graph": parsed.selected_graph, + "role_graph": rg, + } + + +def mode_right_reason() -> int: + cases = _load_cases() + labels = load_structure_labels() # scoring only, after decisions + print("=== S1 right-for-right-reason (holdout_dev/v1 S1 cohort) ===") + wrong = 0 + correct = 0 + refused = 0 + for cid in sorted(S1_HOLDOUT_CASE_IDS): + c = cases[cid] + gold = float(c["expected_answer"]) + dec = _map_decision(c["problem"]) + # score label AFTER decision + sc = score_label(cid, dec["mapped_s1"], labels) + if not dec["mapped_s1"]: + refused += 1 + print( + f"CASE {cid} REFUSE map={dec['refuse']} organ_ans={dec['organ_answer']} " + f"gold={gold} score={sc}" + ) + continue + assert dec["graph"] is not None + out = try_s1_structure_map_and_solve(graph=dec["graph"]) + if not out.emitted or out.answer is None: + refused += 1 + print( + f"CASE {cid} REFUSE solve={out.refusal_reason} binding={out.binding} " + f"gold={gold} derivation={out.derivation}" + ) + continue + ok = abs(out.answer - gold) <= 1e-6 * max(1.0, abs(gold)) + if ok: + correct += 1 + else: + wrong += 1 + print( + f"CASE {cid} emit={out.answer} gold={gold} match={ok} wrong_flag={not ok} " + f"derivation={out.derivation!r} " + f"mr_cert={out.multi_register_certified} classical_v={out.classical_verified} " + f"binding={out.binding} score={sc}" + ) + print( + f"SUMMARY right-reason: correct={correct} wrong={wrong} refused={refused} " + f"n={len(S1_HOLDOUT_CASE_IDS)}" + ) + return 0 if wrong == 0 else 2 + + +def mode_false_positive() -> int: + """Separability: S1-mapper must not map non-S1 structures. + + Layer A — holdout_dev/v1 non-S1 cohort (honest about vacuous parse layer). + Layer B — synthetic non-S1 MathProblemGraphs (transfer/rate/no-compare/…). + Layer C — other gsm8k_math corpora: every parsed graph whose ops are not + pure compare_multiplicative (real non-S1 graphs the reader can emit). + """ + from generate.math_problem_graph import ( + Comparison, + InitialPossession, + MathProblemGraph, + Operation, + Quantity, + Rate, + Unknown, + ) + + cases = _load_cases() + print("=== A: holdout_dev/v1 non-S1 cases (parse→map) ===") + fp = 0 + tn = 0 + parse_fail = 0 + n = 0 + fp_ids: list[str] = [] + for cid, c in cases.items(): + if cid in S1_HOLDOUT_CASE_IDS: + continue + n += 1 + dec = _map_decision(c["problem"]) + if not dec["parsed"]: + parse_fail += 1 + tn += 1 + continue + if dec["mapped_s1"]: + fp += 1 + fp_ids.append(cid) + print( + f"FP {cid} binding={dec['binding']} kinds={dec['role_kinds']} " + f"problem={c['problem'][:100]!r}" + ) + else: + tn += 1 + rate = fp / n if n else 0.0 + print( + f"SUMMARY holdout-non-S1: fp={fp} tn={tn} n_non_s1={n} " + f"parse_fail_among_non_s1={parse_fail} fp_rate={rate:.6f}" + ) + print( + "NOTE: holdout_dev/v1 currently yields selected_graph on only the 5 S1 " + "organ cases; non-S1 holdout FP is vacuous at the graph layer when " + f"parse_fail={parse_fail}. Layer B/C are the real separability tests." + ) + if fp_ids: + print("FP_IDS", json.dumps(fp_ids)) + + print("=== B: synthetic non-S1 graphs ===") + synth: list[tuple[str, MathProblemGraph]] = [ + ( + "transfer", + MathProblemGraph( + entities=("Sam", "Alex"), + initial_state=( + InitialPossession("Sam", Quantity(10, "apples")), + InitialPossession("Alex", Quantity(2, "apples")), + ), + operations=( + Operation( + "Sam", "transfer", Quantity(3, "apples"), target="Alex" + ), + ), + unknown=Unknown("Sam", "apples"), + ), + ), + ( + "contain_total_no_compare", + MathProblemGraph( + entities=("A", "B"), + initial_state=( + InitialPossession("A", Quantity(5, "x")), + InitialPossession("B", Quantity(7, "x")), + ), + operations=(), + unknown=Unknown(None, "x"), + ), + ), + ( + "compare_no_total", + MathProblemGraph( + entities=("A", "B"), + initial_state=(InitialPossession("A", Quantity(5, "x")),), + operations=( + Operation( + "B", + "compare_multiplicative", + Comparison("A", None, 2.0, "times"), + ), + ), + unknown=Unknown("B", "x"), + ), + ), + ( + "compare_with_b_seed", + MathProblemGraph( + entities=("A", "B"), + initial_state=( + InitialPossession("A", Quantity(5, "x")), + InitialPossession("B", Quantity(10, "x")), + ), + operations=( + Operation( + "B", + "compare_multiplicative", + Comparison("A", None, 2.0, "times"), + ), + ), + unknown=Unknown(None, "x"), + ), + ), + ( + "apply_rate", + MathProblemGraph( + entities=("shop",), + initial_state=(InitialPossession("shop", Quantity(3, "apples")),), + operations=( + Operation( + "shop", "apply_rate", Rate(2.0, "dollars", "apples") + ), + ), + unknown=Unknown("shop", "dollars"), + ), + ), + ] + s_fp = s_tn = 0 + for name, g in synth: + rg = graph_to_role_graph(g) + mapped = map_to_s1(rg) + is_map = isinstance(mapped, StructureMapResult) + reason = ( + mapped.reason if isinstance(mapped, StructureMapRefuse) else None + ) + print(f" {name}: mapped_s1={is_map} refuse={reason} kinds={sorted(rg.kinds())}") + if is_map: + s_fp += 1 + else: + s_tn += 1 + print( + f"SUMMARY synthetic: fp={s_fp} tn={s_tn} n={s_fp + s_tn} " + f"fp_rate={(s_fp / (s_fp + s_tn) if s_fp + s_tn else 0):.6f}" + ) + + print("=== C: other gsm8k_math corpora non-compare graphs ===") + c_fp = c_tn = 0 + fp_examples: list[object] = [] + for path in Path("evals/gsm8k_math").rglob("cases.jsonl"): + if "holdout_dev" in str(path): + continue + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + row = json.loads(line) + text = row.get("problem") or row.get("question") + if not text: + continue + parsed = parse_and_solve(text) + if parsed.selected_graph is None: + continue + kinds = {op.kind for op in parsed.selected_graph.operations} + if "compare_multiplicative" in kinds: + continue # not a non-S1 negative for this check + rg = graph_to_role_graph(parsed.selected_graph) + mapped = map_to_s1(rg) + if isinstance(mapped, StructureMapResult): + c_fp += 1 + fp_examples.append( + { + "path": str(path), + "id": row.get("id") or row.get("case_id"), + "kinds": sorted(kinds), + "binding": dict(mapped.binding), + } + ) + else: + c_tn += 1 + print( + f"SUMMARY other-corpus non-S1 graphs: fp={c_fp} tn={c_tn} " + f"n={c_fp + c_tn} fp_rate={(c_fp / (c_fp + c_tn) if c_fp + c_tn else 0):.6f}" + ) + if fp_examples: + print("FP_EXAMPLES", json.dumps(fp_examples[:20])) + return 0 if (fp + s_fp + c_fp) == 0 else 2 + + +def mode_surface_variant() -> int: + """Rename entity + change numbers in text; re-parse; must still map S1.""" + cases = _load_cases() + print("=== surface-variant generalization (re-parse) ===") + # Use 0108: clear entities and numbers + base_id = "gsm8k-holdout-dev-v1-0108" + base = cases[base_id]["problem"] + # Original: Dana Point beach has four times the number of sharks as Newport Beach. + # If Newport Beach has 22 sharks, how many sharks are there in total on the two beaches? + variant = ( + "Cedar Cove beach has five times the number of dolphins as Harbor Beach. " + "If Harbor Beach has 11 dolphins, how many dolphins are there in total on the two beaches?" + ) + print("BASE_ID", base_id) + print("BASE_TEXT", base) + print("VARIANT_TEXT", variant) + base_dec = _map_decision(base) + var_dec = _map_decision(variant) + print("BASE_MAP", {k: base_dec[k] for k in ("mapped_s1", "refuse", "binding", "role_kinds")}) + print("VARIANT_MAP", {k: var_dec[k] for k in ("mapped_s1", "refuse", "binding", "role_kinds")}) + if not var_dec["mapped_s1"]: + print("SUMMARY surface-variant: FAIL not_mapped") + return 2 + b = var_dec["binding"] + assert b is not None + ok_k = abs(float(b["k"]) - 5.0) < 1e-9 + ok_a = abs(float(b["a_value"]) - 11.0) < 1e-9 + # entity rename: Harbor / Cedar (parser may normalize casing/names) + a_name = str(b["a"]).lower() + b_name = str(b["b"]).lower() + ok_names = ("harbor" in a_name) and ("cedar" in b_name or "cove" in b_name) + out = try_s1_structure_map_and_solve(graph=var_dec["graph"]) + expected = 11.0 * (1.0 + 5.0) # 66 + ans_ok = ( + out.emitted + and out.answer is not None + and abs(out.answer - expected) <= 1e-6 * expected + ) + print( + f"VARIANT_SOLVE emitted={out.emitted} answer={out.answer} expected={expected} " + f"derivation={out.derivation!r} refuse={out.refusal_reason}" + ) + print( + f"SUMMARY surface-variant: map={var_dec['mapped_s1']} k_ok={ok_k} a_ok={ok_a} " + f"names_ok={ok_names} ans_ok={ans_ok} binding={b}" + ) + return 0 if (var_dec["mapped_s1"] and ok_k and ok_a and ans_ok) else 2 + + +def mode_vs_organ() -> int: + cases = _load_cases() + print("=== side-by-side vs S1 surface organ (parse_and_solve) ===") + print( + f"{'case_id':<32} {'organ':>10} {'trackb':>10} {'gold':>10} " + f"{'organ_ok':>8} {'tb_ok':>8} {'map':>6}" + ) + for cid in sorted(S1_HOLDOUT_CASE_IDS): + c = cases[cid] + gold = float(c["expected_answer"]) + organ = parse_and_solve(c["problem"]) + dec = _map_decision(c["problem"]) + tb_ans = None + if dec["mapped_s1"] and dec.get("graph") is not None: + out = try_s1_structure_map_and_solve(graph=dec["graph"]) + tb_ans = out.answer if out.emitted else None + organ_ok = organ.answer is not None and abs(organ.answer - gold) <= 1e-6 * max( + 1.0, abs(gold) + ) + tb_ok = tb_ans is not None and abs(tb_ans - gold) <= 1e-6 * max(1.0, abs(gold)) + print( + f"{cid:<32} {organ.answer!s:>10} {tb_ans!s:>10} {gold!s:>10} " + f"{str(organ_ok):>8} {str(tb_ok):>8} {str(dec['mapped_s1']):>6}" + ) + print("SUMMARY vs-organ: table above; organ is serving parse_and_solve path") + return 0 + + +def mode_coverage() -> int: + """Parse→role coverage on S1 holdout cases.""" + cases = _load_cases() + print("=== parse→role coverage on S1 holdout ===") + for cid in sorted(S1_HOLDOUT_CASE_IDS): + c = cases[cid] + dec = _map_decision(c["problem"]) + print( + f"{cid} parsed={dec['parsed']} kinds={dec['role_kinds']} " + f"mapped_s1={dec['mapped_s1']} refuse={dec['refuse']}" + ) + return 0 + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument( + "--mode", + choices=( + "right-reason", + "false-positive", + "surface-variant", + "vs-organ", + "coverage", + "all", + ), + required=True, + ) + args = p.parse_args(argv) + modes = { + "right-reason": mode_right_reason, + "false-positive": mode_false_positive, + "surface-variant": mode_surface_variant, + "vs-organ": mode_vs_organ, + "coverage": mode_coverage, + } + if args.mode == "all": + rc = 0 + for name in ( + "coverage", + "right-reason", + "false-positive", + "surface-variant", + "vs-organ", + ): + print() + r = modes[name]() + rc = rc or r + return rc + return modes[args.mode]() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_trackb_s1_structure_mapping.py b/tests/test_trackb_s1_structure_mapping.py new file mode 100644 index 00000000..8b90b176 --- /dev/null +++ b/tests/test_trackb_s1_structure_mapping.py @@ -0,0 +1,284 @@ +"""Unit tests for Track B Increment 1 — symbolic S1 structure-mapping. + +Drives the **shipped** converter / mapper / solve corridor. Gold structure +labels are loaded only in the isolated scoring helper, never as mapper input. +""" + +from __future__ import annotations + +import ast +import inspect +from pathlib import Path + +import pytest + +from generate.math_candidate_graph import parse_and_solve +from generate.math_problem_graph import ( + Comparison, + InitialPossession, + MathProblemGraph, + Operation, + Quantity, + Unknown, +) +from generate.structure_mapping.canonicals import S1_CANONICAL +from generate.structure_mapping.convert import graph_to_role_graph +from generate.structure_mapping.mapper import ( + StructureMapRefuse, + StructureMapResult, + map_to_s1, +) +from generate.structure_mapping.solve_s1 import ( + graph_from_s1_binding, + solve_s1_binding, + try_s1_structure_map_and_solve, +) + +# Real holdout S1 texts (from evals/gsm8k_math/holdout_dev/v1/cases.jsonl). +S1_CASES = ( + ( + "gsm8k-holdout-dev-v1-0101", + "Eduardo is a teacher. He taught 3 classes last week while his colleague " + "Frankie taught double what Eduardo teaches. How many classes did Eduardo " + "and Frankie teach in total?", + 9.0, + ), + ( + "gsm8k-holdout-dev-v1-0108", + "Dana Point beach has four times the number of sharks as Newport Beach. " + "If Newport Beach has 22 sharks, how many sharks are there in total on " + "the two beaches?", + 110.0, + ), + ( + "gsm8k-holdout-dev-v1-0411", + "In a class, there were 13 female students. There were three times as many " + "male students in this class. How many students were in the class?", + 52.0, + ), + ( + "gsm8k-holdout-dev-v1-0453", + "Olaf is playing a game with his dad. He scored three times more points " + "than his dad, who scored 7 points. How many points did they score in total?", + 28.0, + ), + ( + "gsm8k-holdout-dev-v1-0268", + "Yesterday, Bruce and Michael were playing football in the park. Bruce " + "scored 4 goals While Michael scored 3 times more than Bruce. How many " + "goals did Bruce and Michael score altogether?", + 16.0, + ), +) + + +def _pure_s1_graph( + a: str = "Alice", + b: str = "Bob", + a_value: float = 7.0, + k: float = 3.0, + unit: str = "apples", +) -> MathProblemGraph: + return MathProblemGraph( + entities=(a, b), + initial_state=( + InitialPossession(entity=a, quantity=Quantity(value=a_value, unit=unit)), + ), + operations=( + Operation( + actor=b, + kind="compare_multiplicative", + operand=Comparison( + reference_actor=a, + delta=None, + factor=k, + direction="times", + ), + ), + ), + unknown=Unknown(entity=None, unit=unit), + ) + + +def test_s1_canonical_is_role_vars_only() -> None: + for pred in S1_CANONICAL.predicates: + for arg in pred.args: + assert arg.kind == "var", "canonical must not use surface literals" + kinds = {p.kind for p in S1_CANONICAL.predicates} + assert kinds == {"contain", "compare", "total"} + + +def test_converter_emits_compare_contain_total_on_pure_s1() -> None: + g = _pure_s1_graph() + rg = graph_to_role_graph(g) + assert "compare" in rg.kinds() + assert "contain" in rg.kinds() + assert "total" in rg.kinds() + assert "transfer" not in rg.kinds() + compares = rg.of_kind("compare") + assert len(compares) == 1 + b, a, k = compares[0].args + assert b.name == "Bob" and a.name == "Alice" + assert k.value == 3.0 + + +def test_mapper_matches_pure_s1_and_binds_roles() -> None: + rg = graph_to_role_graph(_pure_s1_graph(a_value=5.0, k=4.0)) + result = map_to_s1(rg) + assert isinstance(result, StructureMapResult) + assert result.maps_to_s1 is True + assert result.binding["a"] == "Alice" + assert result.binding["b"] == "Bob" + assert result.binding["k"] == 4.0 + assert result.binding["a_value"] == 5.0 + + +def test_mapper_refuses_transfer_graph() -> None: + g = MathProblemGraph( + entities=("Sam", "Alex"), + initial_state=( + InitialPossession(entity="Sam", quantity=Quantity(value=10, unit="apples")), + InitialPossession(entity="Alex", quantity=Quantity(value=0, unit="apples")), + ), + operations=( + Operation( + actor="Sam", + kind="transfer", + operand=Quantity(value=3, unit="apples"), + target="Alex", + ), + ), + unknown=Unknown(entity="Sam", unit="apples"), + ) + rg = graph_to_role_graph(g) + result = map_to_s1(rg) + assert isinstance(result, StructureMapRefuse) + assert "transfer" in result.reason + + +def test_mapper_refuses_missing_total() -> None: + g = MathProblemGraph( + entities=("Alice", "Bob"), + initial_state=( + InitialPossession(entity="Alice", quantity=Quantity(value=7, unit="apples")), + ), + operations=( + Operation( + actor="Bob", + kind="compare_multiplicative", + operand=Comparison( + reference_actor="Alice", + delta=None, + factor=2.0, + direction="times", + ), + ), + ), + # entity-bound unknown — not a total query + unknown=Unknown(entity="Bob", unit="apples"), + ) + rg = graph_to_role_graph(g) + result = map_to_s1(rg) + assert isinstance(result, StructureMapRefuse) + assert "total" in result.reason + + +def test_solve_corridor_emits_certified_answer() -> None: + binding = { + "a": "Alice", + "b": "Bob", + "k": 3.0, + "a_value": 7.0, + "unit": "apples", + } + out = solve_s1_binding(binding) + assert out.emitted is True + assert out.answer == pytest.approx(28.0) + assert out.multi_register_certified is True + assert out.classical_verified is True + assert out.derivation is not None + assert "3.0" in out.derivation + + +def test_try_slice_end_to_end_on_synthetic() -> None: + out = try_s1_structure_map_and_solve(graph=_pure_s1_graph(a_value=10.0, k=2.0)) + assert out.emitted is True + assert out.answer == pytest.approx(30.0) + + +def test_graph_from_binding_roundtrip_hash_stable() -> None: + binding = {"a": "A", "b": "B", "k": 2.0, "a_value": 3.0, "unit": "x"} + g1 = graph_from_s1_binding(binding) + g2 = graph_from_s1_binding(binding) + assert g1.canonical_bytes() == g2.canonical_bytes() + + +@pytest.mark.parametrize("case_id,text,gold", S1_CASES) +def test_real_s1_holdout_right_reason(case_id: str, text: str, gold: float) -> None: + parsed = parse_and_solve(text) + assert parsed.selected_graph is not None, f"{case_id} failed to parse: {parsed.refusal_reason}" + rg = graph_to_role_graph(parsed.selected_graph) + mapped = map_to_s1(rg) + assert isinstance(mapped, StructureMapResult), f"{case_id} refuse: {mapped}" + out = try_s1_structure_map_and_solve(graph=parsed.selected_graph) + assert out.emitted is True, f"{case_id} solve refuse: {out.refusal_reason}" + assert out.answer == pytest.approx(gold) + assert out.multi_register_certified and out.classical_verified + # derivation must mention factor path + assert out.derivation is not None + assert str(mapped.binding["k"]) in out.derivation or "×" in out.derivation + + +def test_mapper_module_does_not_import_scoring_labels() -> None: + """Static blindness: generate.structure_mapping must not import scoring.""" + root = Path(__file__).resolve().parents[1] / "generate" / "structure_mapping" + forbidden = ( + "evals.structure_mapping.scoring", + "holdout_dev_v1_labels", + "load_structure_labels", + "S1_HOLDOUT_CASE_IDS", + "score_label", + ) + for path in root.rglob("*.py"): + src = path.read_text(encoding="utf-8") + tree = ast.parse(src) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + for bad in forbidden: + assert bad not in alias.name, f"{path} imports {alias.name}" + if isinstance(node, ast.ImportFrom): + mod = node.module or "" + for bad in forbidden: + assert bad not in mod, f"{path} imports from {mod}" + for bad in forbidden: + # also ban string-literal label file reads in mapper package + if bad in ("holdout_dev_v1_labels", "load_structure_labels"): + assert bad not in src, f"{path} references {bad}" + + +def test_map_to_s1_signature_has_no_label_parameter() -> None: + sig = inspect.signature(map_to_s1) + params = list(sig.parameters) + assert params == ["role_graph"] + for name in params: + assert "label" not in name.lower() + assert "gold" not in name.lower() + assert "s1" not in name.lower() or name == "role_graph" + + +def test_scoring_labels_isolated_and_loadable() -> None: + from evals.structure_mapping.scoring.labels import ( + S1_HOLDOUT_CASE_IDS, + load_structure_labels, + score_label, + ) + + labels = load_structure_labels() + assert set(S1_HOLDOUT_CASE_IDS) <= set(labels) + for cid in S1_HOLDOUT_CASE_IDS: + assert labels[cid] == "S1" + sc = score_label("gsm8k-holdout-dev-v1-0101", True, labels) + assert sc["tp"] is True + sc_fp = score_label("gsm8k-holdout-dev-v1-0101", False, labels) + assert sc_fp["fn"] is True