diff --git a/docs/decisions/ADR-0125-reasoning-isolation-perturbation-suite.md b/docs/decisions/ADR-0125-reasoning-isolation-perturbation-suite.md new file mode 100644 index 00000000..25ff81b0 --- /dev/null +++ b/docs/decisions/ADR-0125-reasoning-isolation-perturbation-suite.md @@ -0,0 +1,161 @@ +# ADR-0125 — Reasoning-Isolation Perturbation Suite + +**Status:** Accepted +**Date:** 2026-05-22 +**Author:** CORE agents + reviewers +**Depends on:** ADR-0114a, ADR-0115, ADR-0116, ADR-0118a + +--- + +## Context + +ADR-0114a Obligation #5 requires a programmatic perturbation suite that +separates concept-stable reasoning from surface pattern matching. +Perturbations are either invariance-preserving, where the answer must +not change, or invariance-breaking, where CORE must produce the +predicted changed graph/trace result. + +ADR-0118a already covered OOD surface variation for entity renaming, +unit renaming, and linear number scaling. This ADR adds the semantic +suite over the same GSM8K-style parser development lane without +changing the parser, solver, graph schema, or authored dev cases. + +--- + +## Decision + +`generate/perturbation_suite.py` exports: + +```text +generate_perturbations(problem, ground_truth_graph, *, seed) +``` + +and the frozen, slotted `Perturbation` record. The generator is pure and +deterministic: same problem, graph, and seed produce byte-equal +perturbation records. + +The suite reuses ADR-0118a registry and rendering helpers for entity and +unit relabeling. It adds semantic transforms that stay inside the +ADR-0115 Phase 1.1 pattern registry: + +| Transform | Kind | Behavior | +|---|---|---| +| `rename_entities` | invariance-preserving | Relabel every entity through the ADR-0118a OOD registry. | +| `rename_units` | invariance-preserving | Relabel every unit through the ADR-0118a OOD registry. | +| `reorder_independent_initial_possessions` | invariance-preserving | Reverse two or more independent initial possession sentences. | +| `reorder_independent_operations` | invariance-preserving | Reverse operations only when their affected `(entity, unit)` state sets are pairwise disjoint. | +| `replace_verb_with_synonym` | invariance-preserving | Replace the first add/subtract/transfer verb with a different parser-registry synonym of the same kind. | +| `add_zero_quantity_entity` | invariance-preserving | Add an unused registry entity with zero of the queried unit. | +| `swap_non_commuting_operations` | invariance-breaking | Swap two same-state operations when the replay trace changes; expected answer and trace hash are computed from the swapped graph. | + +`scale_numbers_by_k` is not duplicated here. ADR-0118a owns that +Obligation #2 transform and already pins the linear scaling ratio. + +`evals/gsm8k_parser_dev/perturbation_score.py` scores the live +parser+solver against generated perturbations, reports explicit skip +counts, prints per-transform ratios, and exits `0` iff both aggregate +invariance classes score 100%. + +--- + +## Invariants + +### `adr_0125_generator_determinism` + +Two calls with the same problem, graph, and seed produce byte-equal +serialized perturbation records. + +### `adr_0125_preserving_answer_stability` + +Every invariance-preserving perturbation solves to the original answer +value. + +### `adr_0125_breaking_predictable_result` + +Every invariance-breaking perturbation solves to the expected answer and +the predicted trace hash computed from the perturbed graph. + +### `adr_0125_parser_registry_boundary` + +All rendered perturbations stay inside the documented parser pattern +registry: direct initial possessions, supported operation verbs, +supported transfer syntax, supported multiply/divide syntax, and one +supported question. + +### `adr_0125_skips_are_explicit` + +Inapplicable transforms are skipped with reported reasons. They are not +counted as silent failures or fabricated successes. + +--- + +## ADR-0114a Obligation Discharge Summary + +This ADR closes ADR-0114a Obligation #5 for the GSM8K-style parser dev +lane by making reasoning-isolation perturbations executable and scored +through the same parser+solver contract used for public cases. + +| Obligation #5 transform | Status under ADR-0125 | +|---|---| +| Rename all entities | Discharged here by reuse of ADR-0118a helpers | +| Rename all units | Discharged here by reuse of ADR-0118a helpers | +| Multiply all numbers by `k` | Discharged by ADR-0118a, not duplicated | +| Reorder independent sentences | Discharged for independent initial possessions; independent operations implemented with 0 applicable current dev cases | +| Swap order of non-commuting operations | Discharged with predicted answer + trace-hash check | +| Replace verb with synonym in registry | Discharged | + +The current 50-case dev split has no pairwise-disjoint operation cases, +so `reorder_independent_operations` reports `0/0` applicable and +`50/50` skipped. The transform is implemented and covered by a synthetic +unit test; future dev/holdout cases that contain independent operations +will be scored by the same gate. + +--- + +## Acceptance Evidence + +Accepted when: + +- `generate/perturbation_suite.py` exports `Perturbation` and + `generate_perturbations` +- `evals/gsm8k_parser_dev/perturbation_score.py` runs as + `python3 -m evals.gsm8k_parser_dev.perturbation_score` +- `tests/test_perturbation_suite.py` is green +- Smoke suite is green +- The perturbation scorer reports: + - `add_zero_quantity_entity`: 50/50 = 1.0000 + - `rename_entities`: 50/50 = 1.0000 + - `rename_units`: 50/50 = 1.0000 + - `reorder_independent_initial_possessions`: 21/21 = 1.0000 + - `reorder_independent_operations`: 0/0 = n/a, 50 skipped + - `replace_verb_with_synonym`: 36/36 = 1.0000 + - `swap_non_commuting_operations`: 17/17 = 1.0000 + - invariance-preserving: 207/207 = 1.0000 + - invariance-breaking: 17/17 = 1.0000 +- ADR linked from `docs/decisions/README.md` index and frontier + +--- + +## Consequences + +- ADR-0114a Obligation #5 now has a deterministic local score lane for + applicable GSM8K-style dev perturbations. +- The scorer distinguishes semantic invariance from source-order graph + identity: reordering may change tuple order in `MathProblemGraph`, but + the answer invariant is still checked through the solver. +- Trace-changing swaps are first-class evidence even when the final + numeric answer remains equal. +- Independent-operation coverage is explicit rather than implied; the + current public dev set has no applicable pairwise-disjoint operations. + +--- + +## Out of Scope + +- Number scaling, which remains owned by ADR-0118a. +- Parser, solver, graph-schema, or dev-case expansion. +- New constructions outside ADR-0115 Phase 1.1. +- Holdout scoring. The generator is holdout-ready, but holdout access + remains governed by ADR-0114a Obligation #1. +- LLMs, sampling, stochastic generation, approximate recall, or + unreviewed mutation. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 9f8a06f5..b1ccf543 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -44,6 +44,7 @@ ADRs record significant architectural decisions: what was decided, why, what alt | [ADR-0123](ADR-0123-symbolic-logic-shape-remap.md) | `symbolic_logic` Lane-Shape Remap (ADR-0109 Amendment) | Accepted (2026-05-22) | | [ADR-0124](ADR-0124-systems-software-audit-passed-promotion.md) | `systems_software` Audit-Passed Promotion | Accepted (2026-05-22) | | [ADR-0123a](ADR-0123a-inference-shape-synonym.md) | `all_three_pass_rate` Synonym in `inference_shape` (ADR-0109 Amendment) | Accepted (2026-05-22) | +| [ADR-0125](ADR-0125-reasoning-isolation-perturbation-suite.md) | Reasoning-Isolation Perturbation Suite | Accepted (2026-05-22) | --- @@ -83,6 +84,7 @@ The ADR-0091..0114 slate is fully accepted (0091..0113) plus one proposed-roadma - `symbolic_logic` Lane-Shape Remap (ADR-0109 amendment) — ADR-0123 - `systems_software` Audit-Passed Promotion (third successful) — ADR-0124 - `all_three_pass_rate` Synonym in `inference_shape` (ADR-0109 Amendment) — ADR-0123a +- Reasoning-Isolation Perturbation Suite (224 deterministic applicable semantic perturbations; discharges ADR-0114a obligation #5 for the GSM8K-style parser dev lane) — ADR-0125 ADR-0080 has also landed: Contemplation Loop Phase 1 adds a read-only frontier-compare miner that emits `SPECULATIVE` findings only. diff --git a/evals/gsm8k_parser_dev/perturbation_score.py b/evals/gsm8k_parser_dev/perturbation_score.py new file mode 100644 index 00000000..0a70e843 --- /dev/null +++ b/evals/gsm8k_parser_dev/perturbation_score.py @@ -0,0 +1,118 @@ +"""Score ADR-0125 semantic perturbations for the parser dev lane.""" + +from __future__ import annotations + +import hashlib +from collections import defaultdict + +from evals.gsm8k_parser_dev.ood_score import load_cases +from generate.math_parser import ParseError, parse_problem +from generate.math_solver import SolutionTrace, SolveError, solve +from generate.perturbation_suite import ( + INVARIANCE_BREAKING, + INVARIANCE_PRESERVING, + Perturbation, + generate_perturbations, + skip_reasons, +) + + +def score_perturbation(perturbation: Perturbation) -> tuple[bool, str]: + try: + graph = parse_problem(perturbation.problem_text) + trace = solve(graph) + except (ParseError, SolveError) as exc: + return False, f"{type(exc).__name__}: {exc}" + + if graph.canonical_bytes() != perturbation.expected_graph.canonical_bytes(): + return False, "parsed graph did not match predicted perturbation graph" + if trace.answer_unit != perturbation.expected_unit: + return False, f"unit {trace.answer_unit!r} != {perturbation.expected_unit!r}" + if trace.answer_value != perturbation.expected_answer: + return ( + False, + f"answer {trace.answer_value!r} != {perturbation.expected_answer!r}", + ) + if perturbation.kind == INVARIANCE_BREAKING: + expected_trace_hash = perturbation.transform_params.get("expected_trace_hash") + if expected_trace_hash is not None and _trace_hash(trace) != expected_trace_hash: + return False, "trace hash did not match predicted perturbation trace" + return True, "ok" + + +def _trace_hash(trace: SolutionTrace) -> str: + return hashlib.sha256(trace.canonical_bytes()).hexdigest() + + +def _seed_from_case_id(case_id: str) -> int: + return int(case_id.rsplit("-", 1)[1]) + + +def _ratio(correct: int, total: int) -> float: + return correct / total if total else 0.0 + + +def _ratio_text(correct: int, total: int) -> str: + if total == 0: + return "n/a" + return f"{_ratio(correct, total):.4f}" + + +def main() -> int: + cases = load_cases() + per_transform: dict[str, list[bool]] = defaultdict(list) + per_kind: dict[str, list[bool]] = defaultdict(list) + skip_counts: dict[str, list[str]] = defaultdict(list) + + for case in cases: + seed = _seed_from_case_id(case.case_id) + perturbations = generate_perturbations(case.problem, case.graph, seed=seed) + skips = skip_reasons(case.problem, case.graph, seed=seed) + for transform, reason in sorted(skips.items()): + skip_counts[transform].append(case.case_id) + print(f"SKIP {case.case_id}:{transform}: {reason}") + + for perturbation in perturbations: + ok, detail = score_perturbation(perturbation) + per_transform[perturbation.transform].append(ok) + per_kind[perturbation.kind].append(ok) + status = "PASS" if ok else "FAIL" + print( + f"{status} {perturbation.perturbation_id} " + f"{perturbation.kind}/{perturbation.transform}: {detail}" + ) + + print() + for transform in sorted(set(per_transform) | set(skip_counts)): + results = per_transform.get(transform, []) + correct = sum(results) + total = len(results) + skipped = len(skip_counts.get(transform, [])) + print( + f"transform {transform}: {correct}/{total} = {_ratio_text(correct, total)} " + f"(skipped {skipped}/{len(cases)})" + ) + + preserving = per_kind[INVARIANCE_PRESERVING] + breaking = per_kind[INVARIANCE_BREAKING] + preserving_correct = sum(preserving) + breaking_correct = sum(breaking) + preserving_total = len(preserving) + breaking_total = len(breaking) + preserving_ratio = _ratio(preserving_correct, preserving_total) + breaking_ratio = _ratio(breaking_correct, breaking_total) + + print( + f"invariance_preserving: {preserving_correct}/{preserving_total} = " + f"{preserving_ratio:.4f}" + ) + print( + f"invariance_breaking: {breaking_correct}/{breaking_total} = " + f"{breaking_ratio:.4f}" + ) + + return 0 if preserving_ratio == 1.0 and breaking_ratio == 1.0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/generate/perturbation_suite.py b/generate/perturbation_suite.py new file mode 100644 index 00000000..c25ff106 --- /dev/null +++ b/generate/perturbation_suite.py @@ -0,0 +1,501 @@ +"""ADR-0125 — semantic perturbation suite for GSM8K-style dev cases. + +This module builds on ADR-0118a's deterministic OOD surface generator. +It keeps every surface inside the ADR-0115 Phase 1.1 pattern registry +while applying semantic perturbations that either preserve the answer or +change the replayed trace in a predicted way. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from typing import Any + +from generate.math_parser import ( + _ADD_VERBS, + _SUBTRACT_VERBS, + _TRANSFER_VERBS, +) +from generate.math_problem_graph import ( + InitialPossession, + MathProblemGraph, + Operation, + Quantity, +) +from generate.math_solver import SolutionTrace, solve +from generate.ood_surface_generator import ( + _ENTITY_REGISTRY, + _entity_map, + _number, + _rename_graph, + _render_graph, + _surface_unit, + _unit_map, +) + + +INVARIANCE_PRESERVING = "invariance_preserving" +INVARIANCE_BREAKING = "invariance_breaking" + +TRANSFORMS: tuple[str, ...] = ( + "rename_entities", + "rename_units", + "reorder_independent_initial_possessions", + "reorder_independent_operations", + "replace_verb_with_synonym", + "add_zero_quantity_entity", + "swap_non_commuting_operations", +) + + +@dataclass(frozen=True, slots=True) +class Perturbation: + original_id: str + perturbation_id: str + kind: str + transform: str + transform_params: dict[str, Any] + problem_text: str + expected_graph: MathProblemGraph + expected_answer: float + expected_unit: str + + +def generate_perturbations( + problem: str, + ground_truth_graph: MathProblemGraph, + *, + seed: int, +) -> list[Perturbation]: + """Return one semantic perturbation per applicable transform. + + Inapplicable transforms are skipped; call ``skip_reasons`` with the + same inputs to report those skips at scoring time. + """ + if not isinstance(problem, str) or not problem.strip(): + raise ValueError("problem must be a non-empty string") + if not isinstance(seed, int) or isinstance(seed, bool): + raise ValueError("seed must be an integer") + + original_id = _original_id_from_seed(seed) + builders = ( + _rename_entities, + _rename_units, + _reorder_independent_initial_possessions, + _reorder_independent_operations, + _replace_verb_with_synonym, + _add_zero_quantity_entity, + _swap_non_commuting_operations, + ) + perturbations: list[Perturbation] = [] + for build in builders: + perturbation = build( + problem=problem, + graph=ground_truth_graph, + original_id=original_id, + seed=seed, + ) + if perturbation is not None: + perturbations.append(perturbation) + return perturbations + + +def skip_reasons( + problem: str, + ground_truth_graph: MathProblemGraph, + *, + seed: int, +) -> dict[str, str]: + """Return deterministic skip reasons for inapplicable transforms.""" + del problem, seed + reasons: dict[str, str] = {} + if len(ground_truth_graph.initial_state) < 2: + reasons[ + "reorder_independent_initial_possessions" + ] = "requires at least two initial possessions" + if _independent_operation_order(ground_truth_graph) is None: + reasons[ + "reorder_independent_operations" + ] = "requires at least two pairwise independent operations" + if _first_synonym_slot(ground_truth_graph) is None: + reasons[ + "replace_verb_with_synonym" + ] = "requires a first add/subtract/transfer operation" + if _zero_entity_name(ground_truth_graph, 0) is None: + reasons["add_zero_quantity_entity"] = "requires an unused registry entity" + if _non_commuting_swap(ground_truth_graph) is None: + reasons[ + "swap_non_commuting_operations" + ] = "requires two same-entity operations whose swap changes trace" + return reasons + + +def _rename_entities( + *, problem: str, graph: MathProblemGraph, original_id: str, seed: int +) -> Perturbation: + del problem + entity_map = _entity_map(graph, seed) + renamed = _rename_graph(graph, entity_map, {u: u for u in _ordered_units(graph)}) + return _perturbation( + original_id=original_id, + suffix="rename_ent", + kind=INVARIANCE_PRESERVING, + transform="rename_entities", + params={"entity_map": entity_map}, + graph=renamed, + ) + + +def _rename_units( + *, problem: str, graph: MathProblemGraph, original_id: str, seed: int +) -> Perturbation: + del problem + unit_map = _unit_map(graph, seed) + renamed = _rename_graph(graph, {e: e for e in graph.entities}, unit_map) + return _perturbation( + original_id=original_id, + suffix="rename_unit", + kind=INVARIANCE_PRESERVING, + transform="rename_units", + params={"unit_map": unit_map}, + graph=renamed, + ) + + +def _reorder_independent_initial_possessions( + *, problem: str, graph: MathProblemGraph, original_id: str, seed: int +) -> Perturbation | None: + del problem, seed + if len(graph.initial_state) < 2: + return None + reordered_initial = tuple(reversed(graph.initial_state)) + reordered_entities = _entities_for( + initial_state=reordered_initial, + operations=graph.operations, + fallback=graph.entities, + ) + reordered = MathProblemGraph( + entities=reordered_entities, + initial_state=reordered_initial, + operations=graph.operations, + unknown=graph.unknown, + ) + return _perturbation( + original_id=original_id, + suffix="reorder_init", + kind=INVARIANCE_PRESERVING, + transform="reorder_independent_initial_possessions", + params={"order": "reversed", "count": len(graph.initial_state)}, + graph=reordered, + ) + + +def _reorder_independent_operations( + *, problem: str, graph: MathProblemGraph, original_id: str, seed: int +) -> Perturbation | None: + del problem, seed + new_order = _independent_operation_order(graph) + if new_order is None: + return None + reordered = MathProblemGraph( + entities=graph.entities, + initial_state=graph.initial_state, + operations=new_order, + unknown=graph.unknown, + ) + return _perturbation( + original_id=original_id, + suffix="reorder_ops", + kind=INVARIANCE_PRESERVING, + transform="reorder_independent_operations", + params={"order": "reversed", "count": len(graph.operations)}, + graph=reordered, + ) + + +def _replace_verb_with_synonym( + *, problem: str, graph: MathProblemGraph, original_id: str, seed: int +) -> Perturbation | None: + slot = _first_synonym_slot(graph) + if slot is None: + return None + index, verbs = slot + original_verb = _first_operation_verb(problem) + verb = _choose_synonym(verbs, original_verb, seed) + if verb is None: + return None + params = { + "operation_index": index, + "replacement_verb": verb, + "original_verb": original_verb, + } + return _perturbation( + original_id=original_id, + suffix="verb_syn", + kind=INVARIANCE_PRESERVING, + transform="replace_verb_with_synonym", + params=params, + graph=graph, + problem_text=_render_graph_with_operation_verbs(graph, {index: verb}), + ) + + +def _add_zero_quantity_entity( + *, problem: str, graph: MathProblemGraph, original_id: str, seed: int +) -> Perturbation | None: + del problem + entity = _zero_entity_name(graph, seed) + if entity is None: + return None + unit = graph.unknown.unit + zero = InitialPossession(entity=entity, quantity=Quantity(value=0, unit=unit)) + expanded = MathProblemGraph( + entities=(entity, *graph.entities), + initial_state=(zero, *graph.initial_state), + operations=graph.operations, + unknown=graph.unknown, + ) + return _perturbation( + original_id=original_id, + suffix="zero_entity", + kind=INVARIANCE_PRESERVING, + transform="add_zero_quantity_entity", + params={"entity": entity, "quantity": 0, "unit": unit}, + graph=expanded, + ) + + +def _swap_non_commuting_operations( + *, problem: str, graph: MathProblemGraph, original_id: str, seed: int +) -> Perturbation | None: + del problem, seed + swap = _non_commuting_swap(graph) + if swap is None: + return None + i, j = swap + operations = list(graph.operations) + operations[i], operations[j] = operations[j], operations[i] + swapped = MathProblemGraph( + entities=graph.entities, + initial_state=graph.initial_state, + operations=tuple(operations), + unknown=graph.unknown, + ) + original_trace = solve(graph) + expected_trace = solve(swapped) + return _perturbation( + original_id=original_id, + suffix="swap_noncomm", + kind=INVARIANCE_BREAKING, + transform="swap_non_commuting_operations", + params={ + "swapped_indices": [i, j], + "original_answer": original_trace.answer_value, + "original_trace_hash": _trace_hash(original_trace), + "expected_trace_hash": _trace_hash(expected_trace), + }, + graph=swapped, + ) + + +def _perturbation( + *, + original_id: str, + suffix: str, + kind: str, + transform: str, + params: dict[str, Any], + graph: MathProblemGraph, + problem_text: str | None = None, +) -> Perturbation: + trace = solve(graph) + return Perturbation( + original_id=original_id, + perturbation_id=f"{original_id}:{suffix}", + kind=kind, + transform=transform, + transform_params=params, + problem_text=problem_text if problem_text is not None else _render_graph(graph), + expected_graph=graph, + expected_answer=trace.answer_value, + expected_unit=trace.answer_unit, + ) + + +def _original_id_from_seed(seed: int) -> str: + if 1 <= seed <= 999: + return f"gpd-{seed:03d}" + return f"seed-{seed}" + + +def _ordered_units(graph: MathProblemGraph) -> tuple[str, ...]: + units: list[str] = [] + + def add(unit: str) -> None: + if unit not in units: + units.append(unit) + + for possession in graph.initial_state: + add(possession.quantity.unit) + for operation in graph.operations: + add(operation.operand.unit) + add(graph.unknown.unit) + return tuple(units) + + +def _entities_for( + *, + initial_state: tuple[InitialPossession, ...], + operations: tuple[Operation, ...], + fallback: tuple[str, ...], +) -> tuple[str, ...]: + entities: list[str] = [] + + def add(entity: str | None) -> None: + if entity is not None and entity not in entities: + entities.append(entity) + + for possession in initial_state: + add(possession.entity) + for operation in operations: + add(operation.actor) + add(operation.target) + for entity in fallback: + add(entity) + return tuple(entities) + + +def _independent_operation_order(graph: MathProblemGraph) -> tuple[Operation, ...] | None: + if len(graph.operations) < 2: + return None + affected: list[set[tuple[str, str]]] = [_affected_state(op) for op in graph.operations] + for i, left in enumerate(affected): + for right in affected[i + 1 :]: + if left & right: + return None + reversed_ops = tuple(reversed(graph.operations)) + if reversed_ops == graph.operations: + return None + return reversed_ops + + +def _affected_state(operation: Operation) -> set[tuple[str, str]]: + out = {(operation.actor, operation.operand.unit)} + if operation.target is not None: + out.add((operation.target, operation.operand.unit)) + return out + + +def _first_synonym_slot( + graph: MathProblemGraph, +) -> tuple[int, tuple[str, ...]] | None: + if not graph.operations: + return None + operation = graph.operations[0] + if operation.kind == "add": + return 0, tuple(sorted(_ADD_VERBS)) + if operation.kind == "subtract": + return 0, tuple(sorted(_SUBTRACT_VERBS)) + if operation.kind == "transfer": + return 0, tuple(sorted(_TRANSFER_VERBS)) + return None + + +def _first_operation_verb(problem: str) -> str | None: + words = problem.replace(",", " ").replace(".", " ").split() + lower_verbs = _ADD_VERBS | _SUBTRACT_VERBS | _TRANSFER_VERBS + for word in words: + lowered = word.lower() + if lowered in lower_verbs: + return lowered + return None + + +def _choose_synonym( + verbs: tuple[str, ...], original_verb: str | None, seed: int +) -> str | None: + candidates = [v for v in verbs if v != original_verb] + if not candidates: + return None + return candidates[seed % len(candidates)] + + +def _zero_entity_name(graph: MathProblemGraph, seed: int) -> str | None: + used = set(graph.entities) + candidates = [name for name in _ENTITY_REGISTRY if name not in used] + if not candidates: + return None + return candidates[seed % len(candidates)] + + +def _non_commuting_swap(graph: MathProblemGraph) -> tuple[int, int] | None: + original_trace = solve(graph) + for i, left in enumerate(graph.operations): + for j in range(i + 1, len(graph.operations)): + right = graph.operations[j] + if _affected_state(left) != _affected_state(right): + continue + operations = list(graph.operations) + operations[i], operations[j] = operations[j], operations[i] + swapped = MathProblemGraph( + entities=graph.entities, + initial_state=graph.initial_state, + operations=tuple(operations), + unknown=graph.unknown, + ) + swapped_trace = solve(swapped) + if swapped_trace.canonical_bytes() != original_trace.canonical_bytes(): + return i, j + return None + + +def _trace_hash(trace: SolutionTrace) -> str: + return hashlib.sha256(trace.canonical_bytes()).hexdigest() + + +def _render_graph_with_operation_verbs( + graph: MathProblemGraph, operation_verbs: dict[int, str] +) -> str: + sentences: list[str] = [] + for possession in graph.initial_state: + value = possession.quantity.value + unit = _surface_unit(possession.quantity.unit, value) + sentences.append(f"{possession.entity} has {_number(value)} {unit}.") + + for index, operation in enumerate(graph.operations): + value = operation.operand.value + unit = _surface_unit(operation.operand.unit, value) + verb = operation_verbs.get(index) + if operation.kind == "add": + chosen = verb if verb is not None else "buys" + sentence = f"{operation.actor} {chosen} {_number(value)} more {unit}." + elif operation.kind == "subtract": + chosen = verb if verb is not None else "loses" + sentence = f"{operation.actor} {chosen} {_number(value)} {unit}." + elif operation.kind == "transfer": + chosen = verb if verb is not None else "gives" + sentence = ( + f"{operation.actor} {chosen} {_number(value)} {unit} " + f"to {operation.target}." + ) + elif operation.kind == "multiply": + chosen = "doubles" if operation.operand.value == 2 else "triples" + sentence = f"{operation.actor} {chosen} his {operation.operand.unit}." + elif operation.kind == "divide": + sentence = ( + f"{operation.actor} splits them evenly into " + f"{_number(value)} groups and keeps one group." + ) + else: + raise ValueError(f"unsupported operation kind: {operation.kind!r}") + sentences.append(sentence) + + question_unit = _surface_unit(graph.unknown.unit, 2) + if graph.unknown.entity is None: + sentences.append(f"How many {question_unit} do they have in total?") + else: + sentences.append( + f"How many {question_unit} does {graph.unknown.entity} have now?" + ) + return " ".join(sentences) diff --git a/tests/test_perturbation_suite.py b/tests/test_perturbation_suite.py new file mode 100644 index 00000000..2185cd23 --- /dev/null +++ b/tests/test_perturbation_suite.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +from evals.gsm8k_parser_dev.perturbation_score import score_perturbation +from generate.math_parser import parse_problem +from generate.math_problem_graph import ( + InitialPossession, + MathProblemGraph, + Operation, + Quantity, + Unknown, + graph_from_dict, +) +from generate.math_solver import solve +from generate.perturbation_suite import ( + INVARIANCE_BREAKING, + INVARIANCE_PRESERVING, + Perturbation, + generate_perturbations, +) + + +_REPO_ROOT = Path(__file__).resolve().parents[1] +_CASES_PATH = _REPO_ROOT / "evals" / "gsm8k_parser_dev" / "cases.jsonl" + + +def _cases() -> list[dict]: + return [ + json.loads(line) + for line in _CASES_PATH.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def _seed(case_id: str) -> int: + return int(case_id.rsplit("-", 1)[1]) + + +def _graph(case: dict) -> MathProblemGraph: + return graph_from_dict(case["ground_truth_graph"]) + + +def _perturbations() -> list[tuple[dict, Perturbation]]: + out: list[tuple[dict, Perturbation]] = [] + for case in _cases(): + graph = _graph(case) + for perturbation in generate_perturbations( + case["problem"], graph, seed=_seed(case["id"]) + ): + out.append((case, perturbation)) + return out + + +def _perturbation_bytes(perturbations: list[Perturbation]) -> bytes: + payload = [ + { + "original_id": p.original_id, + "perturbation_id": p.perturbation_id, + "kind": p.kind, + "transform": p.transform, + "transform_params": p.transform_params, + "problem_text": p.problem_text, + "expected_graph": json.loads(p.expected_graph.canonical_bytes()), + "expected_answer": p.expected_answer, + "expected_unit": p.expected_unit, + } + for p in perturbations + ] + return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def _trace_hash(graph: MathProblemGraph) -> str: + return hashlib.sha256(solve(graph).canonical_bytes()).hexdigest() + + +def test_generator_is_byte_deterministic_for_same_seed() -> None: + case = _cases()[20] + graph = _graph(case) + first = generate_perturbations(case["problem"], graph, seed=_seed(case["id"])) + second = generate_perturbations(case["problem"], graph, seed=_seed(case["id"])) + assert _perturbation_bytes(first) == _perturbation_bytes(second) + + +def test_invariance_preserving_perturbations_keep_original_answer_value() -> None: + for case, perturbation in _perturbations(): + if perturbation.kind != INVARIANCE_PRESERVING: + continue + original_answer = solve(_graph(case)).answer_value + assert perturbation.expected_answer == original_answer, perturbation + + +def test_invariance_breaking_perturbations_match_predicted_graph_solve() -> None: + for _, perturbation in _perturbations(): + if perturbation.kind != INVARIANCE_BREAKING: + continue + trace = solve(perturbation.expected_graph) + assert perturbation.expected_answer == trace.answer_value + assert perturbation.expected_unit == trace.answer_unit + assert perturbation.transform_params["expected_trace_hash"] == _trace_hash( + perturbation.expected_graph + ) + + +def test_aggregate_dev_rates_are_perfect_for_applicable_perturbations() -> None: + preserving: list[bool] = [] + breaking: list[bool] = [] + for _, perturbation in _perturbations(): + ok, detail = score_perturbation(perturbation) + assert ok, f"{perturbation.perturbation_id}: {detail}" + if perturbation.kind == INVARIANCE_PRESERVING: + preserving.append(ok) + elif perturbation.kind == INVARIANCE_BREAKING: + breaking.append(ok) + + assert preserving and sum(preserving) / len(preserving) == 1.0 + assert breaking and sum(breaking) / len(breaking) == 1.0 + + +def test_verb_synonym_perturbations_use_a_different_verb() -> None: + for _, perturbation in _perturbations(): + if perturbation.transform != "replace_verb_with_synonym": + continue + original = perturbation.transform_params["original_verb"] + replacement = perturbation.transform_params["replacement_verb"] + assert original is not None + assert replacement != original + assert f" {replacement} " in perturbation.problem_text + + +def test_reorder_initial_perturbations_change_sentence_order_but_preserve_answer() -> None: + case = next(c for c in _cases() if c["id"] == "gpd-005") + original_graph = _graph(case) + perturbation = next( + p + for p in generate_perturbations(case["problem"], original_graph, seed=_seed(case["id"])) + if p.transform == "reorder_independent_initial_possessions" + ) + + assert perturbation.problem_text != case["problem"] + assert perturbation.expected_graph.initial_state == tuple( + reversed(original_graph.initial_state) + ) + assert perturbation.expected_answer == solve(original_graph).answer_value + + +def test_reorder_independent_operations_changes_sentence_order_but_not_answer() -> None: + graph = MathProblemGraph( + entities=("Tom", "Sara"), + initial_state=( + InitialPossession("Tom", Quantity(4, "apples")), + InitialPossession("Sara", Quantity(7, "apples")), + ), + operations=( + Operation("Tom", "add", Quantity(3, "apples")), + Operation("Sara", "subtract", Quantity(2, "apples")), + ), + unknown=Unknown(entity=None, unit="apples"), + ) + problem = ( + "Tom has 4 apples. Sara has 7 apples. Tom buys 3 more apples. " + "Sara loses 2 apples. How many apples do they have in total?" + ) + perturbation = next( + p + for p in generate_perturbations(problem, graph, seed=777) + if p.transform == "reorder_independent_operations" + ) + + parsed = parse_problem(perturbation.problem_text) + assert perturbation.problem_text.index("Sara loses") < perturbation.problem_text.index( + "Tom buys" + ) + assert parsed.operations == tuple(reversed(graph.operations)) + assert solve(parsed).answer_value == solve(graph).answer_value