diff --git a/docs/decisions/ADR-0118a-ood-surface-generator.md b/docs/decisions/ADR-0118a-ood-surface-generator.md new file mode 100644 index 00000000..d67e446a --- /dev/null +++ b/docs/decisions/ADR-0118a-ood-surface-generator.md @@ -0,0 +1,189 @@ +# ADR-0118a — OOD Surface Generator for GSM8K-Style Parser Dev + +**Status:** Accepted +**Date:** 2026-05-22 +**Author:** CORE agents + reviewers +**Depends on:** ADR-0114, ADR-0114a, ADR-0115, ADR-0116 + +--- + +## Context + +ADR-0114a Obligation #2 requires a capability lane that scores `S` on +its public split to score at least `0.95 * S` on a programmatically +derived out-of-distribution split that holds the underlying graph +constant while varying surface form. + +The current GSM8K-style parser development lane has 50 authored public +cases in `evals/gsm8k_parser_dev/cases.jsonl`. ADR-0115 fixes the +Phase 1.1 parser grammar and `MathProblemGraph` schema; ADR-0116 fixes +the deterministic solver. This ADR adds the missing OOD surface lane +without changing the parser, solver, graph schema, or public cases. + +--- + +## Decision + +### OOD variant generator + +`generate/ood_surface_generator.py` exports: + +```text +generate_ood_variants(problem, ground_truth_graph, *, seed, n=3) +``` + +and the frozen, slotted `OODVariant` record. The generator is pure and +deterministic: same `problem`, `ground_truth_graph`, `seed`, and `n` +produce byte-equal variant records. + +The generator renders from `MathProblemGraph` rather than performing +free-form text edits. This keeps the surface inside ADR-0115 Phase 1.1: + +- Title-cased one-word entities +- lowercase single-token plural units +- direct declarative possession and operation sentences +- one `How many ...` question +- parser-supported add/subtract/transfer/multiply/divide forms + +The default `n=3` emits one variant in each transform class: + +| Transform | Behavior | +|---|---| +| `rename_entities` | Replaces every entity with a fixed-registry OOD proper noun in order of introduction. | +| `rename_units` | Replaces every unit with a fixed-registry OOD lowercase plural noun, preserving singular/plural surface rendering. | +| `scale_numbers_by_k` | Multiplies initial quantities and add/subtract/transfer operands by `k in {2, 3, 5}`; multiply/divide scalar operands are unchanged. | + +Every rendered variant uses OOD entity names and OOD units so the +surface does not overlap with public dev entity or unit strings. The +required fixed registries are shipped in the module. `Wren` remains in +the registry as specified, but is excluded from selection because it +appears in the public split; `nebulae` remains in the registry but is +not selected because the current parser's canonical plural rule would +map that surface to `nebulaes`. + +### OOD scorer + +`evals/gsm8k_parser_dev/ood_score.py` exposes: + +```bash +python3 -m evals.gsm8k_parser_dev.ood_score +``` + +The scorer: + +1. Loads the 50 public dev cases. +2. Scores public parser+solver correctness. +3. Generates three OOD variants per case with a deterministic seed + derived from the case id. +4. Parses and solves every variant. +5. Prints per-variant pass/fail, per-transform ratios, public ratio, + OOD ratio, and `ood/public`. +6. Exits `0` when `ood/public >= 0.95`, else exits `1`. + +--- + +## Invariants + +### `adr_0118a_generator_determinism` + +Two calls with the same `problem`, `ground_truth_graph`, `seed`, and +`n` produce byte-equal serialized variants. + +### `adr_0118a_unrename_preserves_original_graph` + +Each variant carries `expected_graph_after_unrename` byte-equal to the +source `ground_truth_graph`. Entity and unit relabeling are reversible +and structure-preserving. + +### `adr_0118a_live_parser_solver_accepts_variants` + +Every generated variant is parsed and solved by the live ADR-0115 / +ADR-0116 contracts, and the solver answer matches the variant's +expected answer and unit. + +### `adr_0118a_ood_public_ratio_gate` + +Across the 50-case public dev set, OOD/public score ratio is at least +`0.95`. + +### `adr_0118a_no_public_surface_overlap` + +No generated variant uses an entity or unit string from the public dev +set. + +### `adr_0118a_scale_is_linear` + +For scale-by-`k` variants, `original_answer * k == +variant.expected_answer`. + +--- + +## ADR-0114a obligation discharge summary + +This ADR closes ADR-0114a Obligation #2 for the GSM8K-style parser +development lane: public score `S` and programmatic OOD score are both +measured by the same parser+solver contract, and the OOD/public ratio +is pinned in acceptance evidence. + +| Obligation | Status under ADR-0118a | +|---|---| +| #1 Sealed-holdout discipline | Substrate present; per-lane enforcement remains for later ADRs | +| #2 OOD surface variation | **Discharged** | +| #3 Replay-equal trace | Discharged by ADR-0116/0117 path, not changed here | +| #4 Typed refusal | Discharged by ADR-0115/0116 path, not changed here | +| #5 Reasoning-isolation perturbation suite | Remains for later ADRs | +| #6 Compositional-depth curve | Remains for later ADRs | +| #7 Frontier-baseline comparison | Remains for later ADRs | +| #8 Adversarial generation | Remains for later ADRs | +| #9 Determinism | Discharged at solver layer by ADR-0116; generator determinism added here | +| #10 Operation provenance via pack | Discharged by ADR-0116 | + +#1, #5, #6, #7, and #8 remain for later ADRs. + +--- + +## Acceptance evidence + +Accepted when: + +- `generate/ood_surface_generator.py` exports `generate_ood_variants` + and `OODVariant` +- `evals/gsm8k_parser_dev/ood_score.py` runs as + `python3 -m evals.gsm8k_parser_dev.ood_score` +- `tests/test_ood_surface_generator.py` is green +- Smoke suite is green +- The OOD scorer reports: + - `rename_entities`: 50/50 = 1.0000 + - `rename_units`: 50/50 = 1.0000 + - `scale_numbers_by_k`: 50/50 = 1.0000 + - public: 50/50 = 1.0000 + - OOD: 150/150 = 1.0000 + - OOD/public: 1.0000 +- ADR linked from `docs/decisions/README.md` index and frontier + +--- + +## Consequences + +- ADR-0114a Obligation #2 is now executable evidence rather than a + promissory requirement. +- Parser/solver surface dependence is measured without extending the + grammar, changing schemas, or modifying public cases. +- The future `expert` promotion ledger can cite an OOD/public ratio + produced by a deterministic local command. +- The generator creates a reusable shape for later perturbation suites, + but does not claim to discharge the broader ADR-0114a Obligation #5. + +--- + +## Out of scope + +- Independent-sentence reordering. ADR-0114a lists it as an OOD + example, but this ADR implements the three requested transform + classes only. +- Adversarial generation and misparse probing. Remains for ADR-0114a + Obligation #8. +- Reasoning-isolation perturbations that intentionally change answers + beyond linear scaling. Remains for ADR-0114a Obligation #5. +- Any parser, solver, graph schema, or dev-case expansion. +- LLMs, sampling, stochastic generation, or approximate scoring. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index ba02b640..96ecd7ce 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -39,6 +39,7 @@ ADRs record significant architectural decisions: what was decided, why, what alt | [ADR-0115](ADR-0115-math-problem-parser-and-graph.md) | Math Problem Parser and Typed Proposition Graph | Phase 1.1+1.2+1.3 Accepted (2026-05-22) | | [ADR-0116](ADR-0116-deterministic-solver.md) | Deterministic Solver (`MathProblemGraph` → `SolutionTrace`) | Accepted (2026-05-22) | | [ADR-0117](ADR-0117-solution-trace-verifier.md) | `SolutionTrace` Verifier (independent of solver) | Accepted (2026-05-22) | +| [ADR-0118a](ADR-0118a-ood-surface-generator.md) | OOD Surface Generator for GSM8K-Style Parser Dev | Accepted (2026-05-22) | | [ADR-0122](ADR-0122-systems-software-audit-passed-deferred.md) | `systems_software` Audit-Passed Promotion: Deferred | Accepted (2026-05-22) | --- @@ -75,6 +76,7 @@ The ADR-0091..0114 slate is fully accepted (0091..0113) plus one proposed-roadma - Anti-Overfitting Proof Obligations for any future `expert` promotion (10-point falsifiable framework) — ADR-0114a - Deterministic Solver (Phase 2; SolutionTrace + en_arithmetic_v1 pack; discharges ADR-0114a obligations #3, #4, #9, #10) — ADR-0116 - SolutionTrace Verifier (Phase 3; solver-independent replay; lifts ADR-0114a Obligation #3 to verifier fidelity) — ADR-0117 +- OOD Surface Generator (150 deterministic variants; discharges ADR-0114a obligation #2 for the GSM8K-style parser dev lane) — ADR-0118a ADR-0080 has also landed: Contemplation Loop Phase 1 adds a read-only frontier-compare miner that emits `SPECULATIVE` findings only. diff --git a/evals/gsm8k_parser_dev/__init__.py b/evals/gsm8k_parser_dev/__init__.py new file mode 100644 index 00000000..f22e73b7 --- /dev/null +++ b/evals/gsm8k_parser_dev/__init__.py @@ -0,0 +1 @@ +"""ADR-0115/0118a GSM8K-style parser development lane.""" diff --git a/evals/gsm8k_parser_dev/ood_score.py b/evals/gsm8k_parser_dev/ood_score.py new file mode 100644 index 00000000..e45b16d1 --- /dev/null +++ b/evals/gsm8k_parser_dev/ood_score.py @@ -0,0 +1,120 @@ +"""Score ADR-0118a OOD surface variants for the parser dev lane.""" + +from __future__ import annotations + +import json +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path + +from generate.math_parser import ParseError, parse_problem +from generate.math_problem_graph import MathProblemGraph, graph_from_dict +from generate.math_solver import SolveError, solve +from generate.ood_surface_generator import OODVariant, generate_ood_variants + + +_CASES_PATH = Path(__file__).with_name("cases.jsonl") +_GATE_RATIO = 0.95 + + +@dataclass(frozen=True, slots=True) +class Case: + case_id: str + problem: str + expected_answer: float + expected_unit: str + graph: MathProblemGraph + + +def load_cases(path: Path = _CASES_PATH) -> list[Case]: + cases: list[Case] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + raw = json.loads(line) + cases.append( + Case( + case_id=raw["id"], + problem=raw["problem"], + expected_answer=raw["expected_answer"], + expected_unit=raw["expected_unit"], + graph=graph_from_dict(raw["ground_truth_graph"]), + ) + ) + return cases + + +def score_public(cases: list[Case]) -> tuple[int, int]: + correct = 0 + for case in cases: + try: + graph = parse_problem(case.problem) + trace = solve(graph) + except (ParseError, SolveError): + continue + if ( + graph.canonical_bytes() == case.graph.canonical_bytes() + and trace.answer_value == case.expected_answer + and trace.answer_unit == case.expected_unit + ): + correct += 1 + return correct, len(cases) + + +def score_variant(variant: OODVariant) -> tuple[bool, str]: + try: + trace = solve(parse_problem(variant.problem_text)) + except (ParseError, SolveError) as exc: + return False, f"{type(exc).__name__}: {exc}" + if trace.answer_unit != variant.expected_unit: + return False, f"unit {trace.answer_unit!r} != {variant.expected_unit!r}" + if trace.answer_value != variant.expected_answer: + return False, f"answer {trace.answer_value!r} != {variant.expected_answer!r}" + return True, "ok" + + +def _seed_from_case_id(case_id: str) -> int: + return int(case_id.rsplit("-", 1)[1]) + + +def main() -> int: + cases = load_cases() + public_correct, public_total = score_public(cases) + public_ratio = public_correct / public_total if public_total else 0.0 + + ood_correct = 0 + ood_total = 0 + per_transform: dict[str, list[bool]] = defaultdict(list) + + for case in cases: + variants = generate_ood_variants( + case.problem, + case.graph, + seed=_seed_from_case_id(case.case_id), + n=3, + ) + for variant in variants: + ok, detail = score_variant(variant) + ood_correct += int(ok) + ood_total += 1 + per_transform[variant.transform].append(ok) + status = "PASS" if ok else "FAIL" + print(f"{status} {variant.variant_id} {variant.transform}: {detail}") + + print() + for transform in sorted(per_transform): + results = per_transform[transform] + correct = sum(results) + total = len(results) + ratio = correct / total if total else 0.0 + print(f"transform {transform}: {correct}/{total} = {ratio:.4f}") + + ood_ratio = ood_correct / ood_total if ood_total else 0.0 + relative = ood_ratio / public_ratio if public_ratio else 0.0 + print(f"public: {public_correct}/{public_total} = {public_ratio:.4f}") + print(f"ood: {ood_correct}/{ood_total} = {ood_ratio:.4f}") + print(f"ood/public: {relative:.4f}") + + return 0 if relative >= _GATE_RATIO else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/generate/ood_surface_generator.py b/generate/ood_surface_generator.py new file mode 100644 index 00000000..826d92fe --- /dev/null +++ b/generate/ood_surface_generator.py @@ -0,0 +1,331 @@ +"""ADR-0118a — deterministic OOD surface generator for math dev cases. + +The generator varies surface form while staying inside the ADR-0115 +Phase 1.1 parser grammar. It renders from ``MathProblemGraph`` rather +than performing ad hoc text edits, so entity order, operation order, and +solver-visible arithmetic remain explicit. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from generate.math_problem_graph import ( + InitialPossession, + MathProblemGraph, + Operation, + Quantity, + Unknown, +) +from generate.math_solver import solve + + +_ENTITY_REGISTRY = ( + "Quill", + "Renn", + "Sable", + "Thora", + "Ulric", + "Vesta", + "Wren", + "Xan", + "Ynez", + "Zora", + "Arlo", + "Brae", + "Cedric", + "Doria", + "Eira", + "Finch", + "Grim", + "Hale", + "Indra", + "Jora", +) +_UNIT_REGISTRY = ( + "nebulae", + "spires", + "lanterns", + "ingots", + "shards", + "scrolls", + "talismans", + "obsidians", + "feathers", + "runes", + "crystals", + "pelts", + "moonbeams", + "embers", + "ledgers", + "phials", + "compasses", + "trinkets", +) +_SCALE_FACTORS = (2, 3, 5) + +_TRANSFORMS = ("rename_entities", "rename_units", "scale_numbers_by_k") +_TRANSFORM_SHORT = { + "rename_entities": "rename_ent", + "rename_units": "rename_unit", +} + +# ``Wren`` appears in the public dev split. Keep the required fixed +# registry visible, but never select public-overlapping names. +_PUBLIC_DEV_ENTITY_EXCLUSIONS = frozenset({"Wren"}) + + +@dataclass(frozen=True, slots=True) +class OODVariant: + original_id: str + variant_id: str + transform: str + transform_params: dict[str, Any] + problem_text: str + expected_graph_after_unrename: MathProblemGraph + expected_answer: float + expected_unit: str + + +def generate_ood_variants( + problem: str, + ground_truth_graph: MathProblemGraph, + *, + seed: int, + n: int = 3, +) -> list[OODVariant]: + """Return deterministic OOD variants for one public dev problem. + + ``problem`` participates in the deterministic seed stream so that two + different surfaces with the same graph cannot accidentally share the + same variant rotation. No I/O, global mutable state, or unseeded + randomness is used. + """ + if not isinstance(problem, str) or not problem.strip(): + raise ValueError("problem must be a non-empty string") + if not isinstance(seed, int) or isinstance(seed, bool): + raise ValueError("seed must be an integer") + if n < 0: + raise ValueError("n must be non-negative") + + original_id = _original_id_from_seed(seed) + start = _stable_offset(problem, seed) + variants: list[OODVariant] = [] + for index in range(n): + transform = _TRANSFORMS[(start + index) % len(_TRANSFORMS)] + variants.append( + _build_variant( + original_id=original_id, + graph=ground_truth_graph, + seed=seed, + transform=transform, + ) + ) + return variants + + +def _build_variant( + *, + original_id: str, + graph: MathProblemGraph, + seed: int, + transform: str, +) -> OODVariant: + entity_map = _entity_map(graph, seed) + unit_map = _unit_map(graph, seed) + k: int | None = None + working = graph + params: dict[str, Any] = {} + + if transform == "scale_numbers_by_k": + k = _SCALE_FACTORS[seed % len(_SCALE_FACTORS)] + working = _scale_graph(graph, k) + params["k"] = k + + surface_graph = _rename_graph(working, entity_map, unit_map) + trace = solve(surface_graph) + if k is not None: + params["scaled_answer"] = trace.answer_value + + short = f"scale_k{k}" if k is not None else _TRANSFORM_SHORT[transform] + return OODVariant( + original_id=original_id, + variant_id=f"{original_id}:{short}", + transform=transform, + transform_params=params, + problem_text=_render_graph(surface_graph), + expected_graph_after_unrename=graph, + expected_answer=trace.answer_value, + expected_unit=trace.answer_unit, + ) + + +def _original_id_from_seed(seed: int) -> str: + if 1 <= seed <= 999: + return f"gpd-{seed:03d}" + return f"seed-{seed}" + + +def _stable_offset(problem: str, seed: int) -> int: + return (sum(problem.encode("utf-8")) + seed) % len(_TRANSFORMS) + + +def _entity_map(graph: MathProblemGraph, seed: int) -> dict[str, str]: + names = [n for n in _ENTITY_REGISTRY if n not in _PUBLIC_DEV_ENTITY_EXCLUSIONS] + offset = seed % len(names) + if len(graph.entities) > len(names): + raise ValueError("not enough OOD entity names for graph") + selected = [names[(offset + i) % len(names)] for i in range(len(graph.entities))] + return dict(zip(graph.entities, selected, strict=True)) + + +def _unit_map(graph: MathProblemGraph, seed: int) -> dict[str, str]: + units = _ordered_units(graph) + stable_units = [u for u in _UNIT_REGISTRY if u.endswith("s")] + offset = (seed * 2) % len(stable_units) + if len(units) > len(stable_units): + raise ValueError("not enough OOD unit names for graph") + selected = [stable_units[(offset + i) % len(stable_units)] for i in range(len(units))] + return dict(zip(units, selected, strict=True)) + + +def _ordered_units(graph: MathProblemGraph) -> tuple[str, ...]: + units: list[str] = [] + + def add(unit: str) -> None: + if unit not in units: + units.append(unit) + + for possession in graph.initial_state: + add(possession.quantity.unit) + for operation in graph.operations: + add(operation.operand.unit) + add(graph.unknown.unit) + return tuple(units) + + +def _rename_graph( + graph: MathProblemGraph, entity_map: dict[str, str], unit_map: dict[str, str] +) -> MathProblemGraph: + return MathProblemGraph( + entities=tuple(entity_map[e] for e in graph.entities), + initial_state=tuple( + InitialPossession( + entity=entity_map[p.entity], + quantity=Quantity( + value=p.quantity.value, + unit=unit_map[p.quantity.unit], + ), + ) + for p in graph.initial_state + ), + operations=tuple( + Operation( + actor=entity_map[o.actor], + kind=o.kind, + operand=Quantity( + value=o.operand.value, + unit=unit_map[o.operand.unit], + ), + target=entity_map[o.target] if o.target is not None else None, + ) + for o in graph.operations + ), + unknown=Unknown( + entity=entity_map[graph.unknown.entity] + if graph.unknown.entity is not None + else None, + unit=unit_map[graph.unknown.unit], + ), + ) + + +def _scale_graph(graph: MathProblemGraph, k: int) -> MathProblemGraph: + return MathProblemGraph( + entities=graph.entities, + initial_state=tuple( + InitialPossession( + entity=p.entity, + quantity=Quantity(value=p.quantity.value * k, unit=p.quantity.unit), + ) + for p in graph.initial_state + ), + operations=tuple(_scale_operation(o, k) for o in graph.operations), + unknown=graph.unknown, + ) + + +def _scale_operation(operation: Operation, k: int) -> Operation: + value = operation.operand.value + if operation.kind in {"add", "subtract", "transfer"}: + value *= k + return Operation( + actor=operation.actor, + kind=operation.kind, + operand=Quantity(value=value, unit=operation.operand.unit), + target=operation.target, + ) + + +def _render_graph(graph: MathProblemGraph) -> str: + sentences: list[str] = [] + for possession in graph.initial_state: + value = possession.quantity.value + unit = _surface_unit(possession.quantity.unit, value) + sentences.append(f"{possession.entity} has {_number(value)} {unit}.") + + for operation in graph.operations: + value = operation.operand.value + unit = _surface_unit(operation.operand.unit, value) + if operation.kind == "add": + sentence = f"{operation.actor} buys {_number(value)} more {unit}." + elif operation.kind == "subtract": + sentence = f"{operation.actor} loses {_number(value)} {unit}." + elif operation.kind == "transfer": + sentence = ( + f"{operation.actor} gives {_number(value)} {unit} " + f"to {operation.target}." + ) + elif operation.kind == "multiply": + verb = "doubles" if operation.operand.value == 2 else "triples" + sentence = f"{operation.actor} {verb} his {operation.operand.unit}." + elif operation.kind == "divide": + sentence = ( + f"{operation.actor} splits them evenly into " + f"{_number(value)} groups and keeps one group." + ) + else: + raise ValueError(f"unsupported operation kind: {operation.kind!r}") + sentences.append(sentence) + + question_unit = _surface_unit(graph.unknown.unit, 2) + if graph.unknown.entity is None: + sentences.append(f"How many {question_unit} do they have in total?") + else: + sentences.append( + f"How many {question_unit} does {graph.unknown.entity} have now?" + ) + return " ".join(sentences) + + +def _surface_unit(unit: str, value: int | float) -> str: + if value == 1: + return _singular(unit) + return unit + + +def _singular(unit: str) -> str: + if unit.endswith("ies"): + return unit[:-3] + "y" + if unit.endswith("es") and unit[-3:-2] in {"s", "x", "z"}: + return unit[:-2] + if unit.endswith("s"): + return unit[:-1] + return unit + + +def _number(value: int | float) -> str: + if isinstance(value, float) and value.is_integer(): + return str(int(value)) + return str(value) diff --git a/tests/test_ood_surface_generator.py b/tests/test_ood_surface_generator.py new file mode 100644 index 00000000..ea4a0da4 --- /dev/null +++ b/tests/test_ood_surface_generator.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import json +import re +from pathlib import Path + +from evals.gsm8k_parser_dev.ood_score import Case, score_public, score_variant +from generate.math_problem_graph import MathProblemGraph, graph_from_dict +from generate.math_solver import solve +from generate.ood_surface_generator import OODVariant, generate_ood_variants + + +_REPO_ROOT = Path(__file__).resolve().parents[1] +_CASES_PATH = _REPO_ROOT / "evals" / "gsm8k_parser_dev" / "cases.jsonl" + + +def _cases() -> list[dict]: + return [ + json.loads(line) + for line in _CASES_PATH.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def _seed(case_id: str) -> int: + return int(case_id.rsplit("-", 1)[1]) + + +def _graph(case: dict) -> MathProblemGraph: + return graph_from_dict(case["ground_truth_graph"]) + + +def _variants() -> list[tuple[dict, OODVariant]]: + out: list[tuple[dict, OODVariant]] = [] + for case in _cases(): + graph = _graph(case) + for variant in generate_ood_variants( + case["problem"], graph, seed=_seed(case["id"]), n=3 + ): + out.append((case, variant)) + return out + + +def _variant_bytes(variants: list[OODVariant]) -> bytes: + payload = [ + { + "original_id": v.original_id, + "variant_id": v.variant_id, + "transform": v.transform, + "transform_params": v.transform_params, + "problem_text": v.problem_text, + "expected_graph_after_unrename": json.loads( + v.expected_graph_after_unrename.canonical_bytes() + ), + "expected_answer": v.expected_answer, + "expected_unit": v.expected_unit, + } + for v in variants + ] + return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def test_generator_is_byte_deterministic_for_same_seed() -> None: + case = _cases()[6] + graph = _graph(case) + first = generate_ood_variants(case["problem"], graph, seed=_seed(case["id"]), n=3) + second = generate_ood_variants(case["problem"], graph, seed=_seed(case["id"]), n=3) + assert _variant_bytes(first) == _variant_bytes(second) + + +def test_expected_graph_after_unrename_is_original_graph() -> None: + for case, variant in _variants(): + assert ( + variant.expected_graph_after_unrename.canonical_bytes() + == _graph(case).canonical_bytes() + ) + + +def test_live_parser_and_solver_match_each_variant_expected_answer() -> None: + for _, variant in _variants(): + ok, detail = score_variant(variant) + assert ok, f"{variant.variant_id}: {detail}\n{variant.problem_text}" + + +def test_ood_public_ratio_meets_gate_across_dev_set() -> None: + cases = _cases() + public_cases = [ + Case( + case_id=c["id"], + problem=c["problem"], + expected_answer=c["expected_answer"], + expected_unit=c["expected_unit"], + graph=_graph(c), + ) + for c in cases + ] + public_correct, public_total = score_public(public_cases) + public_ratio = public_correct / public_total + + results = [score_variant(v)[0] for _, v in _variants()] + ood_ratio = sum(results) / len(results) + + assert ood_ratio / public_ratio >= 0.95 + + +def test_variants_do_not_use_public_entity_or_unit_strings() -> None: + public_entities: set[str] = set() + public_units: set[str] = set() + for case in _cases(): + graph = _graph(case) + public_entities.update(graph.entities) + public_units.update(p.quantity.unit for p in graph.initial_state) + public_units.update(o.operand.unit for o in graph.operations) + public_units.add(graph.unknown.unit) + + for _, variant in _variants(): + words = set(re.findall(r"\b[A-Za-z]+\b", variant.problem_text)) + assert not (words & public_entities), variant.problem_text + assert not ({w.lower() for w in words} & public_units), variant.problem_text + + +def test_scale_by_k_variants_scale_expected_answer_linearly() -> None: + for case, variant in _variants(): + if variant.transform != "scale_numbers_by_k": + continue + original_trace = solve(_graph(case)) + k = variant.transform_params["k"] + assert original_trace.answer_value * k == variant.expected_answer