diff --git a/docs/decisions/ADR-0115-math-problem-parser-and-graph.md b/docs/decisions/ADR-0115-math-problem-parser-and-graph.md new file mode 100644 index 00000000..dd29062c --- /dev/null +++ b/docs/decisions/ADR-0115-math-problem-parser-and-graph.md @@ -0,0 +1,214 @@ +# ADR-0115 — Math Problem Parser and Typed Proposition Graph + +**Status:** Phase 1.1 Accepted (schema + 5 seed cases + tests); Phases 1.2–1.4 In Progress +**Date:** 2026-05-22 +**Author:** CORE agents + reviewers +**Depends on:** ADR-0114 + +--- + +## Context + +ADR-0114 laid out the path toward an actual `expert` ledger tier. Phase 1 +of that arc is a deterministic parser that turns a grade-school math word +problem into a typed proposition graph the solver (ADR-0116) and verifier +(ADR-0117) will consume. + +This ADR is decomposed into four sub-phases so each lands as its own +auditable step: + +- **Phase 1.1** — Define the typed graph schema, author seed cases, + pin invariants. (**This commit.**) +- **Phase 1.2** — Author the full 50-case curated dev set against the + Phase 1.1 schema. (Delegated to Codex; tracked in PR follow-up.) +- **Phase 1.3** — Implement the deterministic parser. Exit criterion: + ≥ 0.90 parse correctness against the 50-case dev set. +- **Phase 1.4** — Bind the parser to the existing CORE intent/realizer + surface so a math word problem becomes a first-class runtime input. + +Decomposing the phase keeps the schema (1.1) load-bearing for the +parser (1.3) without coupling their cadence to each other. + +--- + +## Decision + +### Phase 1.1 — what landed here + +1. `generate/math_problem_graph.py` defines the schema: + + - `Quantity(value, unit)` — frozen dataclass. + - `InitialPossession(entity, quantity)` — frozen dataclass. + - `Operation(actor, kind, operand, target?)` — frozen dataclass. + `kind ∈ {add, subtract, transfer, multiply, divide}`. `target` + required when `kind=transfer` and must differ from `actor`. + - `Unknown(entity?, unit)` — frozen dataclass; `entity=None` means + "total across every entity holding `unit`". + - `MathProblemGraph(entities, initial_state, operations, unknown)` — + order-of-introduction tuples; validates referential integrity at + construction (every reference to an entity must resolve). + - `graph_from_dict(d)` and `MathProblemGraph.canonical_bytes()` close + the JSON round-trip. Two logically-equal graphs produce byte-equal + canonical serializations (sorted keys, compact separators). + +2. `evals/gsm8k_parser_dev/cases.jsonl` carries the **first five seed + cases** (`gpd-001` … `gpd-005`): + + | id | construction | answer | + |---|---|---| + | gpd-001 | single-entity / single-add | 8 apples | + | gpd-002 | single-entity / single-subtract | 8 candies | + | gpd-003 | single-entity / multi-step (add then subtract) | 12 books | + | gpd-004 | two-entity transfer | 5 marbles | + | gpd-005 | multi-entity sum (no operations) | 11 stickers | + +3. `evals/gsm8k_parser_dev/README.md` is the **authoring contract**: + pattern registry, canonicalization rules, scope boundary for Phase + 1.1, hand-solving rubric, distribution target for the remaining 45 + cases. + +4. `tests/test_math_problem_graph.py` pins five invariants: + + - Each seed case round-trips through `graph_from_dict → as_json` byte-equal. + - `canonical_bytes()` is deterministic across two identical constructions. + - Constructor refuses every malformed graph case listed in the schema. + - Hand-solving each ground-truth graph reproduces the case's + `expected_answer` — catches mis-authored cases. + - Case ids are sequential `gpd-NNN`. + +### Phase 1.1 scope boundary (documented for Phase 1.2 authors) + +The Phase 1.1 schema covers grade-school arithmetic constructions +expressible as a state-mutation event log. The dev-set README enumerates +exactly which patterns are in scope. **Out of scope for Phase 1.1**: + +- Conditional / time-modal phrasing ("If Sam had ..."). +- Rate-and-quantity inference ("Each apple costs $2, Sam buys 4"). +- Compound questions / multiple unknowns per case. +- Generic-plural / implicit entities ("There are 5 boys"). +- Comparative phrasing without explicit numbers ("twice as many as"). + +These are not architectural limits; they are Phase 1.1 cadence limits. +Phase 1.2+ may lift them under their own ADRs. + +### Phase 1.2 — authoring contract (delegated) + +The remaining 45 dev-set cases (`gpd-006` … `gpd-050`) are authored by +following `evals/gsm8k_parser_dev/README.md` against the Phase 1.1 +schema. Distribution target documented there: + +- 30 single-entity cases (`gpd-001` … `gpd-030`) +- 12 two-entity transfer cases (`gpd-031` … `gpd-042`) +- 8 multi-entity sum / no-op cases (`gpd-043` … `gpd-050`) + +Verification: every authored case must (a) pass +`tests/test_math_problem_graph.py::TestSeedCasesRoundTrip`, (b) pass +`TestGroundTruthGraphsAgreeWithExpectedAnswers` (the hand-solver +reproduces `expected_answer`), and (c) tag only patterns from the +registered list. + +### Phase 1.3 — parser exit criterion + +The parser landing under Phase 1.3 produces `MathProblemGraph` instances +from natural-language input deterministically (no LLM, no sampling). +**Exit criterion**: for ≥ 45 of 50 dev-set cases, + +```python +parser(case["problem"]).canonical_bytes() == graph_from_dict(case["ground_truth_graph"]).canonical_bytes() +``` + +i.e. ≥ 0.90 parse-correctness measured by byte-equality of the canonical +graph serialization. A failing case is reported with the diff between +parser output and ground truth. + +### Phase 1.4 — runtime binding + +Once Phase 1.3 lands, the parser is wired through the existing CORE +intent classifier so `RuntimeConfig.math_parser_enabled=True` routes +math-shaped intents through it. Out of scope for this ADR; will be its +own ADR if non-trivial. + +--- + +## Invariants pinned now + +### `adr_0115_schema_round_trip_byte_equal` + +For every case in `evals/gsm8k_parser_dev/cases.jsonl`, +`graph_from_dict → as_json → graph_from_dict` produces byte-equal +`canonical_bytes()`. Tested by `TestSeedCasesRoundTrip`. + +### `adr_0115_schema_validates_construction` + +`MathProblemGraph` rejects graphs with: empty entities, duplicate +entities, references to undefined entities, transfers without a target, +non-transfer operations carrying a target, transfer-to-self. Tested by +`TestSchemaRejectsMalformed`. + +### `adr_0115_ground_truth_graphs_match_expected_answers` + +Hand-solving every seed case's `ground_truth_graph` reproduces its +declared `expected_answer`. This invariant is what makes the dev set +usable as a parser test bed: a wrong ground-truth would silently grade +the parser against itself. Tested by +`TestGroundTruthGraphsAgreeWithExpectedAnswers`. + +--- + +## Acceptance evidence (for Phase 1.1) + +- `generate/math_problem_graph.py` exports the typed dataclasses, + `VALID_OPERATION_KINDS`, `MathGraphError`, and `graph_from_dict` +- `evals/gsm8k_parser_dev/cases.jsonl` contains 5 seed cases with the + documented `gpd-NNN` id pattern +- `evals/gsm8k_parser_dev/README.md` documents the schema, pattern + registry, scope boundary, and authoring contract +- `tests/test_math_problem_graph.py` is 26/26 green and pins the five + invariants above +- README + `docs/decisions/README.md` link this ADR + +--- + +## Consequences + +- Phase 1 of ADR-0114 now has a concrete shape. Subsequent phase ADRs + (0116 solver, 0117 verifier, etc.) consume this graph type. +- The schema is **load-bearing for the dev-set authoring contract**. + Once `gpd-050` lands, changing the schema requires an amendment ADR + plus rewriting cases — so the schema choices here should be sticky. +- The solver (ADR-0116) gets a clean input contract. It must implement + exactly the semantics documented in this ADR's pattern registry + (transfer = subtract+add, multiply/divide on actor's quantity, + unknown-entity=null means sum-across). +- The hand-solver inside the test module is a **reference** + implementation. ADR-0116 supersedes it with a real solver that can + handle multi-step graphs with shared state across operations and + produce a step-trace for the realizer (ADR-0118). + +--- + +## Out of scope + +- The parser itself. Phase 1.3, separate ADR (or this ADR's extension). +- Anything beyond the documented patterns. Phase 1.1 chooses sticky + boundaries deliberately. +- GSM8K corpus integration. Phase 5 (ADR-0119). +- Defining the `expert` ledger tier predicates. Phase 6 (ADR-0120). +- A rate / per-unit pricing pattern. Future Phase 1.X amendment. +- Comparative-without-explicit-numbers phrasing. Future. + +--- + +## Open candidate directions (no ADR yet) + +- **Fractional / decimal answers.** Phase 1.1 keeps `Quantity.value` typed + as `int | float` but every seed case is integer-valued. If a future + pattern needs fractional intermediate state (e.g. "splits evenly into + 3"), the schema already supports it; what changes is the canonical + comparison rule for the parser exit criterion (currently exact + equality). +- **Multi-currency normalization.** Currently all "$" surfaces are + normalized to `unit="dollars"`. Other currencies would need their own + canonical unit string. +- **Time / duration.** Out of scope for Phase 1; will need its own + arithmetic (hours/minutes/days) when introduced. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 91b6f69f..fbf6c834 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -35,6 +35,7 @@ ADRs record significant architectural decisions: what was decided, why, what alt | [ADR-0112](ADR-0112-runnable-expert-demo-showcase.md) | Runnable Audit-Passed Showcase (originally "Expert-Demo") | Accepted (2026-05-22) | | [ADR-0113](ADR-0113-rename-expert-demo-to-audit-passed.md) | Rename `expert-demo` → `audit-passed`; Reserve `expert` for Future Capability Tier | Accepted (2026-05-22) | | [ADR-0114](ADR-0114-expert-capability-roadmap-gsm8k-first.md) | Expert-Capability Roadmap: GSM8K-Math First | Proposed (2026-05-22) | +| [ADR-0115](ADR-0115-math-problem-parser-and-graph.md) | Math Problem Parser and Typed Proposition Graph | Phase 1.1 Accepted (2026-05-22) | --- @@ -66,6 +67,7 @@ The ADR-0091..0114 slate is fully accepted (0091..0113) plus one proposed-roadma - Runnable Audit-Passed Showcase (originally "Expert-Demo"; renamed) — ADR-0112 + ADR-0113 - Rename `expert-demo` → `audit-passed`; reserve `expert` namespace — ADR-0113 - Expert-Capability Roadmap (GSM8K-Math first); proposed — ADR-0114 +- Math Problem Parser & Typed Graph (Phase 1.1 schema + 5 seed cases); Phase 1.2 (45 more cases) delegated — ADR-0115 ADR-0080 has also landed: Contemplation Loop Phase 1 adds a read-only frontier-compare miner that emits `SPECULATIVE` findings only. diff --git a/evals/gsm8k_parser_dev/README.md b/evals/gsm8k_parser_dev/README.md new file mode 100644 index 00000000..db696f3e --- /dev/null +++ b/evals/gsm8k_parser_dev/README.md @@ -0,0 +1,207 @@ +# `gsm8k_parser_dev` — Curated Dev Set for the ADR-0115 Math Problem Parser + +**Status:** ADR-0115 Phase 1.1 (initial seed). 5 of 50 target cases authored. +**Schema source of truth:** `generate/math_problem_graph.py` (typed dataclasses). +**Format:** JSONL — one case per line. + +## Why this dev set is **not** drawn from GSM8K + +The eventual GSM8K eval lane (ADR-0119) treats the actual GSM8K corpus as +sealed test material. To preserve that integrity we author this dev set +independently in the **same style** as GSM8K (grade-school word problems +with integer answers and 1-8 reasoning steps) but with no overlap. + +The dev set measures the **parser**, not the difficulty of the problem. +A correctly-parsed problem is one whose `parser(problem.text) == +problem.ground_truth_graph` byte-equal. + +## Case schema + +Each line is one JSON object: + +```json +{ + "id": "gpd-NNN", + "problem": "", + "expected_answer": , + "expected_unit": "", + "ground_truth_graph": { + "entities": ["", "", ...], + "initial_state": [ + {"entity": "", "quantity": {"unit": "", "value": }}, + ... + ], + "operations": [ + {"actor": "", "kind": "", + "operand": {"unit": "", "value": }, + "target": "" /* required when kind=transfer; omitted otherwise */}, + ... + ], + "unknown": {"entity": "" | null, "unit": ""} + }, + "patterns": ["", "", ...], + "notes": "" +} +``` + +### Field rules + +- **`id`** — `gpd-NNN` zero-padded to 3 digits, sequential across the file. +- **`problem`** — one or more complete English sentences ending in a question. + Use Title-Cased proper names for entities ("Sam", "Anna's Toy Box"). Be + consistent: the same entity always spelled the same way in `problem` and + `ground_truth_graph.entities`. +- **`expected_answer`** — the integer (or float) answer to the question. +- **`expected_unit`** — the unit string the answer is in. Must match + `ground_truth_graph.unknown.unit` byte-for-byte. +- **`ground_truth_graph.entities`** — tuple in **order of first introduction + in the problem text**. Not alphabetical. No duplicates. +- **`ground_truth_graph.initial_state`** — every entity that starts the + problem with a known quantity. Empty list is legal if no initial + possessions are asserted (rare). +- **`ground_truth_graph.operations`** — in **source-text order**. Empty list + is legal (e.g. multi-entity sum questions with no mutations). +- **`ground_truth_graph.unknown.entity`** — set to the entity the question + asks about, or `null` if the question asks for a total across all entities + ("How many ... in total?"; "How many do they have altogether?"). +- **`patterns`** — tag list naming the constructions used. See [Pattern + registry](#pattern-registry) below. +- **`notes`** — author-supplied one-sentence rationale. Read by future + reviewers when the parser fails this case. + +### Canonicalization rules + +- **Units** — lowercase, plural form ("apples", "candies", "dollars", + "hours"). Use "dollars" for "$" quantities; the parser is expected to + rewrite the "$" surface to the canonical unit. +- **Entities** — preserve capitalization as written. Do not lowercase. +- **Numbers** — integers when the text shows integers. Use floats only + if the problem text mentions fractional units explicitly (rare in + grade-school problems). +- **Operation kinds** — exactly one of `add`, `subtract`, `transfer`, + `multiply`, `divide`. Choose the one closest to the verb in the text: + - "buys / gets / receives / earns / finds / adds" → `add` + - "eats / loses / sells / spends / drops / uses / removes" → `subtract` + - "gives / sends / hands / passes / mails / transfers" → `transfer` + (and set `target`) + - "doubles / triples / Nx as many" → `multiply` + - "splits evenly into N / N% of / shares equally with N people" → `divide` + +### What this dev set does NOT cover (Phase 1.1 scope) + +The parser landing under ADR-0115 will handle the following patterns and +no others. Cases violating these constraints belong to a later phase +and should not appear in this file: + +- **Time-modal / conditional phrasing** ("If Sam had 5 apples, ...") — + out of scope for Phase 1.1. Use direct declarative phrasing only. +- **Rate/per-unit pricing requiring inference** ("Each apple costs $2. + Sam buys 4. How much does he spend?") — out of scope. A simpler + variant ("Sam spends $8 on apples. How much does he have left?") IS + in scope. +- **Multi-clause / compound-question problems** ("How many does Sam + have, and how many does Tom have?") — out of scope. One unknown + per case. +- **Implicit-entity / generic plural** ("There are 5 boys. Each has 2 + apples.") — out of scope. Use named entities. +- **Comparative phrasing without explicit numbers** ("Sam has twice as + many as Tom") — out of scope. Use numeric multipliers only + ("Sam has 2 times 3 apples"). + +These exclusions are not permanent — Phase 1.2+ will lift them under +their own ADRs. + +## Pattern registry + +When tagging a case under `patterns`, draw from this list. Add new tags +only when authoring a case that uses a construction not yet covered; +update the parser's pattern table at the same time. + +| Pattern tag | Construction | Example | +|---|---|---| +| `initial_has` | " has ." | "Sam has 5 apples." | +| `initial_there_are` | "There are ." (no entity; rare) | "There are 12 candies on the table." | +| `operation_buy_more` | " buys more." | "He buys 3 more." | +| `operation_get_more` | " gets more ." | "She gets 4 more pencils." | +| `operation_find_adds` | " finds ." | "Sam finds 2 apples on the path." | +| `operation_eat_loses` | " eats ." | "Tom eats 4 candies." | +| `operation_lose_loses` | " loses ." | "Anna loses 3 marbles." | +| `operation_sell_loses` | " sells ." | "Lisa sells 2 books." | +| `operation_donate_loses` | " donates ." | "Lisa donates 3 books." | +| `operation_use_loses` | " uses ." | "He uses 2 sheets of paper." | +| `operation_give_transfer` | " gives to ." | "Anna gives 3 marbles to Ben." | +| `operation_send_transfer` | " sends to ." | "Tom sends 4 letters to Sara." | +| `operation_double` | " doubles ..." | "Sam doubles his savings." | +| `operation_triple` | " triples ..." | "Sam triples his stickers." | +| `operation_split_divide` | "splits/shares evenly" | "They split 12 candies evenly." | +| `question_how_many_entity` | "How many does have?" | "How many apples does Sam have?" | +| `question_how_many_left` | "How many ... left?" | "How many candies does Tom have left?" | +| `question_how_many_total` | "How many ... in total?" / "altogether" | "How many stickers do they have in total?" | +| `question_how_many_now` | "How many ... now?" | "How many marbles does Anna have now?" | + +## How to author a new case (Codex contract) + +For each case: + +1. **Draft the natural-language problem** in the style of the seed cases. + Use the patterns listed above. Stay within Phase 1.1 scope. +2. **Solve it by hand** to determine `expected_answer` and `expected_unit`. +3. **Walk the problem sentence by sentence**, emitting: + - First introduction of an entity → add to `entities`. + - "X has N " → `initial_state` entry. + - Any state-mutating verb → `operations` entry. Choose the right `kind` + from the registry. For `transfer`, set `target`. + - The question sentence → `unknown` field. +4. **Set `patterns`** to the tags used. +5. **Set `notes`** to one sentence explaining the construction or any + gotcha (anaphora resolution, sequence marker, etc.). +6. **Verify**: load the case via `graph_from_dict`. The constructor will + raise `MathGraphError` on schema violations. Use: + +```python +import json +from generate.math_problem_graph import graph_from_dict +case = json.loads(line) +graph = graph_from_dict(case["ground_truth_graph"]) +# canonicalize: parser output is compared against graph.canonical_bytes() +``` + +7. **Re-solve the graph by hand** using the operation semantics: + - `add`/`subtract` on the actor's quantity of that unit + - `transfer` = subtract from actor + add to target (same unit) + - `multiply`/`divide` on the actor's quantity (scalar operand) + - For `Unknown.entity=null`: sum across every entity holding `unit` + - For `Unknown.entity="X"`: look up X's final quantity of `unit` + + The result must equal `expected_answer`. If it doesn't, the graph is wrong. + +## Determinism check + +```bash +python3 -c " +import json +from generate.math_problem_graph import graph_from_dict +with open('evals/gsm8k_parser_dev/cases.jsonl') as f: + for line in f: + c = json.loads(line) + g = graph_from_dict(c['ground_truth_graph']) + print(c['id'], 'OK', g.canonical_bytes().hex()[:16]) +" +``` + +Every case should print `OK` plus a deterministic 16-hex-char prefix. + +## Authoring target + +50 cases by case-id `gpd-050`. Distribution target: + +- 30 single-entity cases (`gpd-001` … `gpd-030`) +- 12 two-entity transfer cases (`gpd-031` … `gpd-042`) +- 8 multi-entity sum/no-op cases (`gpd-043` … `gpd-050`) + +Within each tranche, vary which `operation_*` pattern is used so the +parser is exercised across the registry. + +The parser landing under ADR-0115 will be measured against this file. +Exit criterion: **parse correctness ≥ 0.90** (45 of 50 cases' +ground-truth graphs reproduce byte-equal from the parser's output). diff --git a/evals/gsm8k_parser_dev/cases.jsonl b/evals/gsm8k_parser_dev/cases.jsonl new file mode 100644 index 00000000..8f2f3b48 --- /dev/null +++ b/evals/gsm8k_parser_dev/cases.jsonl @@ -0,0 +1,5 @@ +{"id":"gpd-001","problem":"Sam has 5 apples. He buys 3 more. How many apples does Sam have?","expected_answer":8,"expected_unit":"apples","ground_truth_graph":{"entities":["Sam"],"initial_state":[{"entity":"Sam","quantity":{"unit":"apples","value":5}}],"operations":[{"actor":"Sam","kind":"add","operand":{"unit":"apples","value":3}}],"unknown":{"entity":"Sam","unit":"apples"}},"patterns":["initial_has","operation_buy_more","question_how_many_entity"],"notes":"Single-entity, single-add. The simplest GSM8K-style pattern. 'He' resolves anaphorically to 'Sam'."} +{"id":"gpd-002","problem":"Tom has 12 candies. He eats 4. How many candies does Tom have left?","expected_answer":8,"expected_unit":"candies","ground_truth_graph":{"entities":["Tom"],"initial_state":[{"entity":"Tom","quantity":{"unit":"candies","value":12}}],"operations":[{"actor":"Tom","kind":"subtract","operand":{"unit":"candies","value":4}}],"unknown":{"entity":"Tom","unit":"candies"}},"patterns":["initial_has","operation_eat_loses","question_how_many_left"],"notes":"Single-entity, single-subtract. 'eats' is a loss verb. Question suffix 'left' is a comprehension cue but the unknown shape is identical to gpd-001."} +{"id":"gpd-003","problem":"Lisa has 10 books. She buys 5 more, then donates 3. How many books does Lisa have?","expected_answer":12,"expected_unit":"books","ground_truth_graph":{"entities":["Lisa"],"initial_state":[{"entity":"Lisa","quantity":{"unit":"books","value":10}}],"operations":[{"actor":"Lisa","kind":"add","operand":{"unit":"books","value":5}},{"actor":"Lisa","kind":"subtract","operand":{"unit":"books","value":3}}],"unknown":{"entity":"Lisa","unit":"books"}},"patterns":["initial_has","operation_buy_more","operation_donate_loses","question_how_many_entity"],"notes":"Single-entity, multi-step. 'then' is a sequence marker; operation order must follow text order."} +{"id":"gpd-004","problem":"Anna has 8 marbles. She gives 3 to Ben. How many marbles does Anna have now?","expected_answer":5,"expected_unit":"marbles","ground_truth_graph":{"entities":["Anna","Ben"],"initial_state":[{"entity":"Anna","quantity":{"unit":"marbles","value":8}}],"operations":[{"actor":"Anna","kind":"transfer","operand":{"unit":"marbles","value":3},"target":"Ben"}],"unknown":{"entity":"Anna","unit":"marbles"}},"patterns":["initial_has","operation_give_transfer","question_how_many_entity"],"notes":"Two-entity transfer. Ben appears only as a transfer target — no initial possession is asserted for Ben. The 'transfer' kind decomposes downstream into subtract(actor) + add(target); the parser emits the closer-to-NL form."} +{"id":"gpd-005","problem":"Tom has 4 stickers. Sara has 7 stickers. How many stickers do they have in total?","expected_answer":11,"expected_unit":"stickers","ground_truth_graph":{"entities":["Tom","Sara"],"initial_state":[{"entity":"Tom","quantity":{"unit":"stickers","value":4}},{"entity":"Sara","quantity":{"unit":"stickers","value":7}}],"operations":[],"unknown":{"entity":null,"unit":"stickers"}},"patterns":["initial_has","initial_has","question_how_many_total"],"notes":"Multi-entity initial possessions, no operations, sum question. 'they' refers to all introduced entities; Unknown.entity=null signals total across entities."} diff --git a/generate/math_problem_graph.py b/generate/math_problem_graph.py new file mode 100644 index 00000000..42435607 --- /dev/null +++ b/generate/math_problem_graph.py @@ -0,0 +1,262 @@ +"""ADR-0115 — Typed proposition graph for grade-school math word problems. + +This module defines the structural target of the parser added under ADR-0115. +Parsing a natural-language problem produces a :class:`MathProblemGraph`; the +solver (ADR-0116) and verifier (ADR-0117) consume the same structure. + +Determinism guarantees: + +- Every dataclass is ``frozen=True, slots=True`` and hashes by value. +- :meth:`MathProblemGraph.canonical_bytes` is sorted-keys, compact-separators + JSON — same graph object → byte-identical SHA-256. +- Field order on ``entities``, ``initial_state``, ``operations`` is + **order-of-introduction** in the source text. Two graphs that disagree on + introduction order are NOT equal; this matches CORE's general "preserve + source-text ordering" doctrine. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any, Final, Mapping + + +# Operation kinds correspond to math-pack lemma vocabulary (en_mathematics_logic_v1). +# A future solver under ADR-0116 dispatches on this string. +VALID_OPERATION_KINDS: Final[frozenset[str]] = frozenset( + {"add", "subtract", "transfer", "multiply", "divide"} +) + + +class MathGraphError(ValueError): + """Raised on schema violations in math-problem-graph construction.""" + + +@dataclass(frozen=True, slots=True) +class Quantity: + """A numeric value paired with a textual unit. + + The unit is the canonical noun (lowercase). Equality is exact: + ``Quantity(5, 'apples')`` != ``Quantity(5, 'apple')``. Authors and + parsers must canonicalize units before constructing. + """ + + value: int | float + unit: str + + def __post_init__(self) -> None: + if not isinstance(self.value, (int, float)) or isinstance(self.value, bool): + raise MathGraphError( + f"Quantity.value must be int or float, got " + f"{type(self.value).__name__}" + ) + if not isinstance(self.unit, str) or not self.unit: + raise MathGraphError( + f"Quantity.unit must be a non-empty string, got {self.unit!r}" + ) + + def as_json(self) -> dict[str, Any]: + return {"unit": self.unit, "value": self.value} + + +@dataclass(frozen=True, slots=True) +class InitialPossession: + """Some entity holds some quantity at the start of the problem.""" + + entity: str + quantity: Quantity + + def __post_init__(self) -> None: + if not isinstance(self.entity, str) or not self.entity: + raise MathGraphError( + "InitialPossession.entity must be a non-empty string" + ) + + def as_json(self) -> dict[str, Any]: + return {"entity": self.entity, "quantity": self.quantity.as_json()} + + +@dataclass(frozen=True, slots=True) +class Operation: + """A state-mutating event applied in story order. + + ``transfer`` denotes ``actor → target`` movement of ``operand``. The + solver (ADR-0116) decomposes ``transfer`` into ``subtract`` from actor + plus ``add`` to target; the parser emits ``transfer`` to stay close to + natural-language surface ("gives X to Y"). + + For ``multiply`` / ``divide`` the ``operand`` is the scalar (e.g. a + factor of 3). Unit handling for these kinds is delegated to the solver. + """ + + actor: str + kind: str + operand: Quantity + target: str | None = None + + def __post_init__(self) -> None: + if not isinstance(self.actor, str) or not self.actor: + raise MathGraphError("Operation.actor must be a non-empty string") + if self.kind not in VALID_OPERATION_KINDS: + raise MathGraphError( + f"Operation.kind must be one of {sorted(VALID_OPERATION_KINDS)}, " + f"got {self.kind!r}" + ) + if self.kind == "transfer": + if not self.target: + raise MathGraphError( + "Operation.target required when kind='transfer'" + ) + if self.target == self.actor: + raise MathGraphError( + "Operation.target must differ from Operation.actor for " + "kind='transfer'" + ) + else: + if self.target is not None: + raise MathGraphError( + f"Operation.target only valid for kind='transfer'; got " + f"kind={self.kind!r}" + ) + + def as_json(self) -> dict[str, Any]: + d: dict[str, Any] = { + "actor": self.actor, + "kind": self.kind, + "operand": self.operand.as_json(), + } + if self.target is not None: + d["target"] = self.target + return d + + +@dataclass(frozen=True, slots=True) +class Unknown: + """The quantity the question is asking for. + + ``entity=None`` means "total across every entity holding ``unit``" + (e.g. "How many apples do they have in total?"). For a single-entity + question ("How many apples does Sam have?") set ``entity='Sam'``. + """ + + entity: str | None + unit: str + + def __post_init__(self) -> None: + if not isinstance(self.unit, str) or not self.unit: + raise MathGraphError("Unknown.unit must be a non-empty string") + if self.entity is not None and ( + not isinstance(self.entity, str) or not self.entity + ): + raise MathGraphError( + "Unknown.entity must be a non-empty string or None" + ) + + def as_json(self) -> dict[str, Any]: + return {"entity": self.entity, "unit": self.unit} + + +@dataclass(frozen=True, slots=True) +class MathProblemGraph: + """Typed graph produced by the ADR-0115 parser. + + Field order on tuples is **order of introduction in the source text**, + not alphabetical. ``MathProblemGraph`` equality is element-wise tuple + equality; reordering changes the graph identity. + """ + + entities: tuple[str, ...] + initial_state: tuple[InitialPossession, ...] + operations: tuple[Operation, ...] + unknown: Unknown + + def __post_init__(self) -> None: + if not self.entities: + raise MathGraphError( + "MathProblemGraph.entities must contain at least one entity" + ) + seen: set[str] = set() + for e in self.entities: + if not isinstance(e, str) or not e: + raise MathGraphError( + "MathProblemGraph.entities must be non-empty strings" + ) + if e in seen: + raise MathGraphError( + f"MathProblemGraph.entities contains duplicate {e!r}" + ) + seen.add(e) + entity_set = set(self.entities) + for p in self.initial_state: + if p.entity not in entity_set: + raise MathGraphError( + f"initial_state references unknown entity {p.entity!r}" + ) + for op in self.operations: + if op.actor not in entity_set: + raise MathGraphError( + f"operation references unknown actor {op.actor!r}" + ) + if op.target is not None and op.target not in entity_set: + raise MathGraphError( + f"operation references unknown target {op.target!r}" + ) + if self.unknown.entity is not None and self.unknown.entity not in entity_set: + raise MathGraphError( + f"unknown references unknown entity {self.unknown.entity!r}" + ) + + def as_json(self) -> dict[str, Any]: + return { + "entities": list(self.entities), + "initial_state": [p.as_json() for p in self.initial_state], + "operations": [o.as_json() for o in self.operations], + "unknown": self.unknown.as_json(), + } + + def canonical_bytes(self) -> bytes: + """Deterministic JSON for hashing/byte-equality comparison.""" + return json.dumps( + self.as_json(), sort_keys=True, separators=(",", ":") + ).encode("utf-8") + + +def graph_from_dict(d: Mapping[str, Any]) -> MathProblemGraph: + """Deserialize a graph from its canonical JSON dict. + + The reverse of :meth:`MathProblemGraph.as_json`. Raises + :class:`MathGraphError` on any schema violation surfaced by the + dataclass constructors. + """ + if not isinstance(d, Mapping): + raise MathGraphError(f"graph payload must be a mapping; got {type(d).__name__}") + for required in ("entities", "initial_state", "operations", "unknown"): + if required not in d: + raise MathGraphError(f"graph payload missing required field {required!r}") + + entities = tuple(d["entities"]) + initial_state = tuple( + InitialPossession( + entity=p["entity"], + quantity=Quantity(value=p["quantity"]["value"], unit=p["quantity"]["unit"]), + ) + for p in d["initial_state"] + ) + operations = tuple( + Operation( + actor=o["actor"], + kind=o["kind"], + operand=Quantity(value=o["operand"]["value"], unit=o["operand"]["unit"]), + target=o.get("target"), + ) + for o in d["operations"] + ) + unk = d["unknown"] + unknown = Unknown(entity=unk.get("entity"), unit=unk["unit"]) + return MathProblemGraph( + entities=entities, + initial_state=initial_state, + operations=operations, + unknown=unknown, + ) diff --git a/tests/test_math_problem_graph.py b/tests/test_math_problem_graph.py new file mode 100644 index 00000000..d31ed8a4 --- /dev/null +++ b/tests/test_math_problem_graph.py @@ -0,0 +1,194 @@ +"""ADR-0115 Phase 1.1 — math problem graph schema invariants. + +Pins: + +1. The five seed cases in ``evals/gsm8k_parser_dev/cases.jsonl`` round-trip + through ``graph_from_dict`` → ``as_json`` without changing bytes. + +2. ``MathProblemGraph.canonical_bytes()`` is deterministic: same logical + graph constructed twice produces identical bytes. + +3. Construction-time validation refuses malformed graphs. + +4. Pyhand-solving each seed case from its ground-truth graph reproduces the + ``expected_answer`` — this catches mis-authored ground-truth graphs. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from generate.math_problem_graph import ( + InitialPossession, + MathGraphError, + MathProblemGraph, + Operation, + Quantity, + Unknown, + graph_from_dict, +) + + +_REPO_ROOT = Path(__file__).resolve().parent.parent +_CASES = _REPO_ROOT / "evals" / "gsm8k_parser_dev" / "cases.jsonl" + + +def _load_cases() -> list[dict]: + return [json.loads(line) for line in _CASES.read_text().splitlines() if line.strip()] + + +class TestSeedCasesRoundTrip: + @pytest.mark.parametrize("case", _load_cases(), ids=lambda c: c["id"]) + def test_graph_loads(self, case: dict) -> None: + graph = graph_from_dict(case["ground_truth_graph"]) + assert isinstance(graph, MathProblemGraph) + + @pytest.mark.parametrize("case", _load_cases(), ids=lambda c: c["id"]) + def test_round_trip_byte_equal(self, case: dict) -> None: + graph = graph_from_dict(case["ground_truth_graph"]) + reloaded = graph_from_dict(graph.as_json()) + assert graph.canonical_bytes() == reloaded.canonical_bytes() + + +class TestCanonicalBytesDeterminism: + def test_two_identical_graphs_produce_identical_bytes(self) -> None: + g1 = MathProblemGraph( + entities=("Sam",), + initial_state=( + InitialPossession("Sam", Quantity(5, "apples")), + ), + operations=(Operation("Sam", "add", Quantity(3, "apples")),), + unknown=Unknown("Sam", "apples"), + ) + g2 = MathProblemGraph( + entities=("Sam",), + initial_state=( + InitialPossession("Sam", Quantity(5, "apples")), + ), + operations=(Operation("Sam", "add", Quantity(3, "apples")),), + unknown=Unknown("Sam", "apples"), + ) + assert g1.canonical_bytes() == g2.canonical_bytes() + assert g1 == g2 + + +class TestSchemaRejectsMalformed: + def test_quantity_rejects_string_value(self) -> None: + with pytest.raises(MathGraphError): + Quantity("5", "apples") # type: ignore[arg-type] + + def test_quantity_rejects_empty_unit(self) -> None: + with pytest.raises(MathGraphError): + Quantity(5, "") + + def test_operation_rejects_unknown_kind(self) -> None: + with pytest.raises(MathGraphError): + Operation("Sam", "explode", Quantity(3, "apples")) + + def test_transfer_requires_target(self) -> None: + with pytest.raises(MathGraphError): + Operation("Sam", "transfer", Quantity(3, "apples")) + + def test_non_transfer_rejects_target(self) -> None: + with pytest.raises(MathGraphError): + Operation("Sam", "add", Quantity(3, "apples"), target="Tom") + + def test_transfer_self_rejected(self) -> None: + with pytest.raises(MathGraphError): + Operation("Sam", "transfer", Quantity(3, "apples"), target="Sam") + + def test_graph_rejects_duplicate_entities(self) -> None: + with pytest.raises(MathGraphError): + MathProblemGraph( + entities=("Sam", "Sam"), + initial_state=(), + operations=(), + unknown=Unknown("Sam", "apples"), + ) + + def test_graph_rejects_unknown_entity_in_initial(self) -> None: + with pytest.raises(MathGraphError): + MathProblemGraph( + entities=("Sam",), + initial_state=(InitialPossession("Tom", Quantity(5, "apples")),), + operations=(), + unknown=Unknown("Sam", "apples"), + ) + + def test_graph_rejects_unknown_entity_in_question(self) -> None: + with pytest.raises(MathGraphError): + MathProblemGraph( + entities=("Sam",), + initial_state=(), + operations=(), + unknown=Unknown("Tom", "apples"), + ) + + +def _hand_solve(graph: MathProblemGraph) -> tuple[float, str]: + """Reference solver — ADR-0116 supersedes this with a real solver. + + Used here only to falsify mis-authored ground-truth graphs in the seed + set. Sufficient for the patterns Phase 1.1 covers. + """ + state: dict[tuple[str, str], float] = {} + for p in graph.initial_state: + state[(p.entity, p.quantity.unit)] = float(p.quantity.value) + for op in graph.operations: + key = (op.actor, op.operand.unit) + cur = state.get(key, 0.0) + v = float(op.operand.value) + if op.kind == "add": + state[key] = cur + v + elif op.kind == "subtract": + state[key] = cur - v + elif op.kind == "transfer": + assert op.target is not None + state[key] = cur - v + tgt_key = (op.target, op.operand.unit) + state[tgt_key] = state.get(tgt_key, 0.0) + v + elif op.kind == "multiply": + state[key] = cur * v + elif op.kind == "divide": + state[key] = cur / v + if graph.unknown.entity is None: + total = sum( + v for (_, unit), v in state.items() if unit == graph.unknown.unit + ) + return total, graph.unknown.unit + return state[(graph.unknown.entity, graph.unknown.unit)], graph.unknown.unit + + +class TestGroundTruthGraphsAgreeWithExpectedAnswers: + """Falsifies mis-authored seed cases. + + For each seed case, hand-solving the ground-truth graph using the + documented operation semantics must reproduce ``expected_answer`` and + ``expected_unit``. + """ + + @pytest.mark.parametrize("case", _load_cases(), ids=lambda c: c["id"]) + def test_hand_solve_matches_expected(self, case: dict) -> None: + graph = graph_from_dict(case["ground_truth_graph"]) + computed, unit = _hand_solve(graph) + assert unit == case["expected_unit"], ( + f"{case['id']}: unit mismatch — graph says {unit!r}, " + f"expected {case['expected_unit']!r}" + ) + # Accept int/float equivalence; problems are integer-valued. + assert computed == case["expected_answer"], ( + f"{case['id']}: hand-solve produced {computed} but case " + f"declared expected_answer={case['expected_answer']}" + ) + + +class TestCaseIdsAreSequential: + def test_ids_are_gpd_zero_padded_sequential(self) -> None: + cases = _load_cases() + for i, c in enumerate(cases, start=1): + assert c["id"] == f"gpd-{i:03d}", ( + f"case {i}: expected id 'gpd-{i:03d}', got {c['id']!r}" + )