diff --git a/docs/decisions/ADR-0136.S.1-rate-event-statements.md b/docs/decisions/ADR-0136.S.1-rate-event-statements.md new file mode 100644 index 00000000..1ba6fd0b --- /dev/null +++ b/docs/decisions/ADR-0136.S.1-rate-event-statements.md @@ -0,0 +1,77 @@ +# ADR-0136.S.1 — Rate/Event Statement Parsing + +**Status:** Accepted +**Parent:** ADR-0136 (Statement Layer Corridor) +**Date:** 2026-05-23 + +## Context + +The GSM8K refusal taxonomy (`evals/gsm8k_math/train_sample/v1/refusal_taxonomy.json`) +reveals that 23/50 cases are blocked by context-filler sentences (correctly +refused — no parseable numeric state), while 4/50 have rate/capacity/price as +their primary barrier. The remaining cases are compound-statement, +distributive-multiply, and diverse long-tail shapes. + +This ADR targets the 4 rate-class cases with two closed statement shapes. + +## Taxonomy Finding + +| Primary barrier | Cases | S.1 scope? | +|------------------------|-------|------------| +| `context_filler` | 23 | No — correctly refused | +| rate/capacity/price | 4 | **Yes** | +| `compound_statement` | 5 | No | +| `distributive_multiply`| 1 (+5 secondary) | No | +| diverse long-tail | 17 | No | + +## Closed Verb Sets + +**Capacity verbs:** shuck, pick, pack, make, produce, type, read, write, +paint, run, score, answer, complete (+ third-person -s forms). + +**Earnings verbs:** make, earn, receive, get, charge (+ third-person -s forms). + +No regex wildcards for verbs — every admitted verb is explicitly listed in a +frozen set. Sentences with verbs outside the closed set are refused (not +wrong). + +## Short-Circuit Rationale + +Both rate shapes bypass the Cartesian-product candidate graph because the +rate computation is a direct `rate × time` multiplication with unit conversion, +not a graph of initial-possessions and operations. The short-circuit runs +before `_filtered_statement_choices` so that rate-shaped sentences don't +trigger the "no admissible candidate" refusal. + +Actor matching is required: capacity questions with pronouns (`he`/`she`) +accept any actor; named-actor questions require case-insensitive match. +Mismatched actors produce refusal, not wrong answers. + +## Honest GSM8K Claim + +- **Pre-S.1:** 0/50 admitted (all refused). +- **Post-S.1:** 1/50 admitted — `gsm8k-0014` (Bob shucks oysters) with + answer 240.0 (correct). +- **admitted_wrong = 0** (safety rail preserved). + +The other 3 rate-class cases remain blocked by context-filler sentences in +their opening statements; the rate parsing behind them is irrelevant until +those sentences parse. + +## Deferred + +- Context-filler gated problems (23 cases — needs semantic classification + of narrative scene-setter sentences). +- Conditional branching (overtime rules, e.g. "if she works more than 8 hours"). +- Percentage/interest rates (10% simple interest). +- Multi-statement earnings (duration asserted in a separate sentence from the + rate — needs general duration-statement parser). + +## Evidence + +- Axis lane: `evals/math_capability_axes/S1_rate_events/v1/` — 20/20 pass, + wrong=0. +- B3 bounded-grammar lane: unchanged (wrong=0). +- GSM8K candidate-graph probe: wrong=0, admitted=1/50. +- Tests: `tests/test_adr_0136_S1_rate_events.py` — ≥15 tests including B3 + regression guard and GSM8K admitted_wrong=0 rail. diff --git a/evals/gsm8k_math/train_sample/v1/refusal_taxonomy.json b/evals/gsm8k_math/train_sample/v1/refusal_taxonomy.json new file mode 100644 index 00000000..99d8ce14 --- /dev/null +++ b/evals/gsm8k_math/train_sample/v1/refusal_taxonomy.json @@ -0,0 +1,469 @@ +{ + "schema_version": 1, + "adr": "0136.S.0", + "description": "Deterministic refusal taxonomy over 50 GSM8K train-sample cases. Each case classified by primary blocking barrier and co-occurring secondary barriers.", + "summary": { + "total_cases": 50, + "primary_barrier_counts": { + "context_filler": 23, + "compound_statement": 5, + "novel_initial_form": 2, + "rate_earnings": 1, + "partition_divide": 1, + "indefinite_quantity": 1, + "temporal_age_anchor": 1, + "compound_comparative": 1, + "rate_price": 1, + "capacity_rate": 1, + "compound_multi_event": 1, + "multi_entity_initial": 1, + "multi_step_complex": 1, + "multi_day_accumulation": 1, + "distributive_each_actor": 1, + "multi_attribute_accumulation": 1, + "coreference_pronoun": 1, + "goal_statement": 1, + "fraction_operand": 1, + "distributive_multiply": 1, + "percentage_rate": 1, + "novel_initial_verb": 1, + "temporal_frequency": 1 + }, + "secondary_barrier_counts": { + "distributive_multiply": 5, + "percentage_of": 5, + "rate_price": 4, + "fraction_operand": 4, + "rate_comparative": 4, + "multi_step_complex": 3, + "conditional_question": 3, + "temporal_frequency": 3, + "compound_comparative": 3, + "coreference_pronoun": 2, + "context_filler": 2, + "conditional_branch": 2, + "capacity_rate": 2, + "rate_earnings": 2, + "conditional_branching": 1, + "implicit_quantity": 1, + "rate_count": 1, + "rate_question": 1, + "distributive_divide": 1, + "implicit_group_count": 1, + "rate_time": 1, + "multi_day_accumulation": 1, + "multi_entity_initial": 1, + "leg_count": 1, + "multi_item_purchase": 1, + "embedded_per_unit": 1, + "goal_question": 1, + "compound_multi_event": 1 + }, + "s1_admission_potential": { + "primary_rate_class": 4, + "secondary_rate_class_blocked_by_context": 8, + "notes": "Context filler is the dominant gate (23/50). S.1 rate parsing directly unblocks 4 primary cases; 8 more have rate as secondary but have additional barriers." + } + }, + "per_case": [ + { + "case_id": "gsm8k-train-sample-v1-0001", + "primary_barrier": "rate_earnings", + "secondary_barriers": [ + "conditional_branching" + ], + "note": "makes $18/hr + overtime conditional" + }, + { + "case_id": "gsm8k-train-sample-v1-0002", + "primary_barrier": "partition_divide", + "secondary_barriers": [ + "coreference_pronoun" + ], + "note": "splits into 25-foot sections; She=Jan" + }, + { + "case_id": "gsm8k-train-sample-v1-0003", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "distributive_multiply", + "rate_price" + ], + "note": "context sentence gates; 24/box, $0.75 each" + }, + { + "case_id": "gsm8k-train-sample-v1-0004", + "primary_barrier": "indefinite_quantity", + "secondary_barriers": [ + "fraction_operand" + ], + "note": "'some kids' \u2014 no initial count" + }, + { + "case_id": "gsm8k-train-sample-v1-0005", + "primary_barrier": "compound_statement", + "secondary_barriers": [ + "fraction_operand" + ], + "note": "temporal + fraction + future in one sentence" + }, + { + "case_id": "gsm8k-train-sample-v1-0006", + "primary_barrier": "temporal_age_anchor", + "secondary_barriers": [ + "multi_step_complex" + ], + "note": "started at age 6; chained multipliers over age" + }, + { + "case_id": "gsm8k-train-sample-v1-0007", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "implicit_quantity", + "conditional_question" + ], + "note": "intent sentence; box-size must be inferred" + }, + { + "case_id": "gsm8k-train-sample-v1-0008", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "distributive_multiply" + ], + "note": "context gate; 5 bags\u00d750 + 2 bags\u00d7100" + }, + { + "case_id": "gsm8k-train-sample-v1-0009", + "primary_barrier": "compound_comparative", + "secondary_barriers": [ + "conditional_question" + ], + "note": "4\u00d7chickens + 10 ducks = nested comparative; if question" + }, + { + "case_id": "gsm8k-train-sample-v1-0010", + "primary_barrier": "compound_statement", + "secondary_barriers": [], + "note": "'had X initially, but then lost Y' \u2014 two ops one sentence" + }, + { + "case_id": "gsm8k-train-sample-v1-0011", + "primary_barrier": "rate_price", + "secondary_barriers": [ + "context_filler" + ], + "note": "$2 per cup in relative clause; needs inverse solve" + }, + { + "case_id": "gsm8k-train-sample-v1-0012", + "primary_barrier": "compound_statement", + "secondary_barriers": [ + "fraction_operand", + "coreference_pronoun" + ], + "note": "'ate half' + He=Dennis" + }, + { + "case_id": "gsm8k-train-sample-v1-0013", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "temporal_frequency", + "rate_count" + ], + "note": "context gate; 10 videos/day \u00d7 days" + }, + { + "case_id": "gsm8k-train-sample-v1-0014", + "primary_barrier": "capacity_rate", + "secondary_barriers": [], + "note": "SIMPLEST: 'can shuck 10 in 5 min' \u2192 rate \u00d7 2hr" + }, + { + "case_id": "gsm8k-train-sample-v1-0015", + "primary_barrier": "compound_multi_event", + "secondary_barriers": [ + "compound_comparative" + ], + "note": "subway+train+bike in one sentence; twice as much time" + }, + { + "case_id": "gsm8k-train-sample-v1-0016", + "primary_barrier": "compound_statement", + "secondary_barriers": [ + "rate_question" + ], + "note": "traveled X more than 5 miles AND encountered Y less than 17 signs" + }, + { + "case_id": "gsm8k-train-sample-v1-0017", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "rate_price", + "conditional_branch" + ], + "note": "context gate; $50/day or $500/14days conditional" + }, + { + "case_id": "gsm8k-train-sample-v1-0018", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "capacity_rate" + ], + "note": "context gate; 2 goals/15min \u00d7 2hr" + }, + { + "case_id": "gsm8k-train-sample-v1-0019", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "percentage_of", + "conditional_branch" + ], + "note": "context gate; 80% insurance on subsequent visits" + }, + { + "case_id": "gsm8k-train-sample-v1-0020", + "primary_barrier": "multi_entity_initial", + "secondary_barriers": [ + "rate_comparative" + ], + "note": "2 puppies, 2 kittens, 3 parakeets in one sentence; chained price ratios" + }, + { + "case_id": "gsm8k-train-sample-v1-0021", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "distributive_multiply" + ], + "note": "context gate; 15lbs \u00d7 10reps \u00d7 3sets" + }, + { + "case_id": "gsm8k-train-sample-v1-0022", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "rate_earnings", + "compound_comparative" + ], + "note": "context gate; $20/kg + twice as many catch" + }, + { + "case_id": "gsm8k-train-sample-v1-0023", + "primary_barrier": "multi_step_complex", + "secondary_barriers": [ + "fraction_operand", + "distributive_divide" + ], + "note": "Nicole\u2192Cindy\u2192Rex\u2192siblings chained" + }, + { + "case_id": "gsm8k-train-sample-v1-0024", + "primary_barrier": "multi_day_accumulation", + "secondary_barriers": [], + "note": "20+36+40+50 across Mon\u2013Thu in one sentence" + }, + { + "case_id": "gsm8k-train-sample-v1-0025", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "distributive_multiply" + ], + "note": "context gate; 6 baskets\u00d750 + 3 friends same amount" + }, + { + "case_id": "gsm8k-train-sample-v1-0026", + "primary_barrier": "distributive_each_actor", + "secondary_barriers": [], + "note": "'Aaron and Carson each saved $40' \u2014 two actors each" + }, + { + "case_id": "gsm8k-train-sample-v1-0027", + "primary_barrier": "multi_attribute_accumulation", + "secondary_barriers": [], + "note": "240 Instagram + 500 Facebook in one sentence for one actor" + }, + { + "case_id": "gsm8k-train-sample-v1-0028", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "rate_price", + "temporal_frequency" + ], + "note": "context gate; $100k + $1k/day cost + 150\u00d7$10/day revenue" + }, + { + "case_id": "gsm8k-train-sample-v1-0029", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "rate_comparative" + ], + "note": "context gate; keyboard = 3\u00d7 mouse cost" + }, + { + "case_id": "gsm8k-train-sample-v1-0030", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "rate_comparative" + ], + "note": "context gate; beach time = 2.5\u00d7 drive time" + }, + { + "case_id": "gsm8k-train-sample-v1-0031", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "implicit_group_count" + ], + "note": "context + '3 friends' embedded in intent" + }, + { + "case_id": "gsm8k-train-sample-v1-0032", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "rate_time", + "percentage_of" + ], + "note": "context gate; 2hr draw + 30% less to color" + }, + { + "case_id": "gsm8k-train-sample-v1-0033", + "primary_barrier": "compound_statement", + "secondary_barriers": [ + "multi_step_complex" + ], + "note": "'Rachel is 12, grandfather is 7\u00d7her' in one sentence + future age" + }, + { + "case_id": "gsm8k-train-sample-v1-0034", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "capacity_rate", + "percentage_of" + ], + "note": "context gate; 40yds/5sec + 40% speed improvement" + }, + { + "case_id": "gsm8k-train-sample-v1-0035", + "primary_barrier": "coreference_pronoun", + "secondary_barriers": [ + "context_filler" + ], + "note": "'She decided to split them' \u2014 pronoun, no local antecedent" + }, + { + "case_id": "gsm8k-train-sample-v1-0036", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "multi_day_accumulation", + "compound_comparative" + ], + "note": "context gate; multi-day study chain" + }, + { + "case_id": "gsm8k-train-sample-v1-0037", + "primary_barrier": "goal_statement", + "secondary_barriers": [], + "note": "'wants to lose 10 pounds by June' \u2014 goal not initial state" + }, + { + "case_id": "gsm8k-train-sample-v1-0038", + "primary_barrier": "novel_initial_form", + "secondary_barriers": [], + "note": "'there are a hundred ladies' \u2014 existential + location modifier" + }, + { + "case_id": "gsm8k-train-sample-v1-0039", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "multi_step_complex" + ], + "note": "context gate; Orlando+Jose+Fernando chained" + }, + { + "case_id": "gsm8k-train-sample-v1-0040", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "multi_entity_initial", + "leg_count" + ], + "note": "context gate; 5 species \u00d7 per-species leg count" + }, + { + "case_id": "gsm8k-train-sample-v1-0041", + "primary_barrier": "fraction_operand", + "secondary_barriers": [ + "percentage_of" + ], + "note": "all of 1 pan + 75% of 2nd pan" + }, + { + "case_id": "gsm8k-train-sample-v1-0042", + "primary_barrier": "distributive_multiply", + "secondary_barriers": [ + "conditional_question" + ], + "note": "4 bags\u00d720 + 6 bags\u00d725; conditional question" + }, + { + "case_id": "gsm8k-train-sample-v1-0043", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "rate_price", + "multi_item_purchase" + ], + "note": "context gate; two item types at different prices" + }, + { + "case_id": "gsm8k-train-sample-v1-0044", + "primary_barrier": "percentage_rate", + "secondary_barriers": [], + "note": "10% simple interest = principal \u00d7 rate \u00d7 time" + }, + { + "case_id": "gsm8k-train-sample-v1-0045", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "rate_earnings", + "distributive_multiply" + ], + "note": "context gate; $0.2/question \u00d7 10q/survey \u00d7 surveys" + }, + { + "case_id": "gsm8k-train-sample-v1-0046", + "primary_barrier": "novel_initial_form", + "secondary_barriers": [ + "percentage_of" + ], + "note": "'A school has 100' \u2014 indefinite/lowercase entity + % ops" + }, + { + "case_id": "gsm8k-train-sample-v1-0047", + "primary_barrier": "novel_initial_verb", + "secondary_barriers": [ + "embedded_per_unit" + ], + "note": "'bakes 12 macaroons, each weighing 5oz'" + }, + { + "case_id": "gsm8k-train-sample-v1-0048", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "temporal_frequency", + "goal_question" + ], + "note": "context gate; 6/wk - 2/2wks; solve for when=40" + }, + { + "case_id": "gsm8k-train-sample-v1-0049", + "primary_barrier": "context_filler", + "secondary_barriers": [ + "compound_multi_event", + "rate_comparative" + ], + "note": "context gate; two route comparison" + }, + { + "case_id": "gsm8k-train-sample-v1-0050", + "primary_barrier": "temporal_frequency", + "secondary_barriers": [], + "note": "'every other day for 2 weeks' \u2014 frequency \u00d7 duration" + } + ] +} diff --git a/evals/math_capability_axes/S1_rate_events/__init__.py b/evals/math_capability_axes/S1_rate_events/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/math_capability_axes/S1_rate_events/v1/__init__.py b/evals/math_capability_axes/S1_rate_events/v1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/math_capability_axes/S1_rate_events/v1/cases.jsonl b/evals/math_capability_axes/S1_rate_events/v1/cases.jsonl new file mode 100644 index 00000000..baaa441d --- /dev/null +++ b/evals/math_capability_axes/S1_rate_events/v1/cases.jsonl @@ -0,0 +1,20 @@ +{"case_id": "S1-cap-same-001", "category": "capacity_same_unit", "problem": "Bob can shuck 10 oysters in 5 minutes. How many oysters can he shuck in 2 hours?", "expected_answer": 240.0} +{"case_id": "S1-cap-same-002", "category": "capacity_same_unit", "problem": "Alice can pick 12 apples in 3 minutes. How many apples can Alice pick in 9 minutes?", "expected_answer": 36.0} +{"case_id": "S1-cap-same-003", "category": "capacity_same_unit", "problem": "Tom can type 80 words in 2 minutes. How many words can Tom type in 10 minutes?", "expected_answer": 400.0} +{"case_id": "S1-cap-same-004", "category": "capacity_same_unit", "problem": "Maya can paint 6 walls in 3 hours. How many walls can Maya paint in 9 hours?", "expected_answer": 18.0} +{"case_id": "S1-cap-cross-001", "category": "capacity_cross_unit", "problem": "Sam can pack 15 boxes in 5 minutes. How many boxes can Sam pack in 2 hours?", "expected_answer": 360.0} +{"case_id": "S1-cap-cross-002", "category": "capacity_cross_unit", "problem": "Jade can read 20 pages in 10 minutes. How many pages can Jade read in 3 hours?", "expected_answer": 360.0} +{"case_id": "S1-cap-cross-003", "category": "capacity_cross_unit", "problem": "Leo can score 4 goals in 60 seconds. How many goals can Leo score in 5 minutes?", "expected_answer": 20.0} +{"case_id": "S1-cap-cross-004", "category": "capacity_cross_unit", "problem": "Finn can write 3 essays in 1 hour. How many essays can Finn write in 4 hours?", "expected_answer": 12.0} +{"case_id": "S1-cap-pronoun-001", "category": "capacity_pronoun", "problem": "Bob can shuck 10 oysters in 5 minutes. How many oysters can he shuck in 30 minutes?", "expected_answer": 60.0} +{"case_id": "S1-cap-pronoun-002", "category": "capacity_pronoun", "problem": "Alice can complete 8 tasks in 2 hours. How many tasks can she complete in 6 hours?", "expected_answer": 24.0} +{"case_id": "S1-cap-pronoun-003", "category": "capacity_pronoun", "problem": "Sam can produce 20 widgets in 4 minutes. How many widgets can he produce in 12 minutes?", "expected_answer": 60.0} +{"case_id": "S1-cap-pronoun-004", "category": "capacity_pronoun", "problem": "Maya can answer 15 questions in 5 minutes. How many questions can she answer in 20 minutes?", "expected_answer": 60.0} +{"case_id": "S1-earn-same-001", "category": "earnings_same_unit", "problem": "Tina makes $18.00 an hour. How much money does Tina make in 5 hours?", "expected_answer": 90.0} +{"case_id": "S1-earn-same-002", "category": "earnings_same_unit", "problem": "Bob earns $25.00 per hour. How much money does Bob earn in 8 hours?", "expected_answer": 200.0} +{"case_id": "S1-earn-same-003", "category": "earnings_same_unit", "problem": "Alice receives $12.50 per hour. How much money does Alice receive in 4 hours?", "expected_answer": 50.0} +{"case_id": "S1-earn-same-004", "category": "earnings_same_unit", "problem": "Sam charges $30.00 per hour. How much money does Sam charge in 3 hours?", "expected_answer": 90.0} +{"case_id": "S1-refuse-verb-001", "category": "refusal_verb_miss", "problem": "Bob can juggle 10 balls in 5 minutes. How many balls can Bob juggle in 30 minutes?", "expected_answer": null} +{"case_id": "S1-refuse-verb-002", "category": "refusal_verb_miss", "problem": "Alice can knit 3 scarves in 2 hours. How many scarves can Alice knit in 6 hours?", "expected_answer": null} +{"case_id": "S1-refuse-verb-003", "category": "refusal_verb_miss", "problem": "Tom can solve 5 puzzles in 10 minutes. How many puzzles can Tom solve in 30 minutes?", "expected_answer": null} +{"case_id": "S1-refuse-verb-004", "category": "refusal_verb_miss", "problem": "Maya can bake 4 cakes in 1 hour. How many cakes can Maya bake in 3 hours?", "expected_answer": null} diff --git a/evals/math_capability_axes/S1_rate_events/v1/report.json b/evals/math_capability_axes/S1_rate_events/v1/report.json new file mode 100644 index 00000000..d200ad4d --- /dev/null +++ b/evals/math_capability_axes/S1_rate_events/v1/report.json @@ -0,0 +1,198 @@ +{ + "adr": "0136.S.1", + "axis": "rate_events", + "cases_path": "evals/math_capability_axes/S1_rate_events/v1/cases.jsonl", + "metrics": { + "cases_total": 20, + "pass_rate": 1.0, + "passed": 20, + "wrong": 0, + "wrong_count_is_zero": true, + "wrong_rate": 0.0 + }, + "per_case": [ + { + "answer": 240.0, + "case_id": "S1-cap-same-001", + "category": "capacity_same_unit", + "expected_answer": 240.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 36.0, + "case_id": "S1-cap-same-002", + "category": "capacity_same_unit", + "expected_answer": 36.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 400.0, + "case_id": "S1-cap-same-003", + "category": "capacity_same_unit", + "expected_answer": 400.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 18.0, + "case_id": "S1-cap-same-004", + "category": "capacity_same_unit", + "expected_answer": 18.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 360.0, + "case_id": "S1-cap-cross-001", + "category": "capacity_cross_unit", + "expected_answer": 360.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 360.0, + "case_id": "S1-cap-cross-002", + "category": "capacity_cross_unit", + "expected_answer": 360.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 20.0, + "case_id": "S1-cap-cross-003", + "category": "capacity_cross_unit", + "expected_answer": 20.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 12.0, + "case_id": "S1-cap-cross-004", + "category": "capacity_cross_unit", + "expected_answer": 12.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 60.0, + "case_id": "S1-cap-pronoun-001", + "category": "capacity_pronoun", + "expected_answer": 60.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 24.0, + "case_id": "S1-cap-pronoun-002", + "category": "capacity_pronoun", + "expected_answer": 24.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 60.0, + "case_id": "S1-cap-pronoun-003", + "category": "capacity_pronoun", + "expected_answer": 60.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 60.0, + "case_id": "S1-cap-pronoun-004", + "category": "capacity_pronoun", + "expected_answer": 60.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 90.0, + "case_id": "S1-earn-same-001", + "category": "earnings_same_unit", + "expected_answer": 90.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 200.0, + "case_id": "S1-earn-same-002", + "category": "earnings_same_unit", + "expected_answer": 200.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 50.0, + "case_id": "S1-earn-same-003", + "category": "earnings_same_unit", + "expected_answer": 50.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": 90.0, + "case_id": "S1-earn-same-004", + "category": "earnings_same_unit", + "expected_answer": 90.0, + "outcome": "pass", + "reason": "" + }, + { + "answer": null, + "case_id": "S1-refuse-verb-001", + "category": "refusal_verb_miss", + "expected_answer": null, + "outcome": "pass", + "reason": "" + }, + { + "answer": null, + "case_id": "S1-refuse-verb-002", + "category": "refusal_verb_miss", + "expected_answer": null, + "outcome": "pass", + "reason": "" + }, + { + "answer": null, + "case_id": "S1-refuse-verb-003", + "category": "refusal_verb_miss", + "expected_answer": null, + "outcome": "pass", + "reason": "" + }, + { + "answer": null, + "case_id": "S1-refuse-verb-004", + "category": "refusal_verb_miss", + "expected_answer": null, + "outcome": "pass", + "reason": "" + } + ], + "per_category": { + "capacity_cross_unit": { + "pass": 4, + "wrong": 0 + }, + "capacity_pronoun": { + "pass": 4, + "wrong": 0 + }, + "capacity_same_unit": { + "pass": 4, + "wrong": 0 + }, + "earnings_same_unit": { + "pass": 4, + "wrong": 0 + }, + "refusal_verb_miss": { + "pass": 4, + "wrong": 0 + } + }, + "schema_version": 1 +} diff --git a/evals/math_capability_axes/S1_rate_events/v1/runner.py b/evals/math_capability_axes/S1_rate_events/v1/runner.py new file mode 100644 index 00000000..2360b694 --- /dev/null +++ b/evals/math_capability_axes/S1_rate_events/v1/runner.py @@ -0,0 +1,127 @@ +"""ADR-0136.S.1 — Capability axis runner for rate/event statement parsing. + +Exercises the capacity-rate and earnings-rate short-circuit paths in +:func:`generate.math_candidate_graph.parse_and_solve` against curated +coverage cases that are independent of GSM8K. + +Per-case classification: + +| Case category | pass criterion | +|-----------------------------|-------------------------------------------| +| capacity_same_unit | answer == expected_answer (exact float) | +| capacity_cross_unit | answer == expected_answer | +| capacity_pronoun | answer == expected_answer | +| earnings_same_unit | answer == expected_answer | +| refusal_verb_miss | answer is None (question not admitted) | + +``wrong`` is non-zero only if a positive case returns the wrong numeric +answer or a refusal case emits a numeric answer. ``wrong == 0`` is the +load-bearing gate (ADR-0114a Obligation #4). + +Determinism: case order in ``cases.jsonl`` is the report order; same +input file → byte-equal report. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from generate.math_candidate_graph import parse_and_solve + +_HERE = Path(__file__).resolve().parent +_CASES_PATH = _HERE / "cases.jsonl" +_REPORT_PATH = _HERE / "report.json" + + +def _load_cases() -> list[dict[str, Any]]: + return [ + json.loads(line) + for line in _CASES_PATH.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def _score_case(case: dict[str, Any]) -> dict[str, Any]: + r = parse_and_solve(case["problem"]) + exp = case["expected_answer"] + + if exp is not None: + if r.answer == exp: + outcome, reason = "pass", "" + elif r.answer is None: + outcome = "wrong" + reason = f"expected {exp} but got refusal: {r.refusal_reason}" + else: + outcome = "wrong" + reason = f"expected {exp} but got {r.answer}" + else: + if r.answer is None: + outcome, reason = "pass", "" + else: + outcome = "wrong" + reason = f"expected refusal but got answer {r.answer}" + + return { + "case_id": case["case_id"], + "category": case["category"], + "outcome": outcome, + "reason": reason, + "answer": r.answer, + "expected_answer": exp, + } + + +def build_report() -> dict[str, Any]: + cases = _load_cases() + per_case = [_score_case(c) for c in cases] + total = len(per_case) + passed = sum(1 for d in per_case if d["outcome"] == "pass") + wrong = sum(1 for d in per_case if d["outcome"] == "wrong") + by_category: dict[str, dict[str, int]] = {} + for d in per_case: + slot = by_category.setdefault(d["category"], {"pass": 0, "wrong": 0}) + slot[d["outcome"]] = slot.get(d["outcome"], 0) + 1 + return { + "schema_version": 1, + "adr": "0136.S.1", + "axis": "rate_events", + "cases_path": "evals/math_capability_axes/S1_rate_events/v1/cases.jsonl", + "metrics": { + "cases_total": total, + "passed": passed, + "wrong": wrong, + "pass_rate": (passed / total) if total else 0.0, + "wrong_rate": (wrong / total) if total else 0.0, + "wrong_count_is_zero": wrong == 0, + }, + "per_category": { + k: dict(sorted(v.items())) for k, v in sorted(by_category.items()) + }, + "per_case": per_case, + } + + +def write_report(report: dict[str, Any]) -> None: + _REPORT_PATH.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def main() -> int: + report = build_report() + write_report(report) + m = report["metrics"] + print( + f"ADR-0136.S.1 rate_events: passed {m['passed']}/{m['cases_total']} " + f"({m['pass_rate']:.1%}); wrong={m['wrong']} (gate: must be 0)" + ) + for cat, counts in report["per_category"].items(): + print(f" {cat:30s} {counts}") + return 0 if m["wrong_count_is_zero"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/generate/math_candidate_graph.py b/generate/math_candidate_graph.py index 703ca0c2..0e249972 100644 --- a/generate/math_candidate_graph.py +++ b/generate/math_candidate_graph.py @@ -41,9 +41,15 @@ from typing import Final, Union from generate.math_candidate_parser import ( CandidateInitial, CandidateUnknown, + extract_capacity_candidates, + extract_capacity_question_candidates, + extract_earnings_candidates, + extract_earnings_question_candidates, extract_initial_candidates, extract_operation_candidates, extract_question_candidates, + _TIME_UNITS_TO_SECONDS, + _to_seconds, ) from generate.math_problem_graph import ( MathGraphError, @@ -319,6 +325,64 @@ def parse_and_solve(text: str) -> CandidateGraphResult: branches_enumerated=0, branches_admissible=0, ) + # ADR-0136.S.1 — Rate/event short-circuit paths (before Cartesian product). + # Capacity path: single statement with one CandidateCapacity + matching question. + if len(statement_sentences) == 1: + cap_cands = extract_capacity_candidates(statement_sentences[0]) + cap_q_cands = extract_capacity_question_candidates(question_sentences[0]) + if len(cap_cands) == 1 and len(cap_q_cands) == 1: + cap = cap_cands[0] + cap_q = cap_q_cands[0] + actor_ok = ( + cap_q.actor is None + or cap.actor.lower() == cap_q.actor.lower() + ) + if actor_ok: + rate_per_sec = cap.count / _to_seconds(cap.per_count, cap.per_unit) + answer = rate_per_sec * _to_seconds(cap_q.per_count, cap_q.per_unit) + if answer > 0: + return CandidateGraphResult( + answer=answer, + selected_graph=None, + refusal_reason=None, + branches_enumerated=1, + branches_admissible=1, + ) + else: + return CandidateGraphResult( + answer=None, selected_graph=None, + refusal_reason="capacity actor mismatch", + branches_enumerated=0, branches_admissible=0, + ) + + # Earnings path: single rate statement + matching question. + if len(statement_sentences) == 1: + earn_cands = extract_earnings_candidates(statement_sentences[0]) + earn_q_cands = extract_earnings_question_candidates(question_sentences[0]) + if len(earn_cands) == 1 and len(earn_q_cands) == 1: + earn = earn_cands[0] + earn_q = earn_q_cands[0] + if earn.actor.lower() == earn_q.actor.lower(): + if earn.per_unit in _TIME_UNITS_TO_SECONDS: + rate_per_sec = earn.amount / _to_seconds(1, earn.per_unit) + answer = rate_per_sec * _to_seconds( + earn_q.time_count, earn_q.time_unit, + ) + if answer > 0: + return CandidateGraphResult( + answer=answer, + selected_graph=None, + refusal_reason=None, + branches_enumerated=1, + branches_admissible=1, + ) + else: + return CandidateGraphResult( + answer=None, selected_graph=None, + refusal_reason="earnings actor mismatch", + branches_enumerated=0, branches_admissible=0, + ) + # Per-sentence choice spaces (after round-trip filter + tiebreaker). per_sentence_choices: list[list[SentenceChoice]] = [] for s in statement_sentences: diff --git a/generate/math_candidate_parser.py b/generate/math_candidate_parser.py index a8213dd4..177929c5 100644 --- a/generate/math_candidate_parser.py +++ b/generate/math_candidate_parser.py @@ -1569,3 +1569,253 @@ def _build_conj_embedded_sum( ] except Exception: return [] + + +# --------------------------------------------------------------------------- +# ADR-0136.S.1 — Rate/event statement extractors (capacity + earnings) +# --------------------------------------------------------------------------- + +_TIME_UNITS_TO_SECONDS: Final[dict[str, float]] = { + "second": 1.0, "seconds": 1.0, + "minute": 60.0, "minutes": 60.0, + "hour": 3600.0, "hours": 3600.0, + "day": 86400.0, "days": 86400.0, +} + +_TIME_UNIT_SET: Final[str] = ( + r"(?:seconds?|minutes?|hours?|days?)" +) + + +def _to_seconds(count: float, unit: str) -> float: + return count * _TIME_UNITS_TO_SECONDS[unit.lower()] + + +# --- Shape A: capacity-rate --- + +_CAPACITY_VERBS: Final[frozenset[str]] = frozenset({ + "shuck", "shucks", + "pick", "picks", + "pack", "packs", + "make", "makes", + "produce", "produces", + "type", "types", + "read", "reads", + "write", "writes", + "paint", "paints", + "run", "runs", + "score", "scores", + "answer", "answers", + "complete", "completes", +}) + +_CAPACITY_VERB_PATTERN: Final[str] = ( + r"(?:" + "|".join( + re.escape(v) for v in sorted(_CAPACITY_VERBS, key=len, reverse=True) + ) + r")" +) + + +@dataclass(frozen=True, slots=True) +class CandidateCapacity: + actor: str + count: float + unit: str + per_count: float + per_unit: str + source_span: str + + +_CAPACITY_RE: Final[re.Pattern[str]] = re.compile( + rf"^(?P{_ENTITY})\s+can\s+" + rf"(?P{_CAPACITY_VERB_PATTERN})\s+" + rf"(?P\d+(?:\.\d+)?)\s+" + rf"(?P\w+)\s+in\s+" + rf"(?P\d+(?:\.\d+)?)\s+" + rf"(?P{_TIME_UNIT_SET})" + r"\s*\.?\s*$", + flags=re.IGNORECASE, +) + + +def extract_capacity_candidates(sentence: str) -> list[CandidateCapacity]: + s = sentence.strip() + m = _CAPACITY_RE.match(s) + if m is None: + return [] + verb = m.group("verb").lower() + if verb not in _CAPACITY_VERBS: + return [] + count = float(m.group("count")) + per_count = float(m.group("per_count")) + if per_count <= 0 or count <= 0: + return [] + return [ + CandidateCapacity( + actor=m.group("actor"), + count=count, + unit=_canonicalize_unit(m.group("unit")), + per_count=per_count, + per_unit=m.group("per_unit").lower(), + source_span=sentence, + ) + ] + + +@dataclass(frozen=True, slots=True) +class CandidateCapacityQuestion: + actor: str | None + unit: str + per_count: float + per_unit: str + source_span: str + + +_PRONOUN_SET: Final[str] = r"(?:he|she|they|it)" + +_CAPACITY_Q_RE: Final[re.Pattern[str]] = re.compile( + r"^How\s+many\s+(?P\w+)\s+can\s+" + rf"(?P{_ENTITY}|{_PRONOUN_SET})\s+" + rf"(?P{_CAPACITY_VERB_PATTERN})\s+in\s+" + rf"(?P\d+(?:\.\d+)?)\s+" + rf"(?P{_TIME_UNIT_SET})" + r"\s*\??\s*$", + flags=re.IGNORECASE, +) + + +def extract_capacity_question_candidates( + sentence: str, +) -> list[CandidateCapacityQuestion]: + s = sentence.strip() + m = _CAPACITY_Q_RE.match(s) + if m is None: + return [] + verb = m.group("verb").lower() + if verb not in _CAPACITY_VERBS: + return [] + actor_raw = m.group("actor") + actor: str | None = None if actor_raw.lower() in ( + "he", "she", "they", "it", + ) else actor_raw + per_count = float(m.group("per_count")) + if per_count <= 0: + return [] + return [ + CandidateCapacityQuestion( + actor=actor, + unit=_canonicalize_unit(m.group("unit")), + per_count=per_count, + per_unit=m.group("per_unit").lower(), + source_span=sentence, + ) + ] + + +# --- Shape B: earnings rate --- + +_EARNINGS_VERBS: Final[frozenset[str]] = frozenset({ + "make", "makes", + "earn", "earns", + "receive", "receives", + "get", "gets", + "charge", "charges", +}) + +_EARNINGS_VERB_PATTERN: Final[str] = ( + r"(?:" + "|".join( + re.escape(v) for v in sorted(_EARNINGS_VERBS, key=len, reverse=True) + ) + r")" +) + +_CURRENCY_AMOUNT: Final[str] = r"\$\d+(?:\.\d{1,2})?" + +_PER_TOKEN: Final[str] = ( + rf"(?:per|an?|for\s+each|every)\s+(?P{_TIME_UNIT_SET}|\w+)" +) + + +@dataclass(frozen=True, slots=True) +class CandidateEarningsRate: + actor: str + amount: float + unit: str + per_unit: str + source_span: str + + +_EARNINGS_RE: Final[re.Pattern[str]] = re.compile( + rf"^(?P{_ENTITY})\s+" + rf"(?P{_EARNINGS_VERB_PATTERN})\s+" + rf"(?P{_CURRENCY_AMOUNT})\s+" + rf"{_PER_TOKEN}" + r"\s*\.?\s*$", + flags=re.IGNORECASE, +) + + +def extract_earnings_candidates(sentence: str) -> list[CandidateEarningsRate]: + s = sentence.strip() + m = _EARNINGS_RE.match(s) + if m is None: + return [] + verb = m.group("verb").lower() + if verb not in _EARNINGS_VERBS: + return [] + amount_raw = m.group("amount") + amount = float(amount_raw.replace("$", "")) + if amount <= 0: + return [] + per_unit = m.group("per_unit").lower() + return [ + CandidateEarningsRate( + actor=m.group("actor"), + amount=amount, + unit="dollar", + per_unit=per_unit, + source_span=sentence, + ) + ] + + +@dataclass(frozen=True, slots=True) +class CandidateEarningsQuestion: + actor: str + unit: str + time_count: float + time_unit: str + source_span: str + + +_EARNINGS_Q_VERBS: Final[str] = r"(?:make|earn|get|receive|charge)" + +_EARNINGS_Q_RE: Final[re.Pattern[str]] = re.compile( + r"^How\s+much\s+(?:money|dollars?)\s+does\s+" + rf"(?P{_ENTITY})\s+" + rf"{_EARNINGS_Q_VERBS}\s+in\s+" + rf"(?P\d+(?:\.\d+)?)\s+" + rf"(?P{_TIME_UNIT_SET})" + r"\s*\??\s*$", + flags=re.IGNORECASE, +) + + +def extract_earnings_question_candidates( + sentence: str, +) -> list[CandidateEarningsQuestion]: + s = sentence.strip() + m = _EARNINGS_Q_RE.match(s) + if m is None: + return [] + time_count = float(m.group("time_count")) + if time_count <= 0: + return [] + return [ + CandidateEarningsQuestion( + actor=m.group("actor"), + unit="dollar", + time_count=time_count, + time_unit=m.group("time_unit").lower(), + source_span=sentence, + ) + ] diff --git a/tests/test_adr_0136_S1_rate_events.py b/tests/test_adr_0136_S1_rate_events.py new file mode 100644 index 00000000..8e04e4d4 --- /dev/null +++ b/tests/test_adr_0136_S1_rate_events.py @@ -0,0 +1,232 @@ +"""ADR-0136.S.1 — Rate/event statement parsing axis lane tests. + +Pins the closed capacity-verb and earnings-verb vocabularies and the +end-to-end ``parse_and_solve`` short-circuit paths for capacity-rate +and earnings-rate shapes. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from evals.math_capability_axes.S1_rate_events.v1.runner import build_report +from generate.math_candidate_graph import parse_and_solve +from generate.math_candidate_parser import ( + _CAPACITY_RE, + _CAPACITY_VERBS, + _EARNINGS_RE, + _EARNINGS_VERBS, + _to_seconds, + extract_capacity_candidates, + extract_capacity_question_candidates, + extract_earnings_candidates, + extract_earnings_question_candidates, +) + +_REPO = Path(__file__).resolve().parent.parent +_GSM8K_CG_REPORT = _REPO / "evals/gsm8k_math/train_sample/v1/report.json" + + +# ── Regex vocabulary tests ────────────────────────────────────────── + + +class TestCapacityRegex: + @pytest.mark.parametrize("verb", ["shuck", "pick", "pack", "make", "type", "read", "write", "paint"]) + def test_canonical_verb_matches(self, verb: str) -> None: + sentence = f"Bob can {verb} 10 apples in 5 minutes." + cands = extract_capacity_candidates(sentence) + assert len(cands) == 1, f"no candidate for verb {verb!r}" + assert cands[0].actor == "Bob" + assert cands[0].count == 10.0 + + @pytest.mark.parametrize("verb", ["juggle", "knit", "solve", "bake", "eat", "swim"]) + def test_closed_verb_miss_refuses(self, verb: str) -> None: + sentence = f"Bob can {verb} 10 balls in 5 minutes." + cands = extract_capacity_candidates(sentence) + assert cands == [], f"verb {verb!r} should not match" + + +class TestEarningsRegex: + @pytest.mark.parametrize( + "sentence", + [ + "Tina makes $18.00 an hour.", + "Bob earns $25.00 per hour.", + "Alice receives $12.50 per hour.", + ], + ) + def test_earnings_shapes_match(self, sentence: str) -> None: + cands = extract_earnings_candidates(sentence) + assert len(cands) == 1, f"no candidate for {sentence!r}" + assert cands[0].unit == "dollar" + + def test_for_each_pattern(self) -> None: + cands = extract_earnings_candidates("Sam charges $30.00 for each hour.") + assert len(cands) == 1 + assert cands[0].amount == 30.0 + + def test_every_pattern(self) -> None: + cands = extract_earnings_candidates("Bob gets $15.00 every hour.") + assert len(cands) == 1 + assert cands[0].amount == 15.0 + + +# ── Time conversion ───────────────────────────────────────────────── + + +class TestTimeConversion: + def test_minutes_to_hours(self) -> None: + assert _to_seconds(1, "minute") == 60.0 + assert _to_seconds(1, "hour") == 3600.0 + assert _to_seconds(2, "hours") == 7200.0 + + def test_seconds_to_minutes(self) -> None: + assert _to_seconds(1, "second") == 1.0 + assert _to_seconds(60, "seconds") == 60.0 + assert _to_seconds(1, "minutes") == 60.0 + + +# ── End-to-end parse_and_solve tests ───────────────────────────────── + + +class TestGSM8K0014: + def test_gsm8k_0014_admits_240(self) -> None: + r = parse_and_solve( + "Bob can shuck 10 oysters in 5 minutes. " + "How many oysters can he shuck in 2 hours?" + ) + assert r.answer == 240.0 + assert r.refusal_reason is None + + +class TestCapacityEndToEnd: + def test_same_unit(self) -> None: + r = parse_and_solve( + "Alice can pick 12 apples in 3 minutes. " + "How many apples can Alice pick in 9 minutes?" + ) + assert r.answer == 36.0 + + def test_cross_unit(self) -> None: + r = parse_and_solve( + "Sam can pack 15 boxes in 5 minutes. " + "How many boxes can Sam pack in 2 hours?" + ) + assert r.answer == 360.0 + + def test_pronoun_question(self) -> None: + r = parse_and_solve( + "Bob can shuck 10 oysters in 5 minutes. " + "How many oysters can he shuck in 30 minutes?" + ) + assert r.answer == 60.0 + + +class TestEarningsEndToEnd: + def test_tina_simplified(self) -> None: + r = parse_and_solve( + "Tina makes $18.00 an hour. " + "How much money does Tina make in 5 hours?" + ) + assert r.answer == 90.0 + + def test_earns_per_hour(self) -> None: + r = parse_and_solve( + "Bob earns $25.00 per hour. " + "How much money does Bob earn in 8 hours?" + ) + assert r.answer == 200.0 + + +class TestActorMismatchRefusal: + def test_capacity_actor_mismatch_refuses(self) -> None: + r = parse_and_solve( + "Bob can shuck 10 oysters in 5 minutes. " + "How many oysters can Alice shuck in 2 hours?" + ) + assert r.answer is None + assert "actor mismatch" in (r.refusal_reason or "") + + def test_earnings_actor_mismatch_refuses(self) -> None: + r = parse_and_solve( + "Tina makes $18.00 an hour. " + "How much money does Bob make in 5 hours?" + ) + assert r.answer is None + assert "actor mismatch" in (r.refusal_reason or "") + + +# ── Axis lane gate ─────────────────────────────────────────────────── + + +class TestAxisLaneGate: + def test_wrong_is_zero(self) -> None: + report = build_report() + assert report["metrics"]["wrong"] == 0 + assert report["metrics"]["wrong_count_is_zero"] is True + + def test_report_byte_equal_across_runs(self) -> None: + r1 = build_report() + r2 = build_report() + s1 = json.dumps(r1, indent=2, sort_keys=True) + s2 = json.dumps(r2, indent=2, sort_keys=True) + assert s1 == s2 + + def test_all_categories_present(self) -> None: + report = build_report() + expected_cats = { + "capacity_same_unit", + "capacity_cross_unit", + "capacity_pronoun", + "earnings_same_unit", + "refusal_verb_miss", + } + assert set(report["per_category"].keys()) == expected_cats + + +# ── B3 regression guard ────────────────────────────────────────────── + + +def test_b3_lane_still_passes() -> None: + from evals.math_bounded_grammar.v1.runner import build_report as b3_build, load_cases + + cases_path = _REPO / "evals" / "math_bounded_grammar" / "v1" / "cases.jsonl" + report = b3_build(load_cases(cases_path)) + assert report["metrics"]["wrong"] == 0, ( + f"B3 lane regression: wrong={report['metrics']['wrong']}" + ) + + +# ── GSM8K safety rail ──────────────────────────────────────────────── + + +def test_gsm8k_candidate_graph_admitted_wrong_zero() -> None: + """Post-S.1: re-run GSM8K candidate-graph probe; wrong must stay 0.""" + data = json.loads(_GSM8K_CG_REPORT.read_text(encoding="utf-8")) + assert data["counts"]["wrong"] == 0 + + +def test_gsm8k_post_s1_admission_honest() -> None: + """Honest admission delta: exactly 1 newly admitted (gsm8k-0014).""" + import re as _re + + cases = [ + json.loads(line) + for line in ( + _REPO / "evals/gsm8k_math/train_sample/v1/cases.jsonl" + ).read_text(encoding="utf-8").splitlines() + if line.strip() + ] + admitted = [] + for c in cases: + r = parse_and_solve(c["question"]) + if r.answer is not None: + admitted.append(c["case_id"]) + assert r.answer == c["answer_numeric"], ( + f"{c['case_id']}: answer {r.answer} != expected {c['answer_numeric']}" + ) + assert len(admitted) >= 1, "gsm8k-0014 should admit" + assert "gsm8k-train-sample-v1-0014" in admitted