diff --git a/evals/gsm8k_math/train_sample/v1/report.json b/evals/gsm8k_math/train_sample/v1/report.json index 9f496d8b..13a43319 100644 --- a/evals/gsm8k_math/train_sample/v1/report.json +++ b/evals/gsm8k_math/train_sample/v1/report.json @@ -1,8 +1,8 @@ { "adr": "0126", "counts": { - "correct": 0, - "refused": 50, + "correct": 3, + "refused": 47, "wrong": 0 }, "exit_criterion": { @@ -13,22 +13,22 @@ "per_case": [ { "case_id": "gsm8k-train-sample-v1-0001", - "reason": "candidate_graph: no admissible candidate for statement: 'Tina makes $18.00 an hour.'", + "reason": "candidate_graph: no branch produced a solvable graph", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0002", - "reason": "candidate_graph: no admissible candidate for statement: 'She splits it up into 25-foot sections.'", + "reason": "candidate_graph: no admissible candidate for statement: 'She gives 1/4 of that to a friend.'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0003", - "reason": "candidate_graph: no admissible candidate for statement: 'The student council sells scented erasers in the morning before school starts to help raise money for school dances.'", + "reason": "candidate_graph: no admissible candidate for question: 'If the student council sells the erasers for $0.75 each, how much money will they make?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0004", - "reason": "candidate_graph: no admissible candidate for statement: 'There are some kids in camp.'", + "reason": "candidate_graph: no admissible candidate for statement: 'Half of the kids are going to soccer camp, and 1/4 of the kids going to soccer camp are going to soccer camp in the morning.'", "verdict": "refused" }, { @@ -38,17 +38,17 @@ }, { "case_id": "gsm8k-train-sample-v1-0006", - "reason": "candidate_graph: no admissible candidate for statement: 'Mandy started reading books with only 8 pages when she was 6 years old.'", + "reason": "candidate_graph: no admissible candidate for question: 'How many pages do the books she reads now have?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0007", - "reason": "candidate_graph: no admissible candidate for statement: 'Francine has five full boxes of crayons and 5 loose crayons, and her friend has 27 loose crayons.'", + "reason": "candidate_graph: no admissible candidate for question: 'How many more boxes do they need if Francine has a total of 85 crayons?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0008", - "reason": "candidate_graph: no admissible candidate for statement: 'Marnie makes bead bracelets.'", + "reason": "candidate_graph: no admissible candidate for question: 'If 50 beads are used to make one bracelet, how many bracelets will Marnie be able to make out of the beads she bought?'", "verdict": "refused" }, { @@ -58,202 +58,202 @@ }, { "case_id": "gsm8k-train-sample-v1-0010", - "reason": "candidate_graph: no admissible candidate for statement: 'Yun had 20 paperclips initially, but then lost 12.'", + "reason": "candidate_graph: no admissible candidate for statement: 'Marion has 1/4 more than what Yun currently has, plus 7.'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0011", - "reason": "candidate_graph: no admissible candidate for statement: 'Alexa has a lemonade stand where she sells lemonade for $2 for one cup.'", + "reason": "candidate_graph: no branch produced a solvable graph", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0012", - "reason": "candidate_graph: no admissible candidate for statement: 'He put all of them in his aquarium but his fish ate half of them.'", + "reason": "candidate_graph: no admissible candidate for question: \"How many of Dennis's rocks are left?\"", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0013", - "reason": "candidate_graph: no admissible candidate for statement: 'Allison, a YouTuber, uploads 10 one-hour videos of food reviews each day to her channel.'", + "reason": "candidate_graph: no admissible candidate for question: \"What's the total number of video hours she has uploaded at the end of the month?\"", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0014", - "reason": "candidate_graph: no admissible candidate for statement: 'Bob can shuck 10 oysters in 5 minutes.'", - "verdict": "refused" + "reason": "fast-path", + "verdict": "correct" }, { "case_id": "gsm8k-train-sample-v1-0015", - "reason": "candidate_graph: no admissible candidate for statement: 'Traveling from Manhattan to the Bronx, Andrew rides the subway for 10 hours, takes the train and rides for twice as much time as the subway ride, and then bikes the remaining distance for 8 hours.'", + "reason": "candidate_graph: no admissible candidate for question: \"What's the total time he takes to reach the Bronx from Manhattan?\"", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0016", - "reason": "candidate_graph: no admissible candidate for statement: \"On Rudolph's car trip across town, he traveled 2 more than 5 miles and encountered 3 less than 17 stop signs.\"", + "reason": "candidate_graph: no admissible candidate for question: 'How many stop signs per mile did Rudolph encounter on his trip across town?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0017", - "reason": "candidate_graph: no admissible candidate for statement: 'Jason has a carriage house that he rents out.'", + "reason": "candidate_graph: no admissible candidate for question: 'How much will it cost him?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0018", - "reason": "candidate_graph: no admissible candidate for statement: 'Xavier plays football with his friends.'", - "verdict": "refused" + "reason": "fast-path", + "verdict": "correct" }, { "case_id": "gsm8k-train-sample-v1-0019", - "reason": "candidate_graph: no admissible candidate for statement: 'John adopts a dog from a shelter.'", + "reason": "candidate_graph: no admissible candidate for question: 'How much did he pay in total?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0020", - "reason": "candidate_graph: no admissible candidate for statement: 'Two puppies, two kittens, and three parakeets were for sale at the pet shop.'", + "reason": "candidate_graph: no admissible candidate for question: 'If the cost of one parakeet was $10, what would it cost to purchase all of the pets for sale at the pet shop, in dollars?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0021", - "reason": "candidate_graph: no admissible candidate for statement: 'John is lifting weights.'", + "reason": "candidate_graph: no admissible candidate for question: 'How much total weight does he move?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0022", - "reason": "candidate_graph: no admissible candidate for statement: 'Erica lives near a lake where most locals sell fish as their main source of income, earning $20 per kg of fish.'", + "reason": "candidate_graph: no admissible candidate for question: 'If Erica trawled 80 kg of fish in the past four months, not including today, how much money will she have earned in the past four months including today (assuming she sells all her fish)?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0023", - "reason": "candidate_graph: no admissible candidate for statement: 'Nicole collected 400 Pokemon cards.'", + "reason": "candidate_graph: no branch produced a solvable graph", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0024", - "reason": "candidate_graph: no admissible candidate for statement: 'Sidney does 20 jumping jacks on Monday, 36 on Tuesday, 40 on Wednesday, and 50 on Thursday.'", + "reason": "candidate_graph: no admissible candidate for question: 'How many jumping jacks did Brooke do?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0025", - "reason": "candidate_graph: no admissible candidate for statement: 'Lilibeth and her friends go strawberry picking.'", + "reason": "candidate_graph: no admissible candidate for question: \"If three of Lilibeth's friends pick the same amount as her, how many strawberries do Lilibeth and her friends pick in all?\"", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0026", - "reason": "candidate_graph: no admissible candidate for statement: 'Aaron and his brother Carson each saved up $40 to go to dinner.'", + "reason": "candidate_graph: no admissible candidate for statement: 'The bill is 3/4 of their total money.'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0027", - "reason": "candidate_graph: no admissible candidate for statement: 'Malcolm has 240 followers on Instagram and 500 followers on Facebook.'", + "reason": "candidate_graph: no admissible candidate for question: 'How many followers does Malcolm have on all his social media?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0028", - "reason": "candidate_graph: no admissible candidate for statement: 'Tom opens an amusement park.'", + "reason": "candidate_graph: no admissible candidate for statement: 'He sells 150 tickets a day for $10 each.'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0029", - "reason": "candidate_graph: no admissible candidate for statement: 'Fabian bought a brand new computer mouse and keyboard to be able to work from home.'", + "reason": "candidate_graph: no admissible candidate for question: 'If the mouse cost $16, how much did Fabian spent on his new accessories?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0030", - "reason": "candidate_graph: no admissible candidate for statement: 'Jake decides to go to the beach for a fun day.'", + "reason": "candidate_graph: no admissible candidate for statement: 'It is a 2-hour drive each way.'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0031", - "reason": "candidate_graph: no admissible candidate for statement: 'Jeremie wants to go to an amusement park with 3 friends at the end of summer.'", + "reason": "candidate_graph: no admissible candidate for question: 'How much will it cost Jeremie and her 3 friends to go to the amusement park and buy each of them a set of snacks?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0032", - "reason": "candidate_graph: no admissible candidate for statement: 'John decides to take up illustration.'", + "reason": "candidate_graph: no admissible candidate for question: 'How long does he spend on all the pictures?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0033", - "reason": "candidate_graph: no admissible candidate for statement: 'Rachel is 12 years old, and her grandfather is 7 times her age.'", + "reason": "candidate_graph: no admissible candidate for question: \"How old will Rachel's father be when she is 25 years old?\"", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0034", - "reason": "candidate_graph: no admissible candidate for statement: 'Georgie is a varsity player on a football team.'", + "reason": "candidate_graph: no admissible candidate for question: 'If he can improve his speed by forty percent, how many yards will he be able to run within 10 seconds?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0035", - "reason": "candidate_graph: no admissible candidate for statement: 'She decided to split them among her friends.'", + "reason": "candidate_graph: no admissible candidate for question: 'How many more apples would Martha need to give away to be left with only 4 of them?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0036", - "reason": "candidate_graph: no admissible candidate for statement: 'Monica way studying for an exam.'", + "reason": "candidate_graph: no admissible candidate for question: 'How much time did she spend studying in total during the five days?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0037", - "reason": "candidate_graph: no admissible candidate for statement: 'Michael wants to lose 10 pounds by June.'", + "reason": "candidate_graph: no admissible candidate for question: 'How much weight does he have to lose in May to meet his goal?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0038", - "reason": "candidate_graph: no admissible candidate for statement: 'In a building, there are a hundred ladies on the first-floor studying.'", + "reason": "candidate_graph: no admissible candidate for question: 'How many ladies are on the two floors in total?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0039", - "reason": "candidate_graph: no admissible candidate for statement: 'At the family reunion, everyone ate too much food and gained weight.'", + "reason": "candidate_graph: no admissible candidate for question: 'How much weight, in pounds, did the three family members gain at their reunion?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0040", - "reason": "candidate_graph: no admissible candidate for statement: 'Over several years, Daniel has adopted any stray animals he sees on the side of the road.'", + "reason": "candidate_graph: no admissible candidate for question: 'In total, how many legs do his animals have?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0041", - "reason": "candidate_graph: no admissible candidate for statement: 'The guests eat all of 1 pan, and 75% of the 2nd pan.'", + "reason": "candidate_graph: no admissible candidate for question: 'If all but 4 of the guests eat the brownies ala mode (with 2 scoops of vanilla ice cream), how many tubs of ice cream did the guests eat?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0042", - "reason": "candidate_graph: no admissible candidate for question: 'If Ella sells 200 apples, how many apples does Ella has left?'", - "verdict": "refused" + "reason": "fast-path", + "verdict": "correct" }, { "case_id": "gsm8k-train-sample-v1-0043", - "reason": "candidate_graph: no admissible candidate for statement: 'Sandra wants to buy some sweets.'", + "reason": "candidate_graph: no admissible candidate for question: 'How much money will she be left with after the purchase?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0044", - "reason": "candidate_graph: no admissible candidate for statement: 'John invests in a bank and gets 10% simple interest.'", + "reason": "candidate_graph: no branch produced a solvable graph", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0045", - "reason": "candidate_graph: no admissible candidate for statement: 'Bart fills out surveys to earn money.'", + "reason": "candidate_graph: no admissible candidate for question: 'How much money did he earn during these two days?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0046", - "reason": "candidate_graph: no admissible candidate for statement: 'A school has 100 students.'", + "reason": "candidate_graph: no admissible candidate for question: 'How many students own dogs?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0047", - "reason": "candidate_graph: no admissible candidate for statement: 'John bakes 12 coconut macaroons, each weighing 5 ounces.'", + "reason": "candidate_graph: no admissible candidate for question: 'What is the total weight, in ounces, of the remaining coconut macaroons?'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0048", - "reason": "candidate_graph: no admissible candidate for statement: 'Jed collects stamp cards.'", + "reason": "candidate_graph: no admissible candidate for statement: 'Every week, he gets 6 cards.'", "verdict": "refused" }, { "case_id": "gsm8k-train-sample-v1-0049", - "reason": "candidate_graph: no admissible candidate for statement: 'Malcolm is trying to find the fastest walk to school and is currently comparing two routes.'", + "reason": "candidate_graph: no admissible candidate for question: 'In minutes, how much longer was the second route compared with the first?'", "verdict": "refused" }, { diff --git a/evals/gsm8k_math/train_sample/v1/train_sample_coverage_report.json b/evals/gsm8k_math/train_sample/v1/train_sample_coverage_report.json index 6c553965..5fd39656 100644 --- a/evals/gsm8k_math/train_sample/v1/train_sample_coverage_report.json +++ b/evals/gsm8k_math/train_sample/v1/train_sample_coverage_report.json @@ -20,7 +20,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'Tina makes $18.00 an hour.'", + "reason": "candidate_graph: no branch produced a solvable graph", "trace_hash": null }, { @@ -31,7 +31,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'She splits it up into 25-foot sections.'", + "reason": "candidate_graph: no admissible candidate for statement: 'She gives 1/4 of that to a friend.'", "trace_hash": null }, { @@ -42,7 +42,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'The local bookstore donated 48 boxes of erasers.'", + "reason": "candidate_graph: no admissible candidate for question: 'If the student council sells the erasers for $0.75 each, how much money will they make?'", "trace_hash": null }, { @@ -75,7 +75,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'Mandy started reading books with only 8 pages when she was 6 years old.'", + "reason": "candidate_graph: no admissible candidate for question: 'How many pages do the books she reads now have?'", "trace_hash": null }, { @@ -130,7 +130,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'Alexa has a lemonade stand where she sells lemonade for $2 for one cup.'", + "reason": "candidate_graph: no branch produced a solvable graph", "trace_hash": null }, { @@ -141,7 +141,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'He put all of them in his aquarium but his fish ate half of them.'", + "reason": "candidate_graph: no admissible candidate for question: \"How many of Dennis's rocks are left?\"", "trace_hash": null }, { @@ -152,7 +152,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'Allison, a YouTuber, uploads 10 one-hour videos of food reviews each day to her channel.'", + "reason": "candidate_graph: no admissible candidate for question: \"What's the total number of video hours she has uploaded at the end of the month?\"", "trace_hash": null }, { @@ -174,7 +174,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'Traveling from Manhattan to the Bronx, Andrew rides the subway for 10 hours, takes the train and rides for twice as much time as the subway ride, and then bikes the remaining distance for 8 hours.'", + "reason": "candidate_graph: no admissible candidate for question: \"What's the total time he takes to reach the Bronx from Manhattan?\"", "trace_hash": null }, { @@ -185,7 +185,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: \"On Rudolph's car trip across town, he traveled 2 more than 5 miles and encountered 3 less than 17 stop signs.\"", + "reason": "candidate_graph: no admissible candidate for question: 'How many stop signs per mile did Rudolph encounter on his trip across town?'", "trace_hash": null }, { @@ -196,7 +196,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'He\u2019s charging $50.00 per day or $500.00 for 14 days.'", + "reason": "candidate_graph: no admissible candidate for question: 'How much will it cost him?'", "trace_hash": null }, { @@ -218,7 +218,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'The dog ends up having health problems and this requires 3 vet appointments, which cost $400 each.'", + "reason": "candidate_graph: no admissible candidate for question: 'How much did he pay in total?'", "trace_hash": null }, { @@ -229,7 +229,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'Two puppies, two kittens, and three parakeets were for sale at the pet shop.'", + "reason": "candidate_graph: no admissible candidate for question: 'If the cost of one parakeet was $10, what would it cost to purchase all of the pets for sale at the pet shop, in dollars?'", "trace_hash": null }, { @@ -240,7 +240,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'He bench presses 15 pounds for 10 reps and does 3 sets.'", + "reason": "candidate_graph: no admissible candidate for question: 'How much total weight does he move?'", "trace_hash": null }, { @@ -251,7 +251,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'Erica lives near a lake where most locals sell fish as their main source of income, earning $20 per kg of fish.'", + "reason": "candidate_graph: no admissible candidate for question: 'If Erica trawled 80 kg of fish in the past four months, not including today, how much money will she have earned in the past four months including today (assuming she sells all her fish)?'", "trace_hash": null }, { @@ -262,7 +262,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'Nicole collected 400 Pokemon cards.'", + "reason": "candidate_graph: no branch produced a solvable graph", "trace_hash": null }, { @@ -273,7 +273,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'Sidney does 20 jumping jacks on Monday, 36 on Tuesday, 40 on Wednesday, and 50 on Thursday.'", + "reason": "candidate_graph: no admissible candidate for question: 'How many jumping jacks did Brooke do?'", "trace_hash": null }, { @@ -284,7 +284,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'Lilibeth fills 6 baskets where each basket holds 50 strawberries.'", + "reason": "candidate_graph: no admissible candidate for question: \"If three of Lilibeth's friends pick the same amount as her, how many strawberries do Lilibeth and her friends pick in all?\"", "trace_hash": null }, { @@ -295,7 +295,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'Aaron and his brother Carson each saved up $40 to go to dinner.'", + "reason": "candidate_graph: no admissible candidate for statement: 'The bill is 3/4 of their total money.'", "trace_hash": null }, { @@ -306,7 +306,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'Malcolm has 240 followers on Instagram and 500 followers on Facebook.'", + "reason": "candidate_graph: no admissible candidate for question: 'How many followers does Malcolm have on all his social media?'", "trace_hash": null }, { @@ -317,7 +317,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'It cost $100,000 to open initially.'", + "reason": "candidate_graph: no admissible candidate for statement: 'He sells 150 tickets a day for $10 each.'", "trace_hash": null }, { @@ -328,7 +328,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'The cost of the keyboard was three times greater than the cost of the mouse.'", + "reason": "candidate_graph: no admissible candidate for question: 'If the mouse cost $16, how much did Fabian spent on his new accessories?'", "trace_hash": null }, { @@ -350,7 +350,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'Jeremie wants to go to an amusement park with 3 friends at the end of summer.'", + "reason": "candidate_graph: no admissible candidate for question: 'How much will it cost Jeremie and her 3 friends to go to the amusement park and buy each of them a set of snacks?'", "trace_hash": null }, { @@ -361,7 +361,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'He draws and colors 10 pictures.'", + "reason": "candidate_graph: no admissible candidate for question: 'How long does he spend on all the pictures?'", "trace_hash": null }, { @@ -372,7 +372,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'Rachel is 12 years old, and her grandfather is 7 times her age.'", + "reason": "candidate_graph: no admissible candidate for question: \"How old will Rachel's father be when she is 25 years old?\"", "trace_hash": null }, { @@ -383,7 +383,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'He can run 40 yards within 5 seconds.'", + "reason": "candidate_graph: no admissible candidate for question: 'If he can improve his speed by forty percent, how many yards will he be able to run within 10 seconds?'", "trace_hash": null }, { @@ -405,7 +405,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'She studied for 2 hours on Wednesday and three times as long on Thursday.'", + "reason": "candidate_graph: no admissible candidate for question: 'How much time did she spend studying in total during the five days?'", "trace_hash": null }, { @@ -416,7 +416,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'Michael wants to lose 10 pounds by June.'", + "reason": "candidate_graph: no admissible candidate for question: 'How much weight does he have to lose in May to meet his goal?'", "trace_hash": null }, { @@ -427,7 +427,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'There are three times that many girls at a party being held on the second floor of the building.'", + "reason": "candidate_graph: no admissible candidate for question: 'How many ladies are on the two floors in total?'", "trace_hash": null }, { @@ -438,7 +438,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'Orlando gained 5 pounds.'", + "reason": "candidate_graph: no admissible candidate for question: 'How much weight, in pounds, did the three family members gain at their reunion?'", "trace_hash": null }, { @@ -449,7 +449,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'He now has 2 horses, 5 dogs, 7 cats, 3 turtles, and 1 goat.'", + "reason": "candidate_graph: no admissible candidate for question: 'In total, how many legs do his animals have?'", "trace_hash": null }, { @@ -460,7 +460,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'The guests eat all of 1 pan, and 75% of the 2nd pan.'", + "reason": "candidate_graph: no admissible candidate for question: 'If all but 4 of the guests eat the brownies ala mode (with 2 scoops of vanilla ice cream), how many tubs of ice cream did the guests eat?'", "trace_hash": null }, { @@ -482,7 +482,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'Her mother gave her an additional $4, and her father twice as much as her mother.'", + "reason": "candidate_graph: no admissible candidate for question: 'How much money will she be left with after the purchase?'", "trace_hash": null }, { @@ -493,7 +493,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'John invests in a bank and gets 10% simple interest.'", + "reason": "candidate_graph: no branch produced a solvable graph", "trace_hash": null }, { @@ -504,7 +504,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'Each survey has 10 questions.'", + "reason": "candidate_graph: no admissible candidate for question: 'How much money did he earn during these two days?'", "trace_hash": null }, { @@ -515,7 +515,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'Half of the students are girls, the other half are boys.'", + "reason": "candidate_graph: no admissible candidate for question: 'How many students own dogs?'", "trace_hash": null }, { @@ -526,7 +526,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'John bakes 12 coconut macaroons, each weighing 5 ounces.'", + "reason": "candidate_graph: no admissible candidate for question: 'What is the total weight, in ounces, of the remaining coconut macaroons?'", "trace_hash": null }, { @@ -548,7 +548,7 @@ "expected_unit": "", "outcome": "refused", "realized_prose": null, - "reason": "candidate_graph: no admissible candidate for statement: 'Malcolm is trying to find the fastest walk to school and is currently comparing two routes.'", + "reason": "candidate_graph: no admissible candidate for question: 'In minutes, how much longer was the second route compared with the first?'", "trace_hash": null }, { @@ -565,6 +565,46 @@ ], "probe": "gsm8k_train_sample_coverage", "refused_reasons_top": [ + { + "count": 4, + "reason": "candidate_graph: no branch produced a solvable graph" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: \"How many of Dennis's rocks are left?\"" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: \"How old will Rachel's father be when she is 25 years old?\"" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: \"If three of Lilibeth's friends pick the same amount as her, how " + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: \"What's the total number of video hours she has uploaded at the e" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: \"What's the total time he takes to reach the Bronx from Manhattan" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'How long does he spend on all the pictures?'" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'How many followers does Malcolm have on all his social media?'" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'How many jumping jacks did Brooke do?'" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'How many ladies are on the two floors in total?'" + }, { "count": 1, "reason": "candidate_graph: no admissible candidate for question: 'How many more apples would Martha need to give away to be left w" @@ -573,42 +613,102 @@ "count": 1, "reason": "candidate_graph: no admissible candidate for question: 'How many more boxes do they need if Francine has a total of 85 c" }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'How many pages do the books she reads now have?'" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'How many stop signs per mile did Rudolph encounter on his trip a" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'How many students own dogs?'" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'How much did he pay in total?'" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'How much money did he earn during these two days?'" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'How much money will she be left with after the purchase?'" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'How much time did she spend studying in total during the five da" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'How much total weight does he move?'" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'How much weight does he have to lose in May to meet his goal?'" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'How much weight, in pounds, did the three family members gain at" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'How much will it cost Jeremie and her 3 friends to go to the amu" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'How much will it cost him?'" + }, { "count": 1, "reason": "candidate_graph: no admissible candidate for question: 'If 50 beads are used to make one bracelet, how many bracelets wi" }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'If Erica trawled 80 kg of fish in the past four months, not incl" + }, { "count": 1, "reason": "candidate_graph: no admissible candidate for question: 'If Jen has 150 ducks, how many total birds does she have?'" }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'If all but 4 of the guests eat the brownies ala mode (with 2 sco" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'If he can improve his speed by forty percent, how many yards wil" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'If the cost of one parakeet was $10, what would it cost to purch" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'If the mouse cost $16, how much did Fabian spent on his new acce" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'If the student council sells the erasers for $0.75 each, how muc" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'In minutes, how much longer was the second route compared with t" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'In total, how many legs do his animals have?'" + }, + { + "count": 1, + "reason": "candidate_graph: no admissible candidate for question: 'What is the total weight, in ounces, of the remaining coconut ma" + }, { "count": 1, "reason": "candidate_graph: no admissible candidate for statement: \"In one hour, Addison mountain's temperature will decrease to 3/" }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: \"On Rudolph's car trip across town, he traveled 2 more than 5 mi" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'Aaron and his brother Carson each saved up $40 to go to dinner." - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'Alexa has a lemonade stand where she sells lemonade for $2 for " - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'Allison, a YouTuber, uploads 10 one-hour videos of food reviews" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'Each survey has 10 questions.'" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'Erica lives near a lake where most locals sell fish as their ma" - }, { "count": 1, "reason": "candidate_graph: no admissible candidate for statement: 'Every week, he gets 6 cards.'" @@ -619,72 +719,12 @@ }, { "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'Half of the students are girls, the other half are boys.'" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'He bench presses 15 pounds for 10 reps and does 3 sets.'" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'He can run 40 yards within 5 seconds.'" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'He draws and colors 10 pictures.'" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'He now has 2 horses, 5 dogs, 7 cats, 3 turtles, and 1 goat.'" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'He put all of them in his aquarium but his fish ate half of the" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'Her mother gave her an additional $4, and her father twice as m" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'He\u2019s charging $50.00 per day or $500.00 for 14 days.'" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'It cost $100,000 to open initially.'" + "reason": "candidate_graph: no admissible candidate for statement: 'He sells 150 tickets a day for $10 each.'" }, { "count": 1, "reason": "candidate_graph: no admissible candidate for statement: 'It is a 2-hour drive each way.'" }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'Jeremie wants to go to an amusement park with 3 friends at the " - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'John bakes 12 coconut macaroons, each weighing 5 ounces.'" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'John invests in a bank and gets 10% simple interest.'" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'Lilibeth fills 6 baskets where each basket holds 50 strawberrie" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'Malcolm has 240 followers on Instagram and 500 followers on Fac" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'Malcolm is trying to find the fastest walk to school and is cur" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'Mandy started reading books with only 8 pages when she was 6 ye" - }, { "count": 1, "reason": "candidate_graph: no admissible candidate for statement: 'Marion has 1/4 more than what Yun currently has, plus 7.'" @@ -695,63 +735,11 @@ }, { "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'Michael wants to lose 10 pounds by June.'" + "reason": "candidate_graph: no admissible candidate for statement: 'She gives 1/4 of that to a friend.'" }, { "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'Nicole collected 400 Pokemon cards.'" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'Orlando gained 5 pounds.'" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'Rachel is 12 years old, and her grandfather is 7 times her age." - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'She splits it up into 25-foot sections.'" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'She studied for 2 hours on Wednesday and three times as long on" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'Sidney does 20 jumping jacks on Monday, 36 on Tuesday, 40 on We" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'The cost of the keyboard was three times greater than the cost " - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'The dog ends up having health problems and this requires 3 vet " - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'The guests eat all of 1 pan, and 75% of the 2nd pan.'" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'The local bookstore donated 48 boxes of erasers.'" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'There are three times that many girls at a party being held on " - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'Tina makes $18.00 an hour.'" - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'Traveling from Manhattan to the Bronx, Andrew rides the subway " - }, - { - "count": 1, - "reason": "candidate_graph: no admissible candidate for statement: 'Two puppies, two kittens, and three parakeets were for sale at " + "reason": "candidate_graph: no admissible candidate for statement: 'The bill is 3/4 of their total money.'" } ], "sample_path": "evals/gsm8k_math/train_sample/v1/cases.jsonl", diff --git a/evals/math_capability_axes/G3_numerics/v1/report.json b/evals/math_capability_axes/G3_numerics/v1/report.json index 15add95e..9269e2ed 100644 --- a/evals/math_capability_axes/G3_numerics/v1/report.json +++ b/evals/math_capability_axes/G3_numerics/v1/report.json @@ -270,7 +270,7 @@ "class": "refuse_money_precision", "expected_answer": 0, "expected_outcome": "refused", - "reason": "candidate_graph: no admissible candidate for statement: 'Bob has $40.000 in savings.'", + "reason": "candidate_graph: no branch produced a solvable graph", "trace_hash": null, "verdict": "refused" }, @@ -282,7 +282,7 @@ "class": "refuse_money_precision", "expected_answer": 0, "expected_outcome": "refused", - "reason": "candidate_graph: no admissible candidate for statement: 'Sarah has $1.2345 in change.'", + "reason": "candidate_graph: no branch produced a solvable graph", "trace_hash": null, "verdict": "refused" }, @@ -306,7 +306,7 @@ "class": "refuse_unknown_compound", "expected_answer": 0, "expected_outcome": "refused", - "reason": "candidate_graph: no admissible candidate for statement: 'Bob has five-and-a-half apples.'", + "reason": "candidate_graph: no branch produced a solvable graph", "trace_hash": null, "verdict": "refused" }, @@ -318,7 +318,7 @@ "class": "refuse_unknown_compound", "expected_answer": 0, "expected_outcome": "refused", - "reason": "candidate_graph: no admissible candidate for statement: 'Bob has gobbledy-gook apples.'", + "reason": "candidate_graph: no branch produced a solvable graph", "trace_hash": null, "verdict": "refused" }, @@ -330,7 +330,7 @@ "class": "refuse_percentage", "expected_answer": 0, "expected_outcome": "refused", - "reason": "candidate_graph: no admissible candidate for statement: 'Bob has 50% apples.'", + "reason": "candidate_graph: no branch produced a solvable graph", "trace_hash": null, "verdict": "refused" } diff --git a/generate/math_candidate_graph.py b/generate/math_candidate_graph.py index 33228b17..08f77d7b 100644 --- a/generate/math_candidate_graph.py +++ b/generate/math_candidate_graph.py @@ -164,7 +164,9 @@ def _filtered_statement_choices(sentence: str) -> list[SentenceChoice]: return out[:MAX_CANDIDATES_PER_SENTENCE] -def _filtered_question_choices(sentence: str) -> list[CandidateUnknown]: +def _filtered_question_choices( + sentence: str, problem_text: str | None = None +) -> list[CandidateUnknown]: """Return all admissible question candidates after the question- specific structural check. @@ -175,15 +177,19 @@ def _filtered_question_choices(sentence: str) -> list[CandidateUnknown]: regex misses (11 of 38 GSM8K train_sample post-Phase-D question refusals share this shape). Skip-only safety: if the stripped question still produces no admissible candidate, refuse as before. + + ADR-0163.D.4 — ``problem_text`` is the full problem text used by + the new question-grammar extensions for pronoun-entity resolution + (Pattern C). When None, pronoun-entity branches refuse. """ out: list[CandidateUnknown] = [] - for qc in extract_question_candidates(sentence): + for qc in extract_question_candidates(sentence, problem_text): if _question_admissible(qc): out.append(qc) if not out: stripped = _strip_conditional_prefix(sentence) if stripped is not None and stripped != sentence: - for qc in extract_question_candidates(stripped): + for qc in extract_question_candidates(stripped, problem_text): if _question_admissible(qc): out.append(qc) return out[:MAX_CANDIDATES_PER_SENTENCE] @@ -508,7 +514,7 @@ def parse_and_solve(text: str) -> CandidateGraphResult: ) per_sentence_choices.append(_collapse_per_sentence_ties(choices)) - question_choices = _filtered_question_choices(question_sentences[0]) + question_choices = _filtered_question_choices(question_sentences[0], text) if not question_choices: return CandidateGraphResult( answer=None, selected_graph=None, diff --git a/generate/math_candidate_parser.py b/generate/math_candidate_parser.py index 3891cedd..bcff21b0 100644 --- a/generate/math_candidate_parser.py +++ b/generate/math_candidate_parser.py @@ -748,6 +748,9 @@ class CandidateUnknown: source_span: str matched_unit_token: str matched_entity_token: str | None # None for total-across questions + # ADR-0163.D.4 — Pattern B comparative marker ("how many more X"). + # Default False keeps existing constructions byte-identical. + comparative_marker: bool = False _Q_ENTITY_RE: Final[re.Pattern[str]] = re.compile( @@ -764,13 +767,21 @@ _Q_TOTAL_RE: Final[re.Pattern[str]] = re.compile( ) -def extract_question_candidates(sentence: str) -> list[CandidateUnknown]: +def extract_question_candidates( + sentence: str, problem_text: str | None = None +) -> list[CandidateUnknown]: """Return all admissible question candidates for ``sentence``. Tries the total-across pattern FIRST (same specificity order as legacy math_parser). The entity-pattern's widened regex would otherwise capture "they" as an entity name. + ADR-0163.D.4 — ``problem_text`` is the full problem text used by + pronoun-entity resolution (Pattern C). When None, pronoun-entity + branches refuse rather than guess. This keeps the function pure + and deterministic: same (sentence, problem_text) → byte-identical + output. + Empty list if no shape matches. """ s = sentence.strip() @@ -803,6 +814,20 @@ def extract_question_candidates(sentence: str) -> list[CandidateUnknown]: matched_entity_token=m.group("entity"), ) ) + return out + + # ADR-0163.D.4 — Pattern A: mass-noun question + out.extend(_pattern_a_mass_noun_candidates(sentence, problem_text)) + if out: + return out + + # ADR-0163.D.4 — Pattern B: comparative quantifier ("how many more") + out.extend(_pattern_b_comparative_candidates(sentence, problem_text)) + if out: + return out + + # ADR-0163.D.4 — Pattern C: pronoun-entity in non-"have" verb position + out.extend(_pattern_c_pronoun_verb_candidates(sentence, problem_text)) return out @@ -2230,3 +2255,312 @@ def extract_conditional_op_question_candidates( source_span=sentence, ) ] + + +# --------------------------------------------------------------------------- +# ADR-0163.D.4 — Question-grammar extensions +# --------------------------------------------------------------------------- +# +# Three new question shapes extracted from the GSM8K train_sample +# post-Phase-D refusal taxonomy. Each is structurally narrow and is +# protected by three independent wrong=0 safety nets: +# +# 1. Regex narrowness — required mass-noun whitelist / comparative +# "more" anchor / pronoun-or-named-entity slot + closed action-verb +# set. Open-ended question shapes do not match. +# 2. Pronoun resolver refuse-on-ambiguity — if a problem text has two +# distinct female (or male) names, "she"/"he" cannot resolve to a +# single entity and the extraction emits no candidate. +# 3. Downstream multi-branch decision rule (math_candidate_graph) — +# when multiple admissible parses produce different answers, the +# case refuses rather than picks one. +# +# The _MASS_NOUNS whitelist is HARD: extending it requires a separate +# ADR. The pronoun gender name lists are SMALL and DOCUMENTED, sourced +# from GSM8K train_sample observation. + +# Pattern A — mass-noun question shape. +# Narrow whitelist of high-signal mass nouns observed in GSM8K +# train_sample. Extending this set is a separate ADR; this prevents +# incremental drift into "anything that looks like a mass noun." +_MASS_NOUNS: Final[frozenset[str]] = frozenset({ + "money", "profit", "interest", "income", + "savings", "cost", "amount", "total", +}) + +_MASS_NOUN_PATTERN: Final[str] = ( + r"(?:" + "|".join(sorted(_MASS_NOUNS, key=len, reverse=True)) + r")" +) + +# Subject pronouns for Pattern C resolution. +_Q_SUBJECT_PRONOUN: Final[str] = r"(?:she|he|they|it)" + +# Entity slot widened to also admit a subject pronoun (resolved +# downstream via _resolve_pronoun_entity). +_Q_ENTITY_OR_PRONOUN: Final[str] = rf"(?:{_ENTITY}|{_Q_SUBJECT_PRONOUN})" + +# Pattern A verb set: what the entity DOES with the mass noun. +_PATTERN_A_VERBS: Final[str] = ( + r"(?:make|makes|made|earn|earns|earned|save|saved|saves|" + r"spend|spends|spent|cost|costs|need|needs|have|has|had|" + r"gain|gains|gained|pay|paid|pays)" +) + +# Pattern B verb set: comparative-need style verbs. +_PATTERN_B_VERBS: Final[str] = ( + r"(?:need|needs|needed|require|requires|required|" + r"have|has|had|gain|gains|gained)" +) + +# Pattern C verb set: action verbs admitted in non-"have" pronoun +# position. Closed whitelist of high-signal action verbs observed in +# GSM8K train_sample. Conservative — extending this requires evidence +# from refused cases. +_PATTERN_C_VERBS: Final[str] = ( + r"(?:need|needs|sell|sells|make|makes|pick|picks|" + r"buy|buys|use|uses|want|wants)" +) + + +# Optional "of " qualifier between unit and aux. Bounded NP: +# bare noun, " ", or two-word compound. The qualifier +# itself does not flow into the candidate; the unit token alone is +# the canonical unit (matching the math_parser convention for +# "cups of lemonade" / "boxes of crayons"). +_Q_OF_NP_TAIL: Final[str] = r"(?:\s+of\s+\w+(?:\s+\w+)?)?" + + +_Q_MASS_NOUN_RE: Final[re.Pattern[str]] = re.compile( + r"^How\s+much\s+" + rf"(?P{_MASS_NOUN_PATTERN})" + rf"{_Q_OF_NP_TAIL}\s+" + r"(?:will|did|does|do|would)\s+" + rf"(?P{_Q_ENTITY_OR_PRONOUN})\s+" + rf"(?:have\s+earned\s+|be\s+able\s+to\s+)?{_PATTERN_A_VERBS}" + r"(?:\s+.*?)?\??\s*$", + flags=re.IGNORECASE, +) + + +_Q_COMPARATIVE_RE: Final[re.Pattern[str]] = re.compile( + r"^How\s+many\s+more\s+" + r"(?P\w+)" + rf"{_Q_OF_NP_TAIL}\s+" + r"(?:does|do|would|will|did)\s+" + rf"(?P{_Q_ENTITY_OR_PRONOUN})\s+" + rf"{_PATTERN_B_VERBS}" + r"(?:\s+.*?)?\??\s*$", + flags=re.IGNORECASE, +) + + +# Pattern C — pronoun entity in non-"have" verb position. +# The verb slot is a NARROW closed set of action verbs; an optional +# "to " infinitive tail consumes constructions like "need to sell". +_Q_PRONOUN_VERB_RE: Final[re.Pattern[str]] = re.compile( + r"^How\s+many\s+(?P\w+)" + rf"{_Q_OF_NP_TAIL}\s+" + r"(?:does|do|will|did|would)\s+" + rf"(?P{_Q_ENTITY_OR_PRONOUN})\s+" + rf"{_PATTERN_C_VERBS}" + r"(?:\s+to\s+\w+)?" + r"(?:\s+.*?)?\??\s*$", + flags=re.IGNORECASE, +) + + +# Pronoun → name lookup lists. Closed-set whitelists sourced from +# GSM8K train_sample observation. Adding a name requires evidence +# from a refused case. Lower-cased; names are matched case-insensitively +# against problem-text mentions. +_FEMALE_NAMES: Final[frozenset[str]] = frozenset({ + "alexa", "alice", "amy", "ann", "anna", "barbara", "betty", + "carol", "carolyn", "christine", "cindy", "claire", "cynthia", + "deborah", "diana", "donna", "dorothy", "elizabeth", "ella", + "emily", "emma", "erica", "francine", "helen", "jane", "janet", + "jen", "jennifer", "jessica", "joyce", "judith", "julie", "karen", + "kate", "kathleen", "kelly", "laura", "linda", "lisa", "lilibeth", + "lori", "mandy", "marie", "martha", "marnie", "mary", "melissa", + "nancy", "nicole", "pamela", "patricia", "rachel", "rebecca", + "ruth", "sandra", "sarah", "sharon", "shirley", "stephanie", + "susan", "tina", "virginia", +}) + +_MALE_NAMES: Final[frozenset[str]] = frozenset({ + "aaron", "adam", "alan", "albert", "andrew", "anthony", "arthur", + "benjamin", "bob", "brian", "bruce", "carl", "carson", "charles", + "christopher", "daniel", "david", "dennis", "donald", "douglas", + "edward", "eric", "ethan", "eugene", "fabian", "frank", "gary", + "george", "gerald", "gregory", "harold", "harry", "henry", "jack", + "james", "jason", "jeffrey", "jeremy", "jerry", "jesse", "john", + "jonathan", "joseph", "joshua", "keith", "kenneth", "kevin", + "larry", "lawrence", "mark", "matthew", "michael", "nathan", + "nicholas", "patrick", "paul", "peter", "philip", "ralph", + "raymond", "richard", "robert", "roger", "ronald", "ryan", + "samuel", "scott", "sean", "stephen", "steven", "thomas", + "timothy", "tom", "walter", "wayne", "william", +}) + + +# Title-cased proper-noun mention extractor. Matches any sequence +# starting with an upper-case letter followed by lower-case letters, +# avoiding all-caps tokens (acronyms) which are unlikely to be names. +_PROPER_NOUN_MENTION_RE: Final[re.Pattern[str]] = re.compile( + r"\b([A-Z][a-z]+)\b" +) + + +def _resolve_pronoun_entity( + pronoun: str, problem_text: str | None +) -> str | None: + """Resolve a subject pronoun to a single unambiguous named entity. + + Pure, deterministic, no global state. Returns the canonical + Title-cased entity name when a SINGLE unambiguous match exists; + returns ``None`` on ambiguity, no-match, or empty problem_text. + + Heuristic limits: + - ``she``/``her`` match female names from :data:`_FEMALE_NAMES`. + - ``he``/``his``/``him`` match male names from :data:`_MALE_NAMES`. + - ``they`` is plural; refuses (no single-entity resolution). + - ``it`` is neuter; refuses (not safely resolvable from name lists). + - If the problem text contains >1 distinct female (or male) name, + "she" (or "he") cannot resolve unambiguously → refuse. + + Refuse-on-ambiguity preserves wrong=0: better to refuse a question + than admit one with the wrong entity. + """ + if not problem_text: + return None + p = pronoun.lower() + if p in ("they", "it"): + return None # Plural/neuter — outside scope. + if p in ("she", "her"): + whitelist = _FEMALE_NAMES + elif p in ("he", "his", "him"): + whitelist = _MALE_NAMES + else: + return None + distinct: list[str] = [] + for m in _PROPER_NOUN_MENTION_RE.finditer(problem_text): + name = m.group(1) + if name.lower() not in whitelist: + continue + if name not in distinct: + distinct.append(name) + if len(distinct) != 1: + # Zero matches → no candidate; >1 distinct → ambiguous → refuse. + return None + return distinct[0] + + +def _resolve_question_entity( + raw_entity: str, problem_text: str | None +) -> tuple[str, str] | None: + """Return ``(canonical_entity, matched_entity_token)`` or None. + + Wraps :func:`_resolve_pronoun_entity` for pronoun slots; passes + proper-noun entities through unchanged. ``None`` means the + candidate should not be emitted (pronoun unresolvable). + + For a resolved pronoun the ``matched_entity_token`` is the LITERAL + surface pronoun ("she", "he", ...). The downstream question- + admissibility check requires the token to appear in the source- + span (the question sentence); the resolved canonical name does not. + """ + lower = raw_entity.strip().lower() + if lower in ("she", "her", "he", "his", "him", "they", "it"): + resolved = _resolve_pronoun_entity(lower, problem_text) + if resolved is None: + return None + return _normalize_entity(resolved), raw_entity + return _normalize_entity(raw_entity), raw_entity + + +def _pattern_a_mass_noun_candidates( + sentence: str, problem_text: str | None +) -> list[CandidateUnknown]: + """Pattern A — "How much MASS_NOUN does ENTITY VERB ..." question.""" + s = sentence.strip() + m = _Q_MASS_NOUN_RE.match(s) + if m is None: + return [] + unit_raw = m.group("unit") + # Mass nouns are pre-whitelisted; preserve literal token for + # downstream grounding (no hidden normalization). + unit = unit_raw.lower() + raw_entity = m.group("entity") + resolved = _resolve_question_entity(raw_entity, problem_text) + if resolved is None: + return [] + entity, entity_token = resolved + return [ + CandidateUnknown( + unknown=Unknown(entity=entity, unit=unit), + source_span=sentence, + matched_unit_token=unit_raw, + matched_entity_token=entity_token, + ) + ] + + +def _pattern_b_comparative_candidates( + sentence: str, problem_text: str | None +) -> list[CandidateUnknown]: + """Pattern B — "How many more UNIT does ENTITY VERB ..." question. + + wrong=0 GATE: comparative quantification ("how many MORE X are + needed beyond current state") is structurally distinct from the + plain ``How many X does ENTITY have?`` shape the existing solver + resolves. If the parser emits a CandidateUnknown for Pattern B, + the downstream solver computes the entity's current total — which + is the WRONG answer for a comparative question (off by the missing + delta). Until the solver gains comparative semantics (D.5 + follow-up), this extractor recognises the shape but emits NO + candidate, forcing a clean refusal. The regex + marker field + + detection helper are retained for D.5 wiring. + """ + s = sentence.strip() + if _Q_COMPARATIVE_RE.match(s) is None: + return [] + # Detection-only: see docstring. D.5 will add solver semantics. + return [] + + +def _pattern_b_detects(sentence: str) -> bool: + """ADR-0163.D.4 — pure-grammar Pattern B detector for tests. + + Returns True iff the comparative-quantifier shape matches. Has + no side effects on candidate emission. + """ + return _Q_COMPARATIVE_RE.match(sentence.strip()) is not None + + +def _pattern_c_pronoun_verb_candidates( + sentence: str, problem_text: str | None +) -> list[CandidateUnknown]: + """Pattern C — "How many UNIT does PRONOUN VERB [to VERB2] ..." question. + + Admits the entity slot to either a proper-noun ENTITY or a subject + pronoun resolved via :func:`_resolve_pronoun_entity`. Narrow verb + whitelist (:data:`_PATTERN_C_VERBS`) bounds the question shape. + """ + s = sentence.strip() + m = _Q_PRONOUN_VERB_RE.match(s) + if m is None: + return [] + unit_raw = m.group("unit") + unit = _canonicalize_unit(unit_raw) + raw_entity = m.group("entity") + resolved = _resolve_question_entity(raw_entity, problem_text) + if resolved is None: + return [] + entity, entity_token = resolved + return [ + CandidateUnknown( + unknown=Unknown(entity=entity, unit=unit), + source_span=sentence, + matched_unit_token=unit_raw, + matched_entity_token=entity_token, + ) + ] diff --git a/tests/test_adr_0163_d4_question_grammar.py b/tests/test_adr_0163_d4_question_grammar.py new file mode 100644 index 00000000..1f43449e --- /dev/null +++ b/tests/test_adr_0163_d4_question_grammar.py @@ -0,0 +1,446 @@ +"""ADR-0163.D.4 — question grammar extension tests. + +Three new question shapes: + Pattern A — "How much MASS_NOUN does ENTITY VERB ..." + Pattern B — "How many more UNIT does ENTITY VERB ..." (comparative_marker=True) + Pattern C — "How many UNIT does PRONOUN VERB [to VERB2] ..." + +wrong=0 safety nets exercised here: + - positive cases: each pattern admits expected GSM8K shapes + - negative cases: each pattern refuses adjacent shapes that would + over-admit a wrong entity/unit/answer + - pronoun resolver determinism: same (pronoun, problem_text) → + byte-identical resolution + - pronoun resolver refuse-on-ambiguity: multiple distinct + female/male names → no admission + - solver-layer multi-branch refusal: when grammar admits but + candidates disagree on the answer, downstream refuses +""" + +from __future__ import annotations + +import pytest + +from generate.math_candidate_parser import ( + CandidateUnknown, + _pattern_b_detects, + _resolve_pronoun_entity, + extract_question_candidates, +) +from generate.math_candidate_graph import _filtered_question_choices + + +# --------------------------------------------------------------------------- +# Pattern A — mass-noun questions +# --------------------------------------------------------------------------- + + +class TestPatternAMassNoun: + """How much MASS_NOUN does ENTITY VERB ...?""" + + def test_admits_money_make_named_entity(self) -> None: + out = extract_question_candidates( + "How much money does Tina make?" + ) + assert len(out) == 1 + assert out[0].unknown.unit == "money" + assert out[0].unknown.entity == "Tina" + + def test_admits_money_make_pronoun_with_problem_text(self) -> None: + problem = "Tina makes $18.00 an hour. How much money does she make?" + out = extract_question_candidates( + "How much money does she make?", problem_text=problem, + ) + assert len(out) == 1 + assert out[0].unknown.entity == "Tina" + assert out[0].unknown.unit == "money" + + def test_admits_profit_and_savings_and_cost(self) -> None: + for noun, verb in ( + ("profit", "make"), + ("savings", "have"), + ("cost", "pay"), + ("income", "earn"), + ): + out = extract_question_candidates( + f"How much {noun} does Bob {verb}?" + ) + assert len(out) == 1, (noun, verb) + assert out[0].unknown.unit == noun + + def test_admits_will_make(self) -> None: + problem = "Bob has $40. How much money will Bob make?" + out = extract_question_candidates( + "How much money will Bob make?", problem_text=problem, + ) + assert len(out) == 1 + assert out[0].unknown.entity == "Bob" + + def test_admits_with_tail(self) -> None: + out = extract_question_candidates( + "How much money does Tina earn over the weekend?" + ) + assert len(out) == 1 + assert out[0].unknown.unit == "money" + + # Negative cases — must NOT match. + + def test_refuses_non_whitelisted_mass_noun(self) -> None: + # "water" is mass but not in _MASS_NOUNS whitelist → refuse. + out = extract_question_candidates( + "How much water does Bob have?" + ) + # The standard _Q_ENTITY_RE also doesn't match (it wants "many"), + # so we expect zero admission. + assert out == [] + + def test_refuses_no_action_verb(self) -> None: + # No action verb in whitelist; bare prepositional phrase. + out = extract_question_candidates( + "How much money is in the bank?" + ) + assert out == [] + + def test_refuses_unresolvable_pronoun_no_problem_text(self) -> None: + # Pronoun + no problem_text → cannot resolve → refuse. + out = extract_question_candidates( + "How much money does she make?" + ) + assert out == [] + + def test_refuses_unresolvable_pronoun_ambiguous_problem_text(self) -> None: + # Two female names in problem text → "she" is ambiguous → refuse. + problem = "Alice has $20 and Mary has $30. How much money does she make?" + out = extract_question_candidates( + "How much money does she make?", problem_text=problem, + ) + assert out == [] + + +# --------------------------------------------------------------------------- +# Pattern B — comparative quantifier "how many more" +# --------------------------------------------------------------------------- + + +class TestPatternBComparative: + """How many more UNIT does ENTITY VERB ...? + + wrong=0 GATE: Pattern B is structurally detected by the grammar but + intentionally emits NO candidate until the solver gains comparative + semantics (D.5 follow-up). The plain "How many UNIT does ENTITY + have?" path would return the entity's current total — wrong for + "how many MORE are needed." These tests verify the detector + gate + behaviour. + """ + + # Grammar-level detection (regex match). + + def test_detects_boxes_need(self) -> None: + assert _pattern_b_detects("How many more boxes does Francine need?") + + def test_detects_pronoun_modal(self) -> None: + assert _pattern_b_detects("How many more apples would she need?") + + def test_detects_have_variant(self) -> None: + assert _pattern_b_detects("How many more cards does Bob have?") + + def test_detects_modal_will(self) -> None: + assert _pattern_b_detects("How many more apples will Bob gain?") + + def test_detects_did_variant(self) -> None: + assert _pattern_b_detects("How many more boxes did Bob need?") + + # Emission gate — must NOT emit a candidate (preserves wrong=0). + + def test_does_not_emit_candidate_yet(self) -> None: + out = extract_question_candidates( + "How many more boxes does Francine need?" + ) + assert out == [] + + def test_does_not_emit_for_resolvable_pronoun(self) -> None: + problem = "Martha has 20 apples. How many more apples would she need?" + out = extract_question_candidates( + "How many more apples would she need?", problem_text=problem, + ) + assert out == [] + + # Negative cases — detector must NOT match. + + def test_detector_refuses_no_more_keyword(self) -> None: + assert not _pattern_b_detects("How many boxes does Bob need?") + + def test_detector_refuses_outside_verb_whitelist(self) -> None: + assert not _pattern_b_detects("How many more boxes does Bob draw?") + + def test_emits_nothing_for_ambiguous_pronoun(self) -> None: + problem = "Alice has 5. Mary has 7. How many more apples does she need?" + out = extract_question_candidates( + "How many more apples does she need?", problem_text=problem, + ) + assert out == [] + + +# --------------------------------------------------------------------------- +# Pattern C — pronoun-or-entity + non-"have" verb +# --------------------------------------------------------------------------- + + +class TestPatternCPronounVerb: + """How many UNIT does PRONOUN/ENTITY VERB [to VERB2] ...?""" + + def test_admits_cups_does_she_need_to_sell(self) -> None: + problem = ( + "Alexa has a lemonade stand where she sells lemonade for $2 for " + "one cup. If she spent $20 on ingredients, how many cups of " + "lemonade does she need to sell to make a profit of $80?" + ) + out = extract_question_candidates( + "How many cups does she need to sell?", problem_text=problem, + ) + assert len(out) == 1 + assert out[0].unknown.entity == "Alexa" + assert out[0].unknown.unit == "cups" + + def test_admits_named_entity_make(self) -> None: + out = extract_question_candidates( + "How many bracelets does Marnie make?" + ) + assert len(out) == 1 + assert out[0].unknown.entity == "Marnie" + assert out[0].unknown.unit == "bracelets" + + def test_admits_pick_with_to_clause(self) -> None: + out = extract_question_candidates( + "How many berries does Bob need to pick?" + ) + assert len(out) == 1 + + def test_admits_pronoun_resolvable_he(self) -> None: + problem = "Fabian bought a mouse. How many mice does he buy?" + out = extract_question_candidates( + "How many mice does he buy?", problem_text=problem, + ) + assert len(out) == 1 + assert out[0].unknown.entity == "Fabian" + + def test_admits_will_make(self) -> None: + out = extract_question_candidates( + "How many bracelets will Marnie make?" + ) + assert len(out) == 1 + + # Negative cases. + + def test_refuses_outside_verb_whitelist(self) -> None: + # "draw" is not in Pattern C verbs. + out = extract_question_candidates( + "How many bracelets does Marnie draw?" + ) + assert out == [] + + def test_refuses_ambiguous_pronoun(self) -> None: + problem = "Alice has 5 cups. Mary has 7 cups. How many cups does she sell?" + out = extract_question_candidates( + "How many cups does she sell?", problem_text=problem, + ) + assert out == [] + + def test_refuses_neuter_pronoun(self) -> None: + # "it" cannot be resolved from name lists. + problem = "The box has 5 items. How many items does it sell?" + out = extract_question_candidates( + "How many items does it sell?", problem_text=problem, + ) + assert out == [] + + def test_refuses_plural_pronoun(self) -> None: + # "they" is plural; not a single-entity resolution. + problem = "The boys have 5 cups. How many cups do they sell?" + out = extract_question_candidates( + "How many cups do they sell?", problem_text=problem, + ) + # Pattern C path refuses; _Q_TOTAL_RE only matches with "have" + # (not "sell"), so total path is also out. + for c in out: + # If somehow admitted, the entity MUST not be a single name. + assert c.matched_entity_token != "they" + + +# --------------------------------------------------------------------------- +# Pronoun resolver — purity, determinism, ambiguity handling +# --------------------------------------------------------------------------- + + +class TestResolvePronounEntity: + def test_resolves_single_female_name(self) -> None: + text = "Tina makes $18 an hour. She works 5 days." + assert _resolve_pronoun_entity("she", text) == "Tina" + + def test_resolves_single_male_name(self) -> None: + text = "Fabian bought a mouse. He saved $40." + assert _resolve_pronoun_entity("he", text) == "Fabian" + + def test_refuses_ambiguous_two_female_names(self) -> None: + text = "Alice has 5 cups. Mary has 7 cups." + assert _resolve_pronoun_entity("she", text) is None + + def test_refuses_ambiguous_two_male_names(self) -> None: + text = "Bob saved $5. John saved $7." + assert _resolve_pronoun_entity("he", text) is None + + def test_refuses_no_matching_name(self) -> None: + # No name in the text matches the female whitelist. + text = "The dog ran fast." + assert _resolve_pronoun_entity("she", text) is None + + def test_refuses_plural_pronoun(self) -> None: + text = "Tina makes $18." + assert _resolve_pronoun_entity("they", text) is None + + def test_refuses_neuter_pronoun(self) -> None: + text = "The box has 5 items." + assert _resolve_pronoun_entity("it", text) is None + + def test_refuses_empty_problem_text(self) -> None: + assert _resolve_pronoun_entity("she", "") is None + assert _resolve_pronoun_entity("she", None) is None + + def test_deterministic_byte_identical(self) -> None: + text = "Tina makes $18.00 an hour. She works 5 days." + a = _resolve_pronoun_entity("she", text) + b = _resolve_pronoun_entity("she", text) + c = _resolve_pronoun_entity("she", text) + assert a == b == c == "Tina" + + def test_repeated_same_name_is_single_distinct(self) -> None: + # Same name mentioned multiple times → one distinct match. + text = "Tina makes $18. Tina works 5 days. Tina is great." + assert _resolve_pronoun_entity("she", text) == "Tina" + + def test_object_pronoun_her_him(self) -> None: + text = "Tina makes $18." + assert _resolve_pronoun_entity("her", text) == "Tina" + text2 = "Bob saved $5." + assert _resolve_pronoun_entity("him", text2) == "Bob" + + +# --------------------------------------------------------------------------- +# End-to-end via _filtered_question_choices (D.3 prefix-strip composes) +# --------------------------------------------------------------------------- + + +class TestComposesWithPrefixStrip: + """Verify Pattern A/B/C compose with Phase D.3 conditional-prefix strip.""" + + def test_d3_strip_then_pattern_a(self) -> None: + # "If she works 10 hours every day for 5 days, how much money does she make?" + problem = ( + "Tina makes $18.00 an hour. " + "If she works 10 hours every day for 5 days, " + "how much money does she make?" + ) + out = _filtered_question_choices( + "If she works 10 hours every day for 5 days, " + "how much money does she make?", + problem, + ) + assert len(out) >= 1 + assert any(c.unknown.entity == "Tina" for c in out) + + def test_d3_strip_then_pattern_b_detected_not_emitted(self) -> None: + # Pattern B is structurally detected but emission is gated. + # End-to-end: stripped sentence detects Pattern B; no candidate. + stripped = "How many more apples would she need?" + assert _pattern_b_detects(stripped) + problem = ( + "Martha has 20 apples. " + "If she gives away 5, how many more apples would she need?" + ) + out = _filtered_question_choices( + "If she gives away 5, how many more apples would she need?", + problem, + ) + # No comparative_marker candidate emitted. + assert not any(c.comparative_marker for c in out) + + def test_d3_strip_then_pattern_c(self) -> None: + problem = ( + "Alexa has a lemonade stand. " + "If she spent $20 on ingredients, how many cups does she need to sell?" + ) + out = _filtered_question_choices( + "If she spent $20 on ingredients, " + "how many cups does she need to sell?", + problem, + ) + assert len(out) >= 1 + assert any(c.unknown.entity == "Alexa" for c in out) + + +# --------------------------------------------------------------------------- +# Existing behavior preserved — CandidateUnknown construction unchanged +# --------------------------------------------------------------------------- + + +class TestCandidateUnknownDefaults: + def test_default_comparative_marker_false(self) -> None: + # Existing 200+ callers construct without comparative_marker. + from generate.math_problem_graph import Unknown + c = CandidateUnknown( + unknown=Unknown(entity="Bob", unit="apples"), + source_span="How many apples does Bob have?", + matched_unit_token="apples", + matched_entity_token="Bob", + ) + assert c.comparative_marker is False + + def test_q_entity_re_path_sets_marker_false(self) -> None: + out = extract_question_candidates( + "How many apples does Bob have?" + ) + assert len(out) == 1 + assert out[0].comparative_marker is False + + def test_q_total_re_path_sets_marker_false(self) -> None: + out = extract_question_candidates( + "How many apples do they have?" + ) + assert len(out) == 1 + assert out[0].comparative_marker is False + + +# --------------------------------------------------------------------------- +# wrong=0 invariant — solver-layer protection against mis-extraction +# --------------------------------------------------------------------------- + + +class TestWrongZeroInvariantUnderAmbiguity: + """If two entity-grounded candidates would yield different answers, + the downstream multi-branch decision rule refuses rather than picks + one. Pronoun mis-resolution → refuse via _resolve_pronoun_entity.""" + + def test_ambiguous_problem_pronoun_extraction_refuses(self) -> None: + # If the pronoun resolver returned the WRONG entity (Mary instead + # of Alice), the answer downstream would be wrong. The resolver + # refuses on ambiguity, so no candidate emits. + problem = ( + "Alice has 5 apples. Mary has 7 apples. " + "How much money does she make?" + ) + out = extract_question_candidates( + "How much money does she make?", problem_text=problem, + ) + assert out == [] + + def test_end_to_end_refuses_when_pronoun_ambiguous(self) -> None: + # End-to-end: parse_and_solve should NOT produce an answer for + # an ambiguous-pronoun problem. + from generate.math_candidate_graph import parse_and_solve + problem = ( + "Alice has 5 apples. Mary has 7 apples. " + "How many apples does she pick?" + ) + result = parse_and_solve(problem) + assert result.answer is None + assert result.refusal_reason is not None