diff --git a/evals/gsm8k_math/train_sample/v1/refusal_rescan_v4.json b/evals/gsm8k_math/train_sample/v1/refusal_rescan_v4.json index ad59f919..0527006f 100644 --- a/evals/gsm8k_math/train_sample/v1/refusal_rescan_v4.json +++ b/evals/gsm8k_math/train_sample/v1/refusal_rescan_v4.json @@ -8,7 +8,7 @@ "current_first_refusal": "Tina makes $18.00 an hour.", "current_outcome": "refused", "current_primary_barrier": "rate_earnings", - "current_refusal_reason": "no admissible candidate for statement: 'Tina makes $18.00 an hour.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'Tina makes $18.00 an hour.' (category=rate_with_currency)", "new_admission": false, "notes": "", "previous_first_refusal": "Tina makes $18.00 an hour.", @@ -21,7 +21,7 @@ "current_first_refusal": "She splits it up into 25-foot sections.", "current_outcome": "refused", "current_primary_barrier": "partition_divide", - "current_refusal_reason": "no admissible candidate for statement: 'She splits it up into 25-foot sections.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'She splits it up into 25-foot sections.' (category=discrete_count_statement)", "new_admission": false, "notes": "", "previous_first_refusal": "She splits it up into 25-foot sections.", @@ -29,13 +29,13 @@ "previous_primary_barrier": "partition_divide" }, { - "barrier_shifted": false, + "barrier_shifted": true, "case_id": "gsm8k-train-sample-v1-0003", - "current_first_refusal": "The local bookstore donated 48 boxes of erasers.", - "current_outcome": "refused", - "current_primary_barrier": "novel_initial_verb", - "current_refusal_reason": "no admissible candidate for statement: 'The local bookstore donated 48 boxes of erasers.'", - "new_admission": false, + "current_first_refusal": null, + "current_outcome": "admitted", + "current_primary_barrier": "admitted", + "current_refusal_reason": null, + "new_admission": true, "notes": "", "previous_first_refusal": "The local bookstore donated 48 boxes of erasers.", "previous_outcome": "refused", @@ -73,7 +73,7 @@ "current_first_refusal": "Mandy started reading books with only 8 pages when she was 6 years old.", "current_outcome": "refused", "current_primary_barrier": "temporal_age_anchor", - "current_refusal_reason": "no admissible candidate for statement: 'Mandy started reading books with only 8 pages when she was 6 years old.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'Mandy started reading books with only 8 pages when she was 6 years old.' (category=multiplicative_aggregation)", "new_admission": false, "notes": "", "previous_first_refusal": "Mandy started reading books with only 8 pages when she was 6 years old.", @@ -138,7 +138,7 @@ "current_first_refusal": "Alexa has a lemonade stand where she sells lemonade for $2 for one cup.", "current_outcome": "refused", "current_primary_barrier": "rate_price", - "current_refusal_reason": "no admissible candidate for statement: 'Alexa has a lemonade stand where she sells lemonade for $2 for one cup.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'Alexa has a lemonade stand where she sells lemonade for $2 for one cup.' (category=rate_with_currency)", "new_admission": false, "notes": "", "previous_first_refusal": "Alexa has a lemonade stand where she sells lemonade for $2 for one cup.", @@ -151,7 +151,7 @@ "current_first_refusal": "He put all of them in his aquarium but his fish ate half of them.", "current_outcome": "refused", "current_primary_barrier": "compound_statement", - "current_refusal_reason": "no admissible candidate for statement: 'He put all of them in his aquarium but his fish ate half of them.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'He put all of them in his aquarium but his fish ate half of them.' (category=descriptive_setup_no_quantity)", "new_admission": false, "notes": "", "previous_first_refusal": "He put all of them in his aquarium but his fish ate half of them.", @@ -164,7 +164,7 @@ "current_first_refusal": "Allison, a YouTuber, uploads 10 one-hour videos of food reviews each day to her channel.", "current_outcome": "refused", "current_primary_barrier": "compound_statement", - "current_refusal_reason": "no admissible candidate for statement: 'Allison, a YouTuber, uploads 10 one-hour videos of food reviews each day to her channel.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'Allison, a YouTuber, uploads 10 one-hour videos of food reviews each day to her channel.' (category=multiplicative_aggregation)", "new_admission": false, "notes": "", "previous_first_refusal": "Allison, a YouTuber, uploads 10 one-hour videos of food reviews each day to her channel.", @@ -190,7 +190,7 @@ "current_first_refusal": "Traveling from Manhattan to the Bronx, Andrew rides the subway for 10 hours, takes the train and rides for twice as much time as the subway ride, and then bikes the remaining distance for 8 hours.", "current_outcome": "refused", "current_primary_barrier": "compound_multi_event", - "current_refusal_reason": "no admissible candidate for statement: 'Traveling from Manhattan to the Bronx, Andrew rides the subway for 10 hours, takes the train and rides for twice as much time as the subway ride, and then bikes the remaining distance for 8 hours.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'Traveling from Manhattan to the Bronx, Andrew rides the subway for 10 hours, takes the train and rides for twice as much time as the subway ride, and then bikes the remaining distance for 8 hours.' (category=discrete_count_statement)", "new_admission": false, "notes": "", "previous_first_refusal": "Traveling from Manhattan to the Bronx, Andrew rides the subway for 10 hours, takes the train and rides for twice as much time as the subway ride, and then bikes the remaining distance for 8 hours.", @@ -203,7 +203,7 @@ "current_first_refusal": "On Rudolph's car trip across town, he traveled 2 more than 5 miles and encountered 3 less than 17 stop signs.", "current_outcome": "refused", "current_primary_barrier": "compound_statement", - "current_refusal_reason": "no admissible candidate for statement: \"On Rudolph's car trip across town, he traveled 2 more than 5 miles and encountered 3 less than 17 stop signs.\"", + "current_refusal_reason": "recognizer matched but produced no injection for statement: \"On Rudolph's car trip across town, he traveled 2 more than 5 miles and encountered 3 less than 17 stop signs.\" (category=discrete_count_statement)", "new_admission": false, "notes": "", "previous_first_refusal": "On Rudolph's car trip across town, he traveled 2 more than 5 miles and encountered 3 less than 17 stop signs.", @@ -216,7 +216,7 @@ "current_first_refusal": "He\u2019s charging $50.00 per day or $500.00 for 14 days.", "current_outcome": "refused", "current_primary_barrier": "conditional_branch", - "current_refusal_reason": "no admissible candidate for statement: 'He\u2019s charging $50.00 per day or $500.00 for 14 days.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'He\u2019s charging $50.00 per day or $500.00 for 14 days.' (category=temporal_aggregation)", "new_admission": false, "notes": "", "previous_first_refusal": "He\u2019s charging $50.00 per day or $500.00 for 14 days.", @@ -237,14 +237,14 @@ "previous_primary_barrier": "admitted" }, { - "barrier_shifted": false, + "barrier_shifted": true, "case_id": "gsm8k-train-sample-v1-0019", - "current_first_refusal": "The dog ends up having health problems and this requires 3 vet appointments, which cost $400 each.", + "current_first_refusal": "After the first appointment, John paid $100 for pet insurance that covers 80% of the subsequent visits.", "current_outcome": "refused", - "current_primary_barrier": "rate_price", - "current_refusal_reason": "no admissible candidate for statement: 'The dog ends up having health problems and this requires 3 vet appointments, which cost $400 each.'", + "current_primary_barrier": "percentage_rate", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'After the first appointment, John paid $100 for pet insurance that covers 80% of the subsequent visits.' (category=currency_amount)", "new_admission": false, - "notes": "", + "notes": "Reader now resolves the prior '$400 per vet appointment' barrier; first refusal moved to 'After the first appointment, John paid $100 for pet insurance that covers 80% of the subsequent visits' \u2014 needs percentage_rate (80% of a subsequent-visit set).", "previous_first_refusal": "The dog ends up having health problems and this requires 3 vet appointments, which cost $400 each.", "previous_outcome": "refused", "previous_primary_barrier": "rate_price" @@ -255,7 +255,7 @@ "current_first_refusal": "Two puppies, two kittens, and three parakeets were for sale at the pet shop.", "current_outcome": "refused", "current_primary_barrier": "multi_entity_initial", - "current_refusal_reason": "no admissible candidate for statement: 'Two puppies, two kittens, and three parakeets were for sale at the pet shop.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'Two puppies, two kittens, and three parakeets were for sale at the pet shop.' (category=discrete_count_statement)", "new_admission": false, "notes": "", "previous_first_refusal": "Two puppies, two kittens, and three parakeets were for sale at the pet shop.", @@ -263,13 +263,13 @@ "previous_primary_barrier": "multi_entity_initial" }, { - "barrier_shifted": false, + "barrier_shifted": true, "case_id": "gsm8k-train-sample-v1-0021", - "current_first_refusal": "He bench presses 15 pounds for 10 reps and does 3 sets.", - "current_outcome": "refused", - "current_primary_barrier": "distributive_multiply", - "current_refusal_reason": "no admissible candidate for statement: 'He bench presses 15 pounds for 10 reps and does 3 sets.'", - "new_admission": false, + "current_first_refusal": null, + "current_outcome": "admitted", + "current_primary_barrier": "admitted", + "current_refusal_reason": null, + "new_admission": true, "notes": "", "previous_first_refusal": "He bench presses 15 pounds for 10 reps and does 3 sets.", "previous_outcome": "refused", @@ -281,7 +281,7 @@ "current_first_refusal": "Erica lives near a lake where most locals sell fish as their main source of income, earning $20 per kg of fish.", "current_outcome": "refused", "current_primary_barrier": "context_filler", - "current_refusal_reason": "no admissible candidate for statement: 'Erica lives near a lake where most locals sell fish as their main source of income, earning $20 per kg of fish.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'Erica lives near a lake where most locals sell fish as their main source of income, earning $20 per kg of fish.' (category=rate_with_currency)", "new_admission": false, "notes": "", "previous_first_refusal": "Erica lives near a lake where most locals sell fish as their main source of income, earning $20 per kg of fish.", @@ -289,40 +289,40 @@ "previous_primary_barrier": "context_filler" }, { - "barrier_shifted": false, + "barrier_shifted": true, "case_id": "gsm8k-train-sample-v1-0023", - "current_first_refusal": "Nicole collected 400 Pokemon cards.", + "current_first_refusal": "Cindy collected twice as many, and Rex collected half of Nicole and Cindy's combined total.", "current_outcome": "refused", - "current_primary_barrier": "novel_initial_verb", - "current_refusal_reason": "no admissible candidate for statement: 'Nicole collected 400 Pokemon cards.'", + "current_primary_barrier": "compound_comparative", + "current_refusal_reason": "recognizer matched but produced no injection for statement: \"Cindy collected twice as many, and Rex collected half of Nicole and Cindy's combined total.\" (category=descriptive_setup_no_quantity)", "new_admission": false, - "notes": "", + "notes": "Reader now resolves 'Nicole collected 400 Pokemon cards' (initial); refuses on 'Cindy collected twice as many, and Rex collected half of Nicole and Cindy's combined total' \u2014 needs compound_comparative ('twice as many') plus fraction_operand ('half of combined').", "previous_first_refusal": "Nicole collected 400 Pokemon cards.", "previous_outcome": "refused", "previous_primary_barrier": "novel_initial_verb" }, { - "barrier_shifted": false, + "barrier_shifted": true, "case_id": "gsm8k-train-sample-v1-0024", - "current_first_refusal": "Sidney does 20 jumping jacks on Monday, 36 on Tuesday, 40 on Wednesday, and 50 on Thursday.", - "current_outcome": "refused", - "current_primary_barrier": "multi_day_accumulation", - "current_refusal_reason": "no admissible candidate for statement: 'Sidney does 20 jumping jacks on Monday, 36 on Tuesday, 40 on Wednesday, and 50 on Thursday.'", - "new_admission": false, + "current_first_refusal": null, + "current_outcome": "admitted", + "current_primary_barrier": "admitted", + "current_refusal_reason": null, + "new_admission": true, "notes": "", "previous_first_refusal": "Sidney does 20 jumping jacks on Monday, 36 on Tuesday, 40 on Wednesday, and 50 on Thursday.", "previous_outcome": "refused", "previous_primary_barrier": "multi_day_accumulation" }, { - "barrier_shifted": false, + "barrier_shifted": true, "case_id": "gsm8k-train-sample-v1-0025", - "current_first_refusal": "Lilibeth fills 6 baskets where each basket holds 50 strawberries.", + "current_first_refusal": "If three of Lilibeth's friends pick the same amount as her, how many strawberries do Lilibeth and her friends pick in all?", "current_outcome": "refused", "current_primary_barrier": "distributive_multiply", - "current_refusal_reason": "no admissible candidate for statement: 'Lilibeth fills 6 baskets where each basket holds 50 strawberries.'", + "current_refusal_reason": "no admissible candidate for question: \"If three of Lilibeth's friends pick the same amount as her, how many strawberries do Lilibeth and her friends pick in all?\"", "new_admission": false, - "notes": "", + "notes": "Reader now resolves 'Lilibeth fills 6 baskets where each basket holds 50 strawberries'; refuses on the question 'If three of Lilibeth's friends pick the same amount as her, how many ... in all?' \u2014 needs distributive_multiply (3 friends x her amount) over a total question.", "previous_first_refusal": "Lilibeth fills 6 baskets where each basket holds 50 strawberries.", "previous_outcome": "refused", "previous_primary_barrier": "distributive_multiply" @@ -341,14 +341,14 @@ "previous_primary_barrier": "distributive_each_actor" }, { - "barrier_shifted": false, + "barrier_shifted": true, "case_id": "gsm8k-train-sample-v1-0027", - "current_first_refusal": "Malcolm has 240 followers on Instagram and 500 followers on Facebook.", + "current_first_refusal": "The number of followers he has on Twitter is half the number of followers he has on Instagram and Facebook combined.", "current_outcome": "refused", - "current_primary_barrier": "multi_attribute_accumulation", - "current_refusal_reason": "no admissible candidate for statement: 'Malcolm has 240 followers on Instagram and 500 followers on Facebook.'", + "current_primary_barrier": "fraction_operand", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'The number of followers he has on Twitter is half the number of followers he has on Instagram and Facebook combined.' (category=descriptive_setup_no_quantity)", "new_admission": false, - "notes": "", + "notes": "Reader now resolves 'Malcolm has 240 followers on Instagram and 500 on Facebook'; refuses on 'The number of followers he has on Twitter is half the number ... on Instagram and Facebook combined' \u2014 needs fraction_operand ('half of combined').", "previous_first_refusal": "Malcolm has 240 followers on Instagram and 500 followers on Facebook.", "previous_outcome": "refused", "previous_primary_barrier": "multi_attribute_accumulation" @@ -359,7 +359,7 @@ "current_first_refusal": "It cost $100,000 to open initially.", "current_outcome": "refused", "current_primary_barrier": "novel_initial_form", - "current_refusal_reason": "no admissible candidate for statement: 'It cost $100,000 to open initially.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'It cost $100,000 to open initially.' (category=currency_amount)", "new_admission": false, "notes": "", "previous_first_refusal": "It cost $100,000 to open initially.", @@ -372,7 +372,7 @@ "current_first_refusal": "The cost of the keyboard was three times greater than the cost of the mouse.", "current_outcome": "refused", "current_primary_barrier": "compound_comparative", - "current_refusal_reason": "no admissible candidate for statement: 'The cost of the keyboard was three times greater than the cost of the mouse.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'The cost of the keyboard was three times greater than the cost of the mouse.' (category=discrete_count_statement)", "new_admission": false, "notes": "", "previous_first_refusal": "The cost of the keyboard was three times greater than the cost of the mouse.", @@ -398,7 +398,7 @@ "current_first_refusal": "Jeremie wants to go to an amusement park with 3 friends at the end of summer.", "current_outcome": "refused", "current_primary_barrier": "context_filler", - "current_refusal_reason": "no admissible candidate for statement: 'Jeremie wants to go to an amusement park with 3 friends at the end of summer.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'Jeremie wants to go to an amusement park with 3 friends at the end of summer.' (category=discrete_count_statement)", "new_admission": false, "notes": "", "previous_first_refusal": "Jeremie wants to go to an amusement park with 3 friends at the end of summer.", @@ -411,7 +411,7 @@ "current_first_refusal": "He draws and colors 10 pictures.", "current_outcome": "refused", "current_primary_barrier": "compound_statement", - "current_refusal_reason": "no admissible candidate for statement: 'He draws and colors 10 pictures.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'He draws and colors 10 pictures.' (category=discrete_count_statement)", "new_admission": false, "notes": "", "previous_first_refusal": "He draws and colors 10 pictures.", @@ -424,7 +424,7 @@ "current_first_refusal": "Rachel is 12 years old, and her grandfather is 7 times her age.", "current_outcome": "refused", "current_primary_barrier": "compound_statement", - "current_refusal_reason": "no admissible candidate for statement: 'Rachel is 12 years old, and her grandfather is 7 times her age.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'Rachel is 12 years old, and her grandfather is 7 times her age.' (category=discrete_count_statement)", "new_admission": false, "notes": "", "previous_first_refusal": "Rachel is 12 years old, and her grandfather is 7 times her age.", @@ -437,7 +437,7 @@ "current_first_refusal": "He can run 40 yards within 5 seconds.", "current_outcome": "refused", "current_primary_barrier": "capacity_rate", - "current_refusal_reason": "no admissible candidate for statement: 'He can run 40 yards within 5 seconds.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'He can run 40 yards within 5 seconds.' (category=discrete_count_statement)", "new_admission": false, "notes": "", "previous_first_refusal": "He can run 40 yards within 5 seconds.", @@ -463,7 +463,7 @@ "current_first_refusal": "She studied for 2 hours on Wednesday and three times as long on Thursday.", "current_outcome": "refused", "current_primary_barrier": "compound_comparative", - "current_refusal_reason": "no admissible candidate for statement: 'She studied for 2 hours on Wednesday and three times as long on Thursday.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'She studied for 2 hours on Wednesday and three times as long on Thursday.' (category=discrete_count_statement)", "new_admission": false, "notes": "", "previous_first_refusal": "She studied for 2 hours on Wednesday and three times as long on Thursday.", @@ -476,7 +476,7 @@ "current_first_refusal": "Michael wants to lose 10 pounds by June.", "current_outcome": "refused", "current_primary_barrier": "goal_statement", - "current_refusal_reason": "no admissible candidate for statement: 'Michael wants to lose 10 pounds by June.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'Michael wants to lose 10 pounds by June.' (category=discrete_count_statement)", "new_admission": false, "notes": "", "previous_first_refusal": "Michael wants to lose 10 pounds by June.", @@ -489,7 +489,7 @@ "current_first_refusal": "There are three times that many girls at a party being held on the second floor of the building.", "current_outcome": "refused", "current_primary_barrier": "compound_comparative", - "current_refusal_reason": "no admissible candidate for statement: 'There are three times that many girls at a party being held on the second floor of the building.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'There are three times that many girls at a party being held on the second floor of the building.' (category=discrete_count_statement)", "new_admission": false, "notes": "Sentence 1 (novel_initial_form) now resolves via S.4's _INITIAL_THERE_ARE_PREFIX_RE to (building, 100, ladies). Refuses on sentence 2: 'There are three times that many girls at a party being held on the second floor of the building' \u2014 needs compound_comparative parsing ('three times that many') with cross-sentence reference to the prior 100.", "previous_first_refusal": "In a building, there are a hundred ladies on the first-floor studying.", @@ -502,7 +502,7 @@ "current_first_refusal": "Orlando gained 5 pounds.", "current_outcome": "refused", "current_primary_barrier": "novel_initial_verb", - "current_refusal_reason": "no admissible candidate for statement: 'Orlando gained 5 pounds.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'Orlando gained 5 pounds.' (category=discrete_count_statement)", "new_admission": false, "notes": "", "previous_first_refusal": "Orlando gained 5 pounds.", @@ -515,7 +515,7 @@ "current_first_refusal": "He now has 2 horses, 5 dogs, 7 cats, 3 turtles, and 1 goat.", "current_outcome": "refused", "current_primary_barrier": "multi_entity_initial", - "current_refusal_reason": "no admissible candidate for statement: 'He now has 2 horses, 5 dogs, 7 cats, 3 turtles, and 1 goat.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'He now has 2 horses, 5 dogs, 7 cats, 3 turtles, and 1 goat.' (category=discrete_count_statement)", "new_admission": false, "notes": "", "previous_first_refusal": "He now has 2 horses, 5 dogs, 7 cats, 3 turtles, and 1 goat.", @@ -528,7 +528,7 @@ "current_first_refusal": "The guests eat all of 1 pan, and 75% of the 2nd pan.", "current_outcome": "refused", "current_primary_barrier": "fraction_operand", - "current_refusal_reason": "no admissible candidate for statement: 'The guests eat all of 1 pan, and 75% of the 2nd pan.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'The guests eat all of 1 pan, and 75% of the 2nd pan.' (category=discrete_count_statement)", "new_admission": false, "notes": "", "previous_first_refusal": "The guests eat all of 1 pan, and 75% of the 2nd pan.", @@ -554,7 +554,7 @@ "current_first_refusal": "Her mother gave her an additional $4, and her father twice as much as her mother.", "current_outcome": "refused", "current_primary_barrier": "compound_comparative", - "current_refusal_reason": "no admissible candidate for statement: 'Her mother gave her an additional $4, and her father twice as much as her mother.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'Her mother gave her an additional $4, and her father twice as much as her mother.' (category=currency_amount)", "new_admission": false, "notes": "", "previous_first_refusal": "Her mother gave her an additional $4, and her father twice as much as her mother.", @@ -567,7 +567,7 @@ "current_first_refusal": "John invests in a bank and gets 10% simple interest.", "current_outcome": "refused", "current_primary_barrier": "percentage_rate", - "current_refusal_reason": "no admissible candidate for statement: 'John invests in a bank and gets 10% simple interest.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'John invests in a bank and gets 10% simple interest.' (category=discrete_count_statement)", "new_admission": false, "notes": "", "previous_first_refusal": "John invests in a bank and gets 10% simple interest.", @@ -580,7 +580,7 @@ "current_first_refusal": "Each survey has 10 questions.", "current_outcome": "refused", "current_primary_barrier": "novel_initial_form", - "current_refusal_reason": "no admissible candidate for statement: 'Each survey has 10 questions.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'Each survey has 10 questions.' (category=multiplicative_aggregation)", "new_admission": false, "notes": "", "previous_first_refusal": "Each survey has 10 questions.", @@ -593,7 +593,7 @@ "current_first_refusal": "Half of the students are girls, the other half are boys.", "current_outcome": "refused", "current_primary_barrier": "fraction_operand", - "current_refusal_reason": "no admissible candidate for statement: 'Half of the students are girls, the other half are boys.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'Half of the students are girls, the other half are boys.' (category=descriptive_setup_no_quantity)", "new_admission": false, "notes": "Sentence 1 (novel_initial_form) now resolves via S.4's _INITIAL_HAS_INDEF_RE to (school, 100, students). Refuses on sentence 2: 'Half of the students are girls, the other half are boys' \u2014 needs fraction_operand parsing + downstream percentage_of on the 20%/10% sentences.", "previous_first_refusal": "A school has 100 students.", @@ -601,14 +601,14 @@ "previous_primary_barrier": "novel_initial_form" }, { - "barrier_shifted": false, + "barrier_shifted": true, "case_id": "gsm8k-train-sample-v1-0047", - "current_first_refusal": "John bakes 12 coconut macaroons, each weighing 5 ounces.", + "current_first_refusal": "He then packs an equal number of the macaroons in 4 different brown bags, ready for delivery.", "current_outcome": "refused", - "current_primary_barrier": "novel_initial_verb", - "current_refusal_reason": "no admissible candidate for statement: 'John bakes 12 coconut macaroons, each weighing 5 ounces.'", + "current_primary_barrier": "partition_divide", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'He then packs an equal number of the macaroons in 4 different brown bags, ready for delivery.' (category=discrete_count_statement)", "new_admission": false, - "notes": "", + "notes": "Reader now resolves 'John bakes 12 coconut macaroons, each weighing 5 ounces'; refuses on 'He then packs an equal number of the macaroons in 4 different brown bags' \u2014 needs partition_divide (12 / 4 equal).", "previous_first_refusal": "John bakes 12 coconut macaroons, each weighing 5 ounces.", "previous_outcome": "refused", "previous_primary_barrier": "novel_initial_verb" @@ -632,7 +632,7 @@ "current_first_refusal": "Malcolm is trying to find the fastest walk to school and is currently comparing two routes.", "current_outcome": "refused", "current_primary_barrier": "context_filler", - "current_refusal_reason": "no admissible candidate for statement: 'Malcolm is trying to find the fastest walk to school and is currently comparing two routes.'", + "current_refusal_reason": "recognizer matched but produced no injection for statement: 'Malcolm is trying to find the fastest walk to school and is currently comparing two routes.' (category=discrete_count_statement)", "new_admission": false, "notes": "", "previous_first_refusal": "Malcolm is trying to find the fastest walk to school and is currently comparing two routes.", @@ -655,9 +655,9 @@ ], "schema_version": 1, "summary": { - "admitted": 3, - "barrier_shifted_v3_to_v4": 2, - "refused": 47, + "admitted": 6, + "barrier_shifted_v3_to_v4": 10, + "refused": 44, "total_cases": 50, "wrong": 0 } diff --git a/evals/gsm8k_math/train_sample/v1/refusal_taxonomy_v4.json b/evals/gsm8k_math/train_sample/v1/refusal_taxonomy_v4.json index 7796d287..b492ed6a 100644 --- a/evals/gsm8k_math/train_sample/v1/refusal_taxonomy_v4.json +++ b/evals/gsm8k_math/train_sample/v1/refusal_taxonomy_v4.json @@ -23,7 +23,7 @@ { "case_id": "gsm8k-train-sample-v1-0003", "notes": "", - "primary_barrier": "novel_initial_verb", + "primary_barrier": "admitted", "secondary_barriers": [ "distributive_multiply", "rate_price" @@ -147,11 +147,10 @@ }, { "case_id": "gsm8k-train-sample-v1-0019", - "notes": "", - "primary_barrier": "rate_price", + "notes": "Reader now resolves the prior '$400 per vet appointment' barrier; first refusal moved to 'After the first appointment, John paid $100 for pet insurance that covers 80% of the subsequent visits' \u2014 needs percentage_rate (80% of a subsequent-visit set).", + "primary_barrier": "percentage_rate", "secondary_barriers": [ - "percentage_of", - "conditional_branch" + "rate_price" ] }, { @@ -165,7 +164,7 @@ { "case_id": "gsm8k-train-sample-v1-0021", "notes": "", - "primary_barrier": "distributive_multiply", + "primary_barrier": "admitted", "secondary_barriers": [] }, { @@ -179,24 +178,25 @@ }, { "case_id": "gsm8k-train-sample-v1-0023", - "notes": "", - "primary_barrier": "novel_initial_verb", + "notes": "Reader now resolves 'Nicole collected 400 Pokemon cards' (initial); refuses on 'Cindy collected twice as many, and Rex collected half of Nicole and Cindy's combined total' \u2014 needs compound_comparative ('twice as many') plus fraction_operand ('half of combined').", + "primary_barrier": "compound_comparative", "secondary_barriers": [ - "fraction_operand", - "distributive_divide" + "fraction_operand" ] }, { "case_id": "gsm8k-train-sample-v1-0024", "notes": "", - "primary_barrier": "multi_day_accumulation", + "primary_barrier": "admitted", "secondary_barriers": [] }, { "case_id": "gsm8k-train-sample-v1-0025", - "notes": "", + "notes": "Reader now resolves 'Lilibeth fills 6 baskets where each basket holds 50 strawberries'; refuses on the question 'If three of Lilibeth's friends pick the same amount as her, how many ... in all?' \u2014 needs distributive_multiply (3 friends x her amount) over a total question.", "primary_barrier": "distributive_multiply", - "secondary_barriers": [] + "secondary_barriers": [ + "complex_question" + ] }, { "case_id": "gsm8k-train-sample-v1-0026", @@ -208,10 +208,10 @@ }, { "case_id": "gsm8k-train-sample-v1-0027", - "notes": "", - "primary_barrier": "multi_attribute_accumulation", + "notes": "Reader now resolves 'Malcolm has 240 followers on Instagram and 500 on Facebook'; refuses on 'The number of followers he has on Twitter is half the number ... on Instagram and Facebook combined' \u2014 needs fraction_operand ('half of combined').", + "primary_barrier": "fraction_operand", "secondary_barriers": [ - "multi_step_complex" + "compound_statement" ] }, { @@ -364,11 +364,9 @@ }, { "case_id": "gsm8k-train-sample-v1-0047", - "notes": "", - "primary_barrier": "novel_initial_verb", - "secondary_barriers": [ - "embedded_per_unit" - ] + "notes": "Reader now resolves 'John bakes 12 coconut macaroons, each weighing 5 ounces'; refuses on 'He then packs an equal number of the macaroons in 4 different brown bags' \u2014 needs partition_divide (12 / 4 equal).", + "primary_barrier": "partition_divide", + "secondary_barriers": [] }, { "case_id": "gsm8k-train-sample-v1-0048", @@ -394,8 +392,8 @@ ], "schema_version": 2, "summary": { - "admitted": 3, - "refused": 47, + "admitted": 6, + "refused": 44, "total_cases": 50, "wrong": 0 } diff --git a/evals/gsm8k_math/train_sample/v1/rescan_v4.py b/evals/gsm8k_math/train_sample/v1/rescan_v4.py index c1eb8e1a..75037176 100644 --- a/evals/gsm8k_math/train_sample/v1/rescan_v4.py +++ b/evals/gsm8k_math/train_sample/v1/rescan_v4.py @@ -25,14 +25,81 @@ _V3_RESCAN_PATH = _HERE / "refusal_rescan_v3.json" _RESCAN_V4_PATH = _HERE / "refusal_rescan_v4.json" _TAXONOMY_V4_PATH = _HERE / "refusal_taxonomy_v4.json" +# Matches both refusal-reason shapes the candidate-graph emits: +# "no admissible candidate for {statement|question}: ''" +# "recognizer matched but produced no injection for statement: '' (category=)" +# (PR #359 added the second shape + trailing "(category=...)"). Greedy capture +# with an anchored closing quote tolerates statements containing apostrophes +# (e.g. "Rudolph's"); the optional category suffix is consumed after the quote. _FIRST_REFUSAL_RE = re.compile( - r"no admissible candidate for (?:statement|question): ['\"](.+?)['\"]$" + r"(?:no admissible candidate for|recognizer matched but produced no injection for) " + r"(?:statement|question): ['\"](.+)['\"](?:\s*\(category=[^)]*\))?\s*$" ) # Barrier reclassification for cases whose first-refusal sentence changed since v3. -# S.4 shifted exactly two cases (0038, 0046) by widening initial-state subject -# shapes (indefinite-article + prepositional-prefix existential). +# This rescan re-runs the LIVE reader (see module docstring), so it tracks the +# reader's current divergence from the v3 baseline — not only the S.4 cut. +# +# S.4 originally shifted two cases (0038, 0046) by widening initial-state subject +# shapes (indefinite-article + prepositional-prefix existential). Later capability +# waves (ADR-0163-D.2 discrete-count, 0174 held-hypothesis, 0178 compose, 0191 +# completeness guard, 0192 discrete_count) advanced the reader past five more +# v3-era barriers, shifting their first-refusal one sentence deeper. All shifts +# are soundness-preserving (the cases still refuse; wrong stays 0). Each shifted +# case carries an override documenting its new first-refusal barrier. +# NOTE: because this asserts against the live reader, future reader advances will +# add shifts here — a follow-up should cut a frozen v5 snapshot or derive the +# expected set from the committed artifact instead of the live run. _V4_BARRIER_OVERRIDES: dict[str, dict[str, Any]] = { + "gsm8k-train-sample-v1-0019": { + "primary_barrier": "percentage_rate", + "secondary_barriers": ["rate_price"], + "notes": ( + "Reader now resolves the prior '$400 per vet appointment' barrier; " + "first refusal moved to 'After the first appointment, John paid $100 " + "for pet insurance that covers 80% of the subsequent visits' — needs " + "percentage_rate (80% of a subsequent-visit set)." + ), + }, + "gsm8k-train-sample-v1-0023": { + "primary_barrier": "compound_comparative", + "secondary_barriers": ["fraction_operand"], + "notes": ( + "Reader now resolves 'Nicole collected 400 Pokemon cards' (initial); " + "refuses on 'Cindy collected twice as many, and Rex collected half of " + "Nicole and Cindy's combined total' — needs compound_comparative " + "('twice as many') plus fraction_operand ('half of combined')." + ), + }, + "gsm8k-train-sample-v1-0025": { + "primary_barrier": "distributive_multiply", + "secondary_barriers": ["complex_question"], + "notes": ( + "Reader now resolves 'Lilibeth fills 6 baskets where each basket holds " + "50 strawberries'; refuses on the question 'If three of Lilibeth's " + "friends pick the same amount as her, how many ... in all?' — needs " + "distributive_multiply (3 friends x her amount) over a total question." + ), + }, + "gsm8k-train-sample-v1-0027": { + "primary_barrier": "fraction_operand", + "secondary_barriers": ["compound_statement"], + "notes": ( + "Reader now resolves 'Malcolm has 240 followers on Instagram and 500 " + "on Facebook'; refuses on 'The number of followers he has on Twitter " + "is half the number ... on Instagram and Facebook combined' — needs " + "fraction_operand ('half of combined')." + ), + }, + "gsm8k-train-sample-v1-0047": { + "primary_barrier": "partition_divide", + "secondary_barriers": [], + "notes": ( + "Reader now resolves 'John bakes 12 coconut macaroons, each weighing " + "5 ounces'; refuses on 'He then packs an equal number of the macaroons " + "in 4 different brown bags' — needs partition_divide (12 / 4 equal)." + ), + }, "gsm8k-train-sample-v1-0038": { "primary_barrier": "compound_comparative", "secondary_barriers": ["aggregate_sum"], diff --git a/tests/test_rescan_v4_invariants.py b/tests/test_rescan_v4_invariants.py index a5d8d4a8..d6021da7 100644 --- a/tests/test_rescan_v4_invariants.py +++ b/tests/test_rescan_v4_invariants.py @@ -22,29 +22,60 @@ def test_wrong_is_zero() -> None: assert rescan["summary"]["wrong"] == 0 -def test_admission_set_unchanged() -> None: - """S.4 did not change the admission set: {0014, 0018, 0042}.""" +def test_admission_set_grew_monotonically_over_v3() -> None: + """The reader admits the v3 set plus three new *correct* solves. + + v3 admitted {0014, 0018, 0042}. Later capability waves added three correct + admissions (0003, 0021, 0024) — the +3 that moved the serving metric from + 3 to 6. No v3 admission was lost (monotonic), and wrong stays 0 + (test_wrong_is_zero), so every gain is a sound solve. + """ rescan, _ = build_rescan() admitted = { c["case_id"] for c in rescan["per_case"] if c["current_outcome"] == "admitted" } - assert admitted == { + v3_admitted = { "gsm8k-train-sample-v1-0014", "gsm8k-train-sample-v1-0018", "gsm8k-train-sample-v1-0042", } + # No regression: every case v3 solved is still solved. + assert v3_admitted <= admitted + assert admitted == v3_admitted | { + "gsm8k-train-sample-v1-0003", + "gsm8k-train-sample-v1-0021", + "gsm8k-train-sample-v1-0024", + } -def test_exactly_two_shifts_v3_to_v4() -> None: - """S.4 shifted exactly two cases (gsm8k-0038, gsm8k-0046).""" +def test_barrier_shifts_from_v3_baseline() -> None: + """The live reader has diverged from the v3 baseline on ten cases. + + S.4 shifted two (0038, 0046). Later capability waves (0163-D.2, 0174, 0178, + 0191, 0192) advanced the reader past five more v3-era barriers (0019, 0023, + 0025, 0027, 0047) — still refusing, one sentence deeper — and turned three + refusals into correct admissions (0003, 0021, 0024). All ten are + soundness-preserving (wrong stays 0); the five still-refusing shifts each + carry an override in _V4_BARRIER_OVERRIDES. + """ rescan, _ = build_rescan() - assert rescan["summary"]["barrier_shifted_v3_to_v4"] == 2 + # 10 = 7 first-refusal/barrier shifts + 3 refused->admitted transitions + # (0003, 0021, 0024), which also count as shifts (outcome changed). + assert rescan["summary"]["barrier_shifted_v3_to_v4"] == 10 shifted = sorted( c["case_id"] for c in rescan["per_case"] if c["barrier_shifted"] ) assert shifted == [ + "gsm8k-train-sample-v1-0003", + "gsm8k-train-sample-v1-0019", + "gsm8k-train-sample-v1-0021", + "gsm8k-train-sample-v1-0023", + "gsm8k-train-sample-v1-0024", + "gsm8k-train-sample-v1-0025", + "gsm8k-train-sample-v1-0027", "gsm8k-train-sample-v1-0038", "gsm8k-train-sample-v1-0046", + "gsm8k-train-sample-v1-0047", ]