chore: deprecate orphaned rescan_v4 archaeology layer
The rescan_v2/v3/v4 chain has zero runtime consumers (nothing in core/, generate/, teaching/, chat/ imports it). v2/v3 are frozen disk-snapshot tests; v4 alone re-ran the LIVE reader and asserted a barrier-shift count + admission set, so it broke on every legitimate reader improvement (it was 8 of the red tests). Its only real invariant (wrong=0) is guarded directly by the serving runner and test_runtime_wrong_zero_preserved. No ADR doc references the v4 layer (only S2/S3-post-rescan exist). Remove the brittle live test plus the now-orphaned generator and its two artifacts. Keep the frozen, tested v2/v3 layer as the historical record. Net: -8 brittle tests, removes a maintenance-only archaeology surface rather than greening it.
This commit is contained in:
Shay
parent
eed293c31c
commit
78001d6f78
4 changed files with 0 additions and 1482 deletions
|
|
@ -1,664 +0,0 @@
|
|||
{
|
||||
"adr": "0136.S.4-post-rescan",
|
||||
"description": "Per-case barrier-shift ledger comparing v3 (post-S.3) to v4 (post-S.4) behavior of parse_and_solve.",
|
||||
"per_case": [
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0001",
|
||||
"current_first_refusal": "Tina makes $18.00 an hour.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "rate_earnings",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'Tina makes $18.00 an hour.' (category=rate_with_currency)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "Tina makes $18.00 an hour.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "rate_earnings"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0002",
|
||||
"current_first_refusal": "She splits it up into 25-foot sections.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "partition_divide",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'She splits it up into 25-foot sections.' (category=discrete_count_statement)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "She splits it up into 25-foot sections.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "partition_divide"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": true,
|
||||
"case_id": "gsm8k-train-sample-v1-0003",
|
||||
"current_first_refusal": null,
|
||||
"current_outcome": "admitted",
|
||||
"current_primary_barrier": "admitted",
|
||||
"current_refusal_reason": null,
|
||||
"new_admission": true,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "The local bookstore donated 48 boxes of erasers.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "novel_initial_verb"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0004",
|
||||
"current_first_refusal": "Half of the kids are going to soccer camp, and 1/4 of the kids going to soccer camp are going to soccer camp in the morning.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "fraction_operand",
|
||||
"current_refusal_reason": "no admissible candidate for statement: 'Half of the kids are going to soccer camp, and 1/4 of the kids going to soccer camp are going to soccer camp in the morning.'",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "Half of the kids are going to soccer camp, and 1/4 of the kids going to soccer camp are going to soccer camp in the morning.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "fraction_operand"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0005",
|
||||
"current_first_refusal": "In one hour, Addison mountain's temperature will decrease to 3/4 of its temperature.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "fraction_operand",
|
||||
"current_refusal_reason": "no admissible candidate for statement: \"In one hour, Addison mountain's temperature will decrease to 3/4 of its temperature.\"",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "In one hour, Addison mountain's temperature will decrease to 3/4 of its temperature.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "fraction_operand"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0006",
|
||||
"current_first_refusal": "Mandy started reading books with only 8 pages when she was 6 years old.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "temporal_age_anchor",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'Mandy started reading books with only 8 pages when she was 6 years old.' (category=multiplicative_aggregation)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "Mandy started reading books with only 8 pages when she was 6 years old.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "temporal_age_anchor"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0007",
|
||||
"current_first_refusal": "How many more boxes do they need if Francine has a total of 85 crayons?",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "conditional_question",
|
||||
"current_refusal_reason": "no admissible candidate for question: 'How many more boxes do they need if Francine has a total of 85 crayons?'",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "How many more boxes do they need if Francine has a total of 85 crayons?",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "conditional_question"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0008",
|
||||
"current_first_refusal": "If 50 beads are used to make one bracelet, how many bracelets will Marnie be able to make out of the beads she bought?",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "conditional_question",
|
||||
"current_refusal_reason": "no admissible candidate for question: 'If 50 beads are used to make one bracelet, how many bracelets will Marnie be able to make out of the beads she bought?'",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "If 50 beads are used to make one bracelet, how many bracelets will Marnie be able to make out of the beads she bought?",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "conditional_question"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0009",
|
||||
"current_first_refusal": "If Jen has 150 ducks, how many total birds does she have?",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "conditional_question",
|
||||
"current_refusal_reason": "no admissible candidate for question: 'If Jen has 150 ducks, how many total birds does she have?'",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "If Jen has 150 ducks, how many total birds does she have?",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "conditional_question"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0010",
|
||||
"current_first_refusal": "Marion has 1/4 more than what Yun currently has, plus 7.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "fraction_operand",
|
||||
"current_refusal_reason": "no admissible candidate for statement: 'Marion has 1/4 more than what Yun currently has, plus 7.'",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "Marion has 1/4 more than what Yun currently has, plus 7.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "fraction_operand"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0011",
|
||||
"current_first_refusal": "Alexa has a lemonade stand where she sells lemonade for $2 for one cup.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "rate_price",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'Alexa has a lemonade stand where she sells lemonade for $2 for one cup.' (category=rate_with_currency)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "Alexa has a lemonade stand where she sells lemonade for $2 for one cup.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "rate_price"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0012",
|
||||
"current_first_refusal": "He put all of them in his aquarium but his fish ate half of them.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "compound_statement",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'He put all of them in his aquarium but his fish ate half of them.' (category=descriptive_setup_no_quantity)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "He put all of them in his aquarium but his fish ate half of them.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "compound_statement"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0013",
|
||||
"current_first_refusal": "Allison, a YouTuber, uploads 10 one-hour videos of food reviews each day to her channel.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "compound_statement",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'Allison, a YouTuber, uploads 10 one-hour videos of food reviews each day to her channel.' (category=multiplicative_aggregation)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "Allison, a YouTuber, uploads 10 one-hour videos of food reviews each day to her channel.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "compound_statement"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0014",
|
||||
"current_first_refusal": null,
|
||||
"current_outcome": "admitted",
|
||||
"current_primary_barrier": "admitted",
|
||||
"current_refusal_reason": null,
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": null,
|
||||
"previous_outcome": "admitted",
|
||||
"previous_primary_barrier": "admitted"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0015",
|
||||
"current_first_refusal": "Traveling from Manhattan to the Bronx, Andrew rides the subway for 10 hours, takes the train and rides for twice as much time as the subway ride, and then bikes the remaining distance for 8 hours.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "compound_multi_event",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'Traveling from Manhattan to the Bronx, Andrew rides the subway for 10 hours, takes the train and rides for twice as much time as the subway ride, and then bikes the remaining distance for 8 hours.' (category=discrete_count_statement)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "Traveling from Manhattan to the Bronx, Andrew rides the subway for 10 hours, takes the train and rides for twice as much time as the subway ride, and then bikes the remaining distance for 8 hours.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "compound_multi_event"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0016",
|
||||
"current_first_refusal": "On Rudolph's car trip across town, he traveled 2 more than 5 miles and encountered 3 less than 17 stop signs.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "compound_statement",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: \"On Rudolph's car trip across town, he traveled 2 more than 5 miles and encountered 3 less than 17 stop signs.\" (category=discrete_count_statement)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "On Rudolph's car trip across town, he traveled 2 more than 5 miles and encountered 3 less than 17 stop signs.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "compound_statement"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0017",
|
||||
"current_first_refusal": "He\u2019s charging $50.00 per day or $500.00 for 14 days.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "conditional_branch",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'He\u2019s charging $50.00 per day or $500.00 for 14 days.' (category=temporal_aggregation)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "He\u2019s charging $50.00 per day or $500.00 for 14 days.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "conditional_branch"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0018",
|
||||
"current_first_refusal": null,
|
||||
"current_outcome": "admitted",
|
||||
"current_primary_barrier": "admitted",
|
||||
"current_refusal_reason": null,
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": null,
|
||||
"previous_outcome": "admitted",
|
||||
"previous_primary_barrier": "admitted"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": true,
|
||||
"case_id": "gsm8k-train-sample-v1-0019",
|
||||
"current_first_refusal": "After the first appointment, John paid $100 for pet insurance that covers 80% of the subsequent visits.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "percentage_rate",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'After the first appointment, John paid $100 for pet insurance that covers 80% of the subsequent visits.' (category=currency_amount)",
|
||||
"new_admission": false,
|
||||
"notes": "Reader now resolves the prior '$400 per vet appointment' barrier; first refusal moved to 'After the first appointment, John paid $100 for pet insurance that covers 80% of the subsequent visits' \u2014 needs percentage_rate (80% of a subsequent-visit set).",
|
||||
"previous_first_refusal": "The dog ends up having health problems and this requires 3 vet appointments, which cost $400 each.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "rate_price"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0020",
|
||||
"current_first_refusal": "Two puppies, two kittens, and three parakeets were for sale at the pet shop.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "multi_entity_initial",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'Two puppies, two kittens, and three parakeets were for sale at the pet shop.' (category=discrete_count_statement)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "Two puppies, two kittens, and three parakeets were for sale at the pet shop.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "multi_entity_initial"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": true,
|
||||
"case_id": "gsm8k-train-sample-v1-0021",
|
||||
"current_first_refusal": null,
|
||||
"current_outcome": "admitted",
|
||||
"current_primary_barrier": "admitted",
|
||||
"current_refusal_reason": null,
|
||||
"new_admission": true,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "He bench presses 15 pounds for 10 reps and does 3 sets.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "distributive_multiply"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0022",
|
||||
"current_first_refusal": "Erica lives near a lake where most locals sell fish as their main source of income, earning $20 per kg of fish.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "context_filler",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'Erica lives near a lake where most locals sell fish as their main source of income, earning $20 per kg of fish.' (category=rate_with_currency)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "Erica lives near a lake where most locals sell fish as their main source of income, earning $20 per kg of fish.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "context_filler"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": true,
|
||||
"case_id": "gsm8k-train-sample-v1-0023",
|
||||
"current_first_refusal": "Cindy collected twice as many, and Rex collected half of Nicole and Cindy's combined total.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "compound_comparative",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: \"Cindy collected twice as many, and Rex collected half of Nicole and Cindy's combined total.\" (category=descriptive_setup_no_quantity)",
|
||||
"new_admission": false,
|
||||
"notes": "Reader now resolves 'Nicole collected 400 Pokemon cards' (initial); refuses on 'Cindy collected twice as many, and Rex collected half of Nicole and Cindy's combined total' \u2014 needs compound_comparative ('twice as many') plus fraction_operand ('half of combined').",
|
||||
"previous_first_refusal": "Nicole collected 400 Pokemon cards.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "novel_initial_verb"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": true,
|
||||
"case_id": "gsm8k-train-sample-v1-0024",
|
||||
"current_first_refusal": null,
|
||||
"current_outcome": "admitted",
|
||||
"current_primary_barrier": "admitted",
|
||||
"current_refusal_reason": null,
|
||||
"new_admission": true,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "Sidney does 20 jumping jacks on Monday, 36 on Tuesday, 40 on Wednesday, and 50 on Thursday.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "multi_day_accumulation"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": true,
|
||||
"case_id": "gsm8k-train-sample-v1-0025",
|
||||
"current_first_refusal": "If three of Lilibeth's friends pick the same amount as her, how many strawberries do Lilibeth and her friends pick in all?",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "distributive_multiply",
|
||||
"current_refusal_reason": "no admissible candidate for question: \"If three of Lilibeth's friends pick the same amount as her, how many strawberries do Lilibeth and her friends pick in all?\"",
|
||||
"new_admission": false,
|
||||
"notes": "Reader now resolves 'Lilibeth fills 6 baskets where each basket holds 50 strawberries'; refuses on the question 'If three of Lilibeth's friends pick the same amount as her, how many ... in all?' \u2014 needs distributive_multiply (3 friends x her amount) over a total question.",
|
||||
"previous_first_refusal": "Lilibeth fills 6 baskets where each basket holds 50 strawberries.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "distributive_multiply"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0026",
|
||||
"current_first_refusal": "Aaron and his brother Carson each saved up $40 to go to dinner.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "distributive_each_actor",
|
||||
"current_refusal_reason": "no admissible candidate for statement: 'Aaron and his brother Carson each saved up $40 to go to dinner.'",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "Aaron and his brother Carson each saved up $40 to go to dinner.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "distributive_each_actor"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": true,
|
||||
"case_id": "gsm8k-train-sample-v1-0027",
|
||||
"current_first_refusal": "The number of followers he has on Twitter is half the number of followers he has on Instagram and Facebook combined.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "fraction_operand",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'The number of followers he has on Twitter is half the number of followers he has on Instagram and Facebook combined.' (category=descriptive_setup_no_quantity)",
|
||||
"new_admission": false,
|
||||
"notes": "Reader now resolves 'Malcolm has 240 followers on Instagram and 500 on Facebook'; refuses on 'The number of followers he has on Twitter is half the number ... on Instagram and Facebook combined' \u2014 needs fraction_operand ('half of combined').",
|
||||
"previous_first_refusal": "Malcolm has 240 followers on Instagram and 500 followers on Facebook.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "multi_attribute_accumulation"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0028",
|
||||
"current_first_refusal": "It cost $100,000 to open initially.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "novel_initial_form",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'It cost $100,000 to open initially.' (category=currency_amount)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "It cost $100,000 to open initially.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "novel_initial_form"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0029",
|
||||
"current_first_refusal": "The cost of the keyboard was three times greater than the cost of the mouse.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "compound_comparative",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'The cost of the keyboard was three times greater than the cost of the mouse.' (category=discrete_count_statement)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "The cost of the keyboard was three times greater than the cost of the mouse.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "compound_comparative"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0030",
|
||||
"current_first_refusal": "It is a 2-hour drive each way.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "novel_initial_form",
|
||||
"current_refusal_reason": "no admissible candidate for statement: 'It is a 2-hour drive each way.'",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "It is a 2-hour drive each way.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "novel_initial_form"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0031",
|
||||
"current_first_refusal": "Jeremie wants to go to an amusement park with 3 friends at the end of summer.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "context_filler",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'Jeremie wants to go to an amusement park with 3 friends at the end of summer.' (category=discrete_count_statement)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "Jeremie wants to go to an amusement park with 3 friends at the end of summer.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "context_filler"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0032",
|
||||
"current_first_refusal": "He draws and colors 10 pictures.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "compound_statement",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'He draws and colors 10 pictures.' (category=discrete_count_statement)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "He draws and colors 10 pictures.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "compound_statement"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0033",
|
||||
"current_first_refusal": "Rachel is 12 years old, and her grandfather is 7 times her age.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "compound_statement",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'Rachel is 12 years old, and her grandfather is 7 times her age.' (category=discrete_count_statement)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "Rachel is 12 years old, and her grandfather is 7 times her age.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "compound_statement"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0034",
|
||||
"current_first_refusal": "He can run 40 yards within 5 seconds.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "capacity_rate",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'He can run 40 yards within 5 seconds.' (category=discrete_count_statement)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "He can run 40 yards within 5 seconds.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "capacity_rate"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0035",
|
||||
"current_first_refusal": "How many more apples would Martha need to give away to be left with only 4 of them?",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "complex_question",
|
||||
"current_refusal_reason": "no admissible candidate for question: 'How many more apples would Martha need to give away to be left with only 4 of them?'",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "How many more apples would Martha need to give away to be left with only 4 of them?",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "complex_question"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0036",
|
||||
"current_first_refusal": "She studied for 2 hours on Wednesday and three times as long on Thursday.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "compound_comparative",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'She studied for 2 hours on Wednesday and three times as long on Thursday.' (category=discrete_count_statement)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "She studied for 2 hours on Wednesday and three times as long on Thursday.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "compound_comparative"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0037",
|
||||
"current_first_refusal": "Michael wants to lose 10 pounds by June.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "goal_statement",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'Michael wants to lose 10 pounds by June.' (category=discrete_count_statement)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "Michael wants to lose 10 pounds by June.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "goal_statement"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": true,
|
||||
"case_id": "gsm8k-train-sample-v1-0038",
|
||||
"current_first_refusal": "There are three times that many girls at a party being held on the second floor of the building.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "compound_comparative",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'There are three times that many girls at a party being held on the second floor of the building.' (category=discrete_count_statement)",
|
||||
"new_admission": false,
|
||||
"notes": "Sentence 1 (novel_initial_form) now resolves via S.4's _INITIAL_THERE_ARE_PREFIX_RE to (building, 100, ladies). Refuses on sentence 2: 'There are three times that many girls at a party being held on the second floor of the building' \u2014 needs compound_comparative parsing ('three times that many') with cross-sentence reference to the prior 100.",
|
||||
"previous_first_refusal": "In a building, there are a hundred ladies on the first-floor studying.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "novel_initial_form"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0039",
|
||||
"current_first_refusal": "Orlando gained 5 pounds.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "novel_initial_verb",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'Orlando gained 5 pounds.' (category=discrete_count_statement)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "Orlando gained 5 pounds.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "novel_initial_verb"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0040",
|
||||
"current_first_refusal": "He now has 2 horses, 5 dogs, 7 cats, 3 turtles, and 1 goat.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "multi_entity_initial",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'He now has 2 horses, 5 dogs, 7 cats, 3 turtles, and 1 goat.' (category=discrete_count_statement)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "He now has 2 horses, 5 dogs, 7 cats, 3 turtles, and 1 goat.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "multi_entity_initial"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0041",
|
||||
"current_first_refusal": "The guests eat all of 1 pan, and 75% of the 2nd pan.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "fraction_operand",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'The guests eat all of 1 pan, and 75% of the 2nd pan.' (category=discrete_count_statement)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "The guests eat all of 1 pan, and 75% of the 2nd pan.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "fraction_operand"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0042",
|
||||
"current_first_refusal": null,
|
||||
"current_outcome": "admitted",
|
||||
"current_primary_barrier": "admitted",
|
||||
"current_refusal_reason": null,
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": null,
|
||||
"previous_outcome": "admitted",
|
||||
"previous_primary_barrier": "admitted"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0043",
|
||||
"current_first_refusal": "Her mother gave her an additional $4, and her father twice as much as her mother.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "compound_comparative",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'Her mother gave her an additional $4, and her father twice as much as her mother.' (category=currency_amount)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "Her mother gave her an additional $4, and her father twice as much as her mother.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "compound_comparative"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0044",
|
||||
"current_first_refusal": "John invests in a bank and gets 10% simple interest.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "percentage_rate",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'John invests in a bank and gets 10% simple interest.' (category=discrete_count_statement)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "John invests in a bank and gets 10% simple interest.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "percentage_rate"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0045",
|
||||
"current_first_refusal": "Each survey has 10 questions.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "novel_initial_form",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'Each survey has 10 questions.' (category=multiplicative_aggregation)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "Each survey has 10 questions.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "novel_initial_form"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": true,
|
||||
"case_id": "gsm8k-train-sample-v1-0046",
|
||||
"current_first_refusal": "Half of the students are girls, the other half are boys.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "fraction_operand",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'Half of the students are girls, the other half are boys.' (category=descriptive_setup_no_quantity)",
|
||||
"new_admission": false,
|
||||
"notes": "Sentence 1 (novel_initial_form) now resolves via S.4's _INITIAL_HAS_INDEF_RE to (school, 100, students). Refuses on sentence 2: 'Half of the students are girls, the other half are boys' \u2014 needs fraction_operand parsing + downstream percentage_of on the 20%/10% sentences.",
|
||||
"previous_first_refusal": "A school has 100 students.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "novel_initial_form"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": true,
|
||||
"case_id": "gsm8k-train-sample-v1-0047",
|
||||
"current_first_refusal": "He then packs an equal number of the macaroons in 4 different brown bags, ready for delivery.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "partition_divide",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'He then packs an equal number of the macaroons in 4 different brown bags, ready for delivery.' (category=discrete_count_statement)",
|
||||
"new_admission": false,
|
||||
"notes": "Reader now resolves 'John bakes 12 coconut macaroons, each weighing 5 ounces'; refuses on 'He then packs an equal number of the macaroons in 4 different brown bags' \u2014 needs partition_divide (12 / 4 equal).",
|
||||
"previous_first_refusal": "John bakes 12 coconut macaroons, each weighing 5 ounces.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "novel_initial_verb"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0048",
|
||||
"current_first_refusal": "Every week, he gets 6 cards.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "temporal_frequency",
|
||||
"current_refusal_reason": "no admissible candidate for statement: 'Every week, he gets 6 cards.'",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "Every week, he gets 6 cards.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "temporal_frequency"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0049",
|
||||
"current_first_refusal": "Malcolm is trying to find the fastest walk to school and is currently comparing two routes.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "context_filler",
|
||||
"current_refusal_reason": "recognizer matched but produced no injection for statement: 'Malcolm is trying to find the fastest walk to school and is currently comparing two routes.' (category=discrete_count_statement)",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "Malcolm is trying to find the fastest walk to school and is currently comparing two routes.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "context_filler"
|
||||
},
|
||||
{
|
||||
"barrier_shifted": false,
|
||||
"case_id": "gsm8k-train-sample-v1-0050",
|
||||
"current_first_refusal": "Mark does a gig every other day for 2 weeks.",
|
||||
"current_outcome": "refused",
|
||||
"current_primary_barrier": "temporal_frequency",
|
||||
"current_refusal_reason": "no admissible candidate for statement: 'Mark does a gig every other day for 2 weeks.'",
|
||||
"new_admission": false,
|
||||
"notes": "",
|
||||
"previous_first_refusal": "Mark does a gig every other day for 2 weeks.",
|
||||
"previous_outcome": "refused",
|
||||
"previous_primary_barrier": "temporal_frequency"
|
||||
}
|
||||
],
|
||||
"schema_version": 1,
|
||||
"summary": {
|
||||
"admitted": 6,
|
||||
"barrier_shifted_v3_to_v4": 10,
|
||||
"refused": 44,
|
||||
"total_cases": 50,
|
||||
"wrong": 0
|
||||
}
|
||||
}
|
||||
|
|
@ -1,400 +0,0 @@
|
|||
{
|
||||
"adr": "0136.S.4-post-rescan",
|
||||
"description": "Post-S.4 refusal taxonomy. Schema identical to v3; primary_barrier vocabulary unchanged.",
|
||||
"per_case": [
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0001",
|
||||
"notes": "",
|
||||
"primary_barrier": "rate_earnings",
|
||||
"secondary_barriers": [
|
||||
"conditional_branch",
|
||||
"multi_statement"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0002",
|
||||
"notes": "",
|
||||
"primary_barrier": "partition_divide",
|
||||
"secondary_barriers": [
|
||||
"fraction_operand",
|
||||
"coreference_pronoun"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0003",
|
||||
"notes": "",
|
||||
"primary_barrier": "admitted",
|
||||
"secondary_barriers": [
|
||||
"distributive_multiply",
|
||||
"rate_price"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0004",
|
||||
"notes": "",
|
||||
"primary_barrier": "fraction_operand",
|
||||
"secondary_barriers": [
|
||||
"conditional_question"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0005",
|
||||
"notes": "",
|
||||
"primary_barrier": "fraction_operand",
|
||||
"secondary_barriers": []
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0006",
|
||||
"notes": "",
|
||||
"primary_barrier": "temporal_age_anchor",
|
||||
"secondary_barriers": [
|
||||
"multi_step_complex"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0007",
|
||||
"notes": "",
|
||||
"primary_barrier": "conditional_question",
|
||||
"secondary_barriers": [
|
||||
"multi_entity_initial"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0008",
|
||||
"notes": "",
|
||||
"primary_barrier": "conditional_question",
|
||||
"secondary_barriers": [
|
||||
"distributive_multiply"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0009",
|
||||
"notes": "",
|
||||
"primary_barrier": "conditional_question",
|
||||
"secondary_barriers": [
|
||||
"compound_comparative"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0010",
|
||||
"notes": "Sentence 1 (compound_statement) now resolves via S.3's _INIT_MUTATION_RE to Yun=8 paperclips. Refuses on sentence 2: 'Marion has 1/4 more than what Yun currently has, plus 7' \u2014 needs fraction_operand parsing + coreference to Yun's current quantity + comparative-additive arithmetic.",
|
||||
"primary_barrier": "fraction_operand",
|
||||
"secondary_barriers": [
|
||||
"coreference_quantity",
|
||||
"comparative_arithmetic"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0011",
|
||||
"notes": "",
|
||||
"primary_barrier": "rate_price",
|
||||
"secondary_barriers": [
|
||||
"context_filler"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0012",
|
||||
"notes": "",
|
||||
"primary_barrier": "compound_statement",
|
||||
"secondary_barriers": [
|
||||
"fraction_operand",
|
||||
"coreference_pronoun"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0013",
|
||||
"notes": "",
|
||||
"primary_barrier": "compound_statement",
|
||||
"secondary_barriers": [
|
||||
"temporal_frequency"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0014",
|
||||
"notes": "",
|
||||
"primary_barrier": "admitted",
|
||||
"secondary_barriers": []
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0015",
|
||||
"notes": "",
|
||||
"primary_barrier": "compound_multi_event",
|
||||
"secondary_barriers": [
|
||||
"compound_comparative"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0016",
|
||||
"notes": "",
|
||||
"primary_barrier": "compound_statement",
|
||||
"secondary_barriers": [
|
||||
"rate_question"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0017",
|
||||
"notes": "",
|
||||
"primary_barrier": "conditional_branch",
|
||||
"secondary_barriers": [
|
||||
"rate_price"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0018",
|
||||
"notes": "",
|
||||
"primary_barrier": "admitted",
|
||||
"secondary_barriers": []
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0019",
|
||||
"notes": "Reader now resolves the prior '$400 per vet appointment' barrier; first refusal moved to 'After the first appointment, John paid $100 for pet insurance that covers 80% of the subsequent visits' \u2014 needs percentage_rate (80% of a subsequent-visit set).",
|
||||
"primary_barrier": "percentage_rate",
|
||||
"secondary_barriers": [
|
||||
"rate_price"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0020",
|
||||
"notes": "",
|
||||
"primary_barrier": "multi_entity_initial",
|
||||
"secondary_barriers": [
|
||||
"rate_comparative"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0021",
|
||||
"notes": "",
|
||||
"primary_barrier": "admitted",
|
||||
"secondary_barriers": []
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0022",
|
||||
"notes": "",
|
||||
"primary_barrier": "context_filler",
|
||||
"secondary_barriers": [
|
||||
"rate_earnings",
|
||||
"compound_comparative"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0023",
|
||||
"notes": "Reader now resolves 'Nicole collected 400 Pokemon cards' (initial); refuses on 'Cindy collected twice as many, and Rex collected half of Nicole and Cindy's combined total' \u2014 needs compound_comparative ('twice as many') plus fraction_operand ('half of combined').",
|
||||
"primary_barrier": "compound_comparative",
|
||||
"secondary_barriers": [
|
||||
"fraction_operand"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0024",
|
||||
"notes": "",
|
||||
"primary_barrier": "admitted",
|
||||
"secondary_barriers": []
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0025",
|
||||
"notes": "Reader now resolves 'Lilibeth fills 6 baskets where each basket holds 50 strawberries'; refuses on the question 'If three of Lilibeth's friends pick the same amount as her, how many ... in all?' \u2014 needs distributive_multiply (3 friends x her amount) over a total question.",
|
||||
"primary_barrier": "distributive_multiply",
|
||||
"secondary_barriers": [
|
||||
"complex_question"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0026",
|
||||
"notes": "",
|
||||
"primary_barrier": "distributive_each_actor",
|
||||
"secondary_barriers": [
|
||||
"fraction_operand"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0027",
|
||||
"notes": "Reader now resolves 'Malcolm has 240 followers on Instagram and 500 on Facebook'; refuses on 'The number of followers he has on Twitter is half the number ... on Instagram and Facebook combined' \u2014 needs fraction_operand ('half of combined').",
|
||||
"primary_barrier": "fraction_operand",
|
||||
"secondary_barriers": [
|
||||
"compound_statement"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0028",
|
||||
"notes": "",
|
||||
"primary_barrier": "novel_initial_form",
|
||||
"secondary_barriers": [
|
||||
"rate_price",
|
||||
"temporal_frequency"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0029",
|
||||
"notes": "",
|
||||
"primary_barrier": "compound_comparative",
|
||||
"secondary_barriers": [
|
||||
"rate_comparative"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0030",
|
||||
"notes": "",
|
||||
"primary_barrier": "novel_initial_form",
|
||||
"secondary_barriers": [
|
||||
"compound_comparative"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0031",
|
||||
"notes": "",
|
||||
"primary_barrier": "context_filler",
|
||||
"secondary_barriers": [
|
||||
"implicit_group_count"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0032",
|
||||
"notes": "",
|
||||
"primary_barrier": "compound_statement",
|
||||
"secondary_barriers": [
|
||||
"percentage_of"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0033",
|
||||
"notes": "",
|
||||
"primary_barrier": "compound_statement",
|
||||
"secondary_barriers": [
|
||||
"multi_step_complex"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0034",
|
||||
"notes": "",
|
||||
"primary_barrier": "capacity_rate",
|
||||
"secondary_barriers": [
|
||||
"percentage_of",
|
||||
"capacity_rate"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0035",
|
||||
"notes": "",
|
||||
"primary_barrier": "complex_question",
|
||||
"secondary_barriers": [
|
||||
"coreference_pronoun"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0036",
|
||||
"notes": "",
|
||||
"primary_barrier": "compound_comparative",
|
||||
"secondary_barriers": [
|
||||
"compound_comparative"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0037",
|
||||
"notes": "",
|
||||
"primary_barrier": "goal_statement",
|
||||
"secondary_barriers": []
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0038",
|
||||
"notes": "Sentence 1 (novel_initial_form) now resolves via S.4's _INITIAL_THERE_ARE_PREFIX_RE to (building, 100, ladies). Refuses on sentence 2: 'There are three times that many girls at a party being held on the second floor of the building' \u2014 needs compound_comparative parsing ('three times that many') with cross-sentence reference to the prior 100.",
|
||||
"primary_barrier": "compound_comparative",
|
||||
"secondary_barriers": [
|
||||
"aggregate_sum"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0039",
|
||||
"notes": "",
|
||||
"primary_barrier": "novel_initial_verb",
|
||||
"secondary_barriers": [
|
||||
"multi_step_complex"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0040",
|
||||
"notes": "",
|
||||
"primary_barrier": "multi_entity_initial",
|
||||
"secondary_barriers": [
|
||||
"leg_count"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0041",
|
||||
"notes": "",
|
||||
"primary_barrier": "fraction_operand",
|
||||
"secondary_barriers": [
|
||||
"percentage_of"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0042",
|
||||
"notes": "",
|
||||
"primary_barrier": "admitted",
|
||||
"secondary_barriers": []
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0043",
|
||||
"notes": "",
|
||||
"primary_barrier": "compound_comparative",
|
||||
"secondary_barriers": [
|
||||
"rate_comparative"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0044",
|
||||
"notes": "",
|
||||
"primary_barrier": "percentage_rate",
|
||||
"secondary_barriers": []
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0045",
|
||||
"notes": "",
|
||||
"primary_barrier": "novel_initial_form",
|
||||
"secondary_barriers": [
|
||||
"rate_price"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0046",
|
||||
"notes": "Sentence 1 (novel_initial_form) now resolves via S.4's _INITIAL_HAS_INDEF_RE to (school, 100, students). Refuses on sentence 2: 'Half of the students are girls, the other half are boys' \u2014 needs fraction_operand parsing + downstream percentage_of on the 20%/10% sentences.",
|
||||
"primary_barrier": "fraction_operand",
|
||||
"secondary_barriers": [
|
||||
"percentage_of"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0047",
|
||||
"notes": "Reader now resolves 'John bakes 12 coconut macaroons, each weighing 5 ounces'; refuses on 'He then packs an equal number of the macaroons in 4 different brown bags' \u2014 needs partition_divide (12 / 4 equal).",
|
||||
"primary_barrier": "partition_divide",
|
||||
"secondary_barriers": []
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0048",
|
||||
"notes": "",
|
||||
"primary_barrier": "temporal_frequency",
|
||||
"secondary_barriers": []
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0049",
|
||||
"notes": "",
|
||||
"primary_barrier": "context_filler",
|
||||
"secondary_barriers": [
|
||||
"compound_multi_event",
|
||||
"rate_comparative"
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-train-sample-v1-0050",
|
||||
"notes": "",
|
||||
"primary_barrier": "temporal_frequency",
|
||||
"secondary_barriers": []
|
||||
}
|
||||
],
|
||||
"schema_version": 2,
|
||||
"summary": {
|
||||
"admitted": 6,
|
||||
"refused": 44,
|
||||
"total_cases": 50,
|
||||
"wrong": 0
|
||||
}
|
||||
}
|
||||
|
|
@ -1,293 +0,0 @@
|
|||
"""ADR-0136.S.4 post-rescan — barrier-shift ledger v4 over 50-case GSM8K train sample.
|
||||
|
||||
Measurement-only. Runs ``parse_and_solve`` on every case on current main
|
||||
(post-S.4), compares to v3 taxonomy (ADR-0136.S.3-post-rescan), and writes:
|
||||
|
||||
- ``refusal_rescan_v4.json`` — per-case barrier-shift ledger (v3 → v4).
|
||||
- ``refusal_taxonomy_v4.json`` — updated taxonomy reflecting post-S.4 parser.
|
||||
|
||||
Aborts on any ``wrong`` admission (answer != expected).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from generate.math_candidate_graph import parse_and_solve
|
||||
|
||||
_HERE = Path(__file__).resolve().parent
|
||||
_CASES_PATH = _HERE / "cases.jsonl"
|
||||
_V3_TAXONOMY_PATH = _HERE / "refusal_taxonomy_v3.json"
|
||||
_V3_RESCAN_PATH = _HERE / "refusal_rescan_v3.json"
|
||||
_RESCAN_V4_PATH = _HERE / "refusal_rescan_v4.json"
|
||||
_TAXONOMY_V4_PATH = _HERE / "refusal_taxonomy_v4.json"
|
||||
|
||||
# Matches both refusal-reason shapes the candidate-graph emits:
|
||||
# "no admissible candidate for {statement|question}: '<text>'"
|
||||
# "recognizer matched but produced no injection for statement: '<text>' (category=<c>)"
|
||||
# (PR #359 added the second shape + trailing "(category=...)"). Greedy capture
|
||||
# with an anchored closing quote tolerates statements containing apostrophes
|
||||
# (e.g. "Rudolph's"); the optional category suffix is consumed after the quote.
|
||||
_FIRST_REFUSAL_RE = re.compile(
|
||||
r"(?:no admissible candidate for|recognizer matched but produced no injection for) "
|
||||
r"(?:statement|question): ['\"](.+)['\"](?:\s*\(category=[^)]*\))?\s*$"
|
||||
)
|
||||
|
||||
# Barrier reclassification for cases whose first-refusal sentence changed since v3.
|
||||
# This rescan re-runs the LIVE reader (see module docstring), so it tracks the
|
||||
# reader's current divergence from the v3 baseline — not only the S.4 cut.
|
||||
#
|
||||
# S.4 originally shifted two cases (0038, 0046) by widening initial-state subject
|
||||
# shapes (indefinite-article + prepositional-prefix existential). Later capability
|
||||
# waves (ADR-0163-D.2 discrete-count, 0174 held-hypothesis, 0178 compose, 0191
|
||||
# completeness guard, 0192 discrete_count) advanced the reader past five more
|
||||
# v3-era barriers, shifting their first-refusal one sentence deeper. All shifts
|
||||
# are soundness-preserving (the cases still refuse; wrong stays 0). Each shifted
|
||||
# case carries an override documenting its new first-refusal barrier.
|
||||
# NOTE: because this asserts against the live reader, future reader advances will
|
||||
# add shifts here — a follow-up should cut a frozen v5 snapshot or derive the
|
||||
# expected set from the committed artifact instead of the live run.
|
||||
_V4_BARRIER_OVERRIDES: dict[str, dict[str, Any]] = {
|
||||
"gsm8k-train-sample-v1-0019": {
|
||||
"primary_barrier": "percentage_rate",
|
||||
"secondary_barriers": ["rate_price"],
|
||||
"notes": (
|
||||
"Reader now resolves the prior '$400 per vet appointment' barrier; "
|
||||
"first refusal moved to 'After the first appointment, John paid $100 "
|
||||
"for pet insurance that covers 80% of the subsequent visits' — needs "
|
||||
"percentage_rate (80% of a subsequent-visit set)."
|
||||
),
|
||||
},
|
||||
"gsm8k-train-sample-v1-0023": {
|
||||
"primary_barrier": "compound_comparative",
|
||||
"secondary_barriers": ["fraction_operand"],
|
||||
"notes": (
|
||||
"Reader now resolves 'Nicole collected 400 Pokemon cards' (initial); "
|
||||
"refuses on 'Cindy collected twice as many, and Rex collected half of "
|
||||
"Nicole and Cindy's combined total' — needs compound_comparative "
|
||||
"('twice as many') plus fraction_operand ('half of combined')."
|
||||
),
|
||||
},
|
||||
"gsm8k-train-sample-v1-0025": {
|
||||
"primary_barrier": "distributive_multiply",
|
||||
"secondary_barriers": ["complex_question"],
|
||||
"notes": (
|
||||
"Reader now resolves 'Lilibeth fills 6 baskets where each basket holds "
|
||||
"50 strawberries'; refuses on the question 'If three of Lilibeth's "
|
||||
"friends pick the same amount as her, how many ... in all?' — needs "
|
||||
"distributive_multiply (3 friends x her amount) over a total question."
|
||||
),
|
||||
},
|
||||
"gsm8k-train-sample-v1-0027": {
|
||||
"primary_barrier": "fraction_operand",
|
||||
"secondary_barriers": ["compound_statement"],
|
||||
"notes": (
|
||||
"Reader now resolves 'Malcolm has 240 followers on Instagram and 500 "
|
||||
"on Facebook'; refuses on 'The number of followers he has on Twitter "
|
||||
"is half the number ... on Instagram and Facebook combined' — needs "
|
||||
"fraction_operand ('half of combined')."
|
||||
),
|
||||
},
|
||||
"gsm8k-train-sample-v1-0047": {
|
||||
"primary_barrier": "partition_divide",
|
||||
"secondary_barriers": [],
|
||||
"notes": (
|
||||
"Reader now resolves 'John bakes 12 coconut macaroons, each weighing "
|
||||
"5 ounces'; refuses on 'He then packs an equal number of the macaroons "
|
||||
"in 4 different brown bags' — needs partition_divide (12 / 4 equal)."
|
||||
),
|
||||
},
|
||||
"gsm8k-train-sample-v1-0038": {
|
||||
"primary_barrier": "compound_comparative",
|
||||
"secondary_barriers": ["aggregate_sum"],
|
||||
"notes": (
|
||||
"Sentence 1 (novel_initial_form) now resolves via S.4's "
|
||||
"_INITIAL_THERE_ARE_PREFIX_RE to (building, 100, ladies). Refuses "
|
||||
"on sentence 2: 'There are three times that many girls at a party "
|
||||
"being held on the second floor of the building' — needs "
|
||||
"compound_comparative parsing ('three times that many') with "
|
||||
"cross-sentence reference to the prior 100."
|
||||
),
|
||||
},
|
||||
"gsm8k-train-sample-v1-0046": {
|
||||
"primary_barrier": "fraction_operand",
|
||||
"secondary_barriers": ["percentage_of"],
|
||||
"notes": (
|
||||
"Sentence 1 (novel_initial_form) now resolves via S.4's "
|
||||
"_INITIAL_HAS_INDEF_RE to (school, 100, students). Refuses on "
|
||||
"sentence 2: 'Half of the students are girls, the other half are "
|
||||
"boys' — needs fraction_operand parsing + downstream percentage_of "
|
||||
"on the 20%/10% sentences."
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _load_cases() -> list[dict[str, Any]]:
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in _CASES_PATH.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
|
||||
|
||||
def _extract_first_refusal(reason: str | None) -> str | None:
|
||||
if reason is None:
|
||||
return None
|
||||
body = reason.removeprefix("candidate_graph: ")
|
||||
m = _FIRST_REFUSAL_RE.search(body)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def _classify_current_barrier(
|
||||
case_id: str,
|
||||
v3_barrier: str,
|
||||
cur_outcome: str,
|
||||
prev_first_ref: str | None,
|
||||
cur_first_ref: str | None,
|
||||
) -> str:
|
||||
if cur_outcome == "admitted":
|
||||
return "admitted"
|
||||
if cur_first_ref == prev_first_ref:
|
||||
return v3_barrier
|
||||
override = _V4_BARRIER_OVERRIDES.get(case_id)
|
||||
if override is None:
|
||||
raise RuntimeError(
|
||||
f"{case_id}: first-refusal shifted but no v4 override registered. "
|
||||
f"v3={prev_first_ref!r} v4={cur_first_ref!r}"
|
||||
)
|
||||
return override["primary_barrier"]
|
||||
|
||||
|
||||
def build_rescan() -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
cases = _load_cases()
|
||||
v3_data = json.loads(_V3_RESCAN_PATH.read_text(encoding="utf-8"))
|
||||
v3_by_id = {c["case_id"]: c for c in v3_data["per_case"]}
|
||||
v3_tax = json.loads(_V3_TAXONOMY_PATH.read_text(encoding="utf-8"))
|
||||
v3_tax_by_id = {c["case_id"]: c for c in v3_tax["per_case"]}
|
||||
|
||||
rescan_per_case: list[dict[str, Any]] = []
|
||||
tax_per_case: list[dict[str, Any]] = []
|
||||
admitted: list[str] = []
|
||||
wrong: list[tuple[str, float, float]] = []
|
||||
barrier_shifted_count = 0
|
||||
|
||||
for c in cases:
|
||||
cid = c["case_id"]
|
||||
r = parse_and_solve(c["question"])
|
||||
cur_outcome = "admitted" if r.answer is not None else "refused"
|
||||
cur_first_ref = _extract_first_refusal(r.refusal_reason)
|
||||
cur_reason = r.refusal_reason
|
||||
|
||||
if cur_outcome == "admitted":
|
||||
if r.answer == c["answer_numeric"]:
|
||||
admitted.append(cid)
|
||||
else:
|
||||
wrong.append((cid, r.answer, c["answer_numeric"]))
|
||||
|
||||
prev = v3_by_id[cid]
|
||||
prev_outcome = prev["current_outcome"]
|
||||
prev_first_ref = prev["current_first_refusal"]
|
||||
prev_barrier = v3_tax_by_id[cid]["primary_barrier"]
|
||||
cur_barrier = _classify_current_barrier(
|
||||
cid, prev_barrier, cur_outcome, prev_first_ref, cur_first_ref
|
||||
)
|
||||
|
||||
outcome_changed = cur_outcome != prev_outcome
|
||||
first_ref_changed = cur_first_ref != prev_first_ref
|
||||
barrier_changed = cur_barrier != prev_barrier
|
||||
shifted = outcome_changed or first_ref_changed or barrier_changed
|
||||
if shifted:
|
||||
barrier_shifted_count += 1
|
||||
|
||||
rescan_per_case.append({
|
||||
"case_id": cid,
|
||||
"previous_outcome": prev_outcome,
|
||||
"current_outcome": cur_outcome,
|
||||
"previous_primary_barrier": prev_barrier,
|
||||
"current_primary_barrier": cur_barrier,
|
||||
"barrier_shifted": shifted,
|
||||
"new_admission": cur_outcome == "admitted" and prev_outcome != "admitted",
|
||||
"previous_first_refusal": prev_first_ref,
|
||||
"current_first_refusal": cur_first_ref,
|
||||
"current_refusal_reason": cur_reason,
|
||||
"notes": _V4_BARRIER_OVERRIDES.get(cid, {}).get("notes", ""),
|
||||
})
|
||||
|
||||
v3_tax_entry = v3_tax_by_id[cid]
|
||||
if cid in _V4_BARRIER_OVERRIDES and cur_outcome == "refused":
|
||||
ov = _V4_BARRIER_OVERRIDES[cid]
|
||||
tax_per_case.append({
|
||||
"case_id": cid,
|
||||
"primary_barrier": ov["primary_barrier"],
|
||||
"secondary_barriers": ov["secondary_barriers"],
|
||||
"notes": ov["notes"],
|
||||
})
|
||||
else:
|
||||
tax_per_case.append({
|
||||
"case_id": cid,
|
||||
"primary_barrier": cur_barrier,
|
||||
"secondary_barriers": v3_tax_entry.get("secondary_barriers", []),
|
||||
"notes": v3_tax_entry.get("notes", ""),
|
||||
})
|
||||
|
||||
if wrong:
|
||||
raise RuntimeError(f"wrong admissions detected: {wrong}")
|
||||
|
||||
rescan = {
|
||||
"schema_version": 1,
|
||||
"adr": "0136.S.4-post-rescan",
|
||||
"description": (
|
||||
"Per-case barrier-shift ledger comparing v3 (post-S.3) to v4 "
|
||||
"(post-S.4) behavior of parse_and_solve."
|
||||
),
|
||||
"summary": {
|
||||
"total_cases": len(cases),
|
||||
"admitted": len(admitted),
|
||||
"wrong": len(wrong),
|
||||
"refused": len(cases) - len(admitted),
|
||||
"barrier_shifted_v3_to_v4": barrier_shifted_count,
|
||||
},
|
||||
"per_case": rescan_per_case,
|
||||
}
|
||||
|
||||
taxonomy = {
|
||||
"schema_version": 2,
|
||||
"adr": "0136.S.4-post-rescan",
|
||||
"description": (
|
||||
"Post-S.4 refusal taxonomy. Schema identical to v3; primary_barrier "
|
||||
"vocabulary unchanged."
|
||||
),
|
||||
"summary": {
|
||||
"total_cases": len(cases),
|
||||
"admitted": len(admitted),
|
||||
"wrong": len(wrong),
|
||||
"refused": len(cases) - len(admitted),
|
||||
},
|
||||
"per_case": tax_per_case,
|
||||
}
|
||||
|
||||
return rescan, taxonomy
|
||||
|
||||
|
||||
def main() -> int:
|
||||
rescan, taxonomy = build_rescan()
|
||||
_RESCAN_V4_PATH.write_text(
|
||||
json.dumps(rescan, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
_TAXONOMY_V4_PATH.write_text(
|
||||
json.dumps(taxonomy, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
print(
|
||||
f"v4 rescan: admitted={rescan['summary']['admitted']}/"
|
||||
f"{rescan['summary']['total_cases']} wrong={rescan['summary']['wrong']} "
|
||||
f"shifted_v3_to_v4={rescan['summary']['barrier_shifted_v3_to_v4']}"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
|
@ -1,125 +0,0 @@
|
|||
"""ADR-0136.S.4-post-rescan — invariant tests for refusal rescan v4.
|
||||
|
||||
v4 is the CURRENT snapshot (post-S.4). Unlike v3 (which is now frozen and
|
||||
asserted against disk), v4 tests re-run build_rescan() to confirm the live
|
||||
parser still produces the v4 ledger — until S.5 lands and shifts more.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from evals.gsm8k_math.train_sample.v1.rescan_v4 import build_rescan
|
||||
|
||||
_HERE = Path(__file__).resolve().parent.parent
|
||||
_RESCAN_V4 = _HERE / "evals/gsm8k_math/train_sample/v1/refusal_rescan_v4.json"
|
||||
_TAXONOMY_V4 = _HERE / "evals/gsm8k_math/train_sample/v1/refusal_taxonomy_v4.json"
|
||||
|
||||
|
||||
def test_wrong_is_zero() -> None:
|
||||
rescan, _ = build_rescan()
|
||||
assert rescan["summary"]["wrong"] == 0
|
||||
|
||||
|
||||
def test_admission_set_grew_monotonically_over_v3() -> None:
|
||||
"""The reader admits the v3 set plus three new *correct* solves.
|
||||
|
||||
v3 admitted {0014, 0018, 0042}. Later capability waves added three correct
|
||||
admissions (0003, 0021, 0024) — the +3 that moved the serving metric from
|
||||
3 to 6. No v3 admission was lost (monotonic), and wrong stays 0
|
||||
(test_wrong_is_zero), so every gain is a sound solve.
|
||||
"""
|
||||
rescan, _ = build_rescan()
|
||||
admitted = {
|
||||
c["case_id"] for c in rescan["per_case"] if c["current_outcome"] == "admitted"
|
||||
}
|
||||
v3_admitted = {
|
||||
"gsm8k-train-sample-v1-0014",
|
||||
"gsm8k-train-sample-v1-0018",
|
||||
"gsm8k-train-sample-v1-0042",
|
||||
}
|
||||
# No regression: every case v3 solved is still solved.
|
||||
assert v3_admitted <= admitted
|
||||
assert admitted == v3_admitted | {
|
||||
"gsm8k-train-sample-v1-0003",
|
||||
"gsm8k-train-sample-v1-0021",
|
||||
"gsm8k-train-sample-v1-0024",
|
||||
}
|
||||
|
||||
|
||||
def test_barrier_shifts_from_v3_baseline() -> None:
|
||||
"""The live reader has diverged from the v3 baseline on ten cases.
|
||||
|
||||
S.4 shifted two (0038, 0046). Later capability waves (0163-D.2, 0174, 0178,
|
||||
0191, 0192) advanced the reader past five more v3-era barriers (0019, 0023,
|
||||
0025, 0027, 0047) — still refusing, one sentence deeper — and turned three
|
||||
refusals into correct admissions (0003, 0021, 0024). All ten are
|
||||
soundness-preserving (wrong stays 0); the five still-refusing shifts each
|
||||
carry an override in _V4_BARRIER_OVERRIDES.
|
||||
"""
|
||||
rescan, _ = build_rescan()
|
||||
# 10 = 7 first-refusal/barrier shifts + 3 refused->admitted transitions
|
||||
# (0003, 0021, 0024), which also count as shifts (outcome changed).
|
||||
assert rescan["summary"]["barrier_shifted_v3_to_v4"] == 10
|
||||
shifted = sorted(
|
||||
c["case_id"] for c in rescan["per_case"] if c["barrier_shifted"]
|
||||
)
|
||||
assert shifted == [
|
||||
"gsm8k-train-sample-v1-0003",
|
||||
"gsm8k-train-sample-v1-0019",
|
||||
"gsm8k-train-sample-v1-0021",
|
||||
"gsm8k-train-sample-v1-0023",
|
||||
"gsm8k-train-sample-v1-0024",
|
||||
"gsm8k-train-sample-v1-0025",
|
||||
"gsm8k-train-sample-v1-0027",
|
||||
"gsm8k-train-sample-v1-0038",
|
||||
"gsm8k-train-sample-v1-0046",
|
||||
"gsm8k-train-sample-v1-0047",
|
||||
]
|
||||
|
||||
|
||||
def test_gsm8k_0038_now_refuses_on_sentence_2() -> None:
|
||||
rescan, _ = build_rescan()
|
||||
case = next(
|
||||
c for c in rescan["per_case"]
|
||||
if c["case_id"] == "gsm8k-train-sample-v1-0038"
|
||||
)
|
||||
assert case["previous_first_refusal"].startswith("In a building")
|
||||
assert case["current_first_refusal"].startswith("There are three times")
|
||||
assert case["previous_primary_barrier"] == "novel_initial_form"
|
||||
assert case["current_primary_barrier"] == "compound_comparative"
|
||||
|
||||
|
||||
def test_gsm8k_0046_now_refuses_on_sentence_2() -> None:
|
||||
rescan, _ = build_rescan()
|
||||
case = next(
|
||||
c for c in rescan["per_case"]
|
||||
if c["case_id"] == "gsm8k-train-sample-v1-0046"
|
||||
)
|
||||
assert case["previous_first_refusal"].startswith("A school has 100 students")
|
||||
assert case["current_first_refusal"].startswith("Half of the students")
|
||||
assert case["previous_primary_barrier"] == "novel_initial_form"
|
||||
assert case["current_primary_barrier"] == "fraction_operand"
|
||||
|
||||
|
||||
def test_artifacts_deterministic() -> None:
|
||||
r1, t1 = build_rescan()
|
||||
r2, t2 = build_rescan()
|
||||
assert json.dumps(r1, sort_keys=True) == json.dumps(r2, sort_keys=True)
|
||||
assert json.dumps(t1, sort_keys=True) == json.dumps(t2, sort_keys=True)
|
||||
|
||||
|
||||
def test_taxonomy_v4_has_50_entries() -> None:
|
||||
_, taxonomy = build_rescan()
|
||||
assert len(taxonomy["per_case"]) == 50
|
||||
|
||||
|
||||
def test_disk_artifacts_match_freshly_computed() -> None:
|
||||
if not _RESCAN_V4.exists() or not _TAXONOMY_V4.exists():
|
||||
return
|
||||
rescan, taxonomy = build_rescan()
|
||||
on_disk_rescan = json.loads(_RESCAN_V4.read_text(encoding="utf-8"))
|
||||
on_disk_taxonomy = json.loads(_TAXONOMY_V4.read_text(encoding="utf-8"))
|
||||
assert on_disk_rescan == rescan
|
||||
assert on_disk_taxonomy == taxonomy
|
||||
Loading…
Reference in a new issue