diff --git a/evals/gsm8k_math/train_sample/v1/audit_brief_11.json b/evals/gsm8k_math/train_sample/v1/audit_brief_11.json new file mode 100644 index 00000000..a4ceb242 --- /dev/null +++ b/evals/gsm8k_math/train_sample/v1/audit_brief_11.json @@ -0,0 +1,1209 @@ +{ + "schema_version": 1, + "brief": "Brief 11B \u2014 reader closure audit (per-case)", + "adr_chain": [ + "0164", + "0164.3", + "0164.4", + "0166" + ], + "source_dataset": "gsm8k", + "sample_path": "evals/gsm8k_math/train_sample/v1/cases.jsonl", + "case_count": 50, + "summary": { + "admitted": 0, + "refused": 50, + "refusal_reasons": { + "unknown_word": 11, + "unexpected_category": 14, + "incomplete_operation": 18, + "no_question_target": 1, + "unattached_quantity": 3, + "unresolved_pronoun": 3 + }, + "missing_operators": { + "lexicon_entry": 9, + "pre_frame_filler_sentence": 8, + "descriptive_frame_question": 2, + "fraction_percentage_literal": 3, + "multi_quantity_composition": 8, + "quantity_extraction": 9, + "question_target_slot": 1, + "unit_binding": 3, + "pronoun_resolution": 3, + "compound_time_literal": 1, + "multi_subject_sentence": 1, + "question_frame_slot": 1, + "compound_numeric_literal": 1 + } + }, + "invariants": { + "wrong_count": 0, + "reader_runtime_changes_in_this_pr": false + }, + "per_case": [ + { + "case_id": "gsm8k-train-sample-v1-0001", + "outcome": "refused", + "refusal_reason": "unknown_word", + "missing_operator": "lexicon_entry", + "refusal_detail": "no primitive or lexicon match for 'which'", + "sentence_index": 1, + "token_index": 16, + "token_text": "which", + "recognized_terms": [ + "If", + "she", + "works", + "more", + "than", + "8", + "hours", + "per", + "shift", + ",", + "she", + "is", + "eligible", + "for", + "overtime", + "," + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0002", + "outcome": "refused", + "refusal_reason": "unexpected_category", + "missing_operator": "pre_frame_filler_sentence", + "refusal_detail": "category 'statement_terminator' (word='.') at pre-frame position 7 not handled; may be Phase-3 scope", + "sentence_index": 1, + "token_index": 7, + "token_text": ".", + "recognized_terms": [ + "She", + "splits", + "it", + "up", + "into", + "25-foot", + "sections" + ], + "skipped_frame": null + }, + { + "case_id": "gsm8k-train-sample-v1-0003", + "outcome": "refused", + "refusal_reason": "unexpected_category", + "missing_operator": "descriptive_frame_question", + "refusal_detail": "category 'question_terminator' (word='?') not drainable in descriptive_frame", + "sentence_index": 3, + "token_index": 17, + "token_text": "?", + "recognized_terms": [ + "If", + "the", + "student", + "council", + "sells", + "the", + "erasers", + "for", + "$0.75", + "each", + ",", + "how", + "much", + "money", + "will", + "they", + "make" + ], + "skipped_frame": "descriptive_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0004", + "outcome": "refused", + "refusal_reason": "unexpected_category", + "missing_operator": "fraction_percentage_literal", + "refusal_detail": "fraction/percentage literal at position 11 is out-of-scope (embedded-quantifier aggregate; deferred to Phase 2.1)", + "sentence_index": 1, + "token_index": 11, + "token_text": "1/4", + "recognized_terms": [ + "Half", + "of", + "the", + "kids", + "are", + "going", + "to", + "soccer", + "camp", + ",", + "and" + ], + "skipped_frame": "descriptive_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0005", + "outcome": "refused", + "refusal_reason": "unknown_word", + "missing_operator": "lexicon_entry", + "refusal_detail": "no primitive or lexicon match for 'decrease'", + "sentence_index": 0, + "token_index": 8, + "token_text": "decrease", + "recognized_terms": [ + "In", + "one", + "hour", + ",", + "Addison", + "mountain's", + "temperature", + "will" + ], + "skipped_frame": null + }, + { + "case_id": "gsm8k-train-sample-v1-0006", + "outcome": "refused", + "refusal_reason": "unknown_word", + "missing_operator": "lexicon_entry", + "refusal_detail": "no primitive or lexicon match for 'previous'", + "sentence_index": 2, + "token_index": 10, + "token_text": "previous", + "recognized_terms": [ + "Presently", + ",", + "she", + "reads", + "books", + "that", + "are", + "4", + "times", + "the" + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0007", + "outcome": "refused", + "refusal_reason": "incomplete_operation", + "missing_operator": "multi_quantity_composition", + "refusal_detail": "initial_state_frame has 2 quantities; multi-quantity initial state is Phase-2.1 scope", + "sentence_index": 0, + "token_index": 19, + "token_text": "", + "recognized_terms": [ + "Francine", + "has", + "five", + "full", + "boxes", + "of", + "crayons", + "and", + "5", + "loose", + "crayons", + ",", + "and", + "her", + "friend", + "has", + "27", + "loose", + "crayons", + "." + ], + "skipped_frame": "initial_state_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0008", + "outcome": "refused", + "refusal_reason": "incomplete_operation", + "missing_operator": "quantity_extraction", + "refusal_detail": "operation_frame closed with no quantity", + "sentence_index": 0, + "token_index": 4, + "token_text": "", + "recognized_terms": [ + "Marnie", + "makes", + "bead", + "bracelets", + "." + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0009", + "outcome": "refused", + "refusal_reason": "no_question_target", + "missing_operator": "question_target_slot", + "refusal_detail": "ProblemReadingState has no unknown_target_slot after finalize", + "sentence_index": 2, + "token_index": 0, + "token_text": "", + "recognized_terms": [], + "skipped_frame": null + }, + { + "case_id": "gsm8k-train-sample-v1-0010", + "outcome": "refused", + "refusal_reason": "unattached_quantity", + "missing_operator": "unit_binding", + "refusal_detail": "1 quantities never attached to entity+unit at sentence end", + "sentence_index": 0, + "token_index": 10, + "token_text": "", + "recognized_terms": [ + "Yun", + "had", + "20", + "paperclips", + "initially", + ",", + "but", + "then", + "lost", + "12", + "." + ], + "skipped_frame": "initial_state_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0011", + "outcome": "refused", + "refusal_reason": "unattached_quantity", + "missing_operator": "unit_binding", + "refusal_detail": "1 quantities never attached to entity+unit at sentence end", + "sentence_index": 1, + "token_index": 23, + "token_text": "", + "recognized_terms": [ + "If", + "she", + "spent", + "$20", + "on", + "ingredients", + ",", + "how", + "many", + "cups", + "of", + "lemonade", + "does", + "she", + "need", + "to", + "sell", + "to", + "make", + "a", + "profit", + "of", + "$80", + "?" + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0012", + "outcome": "refused", + "refusal_reason": "unresolved_pronoun", + "missing_operator": "pronoun_resolution", + "refusal_detail": "pronoun 'them' in operation_frame has no compatible entity", + "sentence_index": 1, + "token_index": 14, + "token_text": "them", + "recognized_terms": [ + "He", + "put", + "all", + "of", + "them", + "in", + "his", + "aquarium", + "but", + "his", + "fish", + "ate", + "half", + "of" + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0013", + "outcome": "refused", + "refusal_reason": "unknown_word", + "missing_operator": "compound_time_literal", + "refusal_detail": "no primitive or lexicon match for 'one-hour'", + "sentence_index": 0, + "token_index": 7, + "token_text": "one-hour", + "recognized_terms": [ + "Allison", + ",", + "a", + "YouTuber", + ",", + "uploads", + "10" + ], + "skipped_frame": null + }, + { + "case_id": "gsm8k-train-sample-v1-0014", + "outcome": "refused", + "refusal_reason": "incomplete_operation", + "missing_operator": "multi_quantity_composition", + "refusal_detail": "operation_frame has 2 quantities; multi-quantity operations are Phase-2.1 scope", + "sentence_index": 0, + "token_index": 8, + "token_text": "", + "recognized_terms": [ + "Bob", + "can", + "shuck", + "10", + "oysters", + "in", + "5", + "minutes", + "." + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0015", + "outcome": "refused", + "refusal_reason": "unknown_word", + "missing_operator": "lexicon_entry", + "refusal_detail": "no primitive or lexicon match for 'reach'", + "sentence_index": 1, + "token_index": 7, + "token_text": "reach", + "recognized_terms": [ + "What's", + "the", + "total", + "time", + "he", + "takes", + "to" + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0016", + "outcome": "refused", + "refusal_reason": "unexpected_category", + "missing_operator": "pre_frame_filler_sentence", + "refusal_detail": "category 'statement_terminator' (word='.') at pre-frame position 22 not handled; may be Phase-3 scope", + "sentence_index": 0, + "token_index": 22, + "token_text": ".", + "recognized_terms": [ + "On", + "Rudolph's", + "car", + "trip", + "across", + "town", + ",", + "he", + "traveled", + "2", + "more", + "than", + "5", + "miles", + "and", + "encountered", + "3", + "less", + "than", + "17", + "stop", + "signs" + ], + "skipped_frame": null + }, + { + "case_id": "gsm8k-train-sample-v1-0017", + "outcome": "refused", + "refusal_reason": "incomplete_operation", + "missing_operator": "quantity_extraction", + "refusal_detail": "initial_state_frame closed with no quantity", + "sentence_index": 0, + "token_index": 9, + "token_text": "", + "recognized_terms": [ + "Jason", + "has", + "a", + "carriage", + "house", + "that", + "he", + "rents", + "out", + "." + ], + "skipped_frame": "initial_state_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0018", + "outcome": "refused", + "refusal_reason": "unresolved_pronoun", + "missing_operator": "pronoun_resolution", + "refusal_detail": "pronoun 'his' in operation_frame has no compatible entity", + "sentence_index": 0, + "token_index": 4, + "token_text": "his", + "recognized_terms": [ + "Xavier", + "plays", + "football", + "with" + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0019", + "outcome": "refused", + "refusal_reason": "incomplete_operation", + "missing_operator": "quantity_extraction", + "refusal_detail": "operation_frame closed with no quantity", + "sentence_index": 0, + "token_index": 7, + "token_text": "", + "recognized_terms": [ + "John", + "adopts", + "a", + "dog", + "from", + "a", + "shelter", + "." + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0020", + "outcome": "refused", + "refusal_reason": "unexpected_category", + "missing_operator": "descriptive_frame_question", + "refusal_detail": "category 'question_terminator' (word='?') not drainable in descriptive_frame", + "sentence_index": 2, + "token_index": 28, + "token_text": "?", + "recognized_terms": [ + "If", + "the", + "cost", + "of", + "one", + "parakeet", + "was", + "$10", + ",", + "what", + "would", + "it", + "cost", + "to", + "purchase", + "all", + "of", + "the", + "pets", + "for", + "sale", + "at", + "the", + "pet", + "shop", + ",", + "in", + "dollars" + ], + "skipped_frame": "descriptive_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0021", + "outcome": "refused", + "refusal_reason": "unknown_word", + "missing_operator": "lexicon_entry", + "refusal_detail": "no primitive or lexicon match for 'weights'", + "sentence_index": 0, + "token_index": 3, + "token_text": "weights", + "recognized_terms": [ + "John", + "is", + "lifting" + ], + "skipped_frame": "descriptive_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0022", + "outcome": "refused", + "refusal_reason": "unresolved_pronoun", + "missing_operator": "pronoun_resolution", + "refusal_detail": "pronoun 'their' in operation_frame has no compatible entity", + "sentence_index": 0, + "token_index": 11, + "token_text": "their", + "recognized_terms": [ + "Erica", + "lives", + "near", + "a", + "lake", + "where", + "most", + "locals", + "sell", + "fish", + "as" + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0023", + "outcome": "refused", + "refusal_reason": "incomplete_operation", + "missing_operator": "quantity_extraction", + "refusal_detail": "operation_frame closed with no quantity", + "sentence_index": 1, + "token_index": 16, + "token_text": "", + "recognized_terms": [ + "Cindy", + "collected", + "twice", + "as", + "many", + ",", + "and", + "Rex", + "collected", + "half", + "of", + "Nicole", + "and", + "Cindy's", + "combined", + "total", + "." + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0024", + "outcome": "refused", + "refusal_reason": "unknown_word", + "missing_operator": "lexicon_entry", + "refusal_detail": "no primitive or lexicon match for 'jacks'", + "sentence_index": 0, + "token_index": 4, + "token_text": "jacks", + "recognized_terms": [ + "Sidney", + "does", + "20", + "jumping" + ], + "skipped_frame": null + }, + { + "case_id": "gsm8k-train-sample-v1-0025", + "outcome": "refused", + "refusal_reason": "unexpected_category", + "missing_operator": "pre_frame_filler_sentence", + "refusal_detail": "category 'statement_terminator' (word='.') at pre-frame position 7 not handled; may be Phase-3 scope", + "sentence_index": 0, + "token_index": 7, + "token_text": ".", + "recognized_terms": [ + "Lilibeth", + "and", + "her", + "friends", + "go", + "strawberry", + "picking" + ], + "skipped_frame": null + }, + { + "case_id": "gsm8k-train-sample-v1-0026", + "outcome": "refused", + "refusal_reason": "unexpected_category", + "missing_operator": "multi_subject_sentence", + "refusal_detail": "second entity 'Carson' at pre-frame position 4; multi-subject sentences are Phase-2.1 scope", + "sentence_index": 0, + "token_index": 4, + "token_text": "Carson", + "recognized_terms": [ + "Aaron", + "and", + "his", + "brother" + ], + "skipped_frame": null + }, + { + "case_id": "gsm8k-train-sample-v1-0027", + "outcome": "refused", + "refusal_reason": "incomplete_operation", + "missing_operator": "multi_quantity_composition", + "refusal_detail": "initial_state_frame has 2 quantities; multi-quantity initial state is Phase-2.1 scope", + "sentence_index": 0, + "token_index": 11, + "token_text": "", + "recognized_terms": [ + "Malcolm", + "has", + "240", + "followers", + "on", + "Instagram", + "and", + "500", + "followers", + "on", + "Facebook", + "." + ], + "skipped_frame": "initial_state_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0028", + "outcome": "refused", + "refusal_reason": "unexpected_category", + "missing_operator": "pre_frame_filler_sentence", + "refusal_detail": "category 'statement_terminator' (word='.') at pre-frame position 5 not handled; may be Phase-3 scope", + "sentence_index": 0, + "token_index": 5, + "token_text": ".", + "recognized_terms": [ + "Tom", + "opens", + "an", + "amusement", + "park" + ], + "skipped_frame": null + }, + { + "case_id": "gsm8k-train-sample-v1-0029", + "outcome": "refused", + "refusal_reason": "incomplete_operation", + "missing_operator": "quantity_extraction", + "refusal_detail": "operation_frame closed with no quantity", + "sentence_index": 0, + "token_index": 16, + "token_text": "", + "recognized_terms": [ + "Fabian", + "bought", + "a", + "brand", + "new", + "computer", + "mouse", + "and", + "keyboard", + "to", + "be", + "able", + "to", + "work", + "from", + "home", + "." + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0030", + "outcome": "refused", + "refusal_reason": "unexpected_category", + "missing_operator": "pre_frame_filler_sentence", + "refusal_detail": "category 'statement_terminator' (word='.') at pre-frame position 11 not handled; may be Phase-3 scope", + "sentence_index": 0, + "token_index": 11, + "token_text": ".", + "recognized_terms": [ + "Jake", + "decides", + "to", + "go", + "to", + "the", + "beach", + "for", + "a", + "fun", + "day" + ], + "skipped_frame": null + }, + { + "case_id": "gsm8k-train-sample-v1-0031", + "outcome": "refused", + "refusal_reason": "unattached_quantity", + "missing_operator": "unit_binding", + "refusal_detail": "1 quantities never attached to entity+unit at sentence end", + "sentence_index": 0, + "token_index": 16, + "token_text": "", + "recognized_terms": [ + "Jeremie", + "wants", + "to", + "go", + "to", + "an", + "amusement", + "park", + "with", + "3", + "friends", + "at", + "the", + "end", + "of", + "summer", + "." + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0032", + "outcome": "refused", + "refusal_reason": "incomplete_operation", + "missing_operator": "quantity_extraction", + "refusal_detail": "operation_frame closed with no quantity", + "sentence_index": 0, + "token_index": 6, + "token_text": "", + "recognized_terms": [ + "John", + "decides", + "to", + "take", + "up", + "illustration", + "." + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0033", + "outcome": "refused", + "refusal_reason": "incomplete_operation", + "missing_operator": "question_frame_slot", + "refusal_detail": "question_frame missing required slot(s): unit_class", + "sentence_index": 2, + "token_index": 12, + "token_text": "", + "recognized_terms": [ + "How", + "old", + "will", + "Rachel's", + "father", + "be", + "when", + "she", + "is", + "25", + "years", + "old", + "?" + ], + "skipped_frame": "question_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0034", + "outcome": "refused", + "refusal_reason": "incomplete_operation", + "missing_operator": "multi_quantity_composition", + "refusal_detail": "operation_frame has 2 quantities; multi-quantity operations are Phase-2.1 scope", + "sentence_index": 1, + "token_index": 8, + "token_text": "", + "recognized_terms": [ + "He", + "can", + "run", + "40", + "yards", + "within", + "5", + "seconds", + "." + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0035", + "outcome": "refused", + "refusal_reason": "unexpected_category", + "missing_operator": "pre_frame_filler_sentence", + "refusal_detail": "category 'statement_terminator' (word='.') at pre-frame position 8 not handled; may be Phase-3 scope", + "sentence_index": 1, + "token_index": 8, + "token_text": ".", + "recognized_terms": [ + "She", + "decided", + "to", + "split", + "them", + "among", + "her", + "friends" + ], + "skipped_frame": null + }, + { + "case_id": "gsm8k-train-sample-v1-0036", + "outcome": "refused", + "refusal_reason": "unexpected_category", + "missing_operator": "pre_frame_filler_sentence", + "refusal_detail": "category 'statement_terminator' (word='.') at pre-frame position 6 not handled; may be Phase-3 scope", + "sentence_index": 0, + "token_index": 6, + "token_text": ".", + "recognized_terms": [ + "Monica", + "way", + "studying", + "for", + "an", + "exam" + ], + "skipped_frame": null + }, + { + "case_id": "gsm8k-train-sample-v1-0037", + "outcome": "refused", + "refusal_reason": "incomplete_operation", + "missing_operator": "multi_quantity_composition", + "refusal_detail": "operation_frame has 2 quantities; multi-quantity operations are Phase-2.1 scope", + "sentence_index": 1, + "token_index": 11, + "token_text": "", + "recognized_terms": [ + "He", + "lost", + "3", + "pounds", + "in", + "March", + "and", + "4", + "pounds", + "in", + "April", + "." + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0038", + "outcome": "refused", + "refusal_reason": "unknown_word", + "missing_operator": "compound_numeric_literal", + "refusal_detail": "no primitive or lexicon match for 'hundred'", + "sentence_index": 0, + "token_index": 7, + "token_text": "hundred", + "recognized_terms": [ + "In", + "a", + "building", + ",", + "there", + "are", + "a" + ], + "skipped_frame": "descriptive_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0039", + "outcome": "refused", + "refusal_reason": "unknown_word", + "missing_operator": "lexicon_entry", + "refusal_detail": "no primitive or lexicon match for 'too'", + "sentence_index": 0, + "token_index": 7, + "token_text": "too", + "recognized_terms": [ + "At", + "the", + "family", + "reunion", + ",", + "everyone", + "ate" + ], + "skipped_frame": "descriptive_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0040", + "outcome": "refused", + "refusal_reason": "unknown_word", + "missing_operator": "lexicon_entry", + "refusal_detail": "no primitive or lexicon match for 'stray'", + "sentence_index": 0, + "token_index": 8, + "token_text": "stray", + "recognized_terms": [ + "Over", + "several", + "years", + ",", + "Daniel", + "has", + "adopted", + "any" + ], + "skipped_frame": "initial_state_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0041", + "outcome": "refused", + "refusal_reason": "incomplete_operation", + "missing_operator": "multi_quantity_composition", + "refusal_detail": "operation_frame has 2 quantities; multi-quantity operations are Phase-2.1 scope", + "sentence_index": 0, + "token_index": 13, + "token_text": "", + "recognized_terms": [ + "Troy", + "bakes", + "2", + "pans", + "of", + "brownies", + ",", + "cut", + "into", + "16", + "pieces", + "per", + "pan", + "." + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0042", + "outcome": "refused", + "refusal_reason": "incomplete_operation", + "missing_operator": "multi_quantity_composition", + "refusal_detail": "initial_state_frame has 3 quantities; multi-quantity initial state is Phase-2.1 scope", + "sentence_index": 0, + "token_index": 19, + "token_text": "", + "recognized_terms": [ + "Ella", + "has", + "4", + "bags", + "with", + "20", + "apples", + "in", + "each", + "bag", + "and", + "six", + "bags", + "with", + "25", + "apples", + "in", + "each", + "bag", + "." + ], + "skipped_frame": "initial_state_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0043", + "outcome": "refused", + "refusal_reason": "incomplete_operation", + "missing_operator": "quantity_extraction", + "refusal_detail": "operation_frame closed with no quantity", + "sentence_index": 0, + "token_index": 6, + "token_text": "", + "recognized_terms": [ + "Sandra", + "wants", + "to", + "buy", + "some", + "sweets", + "." + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0044", + "outcome": "refused", + "refusal_reason": "unexpected_category", + "missing_operator": "fraction_percentage_literal", + "refusal_detail": "fraction/percentage literal at position 7 is out-of-scope (embedded-quantifier aggregate; deferred to Phase 2.1)", + "sentence_index": 0, + "token_index": 7, + "token_text": "10%", + "recognized_terms": [ + "John", + "invests", + "in", + "a", + "bank", + "and", + "gets" + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0045", + "outcome": "refused", + "refusal_reason": "incomplete_operation", + "missing_operator": "quantity_extraction", + "refusal_detail": "operation_frame closed with no quantity", + "sentence_index": 0, + "token_index": 7, + "token_text": "", + "recognized_terms": [ + "Bart", + "fills", + "out", + "surveys", + "to", + "earn", + "money", + "." + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0046", + "outcome": "refused", + "refusal_reason": "unexpected_category", + "missing_operator": "fraction_percentage_literal", + "refusal_detail": "fraction/percentage literal at position 0 is out-of-scope (embedded-quantifier aggregate; deferred to Phase 2.1)", + "sentence_index": 2, + "token_index": 0, + "token_text": "20%", + "recognized_terms": [], + "skipped_frame": null + }, + { + "case_id": "gsm8k-train-sample-v1-0047", + "outcome": "refused", + "refusal_reason": "incomplete_operation", + "missing_operator": "multi_quantity_composition", + "refusal_detail": "operation_frame has 2 quantities; multi-quantity operations are Phase-2.1 scope", + "sentence_index": 0, + "token_index": 10, + "token_text": "", + "recognized_terms": [ + "John", + "bakes", + "12", + "coconut", + "macaroons", + ",", + "each", + "weighing", + "5", + "ounces", + "." + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0048", + "outcome": "refused", + "refusal_reason": "incomplete_operation", + "missing_operator": "quantity_extraction", + "refusal_detail": "operation_frame closed with no quantity", + "sentence_index": 0, + "token_index": 4, + "token_text": "", + "recognized_terms": [ + "Jed", + "collects", + "stamp", + "cards", + "." + ], + "skipped_frame": "operation_frame" + }, + { + "case_id": "gsm8k-train-sample-v1-0049", + "outcome": "refused", + "refusal_reason": "unknown_word", + "missing_operator": "lexicon_entry", + "refusal_detail": "no primitive or lexicon match for 'uphill'", + "sentence_index": 1, + "token_index": 10, + "token_text": "uphill", + "recognized_terms": [ + "In", + "the", + "first", + "route", + ",", + "he", + "walks", + "for", + "6", + "minutes" + ], + "skipped_frame": null + }, + { + "case_id": "gsm8k-train-sample-v1-0050", + "outcome": "refused", + "refusal_reason": "unexpected_category", + "missing_operator": "pre_frame_filler_sentence", + "refusal_detail": "category 'statement_terminator' (word='.') at pre-frame position 10 not handled; may be Phase-3 scope", + "sentence_index": 0, + "token_index": 10, + "token_text": ".", + "recognized_terms": [ + "Mark", + "does", + "a", + "gig", + "every", + "other", + "day", + "for", + "2", + "weeks" + ], + "skipped_frame": null + } + ] +} \ No newline at end of file diff --git a/evals/gsm8k_math/train_sample/v1/audit_brief_11.md b/evals/gsm8k_math/train_sample/v1/audit_brief_11.md new file mode 100644 index 00000000..806e63f7 --- /dev/null +++ b/evals/gsm8k_math/train_sample/v1/audit_brief_11.md @@ -0,0 +1,143 @@ +# Brief 11B — Reader Closure Audit (GSM8K train-sample v1) + +This document is the human-readable companion to +`audit_brief_11.json`. It captures the per-case Phase 2 reader audit over the +50-case sealed train sample, with the missing-operator inference extended in +this PR (see `generate/comprehension/audit.py`). + +## Per-case counts + +| Outcome | Count | +|------------|------:| +| admitted | 0 | +| refused | 50 | +| **wrong** | **0** | + +`wrong == 0` is preserved by construction: this PR does **not** modify the +reader runtime; only the audit inference layer and a new artifact are added. + +## Refusal taxonomy + +| refusal_reason | count | +|------------------------|------:| +| incomplete_operation | 18 | +| unexpected_category | 14 | +| unknown_word | 11 | +| unattached_quantity | 3 | +| unresolved_pronoun | 3 | +| no_question_target | 1 | + +## Missing-operator taxonomy (load-bearing) + +This is the work backlog that any subsequent reader-closure PR must address. +Every previously-`None` operator has now been labelled. + +| missing_operator | count | category | +|-------------------------------|------:|-------------------| +| quantity_extraction | 9 | incomplete_operation | +| lexicon_entry | 9 | unknown_word | +| multi_quantity_composition | 8 | incomplete_operation | +| **pre_frame_filler_sentence** | **8** | **unexpected_category (new label)** | +| pronoun_resolution | 3 | unresolved_pronoun | +| fraction_percentage_literal | 3 | unexpected_category | +| unit_binding | 3 | unattached_quantity | +| **descriptive_frame_question**| **2** | **unexpected_category (new label)** | +| compound_numeric_literal | 1 | unknown_word | +| compound_time_literal | 1 | unknown_word | +| multi_subject_sentence | 1 | unexpected_category | +| question_target_slot | 1 | no_question_target | +| **question_frame_slot** | **1** | **incomplete_operation (new label)** | + +## New labels introduced in this PR + +### `pre_frame_filler_sentence` (8 cases) + +A sentence whose verb the reader does not recognise as a frame-opener, so the +sentence reaches its terminator while still at pre-frame state with a subject +entity pinned but no quantity and no transactional verb. Examples: + +- `gsm8k-train-sample-v1-0028`: *"Tom opens an amusement park."* +- `gsm8k-train-sample-v1-0035`: *"... go strawberry picking."* +- `gsm8k-train-sample-v1-0050`: *"Mark does a gig every other day for 2 weeks."* + +### `pre_frame_filler_sentence` — design tension (open for operator review) + +A naive fix (drain `statement_terminator` at pre-frame and route through +`_end_descriptive_frame`) **lifts 2 cases from refused → admitted** but creates +a real `wrong > 0` hazard on case `gsm8k-train-sample-v1-0050`: + +``` +Mark does a gig every other day for 2 weeks. For each gig, he plays 3 songs. +2 of the songs are 5 minutes long and the last song is twice that long. +How many minutes did he play? +``` + +With pre-frame drain enabled, the reader admits a partial graph +`Operation(mark, add, 3, songs)` with unknown unit `minute` and would project to +a wrong answer (~3 vs expected 280). The looser variant of the fix is therefore +**rejected** by this PR per Brief 11 §"Failure modes to avoid §1 — Correct-count +greed." + +A stricter variant (gate the drain on `pending_entity_ref is None and not +sentence_state.quantities`) fires on 0 of the 11 candidate cases — all of them +have a subject entity pinned before the terminator — so it is purely additive +overhead without lift. + +The honest closure path requires either: + +1. **Verb-vocabulary expansion** — promote currently-unknown statement verbs + (`opens`, `does`, `goes`, `plays`, `splits`, `picking`) to recognised + categories that open the correct statement frame; or +2. **Sentence-level intent classification before frame dispatch** — distinguish + *context-setting* from *math-content* statements deterministically before + committing to (or refusing) a frame. + +Both are real capability work, not a one-line drain rule. Brief 11B-step-2 +(future PR) is the right home. + +### `descriptive_frame_question` (2 cases) + +A `?` terminator reached while the sentence has fallen into `descriptive_frame` +without a `question_open` token — the question target slot was missed. +Distinct refusal mechanism from the `no_question_target` finalize check. + +### `question_frame_slot` (1 case) + +A `question_frame` opened (correct intent) but did not receive a required slot +such as `unit_class`. Separated from generic `incomplete_operation` so the +operator can attack question-slot recovery independently of statement-frame +slot recovery. + +## Highest-leverage backlog (Brief 11B-step-2 candidates) + +Ranked by `count × proximity-to-existing-mechanism`: + +1. `lexicon_entry` (9) — vocabulary gap; lowest-risk extension since adding a + lemma to the cognition pack cannot create wrong admissions without also + passing the graph completeness check. +2. `multi_quantity_composition` (8) — requires the reader to emit two + `PartialInitialPossession` or `PartialOperation` entries from a single + compound-noun statement ("5 full boxes and 5 loose crayons"). Real frame + work. +3. `pre_frame_filler_sentence` (8) — see design tension above. +4. `quantity_extraction` (9) — usually co-occurs with `compound_*` literals or + indefinite quantifiers ("some", "a few"); needs a deliberate + indefinite-quantity policy before any admission. + +## Reproducibility + +```bash +uv run python -c " +import json +from generate.comprehension.audit import audit_problem +from generate.comprehension.state import ReaderRefusal +with open('evals/gsm8k_math/train_sample/v1/cases.jsonl') as f: + for line in f: + c = json.loads(line) + r, rows = audit_problem(c['question'], case_id=c['case_id']) + print(c['case_id'], type(r).__name__, + rows[0].missing_operator if rows else '-') +" +``` + +The artifact `audit_brief_11.json` is pinned by `tests/test_brief_11b_audit_artifact.py`. diff --git a/generate/comprehension/audit.py b/generate/comprehension/audit.py index ca6175a1..e7a161dd 100644 --- a/generate/comprehension/audit.py +++ b/generate/comprehension/audit.py @@ -192,6 +192,28 @@ _OPERATOR_INFERENCE_RULES: list[tuple[str, re.Pattern[str], str]] = [ re.compile(r"."), "graph_construction", ), + # Brief 11B: pre-frame statement_terminator — sentence ended without + # opening any frame. May be context filler or a verb the reader cannot + # admit yet. Distinct from multi_subject because no second entity. + ( + "unexpected_category", + re.compile(r"'statement_terminator'.*pre-frame", re.IGNORECASE), + "pre_frame_filler_sentence", + ), + # Brief 11B: pre-frame "?" reached in a descriptive_frame — question + # target slot was missed before the terminator arrived. + ( + "unexpected_category", + re.compile(r"'question_terminator'.*descriptive_frame", re.IGNORECASE), + "descriptive_frame_question", + ), + # Brief 11B: question_frame opened but a required slot (e.g. unit_class) + # never arrived. Labelled separately from generic incomplete_operation. + ( + "incomplete_operation", + re.compile(r"question_frame missing required slot", re.IGNORECASE), + "question_frame_slot", + ), ] diff --git a/tests/test_brief_11b_audit_artifact.py b/tests/test_brief_11b_audit_artifact.py new file mode 100644 index 00000000..1b6c6d7f --- /dev/null +++ b/tests/test_brief_11b_audit_artifact.py @@ -0,0 +1,184 @@ +"""Brief 11B — pin the audit artifact and the extended missing-operator +inference labels added in this PR. + +Hard invariants: + +* ``wrong == 0`` (this PR does not change the reader runtime). +* The audit artifact at ``evals/gsm8k_math/train_sample/v1/audit_brief_11.json`` + is reproducible from ``audit_problem`` over the sealed train-sample. +* Every refused case now carries a non-None ``missing_operator`` label. +* The three new labels introduced in this PR fire on the documented + detail-string patterns. +""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path + +import pytest + +from generate.comprehension.audit import ( + _infer_missing_operator, + audit_problem, +) +from generate.comprehension.state import ReaderRefusal + +REPO_ROOT = Path(__file__).resolve().parents[1] +CASES_PATH = REPO_ROOT / "evals/gsm8k_math/train_sample/v1/cases.jsonl" +ARTIFACT_PATH = REPO_ROOT / "evals/gsm8k_math/train_sample/v1/audit_brief_11.json" + + +@pytest.fixture(scope="module") +def cases() -> list[dict]: + with CASES_PATH.open() as f: + return [json.loads(line) for line in f] + + +@pytest.fixture(scope="module") +def artifact() -> dict: + with ARTIFACT_PATH.open() as f: + return json.load(f) + + +# --------------------------------------------------------------------------- +# Extended inference rule tests (the three new labels). +# --------------------------------------------------------------------------- + + +def test_pre_frame_filler_sentence_label(): + detail = ( + "category 'statement_terminator' (word='.') at pre-frame position 7 " + "not handled; may be Phase-3 scope" + ) + assert ( + _infer_missing_operator("unexpected_category", detail) + == "pre_frame_filler_sentence" + ) + + +def test_descriptive_frame_question_label(): + detail = "category 'question_terminator' (word='?') not drainable in descriptive_frame" + assert ( + _infer_missing_operator("unexpected_category", detail) + == "descriptive_frame_question" + ) + + +def test_question_frame_slot_label(): + detail = "question_frame missing required slot(s): unit_class" + assert ( + _infer_missing_operator("incomplete_operation", detail) + == "question_frame_slot" + ) + + +def test_pre_frame_label_does_not_capture_multi_subject(): + detail = ( + "second entity 'Bob' at pre-frame position 3; multi-subject sentences " + "are Phase-2.1 scope" + ) + assert ( + _infer_missing_operator("unexpected_category", detail) + == "multi_subject_sentence" + ) + + +# --------------------------------------------------------------------------- +# Artifact reproducibility. +# --------------------------------------------------------------------------- + + +def test_artifact_exists_and_has_expected_shape(artifact): + assert artifact["schema_version"] == 1 + assert artifact["case_count"] == 50 + assert artifact["summary"]["admitted"] == 0 + assert artifact["summary"]["refused"] == 50 + assert artifact["invariants"]["wrong_count"] == 0 + assert artifact["invariants"]["reader_runtime_changes_in_this_pr"] is False + assert len(artifact["per_case"]) == 50 + + +def test_artifact_reproduces_from_audit(cases, artifact): + """Every per-case row in the artifact must round-trip from audit_problem.""" + by_id = {row["case_id"]: row for row in artifact["per_case"]} + for case in cases: + cid = case["case_id"] + result, rows = audit_problem(case["question"], case_id=cid) + expected = by_id[cid] + if isinstance(result, ReaderRefusal): + assert expected["outcome"] == "refused" + assert expected["refusal_reason"] == result.reason + assert expected["sentence_index"] == result.sentence_index + assert expected["token_index"] == result.token_index + assert expected["token_text"] == result.token_text + assert expected["refusal_detail"] == result.detail + if rows: + assert expected["missing_operator"] == rows[0].missing_operator + assert expected["recognized_terms"] == list(rows[0].recognized_terms) + assert expected["skipped_frame"] == rows[0].skipped_frame + elif result is None: + assert expected["outcome"] == "no_sentences" + else: + assert expected["outcome"] == "admitted" + + +def test_every_refused_case_has_labeled_operator(artifact): + """No more None-operator refusals — Brief 11B closes the inference gap.""" + for row in artifact["per_case"]: + if row["outcome"] != "refused": + continue + assert row["missing_operator"] is not None, ( + f"{row['case_id']}: refused with no labeled missing_operator " + f"(reason={row['refusal_reason']!r}, detail={row['refusal_detail']!r})" + ) + + +def test_new_labels_present_in_summary(artifact): + """The three new labels must be represented in the artifact.""" + ops = artifact["summary"]["missing_operators"] + assert ops.get("pre_frame_filler_sentence", 0) >= 1 + assert ops.get("descriptive_frame_question", 0) >= 1 + assert ops.get("question_frame_slot", 0) >= 1 + + +def test_invariant_no_wrong_admission(artifact): + """The load-bearing wrong=0 invariant.""" + assert artifact["invariants"]["wrong_count"] == 0 + assert artifact["summary"]["admitted"] == 0 + + +def test_taxonomy_total_matches_refused_count(artifact): + refused = artifact["summary"]["refused"] + reason_total = sum(artifact["summary"]["refusal_reasons"].values()) + op_total = sum(artifact["summary"]["missing_operators"].values()) + assert reason_total == refused + assert op_total == refused + + +# --------------------------------------------------------------------------- +# Audit row sample integrity — ensure recognized_terms is well-formed. +# --------------------------------------------------------------------------- + + +def test_recognized_terms_only_present_on_post_pre_frame_refusals(artifact): + for row in artifact["per_case"]: + if row["outcome"] != "refused": + continue + if row["sentence_index"] == 0 and row["token_index"] == 0: + assert row.get("recognized_terms", []) == [] + + +def test_refusal_reason_distribution_is_stable(artifact): + """Pin the headline refusal counts so any reader runtime change must + explicitly update this test alongside the artifact.""" + expected = { + "incomplete_operation": 18, + "unexpected_category": 14, + "unknown_word": 11, + "unattached_quantity": 3, + "unresolved_pronoun": 3, + "no_question_target": 1, + } + assert Counter(artifact["summary"]["refusal_reasons"]) == Counter(expected)