diff --git a/evals/gsm8k_math/train_sample/v1/audit_brief_11.json b/evals/gsm8k_math/train_sample/v1/audit_brief_11.json index f116a29e..112f9497 100644 --- a/evals/gsm8k_math/train_sample/v1/audit_brief_11.json +++ b/evals/gsm8k_math/train_sample/v1/audit_brief_11.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "brief": "Brief 11B-step-2 \u2014 lexicon closure", + "brief": "Brief 11B-step-2 + B1 \u2014 lexicon closure wave 3", "adr_chain": [ "0164", "0164.3", @@ -14,23 +14,23 @@ "admitted": 0, "refused": 50, "refusal_reasons": { - "unknown_word": 5, + "unknown_word": 3, "unexpected_category": 17, "unattached_quantity": 4, - "incomplete_operation": 20, + "incomplete_operation": 21, "no_question_target": 1, - "unresolved_pronoun": 3 + "unresolved_pronoun": 4 }, "missing_operators": { - "lexicon_entry": 3, + "lexicon_entry": 1, "pre_frame_filler_sentence": 9, "descriptive_frame_question": 2, "fraction_percentage_literal": 4, "unit_binding": 4, "multi_quantity_composition": 8, - "quantity_extraction": 11, + "quantity_extraction": 12, "question_target_slot": 1, - "pronoun_resolution": 3, + "pronoun_resolution": 4, "compound_time_literal": 1, "multi_subject_sentence": 2, "question_frame_slot": 1, @@ -40,7 +40,7 @@ "invariants": { "wrong_count": 0, "reader_runtime_changes_in_this_pr": false, - "pack_changes": "12 drain_token lemmas + 2 aliases added to en_core_math_v1" + "pack_changes": "+2 drain_token lemmas (path, journey, sees-alias-of-see) added to en_core_math_v1" }, "per_case": [ { @@ -1000,12 +1000,12 @@ { "case_id": "gsm8k-train-sample-v1-0040", "outcome": "refused", - "refusal_reason": "unknown_word", - "missing_operator": "lexicon_entry", - "refusal_detail": "no primitive or lexicon match for 'sees'", + "refusal_reason": "incomplete_operation", + "missing_operator": "quantity_extraction", + "refusal_detail": "initial_state_frame closed with no quantity", "sentence_index": 0, - "token_index": 11, - "token_text": "sees", + "token_index": 18, + "token_text": "", "recognized_terms": [ "Over", "several", @@ -1017,7 +1017,15 @@ "any", "stray", "animals", - "he" + "he", + "sees", + "on", + "the", + "side", + "of", + "the", + "road", + "." ], "skipped_frame": "initial_state_frame" }, @@ -1199,12 +1207,12 @@ { "case_id": "gsm8k-train-sample-v1-0049", "outcome": "refused", - "refusal_reason": "unknown_word", - "missing_operator": "lexicon_entry", - "refusal_detail": "no primitive or lexicon match for 'path'", + "refusal_reason": "unresolved_pronoun", + "missing_operator": "pronoun_resolution", + "refusal_detail": "pronoun 'it' in operation_frame has no compatible entity", "sentence_index": 1, - "token_index": 21, - "token_text": "path", + "token_index": 33, + "token_text": "it", "recognized_terms": [ "In", "the", @@ -1226,9 +1234,21 @@ "of", "time", "along", - "a" + "a", + "path", + ",", + "then", + "finishes", + "the", + "journey", + "in", + "a", + "third", + "of", + "the", + "time" ], - "skipped_frame": null + "skipped_frame": "operation_frame" }, { "case_id": "gsm8k-train-sample-v1-0050", diff --git a/language_packs/data/en_core_math_v1/lexicon/drain_token.jsonl b/language_packs/data/en_core_math_v1/lexicon/drain_token.jsonl index 6e798ffe..aa5b2b02 100644 --- a/language_packs/data/en_core_math_v1/lexicon/drain_token.jsonl +++ b/language_packs/data/en_core_math_v1/lexicon/drain_token.jsonl @@ -118,6 +118,7 @@ {"lemma": "insurance", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "into", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} {"lemma": "jacks", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_brief_11b_step2_2026-05-27"} +{"lemma": "journey", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_brief_b1_2026-05-27"} {"lemma": "jumping", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "june", "category": "drain_token", "aliases": ["july", "august", "september", "october", "november", "december", "january", "february", "march", "april", "may"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "junior", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} @@ -169,6 +170,7 @@ {"lemma": "park", "category": "drain_token", "aliases": ["parks"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "party", "category": "drain_token", "aliases": ["parties"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "past", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "path", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_brief_b1_2026-05-27"} {"lemma": "per", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "percent", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "pet", "category": "drain_token", "aliases": ["pets"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} @@ -200,6 +202,7 @@ {"lemma": "same", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} {"lemma": "scented", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "school", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "sees", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_brief_b1_2026-05-27"} {"lemma": "senior", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "several", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "shelter", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} diff --git a/tests/test_brief_11b_audit_artifact.py b/tests/test_brief_11b_audit_artifact.py index 56baf39e..f68d7002 100644 --- a/tests/test_brief_11b_audit_artifact.py +++ b/tests/test_brief_11b_audit_artifact.py @@ -177,12 +177,16 @@ def test_refusal_reason_distribution_is_stable(artifact): # additions). The unknown_word row strictly decreased; previously-hidden # bottlenecks at downstream frames became visible (real new work, not # regression). See `audit_brief_11.md` for the before/after table. + # Counts updated post-B1 (lexicon closure wave 3 adds path/journey/sees). + # unknown_word 5→3 (-2); case 0049 path resolution surfaced its pronoun + # barrier (unresolved_pronoun +1) and quantity gap (incomplete_operation + # +1). wrong=0 invariant preserved (admitted=0). expected = { - "incomplete_operation": 20, + "incomplete_operation": 21, "unexpected_category": 17, - "unknown_word": 5, + "unknown_word": 3, "unattached_quantity": 4, - "unresolved_pronoun": 3, + "unresolved_pronoun": 4, "no_question_target": 1, } assert Counter(artifact["summary"]["refusal_reasons"]) == Counter(expected)