core-labs/core
core-labs/core
1
0
Fork 0
Code Issues 11 Pull requests 2 Projects Releases Packages Wiki Activity Actions

feat(brief-B1): lexicon closure wave 3 — unknown_word 5→3, wrong=0 preserved (#368)

Browse source 
Adds 3 drain_token lemmas to en_core_math_v1 closing 2 of 3 remaining
lexicon_entry refusals from the prior wave:

- path (case 0049, new lemma)
- journey (case 0049 follow-on after path resolved)
- sees → alias of existing "see" lemma (case 0040)

The third remaining lexicon_entry refusal (case 0001, '+') is
deliberately NOT closed: '+' is an arithmetic operator literal, not a
lexical token. Adding it as drain_token would silently drop arithmetic
content from problems like "5 + 3 apples", a wrong=0 hazard. Documented
in the PR body and audit artifact.

Refusal taxonomy shift:
- unknown_word: 5 → 3 (-2)
- unresolved_pronoun: 3 → 4 (+1) — case 0049's pronoun barrier surfaced
- incomplete_operation: 20 → 21 (+1) — case 0049's quantity gap surfaced

Hard invariants:
- wrong == 0 (admitted=0, verified)
- case 0050 hazard pinned (refused at sentence_index=0)
- manifest checksum unchanged (per-category source file edit)
- no teaching-store mutation; no reader runtime change
This commit is contained in:
Shay 2026-05-27 10:13:09 -07:00 committed by GitHub
parent 4beda0bfd7
commit 9792f66f90
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 51 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

62
evals/gsm8k_math/train_sample/v1/audit_brief_11.json
View file

@ -1,6 +1,6 @@
{
"schema_version": 1,
"brief": "Brief 11B-step-2 \u2014 lexicon closure",
"brief": "Brief 11B-step-2 + B1 \u2014 lexicon closure wave 3",
"adr_chain": [
"0164",
"0164.3",
@ -14,23 +14,23 @@
"admitted": 0,
"refused": 50,
"refusal_reasons": {
"unknown_word": 5,
"unknown_word": 3,
"unexpected_category": 17,
"unattached_quantity": 4,
"incomplete_operation": 20,
"incomplete_operation": 21,
"no_question_target": 1,
"unresolved_pronoun": 3
"unresolved_pronoun": 4
},
"missing_operators": {
"lexicon_entry": 3,
"lexicon_entry": 1,
"pre_frame_filler_sentence": 9,
"descriptive_frame_question": 2,
"fraction_percentage_literal": 4,
"unit_binding": 4,
"multi_quantity_composition": 8,
"quantity_extraction": 11,
"quantity_extraction": 12,
"question_target_slot": 1,
"pronoun_resolution": 3,
"pronoun_resolution": 4,
"compound_time_literal": 1,
"multi_subject_sentence": 2,
"question_frame_slot": 1,
@ -40,7 +40,7 @@
"invariants": {
"wrong_count": 0,
"reader_runtime_changes_in_this_pr": false,
"pack_changes": "12 drain_token lemmas + 2 aliases added to en_core_math_v1"
"pack_changes": "+2 drain_token lemmas (path, journey, sees-alias-of-see) added to en_core_math_v1"
},
"per_case": [
{
@ -1000,12 +1000,12 @@
{
"case_id": "gsm8k-train-sample-v1-0040",
"outcome": "refused",
"refusal_reason": "unknown_word",
"missing_operator": "lexicon_entry",
"refusal_detail": "no primitive or lexicon match for 'sees'",
"refusal_reason": "incomplete_operation",
"missing_operator": "quantity_extraction",
"refusal_detail": "initial_state_frame closed with no quantity",
"sentence_index": 0,
"token_index": 11,
"token_text": "sees",
"token_index": 18,
"token_text": "",
"recognized_terms": [
"Over",
"several",
@ -1017,7 +1017,15 @@
"any",
"stray",
"animals",
"he"
"he",
"sees",
"on",
"the",
"side",
"of",
"the",
"road",
"."
],
"skipped_frame": "initial_state_frame"
},
@ -1199,12 +1207,12 @@
{
"case_id": "gsm8k-train-sample-v1-0049",
"outcome": "refused",
"refusal_reason": "unknown_word",
"missing_operator": "lexicon_entry",
"refusal_detail": "no primitive or lexicon match for 'path'",
"refusal_reason": "unresolved_pronoun",
"missing_operator": "pronoun_resolution",
"refusal_detail": "pronoun 'it' in operation_frame has no compatible entity",
"sentence_index": 1,
"token_index": 21,
"token_text": "path",
"token_index": 33,
"token_text": "it",
"recognized_terms": [
"In",
"the",
@ -1226,9 +1234,21 @@
"of",
"time",
"along",
"a"
"a",
"path",
",",
"then",
"finishes",
"the",
"journey",
"in",
"a",
"third",
"of",
"the",
"time"
],
"skipped_frame": null
"skipped_frame": "operation_frame"
},
{
"case_id": "gsm8k-train-sample-v1-0050",

3
language_packs/data/en_core_math_v1/lexicon/drain_token.jsonl
View file

@ -118,6 +118,7 @@
{"lemma": "insurance", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"}
{"lemma": "into", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"}
{"lemma": "jacks", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_brief_11b_step2_2026-05-27"}
{"lemma": "journey", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_brief_b1_2026-05-27"}
{"lemma": "jumping", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"}
{"lemma": "june", "category": "drain_token", "aliases": ["july", "august", "september", "october", "november", "december", "january", "february", "march", "april", "may"], "provenance": "phase_2_reader_gsm8k_2026-05-26"}
{"lemma": "junior", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"}
@ -169,6 +170,7 @@
{"lemma": "park", "category": "drain_token", "aliases": ["parks"], "provenance": "phase_2_reader_gsm8k_2026-05-26"}
{"lemma": "party", "category": "drain_token", "aliases": ["parties"], "provenance": "phase_2_reader_gsm8k_2026-05-26"}
{"lemma": "past", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"}
{"lemma": "path", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_brief_b1_2026-05-27"}
{"lemma": "per", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"}
{"lemma": "percent", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"}
{"lemma": "pet", "category": "drain_token", "aliases": ["pets"], "provenance": "phase_2_reader_gsm8k_2026-05-26"}
@ -200,6 +202,7 @@
{"lemma": "same", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"}
{"lemma": "scented", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"}
{"lemma": "school", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"}
{"lemma": "sees", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_brief_b1_2026-05-27"}
{"lemma": "senior", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"}
{"lemma": "several", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"}
{"lemma": "shelter", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"}

10
tests/test_brief_11b_audit_artifact.py
View file

@ -177,12 +177,16 @@ def test_refusal_reason_distribution_is_stable(artifact):
# additions). The unknown_word row strictly decreased; previously-hidden
# bottlenecks at downstream frames became visible (real new work, not
# regression). See `audit_brief_11.md` for the before/after table.
# Counts updated post-B1 (lexicon closure wave 3 adds path/journey/sees).
# unknown_word 5→3 (-2); case 0049 path resolution surfaced its pronoun
# barrier (unresolved_pronoun +1) and quantity gap (incomplete_operation
# +1). wrong=0 invariant preserved (admitted=0).
expected = {
"incomplete_operation": 20,
"incomplete_operation": 21,
"unexpected_category": 17,
"unknown_word": 5,
"unknown_word": 3,
"unattached_quantity": 4,
"unresolved_pronoun": 3,
"unresolved_pronoun": 4,
"no_question_target": 1,
}
assert Counter(artifact["summary"]["refusal_reasons"]) == Counter(expected)