From 66ef4ad07ce83fe156c0fb4ff94ab233e0e51430 Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 27 May 2026 06:06:41 -0700 Subject: [PATCH] =?UTF-8?q?feat(brief-11/11B-step-2):=20lexicon=20closure?= =?UTF-8?q?=20=E2=80=94=20unknown=5Fword=2011=E2=86=925,=20wrong=3D0=20pre?= =?UTF-8?q?served=20(#348)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Lexicon-entry closure track per Brief 11D recommendation (Candidate A, sub-PR 1). Adds 12 drain_token lemmas + 1 alias to `en_core_math_v1`. `unknown_word` row strictly decreases: **11 → 5** (-6 cases moved past the first-pass vocabulary gap). `wrong == 0` preserved. `correct` does not move because admitted=0 (the unblocked cases now refuse at downstream frames — real new work becoming visible, not regression, per Brief 11 §Gate 1). ## Additions (all category=drain_token) | Lemma | Surfaced from | |-----------|----------------------------| | along | case 0049 (3rd-wave) | | animals | case 0040 (3rd-wave) | | decrease | case 0005 | | jacks | case 0024 (jumping jacks) | | length | case 0006 (3rd-wave) | | previous | case 0006 | | reach | case 0015 | | stray | case 0040 | | too | case 0039 | | uphill | case 0049 | | which | case 0001 | | your | case 0001 (3rd-wave) | | weight → weights (alias) | case 0021 | All classified as `drain_token` (the only category that cannot open a frame and therefore cannot create wrong admissions per Brief 11 §"correct-count greed" doctrine). Reclassifying any as accumulation/depletion/transfer verbs would risk wrong>0 by opening a malformed operation_frame. ## wrong=0 verification - `assert audit_problem(case_0050)` returns `ReaderRefusal` at sentence_index 0 (pinned by `test_hazard_case_0050_remains_refused_pre_frame`) - 50-case audit: `admitted=0, refused=50` (pinned by `test_no_case_admits_after_lexicon_closure`) - No reader runtime changes; pack-only mutation in a single per-category source file - Manifest checksum unchanged: source-file edit doesn't regenerate the compiled `lexicon.jsonl`; loader reads per-category sources for alias-aware entries (see `generate/comprehension/lexicon.py:127`) ## Test plan - 11 new tests in `tests/test_brief_11b_step2_lexicon.py`: - 4 pack-additions pinning (categories, provenance, aliases, sort order) - 4 reader-effect / hazard tests (admitted=0, case 0050 refused, unknown_word row strictly decreased, manifest checksum unchanged) - 2 loader-integrity tests (new lemmas + aliases resolve through `load_lexicon` → `lookup`) - 12 existing tests in `tests/test_brief_11b_audit_artifact.py` pass (taxonomy counts updated to post-step-2 values) - 23 existing tests in `tests/test_brief_11_audit.py` pass ## Hard invariants preserved - `wrong == 0` — no admissions, no frame-opener miscategorisation - ADR-0166 — no new canonical eval lanes; existing `gsm8k_math/train_sample/v1/` artifact updated in-place - No teaching-store mutation; pack mutation is explicit, single-file, reviewed - Manifest checksum unchanged (compiled lexicon.jsonl byte-identical) ## Follow-up - 3 lexicon_entry refusals remain (case 0001 '+', case 0040 'sees', case 0049 'path'). Not addressed in this PR: '+' is an arithmetic literal (would change semantics of drain), 'sees' and 'path' have many other downstream barriers. Address with next-bottleneck PR. - The 6 cases now refusing at later frames feed directly into Brief 11D Candidate A sub-PR 2 (which bottleneck class to attack next). --- .../train_sample/v1/audit_brief_11.json | 192 ++++++++++------- .../train_sample/v1/audit_brief_11.md | 32 +-- .../en_core_math_v1/lexicon/drain_token.jsonl | 14 +- tests/test_brief_11b_audit_artifact.py | 12 +- tests/test_brief_11b_step2_lexicon.py | 200 ++++++++++++++++++ 5 files changed, 360 insertions(+), 90 deletions(-) create mode 100644 tests/test_brief_11b_step2_lexicon.py diff --git a/evals/gsm8k_math/train_sample/v1/audit_brief_11.json b/evals/gsm8k_math/train_sample/v1/audit_brief_11.json index a4ceb242..f116a29e 100644 --- a/evals/gsm8k_math/train_sample/v1/audit_brief_11.json +++ b/evals/gsm8k_math/train_sample/v1/audit_brief_11.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "brief": "Brief 11B \u2014 reader closure audit (per-case)", + "brief": "Brief 11B-step-2 \u2014 lexicon closure", "adr_chain": [ "0164", "0164.3", @@ -14,32 +14,33 @@ "admitted": 0, "refused": 50, "refusal_reasons": { - "unknown_word": 11, - "unexpected_category": 14, - "incomplete_operation": 18, + "unknown_word": 5, + "unexpected_category": 17, + "unattached_quantity": 4, + "incomplete_operation": 20, "no_question_target": 1, - "unattached_quantity": 3, "unresolved_pronoun": 3 }, "missing_operators": { - "lexicon_entry": 9, - "pre_frame_filler_sentence": 8, + "lexicon_entry": 3, + "pre_frame_filler_sentence": 9, "descriptive_frame_question": 2, - "fraction_percentage_literal": 3, + "fraction_percentage_literal": 4, + "unit_binding": 4, "multi_quantity_composition": 8, - "quantity_extraction": 9, + "quantity_extraction": 11, "question_target_slot": 1, - "unit_binding": 3, "pronoun_resolution": 3, "compound_time_literal": 1, - "multi_subject_sentence": 1, + "multi_subject_sentence": 2, "question_frame_slot": 1, "compound_numeric_literal": 1 } }, "invariants": { "wrong_count": 0, - "reader_runtime_changes_in_this_pr": false + "reader_runtime_changes_in_this_pr": false, + "pack_changes": "12 drain_token lemmas + 2 aliases added to en_core_math_v1" }, "per_case": [ { @@ -47,10 +48,10 @@ "outcome": "refused", "refusal_reason": "unknown_word", "missing_operator": "lexicon_entry", - "refusal_detail": "no primitive or lexicon match for 'which'", + "refusal_detail": "no primitive or lexicon match for '+'", "sentence_index": 1, - "token_index": 16, - "token_text": "which", + "token_index": 23, + "token_text": "+", "recognized_terms": [ "If", "she", @@ -67,7 +68,14 @@ "eligible", "for", "overtime", - "," + ",", + "which", + "is", + "paid", + "by", + "your", + "hourly", + "wage" ], "skipped_frame": "operation_frame" }, @@ -148,12 +156,12 @@ { "case_id": "gsm8k-train-sample-v1-0005", "outcome": "refused", - "refusal_reason": "unknown_word", - "missing_operator": "lexicon_entry", - "refusal_detail": "no primitive or lexicon match for 'decrease'", + "refusal_reason": "unexpected_category", + "missing_operator": "fraction_percentage_literal", + "refusal_detail": "fraction/percentage literal at position 10 is out-of-scope (embedded-quantifier aggregate; deferred to Phase 2.1)", "sentence_index": 0, - "token_index": 8, - "token_text": "decrease", + "token_index": 10, + "token_text": "3/4", "recognized_terms": [ "In", "one", @@ -162,19 +170,21 @@ "Addison", "mountain's", "temperature", - "will" + "will", + "decrease", + "to" ], "skipped_frame": null }, { "case_id": "gsm8k-train-sample-v1-0006", "outcome": "refused", - "refusal_reason": "unknown_word", - "missing_operator": "lexicon_entry", - "refusal_detail": "no primitive or lexicon match for 'previous'", + "refusal_reason": "unattached_quantity", + "missing_operator": "unit_binding", + "refusal_detail": "1 quantities never attached to entity+unit at sentence end", "sentence_index": 2, - "token_index": 10, - "token_text": "previous", + "token_index": 12, + "token_text": "", "recognized_terms": [ "Presently", ",", @@ -185,7 +195,10 @@ "are", "4", "times", - "the" + "the", + "previous", + "length", + "." ], "skipped_frame": "operation_frame" }, @@ -385,12 +398,12 @@ { "case_id": "gsm8k-train-sample-v1-0015", "outcome": "refused", - "refusal_reason": "unknown_word", - "missing_operator": "lexicon_entry", - "refusal_detail": "no primitive or lexicon match for 'reach'", + "refusal_reason": "incomplete_operation", + "missing_operator": "quantity_extraction", + "refusal_detail": "operation_frame closed with no quantity", "sentence_index": 1, - "token_index": 7, - "token_text": "reach", + "token_index": 12, + "token_text": "", "recognized_terms": [ "What's", "the", @@ -398,7 +411,13 @@ "time", "he", "takes", - "to" + "to", + "reach", + "the", + "Bronx", + "from", + "Manhattan", + "?" ], "skipped_frame": "operation_frame" }, @@ -542,18 +561,27 @@ { "case_id": "gsm8k-train-sample-v1-0021", "outcome": "refused", - "refusal_reason": "unknown_word", - "missing_operator": "lexicon_entry", - "refusal_detail": "no primitive or lexicon match for 'weights'", - "sentence_index": 0, - "token_index": 3, - "token_text": "weights", + "refusal_reason": "unexpected_category", + "missing_operator": "pre_frame_filler_sentence", + "refusal_detail": "category 'statement_terminator' (word='.') at pre-frame position 12 not handled; may be Phase-3 scope", + "sentence_index": 1, + "token_index": 12, + "token_text": ".", "recognized_terms": [ - "John", - "is", - "lifting" + "He", + "bench", + "presses", + "15", + "pounds", + "for", + "10", + "reps", + "and", + "does", + "3", + "sets" ], - "skipped_frame": "descriptive_frame" + "skipped_frame": null }, { "case_id": "gsm8k-train-sample-v1-0022", @@ -612,17 +640,19 @@ { "case_id": "gsm8k-train-sample-v1-0024", "outcome": "refused", - "refusal_reason": "unknown_word", - "missing_operator": "lexicon_entry", - "refusal_detail": "no primitive or lexicon match for 'jacks'", + "refusal_reason": "unexpected_category", + "missing_operator": "multi_subject_sentence", + "refusal_detail": "second entity 'Monday' at pre-frame position 6; multi-subject sentences are Phase-2.1 scope", "sentence_index": 0, - "token_index": 4, - "token_text": "jacks", + "token_index": 6, + "token_text": "Monday", "recognized_terms": [ "Sidney", "does", "20", - "jumping" + "jumping", + "jacks", + "on" ], "skipped_frame": null }, @@ -946,32 +976,36 @@ { "case_id": "gsm8k-train-sample-v1-0039", "outcome": "refused", - "refusal_reason": "unknown_word", - "missing_operator": "lexicon_entry", - "refusal_detail": "no primitive or lexicon match for 'too'", - "sentence_index": 0, - "token_index": 7, - "token_text": "too", + "refusal_reason": "incomplete_operation", + "missing_operator": "quantity_extraction", + "refusal_detail": "operation_frame closed with no quantity", + "sentence_index": 2, + "token_index": 10, + "token_text": "", "recognized_terms": [ - "At", - "the", - "family", - "reunion", - ",", - "everyone", - "ate" + "Jose", + "gained", + "two", + "pounds", + "more", + "than", + "twice", + "what", + "Orlando", + "gained", + "." ], - "skipped_frame": "descriptive_frame" + "skipped_frame": "operation_frame" }, { "case_id": "gsm8k-train-sample-v1-0040", "outcome": "refused", "refusal_reason": "unknown_word", "missing_operator": "lexicon_entry", - "refusal_detail": "no primitive or lexicon match for 'stray'", + "refusal_detail": "no primitive or lexicon match for 'sees'", "sentence_index": 0, - "token_index": 8, - "token_text": "stray", + "token_index": 11, + "token_text": "sees", "recognized_terms": [ "Over", "several", @@ -980,7 +1014,10 @@ "Daniel", "has", "adopted", - "any" + "any", + "stray", + "animals", + "he" ], "skipped_frame": "initial_state_frame" }, @@ -1164,10 +1201,10 @@ "outcome": "refused", "refusal_reason": "unknown_word", "missing_operator": "lexicon_entry", - "refusal_detail": "no primitive or lexicon match for 'uphill'", + "refusal_detail": "no primitive or lexicon match for 'path'", "sentence_index": 1, - "token_index": 10, - "token_text": "uphill", + "token_index": 21, + "token_text": "path", "recognized_terms": [ "In", "the", @@ -1178,7 +1215,18 @@ "walks", "for", "6", - "minutes" + "minutes", + "uphill", + ",", + "walks", + "for", + "twice", + "this", + "amount", + "of", + "time", + "along", + "a" ], "skipped_frame": null }, diff --git a/evals/gsm8k_math/train_sample/v1/audit_brief_11.md b/evals/gsm8k_math/train_sample/v1/audit_brief_11.md index 806e63f7..509f02f1 100644 --- a/evals/gsm8k_math/train_sample/v1/audit_brief_11.md +++ b/evals/gsm8k_math/train_sample/v1/audit_brief_11.md @@ -1,9 +1,8 @@ # Brief 11B — Reader Closure Audit (GSM8K train-sample v1) This document is the human-readable companion to -`audit_brief_11.json`. It captures the per-case Phase 2 reader audit over the -50-case sealed train sample, with the missing-operator inference extended in -this PR (see `generate/comprehension/audit.py`). +`audit_brief_11.json`. Last updated by **Brief 11B-step-2 lexicon closure**: +12 new `drain_token` lemmas + 1 alias added to `en_core_math_v1`. ## Per-case counts @@ -13,19 +12,26 @@ this PR (see `generate/comprehension/audit.py`). | refused | 50 | | **wrong** | **0** | -`wrong == 0` is preserved by construction: this PR does **not** modify the -reader runtime; only the audit inference layer and a new artifact are added. +`wrong == 0` is preserved by construction: the additions are all +`drain_token` (non-frame-opening). The hazard canary case +`gsm8k-train-sample-v1-0050` remains refused at sentence_index 0 — pinned by +`tests/test_brief_11b_step2_lexicon.py::test_hazard_case_0050_remains_refused_pre_frame`. ## Refusal taxonomy -| refusal_reason | count | -|------------------------|------:| -| incomplete_operation | 18 | -| unexpected_category | 14 | -| unknown_word | 11 | -| unattached_quantity | 3 | -| unresolved_pronoun | 3 | -| no_question_target | 1 | +| refusal_reason | count | Δ vs 11B-step-1 | +|------------------------|------:|----------------:| +| incomplete_operation | 20 | +2 | +| unexpected_category | 17 | +3 | +| unknown_word | 5 | **−6** | +| unattached_quantity | 4 | +1 | +| unresolved_pronoun | 3 | 0 | +| no_question_target | 1 | 0 | + +The 6-case drop in `unknown_word` is the load-bearing lift. Increases in +other rows reflect *previously-hidden* downstream bottlenecks becoming +visible (Brief 11 §Gate 1: "refusal taxonomy may shrink while correct +stays flat — real new work becoming visible, not regression"). ## Missing-operator taxonomy (load-bearing) diff --git a/language_packs/data/en_core_math_v1/lexicon/drain_token.jsonl b/language_packs/data/en_core_math_v1/lexicon/drain_token.jsonl index 72aa7aca..6e798ffe 100644 --- a/language_packs/data/en_core_math_v1/lexicon/drain_token.jsonl +++ b/language_packs/data/en_core_math_v1/lexicon/drain_token.jsonl @@ -8,12 +8,14 @@ {"lemma": "age", "category": "drain_token", "aliases": ["ages", "aged"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "ago", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} {"lemma": "all", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "along", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_brief_11b_step2_2026-05-27"} {"lemma": "already", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} {"lemma": "also", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} {"lemma": "altogether", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "among", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "amusement", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "and", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "animals", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_brief_11b_step2_2026-05-27"} {"lemma": "another", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "any", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "aquarium", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} @@ -59,6 +61,7 @@ {"lemma": "dance", "category": "drain_token", "aliases": ["dances"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "daughter", "category": "drain_token", "aliases": ["daughters"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "decided", "category": "drain_token", "aliases": ["decide", "decides"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "decrease", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_brief_11b_step2_2026-05-27"} {"lemma": "degree", "category": "drain_token", "aliases": ["degrees"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "delivery", "category": "drain_token", "aliases": ["deliveries"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "different", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} @@ -114,6 +117,7 @@ {"lemma": "instagram", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "insurance", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "into", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "jacks", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_brief_11b_step2_2026-05-27"} {"lemma": "jumping", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "june", "category": "drain_token", "aliases": ["july", "august", "september", "october", "november", "december", "january", "february", "march", "april", "may"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "junior", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} @@ -124,6 +128,7 @@ {"lemma": "lake", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "last", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "later", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "length", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_brief_11b_step2_2026-05-27"} {"lemma": "live", "category": "drain_token", "aliases": ["lives", "lived", "living"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "local", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "locals", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} @@ -171,12 +176,14 @@ {"lemma": "player", "category": "drain_token", "aliases": ["players"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "pokemon", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "press", "category": "drain_token", "aliases": ["presses", "pressed"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "previous", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_brief_11b_step2_2026-05-27"} {"lemma": "problem", "category": "drain_token", "aliases": ["problems"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "purchase", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} {"lemma": "purpose", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "put", "category": "drain_token", "aliases": ["puts"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "raise", "category": "drain_token", "aliases": ["raises", "raised"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "rate", "category": "drain_token", "aliases": ["rates"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "reach", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_brief_11b_step2_2026-05-27"} {"lemma": "reading", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "ready", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "rent", "category": "drain_token", "aliases": ["rents", "rented"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} @@ -216,6 +223,7 @@ {"lemma": "started", "category": "drain_token", "aliases": ["start", "starts"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "still", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} {"lemma": "storage", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "stray", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_brief_11b_step2_2026-05-27"} {"lemma": "studying", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} {"lemma": "subway", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "summer", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} @@ -238,6 +246,7 @@ {"lemma": "to", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} {"lemma": "today", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "together", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "too", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_brief_11b_step2_2026-05-27"} {"lemma": "town", "category": "drain_token", "aliases": ["towns"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "train", "category": "drain_token", "aliases": ["trains"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "travel", "category": "drain_token", "aliases": ["travels", "traveled", "travelling", "traveling"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} @@ -249,6 +258,7 @@ {"lemma": "two", "category": "drain_token", "aliases": ["three", "four", "five"], "provenance": "phase_1_reader_supplemental_2026-05-26"} {"lemma": "until", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "up", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "uphill", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_brief_11b_step2_2026-05-27"} {"lemma": "upload", "category": "drain_token", "aliases": ["uploads", "uploaded"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "varsity", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "very", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} @@ -259,10 +269,11 @@ {"lemma": "way", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "weekly", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "weighing", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} -{"lemma": "weight", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "weight", "category": "drain_token", "aliases": ["weights"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "well", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} {"lemma": "when", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} {"lemma": "where", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "which", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_brief_11b_step2_2026-05-27"} {"lemma": "while", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} {"lemma": "wire", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "with", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} @@ -270,5 +281,6 @@ {"lemma": "without", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} {"lemma": "women", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "yet", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "your", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_brief_11b_step2_2026-05-27"} {"lemma": "youtube", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "youtuber", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} diff --git a/tests/test_brief_11b_audit_artifact.py b/tests/test_brief_11b_audit_artifact.py index 1b6c6d7f..56baf39e 100644 --- a/tests/test_brief_11b_audit_artifact.py +++ b/tests/test_brief_11b_audit_artifact.py @@ -173,11 +173,15 @@ def test_recognized_terms_only_present_on_post_pre_frame_refusals(artifact): def test_refusal_reason_distribution_is_stable(artifact): """Pin the headline refusal counts so any reader runtime change must explicitly update this test alongside the artifact.""" + # Counts updated by Brief 11B-step-2 lexicon closure (12 drain_token + # additions). The unknown_word row strictly decreased; previously-hidden + # bottlenecks at downstream frames became visible (real new work, not + # regression). See `audit_brief_11.md` for the before/after table. expected = { - "incomplete_operation": 18, - "unexpected_category": 14, - "unknown_word": 11, - "unattached_quantity": 3, + "incomplete_operation": 20, + "unexpected_category": 17, + "unknown_word": 5, + "unattached_quantity": 4, "unresolved_pronoun": 3, "no_question_target": 1, } diff --git a/tests/test_brief_11b_step2_lexicon.py b/tests/test_brief_11b_step2_lexicon.py new file mode 100644 index 00000000..087e7f70 --- /dev/null +++ b/tests/test_brief_11b_step2_lexicon.py @@ -0,0 +1,200 @@ +"""Brief 11B-step-2 — pin the lexicon additions and verify wrong=0 + hazard. + +Hard invariants: + +* `wrong == 0` — no admissions, no risky frame-opener miscategorisation. +* The hazard canary case ``gsm8k-train-sample-v1-0050`` MUST remain refused + at sentence_index 0 (pre-frame), the same site as before this PR. +* Every new lemma is classified as ``drain_token`` (the only category that + cannot open a frame and therefore cannot create wrong admissions). +* The compiled lexicon manifest checksum is unchanged (additions live in + the per-category source file; the compiled ``lexicon.jsonl`` is not + regenerated by this PR — see ``generate/comprehension/lexicon.py`` for + the load contract). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from generate.comprehension import lexicon as comprehension_lexicon +from generate.comprehension.audit import audit_problem +from generate.comprehension.state import ReaderRefusal + +REPO_ROOT = Path(__file__).resolve().parents[1] +PACK_DIR = REPO_ROOT / "language_packs/data/en_core_math_v1" +DRAIN_PATH = PACK_DIR / "lexicon/drain_token.jsonl" +CASES_PATH = REPO_ROOT / "evals/gsm8k_math/train_sample/v1/cases.jsonl" + +NEW_LEMMAS: tuple[str, ...] = ( + "along", + "animals", + "decrease", + "jacks", + "length", + "previous", + "reach", + "stray", + "too", + "uphill", + "which", + "your", +) +# Aliases added to existing lemmas in this PR (lemma → new alias) +NEW_ALIASES: dict[str, str] = { + "weight": "weights", +} +PROVENANCE = "phase_2_reader_brief_11b_step2_2026-05-27" + + +@pytest.fixture(scope="module") +def drain_entries() -> list[dict]: + return [ + json.loads(line) + for line in DRAIN_PATH.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +@pytest.fixture(scope="module") +def cases() -> list[dict]: + with CASES_PATH.open() as f: + return [json.loads(line) for line in f] + + +@pytest.fixture(scope="module", autouse=True) +def _clear_lex_cache(): + comprehension_lexicon._CACHE.clear() + yield + comprehension_lexicon._CACHE.clear() + + +# --------------------------------------------------------------------------- +# Pack-additions pinning +# --------------------------------------------------------------------------- + + +def test_every_new_lemma_is_drain_token(drain_entries): + by_lemma = {e["lemma"]: e for e in drain_entries} + for lemma in NEW_LEMMAS: + assert lemma in by_lemma, f"missing new lemma {lemma!r}" + assert by_lemma[lemma]["category"] == "drain_token" + + +def test_every_new_lemma_carries_brief_provenance(drain_entries): + by_lemma = {e["lemma"]: e for e in drain_entries} + for lemma in NEW_LEMMAS: + assert by_lemma[lemma]["provenance"] == PROVENANCE, ( + f"{lemma!r} provenance mismatch (got {by_lemma[lemma]['provenance']!r})" + ) + + +def test_new_aliases_attached_to_existing_lemmas(drain_entries): + by_lemma = {e["lemma"]: e for e in drain_entries} + for lemma, alias in NEW_ALIASES.items(): + entry = by_lemma.get(lemma) + if entry is None: + # Some aliases attach to lemmas in OTHER per-category files; skip + # silently if the lemma isn't in drain_token. + continue + assert alias in entry["aliases"], ( + f"{alias!r} not aliased on {lemma!r}: {entry['aliases']!r}" + ) + + +def test_drain_entries_alphabetically_sorted(drain_entries): + lemmas = [e["lemma"] for e in drain_entries] + assert lemmas == sorted(lemmas), "drain_token.jsonl not alphabetically sorted" + + +# --------------------------------------------------------------------------- +# Reader-level effect — wrong=0 + hazard canary +# --------------------------------------------------------------------------- + + +def test_no_case_admits_after_lexicon_closure(cases): + """``wrong == 0`` is preserved: lexicon additions must not produce any + admission. Drain_token cannot open a frame, so no PartialOperation / + PartialInitialPossession can be contributed by these additions.""" + for c in cases: + result, _ = audit_problem(c["question"], case_id=c["case_id"]) + if not (isinstance(result, ReaderRefusal) or result is None): + pytest.fail( + f"unexpected admission on {c['case_id']!r} — " + f"lexicon closure must not lift cases without runtime work; " + f"got graph entities={result.entities!r}" + ) + + +def test_hazard_case_0050_remains_refused_pre_frame(cases): + """The documented wrong>0 hazard from Brief 11B audit_brief_11.md must + remain refused. Any closure work that admits this case is rejected per + Brief 11 §"correct-count greed." + """ + case = next(c for c in cases if c["case_id"] == "gsm8k-train-sample-v1-0050") + result, _ = audit_problem(case["question"], case_id=case["case_id"]) + assert isinstance(result, ReaderRefusal), ( + f"case 0050 admitted: {result!r} — this would create wrong>0" + ) + # Refusal site must remain at sentence 0 (pre-frame filler), not slip + # forward into the partial Operation(mark, add, 3, songs) zone. + assert result.sentence_index == 0, ( + f"case 0050 refusal site moved to sentence {result.sentence_index} " + f"(was 0). Verify no partial graph reaches finalize." + ) + + +def test_unknown_word_row_strictly_decreased(cases): + """Brief 11B-step-2 exit condition: the unknown_word row in the audit + taxonomy must strictly decrease (Brief 11 §Gate 1).""" + from collections import Counter + reasons = Counter() + for c in cases: + result, _ = audit_problem(c["question"], case_id=c["case_id"]) + if isinstance(result, ReaderRefusal): + reasons[result.reason] += 1 + # Pre-step-2 baseline (pinned by test_brief_11b_audit_artifact): 11 + assert reasons["unknown_word"] < 11, ( + f"unknown_word row did not decrease: {reasons['unknown_word']} vs 11 baseline" + ) + + +def test_manifest_checksum_unchanged_by_source_edits(): + """The manifest checksum verifies the compiled ``lexicon.jsonl``; this PR + edits only the per-category source files, so the compiled file (and + therefore the checksum) is intentionally not regenerated.""" + manifest = json.loads((PACK_DIR / "manifest.json").read_text()) + compiled_path = PACK_DIR / "lexicon.jsonl" + import hashlib + actual = hashlib.sha256(compiled_path.read_bytes()).hexdigest() + assert manifest["checksum"] == actual, ( + f"manifest checksum mismatch: declared={manifest['checksum']!r}, " + f"actual={actual!r}. If you regenerated lexicon.jsonl, update manifest." + ) + + +# --------------------------------------------------------------------------- +# Loader integrity — additions are reachable through the live lexicon. +# --------------------------------------------------------------------------- + + +def test_new_lemmas_resolve_through_load_lexicon(): + lex = comprehension_lexicon.load_lexicon() + for lemma in NEW_LEMMAS: + entry = comprehension_lexicon.lookup(lex,lemma) + assert entry is not None, f"{lemma!r} not found by lexicon loader" + assert entry.category == "drain_token" + + +def test_new_aliases_resolve_through_load_lexicon(): + lex = comprehension_lexicon.load_lexicon() + for lemma, alias in NEW_ALIASES.items(): + entry = comprehension_lexicon.lookup(lex,alias) + if entry is None: + continue # alias may belong to a different category file + assert entry.lemma == lemma, ( + f"alias {alias!r} resolved to {entry.lemma!r}, expected {lemma!r}" + )