From 60043973b01d27e81c9bf1c47d536a949def1810 Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 27 May 2026 05:03:56 -0700 Subject: [PATCH] feat(comprehension/10): Phase 2 statement-frame reader (ADR-0164.4) (#335) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the comprehension reader from question-only scope to whole- problem scope. Phase 1 (Brief 8 / #326) implemented question_frame; this brief implements initial_state_frame, operation_frame, and descriptive_frame, plus finalize() projection into a strict ADR-0115 MathProblemGraph. Architecturally correct under ADR-0164.3; not yet productive on GSM8K train_sample. Below-floor measurement documented; specific bottlenecks tabled for Phase 2.1 follow-up. What landed - Frame-opener dispatch in lifecycle.py for the three new statement frames, plus rule handlers (_rule_op_*, _rule_preframe_*, _rule_descriptive_*). - finalize(state) -> MathProblemGraph | ReaderRefusal: pure projection with closure checks (entity registry non-empty, unknown target bound, every op/initial references a known entity, Decimal precision projects losslessly). - _classify extended to 3-tuple (category, surface, decimal_value) with possessive strip retry. Brief 8.2's sentence-initial lookup-first + gender-skip preserved AND extended to mid-sentence (gender is enrichment everywhere, never admission). - Whole-problem coexistence dispatch in math_candidate_graph.py (config.comprehension_reader_questions=True): reader attempts the whole problem; on any ReaderRefusal falls through to existing regex parser. All-or-nothing per the brief. - Lexicon expansion (carried into renamed proper_noun_gender_* files): +2 accumulation_verb (adopt, invest), +2 currency_unit_noun (dollar, cent), +6 capacity_verb (fill, lift, play, work, finish, drive), +5 female names (allison, brooke, jan, marion, sidney), +14 male names (bart, fernando, georgie, jake, jed, jeremie, jose, orlando, rex, rudolph, steve, troy, xavier, yun), +numerous count_unit_noun, drain_token, time_unit_noun. - ADR-0164.4-phase2-statement-frame-reader.md — the architectural rationale and acceptance contract. Measurement (reader_phase2_delta.json): flag-OFF: correct=3 refused=47 wrong=0 flag-ON: correct=3 refused=47 wrong=0 delta: 0/0/0 Below the brief's floor of correct >= 4. Architecture is sound — the reader admits cases as graphs when the structure resolves, refuses cleanly otherwise, preserves wrong=0 across both flag states. Bottleneck table (from per-case attribution): count refusal_class dominant cause ----- ---------------------- ------------------------------------ 18 incomplete_operation multi-quantity ops; no-quantity op 11 unknown_word "hundred", "presently", "one-hour", non-math verbs (compound numerics, lexicon gaps) 6 unexpected_category fraction / percentage literals; multi-subject sentences 6 unresolved_pronoun "them", "their", "his" with no compatible entity 5 unattached_quantity quantity never bound to a unit 1 no_question_target question parsed but slot never set Closing the gate to mixed-bounded [4, 24] is Phase 2.1 scope: extend composition rules for multi-quantity ops, add fraction/percentage primitives (per ADR-0164.1 amendment), expand lexicon for the remaining unknown_word cases, extend pronoun resolution. Invariants preserved - wrong = 0 in both flag states ✓ - flag-OFF byte-identical to today ✓ - determinism (50/50 identical runs) ✓ - Capability axes G1-G5, S1 unchanged ✓ - Reader tests: 19 (Phase 2) + 18 (Phase 1, post-update) + 53 (pack) + 76 (lexicon + primitives) = 166 specific to this change; all pass - core test --suite smoke -q: 67 passed Rebase note This PR was authored against an older base; rebased onto current main to incorporate #333 (Brief 8.2 universal proper_noun_token primitive) and #334 (ADR-0166 measurement discipline). The rebase required: - Lexicon files renamed proper_noun_entity_* -> proper_noun_gender_* (with the Phase 2 additions merged into the gender_* files) - Compiled lexicon.jsonl unchanged from #333's 207-entry state (Phase 2's per-category additions are runtime-visible via the source loader, not via the compiled file) - _classify reconciled with Brief 8.2's sentence-initial dispatch + Phase 2's 3-tuple decimal-value return - All dispatch tables and category checks updated to reference proper_noun_token (singular) instead of proper_noun_entity_{f,m} - Three Phase 1 test expectations updated to reflect Phase 2 behavior (proper noun at position 0 now opens statement pre-frame instead of refusing; pronoun resolution applies per ADR-0164.2) Per ADR-0166's three-question test, this PR is honest measurement: capability exists, at least one case admits, lane distinguishes presence from absence — which the bottleneck table demonstrates. Refs ADR-0164.3 §Phasing Phase 2, ADR-0164.1 amendment (Brief 8.2), ADR-0166 §"Mixed (notable but not blocking)" — except here, below floor. --- ...DR-0164.4-phase2-statement-frame-reader.md | 151 ++ .../train_sample/v1/reader_phase2_delta.json | 20 + generate/comprehension/lifecycle.py | 1459 ++++++++++++++--- generate/comprehension/state.py | 7 +- generate/math_candidate_graph.py | 114 ++ .../lexicon/accumulation_verb.jsonl | 28 +- .../lexicon/capacity_verb.jsonl | 28 +- .../lexicon/count_unit_noun.jsonl | 65 +- .../lexicon/currency_unit_noun.jsonl | 2 + .../en_core_math_v1/lexicon/drain_token.jsonl | 284 +++- .../lexicon/proper_noun_gender_female.jsonl | 5 + .../lexicon/proper_noun_gender_male.jsonl | 14 + .../lexicon/time_unit_noun.jsonl | 7 +- tests/test_en_core_math_v1_pack.py | 28 +- tests/test_reader_phase2.py | 339 ++++ tests/test_reader_question_frame.py | 51 +- 16 files changed, 2320 insertions(+), 282 deletions(-) create mode 100644 docs/decisions/ADR-0164.4-phase2-statement-frame-reader.md create mode 100644 evals/gsm8k_math/train_sample/v1/reader_phase2_delta.json create mode 100644 tests/test_reader_phase2.py diff --git a/docs/decisions/ADR-0164.4-phase2-statement-frame-reader.md b/docs/decisions/ADR-0164.4-phase2-statement-frame-reader.md new file mode 100644 index 00000000..c2fee158 --- /dev/null +++ b/docs/decisions/ADR-0164.4-phase2-statement-frame-reader.md @@ -0,0 +1,151 @@ +# ADR-0164.4 — Phase 2 Statement-Frame Reader + +**Status:** Proposed +**Date:** 2026-05-26 +**Author:** Shay +**Anchor:** [[thesis-decoding-not-generating]] +**Parent:** [ADR-0164 — Incremental Comprehension Reader](./ADR-0164-incremental-comprehension-reader.md) +**Builds on:** [ADR-0164.3 — Cross-Sentence Reading State](./ADR-0164.3-cross-sentence-state.md) +**Related downstream types:** [ADR-0115 — `MathProblemGraph`](./ADR-0115-math-problem-parser-and-graph.md) + +--- + +## Context + +Phase 1 (ADR-0164.3) shipped the `question_frame` reader and the +two-level `ProblemReadingState` / `SentenceReadingState` lifecycle. Phase +2 extends the reader to the three statement-side frames and adds the +`finalize()` projection from `ProblemReadingState` into +`MathProblemGraph`, so a whole problem can be read end-to-end without +the legacy regex parser. + +## Decision + +### Frames ratified + +- **`initial_state_frame`** — `entity possession_verb [count] [unit]`, + emits a `PartialInitialPossession` → `InitialPossession` at + `finalize()`. +- **`operation_frame`** — `entity (accumulation|depletion|transfer| + capacity)_verb [count] [unit] [to entity₂]`, emits a `PartialOperation` + → `Operation` at `finalize()`. `accumulation` → `add`, `depletion` → + `subtract`, `capacity` → `add`, `transfer` → `transfer`. +- **`descriptive_frame`** — opens on `copula_verb` (or subject-dropped + verb position), drains known tokens, emits no math state. Used for + descriptive prose ("Sandra is a baker", "There are some kids in + camp") that does not bind quantities to operations. + +### `finalize()` projection + +Operates on a closed `ProblemReadingState`: + +1. Require `unknown_target_slot` — else `no_question_target`. +2. Build `entities` tuple from `entity_registry` — empty registry yields + `dangling_entity`. +3. `PartialInitialPossession` → `InitialPossession` (requires + `entity`, `quantity.value`, and resolved `unit`). +4. `PartialOperation` → `Operation` (requires `actor`, op-kind from + verb category, `operand.value`, resolved `unit`). +5. `QuestionTargetSlot` → `Unknown` with `unit` derived from the slot's + captured unit lemma or the unit-class default. + +### Integration flag + +`generate.math_candidate_graph.parse_and_solve(text, *, +comprehension_reader: bool = False)` gates the reader-first path. With +the flag `False` (default), behaviour is byte-identical to the existing +regex parser. With the flag `True`, the reader is attempted first; if +any sentence refuses (all-or-nothing) the regex path runs unchanged. + +### Wrong = 0 discipline + +The reader never returns a graph with a wrong answer in the Phase 2 +GSM8K-train sample: any structural ambiguity (multi-quantity ops, +fractions, multi-subject sentences) yields a typed `ReaderRefusal` so +the regex parser handles the case. This preserves the project-wide +`wrong == 0` invariant. + +### Lexicon additions ratified (`phase_2_reader_gsm8k_2026-05-26`) + +- `currency_unit_noun` +2 (`dollar`, `cent`). +- `accumulation_verb` +2 (`adopt`, `invest`). +- `capacity_verb` +6 (`fill`, `lift`, `play`, `work`, `finish`, + `drive`). +- `proper_noun_entity_female` +5 (`allison`, `brooke`, `jan`, + `marion`, `sidney`). +- `proper_noun_entity_male` +14 (`bart`, `fernando`, `georgie`, `jake`, + `jed`, `jeremie`, `jose`, `orlando`, `rex`, `rudolph`, `steve`, + `troy`, `xavier`, `yun`). +- `time_unit_noun` +3 (`year`, `month`, `second`). +- `count_unit_noun` +14 (puppy, kitten, parakeet, coconut, macaroon, + brownie, scoop, section, foot, cable, eraser, crayon, paperclip, + card, …). +- `drain_token` substantial expansion to absorb prose connectives, + written numerals, place names, and non-math verbs that should not + drive frame selection. + +### Possessive handling + +`_classify()` now strips trailing `'s` and re-attempts lexicon lookup, +so `Rudolph's` resolves to `rudolph` (proper_noun_entity_male). Genitive +possessives drain (e.g., "Aaron and his brother") rather than triggering +the multi-subject refusal. + +## Evidence + +`evals/gsm8k_math/train_sample/v1/reader_phase2_delta.json` captures +per-case attribution on the 50-case sample: + +```text +flag-OFF: correct=3 wrong=0 refused=47 +flag-ON: correct=3 wrong=0 refused=47 +reader_accepted (built a Graph): 3 / 50 +``` + +### Observed bottlenecks + +| count | reader refusal class | examples / interpretation | +| ----- | ------------------------------ | -------------------------------------------- | +| 18 | `incomplete_operation` at end | multi-quantity ops ("4 bags with 20 apples in each bag"); no-quantity op_frame | +| 11 | `unknown_word` | `hundred`, `presently`, compound `one-hour`; non-math verbs (`encountered`, `studied`, `holds`) | +| 6 | `unexpected_category` | fraction/percentage literals (Phase 2.1); multi-subject ("Aaron and Carson") | +| 6 | `unresolved_pronoun` | `them`, `their`, `his` with no compatible registry entry | +| 5 | `unattached_quantity` at end | quantity never bound to a unit noun | +| 1 | `no_question_target` at finalize | question sentence parsed but never set the slot | + +### Acceptance gate + +The brief asks for `correct ≥ 25 (preferred)` or +`mixed ∈ [4, 24] with documented bottlenecks`. The current count of +`correct = 3` falls below the gate floor. **The reader is structurally +correct (wrong = 0 holds, flag-OFF byte-identical, determinism holds), +but the lexicon and structural coverage are not yet sufficient to clear +the gate.** Closing the gap requires: + +1. Phase 2.1: embedded-quantifier aggregates ("N X with M Y in each") + and multi-quantity operation handling. +2. Phase 2.2: fraction/percentage literal handling (currently refused + wholesale). +3. Phase 3: multi-subject sentences, descriptive-clause unpacking. + +## Invariants + +- `versor_condition(F) < 1e-6` — Phase 2 reader does not touch field + state; trivially preserved. +- `wrong == 0` — empirically verified on the GSM8K train sample under + both flag settings. +- Flag-OFF byte-identical — `parse_and_solve` short-circuits before any + reader call when the flag is `False`. +- Determinism — identical input yields identical `trace_hash` on + repeated runs (50/50 verified). + +## Out of scope (refused cleanly in Phase 2) + +- Embedded-quantifier aggregates → `embedded-quantifier aggregate; + deferred to Phase 2.1`. +- Fraction / percentage literals → `unexpected_category` with + Phase 2.1 deferred-scope detail. +- Conditional frames ("If X, then Y") → Phase 3+. +- Multi-quantity initial state ("Two puppies, two kittens, …") → + Phase 2.1. +- Multi-subject sentences ("Aaron and Carson saved $40") → Phase 2.1. diff --git a/evals/gsm8k_math/train_sample/v1/reader_phase2_delta.json b/evals/gsm8k_math/train_sample/v1/reader_phase2_delta.json new file mode 100644 index 00000000..1d4a3f00 --- /dev/null +++ b/evals/gsm8k_math/train_sample/v1/reader_phase2_delta.json @@ -0,0 +1,20 @@ +{ + "schema_version": 1, + "brief": "Wave 4 / Brief 10 \u2014 Phase 2 statement-frame reader (post Brief 8.2 rebase)", + "sample_path": "evals/gsm8k_math/train_sample/v1/cases.jsonl", + "flag_off": { + "correct": 3, + "wrong": 0, + "refused": 47 + }, + "flag_on": { + "correct": 3, + "wrong": 0, + "refused": 47 + }, + "delta": { + "correct": 0, + "wrong": 0, + "refused": 0 + } +} \ No newline at end of file diff --git a/generate/comprehension/lifecycle.py b/generate/comprehension/lifecycle.py index 9546dda8..986e4917 100644 --- a/generate/comprehension/lifecycle.py +++ b/generate/comprehension/lifecycle.py @@ -1,16 +1,19 @@ """ADR-0164 / ADR-0164.3 — incremental comprehension reader lifecycle. -Phase 1 scope: ``question_frame`` only. Statement-side frames -(``initial_state_frame``, ``operation_frame``, ``descriptive_frame``) are -Phase 2. +Phase 1 scope: ``question_frame`` only. +Phase 2 scope: ``initial_state_frame``, ``operation_frame``, +``descriptive_frame``, plus ``finalize()`` projection to +:class:`~generate.math_problem_graph.MathProblemGraph`. -The three public functions are pure and deterministic: +The four public functions are pure and deterministic: * :func:`begin_sentence` opens a fresh sentence-local state. * :func:`apply_word` advances one token; returns a new state or a typed :class:`ReaderRefusal`. * :func:`end_sentence` projects the closed sentence into a new :class:`ProblemReadingState` (or refuses). +* :func:`finalize` projects the finished :class:`ProblemReadingState` + into a :class:`~generate.math_problem_graph.MathProblemGraph` (or refuses). ADR-0164 §Decision §3 specifies the four-step token loop: @@ -18,15 +21,11 @@ ADR-0164 §Decision §3 specifies the four-step token loop: 2. Lexicon lookup. 3. Expectation check. 4. Update emit. - -Update rules live in :data:`_QUESTION_FRAME_RULES` as a single readable -table. The table's coverage is intentionally narrow — the five Brief-8 -GSM8K target question sentences plus close variants. Adding a category is -either an entry in this table (mechanical) or a sub-ADR (semantic). """ from __future__ import annotations +from decimal import Decimal from functools import cache from typing import Callable, Final, Literal @@ -37,7 +36,10 @@ from generate.comprehension.state import ( AppliedCategory, EntityRef, FramePayload, + PartialInitialPossession, + PartialOperation, ProblemReadingState, + QuantityRef, QuestionTargetSlot, ReaderRefusal, SentenceReadingState, @@ -45,7 +47,7 @@ from generate.comprehension.state import ( ) # --------------------------------------------------------------------------- -# Cached lexicon — Brief 7's loader is the source of truth once it lands. +# Cached lexicon. # --------------------------------------------------------------------------- @@ -55,7 +57,7 @@ def _get_lexicon() -> Lexicon: # --------------------------------------------------------------------------- -# Category groupings. +# Category groupings and mapping tables. # --------------------------------------------------------------------------- _QUESTION_OPENERS: Final[frozenset[str]] = frozenset({"question_open"}) @@ -71,6 +73,25 @@ _FRAME_CLOSING_VERBS: Final[frozenset[str]] = frozenset( } ) +# Verb categories that determine the statement frame at pre-frame position. +_VERB_TO_FRAME: Final[dict[str, str]] = { + "possession_verb": "initial_state_frame", + "accumulation_verb": "operation_frame", + "depletion_verb": "operation_frame", + "transfer_verb": "operation_frame", + "capacity_verb": "operation_frame", + "copula_verb": "descriptive_frame", +} + +# Verb category → Operation.kind for operation_frame. +# possession_verb is excluded — it produces an InitialPossession, not an Operation. +_VERB_CATEGORY_TO_OP_KIND: Final[dict[str, str]] = { + "accumulation_verb": "add", + "depletion_verb": "subtract", + "transfer_verb": "transfer", + "capacity_verb": "add", +} + # Map qualifier category → QuestionTargetSlot.kind. _KIND_BY_QUALIFIER: Final[dict[str, str]] = { "question_continuous_qty": "continuous_quantity", @@ -79,17 +100,29 @@ _KIND_BY_QUALIFIER: Final[dict[str, str]] = { "aggregate_modifier": "aggregate", } -# Map unit category → unit_class string carried into QuestionTargetSlot. +# Map unit category → unit_class string. _UNIT_CLASS_BY_CATEGORY: Final[dict[str, str]] = { "count_unit_noun": "count", "currency_unit_noun": "currency", "time_unit_noun": "time", } -# Sentinel category recorded in the lookback once the question frame closes. -# After this marker lands, every further token drains into the lookback -# without further state mutation. The marker itself is filtered out of -# the lookback if it would exceed the bounded length. +# Map primitive_name → semantic category used internally. +_PRIMITIVE_CATEGORY_MAP: Final[dict[str, str]] = { + "decimal-currency-literal": "currency_quantity", + "currency-literal": "currency_quantity", + "numeric-literal": "count_quantity", + "time-amount-literal": "time_quantity", + "ordinal-literal": "ordinal_token", + "fraction-literal": "fraction_token", + "percentage-literal": "percentage_token", + "mass-noun-token": "mass_noun_token", +} + +# Internal category produced by "UNIT_CATEGORY_TOKEN" emit (mass-noun-token). +_UNIT_CATEGORY_TOKEN: Final[str] = "UNIT_CATEGORY_TOKEN" + +# Sentinel category recorded in the lookback once any frame closes. _FRAME_CLOSED_MARKER: Final[str] = "_frame_closed" _PRONOUN_GENDER: Final[dict[str, str]] = { @@ -105,6 +138,20 @@ _PRONOUN_GENDER: Final[dict[str, str]] = { "their": "unknown", } +# Categories that are always silently drained in any statement frame. +_STATEMENT_DRAIN_CATEGORIES: Final[frozenset[str]] = frozenset( + { + "drain_token", + "modal_aux", + "residual_modifier", + "aggregate_modifier", + "ordinal_token", + "mass_noun_token", + _UNIT_CATEGORY_TOKEN, + "punctuation_comma", + } +) + # --------------------------------------------------------------------------- # Internal helpers — all pure. @@ -135,12 +182,7 @@ def _resolve_pronoun( """Return a tuple of canonical names compatible with the pronoun's gender. ``None`` means the pronoun's gender is not recognised. Empty tuple means - no compatible entity in the registry. Multi-element means ambiguous; - Phase 1 refuses on >1 candidates. - - The compatibility table is a Phase-1 subset of ADR-0164.2 §2.2: - gender match by exact string; "unknown" gender entries are compatible - with any pronoun gender (single-salient-entity case). + no compatible entity in the registry. """ needed = _PRONOUN_GENDER.get(pronoun.lower()) if needed is None: @@ -158,6 +200,7 @@ def _update_question_target( kind: str | None = None, entity: str | None = None, unit_class: str | None = None, + unit: str | None = None, position: int | None = None, ) -> QuestionTargetSlot: """Build a new QuestionTargetSlot, falling back to existing values.""" @@ -171,6 +214,9 @@ def _update_question_target( new_unit_class = unit_class if unit_class is not None else ( existing.unit_class if existing is not None else None ) + new_unit = unit if unit is not None else ( + existing.unit if existing is not None else None + ) new_position = position if position is not None else ( existing.position if existing is not None else 0 ) @@ -178,10 +224,38 @@ def _update_question_target( kind=new_kind, entity=new_entity, unit_class=new_unit_class, + unit=new_unit, position=new_position, ) +def _close_frame( + sentence_state: SentenceReadingState, + category: str, +) -> SentenceReadingState: + """Push category to lookback then append _FRAME_CLOSED_MARKER.""" + intermediate = _advance(sentence_state, category=category) + closed_lookback = _push_lookback( + intermediate.lookback, + _FRAME_CLOSED_MARKER, + intermediate.token_index - 1, + ) + return SentenceReadingState( + entities=intermediate.entities, + quantities=intermediate.quantities, + operations=intermediate.operations, + question_target=intermediate.question_target, + expectation=intermediate.expectation, + frame=intermediate.frame, + pending_quantities=intermediate.pending_quantities, + pending_entity_ref=intermediate.pending_entity_ref, + pending_verb=intermediate.pending_verb, + token_index=intermediate.token_index, + lookback=closed_lookback, + partial_frame_payload=intermediate.partial_frame_payload, + ) + + # --------------------------------------------------------------------------- # Lifecycle API. # --------------------------------------------------------------------------- @@ -194,10 +268,7 @@ def begin_sentence( """Open a fresh sentence-local state. Per ADR-0164.3 §Lifecycle API. ``sentence_index`` is *not* incremented - here — ``end_sentence`` owns the increment. ``source_text_offset`` is - accepted for parity with the spec; the sentence state itself doesn't - carry it (it lives on ``ProblemReadingState`` and advances at - ``end_sentence``). + here — ``end_sentence`` owns the increment. """ if not isinstance(problem_state, ProblemReadingState): raise TypeError( @@ -232,10 +303,8 @@ def apply_word( ) -> SentenceReadingState | ReaderRefusal: """Advance the reader by one token. Pure / deterministic. - See module docstring for the four-step contract. The Phase-1 update - rules apply only to the ``question_frame``; opening any other frame at - position 0 refuses with ``unexpected_category`` carrying a Phase-2 - diagnostic. + See module docstring for the four-step contract. Phase 2 extends + Phase 1 to handle statement-frame openers at position 0. """ if not isinstance(word, str) or word == "": return ReaderRefusal( @@ -250,12 +319,9 @@ def apply_word( sentence_idx = problem_state.sentence_index # Step 1 + 2 — primitive scan, then lexicon lookup. - category, _surface = _classify(word, token_index=position) + category, _surface, dec_val = _classify(word, token_index=position) - # Step 3 + 4 — expectation + update emit. - # Once the frame is closed, every token drains: classified ones keep - # their category in the lookback; unknowns drain as - # ``unknown_remainder`` so downstream consumers can still see them. + # Once the frame is closed, every token drains. if _frame_closed(sentence_state): return _advance( sentence_state, @@ -271,30 +337,85 @@ def apply_word( token_text=word, ) - # Pure-drain categories at any stage (punctuation, articles, etc.). + # Pure-drain categories at any position and in any frame. if category in {"drain_token", "punctuation_comma"}: return _advance(sentence_state, category=category) - # Phase-1 scope check at position 0. - if sentence_state.frame is None and category not in _QUESTION_OPENERS: + # Fraction/percentage tokens: refuse at any position in any open frame. + # These require Phase 2.1+ handling (embedded-quantifier aggregates). + if category in {"fraction_token", "percentage_token"}: return ReaderRefusal( reason="unexpected_category", detail=( - f"non-question frame at position 0 is Phase-2 scope " - f"(saw category={category!r}, word={word!r})" + f"fraction/percentage literal at position {position} is " + "out-of-scope (embedded-quantifier aggregate; deferred to Phase 2.1)" ), sentence_index=sentence_idx, token_index=position, token_text=word, ) - # Dispatch the rule table. - handler = _QUESTION_FRAME_RULES.get(category, _rule_default_refuse) - return handler( - sentence_state=sentence_state, - problem_state=problem_state, - category=category, - word=word, + # ----------------------------------------------------------------------- + # Pre-frame dispatch (frame is None). + # ----------------------------------------------------------------------- + if sentence_state.frame is None: + return _apply_preframe( + sentence_state=sentence_state, + problem_state=problem_state, + category=category, + word=word, + dec_val=dec_val, + ) + + # ----------------------------------------------------------------------- + # In-frame dispatch. + # ----------------------------------------------------------------------- + if sentence_state.frame == "question_frame": + handler = _QUESTION_FRAME_RULES.get(category, _rule_default_refuse) + return handler( + sentence_state=sentence_state, + problem_state=problem_state, + category=category, + word=word, + dec_val=dec_val, + ) + + if sentence_state.frame == "initial_state_frame": + handler = _INITIAL_STATE_FRAME_RULES.get(category, _rule_statement_refuse) + return handler( + sentence_state=sentence_state, + problem_state=problem_state, + category=category, + word=word, + dec_val=dec_val, + ) + + if sentence_state.frame == "operation_frame": + handler = _OPERATION_FRAME_RULES.get(category, _rule_statement_refuse) + return handler( + sentence_state=sentence_state, + problem_state=problem_state, + category=category, + word=word, + dec_val=dec_val, + ) + + if sentence_state.frame == "descriptive_frame": + handler = _DESCRIPTIVE_FRAME_RULES.get(category, _rule_descriptive_drain_or_refuse) + return handler( + sentence_state=sentence_state, + problem_state=problem_state, + category=category, + word=word, + dec_val=dec_val, + ) + + return ReaderRefusal( + reason="unexpected_category", + detail=f"unknown frame kind {sentence_state.frame!r}", + sentence_index=sentence_idx, + token_index=position, + token_text=word, ) @@ -304,28 +425,21 @@ def end_sentence( ) -> ProblemReadingState | ReaderRefusal: """Close the sentence and fold it into a new ``ProblemReadingState``. - Validation order matches ADR-0164.3 §Lifecycle API: - - 1. ``sentence_state.frame`` must be a legal frame kind. - 2. ``sentence_state.pending_quantities`` must be empty. - 3. If frame is ``question_frame``: target slot must have unit_class - AND a non-default kind set; otherwise ``incomplete_operation``. - 4. Project payload → ``problem_state.unknown_target_slot`` (locked - if already set, refusing). - 5. Append any sentence-introduced entities, fold pronoun resolutions - into the history, increment ``sentence_index``, advance offset. + Validation order per ADR-0164.3 §Lifecycle API. """ sentence_idx = problem_state.sentence_index last_position = max(sentence_state.token_index - 1, 0) if sentence_state.frame is None: - return ReaderRefusal( - reason="unfinished_frame", - detail="sentence ended without a frame being decided", - sentence_index=sentence_idx, - token_index=last_position, - token_text="", - ) + if sentence_state.token_index == 0: + return ReaderRefusal( + reason="unfinished_frame", + detail="sentence ended without a frame being decided", + sentence_index=sentence_idx, + token_index=last_position, + token_text="", + ) + return _end_descriptive_frame(sentence_state, problem_state) if sentence_state.pending_quantities: return ReaderRefusal( @@ -339,135 +453,300 @@ def end_sentence( token_text="", ) + # question_frame — same logic as Phase 1. if sentence_state.frame == "question_frame": - target = sentence_state.question_target - if target is None: - return ReaderRefusal( - reason="incomplete_operation", - detail="question_frame closed with no QuestionTargetSlot", - sentence_index=sentence_idx, - token_index=last_position, - token_text="", - ) - missing: list[str] = [] - if target.unit_class is None: - missing.append("unit_class") - # question_form is encoded in kind; "continuous_quantity" is the - # default at first qualifier — accept any of the four valid kinds. - if missing: - return ReaderRefusal( - reason="incomplete_operation", - detail=( - "question_frame missing required slot(s): " - + ", ".join(missing) - ), - sentence_index=sentence_idx, - token_index=last_position, - token_text="", - ) + return _end_question_frame(sentence_state, problem_state, sentence_idx, last_position) - # Commit unknown_target_slot. Lock-on-set: refuse if already set. - if problem_state.unknown_target_slot is not None: - return ReaderRefusal( - reason="incomplete_operation", - detail=( - "problem already has unknown_target_slot set; " - "second question sentence rejected" - ), - sentence_index=sentence_idx, - token_index=last_position, - token_text="", - ) - new_unknown = target - else: - new_unknown = problem_state.unknown_target_slot + # initial_state_frame — commit PartialInitialPossession. + if sentence_state.frame == "initial_state_frame": + return _end_initial_state_frame(sentence_state, problem_state, sentence_idx, last_position) - # Carry the sentence-introduced entities into the registry. Phase 1 - # only introduces an entity via pending_entity_ref (subject/proper - # noun); pronoun resolutions do NOT introduce new entries. - new_registry = problem_state.entity_registry - if sentence_state.pending_entity_ref is not None: - existing_names = {e.canonical_name for e in new_registry} - candidate = sentence_state.pending_entity_ref - if candidate.canonical_name not in existing_names: - new_registry = new_registry + (candidate,) + # operation_frame — commit PartialOperation. + if sentence_state.frame == "operation_frame": + return _end_operation_frame(sentence_state, problem_state, sentence_idx, last_position) - # Pronoun resolutions recorded in lookback via "_pronoun_resolved:" - # sentinels are not persisted to history here (Phase 1 keeps the - # discipline minimal). The history fold is a Phase-2 sub-ADR; this - # PR preserves the history field untouched on success. - return ProblemReadingState( - entity_registry=new_registry, - accumulated_initial_state=problem_state.accumulated_initial_state, - accumulated_operations=problem_state.accumulated_operations, - unknown_target_slot=new_unknown, - pronoun_resolution_history=problem_state.pronoun_resolution_history, - sentence_index=problem_state.sentence_index + 1, - source_text_offset=problem_state.source_text_offset - + max(sentence_state.token_index, 1), + # descriptive_frame — no math state, just advance. + if sentence_state.frame == "descriptive_frame": + return _end_descriptive_frame(sentence_state, problem_state) + + return ReaderRefusal( + reason="unfinished_frame", + detail=f"unrecognised frame kind {sentence_state.frame!r}", + sentence_index=sentence_idx, + token_index=last_position, + token_text="", ) +def finalize( + problem_state: ProblemReadingState, +) -> "MathProblemGraph | ReaderRefusal": + """Project a finished ProblemReadingState into a MathProblemGraph. + + Called after the last sentence's end_sentence succeeds. + Returns a :class:`ReaderRefusal` if any structural requirement is unmet. + """ + from generate.math_problem_graph import ( + InitialPossession, + MathGraphError, + MathProblemGraph, + Operation, + Quantity, + Unknown, + ) + + # 1. Require a question target. + if problem_state.unknown_target_slot is None: + return ReaderRefusal( + reason="no_question_target", + detail="ProblemReadingState has no unknown_target_slot after finalize", + sentence_index=problem_state.sentence_index, + token_index=0, + token_text="", + ) + + target = problem_state.unknown_target_slot + + # 2. Build entity list from registry. + entities = tuple(e.canonical_name for e in problem_state.entity_registry) + if not entities: + return ReaderRefusal( + reason="dangling_entity", + detail="entity_registry is empty; no entities to build graph", + sentence_index=problem_state.sentence_index, + token_index=0, + token_text="", + ) + + # 3. Project accumulated_initial_state → InitialPossession. + initial_possessions: list[InitialPossession] = [] + for pip in problem_state.accumulated_initial_state: + if pip.entity is None or pip.quantity is None: + return ReaderRefusal( + reason="graph_construction_failure", + detail="PartialInitialPossession missing entity or quantity at finalize", + sentence_index=problem_state.sentence_index, + token_index=0, + token_text="", + ) + qty = pip.quantity + if qty.unit is None: + return ReaderRefusal( + reason="graph_construction_failure", + detail="PartialInitialPossession.quantity has no unit at finalize", + sentence_index=problem_state.sentence_index, + token_index=0, + token_text="", + ) + try: + ip = InitialPossession( + entity=pip.entity, + quantity=Quantity(value=float(qty.value), unit=qty.unit), + ) + except MathGraphError as exc: + return ReaderRefusal( + reason="graph_construction_failure", + detail=f"InitialPossession construction failed: {exc}", + sentence_index=problem_state.sentence_index, + token_index=0, + token_text="", + ) + initial_possessions.append(ip) + + # 4. Project accumulated_operations → Operation. + operations: list[Operation] = [] + for pop in problem_state.accumulated_operations: + if pop.actor is None or pop.kind is None or pop.operand is None: + return ReaderRefusal( + reason="graph_construction_failure", + detail="PartialOperation missing actor/kind/operand at finalize", + sentence_index=problem_state.sentence_index, + token_index=0, + token_text="", + ) + qty = pop.operand + if qty.unit is None: + return ReaderRefusal( + reason="graph_construction_failure", + detail="PartialOperation.operand has no unit at finalize", + sentence_index=problem_state.sentence_index, + token_index=0, + token_text="", + ) + op_kind = _VERB_CATEGORY_TO_OP_KIND.get(pop.kind) + if op_kind is None: + return ReaderRefusal( + reason="graph_construction_failure", + detail=f"unknown verb kind {pop.kind!r} in PartialOperation at finalize", + sentence_index=problem_state.sentence_index, + token_index=0, + token_text="", + ) + try: + op = Operation( + actor=pop.actor, + kind=op_kind, + operand=Quantity(value=float(qty.value), unit=qty.unit), + target=pop.target, + ) + except MathGraphError as exc: + return ReaderRefusal( + reason="graph_construction_failure", + detail=f"Operation construction failed: {exc}", + sentence_index=problem_state.sentence_index, + token_index=0, + token_text="", + ) + operations.append(op) + + # 5. Build Unknown from QuestionTargetSlot. + # unit is the question's unit noun lemma (set by _rule_unit_noun_question). + # Fall back to unit_class if unit was not captured (for currency/time). + unknown_unit = target.unit + if unknown_unit is None: + # Derive a best-effort unit from unit_class — this allows currency/time + # questions without an explicit unit noun to still resolve. + unknown_unit = _UNIT_CLASS_TO_DEFAULT_UNIT.get(target.unit_class or "") + if not unknown_unit: + return ReaderRefusal( + reason="graph_construction_failure", + detail="QuestionTargetSlot has no unit and no unit_class to derive from", + sentence_index=problem_state.sentence_index, + token_index=0, + token_text="", + ) + + try: + unknown = Unknown(entity=target.entity, unit=unknown_unit) + except MathGraphError as exc: + return ReaderRefusal( + reason="graph_construction_failure", + detail=f"Unknown construction failed: {exc}", + sentence_index=problem_state.sentence_index, + token_index=0, + token_text="", + ) + + # 6. Build MathProblemGraph. + try: + graph = MathProblemGraph( + entities=entities, + initial_state=tuple(initial_possessions), + operations=tuple(operations), + unknown=unknown, + ) + except MathGraphError as exc: + return ReaderRefusal( + reason="graph_construction_failure", + detail=f"MathProblemGraph construction failed: {exc}", + sentence_index=problem_state.sentence_index, + token_index=0, + token_text="", + ) + return graph + + +# Default unit strings for unit_class values when the question sentence +# contains no unit noun (e.g. "How much will it cost him?" → unit_class="currency"). +_UNIT_CLASS_TO_DEFAULT_UNIT: Final[dict[str, str]] = { + "currency": "dollars", + "time": "hours", +} + + # --------------------------------------------------------------------------- # Step 1 + 2 — classification. # --------------------------------------------------------------------------- -def _classify(word: str, *, token_index: int) -> tuple[str | None, str]: - """Return (category, surface). Category is None on miss. +def _classify(word: str, *, token_index: int) -> tuple[str | None, str, Decimal | None]: + """Return (category, surface, decimal_value). Category is None on miss. - Dispatch order (ADR-0164.1 §sentence-initial lookup-first): + Dispatch order: + - At token_index == 0 (sentence-initial, ADR-0164.1 amendment via + Brief 8.2): lookup-first, skipping proper_noun_gender_* entries + (those are enrichment, not admission). On miss, primitive scan + catches the universal proper_noun_token primitive. + - At token_index > 0: lookup-first (Phase 2 ordering — lexicon + verbs/units take precedence over primitive coverage); on miss, + possessive strip retry; then primitive scan for numerics, currency + amounts, fractions, and capitalized names. - - token_index == 0 (sentence-initial): lookup-first, skipping - proper_noun_gender_* entries (those are enrichment, not - admission). On miss, primitive scan catches the universal - proper_noun_token primitive. This inverts the question from - "is this a name?" to "is this a known common word?" - - token_index > 0: primitive scan first; when primitive emits - UNIT_CATEGORY_TOKEN, fall through to lexicon so operational - categories (currency_unit_noun, etc.) override the generic - mass-noun emission. Otherwise return primitive, else lexicon. + Numeric primitives extract a Decimal value; non-numeric primitives + return Decimal=None. """ - # (d) Punctuation terminators — reader-internal; not in primitive registry - # or lexicon. Formerly in _interface_stubs._PRIMITIVE_PATTERNS; - # Phase-1 reader-internal dispatch. + # Punctuation terminators — reader-internal dispatch. if word == "?": - return "question_terminator", word + return "question_terminator", word, None if word in (".", "!"): - return "statement_terminator", word + return "statement_terminator", word, None if word == ",": - return "punctuation_comma", word + return "punctuation_comma", word, None lex = _get_lexicon() + def _emit_primitive() -> tuple[str | None, str, Decimal | None]: + primitive: LexemeMatch | None = scan(word) + if primitive is None: + return None, word, None + if primitive.emit_category == _UNIT_CATEGORY_TOKEN: + # Lexicon override for mass-noun tokens with operational meaning. + entry = lookup(lex, word) + if entry is not None: + return entry.category, entry.lemma, None + return "mass_noun_token", primitive.source_text, None + cat = _PRIMITIVE_CATEGORY_MAP.get(primitive.primitive_name, primitive.emit_category) + dec_val: Decimal | None = None + ev = primitive.extracted_values + if "value" in ev: + try: + dec_val = Decimal(ev["value"]) + except Exception: + pass + elif "whole" in ev: + # decimal-currency-literal splits into "whole" + "cents" + whole = ev.get("whole", "0") + cents = ev.get("cents", "0").zfill(2) + try: + dec_val = Decimal(f"{whole}.{cents}") + except Exception: + pass + return cat, primitive.source_text, dec_val + if token_index == 0: - # Sentence-initial: lookup-first, gender enrichment categories - # do not admit (treated as not-found so the primitive's - # proper_noun_token can match instead). + # Sentence-initial: lookup-first, skip gender-enrichment categories + # (per Brief 8.2 — gender is enrichment, not admission). entry: LexiconEntry | None = lookup(lex, word) if entry is not None and entry.category not in { "proper_noun_gender_female", "proper_noun_gender_male", }: - return entry.category, entry.lemma - primitive: LexemeMatch | None = scan(word) - if primitive is not None: - return primitive.emit_category, primitive.source_text - return None, word + return entry.category, entry.lemma, None + # On lookup miss OR gender-only hit: primitive scan picks up the name. + return _emit_primitive() - # Mid-sentence: primitive-first, with UNIT_CATEGORY_TOKEN ceding - # to the operational lexicon if it has a more specific category. - primitive = scan(word) - if primitive is not None: - if primitive.emit_category == "UNIT_CATEGORY_TOKEN": - entry = lookup(lex, word) - if entry is not None: - return entry.category, entry.lemma - return primitive.emit_category, primitive.source_text + # Mid-sentence: lookup-first (Phase 2 ordering), but skip + # proper_noun_gender_* entries (gender is enrichment everywhere, + # per Brief 8.2 — let the primitive emit proper_noun_token so the + # dispatch table sees one consistent category for names). entry = lookup(lex, word) - if entry is not None: - return entry.category, entry.lemma - return None, word + if entry is not None and entry.category not in { + "proper_noun_gender_female", + "proper_noun_gender_male", + }: + return entry.category, entry.lemma, None + + # Possessive strip retry. + if word.endswith("'s") and len(word) > 2: + entry = lookup(lex, word[:-2]) + if entry is not None and entry.category not in { + "proper_noun_gender_female", + "proper_noun_gender_male", + }: + return entry.category, entry.lemma, None + + # Primitive scan for numerics, currency, names, etc. + return _emit_primitive() def gender_of_proper_noun( @@ -494,9 +773,7 @@ def gender_of_proper_noun( # --------------------------------------------------------------------------- -# Update-rule handlers. -# Each handler signature: keyword-only sentence_state, problem_state, -# category, word. Returns a new SentenceReadingState or a ReaderRefusal. +# _advance helper. # --------------------------------------------------------------------------- @@ -529,19 +806,215 @@ def _advance( return SentenceReadingState(**base) +# --------------------------------------------------------------------------- +# Pre-frame handlers (frame is None at the time of the call). +# --------------------------------------------------------------------------- + + +def _apply_preframe( + *, + sentence_state: SentenceReadingState, + problem_state: ProblemReadingState, + category: str, + word: str, + dec_val: Decimal | None, +) -> SentenceReadingState | ReaderRefusal: + """Dispatch token when frame has not yet been determined.""" + position = sentence_state.token_index + sentence_idx = problem_state.sentence_index + + if category in _QUESTION_OPENERS: + return _rule_question_open( + sentence_state=sentence_state, + problem_state=problem_state, + category=category, + word=word, + dec_val=dec_val, + ) + + if category == "proper_noun_token": + return _rule_preframe_entity( + sentence_state=sentence_state, + problem_state=problem_state, + category=category, + word=word, + dec_val=dec_val, + ) + + if category == "entity_pronoun": + return _rule_preframe_pronoun( + sentence_state=sentence_state, + problem_state=problem_state, + category=category, + word=word, + dec_val=dec_val, + ) + + if category in _VERB_TO_FRAME: + if sentence_state.pending_entity_ref is None: + # Subject-dropped: treat as descriptive frame and drain the verb. + return _advance( + sentence_state, + category=category, + frame="descriptive_frame", + ) + return _rule_preframe_verb( + sentence_state=sentence_state, + problem_state=problem_state, + category=category, + word=word, + dec_val=dec_val, + ) + + if category in _STATEMENT_DRAIN_CATEGORIES: + return _advance(sentence_state, category=category) + + # Categories that can safely drain when no frame is set yet. + _PREFRAME_DRAIN: frozenset[str] = frozenset({ + "count_unit_noun", "currency_unit_noun", "time_unit_noun", + "count_quantity", "currency_quantity", "time_quantity", + "question_continuous_qty", "question_discrete_qty", + "question_comparative", + "copula_verb", + }) + if category in _PREFRAME_DRAIN: + return _advance(sentence_state, category=category) + + return ReaderRefusal( + reason="unexpected_category", + detail=( + f"category {category!r} (word={word!r}) at pre-frame position " + f"{position} not handled; may be Phase-3 scope" + ), + sentence_index=sentence_idx, + token_index=position, + token_text=word, + ) + + +def _rule_preframe_entity( + *, + sentence_state: SentenceReadingState, + problem_state: ProblemReadingState, + category: str, + word: str, + dec_val: Decimal | None, # noqa: ARG001 +) -> SentenceReadingState | ReaderRefusal: + """Proper noun at pre-frame position — records subject entity, leaves frame=None.""" + if sentence_state.pending_entity_ref is not None: + return ReaderRefusal( + reason="unexpected_category", + detail=( + f"second entity {word!r} at pre-frame position " + f"{sentence_state.token_index}; multi-subject sentences are " + "Phase-2.1 scope" + ), + sentence_index=problem_state.sentence_index, + token_index=sentence_state.token_index, + token_text=word, + ) + canonical = word.lower() + gender = gender_of_proper_noun(word, _get_lexicon()) + entity_ref = EntityRef( + canonical_name=canonical, + gender=gender, + first_mention_position=sentence_state.token_index, + ) + return _advance( + sentence_state, + category=category, + pending_entity_ref=entity_ref, + ) + + +def _rule_preframe_pronoun( + *, + sentence_state: SentenceReadingState, + problem_state: ProblemReadingState, + category: str, # noqa: ARG001 + word: str, + dec_val: Decimal | None, # noqa: ARG001 +) -> SentenceReadingState | ReaderRefusal: + """Pronoun at pre-frame position — resolves to registry entity, leaves frame=None.""" + if sentence_state.pending_entity_ref is not None: + # Possessive adjective after entity (e.g., "Aaron and his brother") — drain. + return _advance(sentence_state, category="drain_token") + candidates = _resolve_pronoun(word, problem_state.entity_registry) + if candidates is None or len(candidates) == 0: + return ReaderRefusal( + reason="unresolved_pronoun", + detail=( + f"pronoun {word!r} has no compatible entity in registry " + f"(size={len(problem_state.entity_registry)})" + ), + sentence_index=problem_state.sentence_index, + token_index=sentence_state.token_index, + token_text=word, + ) + if len(candidates) > 1: + return ReaderRefusal( + reason="ambiguous_pronoun_referent", + detail=( + f"pronoun {word!r} matches >1 entity: " + ", ".join(candidates) + ), + sentence_index=problem_state.sentence_index, + token_index=sentence_state.token_index, + token_text=word, + ) + resolved_name = candidates[0] + pronoun_lower = word.lower() + gender = _PRONOUN_GENDER.get(pronoun_lower, "unknown") + # Create an EntityRef referencing the already-registered entity (not new). + entity_ref = EntityRef( + canonical_name=resolved_name, + gender=gender, + first_mention_position=sentence_state.token_index, + ) + return _advance( + sentence_state, + category="entity_pronoun", + pending_entity_ref=entity_ref, + ) + + +def _rule_preframe_verb( + *, + sentence_state: SentenceReadingState, + problem_state: ProblemReadingState, # noqa: ARG001 + category: str, + word: str, + dec_val: Decimal | None, # noqa: ARG001 +) -> SentenceReadingState | ReaderRefusal: + """Frame-determining verb — sets frame based on verb category.""" + frame = _VERB_TO_FRAME[category] + verb_ref = VerbReference( + surface=word.lower(), + kind=category, + position=sentence_state.token_index, + ) + return _advance( + sentence_state, + category=category, + frame=frame, + pending_verb=verb_ref, + partial_frame_payload=FramePayload(frame_kind=frame), + ) + + +# --------------------------------------------------------------------------- +# Question-frame handlers. +# --------------------------------------------------------------------------- + + def _rule_question_open( *, sentence_state: SentenceReadingState, problem_state: ProblemReadingState, category: str, word: str, + dec_val: Decimal | None, # noqa: ARG001 ) -> SentenceReadingState | ReaderRefusal: - """Rule: opening word ('How', 'What') begins a question_frame. - - Only legal at position 0 (or after a punctuation token; Phase 1 - restricts to position 0 since within-sentence multi-clause is - Phase 2 scope). - """ + """Opening word ('How', 'What') begins a question_frame.""" if sentence_state.frame is not None: return ReaderRefusal( reason="unexpected_category", @@ -564,6 +1037,7 @@ def _rule_qty_qualifier( problem_state: ProblemReadingState, category: str, word: str, + dec_val: Decimal | None, # noqa: ARG001 ) -> SentenceReadingState | ReaderRefusal: """Rule: 'many'/'much'/'more'/'less'/'longer'/'total'/'combined'.""" if sentence_state.frame != "question_frame": @@ -585,14 +1059,15 @@ def _rule_qty_qualifier( ) -def _rule_unit_noun( +def _rule_unit_noun_question( *, sentence_state: SentenceReadingState, problem_state: ProblemReadingState, category: str, word: str, + dec_val: Decimal | None, # noqa: ARG001 ) -> SentenceReadingState | ReaderRefusal: - """Rule: count/currency/time unit noun sets ``unit_class``.""" + """Rule: count/currency/time unit noun in question_frame sets unit_class + unit.""" if sentence_state.frame != "question_frame": return ReaderRefusal( reason="unexpected_category", @@ -602,7 +1077,13 @@ def _rule_unit_noun( token_text=word, ) unit_class = _UNIT_CLASS_BY_CATEGORY[category] - new_target = _update_question_target(sentence_state, unit_class=unit_class) + # Capture the lemma as the unit string for finalize(). + lex = _get_lexicon() + entry = lookup(lex, word) + unit_lemma = entry.lemma if entry is not None else word.lower() + new_target = _update_question_target( + sentence_state, unit_class=unit_class, unit=unit_lemma + ) return _advance( sentence_state, category=category, @@ -616,6 +1097,7 @@ def _rule_modal_aux( problem_state: ProblemReadingState, category: str, word: str, + dec_val: Decimal | None, # noqa: ARG001 ) -> SentenceReadingState | ReaderRefusal: if sentence_state.frame != "question_frame": return ReaderRefusal( @@ -634,8 +1116,9 @@ def _rule_entity_pronoun( problem_state: ProblemReadingState, category: str, word: str, + dec_val: Decimal | None, # noqa: ARG001 ) -> SentenceReadingState | ReaderRefusal: - """Rule: resolve against ``problem_state.entity_registry`` per ADR-0164.2.""" + """Rule: resolve pronoun against registry (question_frame only).""" if sentence_state.frame != "question_frame": return ReaderRefusal( reason="unexpected_category", @@ -676,12 +1159,13 @@ def _rule_entity_pronoun( ) -def _rule_proper_noun( +def _rule_proper_noun_question( *, sentence_state: SentenceReadingState, problem_state: ProblemReadingState, category: str, word: str, + dec_val: Decimal | None, # noqa: ARG001 ) -> SentenceReadingState | ReaderRefusal: if sentence_state.frame != "question_frame": return ReaderRefusal( @@ -713,27 +1197,23 @@ def _rule_residual_modifier( problem_state: ProblemReadingState, # noqa: ARG001 category: str, word: str, # noqa: ARG001 + dec_val: Decimal | None, # noqa: ARG001 ) -> SentenceReadingState | ReaderRefusal: - """Rule: 'left'/'remaining'/'after' modify residual semantics. - - QuestionTargetSlot.kind has no 'residual' literal; Phase 1 keeps the - current kind (typically continuous_quantity / difference) and records - the residual marker in the lookback for downstream consumers. - """ + """Rule: 'left'/'remaining'/'after' — drain outside question_frame.""" if sentence_state.frame != "question_frame": - # Outside the frame these are drain tokens. return _advance(sentence_state, category="drain_token") return _advance(sentence_state, category=category) -def _rule_frame_closer( +def _rule_frame_closer_question( *, sentence_state: SentenceReadingState, problem_state: ProblemReadingState, category: str, word: str, + dec_val: Decimal | None, # noqa: ARG001 ) -> SentenceReadingState | ReaderRefusal: - """Rule: verb or '?' closes the question frame.""" + """Rule: verb or '?' closes the question_frame.""" if sentence_state.frame != "question_frame": return ReaderRefusal( reason="unexpected_category", @@ -747,9 +1227,13 @@ def _rule_frame_closer( pending_verb = VerbReference( surface=word.lower(), kind=category, position=sentence_state.token_index ) - # First push the category, then the close marker, so trace order is - # preserved. intermediate = _advance(sentence_state, category=category, pending_verb=pending_verb) + return _close_frame_from_intermediate(intermediate) + + +def _close_frame_from_intermediate( + intermediate: SentenceReadingState, +) -> SentenceReadingState: closed_lookback = _push_lookback( intermediate.lookback, _FRAME_CLOSED_MARKER, @@ -777,10 +1261,11 @@ def _rule_default_refuse( problem_state: ProblemReadingState, category: str, word: str, + dec_val: Decimal | None, # noqa: ARG001 ) -> ReaderRefusal: return ReaderRefusal( reason="unexpected_category", - detail=f"category {category!r} not handled by Phase-1 question_frame rules", + detail=f"category {category!r} not handled by question_frame rules", sentence_index=problem_state.sentence_index, token_index=sentence_state.token_index, token_text=word, @@ -788,39 +1273,594 @@ def _rule_default_refuse( # --------------------------------------------------------------------------- -# Phase-1 question_frame rule table. -# Each entry: category → handler. New categories belong here, not in a -# different module. +# Statement-frame handlers (shared across initial_state + operation frames). +# --------------------------------------------------------------------------- + + +def _rule_statement_drain( + *, + sentence_state: SentenceReadingState, + problem_state: ProblemReadingState, # noqa: ARG001 + category: str, + word: str, # noqa: ARG001 + dec_val: Decimal | None, # noqa: ARG001 +) -> SentenceReadingState: + """Drain token in a statement frame — advance without semantic effect.""" + return _advance(sentence_state, category="drain_token") + + +def _rule_statement_quantity( + *, + sentence_state: SentenceReadingState, + problem_state: ProblemReadingState, + category: str, + word: str, + dec_val: Decimal | None, +) -> SentenceReadingState | ReaderRefusal: + """Numeric literal in a statement frame — creates a pending QuantityRef.""" + if dec_val is None: + return ReaderRefusal( + reason="unexpected_category", + detail=f"quantity token {word!r} has no parseable decimal value", + sentence_index=problem_state.sentence_index, + token_index=sentence_state.token_index, + token_text=word, + ) + actor = sentence_state.pending_entity_ref + owner = actor.canonical_name if actor is not None else None + # currency_quantity gets a default unit "dollars" (refined if unit noun follows). + # count_quantity and time_quantity get unit_class="pending" until unit noun arrives. + if category == "currency_quantity": + pending = QuantityRef( + value=dec_val, + unit="dollars", + unit_class="currency", + owner_entity=owner, + mention_position=sentence_state.token_index, + ) + else: + pending = QuantityRef( + value=dec_val, + unit=None, + unit_class="pending", + owner_entity=owner, + mention_position=sentence_state.token_index, + ) + new_pending = sentence_state.pending_quantities + (pending,) + return _advance( + sentence_state, + category=category, + pending_quantities=new_pending, + ) + + +def _rule_unit_noun_statement( + *, + sentence_state: SentenceReadingState, + problem_state: ProblemReadingState, # noqa: ARG001 + category: str, + word: str, + dec_val: Decimal | None, # noqa: ARG001 +) -> SentenceReadingState | ReaderRefusal: + """Unit noun in a statement frame — completes the most-recent pending quantity. + + If no pending quantity exists, the unit noun is a bare descriptor and is + drained (e.g. "Sandra had some bags" — 'bags' has no quantity). + """ + if not sentence_state.pending_quantities: + return _advance(sentence_state, category="drain_token") + + unit_class = _UNIT_CLASS_BY_CATEGORY[category] + lex = _get_lexicon() + entry = lookup(lex, word) + unit_lemma = entry.lemma if entry is not None else word.lower() + + pending = sentence_state.pending_quantities[-1] + complete = QuantityRef( + value=pending.value, + unit=unit_lemma, + unit_class=unit_class, + owner_entity=pending.owner_entity, + mention_position=pending.mention_position, + ) + new_pending = sentence_state.pending_quantities[:-1] + new_quantities = sentence_state.quantities + (complete,) + return _advance( + sentence_state, + category=category, + pending_quantities=new_pending, + quantities=new_quantities, + ) + + +def _rule_statement_closer( + *, + sentence_state: SentenceReadingState, + problem_state: ProblemReadingState, # noqa: ARG001 + category: str, + word: str, # noqa: ARG001 + dec_val: Decimal | None, # noqa: ARG001 +) -> SentenceReadingState: + """Statement terminator — closes the statement frame.""" + return _close_frame(sentence_state, category) + + +def _rule_statement_refuse( + *, + sentence_state: SentenceReadingState, + problem_state: ProblemReadingState, + category: str, + word: str, + dec_val: Decimal | None, # noqa: ARG001 +) -> ReaderRefusal: + return ReaderRefusal( + reason="unexpected_category", + detail=( + f"category {category!r} (word={word!r}) not handled in " + f"{sentence_state.frame!r}" + ), + sentence_index=problem_state.sentence_index, + token_index=sentence_state.token_index, + token_text=word, + ) + + +# --------------------------------------------------------------------------- +# Operation-frame specific handlers. +# --------------------------------------------------------------------------- + + +def _rule_op_proper_noun( + *, + sentence_state: SentenceReadingState, + problem_state: ProblemReadingState, # noqa: ARG001 + category: str, + word: str, + dec_val: Decimal | None, # noqa: ARG001 +) -> SentenceReadingState: + """Proper noun mid-operation frame — potential transfer target. + + Stored in ``entities`` so end_sentence can extract it as the transfer + target when verb kind is transfer_verb. + """ + canonical = word.lower() + gender = gender_of_proper_noun(word, _get_lexicon()) + entity_ref = EntityRef( + canonical_name=canonical, + gender=gender, + first_mention_position=sentence_state.token_index, + ) + new_entities = sentence_state.entities + (entity_ref,) + return _advance( + sentence_state, + category=category, + entities=new_entities, + ) + + +def _rule_op_pronoun( + *, + sentence_state: SentenceReadingState, + problem_state: ProblemReadingState, + category: str, # noqa: ARG001 + word: str, + dec_val: Decimal | None, # noqa: ARG001 +) -> SentenceReadingState | ReaderRefusal: + """Pronoun mid-operation frame — potential transfer target (resolved).""" + candidates = _resolve_pronoun(word, problem_state.entity_registry) + if candidates is None or len(candidates) == 0: + return ReaderRefusal( + reason="unresolved_pronoun", + detail=( + f"pronoun {word!r} in operation_frame has no compatible entity" + ), + sentence_index=problem_state.sentence_index, + token_index=sentence_state.token_index, + token_text=word, + ) + if len(candidates) > 1: + return ReaderRefusal( + reason="ambiguous_pronoun_referent", + detail=( + f"pronoun {word!r} in operation_frame matches >1 entity: " + + ", ".join(candidates) + ), + sentence_index=problem_state.sentence_index, + token_index=sentence_state.token_index, + token_text=word, + ) + resolved_name = candidates[0] + pronoun_lower = word.lower() + gender = _PRONOUN_GENDER.get(pronoun_lower, "unknown") + entity_ref = EntityRef( + canonical_name=resolved_name, + gender=gender, + first_mention_position=sentence_state.token_index, + ) + new_entities = sentence_state.entities + (entity_ref,) + return _advance( + sentence_state, + category="entity_pronoun", + entities=new_entities, + ) + + +# --------------------------------------------------------------------------- +# Descriptive-frame handler. +# --------------------------------------------------------------------------- + + +def _rule_descriptive_drain_or_refuse( + *, + sentence_state: SentenceReadingState, + problem_state: ProblemReadingState, + category: str, + word: str, + dec_val: Decimal | None, # noqa: ARG001 +) -> SentenceReadingState | ReaderRefusal: + """In descriptive_frame, known semantic categories drain; unknowns refuse.""" + _DESCRIPTIVE_DRAIN_CATEGORIES: frozenset[str] = frozenset( + { + "count_unit_noun", + "currency_unit_noun", + "time_unit_noun", + "proper_noun_token", + "entity_pronoun", + "count_quantity", + "currency_quantity", + "time_quantity", + "ordinal_token", + "mass_noun_token", + "accumulation_verb", + "depletion_verb", + "transfer_verb", + "capacity_verb", + "possession_verb", + } + ) + if category in _DESCRIPTIVE_DRAIN_CATEGORIES: + return _advance(sentence_state, category="drain_token") + return ReaderRefusal( + reason="unexpected_category", + detail=f"category {category!r} (word={word!r}) not drainable in descriptive_frame", + sentence_index=problem_state.sentence_index, + token_index=sentence_state.token_index, + token_text=word, + ) + + +# --------------------------------------------------------------------------- +# end_sentence helpers. +# --------------------------------------------------------------------------- + + +def _carry_entity( + sentence_state: SentenceReadingState, + problem_state: ProblemReadingState, +) -> tuple[tuple[EntityRef, ...], ProblemReadingState]: + """Return (registry, updated-problem-state) after carrying sentence entity.""" + new_registry = problem_state.entity_registry + if sentence_state.pending_entity_ref is not None: + existing_names = {e.canonical_name for e in new_registry} + candidate = sentence_state.pending_entity_ref + if candidate.canonical_name not in existing_names: + new_registry = new_registry + (candidate,) + return new_registry, problem_state + + +def _end_question_frame( + sentence_state: SentenceReadingState, + problem_state: ProblemReadingState, + sentence_idx: int, + last_position: int, +) -> ProblemReadingState | ReaderRefusal: + target = sentence_state.question_target + if target is None: + return ReaderRefusal( + reason="incomplete_operation", + detail="question_frame closed with no QuestionTargetSlot", + sentence_index=sentence_idx, + token_index=last_position, + token_text="", + ) + if target.unit_class is None: + return ReaderRefusal( + reason="incomplete_operation", + detail="question_frame missing required slot(s): unit_class", + sentence_index=sentence_idx, + token_index=last_position, + token_text="", + ) + if problem_state.unknown_target_slot is not None: + return ReaderRefusal( + reason="incomplete_operation", + detail=( + "problem already has unknown_target_slot set; " + "second question sentence rejected" + ), + sentence_index=sentence_idx, + token_index=last_position, + token_text="", + ) + new_registry, _ = _carry_entity(sentence_state, problem_state) + return ProblemReadingState( + entity_registry=new_registry, + accumulated_initial_state=problem_state.accumulated_initial_state, + accumulated_operations=problem_state.accumulated_operations, + unknown_target_slot=target, + pronoun_resolution_history=problem_state.pronoun_resolution_history, + sentence_index=problem_state.sentence_index + 1, + source_text_offset=problem_state.source_text_offset + + max(sentence_state.token_index, 1), + ) + + +def _end_initial_state_frame( + sentence_state: SentenceReadingState, + problem_state: ProblemReadingState, + sentence_idx: int, + last_position: int, +) -> ProblemReadingState | ReaderRefusal: + if not sentence_state.quantities: + return ReaderRefusal( + reason="incomplete_operation", + detail="initial_state_frame closed with no quantity", + sentence_index=sentence_idx, + token_index=last_position, + token_text="", + ) + if len(sentence_state.quantities) > 1: + return ReaderRefusal( + reason="incomplete_operation", + detail=( + f"initial_state_frame has {len(sentence_state.quantities)} " + "quantities; multi-quantity initial state is Phase-2.1 scope" + ), + sentence_index=sentence_idx, + token_index=last_position, + token_text="", + ) + actor = sentence_state.pending_entity_ref + if actor is None: + return ReaderRefusal( + reason="incomplete_operation", + detail="initial_state_frame has no subject entity", + sentence_index=sentence_idx, + token_index=last_position, + token_text="", + ) + qty = sentence_state.quantities[0] + pip = PartialInitialPossession(entity=actor.canonical_name, quantity=qty) + new_initial_state = problem_state.accumulated_initial_state + (pip,) + new_registry, _ = _carry_entity(sentence_state, problem_state) + return ProblemReadingState( + entity_registry=new_registry, + accumulated_initial_state=new_initial_state, + accumulated_operations=problem_state.accumulated_operations, + unknown_target_slot=problem_state.unknown_target_slot, + pronoun_resolution_history=problem_state.pronoun_resolution_history, + sentence_index=problem_state.sentence_index + 1, + source_text_offset=problem_state.source_text_offset + + max(sentence_state.token_index, 1), + ) + + +def _end_operation_frame( + sentence_state: SentenceReadingState, + problem_state: ProblemReadingState, + sentence_idx: int, + last_position: int, +) -> ProblemReadingState | ReaderRefusal: + if not sentence_state.quantities: + return ReaderRefusal( + reason="incomplete_operation", + detail="operation_frame closed with no quantity", + sentence_index=sentence_idx, + token_index=last_position, + token_text="", + ) + if len(sentence_state.quantities) > 1: + return ReaderRefusal( + reason="incomplete_operation", + detail=( + f"operation_frame has {len(sentence_state.quantities)} " + "quantities; multi-quantity operations are Phase-2.1 scope" + ), + sentence_index=sentence_idx, + token_index=last_position, + token_text="", + ) + actor = sentence_state.pending_entity_ref + if actor is None: + return ReaderRefusal( + reason="incomplete_operation", + detail="operation_frame has no subject entity", + sentence_index=sentence_idx, + token_index=last_position, + token_text="", + ) + verb = sentence_state.pending_verb + if verb is None: + return ReaderRefusal( + reason="incomplete_operation", + detail="operation_frame has no pending_verb", + sentence_index=sentence_idx, + token_index=last_position, + token_text="", + ) + qty = sentence_state.quantities[0] + # Transfer target: the first entity in sentence_state.entities that is NOT + # the actor (added by _rule_op_proper_noun / _rule_op_pronoun). + transfer_target: str | None = None + if verb.kind == "transfer_verb": + for ent in sentence_state.entities: + if ent.canonical_name != actor.canonical_name: + transfer_target = ent.canonical_name + break + pop = PartialOperation( + actor=actor.canonical_name, + kind=verb.kind, + operand=qty, + target=transfer_target, + ) + new_operations = problem_state.accumulated_operations + (pop,) + # Also carry over any newly-introduced entities from this operation frame. + new_registry = problem_state.entity_registry + for ent in (sentence_state.pending_entity_ref,) + sentence_state.entities: + if ent is not None: + existing_names = {e.canonical_name for e in new_registry} + if ent.canonical_name not in existing_names: + new_registry = new_registry + (ent,) + return ProblemReadingState( + entity_registry=new_registry, + accumulated_initial_state=problem_state.accumulated_initial_state, + accumulated_operations=new_operations, + unknown_target_slot=problem_state.unknown_target_slot, + pronoun_resolution_history=problem_state.pronoun_resolution_history, + sentence_index=problem_state.sentence_index + 1, + source_text_offset=problem_state.source_text_offset + + max(sentence_state.token_index, 1), + ) + + +def _end_descriptive_frame( + sentence_state: SentenceReadingState, + problem_state: ProblemReadingState, +) -> ProblemReadingState: + new_registry, _ = _carry_entity(sentence_state, problem_state) + return ProblemReadingState( + entity_registry=new_registry, + accumulated_initial_state=problem_state.accumulated_initial_state, + accumulated_operations=problem_state.accumulated_operations, + unknown_target_slot=problem_state.unknown_target_slot, + pronoun_resolution_history=problem_state.pronoun_resolution_history, + sentence_index=problem_state.sentence_index + 1, + source_text_offset=problem_state.source_text_offset + + max(sentence_state.token_index, 1), + ) + + +# --------------------------------------------------------------------------- +# Rule tables. # --------------------------------------------------------------------------- _Handler = Callable[..., "SentenceReadingState | ReaderRefusal"] +# question_frame — Phase 1, unchanged in semantics. _QUESTION_FRAME_RULES: Final[dict[str, _Handler]] = { - # Openers "question_open": _rule_question_open, - # Quantifiers / comparatives / aggregate "question_continuous_qty": _rule_qty_qualifier, "question_discrete_qty": _rule_qty_qualifier, "question_comparative": _rule_qty_qualifier, "aggregate_modifier": _rule_qty_qualifier, - # Unit nouns - "count_unit_noun": _rule_unit_noun, - "currency_unit_noun": _rule_unit_noun, - "time_unit_noun": _rule_unit_noun, - # Pivots + "count_unit_noun": _rule_unit_noun_question, + "currency_unit_noun": _rule_unit_noun_question, + "time_unit_noun": _rule_unit_noun_question, "modal_aux": _rule_modal_aux, "entity_pronoun": _rule_entity_pronoun, - "proper_noun_token": _rule_proper_noun, + "proper_noun_token": _rule_proper_noun_question, # Residual marker "residual_modifier": _rule_residual_modifier, - # Frame closers - "accumulation_verb": _rule_frame_closer, - "depletion_verb": _rule_frame_closer, - "transfer_verb": _rule_frame_closer, - "capacity_verb": _rule_frame_closer, - "possession_verb": _rule_frame_closer, - "copula_verb": _rule_frame_closer, - "question_terminator": _rule_frame_closer, + "accumulation_verb": _rule_frame_closer_question, + "depletion_verb": _rule_frame_closer_question, + "transfer_verb": _rule_frame_closer_question, + "capacity_verb": _rule_frame_closer_question, + "possession_verb": _rule_frame_closer_question, + "copula_verb": _rule_frame_closer_question, + "question_terminator": _rule_frame_closer_question, + # Quantity tokens that appear in a post-close portion of a question sentence + # drain safely (frame is already closed before they're reached in practice). + "count_quantity": _rule_statement_drain, + "currency_quantity": _rule_statement_drain, + "time_quantity": _rule_statement_drain, + "ordinal_token": _rule_statement_drain, + "mass_noun_token": _rule_statement_drain, +} + +# initial_state_frame — entity had/has/owned N unit. +_INITIAL_STATE_FRAME_RULES: Final[dict[str, _Handler]] = { + "count_quantity": _rule_statement_quantity, + "currency_quantity": _rule_statement_quantity, + "time_quantity": _rule_statement_quantity, + "count_unit_noun": _rule_unit_noun_statement, + "currency_unit_noun": _rule_unit_noun_statement, + "time_unit_noun": _rule_unit_noun_statement, + "modal_aux": _rule_statement_drain, + "residual_modifier": _rule_statement_drain, + "aggregate_modifier": _rule_statement_drain, + "ordinal_token": _rule_statement_drain, + "mass_noun_token": _rule_statement_drain, + "question_comparative": _rule_statement_drain, + "proper_noun_token": _rule_statement_drain, + "entity_pronoun": _rule_statement_drain, + "accumulation_verb": _rule_statement_drain, + "depletion_verb": _rule_statement_drain, + "transfer_verb": _rule_statement_drain, + "capacity_verb": _rule_statement_drain, + "copula_verb": _rule_statement_drain, + "possession_verb": _rule_statement_drain, + "question_open": _rule_statement_drain, + "question_continuous_qty": _rule_statement_drain, + "question_discrete_qty": _rule_statement_drain, + "statement_terminator": _rule_statement_closer, + "question_terminator": _rule_statement_closer, +} + +# operation_frame — entity verb N unit [to entity2]. +_OPERATION_FRAME_RULES: Final[dict[str, _Handler]] = { + "count_quantity": _rule_statement_quantity, + "currency_quantity": _rule_statement_quantity, + "time_quantity": _rule_statement_quantity, + "count_unit_noun": _rule_unit_noun_statement, + "currency_unit_noun": _rule_unit_noun_statement, + "time_unit_noun": _rule_unit_noun_statement, + "modal_aux": _rule_statement_drain, + "residual_modifier": _rule_statement_drain, + "aggregate_modifier": _rule_statement_drain, + "ordinal_token": _rule_statement_drain, + "mass_noun_token": _rule_statement_drain, + "question_comparative": _rule_statement_drain, + "proper_noun_token": _rule_op_proper_noun, + "entity_pronoun": _rule_op_pronoun, + "accumulation_verb": _rule_statement_drain, + "depletion_verb": _rule_statement_drain, + "transfer_verb": _rule_statement_drain, + "capacity_verb": _rule_statement_drain, + "copula_verb": _rule_statement_drain, + "possession_verb": _rule_statement_drain, + "question_open": _rule_statement_drain, + "question_continuous_qty": _rule_statement_drain, + "question_discrete_qty": _rule_statement_drain, + "statement_terminator": _rule_statement_closer, + "question_terminator": _rule_statement_closer, +} + +# descriptive_frame — drains known categories; closes on terminator. +_DESCRIPTIVE_FRAME_RULES: Final[dict[str, _Handler]] = { + "statement_terminator": _rule_statement_closer, + "modal_aux": _rule_statement_drain, + "residual_modifier": _rule_statement_drain, + "aggregate_modifier": _rule_statement_drain, + "ordinal_token": _rule_statement_drain, + "mass_noun_token": _rule_statement_drain, + "question_comparative": _rule_statement_drain, + "count_unit_noun": _rule_statement_drain, + "currency_unit_noun": _rule_statement_drain, + "time_unit_noun": _rule_statement_drain, + "proper_noun_token": _rule_statement_drain, + "entity_pronoun": _rule_statement_drain, + "count_quantity": _rule_statement_drain, + "currency_quantity": _rule_statement_drain, + "time_quantity": _rule_statement_drain, + "accumulation_verb": _rule_statement_drain, + "depletion_verb": _rule_statement_drain, + "transfer_verb": _rule_statement_drain, + "capacity_verb": _rule_statement_drain, + "possession_verb": _rule_statement_drain, + "copula_verb": _rule_statement_drain, + "question_open": _rule_statement_drain, + "question_continuous_qty": _rule_statement_drain, + "question_discrete_qty": _rule_statement_drain, } @@ -828,4 +1868,5 @@ __all__ = [ "apply_word", "begin_sentence", "end_sentence", + "finalize", ] diff --git a/generate/comprehension/state.py b/generate/comprehension/state.py index 4c7e8dba..3a491c9f 100644 --- a/generate/comprehension/state.py +++ b/generate/comprehension/state.py @@ -244,6 +244,7 @@ class QuestionTargetSlot: entity: str | None unit_class: str | None position: int + unit: str | None = None def __post_init__(self) -> None: if self.kind not in VALID_QUESTION_KINDS: @@ -254,14 +255,18 @@ class QuestionTargetSlot: _require_optional_str(self.entity, "QuestionTargetSlot.entity") _require_optional_str(self.unit_class, "QuestionTargetSlot.unit_class") _require_non_negative_int(self.position, "QuestionTargetSlot.position") + _require_optional_str(self.unit, "QuestionTargetSlot.unit") def as_canonical(self) -> dict[str, Any]: - return { + d: dict[str, Any] = { "entity": self.entity, "kind": self.kind, "position": self.position, "unit_class": self.unit_class, } + if self.unit is not None: + d["unit"] = self.unit + return d @dataclass(frozen=True, slots=True) diff --git a/generate/math_candidate_graph.py b/generate/math_candidate_graph.py index 01737ce8..37f616e9 100644 --- a/generate/math_candidate_graph.py +++ b/generate/math_candidate_graph.py @@ -412,6 +412,109 @@ def _build_graph( return None +# --------------------------------------------------------------------------- +# Comprehension reader integration (ADR-0164 Phase 2) +# --------------------------------------------------------------------------- + + +def _tokenize_sentence(sentence: str) -> list[str]: + """Split a sentence string into tokens for the reader. + + Handles punctuation attachment: "." and "?" are emitted as separate + tokens if attached to a word (e.g. "dollars." → ["dollars", "."]). + Commas are split off too. + """ + import re as _re + raw = _re.split(r"\s+", sentence.strip()) + out: list[str] = [] + for tok in raw: + if not tok: + continue + # Detach trailing punctuation. + if len(tok) > 1 and tok[-1] in (".", "?", "!"): + out.append(tok[:-1]) + out.append(tok[-1]) + elif len(tok) > 1 and tok[-1] == ",": + out.append(tok[:-1]) + out.append(",") + else: + out.append(tok) + return [t for t in out if t] + + +def _try_comprehension_reader(text: str) -> CandidateGraphResult | None: + """Attempt to parse and solve *text* via the incremental reader. + + Returns a :class:`CandidateGraphResult` on success or reader refusal, + or ``None`` if the reader itself errors unexpectedly (caller falls + through to regex path). + + All-or-nothing: if the reader refuses on ANY sentence, returns None + so the existing regex path can try. This guarantees wrong == 0 — + the reader either produces a verified graph or steps aside. + """ + try: + from generate.comprehension.lifecycle import ( + apply_word, + begin_sentence, + end_sentence, + finalize, + ) + from generate.comprehension.state import ( + EntityRef, + ProblemReadingState, + ReaderRefusal, + ) + from generate.math_solver import SolveError, solve + except Exception: + return None + + sentences = _split_sentences(text) + if not sentences: + return None + + ps = ProblemReadingState( + entity_registry=(), + accumulated_initial_state=(), + accumulated_operations=(), + unknown_target_slot=None, + pronoun_resolution_history=(), + sentence_index=0, + source_text_offset=0, + ) + + for sentence in sentences: + tokens = _tokenize_sentence(sentence) + ss = begin_sentence(ps, ps.source_text_offset) + for tok in tokens: + result = apply_word(ss, ps, tok) + if isinstance(result, ReaderRefusal): + return None # fall through to regex + ss = result + end = end_sentence(ss, ps) + if isinstance(end, ReaderRefusal): + return None # fall through to regex + ps = end + + graph_or_refusal = finalize(ps) + if isinstance(graph_or_refusal, ReaderRefusal): + return None # fall through to regex + + graph = graph_or_refusal + try: + answer = solve(graph) + except (SolveError, Exception): + return None # fall through to regex + + return CandidateGraphResult( + answer=answer, + selected_graph=graph, + refusal_reason=None, + branches_enumerated=1, + branches_admissible=1, + ) + + # --------------------------------------------------------------------------- # Orchestrator # --------------------------------------------------------------------------- @@ -456,6 +559,17 @@ def parse_and_solve( branches_enumerated=0, branches_admissible=0, ) + # ADR-0164 Phase 2 — comprehension reader path (flag-gated, off by default). + # All-or-nothing: reader either solves the whole problem or falls through. + # Reuses the same RuntimeConfig.comprehension_reader_questions flag that + # Phase 1 introduced (#331). Whole-problem reader takes priority over the + # Phase 1 question-only hybrid path because finalize() emits a complete + # MathProblemGraph that the existing solver consumes unchanged. + if config is not None and config.comprehension_reader_questions: + reader_result = _try_comprehension_reader(text) + if reader_result is not None: + return reader_result + sentences = _split_sentences(text) if not sentences: return CandidateGraphResult( diff --git a/language_packs/data/en_core_math_v1/lexicon/accumulation_verb.jsonl b/language_packs/data/en_core_math_v1/lexicon/accumulation_verb.jsonl index d9f222c7..76940cef 100644 --- a/language_packs/data/en_core_math_v1/lexicon/accumulation_verb.jsonl +++ b/language_packs/data/en_core_math_v1/lexicon/accumulation_verb.jsonl @@ -1,19 +1,21 @@ -{"lemma": "buy", "category": "accumulation_verb", "aliases": ["buys", "bought"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "get", "category": "accumulation_verb", "aliases": ["gets", "got"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "find", "category": "accumulation_verb", "aliases": ["finds", "found"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "receive", "category": "accumulation_verb", "aliases": ["receives", "received"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "earn", "category": "accumulation_verb", "aliases": ["earns", "earned"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} {"lemma": "add", "category": "accumulation_verb", "aliases": ["adds", "added"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "collect", "category": "accumulation_verb", "aliases": ["collects", "collected"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "gather", "category": "accumulation_verb", "aliases": ["gathers", "gathered"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "catch", "category": "accumulation_verb", "aliases": ["catches", "caught"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "save", "category": "accumulation_verb", "aliases": ["saves", "saved"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "adopt", "category": "accumulation_verb", "aliases": ["adopts", "adopted"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "bake", "category": "accumulation_verb", "aliases": ["bakes", "baked"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "cook", "category": "accumulation_verb", "aliases": ["cooks", "cooked"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "slice", "category": "accumulation_verb", "aliases": ["slices", "sliced"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} {"lemma": "build", "category": "accumulation_verb", "aliases": ["builds", "built"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "grow", "category": "accumulation_verb", "aliases": ["grows", "grew"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "gain", "category": "accumulation_verb", "aliases": ["gains", "gained"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "buy", "category": "accumulation_verb", "aliases": ["buys", "bought"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "catch", "category": "accumulation_verb", "aliases": ["catches", "caught"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} {"lemma": "charge", "category": "accumulation_verb", "aliases": ["charges", "charged"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "collect", "category": "accumulation_verb", "aliases": ["collects", "collected"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "cook", "category": "accumulation_verb", "aliases": ["cooks", "cooked"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "earn", "category": "accumulation_verb", "aliases": ["earns", "earned"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "find", "category": "accumulation_verb", "aliases": ["finds", "found"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "gain", "category": "accumulation_verb", "aliases": ["gains", "gained"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "gather", "category": "accumulation_verb", "aliases": ["gathers", "gathered"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "get", "category": "accumulation_verb", "aliases": ["gets", "got"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "grow", "category": "accumulation_verb", "aliases": ["grows", "grew"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "invest", "category": "accumulation_verb", "aliases": ["invests", "invested"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "need", "category": "accumulation_verb", "aliases": ["needs", "needed"], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "receive", "category": "accumulation_verb", "aliases": ["receives", "received"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "save", "category": "accumulation_verb", "aliases": ["saves", "saved"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "slice", "category": "accumulation_verb", "aliases": ["slices", "sliced"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} {"lemma": "want", "category": "accumulation_verb", "aliases": ["wants", "wanted"], "provenance": "phase_1_reader_supplemental_2026-05-26"} diff --git a/language_packs/data/en_core_math_v1/lexicon/capacity_verb.jsonl b/language_packs/data/en_core_math_v1/lexicon/capacity_verb.jsonl index 9c76eb7c..2d8963df 100644 --- a/language_packs/data/en_core_math_v1/lexicon/capacity_verb.jsonl +++ b/language_packs/data/en_core_math_v1/lexicon/capacity_verb.jsonl @@ -1,13 +1,19 @@ -{"lemma": "shuck", "category": "capacity_verb", "aliases": ["shucks"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "pick", "category": "capacity_verb", "aliases": ["picks"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "pack", "category": "capacity_verb", "aliases": ["packs"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "make", "category": "capacity_verb", "aliases": ["makes", "made"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "produce", "category": "capacity_verb", "aliases": ["produces", "produced"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "type", "category": "capacity_verb", "aliases": ["types", "typed"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "read", "category": "capacity_verb", "aliases": ["reads"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "write", "category": "capacity_verb", "aliases": ["writes", "wrote"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "paint", "category": "capacity_verb", "aliases": ["paints", "painted"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "run", "category": "capacity_verb", "aliases": ["runs", "ran"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} -{"lemma": "score", "category": "capacity_verb", "aliases": ["scores", "scored"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} {"lemma": "answer", "category": "capacity_verb", "aliases": ["answers", "answered"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} {"lemma": "complete", "category": "capacity_verb", "aliases": ["completes", "completed"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "drive", "category": "capacity_verb", "aliases": ["drives", "drove"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "fill", "category": "capacity_verb", "aliases": ["fills", "filled"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "finish", "category": "capacity_verb", "aliases": ["finishes", "finished"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "lift", "category": "capacity_verb", "aliases": ["lifts", "lifted", "lifting"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "make", "category": "capacity_verb", "aliases": ["makes", "made"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "pack", "category": "capacity_verb", "aliases": ["packs"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "paint", "category": "capacity_verb", "aliases": ["paints", "painted"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "pick", "category": "capacity_verb", "aliases": ["picks"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "play", "category": "capacity_verb", "aliases": ["plays", "played"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "produce", "category": "capacity_verb", "aliases": ["produces", "produced"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "read", "category": "capacity_verb", "aliases": ["reads"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "run", "category": "capacity_verb", "aliases": ["runs", "ran"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "score", "category": "capacity_verb", "aliases": ["scores", "scored"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "shuck", "category": "capacity_verb", "aliases": ["shucks"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "type", "category": "capacity_verb", "aliases": ["types", "typed"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} +{"lemma": "work", "category": "capacity_verb", "aliases": ["works", "worked"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "write", "category": "capacity_verb", "aliases": ["writes", "wrote"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} diff --git a/language_packs/data/en_core_math_v1/lexicon/count_unit_noun.jsonl b/language_packs/data/en_core_math_v1/lexicon/count_unit_noun.jsonl index b861e9bf..cfc4d6bd 100644 --- a/language_packs/data/en_core_math_v1/lexicon/count_unit_noun.jsonl +++ b/language_packs/data/en_core_math_v1/lexicon/count_unit_noun.jsonl @@ -1,6 +1,63 @@ -{"lemma": "box", "category": "count_unit_noun", "aliases": ["boxes"], "provenance": "phase_1_reader_supplemental_2026-05-26"} -{"lemma": "crayon", "category": "count_unit_noun", "aliases": ["crayons"], "provenance": "phase_1_reader_supplemental_2026-05-26"} -{"lemma": "follower", "category": "count_unit_noun", "aliases": ["followers"], "provenance": "phase_1_reader_supplemental_2026-05-26"} -{"lemma": "candy", "category": "count_unit_noun", "aliases": ["candies"], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "apple", "category": "count_unit_noun", "aliases": ["apples"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "appointment", "category": "count_unit_noun", "aliases": ["appointments"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "bag", "category": "count_unit_noun", "aliases": ["bags"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "basket", "category": "count_unit_noun", "aliases": ["baskets"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "bead", "category": "count_unit_noun", "aliases": ["beads"], "provenance": "phase_2_reader_2026-05-26"} {"lemma": "bean", "category": "count_unit_noun", "aliases": ["beans"], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "bird", "category": "count_unit_noun", "aliases": ["birds"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "book", "category": "count_unit_noun", "aliases": ["books"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "box", "category": "count_unit_noun", "aliases": ["boxes"], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "bracelet", "category": "count_unit_noun", "aliases": ["bracelets"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "brownie", "category": "count_unit_noun", "aliases": ["brownies"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "cable", "category": "count_unit_noun", "aliases": ["cables"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "candy", "category": "count_unit_noun", "aliases": ["candies"], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "card", "category": "count_unit_noun", "aliases": ["cards"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "cat", "category": "count_unit_noun", "aliases": ["cats"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "chicken", "category": "count_unit_noun", "aliases": ["chickens"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "child", "category": "count_unit_noun", "aliases": ["children"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "coconut", "category": "count_unit_noun", "aliases": ["coconuts"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "coin", "category": "count_unit_noun", "aliases": ["coins"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "crayon", "category": "count_unit_noun", "aliases": ["crayons"], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "cup", "category": "count_unit_noun", "aliases": ["cups"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "dog", "category": "count_unit_noun", "aliases": ["dogs"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "duck", "category": "count_unit_noun", "aliases": ["ducks"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "eraser", "category": "count_unit_noun", "aliases": ["erasers"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "fish", "category": "count_unit_noun", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "follower", "category": "count_unit_noun", "aliases": ["followers"], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "foot", "category": "count_unit_noun", "aliases": ["feet"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "gig", "category": "count_unit_noun", "aliases": ["gigs"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "goal", "category": "count_unit_noun", "aliases": ["goals"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "horse", "category": "count_unit_noun", "aliases": ["horses"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "ingredient", "category": "count_unit_noun", "aliases": ["ingredients"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "kid", "category": "count_unit_noun", "aliases": ["kids"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "kitten", "category": "count_unit_noun", "aliases": ["kittens"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "km", "category": "count_unit_noun", "aliases": ["kilometer", "kilometers"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "lemonade", "category": "count_unit_noun", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "macaroon", "category": "count_unit_noun", "aliases": ["macaroons"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "mile", "category": "count_unit_noun", "aliases": ["miles"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "ounce", "category": "count_unit_noun", "aliases": ["ounces"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "oyster", "category": "count_unit_noun", "aliases": ["oysters"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "pan", "category": "count_unit_noun", "aliases": ["pans"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "paperclip", "category": "count_unit_noun", "aliases": ["paperclips"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "parakeet", "category": "count_unit_noun", "aliases": ["parakeets"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "person", "category": "count_unit_noun", "aliases": ["people"], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "picture", "category": "count_unit_noun", "aliases": ["pictures"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "piece", "category": "count_unit_noun", "aliases": ["pieces"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "point", "category": "count_unit_noun", "aliases": ["points"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "pound", "category": "count_unit_noun", "aliases": ["pounds"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "puppy", "category": "count_unit_noun", "aliases": ["puppies"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "question", "category": "count_unit_noun", "aliases": ["questions"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "rock", "category": "count_unit_noun", "aliases": ["rocks"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "scoop", "category": "count_unit_noun", "aliases": ["scoops"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "section", "category": "count_unit_noun", "aliases": ["sections"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "set", "category": "count_unit_noun", "aliases": ["sets"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "sign", "category": "count_unit_noun", "aliases": ["signs"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "song", "category": "count_unit_noun", "aliases": ["songs"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "stamp", "category": "count_unit_noun", "aliases": ["stamps"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "stop", "category": "count_unit_noun", "aliases": ["stops"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "strawberry", "category": "count_unit_noun", "aliases": ["strawberries"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "student", "category": "count_unit_noun", "aliases": ["students"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "survey", "category": "count_unit_noun", "aliases": ["surveys"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "ticket", "category": "count_unit_noun", "aliases": ["tickets"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "video", "category": "count_unit_noun", "aliases": ["videos"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "yard", "category": "count_unit_noun", "aliases": ["yards"], "provenance": "phase_2_reader_2026-05-26"} diff --git a/language_packs/data/en_core_math_v1/lexicon/currency_unit_noun.jsonl b/language_packs/data/en_core_math_v1/lexicon/currency_unit_noun.jsonl index be07bb84..9b258e8a 100644 --- a/language_packs/data/en_core_math_v1/lexicon/currency_unit_noun.jsonl +++ b/language_packs/data/en_core_math_v1/lexicon/currency_unit_noun.jsonl @@ -1,3 +1,5 @@ +{"lemma": "cent", "category": "currency_unit_noun", "aliases": ["cents"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "dollar", "category": "currency_unit_noun", "aliases": ["dollars"], "provenance": "phase_2_reader_2026-05-26"} {"lemma": "money", "category": "currency_unit_noun", "aliases": [], "provenance": "ported_from_math_candidate_parser_2026-05-26"} {"lemma": "profit", "category": "currency_unit_noun", "aliases": [], "provenance": "ported_from_math_candidate_parser_2026-05-26"} {"lemma": "interest", "category": "currency_unit_noun", "aliases": [], "provenance": "ported_from_math_candidate_parser_2026-05-26"} diff --git a/language_packs/data/en_core_math_v1/lexicon/drain_token.jsonl b/language_packs/data/en_core_math_v1/lexicon/drain_token.jsonl index b7ade49d..72aa7aca 100644 --- a/language_packs/data/en_core_math_v1/lexicon/drain_token.jsonl +++ b/language_packs/data/en_core_math_v1/lexicon/drain_token.jsonl @@ -1,18 +1,274 @@ {"lemma": "a", "category": "drain_token", "aliases": ["an"], "provenance": "phase_1_reader_supplemental_2026-05-26"} -{"lemma": "the", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} -{"lemma": "of", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} -{"lemma": "on", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} -{"lemma": "in", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} -{"lemma": "at", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} -{"lemma": "for", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} -{"lemma": "with", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} -{"lemma": "if", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "able", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "about", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "across", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "addison", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "additional", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "afternoon", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "age", "category": "drain_token", "aliases": ["ages", "aged"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "ago", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} {"lemma": "all", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} -{"lemma": "some", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} -{"lemma": "this", "category": "drain_token", "aliases": ["that"], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "already", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "also", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "altogether", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "among", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "amusement", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "and", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "another", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "any", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "aquarium", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "as", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "at", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "average", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "away", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "back", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "bank", "category": "drain_token", "aliases": ["banks"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "beach", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "because", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "before", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "behind", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "bench", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "best", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "between", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "bike", "category": "drain_token", "aliases": ["bikes"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "bookstore", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "boy", "category": "drain_token", "aliases": ["boys"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "brand", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "bronx", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "brother", "category": "drain_token", "aliases": ["brothers"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "brown", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "building", "category": "drain_token", "aliases": ["buildings"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "but", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "by", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "camp", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "car", "category": "drain_token", "aliases": ["cars"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "carriage", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "channel", "category": "drain_token", "aliases": ["channels"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "cheap", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "class", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "color", "category": "drain_token", "aliases": ["colors", "colored"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "compare", "category": "drain_token", "aliases": ["compares", "compared"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "comparing", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "computer", "category": "drain_token", "aliases": ["computers"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "council", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "cream", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "current", "category": "drain_token", "aliases": ["currently"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "currently", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "cut", "category": "drain_token", "aliases": ["cuts"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "daily", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "dance", "category": "drain_token", "aliases": ["dances"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "daughter", "category": "drain_token", "aliases": ["daughters"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "decided", "category": "drain_token", "aliases": ["decide", "decides"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "degree", "category": "drain_token", "aliases": ["degrees"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "delivery", "category": "drain_token", "aliases": ["deliveries"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "different", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "distance", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "divide", "category": "drain_token", "aliases": ["divides", "divided"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "down", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "draw", "category": "drain_token", "aliases": ["draws", "drew"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "during", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} -{"lemma": "two", "category": "drain_token", "aliases": ["three", "four", "five"], "provenance": "phase_1_reader_supplemental_2026-05-26"} -{"lemma": "studying", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} -{"lemma": "purchase", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} -{"lemma": "social", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "each", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "eligible", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "encounter", "category": "drain_token", "aliases": ["encounters", "encountered"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "end", "category": "drain_token", "aliases": ["ends"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "enough", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "equally", "category": "drain_token", "aliases": ["equal"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "evening", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "every", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "everybody", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "everyone", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "exam", "category": "drain_token", "aliases": ["exams"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "expensive", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "facebook", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "family", "category": "drain_token", "aliases": ["families"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "fast", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "fastest", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "father", "category": "drain_token", "aliases": ["fathers"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "finally", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "first", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "floor", "category": "drain_token", "aliases": ["floors"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "food", "category": "drain_token", "aliases": ["foods"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "football", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "for", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "friend", "category": "drain_token", "aliases": ["friends"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "from", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "full", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "fun", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "girl", "category": "drain_token", "aliases": ["girls"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "go", "category": "drain_token", "aliases": ["goes", "went", "gone"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "going", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "grandfather", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "guest", "category": "drain_token", "aliases": ["guests"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "half", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "health", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "help", "category": "drain_token", "aliases": ["helps", "helped"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "home", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "hourly", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "house", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "ice", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "if", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "illustration", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "improve", "category": "drain_token", "aliases": ["improves", "improved"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "in", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "initially", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "instagram", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "insurance", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "into", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "jumping", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "june", "category": "drain_token", "aliases": ["july", "august", "september", "october", "november", "december", "january", "february", "march", "april", "may"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "junior", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "just", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "keep", "category": "drain_token", "aliases": ["keeps", "kept"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "keyboard", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "lady", "category": "drain_token", "aliases": ["ladies"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "lake", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "last", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "later", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "live", "category": "drain_token", "aliases": ["lives", "lived", "living"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "local", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "locals", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "locker", "category": "drain_token", "aliases": ["lockers"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "long", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "loose", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "manhattan", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} {"lemma": "media", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "men", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "morning", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "most", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "mother", "category": "drain_token", "aliases": ["mothers"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "mountain", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "mouse", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "near", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "new", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "next", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "night", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "no", "category": "drain_token", "aliases": ["not"], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "now", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "number", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "of", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "off", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "old", "category": "drain_token", "aliases": ["older"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "on", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "once", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "one", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "only", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "open", "category": "drain_token", "aliases": ["opens", "opened"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "other", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "out", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "over", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "overtime", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "own", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "pace", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "page", "category": "drain_token", "aliases": ["pages"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "parent", "category": "drain_token", "aliases": ["parents"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "park", "category": "drain_token", "aliases": ["parks"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "party", "category": "drain_token", "aliases": ["parties"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "past", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "per", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "percent", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "pet", "category": "drain_token", "aliases": ["pets"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "picking", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "player", "category": "drain_token", "aliases": ["players"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "pokemon", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "press", "category": "drain_token", "aliases": ["presses", "pressed"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "problem", "category": "drain_token", "aliases": ["problems"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "purchase", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "purpose", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "put", "category": "drain_token", "aliases": ["puts"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "raise", "category": "drain_token", "aliases": ["raises", "raised"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "rate", "category": "drain_token", "aliases": ["rates"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "reading", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "ready", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "rent", "category": "drain_token", "aliases": ["rents", "rented"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "rep", "category": "drain_token", "aliases": ["reps"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "rest", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "reunion", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "review", "category": "drain_token", "aliases": ["reviews"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "ride", "category": "drain_token", "aliases": ["rides", "rode"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "right", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "road", "category": "drain_token", "aliases": ["roads"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "route", "category": "drain_token", "aliases": ["routes"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "running", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "sale", "category": "drain_token", "aliases": ["sales"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "same", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "scented", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "school", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "senior", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "several", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "shelter", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "shift", "category": "drain_token", "aliases": ["shifts"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "shop", "category": "drain_token", "aliases": ["shops"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "side", "category": "drain_token", "aliases": ["sides"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "since", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "sister", "category": "drain_token", "aliases": ["sisters"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "six", "category": "drain_token", "aliases": ["seven", "eight", "nine", "ten"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "slow", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "snack", "category": "drain_token", "aliases": ["snacks"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "so", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "soccer", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "social", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "some", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "son", "category": "drain_token", "aliases": ["sons"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "speed", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "spit", "category": "drain_token", "aliases": ["spits"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "split", "category": "drain_token", "aliases": ["splits"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "stand", "category": "drain_token", "aliases": ["stands"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "started", "category": "drain_token", "aliases": ["start", "starts"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "still", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "storage", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "studying", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "subway", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "summer", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "sweet", "category": "drain_token", "aliases": ["sweets"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "swimming", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "teacher", "category": "drain_token", "aliases": ["teachers"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "team", "category": "drain_token", "aliases": ["teams"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "temperature", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "than", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "that", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "the", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "then", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "there", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "third", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "this", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "through", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "throughout", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "tiktok", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "times", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "to", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "today", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "together", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "town", "category": "drain_token", "aliases": ["towns"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "train", "category": "drain_token", "aliases": ["trains"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "travel", "category": "drain_token", "aliases": ["travels", "traveled", "travelling", "traveling"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "trip", "category": "drain_token", "aliases": ["trips"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "try", "category": "drain_token", "aliases": ["tries", "tried", "trying"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "tub", "category": "drain_token", "aliases": ["tubs"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "twice", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "twitter", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "two", "category": "drain_token", "aliases": ["three", "four", "five"], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "until", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "up", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "upload", "category": "drain_token", "aliases": ["uploads", "uploaded"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "varsity", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "very", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "vet", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "wage", "category": "drain_token", "aliases": ["wages"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "walk", "category": "drain_token", "aliases": ["walks", "walked", "walking"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "water", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "way", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "weekly", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "weighing", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "weight", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "well", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "when", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "where", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "while", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "wire", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "with", "category": "drain_token", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "within", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "without", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "women", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "yet", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_2026-05-26"} +{"lemma": "youtube", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "youtuber", "category": "drain_token", "aliases": [], "provenance": "phase_2_reader_gsm8k_2026-05-26"} diff --git a/language_packs/data/en_core_math_v1/lexicon/proper_noun_gender_female.jsonl b/language_packs/data/en_core_math_v1/lexicon/proper_noun_gender_female.jsonl index aa58b0fb..5a64011d 100644 --- a/language_packs/data/en_core_math_v1/lexicon/proper_noun_gender_female.jsonl +++ b/language_packs/data/en_core_math_v1/lexicon/proper_noun_gender_female.jsonl @@ -60,3 +60,8 @@ {"lemma": "tina", "category": "proper_noun_gender_female", "aliases": ["Tina"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} {"lemma": "virginia", "category": "proper_noun_gender_female", "aliases": ["Virginia"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} {"lemma": "monica", "category": "proper_noun_gender_female", "aliases": ["Monica"], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "allison", "category": "proper_noun_gender_female", "aliases": ["Allison"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "brooke", "category": "proper_noun_gender_female", "aliases": ["Brooke"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "jan", "category": "proper_noun_gender_female", "aliases": ["Jan"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "marion", "category": "proper_noun_gender_female", "aliases": ["Marion"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "sidney", "category": "proper_noun_gender_female", "aliases": ["Sidney"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} diff --git a/language_packs/data/en_core_math_v1/lexicon/proper_noun_gender_male.jsonl b/language_packs/data/en_core_math_v1/lexicon/proper_noun_gender_male.jsonl index 99c1f6f5..1321417e 100644 --- a/language_packs/data/en_core_math_v1/lexicon/proper_noun_gender_male.jsonl +++ b/language_packs/data/en_core_math_v1/lexicon/proper_noun_gender_male.jsonl @@ -75,3 +75,17 @@ {"lemma": "wayne", "category": "proper_noun_gender_male", "aliases": ["Wayne"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} {"lemma": "william", "category": "proper_noun_gender_male", "aliases": ["William"], "provenance": "ported_from_math_candidate_parser_2026-05-26"} {"lemma": "malcolm", "category": "proper_noun_gender_male", "aliases": ["Malcolm"], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "bart", "category": "proper_noun_gender_male", "aliases": ["Bart"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "fernando", "category": "proper_noun_gender_male", "aliases": ["Fernando"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "georgie", "category": "proper_noun_gender_male", "aliases": ["Georgie"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "jake", "category": "proper_noun_gender_male", "aliases": ["Jake"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "jed", "category": "proper_noun_gender_male", "aliases": ["Jed"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "jeremie", "category": "proper_noun_gender_male", "aliases": ["Jeremie"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "jose", "category": "proper_noun_gender_male", "aliases": ["Jose"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "orlando", "category": "proper_noun_gender_male", "aliases": ["Orlando"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "rex", "category": "proper_noun_gender_male", "aliases": ["Rex"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "rudolph", "category": "proper_noun_gender_male", "aliases": ["Rudolph"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "steve", "category": "proper_noun_gender_male", "aliases": ["Steve"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "troy", "category": "proper_noun_gender_male", "aliases": ["Troy"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "xavier", "category": "proper_noun_gender_male", "aliases": ["Xavier"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "yun", "category": "proper_noun_gender_male", "aliases": ["Yun"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} diff --git a/language_packs/data/en_core_math_v1/lexicon/time_unit_noun.jsonl b/language_packs/data/en_core_math_v1/lexicon/time_unit_noun.jsonl index caf1304c..1cdce296 100644 --- a/language_packs/data/en_core_math_v1/lexicon/time_unit_noun.jsonl +++ b/language_packs/data/en_core_math_v1/lexicon/time_unit_noun.jsonl @@ -1,5 +1,8 @@ -{"lemma": "time", "category": "time_unit_noun", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} -{"lemma": "hour", "category": "time_unit_noun", "aliases": ["hours"], "provenance": "phase_1_reader_supplemental_2026-05-26"} {"lemma": "day", "category": "time_unit_noun", "aliases": ["days"], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "hour", "category": "time_unit_noun", "aliases": ["hours"], "provenance": "phase_1_reader_supplemental_2026-05-26"} {"lemma": "minute", "category": "time_unit_noun", "aliases": ["minutes"], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "month", "category": "time_unit_noun", "aliases": ["months"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "second", "category": "time_unit_noun", "aliases": ["seconds"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} +{"lemma": "time", "category": "time_unit_noun", "aliases": [], "provenance": "phase_1_reader_supplemental_2026-05-26"} {"lemma": "week", "category": "time_unit_noun", "aliases": ["weeks"], "provenance": "phase_1_reader_supplemental_2026-05-26"} +{"lemma": "year", "category": "time_unit_noun", "aliases": ["years"], "provenance": "phase_2_reader_gsm8k_2026-05-26"} diff --git a/tests/test_en_core_math_v1_pack.py b/tests/test_en_core_math_v1_pack.py index d1704c77..d629e032 100644 --- a/tests/test_en_core_math_v1_pack.py +++ b/tests/test_en_core_math_v1_pack.py @@ -27,7 +27,15 @@ _LEXICON_DIR = _PACK_DIR / "lexicon" PROVENANCE_TAG = "ported_from_math_candidate_parser_2026-05-26" # Phase-1 reader integration added supplemental entries under this tag. _SUPPLEMENTAL_PROVENANCE_TAG = "phase_1_reader_supplemental_2026-05-26" -VALID_PROVENANCE_TAGS = {PROVENANCE_TAG, _SUPPLEMENTAL_PROVENANCE_TAG} +# Phase-2 statement-frame reader integration adds GSM8K coverage entries. +_PHASE_2_PROVENANCE_TAG = "phase_2_reader_gsm8k_2026-05-26" +_PHASE_2_DOLLAR_TAG = "phase_2_reader_2026-05-26" +VALID_PROVENANCE_TAGS = { + PROVENANCE_TAG, + _SUPPLEMENTAL_PROVENANCE_TAG, + _PHASE_2_PROVENANCE_TAG, + _PHASE_2_DOLLAR_TAG, +} # Expected lemma counts sourced directly from the ported whitelist constants. # Change only when the source constant changes AND an ADR ratifies the delta. @@ -39,16 +47,24 @@ VALID_PROVENANCE_TAGS = {PROVENANCE_TAG, _SUPPLEMENTAL_PROVENANCE_TAG} # Brief 8.2 (2026-05-27) renamed proper_noun_entity_{f,m} → proper_noun_gender_{f,m} # as enrichment categories per ADR-0164.1 amendment; admission is now via the # universal proper_noun_token primitive (lexeme_primitives.py). +# ADR-0164 Phase-2 reader integration (Brief 10) ratified additional deltas: +# accumulation_verb +2 (adopt, invest) +# currency_unit_noun +2 (dollar, cent) +# proper_noun_gender_female +5 (allison, brooke, jan, marion, sidney) +# proper_noun_gender_male +14 (bart, fernando, georgie, jake, jed, jeremie, +# jose, orlando, rex, rudolph, steve, troy, +# xavier, yun) +# capacity_verb +6 (fill, lift, play, work, finish, drive) EXPECTED_CATEGORY_COUNTS: dict[str, int] = { - "accumulation_verb": 19, + "accumulation_verb": 21, "depletion_verb": 15, "transfer_verb": 7, - "currency_unit_noun": 7, + "currency_unit_noun": 9, "entity_pronoun": 4, - "proper_noun_gender_female": 62, - "proper_noun_gender_male": 77, + "proper_noun_gender_female": 67, + "proper_noun_gender_male": 91, "possession_verb": 1, - "capacity_verb": 13, + "capacity_verb": 19, "question_open": 2, "residual_modifier": 3, } diff --git a/tests/test_reader_phase2.py b/tests/test_reader_phase2.py new file mode 100644 index 00000000..63990a03 --- /dev/null +++ b/tests/test_reader_phase2.py @@ -0,0 +1,339 @@ +"""Phase 2 statement-frame reader tests (ADR-0164 Phase 2).""" + +from __future__ import annotations + +from decimal import Decimal + +import pytest + +from generate.comprehension.lifecycle import ( + apply_word, + begin_sentence, + end_sentence, + finalize, +) +from generate.comprehension.state import ( + EntityRef, + ProblemReadingState, + ReaderRefusal, + SentenceReadingState, +) +from generate.math_problem_graph import MathProblemGraph + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _empty_problem( + *, + registry: tuple[EntityRef, ...] = (), + sentence_index: int = 0, +) -> ProblemReadingState: + return ProblemReadingState( + entity_registry=registry, + accumulated_initial_state=(), + accumulated_operations=(), + unknown_target_slot=None, + pronoun_resolution_history=(), + sentence_index=sentence_index, + source_text_offset=0, + ) + + +def _read_sentence( + words: list[str], + problem_state: ProblemReadingState, +) -> SentenceReadingState | ReaderRefusal: + state: SentenceReadingState | ReaderRefusal = begin_sentence(problem_state, 0) + assert isinstance(state, SentenceReadingState) + for word in words: + result = apply_word(state, problem_state, word) + if isinstance(result, ReaderRefusal): + return result + state = result + return state + + +def _read_problem(sentences: list[list[str]]) -> ProblemReadingState | ReaderRefusal: + """Drive a list of tokenised sentences through the full lifecycle.""" + ps: ProblemReadingState = _empty_problem() + for words in sentences: + ss = _read_sentence(words, ps) + if isinstance(ss, ReaderRefusal): + return ss + end = end_sentence(ss, ps) + if isinstance(end, ReaderRefusal): + return end + ps = end + return ps + + +# --------------------------------------------------------------------------- +# Initial-state frame +# --------------------------------------------------------------------------- + + +class TestInitialStateFrame: + def test_proper_noun_possession_verb_count_unit(self) -> None: + """Sandra had 600 dollars.""" + ps = _empty_problem() + words = ["Sandra", "had", "600", "dollars", "."] + ss = _read_sentence(words, ps) + assert isinstance(ss, SentenceReadingState), ss + end = end_sentence(ss, ps) + assert isinstance(end, ProblemReadingState), end + assert len(end.accumulated_initial_state) == 1 + pip = end.accumulated_initial_state[0] + assert pip.entity == "sandra" + assert pip.quantity is not None + assert pip.quantity.value == Decimal("600") + assert pip.quantity.unit == "dollar" + assert "sandra" in {e.canonical_name for e in end.entity_registry} + + def test_proper_noun_has_count_unit(self) -> None: + """Tom has 5 apples.""" + ps = _empty_problem() + words = ["Tom", "has", "5", "apples", "."] + ss = _read_sentence(words, ps) + assert isinstance(ss, SentenceReadingState), ss + end = end_sentence(ss, ps) + assert isinstance(end, ProblemReadingState), end + pip = end.accumulated_initial_state[0] + assert pip.entity == "tom" + assert pip.quantity.unit == "apple" + + def test_sentence_index_advances(self) -> None: + ps = _empty_problem() + words = ["Sandra", "had", "600", "dollars", "."] + ss = _read_sentence(words, ps) + assert isinstance(ss, SentenceReadingState) + end = end_sentence(ss, ps) + assert isinstance(end, ProblemReadingState) + assert end.sentence_index == 1 + + def test_entity_added_to_registry(self) -> None: + ps = _empty_problem() + words = ["Monica", "had", "5", "apples", "."] + ss = _read_sentence(words, ps) + assert isinstance(ss, SentenceReadingState) + end = end_sentence(ss, ps) + assert isinstance(end, ProblemReadingState) + names = [e.canonical_name for e in end.entity_registry] + assert "monica" in names + + def test_refuse_no_quantity(self) -> None: + """initial_state_frame with no quantity → incomplete_operation.""" + ps = _empty_problem() + words = ["Sandra", "had", "."] + ss = _read_sentence(words, ps) + assert isinstance(ss, SentenceReadingState) + end = end_sentence(ss, ps) + assert isinstance(end, ReaderRefusal) + assert end.reason == "incomplete_operation" + + +# --------------------------------------------------------------------------- +# Operation frame +# --------------------------------------------------------------------------- + + +class TestOperationFrame: + def test_depletion_verb_count(self) -> None: + """She spent 200 dollars.""" + ps = _empty_problem(registry=(EntityRef("sandra", "female", 0),)) + words = ["She", "spent", "200", "dollars", "."] + ss = _read_sentence(words, ps) + assert isinstance(ss, SentenceReadingState), ss + end = end_sentence(ss, ps) + assert isinstance(end, ProblemReadingState), end + assert len(end.accumulated_operations) == 1 + pop = end.accumulated_operations[0] + assert pop.actor == "sandra" + assert pop.kind == "depletion_verb" + assert pop.operand is not None + assert pop.operand.value == Decimal("200") + assert pop.operand.unit == "dollar" + + def test_accumulation_verb_count(self) -> None: + """Tom earned 3 books.""" + ps = _empty_problem(registry=(EntityRef("tom", "male", 0),)) + words = ["Tom", "earned", "3", "books", "."] + ss = _read_sentence(words, ps) + assert isinstance(ss, SentenceReadingState), ss + end = end_sentence(ss, ps) + assert isinstance(end, ProblemReadingState), end + pop = end.accumulated_operations[0] + assert pop.actor == "tom" + assert pop.kind == "accumulation_verb" + assert pop.operand.unit == "book" + + def test_pronoun_subject(self) -> None: + """He spent 50 dollars — pronoun resolved from registry.""" + ps = _empty_problem(registry=(EntityRef("eric", "male", 0),)) + words = ["He", "spent", "50", "dollars", "."] + ss = _read_sentence(words, ps) + assert isinstance(ss, SentenceReadingState), ss + end = end_sentence(ss, ps) + assert isinstance(end, ProblemReadingState), end + pop = end.accumulated_operations[0] + assert pop.actor == "eric" + + def test_refuse_no_quantity(self) -> None: + ps = _empty_problem(registry=(EntityRef("sandra", "female", 0),)) + words = ["She", "spent", "."] + ss = _read_sentence(words, ps) + assert isinstance(ss, SentenceReadingState) + end = end_sentence(ss, ps) + assert isinstance(end, ReaderRefusal) + assert end.reason == "incomplete_operation" + + +# --------------------------------------------------------------------------- +# Descriptive frame +# --------------------------------------------------------------------------- + + +class TestDescriptiveFrame: + def test_copula_drains_advances(self) -> None: + """Sandra is a baker. — descriptive_frame, no math state.""" + ps = _empty_problem() + words = ["Sandra", "is", "a", "baker", "."] + ss = _read_sentence(words, ps) + # "a" drains, "baker" → unknown_word refusal (not in lexicon) + # This is expected for Phase 2 scope + assert isinstance(ss, (SentenceReadingState, ReaderRefusal)) + + def test_copula_with_known_tokens_only(self) -> None: + """Sandra is the student. — all known tokens drain.""" + ps = _empty_problem() + words = ["Sandra", "is", "the", "student", "."] + ss = _read_sentence(words, ps) + assert isinstance(ss, SentenceReadingState), ss + end = end_sentence(ss, ps) + assert isinstance(end, ProblemReadingState), end + assert len(end.accumulated_initial_state) == 0 + assert len(end.accumulated_operations) == 0 + assert end.sentence_index == 1 + + +# --------------------------------------------------------------------------- +# Full problem round-trip with finalize() +# --------------------------------------------------------------------------- + + +class TestFinalize: + def test_simple_two_sentence_problem(self) -> None: + """Sandra had 600 dollars. She spent 200 dollars. How much is left?""" + sentences = [ + ["Sandra", "had", "600", "dollars", "."], + ["She", "spent", "200", "dollars", "."], + ["How", "much", "money", "will", "she", "be", "left", "with", "?"], + ] + ps = _read_problem(sentences) + assert isinstance(ps, ProblemReadingState), ps + graph = finalize(ps) + assert isinstance(graph, MathProblemGraph), graph + assert "sandra" in graph.entities + assert len(graph.initial_state) == 1 + assert graph.initial_state[0].entity == "sandra" + assert graph.initial_state[0].quantity.value == 600.0 + assert len(graph.operations) == 1 + assert graph.operations[0].kind == "subtract" + assert graph.operations[0].operand.value == 200.0 + assert graph.unknown.entity == "sandra" + + def test_finalize_no_question_target_refuses(self) -> None: + ps = _empty_problem() + result = finalize(ps) + assert isinstance(result, ReaderRefusal) + assert result.reason == "no_question_target" + + def test_finalize_empty_registry_refuses(self) -> None: + from generate.comprehension.state import QuestionTargetSlot + qs = QuestionTargetSlot( + kind="continuous_quantity", + entity="sandra", + unit_class="currency", + unit="dollar", + position=0, + ) + ps = ProblemReadingState( + entity_registry=(), # empty + accumulated_initial_state=(), + accumulated_operations=(), + unknown_target_slot=qs, + pronoun_resolution_history=(), + sentence_index=1, + source_text_offset=0, + ) + result = finalize(ps) + assert isinstance(result, ReaderRefusal) + assert result.reason == "dangling_entity" + + def test_determinism(self) -> None: + """Same input → same trace hash.""" + sentences = [ + ["Sandra", "had", "600", "dollars", "."], + ["She", "spent", "200", "dollars", "."], + ["How", "much", "money", "will", "she", "be", "left", "with", "?"], + ] + ps1 = _read_problem(sentences) + ps2 = _read_problem(sentences) + assert isinstance(ps1, ProblemReadingState) + assert isinstance(ps2, ProblemReadingState) + assert ps1.canonical_hash() == ps2.canonical_hash() + + +# --------------------------------------------------------------------------- +# Refusal coverage +# --------------------------------------------------------------------------- + + +class TestPhase2Refusals: + def test_fraction_token_refused(self) -> None: + """Fraction literals are out of Phase 2 scope.""" + ps = _empty_problem() + s = begin_sentence(ps, 0) + result = apply_word(s, ps, "1/2") + assert isinstance(result, ReaderRefusal) + assert result.reason == "unexpected_category" + assert "Phase 2.1" in result.detail + + def test_verb_without_entity_opens_descriptive(self) -> None: + """Verb before entity (subject dropped) opens descriptive_frame.""" + ps = _empty_problem() + s = begin_sentence(ps, 0) + result = apply_word(s, ps, "spent") + assert isinstance(result, SentenceReadingState) + assert result.frame == "descriptive_frame" + + def test_unresolved_pronoun_statement_frame(self) -> None: + """Pronoun with empty registry refuses at pre-frame.""" + ps = _empty_problem() + s = begin_sentence(ps, 0) + result = apply_word(s, ps, "She") + assert isinstance(result, ReaderRefusal) + assert result.reason == "unresolved_pronoun" + + def test_multi_sentence_wrong_zero(self) -> None: + """All-or-nothing: if one sentence fails, the whole problem refuses.""" + # First sentence succeeds, second has unknown word "baker" + ps0 = _empty_problem() + words1 = ["Sandra", "had", "600", "dollars", "."] + ss1 = _read_sentence(words1, ps0) + assert isinstance(ss1, SentenceReadingState) + ps1 = end_sentence(ss1, ps0) + assert isinstance(ps1, ProblemReadingState) + + # Second sentence: "baker" is unknown → refusal + words2 = ["She", "is", "a", "baker", "."] + ss2 = _read_sentence(words2, ps1) + # "baker" not in lexicon → unknown_word refusal + assert isinstance(ss2, ReaderRefusal) + assert ss2.reason == "unknown_word" + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_reader_question_frame.py b/tests/test_reader_question_frame.py index 795dc139..7150bf2a 100644 --- a/tests/test_reader_question_frame.py +++ b/tests/test_reader_question_frame.py @@ -183,20 +183,20 @@ class TestRefusals: assert r.reason == "unknown_word" assert r.token_text == "@@@" - def test_unexpected_category_non_question_opener(self) -> None: - """A statement-frame opener at position 0 is Phase-2 scope.""" + def test_statement_frame_opener_accepted(self) -> None: + """Phase 2 (ADR-0164.4): proper noun at position 0 opens a statement + pre-frame. After Brief 8.2's gender-enrichment refactor, the lookup + skips proper_noun_gender_* categories and the proper_noun_token + primitive admits "Francine" — Phase 2 then routes it to the + statement-frame pre-frame entity slot instead of refusing. + """ ps = _empty_problem(registry=(EntityRef("francine", "female", 0),)) s = begin_sentence(ps, 0) - # "francine" is a known word in the gender-enrichment lexicon. - # Under sentence-initial lookup-first dispatch, gender categories - # do not admit — they are enrichment, not admission. The - # proper_noun_token primitive then admits "Francine" as a name, - # but at sentence position 0 (statement frame opener) Phase 1 - # refuses. r = apply_word(s, ps, "Francine") - assert isinstance(r, ReaderRefusal) - assert r.reason == "unexpected_category" - assert "Phase-2" in r.detail + assert isinstance(r, SentenceReadingState) + assert r.frame is None # frame determined on next verb + assert r.pending_entity_ref is not None + assert r.pending_entity_ref.canonical_name == "francine" def test_unresolved_pronoun_empty_registry(self) -> None: """A pronoun with no compatible entity refuses cleanly.""" @@ -311,7 +311,9 @@ class TestInitialDispatchAndUnknownGender: state = begin_sentence(ps, 0) out = apply_word(state, ps, "She") assert isinstance(out, ReaderRefusal) - assert out.reason == "unexpected_category" + # Phase 2: pronoun at position 0 attempts resolution; empty + # registry → unresolved_pronoun per ADR-0164.2. + assert out.reason == "unresolved_pronoun" state = begin_sentence(ps, 0) out = apply_word(state, ps, "How") @@ -320,24 +322,29 @@ class TestInitialDispatchAndUnknownGender: def test_sentence_initial_marnie_is_not_gated_by_gender_list(self) -> None: """'Marnie' is not in proper_noun_gender_female (after Brief 8.2 - rename dropped marnie from the female list). Reader still admits - her as a proper_noun_token at sentence-initial position, then - refuses with Phase-2-scope detail (statement frame opener).""" + rename dropped marnie from the female list). Reader admits her + as a proper_noun_token at sentence-initial position. Under + Phase 2, this opens a statement pre-frame (not a refusal). + EntityRef carries gender="unknown" because the name is outside + the curated gender lists. + """ ps = _empty_problem() out = apply_word(begin_sentence(ps, 0), ps, "Marnie") - assert isinstance(out, ReaderRefusal) - assert out.reason == "unexpected_category" - assert "Phase-2" in out.detail + assert isinstance(out, SentenceReadingState) + assert out.pending_entity_ref is not None + assert out.pending_entity_ref.gender == "unknown" def test_sentence_initial_novel_name_uses_primitive_and_unknown_gender(self) -> None: """A name not in either gender list (e.g. 'Zelda') still admits - via the universal proper_noun_token primitive.""" + via the universal proper_noun_token primitive. Under Phase 2, + this opens a statement pre-frame with gender='unknown'.""" ps = _empty_problem() state = begin_sentence(ps, 0) out = apply_word(state, ps, "Zelda") - assert isinstance(out, ReaderRefusal) - assert out.reason == "unexpected_category" - assert "Phase-2" in out.detail + assert isinstance(out, SentenceReadingState) + assert out.pending_entity_ref is not None + assert out.pending_entity_ref.canonical_name == "zelda" + assert out.pending_entity_ref.gender == "unknown" def test_pronoun_single_unknown_entity_resolves(self) -> None: """ADR-0164.2 single-salient fallback: one gender-unknown entity