From 957e7c6642b1c92a8d0d1282046e6d897f5b5a5e Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 26 May 2026 19:54:17 -0700 Subject: [PATCH] feat(comprehension): split ComprehensionState into ProblemReadingState + SentenceReadingState (ADR-0164.3) (#323) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reconciles the #321 skeleton with ADR-0164.3's two-level state model. Changes: - Renames ComprehensionState → SentenceReadingState (backward-compat alias kept; existing callers need not change) - Adds 7 new fields to SentenceReadingState (all defaulted so existing construction still compiles): frame, pending_quantities, pending_entity_ref, pending_verb, token_index, lookback (≤8 entries, validated), partial_frame_payload - Introduces SentenceFrame (Literal), VerbReference, AppliedCategory, FramePayload (stub, frame_kind validated) - Adds ProblemReadingState (outer, problem-scoped) with all 7 fields per ADR-0164.3 table order, no defaults (explicit construction required) - Introduces PartialInitialPossession and PartialOperation (nullable precursors to ADR-0115 types), PronounResolution - Adds READER_REFUSAL_REASONS (11-member frozenset, closed/ADR-tracked) and ReaderRefusal dataclass with reason validation - Adds to_canonical_bytes() standalone function implementing ADR-0164.3 §Canonical-bytes rules: sort keys, omit None, Decimal→str; handles ProblemReadingState, SentenceReadingState, ReaderRefusal - SentenceReadingState.canonical_bytes() kept backward-compatible (original 5 fields, null for None) — existing pinned-bytes tests pass - 47 tests: all original tests pass; new tests cover ProblemReadingState construction, determinism gate, sensitivity gate, ReaderRefusal construction and every READER_REFUSAL_REASONS entry Refs: #320 (ADR-0164.3), #321 (comprehension-state-skeleton) --- generate/comprehension/state.py | 536 +++++++++++++++++++++++++++++- tests/test_comprehension_state.py | 509 +++++++++++++++++++++++++++- 2 files changed, 1023 insertions(+), 22 deletions(-) diff --git a/generate/comprehension/state.py b/generate/comprehension/state.py index 1db62473..4c7e8dba 100644 --- a/generate/comprehension/state.py +++ b/generate/comprehension/state.py @@ -1,9 +1,18 @@ -"""ADR-0164 — immutable partial-comprehension state skeleton. +"""ADR-0164 / ADR-0164.3 — two-level immutable comprehension-state types. -This module defines the typed state container the incremental -comprehension reader will accumulate. It is intentionally pure data: -frozen dataclasses, refusal-first validation, and canonical-bytes -serialization for deterministic replay and trace hashing. +This module defines the typed state containers the incremental comprehension +reader accumulates. It is intentionally pure data: frozen dataclasses, +refusal-first validation, and canonical-bytes serialisation for deterministic +replay and trace hashing. + +Two levels (ADR-0164.3 §Decision): + - ``ProblemReadingState`` — outer, problem-scoped. Persists across sentence + boundaries. Mutated only by ``end_sentence``. + - ``SentenceReadingState`` — inner, sentence-scoped. Lifetime = one sentence. + Created by ``begin_sentence``, mutated by ``apply_word``. + +``ComprehensionState`` is a backward-compatibility alias for +``SentenceReadingState``; existing importers need not change. """ from __future__ import annotations @@ -14,6 +23,10 @@ from dataclasses import dataclass from decimal import Decimal from typing import Any, Final, Literal +# --------------------------------------------------------------------------- +# Closed-set constants +# --------------------------------------------------------------------------- + VALID_GENDERS: Final[frozenset[str]] = frozenset( {"female", "male", "neuter", "unknown"} ) @@ -36,11 +49,59 @@ VALID_EXPECTATION_KINDS: Final[frozenset[str]] = frozenset( } ) +VALID_SENTENCE_FRAME_KINDS: Final[frozenset[str]] = frozenset( + { + "initial_state_frame", + "operation_frame", + "question_frame", + "descriptive_frame", + } +) + +# ADR-0164.3 §ReaderRefusal — closed, ADR-tracked. +# New reasons require an ADR amendment. +READER_REFUSAL_REASONS: Final[frozenset[str]] = frozenset( + { + # apply_word — token-level + "unknown_word", + "unexpected_category", + "expectation_collision", + "unresolved_pronoun", + "ambiguous_pronoun_referent", + # end_sentence — sentence-level + "unfinished_frame", + "unattached_quantity", + "incomplete_operation", + # finalization predicate — problem-level + "no_question_target", + "dangling_entity", + "graph_construction_failure", + } +) + +# SentenceFrame is a Literal over the four discriminator values. +SentenceFrame = Literal[ + "initial_state_frame", + "operation_frame", + "question_frame", + "descriptive_frame", +] + +_LOOKBACK_MAX: Final[int] = 8 + +# --------------------------------------------------------------------------- +# Error +# --------------------------------------------------------------------------- + class ComprehensionStateError(ValueError): """Raised on invalid comprehension-state construction.""" +# --------------------------------------------------------------------------- +# Internal validators (unchanged from #321) +# --------------------------------------------------------------------------- + def _require_non_empty_str(value: object, field_name: str) -> None: if not isinstance(value, str) or value == "": raise ComprehensionStateError( @@ -64,7 +125,7 @@ def _require_int(value: object, field_name: str) -> None: def _require_non_negative_int(value: object, field_name: str) -> None: _require_int(value, field_name) - if value < 0: + if value < 0: # type: ignore[operator] raise ComprehensionStateError(f"{field_name} must be >= 0; got {value}") @@ -86,6 +147,10 @@ def _canonical_decimal(value: Decimal) -> str: return format(normalized, "f") +# --------------------------------------------------------------------------- +# Shared leaf types (unchanged from #321) +# --------------------------------------------------------------------------- + @dataclass(frozen=True, slots=True) class EntityRef: canonical_name: str @@ -229,60 +294,347 @@ class ExpectationFrame: } +# --------------------------------------------------------------------------- +# New leaf types for SentenceReadingState (ADR-0164.3) +# --------------------------------------------------------------------------- + @dataclass(frozen=True, slots=True) -class ComprehensionState: +class VerbReference: + """The verb captured at frame-determining position, awaiting completion.""" + + surface: str + kind: str + position: int + + def __post_init__(self) -> None: + _require_non_empty_str(self.surface, "VerbReference.surface") + _require_non_empty_str(self.kind, "VerbReference.kind") + _require_non_negative_int(self.position, "VerbReference.position") + + def as_canonical(self) -> dict[str, Any]: + return { + "kind": self.kind, + "position": self.position, + "surface": self.surface, + } + + +@dataclass(frozen=True, slots=True) +class AppliedCategory: + """One entry in the lookback window: a category applied at a position.""" + + category: str + position: int + + def __post_init__(self) -> None: + _require_non_empty_str(self.category, "AppliedCategory.category") + _require_non_negative_int(self.position, "AppliedCategory.position") + + def as_canonical(self) -> dict[str, Any]: + return {"category": self.category, "position": self.position} + + +@dataclass(frozen=True, slots=True) +class FramePayload: + """Stub container for the in-construction frame payload. + + The reader (Brief 5 Phase 1) populates sub-fields specific to each + frame kind. This stub carries only the frame_kind discriminator so + the two-level state model can be typed and tested without coupling + to the reader implementation. + """ + + frame_kind: str + + def __post_init__(self) -> None: + if self.frame_kind not in VALID_SENTENCE_FRAME_KINDS: + raise ComprehensionStateError( + "FramePayload.frame_kind must be one of " + f"{sorted(VALID_SENTENCE_FRAME_KINDS)}; got {self.frame_kind!r}" + ) + + def as_canonical(self) -> dict[str, Any]: + return {"frame_kind": self.frame_kind} + + +# --------------------------------------------------------------------------- +# New leaf types for ProblemReadingState (ADR-0164.3) +# --------------------------------------------------------------------------- + +@dataclass(frozen=True, slots=True) +class PartialInitialPossession: + """Precursor to ADR-0115 InitialPossession during reader construction. + + Every field is nullable: the reader builds this incrementally as + tokens arrive. A fully-specified instance (no None fields) projects + to a strict ``InitialPossession`` at ``end_sentence``. + """ + + entity: str | None + quantity: QuantityRef | None + + def __post_init__(self) -> None: + if self.entity is not None: + _require_non_empty_str(self.entity, "PartialInitialPossession.entity") + if self.quantity is not None and not isinstance(self.quantity, QuantityRef): + raise ComprehensionStateError( + "PartialInitialPossession.quantity must be QuantityRef | None; " + f"got {type(self.quantity).__name__}" + ) + + def as_canonical(self) -> dict[str, Any]: + d: dict[str, Any] = {} + if self.entity is not None: + d["entity"] = self.entity + if self.quantity is not None: + d["quantity"] = self.quantity.as_canonical() + return d + + +@dataclass(frozen=True, slots=True) +class PartialOperation: + """Precursor to ADR-0115 Operation during reader construction. + + Every field is nullable: the reader builds this incrementally as + tokens arrive. A fully-specified instance projects to a strict + ``Operation`` at ``end_sentence``. + """ + + actor: str | None + kind: str | None + operand: QuantityRef | None + target: str | None + + def __post_init__(self) -> None: + if self.actor is not None: + _require_non_empty_str(self.actor, "PartialOperation.actor") + if self.kind is not None: + _require_non_empty_str(self.kind, "PartialOperation.kind") + if self.target is not None: + _require_non_empty_str(self.target, "PartialOperation.target") + if self.operand is not None and not isinstance(self.operand, QuantityRef): + raise ComprehensionStateError( + "PartialOperation.operand must be QuantityRef | None; " + f"got {type(self.operand).__name__}" + ) + + def as_canonical(self) -> dict[str, Any]: + d: dict[str, Any] = {} + if self.actor is not None: + d["actor"] = self.actor + if self.kind is not None: + d["kind"] = self.kind + if self.operand is not None: + d["operand"] = self.operand.as_canonical() + if self.target is not None: + d["target"] = self.target + return d + + +@dataclass(frozen=True, slots=True) +class PronounResolution: + """Replay-deterministic record of one pronoun resolution event. + + Per ADR-0164.2. Appended to ``ProblemReadingState.pronoun_resolution_history`` + only when the containing sentence closes successfully. + """ + + pronoun: str + resolved_to: str + at_sentence: int + at_position: int + + def __post_init__(self) -> None: + _require_non_empty_str(self.pronoun, "PronounResolution.pronoun") + _require_non_empty_str(self.resolved_to, "PronounResolution.resolved_to") + _require_non_negative_int(self.at_sentence, "PronounResolution.at_sentence") + _require_non_negative_int(self.at_position, "PronounResolution.at_position") + + def as_canonical(self) -> dict[str, Any]: + return { + "at_position": self.at_position, + "at_sentence": self.at_sentence, + "pronoun": self.pronoun, + "resolved_to": self.resolved_to, + } + + +# --------------------------------------------------------------------------- +# ReaderRefusal (ADR-0164.3 §ReaderRefusal) +# --------------------------------------------------------------------------- + +@dataclass(frozen=True, slots=True) +class ReaderRefusal: + """Typed refusal record. Carries one of the closed READER_REFUSAL_REASONS. + + ``token_text`` may be empty string for sentence-level or problem-level + refusals where no single token is in question. + """ + + reason: str + detail: str + sentence_index: int + token_index: int + token_text: str + + def __post_init__(self) -> None: + if self.reason not in READER_REFUSAL_REASONS: + raise ComprehensionStateError( + "ReaderRefusal.reason must be a member of READER_REFUSAL_REASONS; " + f"got {self.reason!r}" + ) + _require_non_empty_str(self.detail, "ReaderRefusal.detail") + _require_non_negative_int(self.sentence_index, "ReaderRefusal.sentence_index") + _require_non_negative_int(self.token_index, "ReaderRefusal.token_index") + if not isinstance(self.token_text, str): + raise ComprehensionStateError( + "ReaderRefusal.token_text must be str; " + f"got {type(self.token_text).__name__}" + ) + + def as_canonical(self) -> dict[str, Any]: + return { + "detail": self.detail, + "reason": self.reason, + "sentence_index": self.sentence_index, + "token_index": self.token_index, + "token_text": self.token_text, + } + + def canonical_bytes(self) -> bytes: + return to_canonical_bytes(self) + + def canonical_hash(self) -> str: + return hashlib.sha256(self.canonical_bytes()).hexdigest() + + +# --------------------------------------------------------------------------- +# SentenceReadingState (inner, sentence-scoped) — ADR-0164.3 §Decision +# Renamed from ComprehensionState (#321). Original five fields stay verbatim. +# Seven new fields added (all with defaults) per ADR-0164.3 §SentenceReadingState. +# --------------------------------------------------------------------------- + +@dataclass(frozen=True, slots=True) +class SentenceReadingState: + # --- original five fields (verbatim from #321) --- entities: tuple[EntityRef, ...] quantities: tuple[QuantityRef, ...] operations: tuple[PartialOp, ...] question_target: QuestionTargetSlot | None = None expectation: ExpectationFrame | None = None + # --- ADR-0164.3 §SentenceReadingState new fields --- + frame: SentenceFrame | None = None + pending_quantities: tuple[QuantityRef, ...] = () + pending_entity_ref: EntityRef | None = None + pending_verb: VerbReference | None = None + token_index: int = 0 + lookback: tuple[AppliedCategory, ...] = () + partial_frame_payload: FramePayload | None = None + def __post_init__(self) -> None: + # --- validate original fields --- if not isinstance(self.entities, tuple): raise ComprehensionStateError( - "ComprehensionState.entities must be tuple[EntityRef, ...]" + "SentenceReadingState.entities must be tuple[EntityRef, ...]" ) if not isinstance(self.quantities, tuple): raise ComprehensionStateError( - "ComprehensionState.quantities must be tuple[QuantityRef, ...]" + "SentenceReadingState.quantities must be tuple[QuantityRef, ...]" ) if not isinstance(self.operations, tuple): raise ComprehensionStateError( - "ComprehensionState.operations must be tuple[PartialOp, ...]" + "SentenceReadingState.operations must be tuple[PartialOp, ...]" ) for idx, entity in enumerate(self.entities): if not isinstance(entity, EntityRef): raise ComprehensionStateError( - f"ComprehensionState.entities[{idx}] must be EntityRef; " + f"SentenceReadingState.entities[{idx}] must be EntityRef; " f"got {type(entity).__name__}" ) for idx, quantity in enumerate(self.quantities): if not isinstance(quantity, QuantityRef): raise ComprehensionStateError( - f"ComprehensionState.quantities[{idx}] must be QuantityRef; " + f"SentenceReadingState.quantities[{idx}] must be QuantityRef; " f"got {type(quantity).__name__}" ) for idx, operation in enumerate(self.operations): if not isinstance(operation, PartialOp): raise ComprehensionStateError( - f"ComprehensionState.operations[{idx}] must be PartialOp; " + f"SentenceReadingState.operations[{idx}] must be PartialOp; " f"got {type(operation).__name__}" ) if self.question_target is not None and not isinstance( self.question_target, QuestionTargetSlot ): raise ComprehensionStateError( - "ComprehensionState.question_target must be " + "SentenceReadingState.question_target must be " f"QuestionTargetSlot | None; got {type(self.question_target).__name__}" ) if self.expectation is not None and not isinstance( self.expectation, ExpectationFrame ): raise ComprehensionStateError( - "ComprehensionState.expectation must be " + "SentenceReadingState.expectation must be " f"ExpectationFrame | None; got {type(self.expectation).__name__}" ) + # --- validate new fields --- + if self.frame is not None and self.frame not in VALID_SENTENCE_FRAME_KINDS: + raise ComprehensionStateError( + "SentenceReadingState.frame must be a SentenceFrame literal or None; " + f"got {self.frame!r}" + ) + if not isinstance(self.pending_quantities, tuple): + raise ComprehensionStateError( + "SentenceReadingState.pending_quantities must be tuple[QuantityRef, ...]" + ) + for idx, pq in enumerate(self.pending_quantities): + if not isinstance(pq, QuantityRef): + raise ComprehensionStateError( + f"SentenceReadingState.pending_quantities[{idx}] must be " + f"QuantityRef; got {type(pq).__name__}" + ) + if self.pending_entity_ref is not None and not isinstance( + self.pending_entity_ref, EntityRef + ): + raise ComprehensionStateError( + "SentenceReadingState.pending_entity_ref must be EntityRef | None; " + f"got {type(self.pending_entity_ref).__name__}" + ) + if self.pending_verb is not None and not isinstance( + self.pending_verb, VerbReference + ): + raise ComprehensionStateError( + "SentenceReadingState.pending_verb must be VerbReference | None; " + f"got {type(self.pending_verb).__name__}" + ) + _require_non_negative_int(self.token_index, "SentenceReadingState.token_index") + if not isinstance(self.lookback, tuple): + raise ComprehensionStateError( + "SentenceReadingState.lookback must be tuple[AppliedCategory, ...]" + ) + if len(self.lookback) > _LOOKBACK_MAX: + raise ComprehensionStateError( + f"SentenceReadingState.lookback must be ≤{_LOOKBACK_MAX} entries; " + f"got {len(self.lookback)}" + ) + for idx, ac in enumerate(self.lookback): + if not isinstance(ac, AppliedCategory): + raise ComprehensionStateError( + f"SentenceReadingState.lookback[{idx}] must be AppliedCategory; " + f"got {type(ac).__name__}" + ) + if self.partial_frame_payload is not None and not isinstance( + self.partial_frame_payload, FramePayload + ): + raise ComprehensionStateError( + "SentenceReadingState.partial_frame_payload must be " + f"FramePayload | None; got {type(self.partial_frame_payload).__name__}" + ) + + # --- backward-compatible serialisation (original 5 fields only, null for None) --- + def as_canonical(self) -> dict[str, Any]: return { "entities": [entity.as_canonical() for entity in self.entities], @@ -315,3 +667,157 @@ class ComprehensionState: def canonical_hash(self) -> str: return hashlib.sha256(self.canonical_bytes()).hexdigest() + +# --------------------------------------------------------------------------- +# ProblemReadingState (outer, problem-scoped) — ADR-0164.3 §Decision +# Field order matches ADR-0164.3 §ProblemReadingState table exactly. +# All fields required (no defaults) — initial construction is explicit. +# --------------------------------------------------------------------------- + +@dataclass(frozen=True, slots=True) +class ProblemReadingState: + entity_registry: tuple[EntityRef, ...] + accumulated_initial_state: tuple[PartialInitialPossession, ...] + accumulated_operations: tuple[PartialOperation, ...] + unknown_target_slot: QuestionTargetSlot | None + pronoun_resolution_history: tuple[PronounResolution, ...] + sentence_index: int + source_text_offset: int + + def __post_init__(self) -> None: + if not isinstance(self.entity_registry, tuple): + raise ComprehensionStateError( + "ProblemReadingState.entity_registry must be tuple[EntityRef, ...]" + ) + for idx, e in enumerate(self.entity_registry): + if not isinstance(e, EntityRef): + raise ComprehensionStateError( + f"ProblemReadingState.entity_registry[{idx}] must be EntityRef; " + f"got {type(e).__name__}" + ) + if not isinstance(self.accumulated_initial_state, tuple): + raise ComprehensionStateError( + "ProblemReadingState.accumulated_initial_state must be " + "tuple[PartialInitialPossession, ...]" + ) + for idx, pip in enumerate(self.accumulated_initial_state): + if not isinstance(pip, PartialInitialPossession): + raise ComprehensionStateError( + f"ProblemReadingState.accumulated_initial_state[{idx}] must be " + f"PartialInitialPossession; got {type(pip).__name__}" + ) + if not isinstance(self.accumulated_operations, tuple): + raise ComprehensionStateError( + "ProblemReadingState.accumulated_operations must be " + "tuple[PartialOperation, ...]" + ) + for idx, po in enumerate(self.accumulated_operations): + if not isinstance(po, PartialOperation): + raise ComprehensionStateError( + f"ProblemReadingState.accumulated_operations[{idx}] must be " + f"PartialOperation; got {type(po).__name__}" + ) + if self.unknown_target_slot is not None and not isinstance( + self.unknown_target_slot, QuestionTargetSlot + ): + raise ComprehensionStateError( + "ProblemReadingState.unknown_target_slot must be " + f"QuestionTargetSlot | None; got {type(self.unknown_target_slot).__name__}" + ) + if not isinstance(self.pronoun_resolution_history, tuple): + raise ComprehensionStateError( + "ProblemReadingState.pronoun_resolution_history must be " + "tuple[PronounResolution, ...]" + ) + for idx, pr in enumerate(self.pronoun_resolution_history): + if not isinstance(pr, PronounResolution): + raise ComprehensionStateError( + f"ProblemReadingState.pronoun_resolution_history[{idx}] must be " + f"PronounResolution; got {type(pr).__name__}" + ) + _require_non_negative_int( + self.sentence_index, "ProblemReadingState.sentence_index" + ) + _require_non_negative_int( + self.source_text_offset, "ProblemReadingState.source_text_offset" + ) + + def canonical_bytes(self) -> bytes: + return to_canonical_bytes(self) + + def canonical_hash(self) -> str: + return hashlib.sha256(self.canonical_bytes()).hexdigest() + + +# --------------------------------------------------------------------------- +# Canonical-bytes serialisation — ADR-0164.3 §Canonical-bytes +# Handles ProblemReadingState, SentenceReadingState, and ReaderRefusal. +# Rules: sort keys, compact separators, tuple→list, Decimal→str, +# None→OMITTED (not null), dataclass→sorted-key dict. +# --------------------------------------------------------------------------- + +def _canonical_dict_omit_none(obj: Any) -> Any: + """Recursively convert to a canonical JSON-serialisable value. + + None values are returned as the sentinel _OMIT; callers drop them + from dict outputs. This matches ADR-0164.3 §Canonical-bytes rule 7. + """ + if obj is None: + return _OMIT + if isinstance(obj, bool): + return obj + if isinstance(obj, int): + return obj + if isinstance(obj, Decimal): + return _canonical_decimal(obj) + if isinstance(obj, str): + return obj + if isinstance(obj, (tuple, list)): + return [_canonical_dict_omit_none(item) for item in obj] + if hasattr(obj, "__dataclass_fields__"): + out: dict[str, Any] = {} + for key in sorted(obj.__dataclass_fields__.keys()): + val = _canonical_dict_omit_none(getattr(obj, key)) + if val is not _OMIT: + out[key] = val + return out + raise ComprehensionStateError( + f"to_canonical_bytes: cannot serialise {type(obj).__name__}" + ) + + +class _OmitSentinel: + """Sentinel returned by _canonical_dict_omit_none for None values.""" + __slots__ = () + + +_OMIT = _OmitSentinel() + + +def to_canonical_bytes( + state: ProblemReadingState | SentenceReadingState | ReaderRefusal, +) -> bytes: + """Sorted-keys, compact-separators JSON per ADR-0164.3 §Canonical-bytes. + + Optional fields whose value is None are OMITTED from the output + (not serialised as ``null``). Tuples become JSON arrays. Decimal + values are serialised as strings to preserve precision. + + Identical state → byte-identical output (determinism gate). + """ + d = _canonical_dict_omit_none(state) + return json.dumps( + d, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +# --------------------------------------------------------------------------- +# Backward-compatibility alias +# --------------------------------------------------------------------------- + +#: Alias for code that imported ComprehensionState from #321. +#: ``SentenceReadingState`` is the canonical name per ADR-0164.3. +ComprehensionState = SentenceReadingState diff --git a/tests/test_comprehension_state.py b/tests/test_comprehension_state.py index 6275301f..5b9ee620 100644 --- a/tests/test_comprehension_state.py +++ b/tests/test_comprehension_state.py @@ -5,18 +5,78 @@ from decimal import Decimal import pytest from generate.comprehension.state import ( - ComprehensionState, - ComprehensionStateError, + # inner level (renamed from ComprehensionState) + SentenceReadingState, + # outer level (new) + ProblemReadingState, + # shared leaf types EntityRef, ExpectationFrame, PartialOp, QuantityRef, QuestionTargetSlot, + # new inner-level leaf types + AppliedCategory, + FramePayload, + VerbReference, + # new outer-level leaf types + PartialInitialPossession, + PartialOperation, + PronounResolution, + # refusal + ReaderRefusal, + READER_REFUSAL_REASONS, + # error + ComprehensionStateError, + # canonical-bytes function + to_canonical_bytes, + # backward-compat alias (must still resolve) + ComprehensionState, ) +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_entity(name: str = "tina", gender: str = "female", pos: int = 0) -> EntityRef: + return EntityRef(canonical_name=name, gender=gender, first_mention_position=pos) + + +def _make_qty( + val: str = "12.00", + unit: str = "dollars", + unit_class: str = "currency", + pos: int = 1, +) -> QuantityRef: + return QuantityRef( + value=Decimal(val), + unit=unit, + unit_class=unit_class, + owner_entity=None, + mention_position=pos, + ) + + + +def _make_empty_problem_state() -> ProblemReadingState: + return ProblemReadingState( + entity_registry=(), + accumulated_initial_state=(), + accumulated_operations=(), + unknown_target_slot=None, + pronoun_resolution_history=(), + sentence_index=0, + source_text_offset=0, + ) + + +# =========================================================================== +# EXISTING TESTS — now exercising SentenceReadingState (pure rename from #321) +# =========================================================================== + def test_canonical_bytes_are_deterministic_for_equal_states() -> None: - first = ComprehensionState( + first = SentenceReadingState( entities=( EntityRef( canonical_name="tina", @@ -53,7 +113,7 @@ def test_canonical_bytes_are_deterministic_for_equal_states() -> None: reason="question frame remains open", ), ) - second = ComprehensionState( + second = SentenceReadingState( entities=( EntityRef( canonical_name="tina", @@ -100,7 +160,7 @@ def test_canonical_bytes_are_deterministic_for_equal_states() -> None: def test_optional_fields_serialize_as_null() -> None: - state = ComprehensionState( + state = SentenceReadingState( entities=(), quantities=(), operations=(), @@ -160,14 +220,449 @@ def test_expectation_frame_refuses_empty_or_unknown_categories() -> None: def test_comprehension_state_refuses_non_tuple_and_wrong_member_types() -> None: with pytest.raises(ComprehensionStateError, match="entities must be tuple"): - ComprehensionState( + SentenceReadingState( entities=[], # type: ignore[arg-type] quantities=(), operations=(), ) with pytest.raises(ComprehensionStateError, match="quantities\\[0\\]"): - ComprehensionState( + SentenceReadingState( entities=(), quantities=("bad",), # type: ignore[arg-type] operations=(), ) + + +# =========================================================================== +# NEW TESTS — backward-compat alias +# =========================================================================== + +def test_comprehension_state_alias_resolves_to_sentence_reading_state() -> None: + """ComprehensionState must still resolve so callers from #321 need not change.""" + assert ComprehensionState is SentenceReadingState + + +# =========================================================================== +# NEW TESTS — SentenceReadingState new fields +# =========================================================================== + +def test_sentence_reading_state_new_fields_default_to_safe_values() -> None: + state = SentenceReadingState(entities=(), quantities=(), operations=()) + assert state.frame is None + assert state.pending_quantities == () + assert state.pending_entity_ref is None + assert state.pending_verb is None + assert state.token_index == 0 + assert state.lookback == () + assert state.partial_frame_payload is None + + +def test_sentence_reading_state_accepts_new_fields() -> None: + verb_ref = VerbReference(surface="makes", kind="rate_emit", position=1) + applied = AppliedCategory(category="accumulation_verb", position=1) + payload = FramePayload(frame_kind="operation_frame") + entity = _make_entity() + qty = _make_qty() + + state = SentenceReadingState( + entities=(entity,), + quantities=(qty,), + operations=(), + frame="operation_frame", + pending_quantities=(qty,), + pending_entity_ref=entity, + pending_verb=verb_ref, + token_index=3, + lookback=(applied,), + partial_frame_payload=payload, + ) + + assert state.frame == "operation_frame" + assert state.pending_verb is verb_ref + assert state.token_index == 3 + assert len(state.lookback) == 1 + assert state.partial_frame_payload is payload + + +def test_sentence_reading_state_refuses_invalid_frame() -> None: + with pytest.raises(ComprehensionStateError, match="frame"): + SentenceReadingState( + entities=(), quantities=(), operations=(), + frame="nonsense_frame", # type: ignore[arg-type] + ) + + +def test_sentence_reading_state_refuses_lookback_overflow() -> None: + cats = tuple(AppliedCategory(category="accumulation_verb", position=i) for i in range(9)) + with pytest.raises(ComprehensionStateError, match="lookback.*≤8"): + SentenceReadingState(entities=(), quantities=(), operations=(), lookback=cats) + + +def test_verb_reference_construction_and_validation() -> None: + vr = VerbReference(surface="earns", kind="accumulation_verb", position=2) + assert vr.surface == "earns" + assert vr.kind == "accumulation_verb" + assert vr.as_canonical() == {"kind": "accumulation_verb", "position": 2, "surface": "earns"} + + with pytest.raises(ComprehensionStateError): + VerbReference(surface="", kind="x", position=0) + with pytest.raises(ComprehensionStateError): + VerbReference(surface="x", kind="x", position=-1) + + +def test_applied_category_construction() -> None: + ac = AppliedCategory(category="depletion_verb", position=5) + assert ac.category == "depletion_verb" + assert ac.as_canonical() == {"category": "depletion_verb", "position": 5} + + +def test_frame_payload_construction_and_validation() -> None: + for kind in ("initial_state_frame", "operation_frame", "question_frame", "descriptive_frame"): + fp = FramePayload(frame_kind=kind) + assert fp.frame_kind == kind + + with pytest.raises(ComprehensionStateError, match="frame_kind"): + FramePayload(frame_kind="bad_frame") + + +# =========================================================================== +# NEW TESTS — PartialInitialPossession and PartialOperation +# =========================================================================== + +def test_partial_initial_possession_fully_nullable() -> None: + pip = PartialInitialPossession(entity=None, quantity=None) + assert pip.entity is None + assert pip.quantity is None + assert pip.as_canonical() == {} + + +def test_partial_initial_possession_with_values() -> None: + qty = _make_qty() + pip = PartialInitialPossession(entity="tina", quantity=qty) + canon = pip.as_canonical() + assert canon["entity"] == "tina" + assert "quantity" in canon + + +def test_partial_initial_possession_refuses_empty_entity() -> None: + with pytest.raises(ComprehensionStateError): + PartialInitialPossession(entity="", quantity=None) + + +def test_partial_operation_fully_nullable() -> None: + po = PartialOperation(actor=None, kind=None, operand=None, target=None) + assert po.as_canonical() == {} + + +def test_partial_operation_with_values() -> None: + qty = _make_qty() + po = PartialOperation(actor="tina", kind="accumulation", operand=qty, target=None) + canon = po.as_canonical() + assert canon["actor"] == "tina" + assert canon["kind"] == "accumulation" + assert "operand" in canon + assert "target" not in canon # None → omitted + + +def test_pronoun_resolution_construction() -> None: + pr = PronounResolution(pronoun="she", resolved_to="Tina", at_sentence=1, at_position=0) + assert pr.resolved_to == "Tina" + canon = pr.as_canonical() + assert canon == { + "at_position": 0, + "at_sentence": 1, + "pronoun": "she", + "resolved_to": "Tina", + } + + +def test_pronoun_resolution_refuses_empty_fields() -> None: + with pytest.raises(ComprehensionStateError, match="PronounResolution.pronoun"): + PronounResolution(pronoun="", resolved_to="Tina", at_sentence=0, at_position=0) + with pytest.raises(ComprehensionStateError, match="PronounResolution.resolved_to"): + PronounResolution(pronoun="she", resolved_to="", at_sentence=0, at_position=0) + + +# =========================================================================== +# NEW TESTS — ProblemReadingState construction +# =========================================================================== + +def test_problem_reading_state_empty_construction() -> None: + ps = _make_empty_problem_state() + assert ps.entity_registry == () + assert ps.accumulated_initial_state == () + assert ps.accumulated_operations == () + assert ps.unknown_target_slot is None + assert ps.pronoun_resolution_history == () + assert ps.sentence_index == 0 + assert ps.source_text_offset == 0 + + +def test_problem_reading_state_with_entities_and_operations() -> None: + entity = _make_entity() + qty = _make_qty() + pip = PartialInitialPossession(entity="tina", quantity=qty) + po = PartialOperation(actor="tina", kind="accumulation", operand=qty, target=None) + pr = PronounResolution(pronoun="she", resolved_to="tina", at_sentence=1, at_position=0) + qt = QuestionTargetSlot(kind="continuous_quantity", entity="tina", unit_class="currency", position=0) + + ps = ProblemReadingState( + entity_registry=(entity,), + accumulated_initial_state=(pip,), + accumulated_operations=(po,), + unknown_target_slot=qt, + pronoun_resolution_history=(pr,), + sentence_index=2, + source_text_offset=47, + ) + + assert len(ps.entity_registry) == 1 + assert ps.sentence_index == 2 + assert ps.source_text_offset == 47 + assert ps.unknown_target_slot is qt + + +def test_problem_reading_state_refuses_wrong_member_types() -> None: + with pytest.raises(ComprehensionStateError, match="entity_registry"): + ProblemReadingState( + entity_registry="bad", # type: ignore[arg-type] + accumulated_initial_state=(), + accumulated_operations=(), + unknown_target_slot=None, + pronoun_resolution_history=(), + sentence_index=0, + source_text_offset=0, + ) + with pytest.raises(ComprehensionStateError, match="entity_registry\\[0\\]"): + ProblemReadingState( + entity_registry=("bad",), # type: ignore[arg-type] + accumulated_initial_state=(), + accumulated_operations=(), + unknown_target_slot=None, + pronoun_resolution_history=(), + sentence_index=0, + source_text_offset=0, + ) + with pytest.raises(ComprehensionStateError, match="sentence_index"): + ProblemReadingState( + entity_registry=(), + accumulated_initial_state=(), + accumulated_operations=(), + unknown_target_slot=None, + pronoun_resolution_history=(), + sentence_index=-1, + source_text_offset=0, + ) + + +# =========================================================================== +# NEW TESTS — ProblemReadingState canonical-bytes determinism gate +# =========================================================================== + +def test_problem_reading_state_canonical_bytes_determinism() -> None: + """Two equal ProblemReadingState instances must produce byte-equal output.""" + entity = _make_entity() + pip = PartialInitialPossession(entity="tina", quantity=None) + + ps1 = ProblemReadingState( + entity_registry=(entity,), + accumulated_initial_state=(pip,), + accumulated_operations=(), + unknown_target_slot=None, + pronoun_resolution_history=(), + sentence_index=1, + source_text_offset=33, + ) + ps2 = ProblemReadingState( + entity_registry=(entity,), + accumulated_initial_state=(pip,), + accumulated_operations=(), + unknown_target_slot=None, + pronoun_resolution_history=(), + sentence_index=1, + source_text_offset=33, + ) + + assert ps1.canonical_bytes() == ps2.canonical_bytes() + assert ps1.canonical_hash() == ps2.canonical_hash() + assert to_canonical_bytes(ps1) == to_canonical_bytes(ps2) + + +def test_problem_reading_state_canonical_bytes_sensitivity() -> None: + """Single-field differences must produce different canonical bytes.""" + base = _make_empty_problem_state() + + # sentence_index differs + incremented = ProblemReadingState( + entity_registry=(), + accumulated_initial_state=(), + accumulated_operations=(), + unknown_target_slot=None, + pronoun_resolution_history=(), + sentence_index=1, + source_text_offset=0, + ) + assert base.canonical_bytes() != incremented.canonical_bytes() + + # source_text_offset differs + offset = ProblemReadingState( + entity_registry=(), + accumulated_initial_state=(), + accumulated_operations=(), + unknown_target_slot=None, + pronoun_resolution_history=(), + sentence_index=0, + source_text_offset=10, + ) + assert base.canonical_bytes() != offset.canonical_bytes() + + # entity_registry populated + with_entity = ProblemReadingState( + entity_registry=(_make_entity(),), + accumulated_initial_state=(), + accumulated_operations=(), + unknown_target_slot=None, + pronoun_resolution_history=(), + sentence_index=0, + source_text_offset=0, + ) + assert base.canonical_bytes() != with_entity.canonical_bytes() + + +def test_problem_reading_state_none_fields_omitted_from_canonical() -> None: + """Per ADR-0164.3 §Canonical-bytes: None → omitted, not serialised as null.""" + ps = _make_empty_problem_state() + raw = to_canonical_bytes(ps) + assert b"null" not in raw, ( + "None fields must be OMITTED from canonical bytes, not serialised as null" + ) + + +# =========================================================================== +# NEW TESTS — to_canonical_bytes handles SentenceReadingState too +# =========================================================================== + +def test_to_canonical_bytes_sentence_state_none_omitted() -> None: + state = SentenceReadingState(entities=(), quantities=(), operations=()) + raw = to_canonical_bytes(state) + assert b"null" not in raw + # token_index=0 is a non-None int — must be present + assert b'"token_index"' in raw + + +def test_to_canonical_bytes_sentence_state_determinism() -> None: + s1 = SentenceReadingState( + entities=(_make_entity(),), + quantities=(), + operations=(), + token_index=2, + ) + s2 = SentenceReadingState( + entities=(_make_entity(),), + quantities=(), + operations=(), + token_index=2, + ) + assert to_canonical_bytes(s1) == to_canonical_bytes(s2) + + +def test_to_canonical_bytes_sentence_state_sensitivity() -> None: + base = SentenceReadingState(entities=(), quantities=(), operations=(), token_index=0) + diff = SentenceReadingState(entities=(), quantities=(), operations=(), token_index=1) + assert to_canonical_bytes(base) != to_canonical_bytes(diff) + + +# =========================================================================== +# NEW TESTS — ReaderRefusal construction and canonical bytes +# =========================================================================== + +def test_reader_refusal_reasons_has_11_entries() -> None: + assert len(READER_REFUSAL_REASONS) == 11 + + +@pytest.mark.parametrize("reason", sorted(READER_REFUSAL_REASONS)) +def test_every_refusal_reason_is_constructible(reason: str) -> None: + rf = ReaderRefusal( + reason=reason, + detail=f"test refusal for {reason}", + sentence_index=0, + token_index=0, + token_text="", + ) + assert rf.reason == reason + + +def test_reader_refusal_refuses_unknown_reason() -> None: + with pytest.raises(ComprehensionStateError, match="READER_REFUSAL_REASONS"): + ReaderRefusal( + reason="made_up_reason", + detail="detail", + sentence_index=0, + token_index=0, + token_text="", + ) + + +def test_reader_refusal_refuses_empty_detail() -> None: + with pytest.raises(ComprehensionStateError, match="ReaderRefusal.detail"): + ReaderRefusal( + reason="unknown_word", + detail="", + sentence_index=0, + token_index=0, + token_text="if", + ) + + +def test_reader_refusal_canonical_bytes_round_trip() -> None: + rf = ReaderRefusal( + reason="unexpected_category", + detail="conditional_open at sentence_index=1, position=0", + sentence_index=1, + token_index=0, + token_text="If", + ) + b1 = rf.canonical_bytes() + b2 = to_canonical_bytes(rf) + assert b1 == b2 + + # same values → byte-equal + rf2 = ReaderRefusal( + reason="unexpected_category", + detail="conditional_open at sentence_index=1, position=0", + sentence_index=1, + token_index=0, + token_text="If", + ) + assert rf.canonical_bytes() == rf2.canonical_bytes() + assert rf.canonical_hash() == rf2.canonical_hash() + + +def test_reader_refusal_empty_token_text_allowed() -> None: + """Sentence-level and problem-level refusals have no single token.""" + rf = ReaderRefusal( + reason="unfinished_frame", + detail="frame never decided", + sentence_index=2, + token_index=0, + token_text="", + ) + assert rf.token_text == "" + + +def test_reader_refusal_canonical_bytes_sensitivity() -> None: + rf1 = ReaderRefusal( + reason="unknown_word", + detail="detail", + sentence_index=0, + token_index=0, + token_text="foo", + ) + rf2 = ReaderRefusal( + reason="unknown_word", + detail="detail", + sentence_index=0, + token_index=1, + token_text="foo", + ) + assert rf1.canonical_bytes() != rf2.canonical_bytes()