feat(comprehension): split ComprehensionState into ProblemReadingState + SentenceReadingState (ADR-0164.3) (#323)
Reconciles the #321 skeleton with ADR-0164.3's two-level state model. Changes: - Renames ComprehensionState → SentenceReadingState (backward-compat alias kept; existing callers need not change) - Adds 7 new fields to SentenceReadingState (all defaulted so existing construction still compiles): frame, pending_quantities, pending_entity_ref, pending_verb, token_index, lookback (≤8 entries, validated), partial_frame_payload - Introduces SentenceFrame (Literal), VerbReference, AppliedCategory, FramePayload (stub, frame_kind validated) - Adds ProblemReadingState (outer, problem-scoped) with all 7 fields per ADR-0164.3 table order, no defaults (explicit construction required) - Introduces PartialInitialPossession and PartialOperation (nullable precursors to ADR-0115 types), PronounResolution - Adds READER_REFUSAL_REASONS (11-member frozenset, closed/ADR-tracked) and ReaderRefusal dataclass with reason validation - Adds to_canonical_bytes() standalone function implementing ADR-0164.3 §Canonical-bytes rules: sort keys, omit None, Decimal→str; handles ProblemReadingState, SentenceReadingState, ReaderRefusal - SentenceReadingState.canonical_bytes() kept backward-compatible (original 5 fields, null for None) — existing pinned-bytes tests pass - 47 tests: all original tests pass; new tests cover ProblemReadingState construction, determinism gate, sensitivity gate, ReaderRefusal construction and every READER_REFUSAL_REASONS entry Refs: #320 (ADR-0164.3), #321 (comprehension-state-skeleton)
This commit is contained in:
Shay
GitHub
parent
48ea34bd52
commit
957e7c6642
2 changed files with 1023 additions and 22 deletions
|
|
@ -1,9 +1,18 @@
|
|||
"""ADR-0164 — immutable partial-comprehension state skeleton.
|
||||
"""ADR-0164 / ADR-0164.3 — two-level immutable comprehension-state types.
|
||||
|
||||
This module defines the typed state container the incremental
|
||||
comprehension reader will accumulate. It is intentionally pure data:
|
||||
frozen dataclasses, refusal-first validation, and canonical-bytes
|
||||
serialization for deterministic replay and trace hashing.
|
||||
This module defines the typed state containers the incremental comprehension
|
||||
reader accumulates. It is intentionally pure data: frozen dataclasses,
|
||||
refusal-first validation, and canonical-bytes serialisation for deterministic
|
||||
replay and trace hashing.
|
||||
|
||||
Two levels (ADR-0164.3 §Decision):
|
||||
- ``ProblemReadingState`` — outer, problem-scoped. Persists across sentence
|
||||
boundaries. Mutated only by ``end_sentence``.
|
||||
- ``SentenceReadingState`` — inner, sentence-scoped. Lifetime = one sentence.
|
||||
Created by ``begin_sentence``, mutated by ``apply_word``.
|
||||
|
||||
``ComprehensionState`` is a backward-compatibility alias for
|
||||
``SentenceReadingState``; existing importers need not change.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -14,6 +23,10 @@ from dataclasses import dataclass
|
|||
from decimal import Decimal
|
||||
from typing import Any, Final, Literal
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Closed-set constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
VALID_GENDERS: Final[frozenset[str]] = frozenset(
|
||||
{"female", "male", "neuter", "unknown"}
|
||||
)
|
||||
|
|
@ -36,11 +49,59 @@ VALID_EXPECTATION_KINDS: Final[frozenset[str]] = frozenset(
|
|||
}
|
||||
)
|
||||
|
||||
VALID_SENTENCE_FRAME_KINDS: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
"initial_state_frame",
|
||||
"operation_frame",
|
||||
"question_frame",
|
||||
"descriptive_frame",
|
||||
}
|
||||
)
|
||||
|
||||
# ADR-0164.3 §ReaderRefusal — closed, ADR-tracked.
|
||||
# New reasons require an ADR amendment.
|
||||
READER_REFUSAL_REASONS: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
# apply_word — token-level
|
||||
"unknown_word",
|
||||
"unexpected_category",
|
||||
"expectation_collision",
|
||||
"unresolved_pronoun",
|
||||
"ambiguous_pronoun_referent",
|
||||
# end_sentence — sentence-level
|
||||
"unfinished_frame",
|
||||
"unattached_quantity",
|
||||
"incomplete_operation",
|
||||
# finalization predicate — problem-level
|
||||
"no_question_target",
|
||||
"dangling_entity",
|
||||
"graph_construction_failure",
|
||||
}
|
||||
)
|
||||
|
||||
# SentenceFrame is a Literal over the four discriminator values.
|
||||
SentenceFrame = Literal[
|
||||
"initial_state_frame",
|
||||
"operation_frame",
|
||||
"question_frame",
|
||||
"descriptive_frame",
|
||||
]
|
||||
|
||||
_LOOKBACK_MAX: Final[int] = 8
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Error
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ComprehensionStateError(ValueError):
|
||||
"""Raised on invalid comprehension-state construction."""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal validators (unchanged from #321)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _require_non_empty_str(value: object, field_name: str) -> None:
|
||||
if not isinstance(value, str) or value == "":
|
||||
raise ComprehensionStateError(
|
||||
|
|
@ -64,7 +125,7 @@ def _require_int(value: object, field_name: str) -> None:
|
|||
|
||||
def _require_non_negative_int(value: object, field_name: str) -> None:
|
||||
_require_int(value, field_name)
|
||||
if value < 0:
|
||||
if value < 0: # type: ignore[operator]
|
||||
raise ComprehensionStateError(f"{field_name} must be >= 0; got {value}")
|
||||
|
||||
|
||||
|
|
@ -86,6 +147,10 @@ def _canonical_decimal(value: Decimal) -> str:
|
|||
return format(normalized, "f")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared leaf types (unchanged from #321)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class EntityRef:
|
||||
canonical_name: str
|
||||
|
|
@ -229,60 +294,347 @@ class ExpectationFrame:
|
|||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# New leaf types for SentenceReadingState (ADR-0164.3)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ComprehensionState:
|
||||
class VerbReference:
|
||||
"""The verb captured at frame-determining position, awaiting completion."""
|
||||
|
||||
surface: str
|
||||
kind: str
|
||||
position: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_require_non_empty_str(self.surface, "VerbReference.surface")
|
||||
_require_non_empty_str(self.kind, "VerbReference.kind")
|
||||
_require_non_negative_int(self.position, "VerbReference.position")
|
||||
|
||||
def as_canonical(self) -> dict[str, Any]:
|
||||
return {
|
||||
"kind": self.kind,
|
||||
"position": self.position,
|
||||
"surface": self.surface,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AppliedCategory:
|
||||
"""One entry in the lookback window: a category applied at a position."""
|
||||
|
||||
category: str
|
||||
position: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_require_non_empty_str(self.category, "AppliedCategory.category")
|
||||
_require_non_negative_int(self.position, "AppliedCategory.position")
|
||||
|
||||
def as_canonical(self) -> dict[str, Any]:
|
||||
return {"category": self.category, "position": self.position}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FramePayload:
|
||||
"""Stub container for the in-construction frame payload.
|
||||
|
||||
The reader (Brief 5 Phase 1) populates sub-fields specific to each
|
||||
frame kind. This stub carries only the frame_kind discriminator so
|
||||
the two-level state model can be typed and tested without coupling
|
||||
to the reader implementation.
|
||||
"""
|
||||
|
||||
frame_kind: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.frame_kind not in VALID_SENTENCE_FRAME_KINDS:
|
||||
raise ComprehensionStateError(
|
||||
"FramePayload.frame_kind must be one of "
|
||||
f"{sorted(VALID_SENTENCE_FRAME_KINDS)}; got {self.frame_kind!r}"
|
||||
)
|
||||
|
||||
def as_canonical(self) -> dict[str, Any]:
|
||||
return {"frame_kind": self.frame_kind}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# New leaf types for ProblemReadingState (ADR-0164.3)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PartialInitialPossession:
|
||||
"""Precursor to ADR-0115 InitialPossession during reader construction.
|
||||
|
||||
Every field is nullable: the reader builds this incrementally as
|
||||
tokens arrive. A fully-specified instance (no None fields) projects
|
||||
to a strict ``InitialPossession`` at ``end_sentence``.
|
||||
"""
|
||||
|
||||
entity: str | None
|
||||
quantity: QuantityRef | None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.entity is not None:
|
||||
_require_non_empty_str(self.entity, "PartialInitialPossession.entity")
|
||||
if self.quantity is not None and not isinstance(self.quantity, QuantityRef):
|
||||
raise ComprehensionStateError(
|
||||
"PartialInitialPossession.quantity must be QuantityRef | None; "
|
||||
f"got {type(self.quantity).__name__}"
|
||||
)
|
||||
|
||||
def as_canonical(self) -> dict[str, Any]:
|
||||
d: dict[str, Any] = {}
|
||||
if self.entity is not None:
|
||||
d["entity"] = self.entity
|
||||
if self.quantity is not None:
|
||||
d["quantity"] = self.quantity.as_canonical()
|
||||
return d
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PartialOperation:
|
||||
"""Precursor to ADR-0115 Operation during reader construction.
|
||||
|
||||
Every field is nullable: the reader builds this incrementally as
|
||||
tokens arrive. A fully-specified instance projects to a strict
|
||||
``Operation`` at ``end_sentence``.
|
||||
"""
|
||||
|
||||
actor: str | None
|
||||
kind: str | None
|
||||
operand: QuantityRef | None
|
||||
target: str | None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.actor is not None:
|
||||
_require_non_empty_str(self.actor, "PartialOperation.actor")
|
||||
if self.kind is not None:
|
||||
_require_non_empty_str(self.kind, "PartialOperation.kind")
|
||||
if self.target is not None:
|
||||
_require_non_empty_str(self.target, "PartialOperation.target")
|
||||
if self.operand is not None and not isinstance(self.operand, QuantityRef):
|
||||
raise ComprehensionStateError(
|
||||
"PartialOperation.operand must be QuantityRef | None; "
|
||||
f"got {type(self.operand).__name__}"
|
||||
)
|
||||
|
||||
def as_canonical(self) -> dict[str, Any]:
|
||||
d: dict[str, Any] = {}
|
||||
if self.actor is not None:
|
||||
d["actor"] = self.actor
|
||||
if self.kind is not None:
|
||||
d["kind"] = self.kind
|
||||
if self.operand is not None:
|
||||
d["operand"] = self.operand.as_canonical()
|
||||
if self.target is not None:
|
||||
d["target"] = self.target
|
||||
return d
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PronounResolution:
|
||||
"""Replay-deterministic record of one pronoun resolution event.
|
||||
|
||||
Per ADR-0164.2. Appended to ``ProblemReadingState.pronoun_resolution_history``
|
||||
only when the containing sentence closes successfully.
|
||||
"""
|
||||
|
||||
pronoun: str
|
||||
resolved_to: str
|
||||
at_sentence: int
|
||||
at_position: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_require_non_empty_str(self.pronoun, "PronounResolution.pronoun")
|
||||
_require_non_empty_str(self.resolved_to, "PronounResolution.resolved_to")
|
||||
_require_non_negative_int(self.at_sentence, "PronounResolution.at_sentence")
|
||||
_require_non_negative_int(self.at_position, "PronounResolution.at_position")
|
||||
|
||||
def as_canonical(self) -> dict[str, Any]:
|
||||
return {
|
||||
"at_position": self.at_position,
|
||||
"at_sentence": self.at_sentence,
|
||||
"pronoun": self.pronoun,
|
||||
"resolved_to": self.resolved_to,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ReaderRefusal (ADR-0164.3 §ReaderRefusal)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReaderRefusal:
|
||||
"""Typed refusal record. Carries one of the closed READER_REFUSAL_REASONS.
|
||||
|
||||
``token_text`` may be empty string for sentence-level or problem-level
|
||||
refusals where no single token is in question.
|
||||
"""
|
||||
|
||||
reason: str
|
||||
detail: str
|
||||
sentence_index: int
|
||||
token_index: int
|
||||
token_text: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.reason not in READER_REFUSAL_REASONS:
|
||||
raise ComprehensionStateError(
|
||||
"ReaderRefusal.reason must be a member of READER_REFUSAL_REASONS; "
|
||||
f"got {self.reason!r}"
|
||||
)
|
||||
_require_non_empty_str(self.detail, "ReaderRefusal.detail")
|
||||
_require_non_negative_int(self.sentence_index, "ReaderRefusal.sentence_index")
|
||||
_require_non_negative_int(self.token_index, "ReaderRefusal.token_index")
|
||||
if not isinstance(self.token_text, str):
|
||||
raise ComprehensionStateError(
|
||||
"ReaderRefusal.token_text must be str; "
|
||||
f"got {type(self.token_text).__name__}"
|
||||
)
|
||||
|
||||
def as_canonical(self) -> dict[str, Any]:
|
||||
return {
|
||||
"detail": self.detail,
|
||||
"reason": self.reason,
|
||||
"sentence_index": self.sentence_index,
|
||||
"token_index": self.token_index,
|
||||
"token_text": self.token_text,
|
||||
}
|
||||
|
||||
def canonical_bytes(self) -> bytes:
|
||||
return to_canonical_bytes(self)
|
||||
|
||||
def canonical_hash(self) -> str:
|
||||
return hashlib.sha256(self.canonical_bytes()).hexdigest()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SentenceReadingState (inner, sentence-scoped) — ADR-0164.3 §Decision
|
||||
# Renamed from ComprehensionState (#321). Original five fields stay verbatim.
|
||||
# Seven new fields added (all with defaults) per ADR-0164.3 §SentenceReadingState.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SentenceReadingState:
|
||||
# --- original five fields (verbatim from #321) ---
|
||||
entities: tuple[EntityRef, ...]
|
||||
quantities: tuple[QuantityRef, ...]
|
||||
operations: tuple[PartialOp, ...]
|
||||
question_target: QuestionTargetSlot | None = None
|
||||
expectation: ExpectationFrame | None = None
|
||||
|
||||
# --- ADR-0164.3 §SentenceReadingState new fields ---
|
||||
frame: SentenceFrame | None = None
|
||||
pending_quantities: tuple[QuantityRef, ...] = ()
|
||||
pending_entity_ref: EntityRef | None = None
|
||||
pending_verb: VerbReference | None = None
|
||||
token_index: int = 0
|
||||
lookback: tuple[AppliedCategory, ...] = ()
|
||||
partial_frame_payload: FramePayload | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
# --- validate original fields ---
|
||||
if not isinstance(self.entities, tuple):
|
||||
raise ComprehensionStateError(
|
||||
"ComprehensionState.entities must be tuple[EntityRef, ...]"
|
||||
"SentenceReadingState.entities must be tuple[EntityRef, ...]"
|
||||
)
|
||||
if not isinstance(self.quantities, tuple):
|
||||
raise ComprehensionStateError(
|
||||
"ComprehensionState.quantities must be tuple[QuantityRef, ...]"
|
||||
"SentenceReadingState.quantities must be tuple[QuantityRef, ...]"
|
||||
)
|
||||
if not isinstance(self.operations, tuple):
|
||||
raise ComprehensionStateError(
|
||||
"ComprehensionState.operations must be tuple[PartialOp, ...]"
|
||||
"SentenceReadingState.operations must be tuple[PartialOp, ...]"
|
||||
)
|
||||
for idx, entity in enumerate(self.entities):
|
||||
if not isinstance(entity, EntityRef):
|
||||
raise ComprehensionStateError(
|
||||
f"ComprehensionState.entities[{idx}] must be EntityRef; "
|
||||
f"SentenceReadingState.entities[{idx}] must be EntityRef; "
|
||||
f"got {type(entity).__name__}"
|
||||
)
|
||||
for idx, quantity in enumerate(self.quantities):
|
||||
if not isinstance(quantity, QuantityRef):
|
||||
raise ComprehensionStateError(
|
||||
f"ComprehensionState.quantities[{idx}] must be QuantityRef; "
|
||||
f"SentenceReadingState.quantities[{idx}] must be QuantityRef; "
|
||||
f"got {type(quantity).__name__}"
|
||||
)
|
||||
for idx, operation in enumerate(self.operations):
|
||||
if not isinstance(operation, PartialOp):
|
||||
raise ComprehensionStateError(
|
||||
f"ComprehensionState.operations[{idx}] must be PartialOp; "
|
||||
f"SentenceReadingState.operations[{idx}] must be PartialOp; "
|
||||
f"got {type(operation).__name__}"
|
||||
)
|
||||
if self.question_target is not None and not isinstance(
|
||||
self.question_target, QuestionTargetSlot
|
||||
):
|
||||
raise ComprehensionStateError(
|
||||
"ComprehensionState.question_target must be "
|
||||
"SentenceReadingState.question_target must be "
|
||||
f"QuestionTargetSlot | None; got {type(self.question_target).__name__}"
|
||||
)
|
||||
if self.expectation is not None and not isinstance(
|
||||
self.expectation, ExpectationFrame
|
||||
):
|
||||
raise ComprehensionStateError(
|
||||
"ComprehensionState.expectation must be "
|
||||
"SentenceReadingState.expectation must be "
|
||||
f"ExpectationFrame | None; got {type(self.expectation).__name__}"
|
||||
)
|
||||
|
||||
# --- validate new fields ---
|
||||
if self.frame is not None and self.frame not in VALID_SENTENCE_FRAME_KINDS:
|
||||
raise ComprehensionStateError(
|
||||
"SentenceReadingState.frame must be a SentenceFrame literal or None; "
|
||||
f"got {self.frame!r}"
|
||||
)
|
||||
if not isinstance(self.pending_quantities, tuple):
|
||||
raise ComprehensionStateError(
|
||||
"SentenceReadingState.pending_quantities must be tuple[QuantityRef, ...]"
|
||||
)
|
||||
for idx, pq in enumerate(self.pending_quantities):
|
||||
if not isinstance(pq, QuantityRef):
|
||||
raise ComprehensionStateError(
|
||||
f"SentenceReadingState.pending_quantities[{idx}] must be "
|
||||
f"QuantityRef; got {type(pq).__name__}"
|
||||
)
|
||||
if self.pending_entity_ref is not None and not isinstance(
|
||||
self.pending_entity_ref, EntityRef
|
||||
):
|
||||
raise ComprehensionStateError(
|
||||
"SentenceReadingState.pending_entity_ref must be EntityRef | None; "
|
||||
f"got {type(self.pending_entity_ref).__name__}"
|
||||
)
|
||||
if self.pending_verb is not None and not isinstance(
|
||||
self.pending_verb, VerbReference
|
||||
):
|
||||
raise ComprehensionStateError(
|
||||
"SentenceReadingState.pending_verb must be VerbReference | None; "
|
||||
f"got {type(self.pending_verb).__name__}"
|
||||
)
|
||||
_require_non_negative_int(self.token_index, "SentenceReadingState.token_index")
|
||||
if not isinstance(self.lookback, tuple):
|
||||
raise ComprehensionStateError(
|
||||
"SentenceReadingState.lookback must be tuple[AppliedCategory, ...]"
|
||||
)
|
||||
if len(self.lookback) > _LOOKBACK_MAX:
|
||||
raise ComprehensionStateError(
|
||||
f"SentenceReadingState.lookback must be ≤{_LOOKBACK_MAX} entries; "
|
||||
f"got {len(self.lookback)}"
|
||||
)
|
||||
for idx, ac in enumerate(self.lookback):
|
||||
if not isinstance(ac, AppliedCategory):
|
||||
raise ComprehensionStateError(
|
||||
f"SentenceReadingState.lookback[{idx}] must be AppliedCategory; "
|
||||
f"got {type(ac).__name__}"
|
||||
)
|
||||
if self.partial_frame_payload is not None and not isinstance(
|
||||
self.partial_frame_payload, FramePayload
|
||||
):
|
||||
raise ComprehensionStateError(
|
||||
"SentenceReadingState.partial_frame_payload must be "
|
||||
f"FramePayload | None; got {type(self.partial_frame_payload).__name__}"
|
||||
)
|
||||
|
||||
# --- backward-compatible serialisation (original 5 fields only, null for None) ---
|
||||
|
||||
def as_canonical(self) -> dict[str, Any]:
|
||||
return {
|
||||
"entities": [entity.as_canonical() for entity in self.entities],
|
||||
|
|
@ -315,3 +667,157 @@ class ComprehensionState:
|
|||
def canonical_hash(self) -> str:
|
||||
return hashlib.sha256(self.canonical_bytes()).hexdigest()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ProblemReadingState (outer, problem-scoped) — ADR-0164.3 §Decision
|
||||
# Field order matches ADR-0164.3 §ProblemReadingState table exactly.
|
||||
# All fields required (no defaults) — initial construction is explicit.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProblemReadingState:
|
||||
entity_registry: tuple[EntityRef, ...]
|
||||
accumulated_initial_state: tuple[PartialInitialPossession, ...]
|
||||
accumulated_operations: tuple[PartialOperation, ...]
|
||||
unknown_target_slot: QuestionTargetSlot | None
|
||||
pronoun_resolution_history: tuple[PronounResolution, ...]
|
||||
sentence_index: int
|
||||
source_text_offset: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not isinstance(self.entity_registry, tuple):
|
||||
raise ComprehensionStateError(
|
||||
"ProblemReadingState.entity_registry must be tuple[EntityRef, ...]"
|
||||
)
|
||||
for idx, e in enumerate(self.entity_registry):
|
||||
if not isinstance(e, EntityRef):
|
||||
raise ComprehensionStateError(
|
||||
f"ProblemReadingState.entity_registry[{idx}] must be EntityRef; "
|
||||
f"got {type(e).__name__}"
|
||||
)
|
||||
if not isinstance(self.accumulated_initial_state, tuple):
|
||||
raise ComprehensionStateError(
|
||||
"ProblemReadingState.accumulated_initial_state must be "
|
||||
"tuple[PartialInitialPossession, ...]"
|
||||
)
|
||||
for idx, pip in enumerate(self.accumulated_initial_state):
|
||||
if not isinstance(pip, PartialInitialPossession):
|
||||
raise ComprehensionStateError(
|
||||
f"ProblemReadingState.accumulated_initial_state[{idx}] must be "
|
||||
f"PartialInitialPossession; got {type(pip).__name__}"
|
||||
)
|
||||
if not isinstance(self.accumulated_operations, tuple):
|
||||
raise ComprehensionStateError(
|
||||
"ProblemReadingState.accumulated_operations must be "
|
||||
"tuple[PartialOperation, ...]"
|
||||
)
|
||||
for idx, po in enumerate(self.accumulated_operations):
|
||||
if not isinstance(po, PartialOperation):
|
||||
raise ComprehensionStateError(
|
||||
f"ProblemReadingState.accumulated_operations[{idx}] must be "
|
||||
f"PartialOperation; got {type(po).__name__}"
|
||||
)
|
||||
if self.unknown_target_slot is not None and not isinstance(
|
||||
self.unknown_target_slot, QuestionTargetSlot
|
||||
):
|
||||
raise ComprehensionStateError(
|
||||
"ProblemReadingState.unknown_target_slot must be "
|
||||
f"QuestionTargetSlot | None; got {type(self.unknown_target_slot).__name__}"
|
||||
)
|
||||
if not isinstance(self.pronoun_resolution_history, tuple):
|
||||
raise ComprehensionStateError(
|
||||
"ProblemReadingState.pronoun_resolution_history must be "
|
||||
"tuple[PronounResolution, ...]"
|
||||
)
|
||||
for idx, pr in enumerate(self.pronoun_resolution_history):
|
||||
if not isinstance(pr, PronounResolution):
|
||||
raise ComprehensionStateError(
|
||||
f"ProblemReadingState.pronoun_resolution_history[{idx}] must be "
|
||||
f"PronounResolution; got {type(pr).__name__}"
|
||||
)
|
||||
_require_non_negative_int(
|
||||
self.sentence_index, "ProblemReadingState.sentence_index"
|
||||
)
|
||||
_require_non_negative_int(
|
||||
self.source_text_offset, "ProblemReadingState.source_text_offset"
|
||||
)
|
||||
|
||||
def canonical_bytes(self) -> bytes:
|
||||
return to_canonical_bytes(self)
|
||||
|
||||
def canonical_hash(self) -> str:
|
||||
return hashlib.sha256(self.canonical_bytes()).hexdigest()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Canonical-bytes serialisation — ADR-0164.3 §Canonical-bytes
|
||||
# Handles ProblemReadingState, SentenceReadingState, and ReaderRefusal.
|
||||
# Rules: sort keys, compact separators, tuple→list, Decimal→str,
|
||||
# None→OMITTED (not null), dataclass→sorted-key dict.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _canonical_dict_omit_none(obj: Any) -> Any:
|
||||
"""Recursively convert to a canonical JSON-serialisable value.
|
||||
|
||||
None values are returned as the sentinel _OMIT; callers drop them
|
||||
from dict outputs. This matches ADR-0164.3 §Canonical-bytes rule 7.
|
||||
"""
|
||||
if obj is None:
|
||||
return _OMIT
|
||||
if isinstance(obj, bool):
|
||||
return obj
|
||||
if isinstance(obj, int):
|
||||
return obj
|
||||
if isinstance(obj, Decimal):
|
||||
return _canonical_decimal(obj)
|
||||
if isinstance(obj, str):
|
||||
return obj
|
||||
if isinstance(obj, (tuple, list)):
|
||||
return [_canonical_dict_omit_none(item) for item in obj]
|
||||
if hasattr(obj, "__dataclass_fields__"):
|
||||
out: dict[str, Any] = {}
|
||||
for key in sorted(obj.__dataclass_fields__.keys()):
|
||||
val = _canonical_dict_omit_none(getattr(obj, key))
|
||||
if val is not _OMIT:
|
||||
out[key] = val
|
||||
return out
|
||||
raise ComprehensionStateError(
|
||||
f"to_canonical_bytes: cannot serialise {type(obj).__name__}"
|
||||
)
|
||||
|
||||
|
||||
class _OmitSentinel:
|
||||
"""Sentinel returned by _canonical_dict_omit_none for None values."""
|
||||
__slots__ = ()
|
||||
|
||||
|
||||
_OMIT = _OmitSentinel()
|
||||
|
||||
|
||||
def to_canonical_bytes(
|
||||
state: ProblemReadingState | SentenceReadingState | ReaderRefusal,
|
||||
) -> bytes:
|
||||
"""Sorted-keys, compact-separators JSON per ADR-0164.3 §Canonical-bytes.
|
||||
|
||||
Optional fields whose value is None are OMITTED from the output
|
||||
(not serialised as ``null``). Tuples become JSON arrays. Decimal
|
||||
values are serialised as strings to preserve precision.
|
||||
|
||||
Identical state → byte-identical output (determinism gate).
|
||||
"""
|
||||
d = _canonical_dict_omit_none(state)
|
||||
return json.dumps(
|
||||
d,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Backward-compatibility alias
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Alias for code that imported ComprehensionState from #321.
|
||||
#: ``SentenceReadingState`` is the canonical name per ADR-0164.3.
|
||||
ComprehensionState = SentenceReadingState
|
||||
|
|
|
|||
|
|
@ -5,18 +5,78 @@ from decimal import Decimal
|
|||
import pytest
|
||||
|
||||
from generate.comprehension.state import (
|
||||
ComprehensionState,
|
||||
ComprehensionStateError,
|
||||
# inner level (renamed from ComprehensionState)
|
||||
SentenceReadingState,
|
||||
# outer level (new)
|
||||
ProblemReadingState,
|
||||
# shared leaf types
|
||||
EntityRef,
|
||||
ExpectationFrame,
|
||||
PartialOp,
|
||||
QuantityRef,
|
||||
QuestionTargetSlot,
|
||||
# new inner-level leaf types
|
||||
AppliedCategory,
|
||||
FramePayload,
|
||||
VerbReference,
|
||||
# new outer-level leaf types
|
||||
PartialInitialPossession,
|
||||
PartialOperation,
|
||||
PronounResolution,
|
||||
# refusal
|
||||
ReaderRefusal,
|
||||
READER_REFUSAL_REASONS,
|
||||
# error
|
||||
ComprehensionStateError,
|
||||
# canonical-bytes function
|
||||
to_canonical_bytes,
|
||||
# backward-compat alias (must still resolve)
|
||||
ComprehensionState,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_entity(name: str = "tina", gender: str = "female", pos: int = 0) -> EntityRef:
|
||||
return EntityRef(canonical_name=name, gender=gender, first_mention_position=pos)
|
||||
|
||||
|
||||
def _make_qty(
|
||||
val: str = "12.00",
|
||||
unit: str = "dollars",
|
||||
unit_class: str = "currency",
|
||||
pos: int = 1,
|
||||
) -> QuantityRef:
|
||||
return QuantityRef(
|
||||
value=Decimal(val),
|
||||
unit=unit,
|
||||
unit_class=unit_class,
|
||||
owner_entity=None,
|
||||
mention_position=pos,
|
||||
)
|
||||
|
||||
|
||||
|
||||
def _make_empty_problem_state() -> ProblemReadingState:
|
||||
return ProblemReadingState(
|
||||
entity_registry=(),
|
||||
accumulated_initial_state=(),
|
||||
accumulated_operations=(),
|
||||
unknown_target_slot=None,
|
||||
pronoun_resolution_history=(),
|
||||
sentence_index=0,
|
||||
source_text_offset=0,
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# EXISTING TESTS — now exercising SentenceReadingState (pure rename from #321)
|
||||
# ===========================================================================
|
||||
|
||||
def test_canonical_bytes_are_deterministic_for_equal_states() -> None:
|
||||
first = ComprehensionState(
|
||||
first = SentenceReadingState(
|
||||
entities=(
|
||||
EntityRef(
|
||||
canonical_name="tina",
|
||||
|
|
@ -53,7 +113,7 @@ def test_canonical_bytes_are_deterministic_for_equal_states() -> None:
|
|||
reason="question frame remains open",
|
||||
),
|
||||
)
|
||||
second = ComprehensionState(
|
||||
second = SentenceReadingState(
|
||||
entities=(
|
||||
EntityRef(
|
||||
canonical_name="tina",
|
||||
|
|
@ -100,7 +160,7 @@ def test_canonical_bytes_are_deterministic_for_equal_states() -> None:
|
|||
|
||||
|
||||
def test_optional_fields_serialize_as_null() -> None:
|
||||
state = ComprehensionState(
|
||||
state = SentenceReadingState(
|
||||
entities=(),
|
||||
quantities=(),
|
||||
operations=(),
|
||||
|
|
@ -160,14 +220,449 @@ def test_expectation_frame_refuses_empty_or_unknown_categories() -> None:
|
|||
|
||||
def test_comprehension_state_refuses_non_tuple_and_wrong_member_types() -> None:
|
||||
with pytest.raises(ComprehensionStateError, match="entities must be tuple"):
|
||||
ComprehensionState(
|
||||
SentenceReadingState(
|
||||
entities=[], # type: ignore[arg-type]
|
||||
quantities=(),
|
||||
operations=(),
|
||||
)
|
||||
with pytest.raises(ComprehensionStateError, match="quantities\\[0\\]"):
|
||||
ComprehensionState(
|
||||
SentenceReadingState(
|
||||
entities=(),
|
||||
quantities=("bad",), # type: ignore[arg-type]
|
||||
operations=(),
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# NEW TESTS — backward-compat alias
|
||||
# ===========================================================================
|
||||
|
||||
def test_comprehension_state_alias_resolves_to_sentence_reading_state() -> None:
|
||||
"""ComprehensionState must still resolve so callers from #321 need not change."""
|
||||
assert ComprehensionState is SentenceReadingState
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# NEW TESTS — SentenceReadingState new fields
|
||||
# ===========================================================================
|
||||
|
||||
def test_sentence_reading_state_new_fields_default_to_safe_values() -> None:
|
||||
state = SentenceReadingState(entities=(), quantities=(), operations=())
|
||||
assert state.frame is None
|
||||
assert state.pending_quantities == ()
|
||||
assert state.pending_entity_ref is None
|
||||
assert state.pending_verb is None
|
||||
assert state.token_index == 0
|
||||
assert state.lookback == ()
|
||||
assert state.partial_frame_payload is None
|
||||
|
||||
|
||||
def test_sentence_reading_state_accepts_new_fields() -> None:
|
||||
verb_ref = VerbReference(surface="makes", kind="rate_emit", position=1)
|
||||
applied = AppliedCategory(category="accumulation_verb", position=1)
|
||||
payload = FramePayload(frame_kind="operation_frame")
|
||||
entity = _make_entity()
|
||||
qty = _make_qty()
|
||||
|
||||
state = SentenceReadingState(
|
||||
entities=(entity,),
|
||||
quantities=(qty,),
|
||||
operations=(),
|
||||
frame="operation_frame",
|
||||
pending_quantities=(qty,),
|
||||
pending_entity_ref=entity,
|
||||
pending_verb=verb_ref,
|
||||
token_index=3,
|
||||
lookback=(applied,),
|
||||
partial_frame_payload=payload,
|
||||
)
|
||||
|
||||
assert state.frame == "operation_frame"
|
||||
assert state.pending_verb is verb_ref
|
||||
assert state.token_index == 3
|
||||
assert len(state.lookback) == 1
|
||||
assert state.partial_frame_payload is payload
|
||||
|
||||
|
||||
def test_sentence_reading_state_refuses_invalid_frame() -> None:
|
||||
with pytest.raises(ComprehensionStateError, match="frame"):
|
||||
SentenceReadingState(
|
||||
entities=(), quantities=(), operations=(),
|
||||
frame="nonsense_frame", # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
|
||||
def test_sentence_reading_state_refuses_lookback_overflow() -> None:
|
||||
cats = tuple(AppliedCategory(category="accumulation_verb", position=i) for i in range(9))
|
||||
with pytest.raises(ComprehensionStateError, match="lookback.*≤8"):
|
||||
SentenceReadingState(entities=(), quantities=(), operations=(), lookback=cats)
|
||||
|
||||
|
||||
def test_verb_reference_construction_and_validation() -> None:
|
||||
vr = VerbReference(surface="earns", kind="accumulation_verb", position=2)
|
||||
assert vr.surface == "earns"
|
||||
assert vr.kind == "accumulation_verb"
|
||||
assert vr.as_canonical() == {"kind": "accumulation_verb", "position": 2, "surface": "earns"}
|
||||
|
||||
with pytest.raises(ComprehensionStateError):
|
||||
VerbReference(surface="", kind="x", position=0)
|
||||
with pytest.raises(ComprehensionStateError):
|
||||
VerbReference(surface="x", kind="x", position=-1)
|
||||
|
||||
|
||||
def test_applied_category_construction() -> None:
|
||||
ac = AppliedCategory(category="depletion_verb", position=5)
|
||||
assert ac.category == "depletion_verb"
|
||||
assert ac.as_canonical() == {"category": "depletion_verb", "position": 5}
|
||||
|
||||
|
||||
def test_frame_payload_construction_and_validation() -> None:
|
||||
for kind in ("initial_state_frame", "operation_frame", "question_frame", "descriptive_frame"):
|
||||
fp = FramePayload(frame_kind=kind)
|
||||
assert fp.frame_kind == kind
|
||||
|
||||
with pytest.raises(ComprehensionStateError, match="frame_kind"):
|
||||
FramePayload(frame_kind="bad_frame")
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# NEW TESTS — PartialInitialPossession and PartialOperation
|
||||
# ===========================================================================
|
||||
|
||||
def test_partial_initial_possession_fully_nullable() -> None:
|
||||
pip = PartialInitialPossession(entity=None, quantity=None)
|
||||
assert pip.entity is None
|
||||
assert pip.quantity is None
|
||||
assert pip.as_canonical() == {}
|
||||
|
||||
|
||||
def test_partial_initial_possession_with_values() -> None:
|
||||
qty = _make_qty()
|
||||
pip = PartialInitialPossession(entity="tina", quantity=qty)
|
||||
canon = pip.as_canonical()
|
||||
assert canon["entity"] == "tina"
|
||||
assert "quantity" in canon
|
||||
|
||||
|
||||
def test_partial_initial_possession_refuses_empty_entity() -> None:
|
||||
with pytest.raises(ComprehensionStateError):
|
||||
PartialInitialPossession(entity="", quantity=None)
|
||||
|
||||
|
||||
def test_partial_operation_fully_nullable() -> None:
|
||||
po = PartialOperation(actor=None, kind=None, operand=None, target=None)
|
||||
assert po.as_canonical() == {}
|
||||
|
||||
|
||||
def test_partial_operation_with_values() -> None:
|
||||
qty = _make_qty()
|
||||
po = PartialOperation(actor="tina", kind="accumulation", operand=qty, target=None)
|
||||
canon = po.as_canonical()
|
||||
assert canon["actor"] == "tina"
|
||||
assert canon["kind"] == "accumulation"
|
||||
assert "operand" in canon
|
||||
assert "target" not in canon # None → omitted
|
||||
|
||||
|
||||
def test_pronoun_resolution_construction() -> None:
|
||||
pr = PronounResolution(pronoun="she", resolved_to="Tina", at_sentence=1, at_position=0)
|
||||
assert pr.resolved_to == "Tina"
|
||||
canon = pr.as_canonical()
|
||||
assert canon == {
|
||||
"at_position": 0,
|
||||
"at_sentence": 1,
|
||||
"pronoun": "she",
|
||||
"resolved_to": "Tina",
|
||||
}
|
||||
|
||||
|
||||
def test_pronoun_resolution_refuses_empty_fields() -> None:
|
||||
with pytest.raises(ComprehensionStateError, match="PronounResolution.pronoun"):
|
||||
PronounResolution(pronoun="", resolved_to="Tina", at_sentence=0, at_position=0)
|
||||
with pytest.raises(ComprehensionStateError, match="PronounResolution.resolved_to"):
|
||||
PronounResolution(pronoun="she", resolved_to="", at_sentence=0, at_position=0)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# NEW TESTS — ProblemReadingState construction
|
||||
# ===========================================================================
|
||||
|
||||
def test_problem_reading_state_empty_construction() -> None:
|
||||
ps = _make_empty_problem_state()
|
||||
assert ps.entity_registry == ()
|
||||
assert ps.accumulated_initial_state == ()
|
||||
assert ps.accumulated_operations == ()
|
||||
assert ps.unknown_target_slot is None
|
||||
assert ps.pronoun_resolution_history == ()
|
||||
assert ps.sentence_index == 0
|
||||
assert ps.source_text_offset == 0
|
||||
|
||||
|
||||
def test_problem_reading_state_with_entities_and_operations() -> None:
|
||||
entity = _make_entity()
|
||||
qty = _make_qty()
|
||||
pip = PartialInitialPossession(entity="tina", quantity=qty)
|
||||
po = PartialOperation(actor="tina", kind="accumulation", operand=qty, target=None)
|
||||
pr = PronounResolution(pronoun="she", resolved_to="tina", at_sentence=1, at_position=0)
|
||||
qt = QuestionTargetSlot(kind="continuous_quantity", entity="tina", unit_class="currency", position=0)
|
||||
|
||||
ps = ProblemReadingState(
|
||||
entity_registry=(entity,),
|
||||
accumulated_initial_state=(pip,),
|
||||
accumulated_operations=(po,),
|
||||
unknown_target_slot=qt,
|
||||
pronoun_resolution_history=(pr,),
|
||||
sentence_index=2,
|
||||
source_text_offset=47,
|
||||
)
|
||||
|
||||
assert len(ps.entity_registry) == 1
|
||||
assert ps.sentence_index == 2
|
||||
assert ps.source_text_offset == 47
|
||||
assert ps.unknown_target_slot is qt
|
||||
|
||||
|
||||
def test_problem_reading_state_refuses_wrong_member_types() -> None:
|
||||
with pytest.raises(ComprehensionStateError, match="entity_registry"):
|
||||
ProblemReadingState(
|
||||
entity_registry="bad", # type: ignore[arg-type]
|
||||
accumulated_initial_state=(),
|
||||
accumulated_operations=(),
|
||||
unknown_target_slot=None,
|
||||
pronoun_resolution_history=(),
|
||||
sentence_index=0,
|
||||
source_text_offset=0,
|
||||
)
|
||||
with pytest.raises(ComprehensionStateError, match="entity_registry\\[0\\]"):
|
||||
ProblemReadingState(
|
||||
entity_registry=("bad",), # type: ignore[arg-type]
|
||||
accumulated_initial_state=(),
|
||||
accumulated_operations=(),
|
||||
unknown_target_slot=None,
|
||||
pronoun_resolution_history=(),
|
||||
sentence_index=0,
|
||||
source_text_offset=0,
|
||||
)
|
||||
with pytest.raises(ComprehensionStateError, match="sentence_index"):
|
||||
ProblemReadingState(
|
||||
entity_registry=(),
|
||||
accumulated_initial_state=(),
|
||||
accumulated_operations=(),
|
||||
unknown_target_slot=None,
|
||||
pronoun_resolution_history=(),
|
||||
sentence_index=-1,
|
||||
source_text_offset=0,
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# NEW TESTS — ProblemReadingState canonical-bytes determinism gate
|
||||
# ===========================================================================
|
||||
|
||||
def test_problem_reading_state_canonical_bytes_determinism() -> None:
|
||||
"""Two equal ProblemReadingState instances must produce byte-equal output."""
|
||||
entity = _make_entity()
|
||||
pip = PartialInitialPossession(entity="tina", quantity=None)
|
||||
|
||||
ps1 = ProblemReadingState(
|
||||
entity_registry=(entity,),
|
||||
accumulated_initial_state=(pip,),
|
||||
accumulated_operations=(),
|
||||
unknown_target_slot=None,
|
||||
pronoun_resolution_history=(),
|
||||
sentence_index=1,
|
||||
source_text_offset=33,
|
||||
)
|
||||
ps2 = ProblemReadingState(
|
||||
entity_registry=(entity,),
|
||||
accumulated_initial_state=(pip,),
|
||||
accumulated_operations=(),
|
||||
unknown_target_slot=None,
|
||||
pronoun_resolution_history=(),
|
||||
sentence_index=1,
|
||||
source_text_offset=33,
|
||||
)
|
||||
|
||||
assert ps1.canonical_bytes() == ps2.canonical_bytes()
|
||||
assert ps1.canonical_hash() == ps2.canonical_hash()
|
||||
assert to_canonical_bytes(ps1) == to_canonical_bytes(ps2)
|
||||
|
||||
|
||||
def test_problem_reading_state_canonical_bytes_sensitivity() -> None:
|
||||
"""Single-field differences must produce different canonical bytes."""
|
||||
base = _make_empty_problem_state()
|
||||
|
||||
# sentence_index differs
|
||||
incremented = ProblemReadingState(
|
||||
entity_registry=(),
|
||||
accumulated_initial_state=(),
|
||||
accumulated_operations=(),
|
||||
unknown_target_slot=None,
|
||||
pronoun_resolution_history=(),
|
||||
sentence_index=1,
|
||||
source_text_offset=0,
|
||||
)
|
||||
assert base.canonical_bytes() != incremented.canonical_bytes()
|
||||
|
||||
# source_text_offset differs
|
||||
offset = ProblemReadingState(
|
||||
entity_registry=(),
|
||||
accumulated_initial_state=(),
|
||||
accumulated_operations=(),
|
||||
unknown_target_slot=None,
|
||||
pronoun_resolution_history=(),
|
||||
sentence_index=0,
|
||||
source_text_offset=10,
|
||||
)
|
||||
assert base.canonical_bytes() != offset.canonical_bytes()
|
||||
|
||||
# entity_registry populated
|
||||
with_entity = ProblemReadingState(
|
||||
entity_registry=(_make_entity(),),
|
||||
accumulated_initial_state=(),
|
||||
accumulated_operations=(),
|
||||
unknown_target_slot=None,
|
||||
pronoun_resolution_history=(),
|
||||
sentence_index=0,
|
||||
source_text_offset=0,
|
||||
)
|
||||
assert base.canonical_bytes() != with_entity.canonical_bytes()
|
||||
|
||||
|
||||
def test_problem_reading_state_none_fields_omitted_from_canonical() -> None:
|
||||
"""Per ADR-0164.3 §Canonical-bytes: None → omitted, not serialised as null."""
|
||||
ps = _make_empty_problem_state()
|
||||
raw = to_canonical_bytes(ps)
|
||||
assert b"null" not in raw, (
|
||||
"None fields must be OMITTED from canonical bytes, not serialised as null"
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# NEW TESTS — to_canonical_bytes handles SentenceReadingState too
|
||||
# ===========================================================================
|
||||
|
||||
def test_to_canonical_bytes_sentence_state_none_omitted() -> None:
|
||||
state = SentenceReadingState(entities=(), quantities=(), operations=())
|
||||
raw = to_canonical_bytes(state)
|
||||
assert b"null" not in raw
|
||||
# token_index=0 is a non-None int — must be present
|
||||
assert b'"token_index"' in raw
|
||||
|
||||
|
||||
def test_to_canonical_bytes_sentence_state_determinism() -> None:
|
||||
s1 = SentenceReadingState(
|
||||
entities=(_make_entity(),),
|
||||
quantities=(),
|
||||
operations=(),
|
||||
token_index=2,
|
||||
)
|
||||
s2 = SentenceReadingState(
|
||||
entities=(_make_entity(),),
|
||||
quantities=(),
|
||||
operations=(),
|
||||
token_index=2,
|
||||
)
|
||||
assert to_canonical_bytes(s1) == to_canonical_bytes(s2)
|
||||
|
||||
|
||||
def test_to_canonical_bytes_sentence_state_sensitivity() -> None:
|
||||
base = SentenceReadingState(entities=(), quantities=(), operations=(), token_index=0)
|
||||
diff = SentenceReadingState(entities=(), quantities=(), operations=(), token_index=1)
|
||||
assert to_canonical_bytes(base) != to_canonical_bytes(diff)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# NEW TESTS — ReaderRefusal construction and canonical bytes
|
||||
# ===========================================================================
|
||||
|
||||
def test_reader_refusal_reasons_has_11_entries() -> None:
|
||||
assert len(READER_REFUSAL_REASONS) == 11
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reason", sorted(READER_REFUSAL_REASONS))
|
||||
def test_every_refusal_reason_is_constructible(reason: str) -> None:
|
||||
rf = ReaderRefusal(
|
||||
reason=reason,
|
||||
detail=f"test refusal for {reason}",
|
||||
sentence_index=0,
|
||||
token_index=0,
|
||||
token_text="",
|
||||
)
|
||||
assert rf.reason == reason
|
||||
|
||||
|
||||
def test_reader_refusal_refuses_unknown_reason() -> None:
|
||||
with pytest.raises(ComprehensionStateError, match="READER_REFUSAL_REASONS"):
|
||||
ReaderRefusal(
|
||||
reason="made_up_reason",
|
||||
detail="detail",
|
||||
sentence_index=0,
|
||||
token_index=0,
|
||||
token_text="",
|
||||
)
|
||||
|
||||
|
||||
def test_reader_refusal_refuses_empty_detail() -> None:
|
||||
with pytest.raises(ComprehensionStateError, match="ReaderRefusal.detail"):
|
||||
ReaderRefusal(
|
||||
reason="unknown_word",
|
||||
detail="",
|
||||
sentence_index=0,
|
||||
token_index=0,
|
||||
token_text="if",
|
||||
)
|
||||
|
||||
|
||||
def test_reader_refusal_canonical_bytes_round_trip() -> None:
|
||||
rf = ReaderRefusal(
|
||||
reason="unexpected_category",
|
||||
detail="conditional_open at sentence_index=1, position=0",
|
||||
sentence_index=1,
|
||||
token_index=0,
|
||||
token_text="If",
|
||||
)
|
||||
b1 = rf.canonical_bytes()
|
||||
b2 = to_canonical_bytes(rf)
|
||||
assert b1 == b2
|
||||
|
||||
# same values → byte-equal
|
||||
rf2 = ReaderRefusal(
|
||||
reason="unexpected_category",
|
||||
detail="conditional_open at sentence_index=1, position=0",
|
||||
sentence_index=1,
|
||||
token_index=0,
|
||||
token_text="If",
|
||||
)
|
||||
assert rf.canonical_bytes() == rf2.canonical_bytes()
|
||||
assert rf.canonical_hash() == rf2.canonical_hash()
|
||||
|
||||
|
||||
def test_reader_refusal_empty_token_text_allowed() -> None:
|
||||
"""Sentence-level and problem-level refusals have no single token."""
|
||||
rf = ReaderRefusal(
|
||||
reason="unfinished_frame",
|
||||
detail="frame never decided",
|
||||
sentence_index=2,
|
||||
token_index=0,
|
||||
token_text="",
|
||||
)
|
||||
assert rf.token_text == ""
|
||||
|
||||
|
||||
def test_reader_refusal_canonical_bytes_sensitivity() -> None:
|
||||
rf1 = ReaderRefusal(
|
||||
reason="unknown_word",
|
||||
detail="detail",
|
||||
sentence_index=0,
|
||||
token_index=0,
|
||||
token_text="foo",
|
||||
)
|
||||
rf2 = ReaderRefusal(
|
||||
reason="unknown_word",
|
||||
detail="detail",
|
||||
sentence_index=0,
|
||||
token_index=1,
|
||||
token_text="foo",
|
||||
)
|
||||
assert rf1.canonical_bytes() != rf2.canonical_bytes()
|
||||
|
|
|
|||
Loading…
Reference in a new issue