diff --git a/chat/runtime.py b/chat/runtime.py index ee8dbde7..e9ed663b 100644 --- a/chat/runtime.py +++ b/chat/runtime.py @@ -535,6 +535,11 @@ class ChatRuntime: # is True AND the planner actually engaged on the turn. Exposed # via the ``last_plan_findings`` property below. self._last_plan_findings: tuple[Any, ...] = () + # Phase 4 — most-recent plan-articulation metrics (PlanMetrics). + # Reset to ``None`` between turns. Populated under the same + # gating discipline as ``_last_plan_findings``: requires + # ``config.discourse_contemplation`` + an engaged planner. + self._last_plan_metrics: Any | None = None @property def session(self) -> SessionContext: @@ -554,6 +559,22 @@ class ChatRuntime: """ return self._last_plan_findings + @property + def last_plan_metrics(self) -> Any | None: + """Phase 4 — most-recent plan articulation metrics. + + ``core.contemplation.plan_metrics.PlanMetrics`` instance + when the discourse planner engaged on the most recent turn + AND ``config.discourse_contemplation`` is True; ``None`` + otherwise. Read-only quantitative companion to + ``last_plan_findings`` (which carries the qualitative + SPECULATIVE concerns). Designed for downstream aggregation + — Phase 5's offline contemplation miner streams these + across turns to score plan-quality patterns the runtime + never tries to act on alone. + """ + return self._last_plan_metrics + def attach_telemetry_sink( self, sink: TurnEventSink | None, @@ -1016,10 +1037,12 @@ class ChatRuntime: * Returns ``None`` when the renderer produces an empty string. """ - # Phase 3 — reset plan-contemplation findings at the start of - # every call so they never leak across turns; only successfully - # rendered plans (with contemplation enabled) repopulate them. + # Phase 3 + 4 — reset plan-contemplation findings AND plan + # metrics at the start of every call so they never leak across + # turns; only successfully rendered plans (with contemplation + # enabled) repopulate them. self._last_plan_findings = () + self._last_plan_metrics = None if not self.config.discourse_planner: return None from generate.discourse_planner import ( @@ -1099,15 +1122,19 @@ class ChatRuntime: plan = plan_discourse(intent, mode, bundle) if len(plan.moves) <= 1: return None - # Phase 3 — plan-level contemplation pre-flight. Read-only, - # SPECULATIVE-only; stores findings on the runtime for the - # offline contemplation miner to consume. Does not mutate - # the plan or block rendering — emits side observations only. + # Phase 3 + 4 — plan-level contemplation pre-flight + metrics. + # Read-only, SPECULATIVE-only on the findings side; pure + # measurements on the metrics side. Stores both on the + # runtime for offline miner consumption. Does not mutate the + # plan or block rendering — emits side observations only. if self.config.discourse_contemplation: + from core.contemplation.plan_metrics import compute_plan_metrics from core.contemplation.plan_preflight import contemplate_plan self._last_plan_findings = contemplate_plan(plan) + self._last_plan_metrics = compute_plan_metrics(plan) else: self._last_plan_findings = () + self._last_plan_metrics = None # Phase 2 — reflective rendering pronominalizes the focus # subject across consecutive same-subject moves, eliminating # the mechanical "Truth ... Truth ... Truth ..." cascade the diff --git a/core/contemplation/plan_metrics.py b/core/contemplation/plan_metrics.py new file mode 100644 index 00000000..f5db678e --- /dev/null +++ b/core/contemplation/plan_metrics.py @@ -0,0 +1,247 @@ +"""Phase 4 — per-plan articulation telemetry metrics. + +Pure-function projection of a ``DiscoursePlan`` into structured +quantitative measurements. Mirrors Phase 3's ``plan_preflight`` +contemplation: + + Phase 3 (plan_preflight) → typed SPECULATIVE *findings* (qualitative) + Phase 4 (plan_metrics) → typed *measurements* (quantitative) + +Both run after the planner finishes; neither mutates anything. +Findings answer "what's wrong with this plan?". Metrics answer +"what shape does this plan have?". Together they give downstream +consumers (offline contemplation miner, operator dashboards) the +signal they need to score plan quality across many turns. + +Why a separate dataclass and not just a dict +-------------------------------------------- + +* **Typed boundary.** ``PlanMetrics`` field types make the + serialization contract explicit; a downstream consumer can't + silently break on a renamed key. +* **Deterministic identity.** ``frozen=True`` + ``slots=True`` + + positional ``as_dict()`` keys means two metrics objects built from + byte-equal plans serialize identically. This is what lets the + offline miner aggregate over time without "is this the same + metric?" ambiguity. +* **Cheap.** Computation is O(moves); no allocation per move + beyond the dataclass itself. + +Doctrine notes +-------------- + +Metrics are pure measurements, not opinions. They never mutate +the plan, the runtime state, or the memory tiers. Promotion to +memory still flows through the existing proposal-review-ratify +chain. Where Phase 3 emits SPECULATIVE *findings* (which downstream +review may accept), Phase 4 emits raw numbers (which downstream +analytics may aggregate). +""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass +from typing import Any + +from generate.discourse_planner import ( + DiscourseMoveKind, + DiscoursePlan, +) + + +@dataclass(frozen=True, slots=True) +class PlanMetrics: + """Quantitative measurements of one ``DiscoursePlan``. + + Every field is a pure function of the plan; same plan in → + byte-identical metrics out. Used by Phase 5 (offline miner) + to aggregate plan-quality signal across many turns and surface + deeper structural patterns that single-plan contemplation + (Phase 3) cannot see. + """ + + # ------ Structure ------ + + move_count: int + """Total moves in the plan, including those without facts (e.g. + bridge ``TRANSITION`` moves with ``fact=None`` and ``CLOSURE`` + moves without summary facts).""" + + fact_bearing_count: int + """Moves with ``fact is not None`` — these are the moves the + renderer actually emits clauses for. ``fact_bearing_count`` + < ``move_count`` indicates structural moves (bridges, closures) + the renderer elides.""" + + # ------ Move-kind distribution ------ + + anchor_count: int + support_count: int + relation_count: int + transition_count: int + closure_count: int + + # ------ Diversity ------ + + unique_predicates: int + """Number of distinct predicate strings across fact-bearing moves. + Low absolute counts paired with high move_count signal predicate + monotony (the WEAK_SURFACE finding from Phase 3).""" + + unique_subjects: int + """Number of distinct subject lemmas across fact-bearing moves.""" + + unique_sources: int + """Number of distinct ``FactSource`` values across fact-bearing + moves. ``unique_sources == 1`` with multi-move plans signals + the COVERAGE_GAP finding from Phase 3.""" + + # ------ Topic dynamics ------ + + topic_shift_count: int + """Number of consecutive-move pairs where the fact subject + changed. Counts transitions across the visible focus channel + that Phase 2 reflective rendering uses; ``topic_shift_count`` + + ``pronominalization_opportunities`` + 1 (for the anchor) sums + to ``fact_bearing_count`` minus zero-fact moves.""" + + pronominalization_opportunities: int + """Number of consecutive-move pairs where the fact subject + repeated. Phase 2's reflective renderer takes each opportunity + to swap the subject token to ``it``.""" + + def as_dict(self) -> dict[str, Any]: + return { + "move_count": self.move_count, + "fact_bearing_count": self.fact_bearing_count, + "anchor_count": self.anchor_count, + "support_count": self.support_count, + "relation_count": self.relation_count, + "transition_count": self.transition_count, + "closure_count": self.closure_count, + "unique_predicates": self.unique_predicates, + "unique_subjects": self.unique_subjects, + "unique_sources": self.unique_sources, + "topic_shift_count": self.topic_shift_count, + "pronominalization_opportunities": ( + self.pronominalization_opportunities + ), + # Derived ratios — included in the wire format so consumers + # don't recompute them inconsistently. ``None`` when undefined + # (e.g. empty plan, single-move plan with no pairs). + "predicate_diversity_ratio": self.predicate_diversity_ratio, + "subject_focus_ratio": self.subject_focus_ratio, + } + + # ---- Derived ratios ---- + + @property + def predicate_diversity_ratio(self) -> float | None: + """``unique_predicates / fact_bearing_count`` — ``None`` when + no fact-bearing moves (nothing to divide by). + + 1.0 = every fact-bearing move uses a distinct predicate (most + diverse). Trending toward 0 = predicates repeating (Phase 3 + ``WEAK_SURFACE`` candidate). + """ + if self.fact_bearing_count == 0: + return None + return self.unique_predicates / self.fact_bearing_count + + @property + def subject_focus_ratio(self) -> float | None: + """Fraction of consecutive-move pairs that held subject focus + (i.e. the inverse of topic-shift rate). ``None`` when there + are no consecutive pairs (< 2 fact-bearing moves). + + 1.0 = perfectly stuck on one topic (every pronominalization + opportunity engaged). Trending toward 0 = topic shifts on + every move (compound or wandering plan). + """ + total_pairs = ( + self.pronominalization_opportunities + self.topic_shift_count + ) + if total_pairs == 0: + return None + return self.pronominalization_opportunities / total_pairs + + +def compute_plan_metrics(plan: DiscoursePlan) -> PlanMetrics: + """Project a :class:`DiscoursePlan` into a :class:`PlanMetrics`. + + Pure deterministic function: ``compute_plan_metrics(p) == + compute_plan_metrics(p)`` byte-identical for any plan ``p``. + + Empty plans yield a zero-valued ``PlanMetrics`` so downstream + consumers can use the same shape regardless of plan engagement. + """ + + if plan.is_empty(): + return PlanMetrics( + move_count=0, + fact_bearing_count=0, + anchor_count=0, + support_count=0, + relation_count=0, + transition_count=0, + closure_count=0, + unique_predicates=0, + unique_subjects=0, + unique_sources=0, + topic_shift_count=0, + pronominalization_opportunities=0, + ) + + kind_counts: Counter[DiscourseMoveKind] = Counter() + predicates: set[str] = set() + subjects: set[str] = set() + sources: set[Any] = set() + fact_bearing = 0 + prior_subject: str | None = None + topic_shifts = 0 + pronominalizations = 0 + + for move in plan.moves: + kind_counts[move.kind] += 1 + if move.fact is None: + # Bridge / closure-without-summary moves don't carry a + # subject focus — they reset the channel. Track a topic + # shift so the focus_ratio reflects the discontinuity but + # do NOT update prior_subject (the next fact-bearing move + # establishes new focus from scratch). + if prior_subject is not None: + topic_shifts += 1 + prior_subject = None + continue + fact_bearing += 1 + predicates.add(move.fact.predicate) + subjects.add(move.fact.subject) + sources.add(move.fact.source) + if prior_subject is not None: + if move.fact.subject == prior_subject: + pronominalizations += 1 + else: + topic_shifts += 1 + prior_subject = move.fact.subject + + return PlanMetrics( + move_count=len(plan.moves), + fact_bearing_count=fact_bearing, + anchor_count=kind_counts.get(DiscourseMoveKind.ANCHOR, 0), + support_count=kind_counts.get(DiscourseMoveKind.SUPPORT, 0), + relation_count=kind_counts.get(DiscourseMoveKind.RELATION, 0), + transition_count=kind_counts.get(DiscourseMoveKind.TRANSITION, 0), + closure_count=kind_counts.get(DiscourseMoveKind.CLOSURE, 0), + unique_predicates=len(predicates), + unique_subjects=len(subjects), + unique_sources=len(sources), + topic_shift_count=topic_shifts, + pronominalization_opportunities=pronominalizations, + ) + + +__all__ = [ + "PlanMetrics", + "compute_plan_metrics", +] diff --git a/tests/test_plan_metrics.py b/tests/test_plan_metrics.py new file mode 100644 index 00000000..0aafc971 --- /dev/null +++ b/tests/test_plan_metrics.py @@ -0,0 +1,351 @@ +"""Phase 4 — per-plan articulation telemetry metrics. + +Pins ``core.contemplation.plan_metrics.compute_plan_metrics`` against: + + * Trivial cases (empty plan, single anchor) + * Structural counts (move_kind distribution) + * Diversity counts (unique predicates / subjects / sources) + * Topic dynamics (pronominalization opportunities, topic shifts) + * Derived ratios (predicate_diversity_ratio, subject_focus_ratio) + * Determinism (same plan → byte-equal metrics dict) + * Bridge-move handling (fact=None resets focus channel) +""" + +from __future__ import annotations + +from core.contemplation.plan_metrics import compute_plan_metrics +from generate.discourse_planner import ( + DiscourseMove, + DiscourseMoveKind, + DiscoursePlan, + FactSource, + GroundedFact, +) +from generate.intent import DialogueIntent, IntentTag, ResponseMode + + +def _fact( + subject: str, + predicate: str, + obj: str, + *, + source: FactSource = FactSource.PACK, + source_id: str = "test_pack_v1", +) -> GroundedFact: + return GroundedFact( + subject=subject, + predicate=predicate, + obj=obj, + source=source, + source_id=source_id, + ) + + +def _intent(subject: str = "truth") -> DialogueIntent: + return DialogueIntent(tag=IntentTag.DEFINITION, subject=subject) + + +def _move( + kind: DiscourseMoveKind, fact: GroundedFact | None = None, +) -> DiscourseMove: + topic = fact.subject if fact is not None else "" + return DiscourseMove( + kind=kind, topic=topic, given=(), new=(), + relation_to_previous=None, fact=fact, + ) + + +# --------------------------------------------------------------------------- +# Empty plan +# --------------------------------------------------------------------------- + + +def test_empty_plan_yields_zero_metrics() -> None: + plan = DiscoursePlan(intent=_intent(), mode=ResponseMode.BRIEF, moves=()) + m = compute_plan_metrics(plan) + assert m.move_count == 0 + assert m.fact_bearing_count == 0 + assert m.anchor_count == 0 + assert m.unique_predicates == 0 + assert m.unique_subjects == 0 + assert m.unique_sources == 0 + assert m.topic_shift_count == 0 + assert m.pronominalization_opportunities == 0 + assert m.predicate_diversity_ratio is None + assert m.subject_focus_ratio is None + + +# --------------------------------------------------------------------------- +# Single-anchor plan +# --------------------------------------------------------------------------- + + +def test_single_anchor_plan_metrics() -> None: + plan = DiscoursePlan( + intent=_intent(), + mode=ResponseMode.BRIEF, + moves=( + _move( + DiscourseMoveKind.ANCHOR, + _fact("truth", "is_defined_as", "what is true"), + ), + ), + ) + m = compute_plan_metrics(plan) + assert m.move_count == 1 + assert m.fact_bearing_count == 1 + assert m.anchor_count == 1 + assert m.support_count == 0 + assert m.unique_predicates == 1 + assert m.unique_subjects == 1 + assert m.unique_sources == 1 + assert m.topic_shift_count == 0 + assert m.pronominalization_opportunities == 0 + assert m.predicate_diversity_ratio == 1.0 + # No consecutive pairs to measure — ratio undefined + assert m.subject_focus_ratio is None + + +# --------------------------------------------------------------------------- +# Move-kind distribution +# --------------------------------------------------------------------------- + + +def test_move_kind_distribution_counts() -> None: + plan = DiscoursePlan( + intent=_intent(), + mode=ResponseMode.PARAGRAPH, + moves=( + _move( + DiscourseMoveKind.ANCHOR, + _fact("truth", "is_defined_as", "what is true"), + ), + _move( + DiscourseMoveKind.SUPPORT, + _fact("truth", "belongs_to", "cognition.truth"), + ), + _move( + DiscourseMoveKind.RELATION, + _fact("truth", "grounds", "knowledge"), + ), + _move( + DiscourseMoveKind.TRANSITION, + _fact("knowledge", "belongs_to", "cognition.knowledge"), + ), + _move(DiscourseMoveKind.CLOSURE), # fact=None + ), + ) + m = compute_plan_metrics(plan) + assert m.move_count == 5 + assert m.fact_bearing_count == 4 + assert m.anchor_count == 1 + assert m.support_count == 1 + assert m.relation_count == 1 + assert m.transition_count == 1 + assert m.closure_count == 1 + + +# --------------------------------------------------------------------------- +# Pronominalization opportunities vs. topic shifts +# --------------------------------------------------------------------------- + + +def test_three_same_subject_moves_yield_two_pronominalization_opportunities() -> None: + plan = DiscoursePlan( + intent=_intent(), + mode=ResponseMode.PARAGRAPH, + moves=( + _move( + DiscourseMoveKind.ANCHOR, + _fact("truth", "is_defined_as", "what is true"), + ), + _move( + DiscourseMoveKind.SUPPORT, + _fact("truth", "belongs_to", "cognition.truth"), + ), + _move( + DiscourseMoveKind.RELATION, + _fact("truth", "grounds", "knowledge"), + ), + ), + ) + m = compute_plan_metrics(plan) + assert m.pronominalization_opportunities == 2 + assert m.topic_shift_count == 0 + assert m.subject_focus_ratio == 1.0 + + +def test_topic_shift_counted_when_subject_changes() -> None: + plan = DiscoursePlan( + intent=_intent(), + mode=ResponseMode.PARAGRAPH, + moves=( + _move( + DiscourseMoveKind.ANCHOR, + _fact("truth", "is_defined_as", "what is true"), + ), + _move( + DiscourseMoveKind.TRANSITION, + _fact("knowledge", "belongs_to", "cognition.knowledge"), + ), + ), + ) + m = compute_plan_metrics(plan) + assert m.topic_shift_count == 1 + assert m.pronominalization_opportunities == 0 + assert m.subject_focus_ratio == 0.0 + + +def test_bridge_move_resets_focus_channel() -> None: + """A fact-bearing move followed by a bridge (``fact=None``) followed + by another fact-bearing move with the SAME subject must not count + as a pronominalization opportunity — the bridge breaks the + consecutive-pair channel.""" + plan = DiscoursePlan( + intent=_intent(), + mode=ResponseMode.PARAGRAPH, + moves=( + _move( + DiscourseMoveKind.ANCHOR, + _fact("truth", "is_defined_as", "what is true"), + ), + _move(DiscourseMoveKind.TRANSITION), # bridge, fact=None + _move( + DiscourseMoveKind.SUPPORT, + _fact("truth", "belongs_to", "cognition.truth"), + ), + ), + ) + m = compute_plan_metrics(plan) + # Bridge counts as a shift; no pronominalization opportunity even + # though both fact-bearing moves share subject "truth". + assert m.topic_shift_count == 1 + assert m.pronominalization_opportunities == 0 + + +# --------------------------------------------------------------------------- +# Diversity counts +# --------------------------------------------------------------------------- + + +def test_predicate_diversity_ratio_reflects_monotony() -> None: + """Three moves with the same predicate → diversity ratio 1/3.""" + plan = DiscoursePlan( + intent=_intent(), + mode=ResponseMode.PARAGRAPH, + moves=( + _move( + DiscourseMoveKind.ANCHOR, + _fact("truth", "belongs_to", "cognition.truth"), + ), + _move( + DiscourseMoveKind.SUPPORT, + _fact("truth", "belongs_to", "epistemic.ground"), + ), + _move( + DiscourseMoveKind.RELATION, + _fact("truth", "belongs_to", "logos.core"), + ), + ), + ) + m = compute_plan_metrics(plan) + assert m.unique_predicates == 1 + assert m.fact_bearing_count == 3 + assert m.predicate_diversity_ratio is not None + assert abs(m.predicate_diversity_ratio - (1.0 / 3.0)) < 1e-9 + + +def test_source_diversity_counts_pack_plus_teaching() -> None: + plan = DiscoursePlan( + intent=_intent(), + mode=ResponseMode.EXPLAIN, + moves=( + _move( + DiscourseMoveKind.ANCHOR, + _fact( + "truth", "is_defined_as", "what is true", + source=FactSource.PACK, + ), + ), + _move( + DiscourseMoveKind.RELATION, + _fact( + "truth", "grounds", "knowledge", + source=FactSource.TEACHING, + source_id="cognition_chains_v1", + ), + ), + ), + ) + m = compute_plan_metrics(plan) + assert m.unique_sources == 2 + + +# --------------------------------------------------------------------------- +# Determinism +# --------------------------------------------------------------------------- + + +def test_metrics_are_deterministic_and_byte_equal_as_dict() -> None: + plan = DiscoursePlan( + intent=_intent(), + mode=ResponseMode.PARAGRAPH, + moves=( + _move( + DiscourseMoveKind.ANCHOR, + _fact("truth", "is_defined_as", "what is true"), + ), + _move( + DiscourseMoveKind.SUPPORT, + _fact("truth", "belongs_to", "cognition.truth"), + ), + _move( + DiscourseMoveKind.RELATION, + _fact("truth", "grounds", "knowledge"), + ), + ), + ) + a = compute_plan_metrics(plan) + b = compute_plan_metrics(plan) + assert a == b + assert a.as_dict() == b.as_dict() + + +# --------------------------------------------------------------------------- +# as_dict surface +# --------------------------------------------------------------------------- + + +def test_as_dict_includes_every_field_and_derived_ratios() -> None: + plan = DiscoursePlan( + intent=_intent(), + mode=ResponseMode.EXPLAIN, + moves=( + _move( + DiscourseMoveKind.ANCHOR, + _fact("truth", "is_defined_as", "what is true"), + ), + _move( + DiscourseMoveKind.SUPPORT, + _fact("truth", "belongs_to", "cognition.truth"), + ), + ), + ) + d = compute_plan_metrics(plan).as_dict() + for required_field in ( + "move_count", + "fact_bearing_count", + "anchor_count", + "support_count", + "relation_count", + "transition_count", + "closure_count", + "unique_predicates", + "unique_subjects", + "unique_sources", + "topic_shift_count", + "pronominalization_opportunities", + "predicate_diversity_ratio", + "subject_focus_ratio", + ): + assert required_field in d, f"missing field {required_field!r}" diff --git a/tests/test_plan_metrics_runtime.py b/tests/test_plan_metrics_runtime.py new file mode 100644 index 00000000..8ca1ec6f --- /dev/null +++ b/tests/test_plan_metrics_runtime.py @@ -0,0 +1,120 @@ +"""Phase 4 — end-to-end ``last_plan_metrics`` runtime wiring. + +Mirrors ``tests/test_plan_contemplation_runtime.py`` for the +quantitative companion. Pins: + + * Disabled by default — ``last_plan_metrics`` stays ``None`` even + when the planner engages. + * Enabled — metrics are populated whenever the planner engages. + * BRIEF prompts (fast-path) yield ``None`` metrics. + * Metrics do not leak across turns. + * Same prompt → byte-equal ``as_dict()`` (determinism). +""" + +from __future__ import annotations + +import pytest + +from chat.runtime import ChatRuntime +from core.config import RuntimeConfig + + +# --------------------------------------------------------------------------- +# Disabled by default +# --------------------------------------------------------------------------- + + +def test_metrics_none_when_contemplation_disabled() -> None: + rt = ChatRuntime(config=RuntimeConfig(discourse_contemplation=False)) + rt.chat("What is truth, and why does it matter?") + assert rt.last_plan_metrics is None + + +# --------------------------------------------------------------------------- +# Enabled — multi-move plan populates structured metrics +# --------------------------------------------------------------------------- + + +def test_compound_prompt_yields_expected_shape() -> None: + rt = ChatRuntime(config=RuntimeConfig(discourse_contemplation=True)) + rt.chat("What is truth, and why does it matter?") + m = rt.last_plan_metrics + assert m is not None + # Compound prompt routes through two sub-plans plus a bridge. + assert m.move_count >= 4 + assert m.fact_bearing_count >= 4 + # Plan re-uses the truth subject across multiple moves; should + # therefore expose pronominalization opportunities. + assert m.pronominalization_opportunities >= 1 + # Diversity ratios resolve to real numbers (no None) on a + # multi-move plan with >= 1 fact-bearing move. + assert m.predicate_diversity_ratio is not None + assert 0.0 < m.predicate_diversity_ratio <= 1.0 + assert m.subject_focus_ratio is not None + assert 0.0 <= m.subject_focus_ratio <= 1.0 + + +# --------------------------------------------------------------------------- +# BRIEF prompts (fast-path) yield no metrics +# --------------------------------------------------------------------------- + + +def test_brief_prompt_yields_no_metrics() -> None: + rt = ChatRuntime(config=RuntimeConfig(discourse_contemplation=True)) + rt.chat("What is knowledge?") + assert rt.last_plan_metrics is None + + +# --------------------------------------------------------------------------- +# Metrics do not leak across turns +# --------------------------------------------------------------------------- + + +def test_metrics_reset_between_turns() -> None: + rt = ChatRuntime(config=RuntimeConfig(discourse_contemplation=True)) + rt.chat("What is truth, and why does it matter?") + assert rt.last_plan_metrics is not None # sanity + rt.chat("What is knowledge?") # BRIEF — should clear + assert rt.last_plan_metrics is None + + +# --------------------------------------------------------------------------- +# Determinism across two runs +# --------------------------------------------------------------------------- + + +def test_metrics_byte_equal_across_runs() -> None: + rt1 = ChatRuntime(config=RuntimeConfig(discourse_contemplation=True)) + rt2 = ChatRuntime(config=RuntimeConfig(discourse_contemplation=True)) + rt1.chat("Tell me about memory.") + rt2.chat("Tell me about memory.") + m1 = rt1.last_plan_metrics + m2 = rt2.last_plan_metrics + assert m1 is not None and m2 is not None + assert m1.as_dict() == m2.as_dict() + + +# --------------------------------------------------------------------------- +# Findings and metrics co-populate cleanly +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "prompt", + [ + "Tell me about memory.", + "Explain truth.", + "What is truth, and why does it matter?", + ], +) +def test_findings_and_metrics_populate_together(prompt: str) -> None: + rt = ChatRuntime(config=RuntimeConfig(discourse_contemplation=True)) + rt.chat(prompt) + metrics = rt.last_plan_metrics + # Whenever metrics is populated, the planner engaged; findings + # is at least an empty tuple (never None on engaged turns). + assert metrics is not None + assert isinstance(rt.last_plan_findings, tuple) + # And the metrics' fact_bearing_count is non-zero on every + # engaged turn. + assert metrics.fact_bearing_count >= 1