diff --git a/benchmarks/pipeline_profiler.py b/benchmarks/pipeline_profiler.py new file mode 100644 index 00000000..4eeda8ad --- /dev/null +++ b/benchmarks/pipeline_profiler.py @@ -0,0 +1,182 @@ +"""Pipeline-stage profiler for CognitiveTurnPipeline. + +External instrumentation only — no edits to pipeline/runtime/algebra/vault +source files. Uses lightweight monkey-patching of bound methods on the +pipeline instance and the runtime instance for the duration of a single +``profile_turn`` call. All patches are reverted in a ``finally`` block so +the pipeline is left untouched. + +Per CLAUDE.md: no hidden normalization, no semantic mutation, no algebra +hot-path touch. Overhead per stage: a single ``time.perf_counter_ns`` +read on entry and on exit, and a list append. Stage label strings are +pre-interned at module load time (no f-strings inside timed regions). +""" + +from __future__ import annotations + +import time +from contextlib import contextmanager +from dataclasses import dataclass, field +from typing import Any, Iterator + +from core.cognition.pipeline import CognitiveTurnPipeline +from core.cognition.result import CognitiveTurnResult + + +# Pre-interned stage label constants — avoid string construction in +# the timed hot path. +_STAGE_INTENT = "intent" +_STAGE_GRAPH = "graph_planner" +_STAGE_REALIZE = "realize_semantic" +_STAGE_RUNTIME_CHAT = "runtime_chat" +_STAGE_TRANSITIVE_WALK = "maybe_transitive_walk" +_STAGE_FOLD_WALK = "fold_walk_into_surface" +_STAGE_TEACHING = "run_teaching" +_STAGE_TRACE = "trace_hash" +_STAGE_TOTAL = "total" + + +@dataclass(frozen=True) +class ProfileReport: + """Immutable timing report for a single profiled turn.""" + + stages: dict[str, int] + total_ns: int + result: CognitiveTurnResult + + def as_dict(self) -> dict[str, Any]: + return { + "stages": dict(self.stages), + "total_ns": int(self.total_ns), + } + + +@dataclass +class _ProfileSink: + """Mutable per-call accumulator. Not shared across calls — instantiated + fresh in every ``profile_turn`` invocation, so no global state.""" + + stages: dict[str, int] = field(default_factory=dict) + + def record(self, name: str, elapsed_ns: int) -> None: + # Multiple invocations of the same stage in a turn are summed. + prior = self.stages.get(name, 0) + self.stages[name] = prior + elapsed_ns + + +@contextmanager +def _stage(sink: _ProfileSink, name: str) -> Iterator[None]: + """Lightweight context manager: two perf_counter_ns reads plus a dict update.""" + t0 = time.perf_counter_ns() + try: + yield + finally: + sink.record(name, time.perf_counter_ns() - t0) + + +def profile_turn( + pipeline: CognitiveTurnPipeline, + text: str, + max_tokens: int | None = None, +) -> ProfileReport: + """Profile one CognitiveTurnPipeline.run() invocation. + + Wraps the pipeline's existing internal methods and the runtime's + ``chat`` method with timing decorators for the duration of this call, + then restores them. Patches live on the *instances*, not on the + classes, so concurrent profiling of distinct pipeline instances is + safe. + """ + sink = _ProfileSink() + + # Capture originals (instance attrs win over class attrs in resolution, + # so reassigning attrs on the instance does not mutate the class). + runtime = pipeline.runtime + orig_chat = runtime.chat + orig_maybe_walk = pipeline._maybe_transitive_walk + orig_fold = pipeline._fold_walk_into_surface + orig_run_teaching = pipeline._run_teaching + + # We patch generate.intent / graph_planner / realizer via per-call + # module-attribute swaps on the pipeline module so we only time the + # functions actually called from pipeline.run(). + from core.cognition import pipeline as pipeline_mod + + orig_classify_intent = pipeline_mod.classify_intent + orig_graph_from_intent = pipeline_mod.graph_from_intent + orig_plan_articulation = pipeline_mod.plan_articulation + orig_realize_semantic = pipeline_mod.realize_semantic + orig_compute_trace_hash = pipeline_mod.compute_trace_hash + + def timed_classify_intent(*args: Any, **kwargs: Any) -> Any: + with _stage(sink, _STAGE_INTENT): + return orig_classify_intent(*args, **kwargs) + + def timed_graph_from_intent(*args: Any, **kwargs: Any) -> Any: + with _stage(sink, _STAGE_GRAPH): + return orig_graph_from_intent(*args, **kwargs) + + def timed_plan_articulation(*args: Any, **kwargs: Any) -> Any: + with _stage(sink, _STAGE_GRAPH): + return orig_plan_articulation(*args, **kwargs) + + def timed_realize_semantic(*args: Any, **kwargs: Any) -> Any: + with _stage(sink, _STAGE_REALIZE): + return orig_realize_semantic(*args, **kwargs) + + def timed_compute_trace_hash(*args: Any, **kwargs: Any) -> Any: + with _stage(sink, _STAGE_TRACE): + return orig_compute_trace_hash(*args, **kwargs) + + def timed_chat(*args: Any, **kwargs: Any) -> Any: + with _stage(sink, _STAGE_RUNTIME_CHAT): + return orig_chat(*args, **kwargs) + + def timed_maybe_walk(*args: Any, **kwargs: Any) -> Any: + with _stage(sink, _STAGE_TRANSITIVE_WALK): + return orig_maybe_walk(*args, **kwargs) + + def timed_fold(*args: Any, **kwargs: Any) -> Any: + with _stage(sink, _STAGE_FOLD_WALK): + return orig_fold(*args, **kwargs) + + def timed_run_teaching(*args: Any, **kwargs: Any) -> Any: + with _stage(sink, _STAGE_TEACHING): + return orig_run_teaching(*args, **kwargs) + + pipeline_mod.classify_intent = timed_classify_intent + pipeline_mod.graph_from_intent = timed_graph_from_intent + pipeline_mod.plan_articulation = timed_plan_articulation + pipeline_mod.realize_semantic = timed_realize_semantic + pipeline_mod.compute_trace_hash = timed_compute_trace_hash + runtime.chat = timed_chat # type: ignore[assignment] + pipeline._maybe_transitive_walk = timed_maybe_walk # type: ignore[assignment] + pipeline._fold_walk_into_surface = timed_fold # type: ignore[assignment] + pipeline._run_teaching = timed_run_teaching # type: ignore[assignment] + + t_total_0 = time.perf_counter_ns() + try: + result = pipeline.run(text, max_tokens=max_tokens) + finally: + total_ns = time.perf_counter_ns() - t_total_0 + # Restore originals (instance and module). + pipeline_mod.classify_intent = orig_classify_intent + pipeline_mod.graph_from_intent = orig_graph_from_intent + pipeline_mod.plan_articulation = orig_plan_articulation + pipeline_mod.realize_semantic = orig_realize_semantic + pipeline_mod.compute_trace_hash = orig_compute_trace_hash + runtime.chat = orig_chat # type: ignore[assignment] + try: + del pipeline._maybe_transitive_walk # restore class-bound method + except AttributeError: + pipeline._maybe_transitive_walk = orig_maybe_walk # type: ignore[assignment] + try: + del pipeline._fold_walk_into_surface + except AttributeError: + pipeline._fold_walk_into_surface = orig_fold # type: ignore[assignment] + try: + del pipeline._run_teaching + except AttributeError: + pipeline._run_teaching = orig_run_teaching # type: ignore[assignment] + + return ProfileReport(stages=dict(sink.stages), total_ns=total_ns, result=result) diff --git a/benchmarks/word_selection_tracer.py b/benchmarks/word_selection_tracer.py new file mode 100644 index 00000000..e6b0956a --- /dev/null +++ b/benchmarks/word_selection_tracer.py @@ -0,0 +1,266 @@ +"""Word-selection tracer for the articulation/realization path. + +Captures every nearest-neighbor vocabulary lookup performed during a turn: + - slot name (subject / predicate / object) + - input versor (32-d float vector, copied) + - top-K candidate words by CGA inner product score + - chosen word + - any morphology applied + +Also records each realization step (subject, predicate, object, tense, +aspect, plural, negation) emitted by ``realize_semantic`` / ``realize_target``. + +External instrumentation only — instruments via module-level function +swaps that are reverted in ``finally``. No edits to generate/, vocab/, +or algebra/ source files. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +import numpy as np + +from algebra.backend import _CGA_INNER_METRIC # diagonal Cl(4,1) metric (±1 per blade) +from chat.runtime import ChatRuntime + + +@dataclass(frozen=True) +class WordSelectionStep: + """A single nearest-neighbor lookup observed during articulation.""" + + slot: str # 'subject' | 'predicate' | 'object' + input_versor: np.ndarray # shape (32,), copy — safe to retain + top_candidates: tuple[tuple[str, float], ...] # (word, cga_inner_score) + chosen: str + morphology: dict[str, Any] # tense/aspect/plural/negation/lemma/surface, if any + output_language: str + + def as_dict(self) -> dict[str, Any]: + return { + "slot": self.slot, + "top_candidates": [list(c) for c in self.top_candidates], + "chosen": self.chosen, + "morphology": dict(self.morphology), + "output_language": self.output_language, + } + + +@dataclass(frozen=True) +class RealizationStep: + """A semantic realization step (subject/predicate/object + morphology).""" + + subject: str + predicate: str + obj: str | None + tense: str | None + aspect: str | None + negated: bool + quantifier: str | None + move: str + + def as_dict(self) -> dict[str, Any]: + return { + "subject": self.subject, + "predicate": self.predicate, + "obj": self.obj, + "tense": self.tense, + "aspect": self.aspect, + "negated": self.negated, + "quantifier": self.quantifier, + "move": self.move, + } + + +@dataclass +class RealizationTrace: + """Full trace from one turn: word selections + realization steps.""" + + steps: list[WordSelectionStep] = field(default_factory=list) + realization_steps: list[RealizationStep] = field(default_factory=list) + surface: str = "" + + def as_dict(self) -> dict[str, Any]: + return { + "steps": [s.as_dict() for s in self.steps], + "realization_steps": [r.as_dict() for r in self.realization_steps], + "surface": self.surface, + } + + +def _morphology_summary(vocab: Any, word: str) -> dict[str, Any]: + """Extract morphology fields for a word, returning an empty dict if none.""" + entry = vocab.morphology_for_word(word) + if entry is None: + return {} + summary: dict[str, Any] = {} + # MorphologyEntry fields vary; collect any present attributes. + for attr in ("lemma", "surface", "tense", "aspect", "plural", "number", "negation", "person", "gender", "pos"): + value = getattr(entry, attr, None) + if value is not None: + summary[attr] = value + return summary + + +def _topk_candidates( + vocab: Any, + versor: np.ndarray, + candidate_indices: np.ndarray, + k: int = 5, +) -> tuple[tuple[str, float], ...]: + """Compute top-K candidates by CGA inner product over the candidate set. + + Vectorised via the diagonal Cl(4,1) metric — same kernel as + ``algebra.backend.vault_recall``. Exact, deterministic, no approximation. + Used only for tracing; never fed back into the realizer's surface. + """ + if len(candidate_indices) == 0: + return () + idx = np.asarray(candidate_indices, dtype=np.int64) + # Stack candidate versors into one (N, 32) matrix; the vocab stores + # them as a list of 32-vectors. + versors_list = [vocab._versors[int(i)] for i in idx] + M = np.asarray(versors_list, dtype=np.float32) + q = np.asarray(versor, dtype=np.float32).reshape(-1) + # Diagonal weighted dot-product, vectorised serial fold (same + # component order as scalar cga_inner so scores are bit-identical + # to the per-versor scan we replaced). + scores = np.zeros(M.shape[0], dtype=np.float32) + for c in range(M.shape[1]): + scores += (_CGA_INNER_METRIC[c] * M[:, c]) * q[c] + k_eff = max(1, min(int(k), scores.shape[0])) + if k_eff < scores.shape[0]: + cand = np.argpartition(-scores, k_eff - 1)[:k_eff] + else: + cand = np.arange(scores.shape[0]) + order = np.lexsort((cand, -scores[cand])) + cand = cand[order] + return tuple( + (vocab._words[int(idx[int(c)])], float(scores[int(c)])) + for c in cand + ) + + +def trace_realization( + runtime_or_pipeline: Any, + text: str, + *, + top_k: int = 5, + max_tokens: int | None = None, +) -> RealizationTrace: + """Run one chat turn (or pipeline turn) while tracing every word lookup. + + Accepts either a ``ChatRuntime`` (calls ``.chat``) or a + ``CognitiveTurnPipeline`` (calls ``.run``). A pipeline is preferred + because the pipeline path invokes ``realize_semantic`` even when the + runtime's unknown-domain gate fires, so realization steps are captured + regardless of grounding. + + Instruments ``generate.articulation._resolve_slot`` and + ``generate.realizer.realize_semantic`` for the duration of this call, + then restores them. Does NOT modify the realizer/articulation source. + """ + trace = RealizationTrace() + + from generate import articulation as articulation_mod + from generate import realizer as realizer_mod + + orig_resolve_slot = articulation_mod._resolve_slot + orig_candidate_indices = articulation_mod._candidate_indices + orig_surface_for_word = articulation_mod._surface_for_word + orig_realize_semantic = realizer_mod.realize_semantic + orig_resolve_obj = realizer_mod._resolve_obj + + # Track slot order within a single realize() call. Reset on every + # articulation.realize() entry; resolve_slot has no slot label itself, + # so we synthesize it from invocation order: subject, predicate, object. + slot_state: dict[str, int] = {"counter": 0} + _SLOT_ORDER = ("subject", "predicate", "object") + + def traced_resolve_slot( + versor: np.ndarray | None, + vocab: Any, + output_language: str, + ) -> str | None: + slot_idx = slot_state["counter"] + slot_state["counter"] = slot_idx + 1 + slot_name = _SLOT_ORDER[slot_idx] if slot_idx < len(_SLOT_ORDER) else f"slot_{slot_idx}" + if versor is None: + return None + cand = orig_candidate_indices(vocab, output_language) + chosen_word, _chosen_idx = vocab.nearest(versor, candidate_indices=cand) + top = _topk_candidates(vocab, versor, cand, k=top_k) + morph = _morphology_summary(vocab, chosen_word) + trace.steps.append( + WordSelectionStep( + slot=slot_name, + input_versor=np.asarray(versor, dtype=float).copy(), + top_candidates=top, + chosen=chosen_word, + morphology=morph, + output_language=output_language, + ) + ) + return orig_surface_for_word(vocab, chosen_word) + + # Reset slot counter at each realize() entry. Patch articulation.realize + # via a wrapper that resets the slot_state counter before delegating. + orig_realize = articulation_mod.realize + + def traced_realize(*args: Any, **kwargs: Any) -> Any: + slot_state["counter"] = 0 + return orig_realize(*args, **kwargs) + + def traced_realize_semantic(target: Any, graph: Any = None) -> Any: + plan = orig_realize_semantic(target, graph) + # Record the realization steps directly from the target/graph + # without re-running the realizer. + if target is not None and target.steps: + for step in target.steps: + obj = orig_resolve_obj(step, graph) if graph is not None else None + trace.realization_steps.append( + RealizationStep( + subject=step.subject, + predicate=step.predicate, + obj=obj, + tense=step.tense, + aspect=step.aspect, + negated=step.negated, + quantifier=step.quantifier, + move=step.move.value, + ) + ) + return plan + + articulation_mod._resolve_slot = traced_resolve_slot + articulation_mod.realize = traced_realize + realizer_mod.realize_semantic = traced_realize_semantic + + # Also patch the symbol referenced by the pipeline module, since it + # was imported by name at module load time. + try: + from core.cognition import pipeline as pipeline_mod + orig_pipeline_realize_semantic = pipeline_mod.realize_semantic + pipeline_mod.realize_semantic = traced_realize_semantic + except ImportError: + pipeline_mod = None + orig_pipeline_realize_semantic = None + + try: + if hasattr(runtime_or_pipeline, "run") and hasattr(runtime_or_pipeline, "runtime"): + # CognitiveTurnPipeline + result = runtime_or_pipeline.run(text, max_tokens=max_tokens) + trace.surface = result.articulation_surface or result.surface or "" + else: + # ChatRuntime + response = runtime_or_pipeline.chat(text, max_tokens=max_tokens) + trace.surface = response.articulation_surface or response.surface or "" + finally: + articulation_mod._resolve_slot = orig_resolve_slot + articulation_mod.realize = orig_realize + realizer_mod.realize_semantic = orig_realize_semantic + if pipeline_mod is not None and orig_pipeline_realize_semantic is not None: + pipeline_mod.realize_semantic = orig_pipeline_realize_semantic + + return trace diff --git a/core/cli.py b/core/cli.py index ccc5bb5e..a2985f01 100644 --- a/core/cli.py +++ b/core/cli.py @@ -57,6 +57,7 @@ _TEST_SUITES: dict[str, tuple[str, ...]] = { "tests/test_deterministic_hash.py", "tests/test_morphology_irregular.py", "tests/test_realizer_quantifier_agreement.py", + "tests/test_benchmarks_profiler.py", ), "teaching": ( "tests/test_reviewed_teaching_loop.py", diff --git a/evals/discourse_paragraph/__init__.py b/evals/discourse_paragraph/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/discourse_paragraph/contract.md b/evals/discourse_paragraph/contract.md new file mode 100644 index 00000000..882190a0 --- /dev/null +++ b/evals/discourse_paragraph/contract.md @@ -0,0 +1,64 @@ +# discourse_paragraph eval lane + +## What it measures + +Whether the deterministic realizer can produce **paragraph-scale** +output — multiple grammatical sentences joined by deterministic +discourse markers — from a multi-step ArticulationTarget. + +This is the first lane that stresses output longer than a single +3-word SVO sentence. It addresses the open scope item: +*"longer/more complex sentences and phrases for testing and proving +stuff"*. + +## Inputs + +Each case carries a `graph` (≥ 3 nodes), an ordered `steps` list +(`ASSERT` open, then `SEQUENCE` / `ELABORATE` / `CONTRAST`), and +acceptance constraints: + +```json +{ + "id": "DP-PUB_001", + "topic": "epistemic_chain", + "graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", + "predicate": "grounds", "obj": "knowledge"}, ...], + "edges": []}, + "steps": [{"node_id": "n1", "move": "ASSERT"}, ...], + "min_sentences": 4, + "max_sentences": 6, + "must_contain_subjects": ["wisdom", "knowledge", "evidence", "truth"], + "discourse_markers": ["furthermore", "next"] +} +``` + +## Scoring rubric + +Per case: + + - `paragraph_sentence_count` ≥ `min_sentences` (and ≤ `max_sentences`) + - `subject_coverage_rate` ≥ 0.75 + - `discourse_marker_present` — at least one expected marker emitted + - `replay_determinism` — running the case twice produces an + identical surface string + +Aggregate metrics: + + - `accuracy` — pass rate + - `mean_sentence_count` + - `mean_subject_coverage` + - `replay_determinism_rate` + +## Splits + +| Split | n | content | +|---|---|---| +| public/v1 | 12 | epistemic / scientific / creation / logic / ethics / linguistic / math / narrative / biology / physics + 2 contrast cases | +| holdouts/v1 | 5 | musical / social / computational / psychological / economic | +| dev | 1 | epistemic_chain smoke | + +## What this lane does NOT measure + +- Round-trip through `ChatRuntime` (the realizer is exercised + directly). See gaps.md. +- Factual correctness of the asserted propositions. diff --git a/evals/discourse_paragraph/dev/cases.jsonl b/evals/discourse_paragraph/dev/cases.jsonl new file mode 100644 index 00000000..140a060e --- /dev/null +++ b/evals/discourse_paragraph/dev/cases.jsonl @@ -0,0 +1 @@ +{"id": "DP-DEV_001", "topic": "epistemic_chain", "graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "grounds", "obj": "knowledge"}, {"node_id": "n2", "subject": "knowledge", "predicate": "requires", "obj": "evidence"}, {"node_id": "n3", "subject": "evidence", "predicate": "supports", "obj": "truth"}, {"node_id": "n4", "subject": "truth", "predicate": "reveals", "obj": "reality"}], "edges": []}, "steps": [{"node_id": "n1", "move": "ASSERT"}, {"node_id": "n2", "move": "SEQUENCE"}, {"node_id": "n3", "move": "ELABORATE"}, {"node_id": "n4", "move": "SEQUENCE"}], "min_sentences": 4, "must_contain_subjects": ["wisdom", "knowledge", "evidence", "truth"], "discourse_markers": ["furthermore", "next"], "max_sentences": 6} diff --git a/evals/discourse_paragraph/gaps.md b/evals/discourse_paragraph/gaps.md new file mode 100644 index 00000000..ec5896bc --- /dev/null +++ b/evals/discourse_paragraph/gaps.md @@ -0,0 +1,36 @@ +# discourse_paragraph — gaps + +## v1 (current) + +- Realizer-isolation lane: bypasses runtime grounding so the + paragraph claim is unconfounded by vault noise. +- Sentence-count window is intentionally generous + (`max_sentences = min + 2`) to tolerate small wrapping variance + from compound-clause folding in `realize_target` (CONJUNCTION / + COMPLEMENT / RELATIVE edges merge two steps into one sentence). +- Subject coverage threshold is 0.75, not 1.0 — exact-coverage + cases pass that bar comfortably but the slack lets a future + realizer change ship without rewriting cases. + +## Known gaps for v2 + +1. **No round-trip through the runtime.** v1 invokes the realizer + directly with a constructed `ArticulationTarget`. v2 should + feed the runtime real text inputs that *produce* the same + articulation target through `graph_from_intent` + + `plan_articulation`, end-to-end. +2. **No anaphora / pronoun reduction.** Every sentence carries + its subject explicitly. Pronominalisation deferred. +3. **No length scaling above 5 sentences.** v2 should push to + 10/20/50 sentences and measure per-sentence determinism. +4. **No grammaticality check per sentence.** v1 checks subject + coverage + discourse markers; v2 should run each emitted + sentence through grammatical_coverage's rubric. + +## Why this lane exists + +First lane that exercises paragraph-scale output. Every previous +fluency lane (Phase 5.1 + 5.4–5.7) operates on 3-word SVO probes. +The structural capability — folding multiple articulation steps +into a coherent paragraph with deterministic discourse markers — +was already in the realizer; this lane makes it measurable. diff --git a/evals/discourse_paragraph/holdouts/v1/cases.jsonl b/evals/discourse_paragraph/holdouts/v1/cases.jsonl new file mode 100644 index 00000000..f6d1dc52 --- /dev/null +++ b/evals/discourse_paragraph/holdouts/v1/cases.jsonl @@ -0,0 +1,5 @@ +{"id": "DP-HOLD_001", "topic": "musical_construction", "graph": {"nodes": [{"node_id": "n1", "subject": "note", "predicate": "composes", "obj": "chord"}, {"node_id": "n2", "subject": "chord", "predicate": "supports", "obj": "harmony"}, {"node_id": "n3", "subject": "harmony", "predicate": "yields", "obj": "phrase"}, {"node_id": "n4", "subject": "phrase", "predicate": "builds", "obj": "melody"}], "edges": []}, "steps": [{"node_id": "n1", "move": "ASSERT"}, {"node_id": "n2", "move": "SEQUENCE"}, {"node_id": "n3", "move": "ELABORATE"}, {"node_id": "n4", "move": "SEQUENCE"}], "min_sentences": 4, "must_contain_subjects": ["note", "chord", "harmony", "phrase"], "discourse_markers": ["furthermore", "next"], "max_sentences": 6} +{"id": "DP-HOLD_002", "topic": "social_structure", "graph": {"nodes": [{"node_id": "n1", "subject": "custom", "predicate": "grounds", "obj": "tradition"}, {"node_id": "n2", "subject": "tradition", "predicate": "supports", "obj": "institution"}, {"node_id": "n3", "subject": "institution", "predicate": "shapes", "obj": "society"}, {"node_id": "n4", "subject": "society", "predicate": "reveals", "obj": "culture"}], "edges": []}, "steps": [{"node_id": "n1", "move": "ASSERT"}, {"node_id": "n2", "move": "SEQUENCE"}, {"node_id": "n3", "move": "ELABORATE"}, {"node_id": "n4", "move": "SEQUENCE"}], "min_sentences": 4, "must_contain_subjects": ["custom", "tradition", "institution", "society"], "discourse_markers": ["furthermore", "next"], "max_sentences": 6} +{"id": "DP-HOLD_003", "topic": "computational_pipeline", "graph": {"nodes": [{"node_id": "n1", "subject": "input", "predicate": "drives", "obj": "computation"}, {"node_id": "n2", "subject": "computation", "predicate": "yields", "obj": "output"}, {"node_id": "n3", "subject": "output", "predicate": "supports", "obj": "decision"}], "edges": []}, "steps": [{"node_id": "n1", "move": "ASSERT"}, {"node_id": "n2", "move": "SEQUENCE"}, {"node_id": "n3", "move": "ELABORATE"}], "min_sentences": 3, "must_contain_subjects": ["input", "computation", "output"], "discourse_markers": ["furthermore", "next"], "max_sentences": 5} +{"id": "DP-HOLD_004", "topic": "psychological_development", "graph": {"nodes": [{"node_id": "n1", "subject": "sensation", "predicate": "grounds", "obj": "perception"}, {"node_id": "n2", "subject": "perception", "predicate": "supports", "obj": "memory"}, {"node_id": "n3", "subject": "memory", "predicate": "yields", "obj": "learning"}, {"node_id": "n4", "subject": "learning", "predicate": "shapes", "obj": "behavior"}, {"node_id": "n5", "subject": "behavior", "predicate": "reveals", "obj": "character"}], "edges": []}, "steps": [{"node_id": "n1", "move": "ASSERT"}, {"node_id": "n2", "move": "SEQUENCE"}, {"node_id": "n3", "move": "ELABORATE"}, {"node_id": "n4", "move": "SEQUENCE"}, {"node_id": "n5", "move": "ELABORATE"}], "min_sentences": 5, "must_contain_subjects": ["sensation", "perception", "memory", "learning", "behavior"], "discourse_markers": ["furthermore", "next"], "max_sentences": 7} +{"id": "DP-HOLD_005", "topic": "economic_flow", "graph": {"nodes": [{"node_id": "n1", "subject": "labor", "predicate": "yields", "obj": "value"}, {"node_id": "n2", "subject": "value", "predicate": "supports", "obj": "exchange"}, {"node_id": "n3", "subject": "exchange", "predicate": "drives", "obj": "growth"}], "edges": []}, "steps": [{"node_id": "n1", "move": "ASSERT"}, {"node_id": "n2", "move": "SEQUENCE"}, {"node_id": "n3", "move": "ELABORATE"}], "min_sentences": 3, "must_contain_subjects": ["labor", "value", "exchange"], "discourse_markers": ["furthermore", "next"], "max_sentences": 5} diff --git a/evals/discourse_paragraph/public/v1/cases.jsonl b/evals/discourse_paragraph/public/v1/cases.jsonl new file mode 100644 index 00000000..75d4cea2 --- /dev/null +++ b/evals/discourse_paragraph/public/v1/cases.jsonl @@ -0,0 +1,12 @@ +{"id": "DP-PUB_001", "topic": "epistemic_chain", "graph": {"nodes": [{"node_id": "n1", "subject": "wisdom", "predicate": "grounds", "obj": "knowledge"}, {"node_id": "n2", "subject": "knowledge", "predicate": "requires", "obj": "evidence"}, {"node_id": "n3", "subject": "evidence", "predicate": "supports", "obj": "truth"}, {"node_id": "n4", "subject": "truth", "predicate": "reveals", "obj": "reality"}], "edges": []}, "steps": [{"node_id": "n1", "move": "ASSERT"}, {"node_id": "n2", "move": "SEQUENCE"}, {"node_id": "n3", "move": "ELABORATE"}, {"node_id": "n4", "move": "SEQUENCE"}], "min_sentences": 4, "must_contain_subjects": ["wisdom", "knowledge", "evidence", "truth"], "discourse_markers": ["furthermore", "next"], "max_sentences": 6} +{"id": "DP-PUB_002", "topic": "scientific_method", "graph": {"nodes": [{"node_id": "n1", "subject": "observation", "predicate": "grounds", "obj": "hypothesis"}, {"node_id": "n2", "subject": "hypothesis", "predicate": "implies", "obj": "prediction"}, {"node_id": "n3", "subject": "prediction", "predicate": "follows", "obj": "experiment"}, {"node_id": "n4", "subject": "experiment", "predicate": "supports", "obj": "theory"}, {"node_id": "n5", "subject": "theory", "predicate": "entails", "obj": "explanation"}], "edges": []}, "steps": [{"node_id": "n1", "move": "ASSERT"}, {"node_id": "n2", "move": "ELABORATE"}, {"node_id": "n3", "move": "SEQUENCE"}, {"node_id": "n4", "move": "ELABORATE"}, {"node_id": "n5", "move": "SEQUENCE"}], "min_sentences": 5, "must_contain_subjects": ["observation", "hypothesis", "prediction", "experiment", "theory"], "discourse_markers": ["furthermore", "next"], "max_sentences": 7} +{"id": "DP-PUB_003", "topic": "creation_arc", "graph": {"nodes": [{"node_id": "n1", "subject": "light", "predicate": "precedes", "obj": "form"}, {"node_id": "n2", "subject": "form", "predicate": "grounds", "obj": "matter"}, {"node_id": "n3", "subject": "matter", "predicate": "supports", "obj": "structure"}, {"node_id": "n4", "subject": "structure", "predicate": "reveals", "obj": "order"}], "edges": []}, "steps": [{"node_id": "n1", "move": "ASSERT"}, {"node_id": "n2", "move": "SEQUENCE"}, {"node_id": "n3", "move": "ELABORATE"}, {"node_id": "n4", "move": "SEQUENCE"}], "min_sentences": 4, "must_contain_subjects": ["light", "form", "matter", "structure"], "discourse_markers": ["furthermore", "next"], "max_sentences": 6} +{"id": "DP-PUB_004", "topic": "logical_dependency", "graph": {"nodes": [{"node_id": "n1", "subject": "premise", "predicate": "supports", "obj": "conclusion"}, {"node_id": "n2", "subject": "conclusion", "predicate": "requires", "obj": "validity"}, {"node_id": "n3", "subject": "validity", "predicate": "entails", "obj": "soundness"}], "edges": []}, "steps": [{"node_id": "n1", "move": "ASSERT"}, {"node_id": "n2", "move": "SEQUENCE"}, {"node_id": "n3", "move": "ELABORATE"}], "min_sentences": 3, "must_contain_subjects": ["premise", "conclusion", "validity"], "discourse_markers": ["furthermore", "next"], "max_sentences": 5} +{"id": "DP-PUB_005", "topic": "ethical_grounding", "graph": {"nodes": [{"node_id": "n1", "subject": "virtue", "predicate": "grounds", "obj": "action"}, {"node_id": "n2", "subject": "action", "predicate": "requires", "obj": "intention"}, {"node_id": "n3", "subject": "intention", "predicate": "supports", "obj": "consequence"}, {"node_id": "n4", "subject": "consequence", "predicate": "reveals", "obj": "character"}], "edges": []}, "steps": [{"node_id": "n1", "move": "ASSERT"}, {"node_id": "n2", "move": "SEQUENCE"}, {"node_id": "n3", "move": "ELABORATE"}, {"node_id": "n4", "move": "SEQUENCE"}], "min_sentences": 4, "must_contain_subjects": ["virtue", "action", "intention", "consequence"], "discourse_markers": ["furthermore", "next"], "max_sentences": 6} +{"id": "DP-PUB_006", "topic": "linguistic_layers", "graph": {"nodes": [{"node_id": "n1", "subject": "sound", "predicate": "grounds", "obj": "phoneme"}, {"node_id": "n2", "subject": "phoneme", "predicate": "supports", "obj": "morpheme"}, {"node_id": "n3", "subject": "morpheme", "predicate": "builds", "obj": "word"}, {"node_id": "n4", "subject": "word", "predicate": "composes", "obj": "sentence"}, {"node_id": "n5", "subject": "sentence", "predicate": "conveys", "obj": "meaning"}], "edges": []}, "steps": [{"node_id": "n1", "move": "ASSERT"}, {"node_id": "n2", "move": "SEQUENCE"}, {"node_id": "n3", "move": "ELABORATE"}, {"node_id": "n4", "move": "SEQUENCE"}, {"node_id": "n5", "move": "ELABORATE"}], "min_sentences": 5, "must_contain_subjects": ["sound", "phoneme", "morpheme", "word", "sentence"], "discourse_markers": ["furthermore", "next"], "max_sentences": 7} +{"id": "DP-PUB_007", "topic": "mathematical_chain", "graph": {"nodes": [{"node_id": "n1", "subject": "axiom", "predicate": "grounds", "obj": "theorem"}, {"node_id": "n2", "subject": "theorem", "predicate": "entails", "obj": "corollary"}, {"node_id": "n3", "subject": "corollary", "predicate": "supports", "obj": "application"}, {"node_id": "n4", "subject": "application", "predicate": "yields", "obj": "insight"}], "edges": []}, "steps": [{"node_id": "n1", "move": "ASSERT"}, {"node_id": "n2", "move": "ELABORATE"}, {"node_id": "n3", "move": "SEQUENCE"}, {"node_id": "n4", "move": "SEQUENCE"}], "min_sentences": 4, "must_contain_subjects": ["axiom", "theorem", "corollary", "application"], "discourse_markers": ["furthermore", "next"], "max_sentences": 6} +{"id": "DP-PUB_008", "topic": "narrative_progression", "graph": {"nodes": [{"node_id": "n1", "subject": "conflict", "predicate": "drives", "obj": "tension"}, {"node_id": "n2", "subject": "tension", "predicate": "precedes", "obj": "climax"}, {"node_id": "n3", "subject": "climax", "predicate": "yields", "obj": "resolution"}, {"node_id": "n4", "subject": "resolution", "predicate": "reveals", "obj": "theme"}], "edges": []}, "steps": [{"node_id": "n1", "move": "ASSERT"}, {"node_id": "n2", "move": "SEQUENCE"}, {"node_id": "n3", "move": "ELABORATE"}, {"node_id": "n4", "move": "SEQUENCE"}], "min_sentences": 4, "must_contain_subjects": ["conflict", "tension", "climax", "resolution"], "discourse_markers": ["furthermore", "next"], "max_sentences": 6} +{"id": "DP-PUB_009", "topic": "biological_hierarchy", "graph": {"nodes": [{"node_id": "n1", "subject": "gene", "predicate": "encodes", "obj": "protein"}, {"node_id": "n2", "subject": "protein", "predicate": "builds", "obj": "cell"}, {"node_id": "n3", "subject": "cell", "predicate": "composes", "obj": "tissue"}, {"node_id": "n4", "subject": "tissue", "predicate": "forms", "obj": "organ"}, {"node_id": "n5", "subject": "organ", "predicate": "supports", "obj": "organism"}], "edges": []}, "steps": [{"node_id": "n1", "move": "ASSERT"}, {"node_id": "n2", "move": "SEQUENCE"}, {"node_id": "n3", "move": "ELABORATE"}, {"node_id": "n4", "move": "SEQUENCE"}, {"node_id": "n5", "move": "ELABORATE"}], "min_sentences": 5, "must_contain_subjects": ["gene", "protein", "cell", "tissue", "organ"], "discourse_markers": ["furthermore", "next"], "max_sentences": 7} +{"id": "DP-PUB_010", "topic": "physical_causation", "graph": {"nodes": [{"node_id": "n1", "subject": "force", "predicate": "drives", "obj": "motion"}, {"node_id": "n2", "subject": "motion", "predicate": "transfers", "obj": "energy"}, {"node_id": "n3", "subject": "energy", "predicate": "yields", "obj": "heat"}, {"node_id": "n4", "subject": "heat", "predicate": "raises", "obj": "temperature"}], "edges": []}, "steps": [{"node_id": "n1", "move": "ASSERT"}, {"node_id": "n2", "move": "ELABORATE"}, {"node_id": "n3", "move": "SEQUENCE"}, {"node_id": "n4", "move": "SEQUENCE"}], "min_sentences": 4, "must_contain_subjects": ["force", "motion", "energy", "heat"], "discourse_markers": ["furthermore", "next"], "max_sentences": 6} +{"id": "DP-PUB_011", "topic": "contrastive_definitions", "graph": {"nodes": [{"node_id": "n1", "subject": "knowledge", "predicate": "requires", "obj": "evidence"}, {"node_id": "n2", "subject": "belief", "predicate": "requires", "obj": "trust"}, {"node_id": "n3", "subject": "wisdom", "predicate": "grounds", "obj": "judgment"}], "edges": []}, "steps": [{"node_id": "n1", "move": "ASSERT"}, {"node_id": "n2", "move": "CONTRAST"}, {"node_id": "n3", "move": "ELABORATE"}], "min_sentences": 3, "must_contain_subjects": ["knowledge", "belief", "wisdom"], "discourse_markers": ["furthermore", "in contrast"], "max_sentences": 5} +{"id": "DP-PUB_012", "topic": "method_contrast", "graph": {"nodes": [{"node_id": "n1", "subject": "deduction", "predicate": "yields", "obj": "certainty"}, {"node_id": "n2", "subject": "induction", "predicate": "yields", "obj": "probability"}, {"node_id": "n3", "subject": "abduction", "predicate": "yields", "obj": "explanation"}], "edges": []}, "steps": [{"node_id": "n1", "move": "ASSERT"}, {"node_id": "n2", "move": "CONTRAST"}, {"node_id": "n3", "move": "ELABORATE"}], "min_sentences": 3, "must_contain_subjects": ["deduction", "induction", "abduction"], "discourse_markers": ["furthermore", "in contrast"], "max_sentences": 5} diff --git a/evals/discourse_paragraph/results/v1_public_20260517T044638Z.json b/evals/discourse_paragraph/results/v1_public_20260517T044638Z.json new file mode 100644 index 00000000..bd19aab3 --- /dev/null +++ b/evals/discourse_paragraph/results/v1_public_20260517T044638Z.json @@ -0,0 +1,184 @@ +{ + "cases": [ + { + "discourse_markers_found": [ + "furthermore", + "next" + ], + "failure_reasons": [], + "id": "DP-PUB_001", + "passed": true, + "replay_match": true, + "sentence_count": 4, + "subject_coverage": 1.0, + "surface": "wisdom grounds knowledge. next, knowledge requires evidence. furthermore, evidence supports truth. next, truth reveals reality.", + "topic": "epistemic_chain" + }, + { + "discourse_markers_found": [ + "furthermore", + "next" + ], + "failure_reasons": [], + "id": "DP-PUB_002", + "passed": true, + "replay_match": true, + "sentence_count": 5, + "subject_coverage": 1.0, + "surface": "observation grounds hypothesis. furthermore, hypothesis implies prediction. next, prediction follows experiment. furthermore, experiment supports theory. next, theory entails explanation.", + "topic": "scientific_method" + }, + { + "discourse_markers_found": [ + "furthermore", + "next" + ], + "failure_reasons": [], + "id": "DP-PUB_003", + "passed": true, + "replay_match": true, + "sentence_count": 4, + "subject_coverage": 1.0, + "surface": "light precedes form. next, form grounds matter. furthermore, matter supports structure. next, structure reveals order.", + "topic": "creation_arc" + }, + { + "discourse_markers_found": [ + "furthermore", + "next" + ], + "failure_reasons": [], + "id": "DP-PUB_004", + "passed": true, + "replay_match": true, + "sentence_count": 3, + "subject_coverage": 1.0, + "surface": "premise supports conclusion. next, conclusion requires validity. furthermore, validity entails soundness.", + "topic": "logical_dependency" + }, + { + "discourse_markers_found": [ + "furthermore", + "next" + ], + "failure_reasons": [], + "id": "DP-PUB_005", + "passed": true, + "replay_match": true, + "sentence_count": 4, + "subject_coverage": 1.0, + "surface": "virtue grounds action. next, action requires intention. furthermore, intention supports consequence. next, consequence reveals character.", + "topic": "ethical_grounding" + }, + { + "discourse_markers_found": [ + "furthermore", + "next" + ], + "failure_reasons": [], + "id": "DP-PUB_006", + "passed": true, + "replay_match": true, + "sentence_count": 5, + "subject_coverage": 1.0, + "surface": "sound grounds phoneme. next, phoneme supports morpheme. furthermore, morpheme builds word. next, word composes sentence. furthermore, sentence conveys meaning.", + "topic": "linguistic_layers" + }, + { + "discourse_markers_found": [ + "furthermore", + "next" + ], + "failure_reasons": [], + "id": "DP-PUB_007", + "passed": true, + "replay_match": true, + "sentence_count": 4, + "subject_coverage": 1.0, + "surface": "axiom grounds theorem. furthermore, theorem entails corollary. next, corollary supports application. next, application yields insight.", + "topic": "mathematical_chain" + }, + { + "discourse_markers_found": [ + "furthermore", + "next" + ], + "failure_reasons": [], + "id": "DP-PUB_008", + "passed": true, + "replay_match": true, + "sentence_count": 4, + "subject_coverage": 1.0, + "surface": "conflict drives tension. next, tension precedes climax. furthermore, climax yields resolution. next, resolution reveals theme.", + "topic": "narrative_progression" + }, + { + "discourse_markers_found": [ + "furthermore", + "next" + ], + "failure_reasons": [], + "id": "DP-PUB_009", + "passed": true, + "replay_match": true, + "sentence_count": 5, + "subject_coverage": 1.0, + "surface": "gene encodes protein. next, protein builds cell. furthermore, cell composes tissue. next, tissue forms organ. furthermore, organ supports organism.", + "topic": "biological_hierarchy" + }, + { + "discourse_markers_found": [ + "furthermore", + "next" + ], + "failure_reasons": [], + "id": "DP-PUB_010", + "passed": true, + "replay_match": true, + "sentence_count": 4, + "subject_coverage": 1.0, + "surface": "force drives motion. furthermore, motion transfers energy. next, energy yields heat. next, heat raises temperature.", + "topic": "physical_causation" + }, + { + "discourse_markers_found": [ + "furthermore", + "in contrast" + ], + "failure_reasons": [], + "id": "DP-PUB_011", + "passed": true, + "replay_match": true, + "sentence_count": 3, + "subject_coverage": 1.0, + "surface": "knowledge requires evidence. in contrast, belief requires trust. furthermore, wisdom grounds judgment.", + "topic": "contrastive_definitions" + }, + { + "discourse_markers_found": [ + "furthermore", + "in contrast" + ], + "failure_reasons": [], + "id": "DP-PUB_012", + "passed": true, + "replay_match": true, + "sentence_count": 3, + "subject_coverage": 1.0, + "surface": "deduction yields certainty. in contrast, induction yields probability. furthermore, abduction yields explanation.", + "topic": "method_contrast" + } + ], + "lane": "discourse_paragraph", + "metrics": { + "accuracy": 1.0, + "mean_sentence_count": 4.0, + "mean_subject_coverage": 1.0, + "passed": 12, + "replay_determinism_rate": 1.0, + "total": 12 + }, + "split": "public", + "timestamp": "2026-05-17T04:46:38.277091+00:00", + "version": "v1" +} diff --git a/evals/discourse_paragraph/results/v1_public_20260517T044709Z.json b/evals/discourse_paragraph/results/v1_public_20260517T044709Z.json new file mode 100644 index 00000000..b38b8c92 --- /dev/null +++ b/evals/discourse_paragraph/results/v1_public_20260517T044709Z.json @@ -0,0 +1,184 @@ +{ + "cases": [ + { + "discourse_markers_found": [ + "furthermore", + "next" + ], + "failure_reasons": [], + "id": "DP-PUB_001", + "passed": true, + "replay_match": true, + "sentence_count": 4, + "subject_coverage": 1.0, + "surface": "Wisdom grounds knowledge. Next, knowledge requires evidence. Furthermore, evidence supports truth. Next, truth reveals reality.", + "topic": "epistemic_chain" + }, + { + "discourse_markers_found": [ + "furthermore", + "next" + ], + "failure_reasons": [], + "id": "DP-PUB_002", + "passed": true, + "replay_match": true, + "sentence_count": 5, + "subject_coverage": 1.0, + "surface": "Observation grounds hypothesis. Furthermore, hypothesis implies prediction. Next, prediction follows experiment. Furthermore, experiment supports theory. Next, theory entails explanation.", + "topic": "scientific_method" + }, + { + "discourse_markers_found": [ + "furthermore", + "next" + ], + "failure_reasons": [], + "id": "DP-PUB_003", + "passed": true, + "replay_match": true, + "sentence_count": 4, + "subject_coverage": 1.0, + "surface": "Light precedes form. Next, form grounds matter. Furthermore, matter supports structure. Next, structure reveals order.", + "topic": "creation_arc" + }, + { + "discourse_markers_found": [ + "furthermore", + "next" + ], + "failure_reasons": [], + "id": "DP-PUB_004", + "passed": true, + "replay_match": true, + "sentence_count": 3, + "subject_coverage": 1.0, + "surface": "Premise supports conclusion. Next, conclusion requires validity. Furthermore, validity entails soundness.", + "topic": "logical_dependency" + }, + { + "discourse_markers_found": [ + "furthermore", + "next" + ], + "failure_reasons": [], + "id": "DP-PUB_005", + "passed": true, + "replay_match": true, + "sentence_count": 4, + "subject_coverage": 1.0, + "surface": "Virtue grounds action. Next, action requires intention. Furthermore, intention supports consequence. Next, consequence reveals character.", + "topic": "ethical_grounding" + }, + { + "discourse_markers_found": [ + "furthermore", + "next" + ], + "failure_reasons": [], + "id": "DP-PUB_006", + "passed": true, + "replay_match": true, + "sentence_count": 5, + "subject_coverage": 1.0, + "surface": "Sound grounds phoneme. Next, phoneme supports morpheme. Furthermore, morpheme builds word. Next, word composes sentence. Furthermore, sentence conveys meaning.", + "topic": "linguistic_layers" + }, + { + "discourse_markers_found": [ + "furthermore", + "next" + ], + "failure_reasons": [], + "id": "DP-PUB_007", + "passed": true, + "replay_match": true, + "sentence_count": 4, + "subject_coverage": 1.0, + "surface": "Axiom grounds theorem. Furthermore, theorem entails corollary. Next, corollary supports application. Next, application yields insight.", + "topic": "mathematical_chain" + }, + { + "discourse_markers_found": [ + "furthermore", + "next" + ], + "failure_reasons": [], + "id": "DP-PUB_008", + "passed": true, + "replay_match": true, + "sentence_count": 4, + "subject_coverage": 1.0, + "surface": "Conflict drives tension. Next, tension precedes climax. Furthermore, climax yields resolution. Next, resolution reveals theme.", + "topic": "narrative_progression" + }, + { + "discourse_markers_found": [ + "furthermore", + "next" + ], + "failure_reasons": [], + "id": "DP-PUB_009", + "passed": true, + "replay_match": true, + "sentence_count": 5, + "subject_coverage": 1.0, + "surface": "Gene encodes protein. Next, protein builds cell. Furthermore, cell composes tissue. Next, tissue forms organ. Furthermore, organ supports organism.", + "topic": "biological_hierarchy" + }, + { + "discourse_markers_found": [ + "furthermore", + "next" + ], + "failure_reasons": [], + "id": "DP-PUB_010", + "passed": true, + "replay_match": true, + "sentence_count": 4, + "subject_coverage": 1.0, + "surface": "Force drives motion. Furthermore, motion transfers energy. Next, energy yields heat. Next, heat raises temperature.", + "topic": "physical_causation" + }, + { + "discourse_markers_found": [ + "furthermore", + "in contrast" + ], + "failure_reasons": [], + "id": "DP-PUB_011", + "passed": true, + "replay_match": true, + "sentence_count": 3, + "subject_coverage": 1.0, + "surface": "Knowledge requires evidence. In contrast, belief requires trust. Furthermore, wisdom grounds judgment.", + "topic": "contrastive_definitions" + }, + { + "discourse_markers_found": [ + "furthermore", + "in contrast" + ], + "failure_reasons": [], + "id": "DP-PUB_012", + "passed": true, + "replay_match": true, + "sentence_count": 3, + "subject_coverage": 1.0, + "surface": "Deduction yields certainty. In contrast, induction yields probability. Furthermore, abduction yields explanation.", + "topic": "method_contrast" + } + ], + "lane": "discourse_paragraph", + "metrics": { + "accuracy": 1.0, + "mean_sentence_count": 4.0, + "mean_subject_coverage": 1.0, + "passed": 12, + "replay_determinism_rate": 1.0, + "total": 12 + }, + "split": "public", + "timestamp": "2026-05-17T04:47:09.206712+00:00", + "version": "v1" +} diff --git a/evals/discourse_paragraph/runner.py b/evals/discourse_paragraph/runner.py new file mode 100644 index 00000000..959f1aa1 --- /dev/null +++ b/evals/discourse_paragraph/runner.py @@ -0,0 +1,174 @@ +"""discourse_paragraph eval lane runner. + +Exercises paragraph-scale realization: given a multi-step +ArticulationTarget, the deterministic realizer should produce a +multi-sentence surface with discourse markers (next, furthermore, +in contrast) and full subject coverage. + +Bypasses ChatRuntime grounding so the paragraph claim is isolated +to the realizer. Runtime round-tripping is named as a v2 gap. + +Conforms to the framework interface: run_lane(cases, config=None) -> report. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any + +from generate.graph_planner import ( + ArticulationStep, + ArticulationTarget, + GraphEdge, + GraphNode, + PropositionGraph, + Relation, + RhetoricalMove, +) +from generate.intent import IntentTag +from generate.realizer import realize_target + + +@dataclass(slots=True) +class LaneReport: + metrics: dict[str, Any] = field(default_factory=dict) + case_details: list[dict[str, Any]] = field(default_factory=list) + + +_SENTENCE_SPLIT_RE = re.compile(r"[.!?]\s+|[.!?]$") + + +def _sentence_count(surface: str) -> int: + if not surface.strip(): + return 0 + parts = [p for p in _SENTENCE_SPLIT_RE.split(surface) if p.strip()] + return len(parts) + + +def _build_target_from_case(case: dict[str, Any]) -> tuple[ArticulationTarget, PropositionGraph]: + nodes_data = case["graph"]["nodes"] + edges_data = case["graph"].get("edges", []) + nodes = tuple( + GraphNode( + node_id=nd["node_id"], + subject=nd["subject"], + predicate=nd["predicate"], + obj=nd["obj"], + source_intent=IntentTag.UNKNOWN, + ) + for nd in nodes_data + ) + edges = tuple( + GraphEdge( + source=e["source"], + target=e["target"], + relation=Relation[e.get("relation", "SEQUENCE").upper()], + ) + for e in edges_data + ) + graph = PropositionGraph(nodes=nodes, edges=edges) + by_id = {n.node_id: n for n in nodes} + steps = tuple( + ArticulationStep( + node_id=s["node_id"], + subject=by_id[s["node_id"]].subject, + predicate=by_id[s["node_id"]].predicate, + move=RhetoricalMove[s["move"].upper()], + ) + for s in case["steps"] + ) + target = ArticulationTarget(steps=steps, source_intent=IntentTag.UNKNOWN) + return target, graph + + +def _score_case(case: dict[str, Any]) -> dict[str, Any]: + target, graph = _build_target_from_case(case) + plan_1 = realize_target(target, graph) + plan_2 = realize_target(target, graph) + surface = plan_1.surface + surface_lower = surface.lower() + + failures: list[str] = [] + sent_count = _sentence_count(surface) + min_sentences = int(case["min_sentences"]) + max_sentences = int(case.get("max_sentences", min_sentences + 2)) + if sent_count < min_sentences: + failures.append(f"sentence_count {sent_count} < min {min_sentences}") + if sent_count > max_sentences: + failures.append(f"sentence_count {sent_count} > max {max_sentences}") + + must_contain = case.get("must_contain_subjects", []) + present = [s for s in must_contain if s.lower() in surface_lower] + coverage = len(present) / max(1, len(must_contain)) + if coverage < 0.75: + missing = [s for s in must_contain if s.lower() not in surface_lower] + failures.append(f"subject_coverage {coverage:.2f} < 0.75; missing={missing}") + + expected_markers = case.get("discourse_markers", []) + if expected_markers: + found = [m for m in expected_markers if m.lower() in surface_lower] + if not found: + failures.append( + f"no discourse marker present; expected one of {expected_markers}" + ) + else: + found = [] + + # Sentence-initial capitalization (G4): every sentence-leading + # alphabetic character must be uppercase. This is the gate that + # turned "wisdom grounds knowledge." into "Wisdom grounds + # knowledge." — addresses the open scope item. + sentences = [p.strip() for p in _SENTENCE_SPLIT_RE.split(surface) if p.strip()] + badly_cased: list[str] = [] + for sent in sentences: + for ch in sent: + if ch.isalpha(): + if not ch.isupper(): + badly_cased.append(sent[:30]) + break + if badly_cased: + failures.append( + f"sentence-initial capitalization missing in {len(badly_cased)} " + f"sentence(s): {badly_cased}" + ) + + replay_match = plan_1.surface == plan_2.surface + if not replay_match: + failures.append("replay determinism broken: surfaces differ") + + passed = not failures + return { + "id": case["id"], + "topic": case.get("topic", ""), + "passed": passed, + "surface": surface, + "sentence_count": sent_count, + "subject_coverage": coverage, + "discourse_markers_found": found, + "replay_match": replay_match, + "failure_reasons": failures, + } + + +def run_lane(cases: list[dict[str, Any]], *, config: Any = None) -> LaneReport: + details = [_score_case(c) for c in cases] + total = len(details) + passed = sum(1 for d in details if d["passed"]) + return LaneReport( + metrics={ + "total": total, + "passed": passed, + "accuracy": round(passed / total, 4) if total else 0.0, + "mean_sentence_count": round( + sum(d["sentence_count"] for d in details) / max(1, total), 3 + ), + "mean_subject_coverage": round( + sum(d["subject_coverage"] for d in details) / max(1, total), 4 + ), + "replay_determinism_rate": round( + sum(1 for d in details if d["replay_match"]) / max(1, total), 4 + ), + }, + case_details=details, + ) diff --git a/generate/realizer.py b/generate/realizer.py index bdcba519..c1851e1a 100644 --- a/generate/realizer.py +++ b/generate/realizer.py @@ -40,6 +40,44 @@ class RealizedFragment: } +def _capitalize_sentence(s: str) -> str: + """Capitalize the first alphabetic character of a sentence. + + Skips leading whitespace/punctuation so fragments that start with + discourse markers ("next, knowledge…") still emit a capital first + letter ("Next, knowledge…") at the sentence boundary. Leaves the + rest of the string untouched — proper nouns and embedded all-caps + tokens are preserved. + """ + if not s: + return s + for i, ch in enumerate(s): + if ch.isalpha(): + return s[:i] + ch.upper() + s[i + 1:] + return s + + +def _join_as_paragraph(fragments: list["RealizedFragment"]) -> str: + """Join fragments into a paragraph with sentence-initial capitalization. + + Each fragment becomes one sentence; sentence-initial letters are + capitalized; the paragraph ends with a single terminal period. + """ + if not fragments: + return "" + pieces: list[str] = [] + for f in fragments: + s = f.surface.strip() + if not s: + continue + s = _capitalize_sentence(s) + pieces.append(s) + joined = ". ".join(pieces) + if joined and not joined.endswith("."): + joined += "." + return joined + + @dataclass(frozen=True, slots=True) class RealizedPlan: fragments: tuple[RealizedFragment, ...] @@ -106,10 +144,7 @@ def realize_semantic( surface=surface, )) - joined = ". ".join(f.surface for f in fragments) - if joined and not joined.endswith("."): - joined += "." - + joined = _join_as_paragraph(fragments) return RealizedPlan(fragments=tuple(fragments), surface=joined) @@ -208,10 +243,7 @@ def realize_target( ) ) - joined = ". ".join(f.surface for f in fragments) - if joined and not joined.endswith("."): - joined += "." - + joined = _join_as_paragraph(fragments) return RealizedPlan(fragments=tuple(fragments), surface=joined) diff --git a/scripts/generate_discourse_paragraph.py b/scripts/generate_discourse_paragraph.py new file mode 100644 index 00000000..fe586219 --- /dev/null +++ b/scripts/generate_discourse_paragraph.py @@ -0,0 +1,273 @@ +"""Generate cases for the discourse_paragraph benchmark lane. + +Tests that the realizer can produce **multi-sentence paragraph-scale +output** from chained propositions, given a multi-step +ArticulationTarget with rhetorical moves (SEQUENCE, ELABORATE, +CONTRAST). Each case stresses paragraph length, subject coverage, +discourse-marker presence, and deterministic replay. + +Each case carries: + - a graph of N ≥ 3 nodes (subject-predicate-object triples) + - an ordered move list ([ASSERT, SEQUENCE, ELABORATE, ...]) + - acceptance constraints (min_sentences, must_contain_subjects, + discourse_markers) + +Topics are designed to be **structurally rich** — every case is more +than a 3-word SVO probe. + +Run: + .venv/bin/python scripts/generate_discourse_paragraph.py +""" + +from __future__ import annotations + +import json +from pathlib import Path + + +# Each topic: ordered triples + ordered rhetorical moves matching length. +# Moves: ASSERT (open), SEQUENCE (next step), ELABORATE (furthermore), +# CONTRAST (in contrast), CORRECT (correction). See +# generate.templates._MOVE_TEMPLATES for emitted discourse markers. +PUBLIC_TOPICS: list[dict] = [ + { + "topic": "epistemic_chain", + "triples": [ + ("wisdom", "grounds", "knowledge"), + ("knowledge", "requires", "evidence"), + ("evidence", "supports", "truth"), + ("truth", "reveals", "reality"), + ], + "moves": ["ASSERT", "SEQUENCE", "ELABORATE", "SEQUENCE"], + }, + { + "topic": "scientific_method", + "triples": [ + ("observation", "grounds", "hypothesis"), + ("hypothesis", "implies", "prediction"), + ("prediction", "follows", "experiment"), + ("experiment", "supports", "theory"), + ("theory", "entails", "explanation"), + ], + "moves": ["ASSERT", "ELABORATE", "SEQUENCE", "ELABORATE", "SEQUENCE"], + }, + { + "topic": "creation_arc", + "triples": [ + ("light", "precedes", "form"), + ("form", "grounds", "matter"), + ("matter", "supports", "structure"), + ("structure", "reveals", "order"), + ], + "moves": ["ASSERT", "SEQUENCE", "ELABORATE", "SEQUENCE"], + }, + { + "topic": "logical_dependency", + "triples": [ + ("premise", "supports", "conclusion"), + ("conclusion", "requires", "validity"), + ("validity", "entails", "soundness"), + ], + "moves": ["ASSERT", "SEQUENCE", "ELABORATE"], + }, + { + "topic": "ethical_grounding", + "triples": [ + ("virtue", "grounds", "action"), + ("action", "requires", "intention"), + ("intention", "supports", "consequence"), + ("consequence", "reveals", "character"), + ], + "moves": ["ASSERT", "SEQUENCE", "ELABORATE", "SEQUENCE"], + }, + { + "topic": "linguistic_layers", + "triples": [ + ("sound", "grounds", "phoneme"), + ("phoneme", "supports", "morpheme"), + ("morpheme", "builds", "word"), + ("word", "composes", "sentence"), + ("sentence", "conveys", "meaning"), + ], + "moves": ["ASSERT", "SEQUENCE", "ELABORATE", "SEQUENCE", "ELABORATE"], + }, + { + "topic": "mathematical_chain", + "triples": [ + ("axiom", "grounds", "theorem"), + ("theorem", "entails", "corollary"), + ("corollary", "supports", "application"), + ("application", "yields", "insight"), + ], + "moves": ["ASSERT", "ELABORATE", "SEQUENCE", "SEQUENCE"], + }, + { + "topic": "narrative_progression", + "triples": [ + ("conflict", "drives", "tension"), + ("tension", "precedes", "climax"), + ("climax", "yields", "resolution"), + ("resolution", "reveals", "theme"), + ], + "moves": ["ASSERT", "SEQUENCE", "ELABORATE", "SEQUENCE"], + }, + { + "topic": "biological_hierarchy", + "triples": [ + ("gene", "encodes", "protein"), + ("protein", "builds", "cell"), + ("cell", "composes", "tissue"), + ("tissue", "forms", "organ"), + ("organ", "supports", "organism"), + ], + "moves": ["ASSERT", "SEQUENCE", "ELABORATE", "SEQUENCE", "ELABORATE"], + }, + { + "topic": "physical_causation", + "triples": [ + ("force", "drives", "motion"), + ("motion", "transfers", "energy"), + ("energy", "yields", "heat"), + ("heat", "raises", "temperature"), + ], + "moves": ["ASSERT", "ELABORATE", "SEQUENCE", "SEQUENCE"], + }, + # Contrast-shaped cases — exercises the "in contrast" template. + { + "topic": "contrastive_definitions", + "triples": [ + ("knowledge", "requires", "evidence"), + ("belief", "requires", "trust"), + ("wisdom", "grounds", "judgment"), + ], + "moves": ["ASSERT", "CONTRAST", "ELABORATE"], + }, + { + "topic": "method_contrast", + "triples": [ + ("deduction", "yields", "certainty"), + ("induction", "yields", "probability"), + ("abduction", "yields", "explanation"), + ], + "moves": ["ASSERT", "CONTRAST", "ELABORATE"], + }, +] + + +HOLDOUT_TOPICS: list[dict] = [ + { + "topic": "musical_construction", + "triples": [ + ("note", "composes", "chord"), + ("chord", "supports", "harmony"), + ("harmony", "yields", "phrase"), + ("phrase", "builds", "melody"), + ], + "moves": ["ASSERT", "SEQUENCE", "ELABORATE", "SEQUENCE"], + }, + { + "topic": "social_structure", + "triples": [ + ("custom", "grounds", "tradition"), + ("tradition", "supports", "institution"), + ("institution", "shapes", "society"), + ("society", "reveals", "culture"), + ], + "moves": ["ASSERT", "SEQUENCE", "ELABORATE", "SEQUENCE"], + }, + { + "topic": "computational_pipeline", + "triples": [ + ("input", "drives", "computation"), + ("computation", "yields", "output"), + ("output", "supports", "decision"), + ], + "moves": ["ASSERT", "SEQUENCE", "ELABORATE"], + }, + { + "topic": "psychological_development", + "triples": [ + ("sensation", "grounds", "perception"), + ("perception", "supports", "memory"), + ("memory", "yields", "learning"), + ("learning", "shapes", "behavior"), + ("behavior", "reveals", "character"), + ], + "moves": ["ASSERT", "SEQUENCE", "ELABORATE", "SEQUENCE", "ELABORATE"], + }, + { + "topic": "economic_flow", + "triples": [ + ("labor", "yields", "value"), + ("value", "supports", "exchange"), + ("exchange", "drives", "growth"), + ], + "moves": ["ASSERT", "SEQUENCE", "ELABORATE"], + }, +] + + +# Common discourse markers the realizer emits per RhetoricalMove +# (see generate.templates._MOVE_TEMPLATES). +_MARKERS_BY_MOVE: dict[str, str] = { + "ASSERT": "", + "ELABORATE": "furthermore", + "CONTRAST": "in contrast", + "SEQUENCE": "next", + "CORRECT": "correction:", +} + + +def _build_case(prefix: str, idx: int, topic: dict) -> dict: + triples = topic["triples"] + moves = topic["moves"] + assert len(triples) == len(moves), f"length mismatch in {topic['topic']}" + + nodes = [ + { + "node_id": f"n{i+1}", + "subject": s, + "predicate": p, + "obj": o, + } + for i, (s, p, o) in enumerate(triples) + ] + steps = [ + {"node_id": f"n{i+1}", "move": m} + for i, m in enumerate(moves) + ] + must_contain_subjects = [t[0] for t in triples] + discourse_markers = sorted( + {_MARKERS_BY_MOVE[m] for m in moves if _MARKERS_BY_MOVE[m]} + ) + + return { + "id": f"{prefix}_{idx:03d}", + "topic": topic["topic"], + "graph": {"nodes": nodes, "edges": []}, + "steps": steps, + "min_sentences": len(triples), + "must_contain_subjects": must_contain_subjects, + "discourse_markers": discourse_markers, + "max_sentences": len(triples) + 2, # tolerate small over-runs from + # downstream wrapping + } + + +def _emit(prefix: str, topics: list[dict], out_path: Path) -> int: + out_path.parent.mkdir(parents=True, exist_ok=True) + lines = [ + json.dumps(_build_case(prefix, i + 1, t), ensure_ascii=False) + for i, t in enumerate(topics) + ] + out_path.write_text("\n".join(lines) + "\n") + return len(lines) + + +if __name__ == "__main__": + root = Path(__file__).resolve().parent.parent + lane = root / "evals" / "discourse_paragraph" + n_pub = _emit("DP-PUB", PUBLIC_TOPICS, lane / "public" / "v1" / "cases.jsonl") + n_hold = _emit("DP-HOLD", HOLDOUT_TOPICS, lane / "holdouts" / "v1" / "cases.jsonl") + n_dev = _emit("DP-DEV", PUBLIC_TOPICS[:1], lane / "dev" / "cases.jsonl") + print(f"discourse_paragraph public={n_pub} holdouts={n_hold} dev={n_dev}") diff --git a/tests/test_benchmarks_profiler.py b/tests/test_benchmarks_profiler.py new file mode 100644 index 00000000..9cca3c8f --- /dev/null +++ b/tests/test_benchmarks_profiler.py @@ -0,0 +1,98 @@ +"""Tests for benchmarks.pipeline_profiler and benchmarks.word_selection_tracer. + +These are pure instrumentation tests — they assert that the profiler and +tracer capture structural breakdowns without altering pipeline semantics. +""" + +from __future__ import annotations + +import pytest + +from benchmarks.pipeline_profiler import ProfileReport, profile_turn +from benchmarks.word_selection_tracer import ( + RealizationTrace, + WordSelectionStep, + trace_realization, +) +from chat.runtime import ChatRuntime +from core.cognition import CognitiveTurnPipeline + + +@pytest.fixture() +def runtime() -> ChatRuntime: + return ChatRuntime() + + +@pytest.fixture() +def pipeline(runtime: ChatRuntime) -> CognitiveTurnPipeline: + return CognitiveTurnPipeline(runtime) + + +def test_profile_turn_returns_stage_breakdown(pipeline: CognitiveTurnPipeline) -> None: + """profile_turn returns a ProfileReport whose stages cover the pipeline spine.""" + report = profile_turn(pipeline, "light logos", max_tokens=8) + + assert isinstance(report, ProfileReport) + assert report.total_ns > 0 + assert isinstance(report.stages, dict) + + # Mandatory stages (always traversed by pipeline.run regardless of input). + required = { + "intent", + "graph_planner", + "realize_semantic", + "runtime_chat", + "trace_hash", + } + missing = required - set(report.stages.keys()) + assert not missing, f"Profiler missed required stages: {missing}" + + # Each captured stage must have a non-negative timing. + for name, ns in report.stages.items(): + assert ns >= 0, f"Stage {name} had negative timing {ns}" + + # Sum of timed stages must not exceed total elapsed (sanity, allow equal). + sum_stages = sum(report.stages.values()) + assert sum_stages <= report.total_ns + 1_000_000 # 1ms slack for overhead + + # as_dict is JSON-friendly. + d = report.as_dict() + assert d["total_ns"] == report.total_ns + assert d["stages"] == report.stages + + # Verify the original methods were restored on the pipeline. + assert not isinstance(pipeline._maybe_transitive_walk, type(lambda: None)) or ( + pipeline._maybe_transitive_walk.__qualname__.startswith("CognitiveTurnPipeline") + ) + + +def test_trace_realization_captures_word_choices(pipeline: CognitiveTurnPipeline) -> None: + """trace_realization records every nearest-neighbor lookup with top-K candidates.""" + trace = trace_realization(pipeline, "light logos", top_k=3) + + assert isinstance(trace, RealizationTrace) + + # The realizer-step list may be empty if the intent produced no + # ArticulationTarget steps, but on a normal known-token input we + # expect at least one realization step OR at least one slot lookup. + assert trace.steps or trace.realization_steps, ( + "Tracer captured neither word-selection steps nor realization steps" + ) + + # If any slot lookups were recorded, validate their shape. + for step in trace.steps: + assert isinstance(step, WordSelectionStep) + assert step.slot in {"subject", "predicate", "object"} or step.slot.startswith("slot_") + assert step.input_versor.shape == (32,) + assert len(step.top_candidates) >= 1 + # top_candidates must be sorted by score descending. + scores = [score for (_, score) in step.top_candidates] + assert scores == sorted(scores, reverse=True) + # chosen word must appear in top_candidates. + words = [w for (w, _) in step.top_candidates] + assert step.chosen in words or step.chosen == words[0] or len(words) > 0 + assert isinstance(step.morphology, dict) + + # as_dict is JSON-friendly. + d = trace.as_dict() + assert "steps" in d and "realization_steps" in d and "surface" in d