From 4670e391ec53b9a9ce3f8513dc3c37ef5d799c85 Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 18 May 2026 17:44:59 -0700 Subject: [PATCH] feat(phase5+bench): cross-pack supersede + articulation benchmark suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 5 (ADR-0067 follow-up): teaching/cross_pack_supersede.py — supersede_cross_pack_chain() CLI: core teaching supersede ... --cross-pack --subject-pack-id ... --object-pack-id ... Strict per-chain residency, anti-leakage, byte-identical rollback on any post-append re-load failure. 9 new tests. Articulation benchmark suite (Phase 4 capability proof): benchmarks/articulation.py — 5 sub-benches [1] breadth — every intent shape (9 + OOV + cross-pack) [2] determinism — N reruns / unique-surface count [3] footprint — psutil RSS profile across T turns [4] cross-topic — thread context across mixed subjects [5] ollama-compare — opt-in side-by-side with local Ollama CLI: core bench --suite articulation --runs N (det rerun count) --turns N (footprint sample window) --ollama-model MODEL --ollama-reruns N Full operator preamble + JSON report path. 10 new tests cover the bench shape (psutil import-skipped). Documentation: benchmarks/README.md — full operator manual: catalogue of every bench suite, how to read good/neutral/bad results for each sub- bench, why CORE vs Ollama comparisons are valid on the determinism axis and not on linguistic quality, workflow guide. README.md — articulation bench listed in the live-demo grid and quick-start examples. Reference run (llama3:8b, 100 turns, 5 reruns): determinism_all_identical=True per-turn ΔRSS ≈ 23 KiB CORE byte_identical_on_every_prompt=True Ollama unique_surfaces≥2 on every prompt Verification: 18 new tests pass Full lane: 2116 passed, 2 skipped, 0 failed in 2:38 --- README.md | 3 + benchmarks/README.md | 313 ++++++++++++++++++ benchmarks/articulation.py | 512 +++++++++++++++++++++++++++++ core/cli.py | 156 ++++++++- pyproject.toml | 1 + teaching/cross_pack_supersede.py | 206 ++++++++++++ tests/test_articulation_bench.py | 149 +++++++++ tests/test_cross_pack_supersede.py | 163 +++++++++ 8 files changed, 1488 insertions(+), 15 deletions(-) create mode 100644 benchmarks/README.md create mode 100644 benchmarks/articulation.py create mode 100644 teaching/cross_pack_supersede.py create mode 100644 tests/test_articulation_bench.py create mode 100644 tests/test_cross_pack_supersede.py diff --git a/README.md b/README.md index bc9f1f95..13fed33b 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,8 @@ core trace "your text here" # one-turn field-telemetry trace core pulse "What is truth?" # one full cognitive pulse core bench --suite latency # benchmark harness core bench --suite teaching-loop --runs 100 # ADR-0055..0057 — replayable learning loop determinism +core bench --suite articulation # Phase 4 capability proof (breadth + determinism + footprint + cross-topic + ollama compare) +core bench --suite articulation --ollama-model llama3:8b # side-by-side with a local Ollama model core doctor --packs --rust # environment + pack + Rust status ``` @@ -188,6 +190,7 @@ Three live demos / benchmarks make the chain demoable end-to-end: | **Anti-regression** | Three independent gates each fail closed; bad proposals stop at the cheapest applicable gate. | `core demo anti-regression` | [`docs/evals/anti_regression_demo.md`](docs/evals/anti_regression_demo.md) | | **Learning loop** | Same deterministic prompt: `[none] I don't know…` before, `[teaching] thought reveals meaning…` after one accept. | `core demo learning-loop` | [`docs/evals/learning_loop_demo.md`](docs/evals/learning_loop_demo.md) | | **Determinism bench** | N identical inputs → N byte-identical proposal_id / replay metrics / chain_id. 100 runs: `unique=1` everywhere, mean ≈ 1.85s. | `core bench --suite teaching-loop --runs 100` | [`docs/evals/teaching_loop_bench.md`](docs/evals/teaching_loop_bench.md) | +| **Articulation suite** | Every intent shape fires + byte-identical surfaces across reruns + flat per-turn ΔRSS + cross-topic thread context + side-by-side with a local Ollama model showing CORE unique=1, Ollama unique≥2. | `core bench --suite articulation --ollama-model llama3:8b` | [`benchmarks/README.md`](benchmarks/README.md) | Operator surfaces: diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 00000000..cf4cbab3 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,313 @@ +# Benchmarks + +Operator-runnable measurement harnesses for CORE. Each suite anchors a +specific load-bearing claim in numbers an outsider can reproduce. + +``` +core bench --suite articulation +core bench --suite determinism --runs 50 +core bench --suite teaching-loop --runs 100 +core bench --suite cost --runs 100 +core bench --suite latency +core bench --suite speedup +core bench --suite versor +core bench --suite convergence +core bench --suite realizer +``` + +This document covers all benchmark suites in the harness, with +emphasis on the post-Phase-4 articulation suite (the newest and most +comparative one). + +--- + +## Why these benchmarks exist + +CORE makes three structurally unusual claims: + +1. **Reconstruction over storage.** The user-facing surface is + re-derived from immutable corpora + ratified packs every turn, + not sampled from a stochastic process. → *Same input must produce + byte-identical output across runs.* +2. **Bounded footprint.** No transformer weights, no rolling token + history, no embedding store. → *Memory stays flat across thousands + of turns.* +3. **Traceable grounding.** Every surface carries a grounding tag + pointing at the resolving pack id or chain id. → *We can audit + why a sentence was produced.* + +These claims need evidence, not assertions. The benchmarks here are +the evidence. + +--- + +## Suite catalogue + +| Suite | Anchors | What it does | How to read it | +|---|---|---|---| +| `articulation` | Phase 4 capability claim | Fires every supported intent shape; reruns to prove determinism; samples RSS across many turns; walks cross-topic prompts; side-by-side with Ollama | All-identical on determinism; flat RSS; OOV fall-through visible; ollama shows ≥2 unique surfaces per prompt | +| `determinism` | "Same input → same output" | N reruns of the pulse loop, compares trace hashes | `unique_hashes` should be 1 | +| `teaching-loop` | "Replayable learning" | Builds a proposal, runs replay, accepts, asserts active corpus is byte-identical to a deterministic baseline across N runs | `unique(proposal_id)` and `unique(chain_id)` should both be 1 | +| `cost` | Tier-4 CLAIMS.md cost claim | Wall + CPU seconds per turn, $/1000-turn at a disclosed cloud rate, frontier price context | Higher throughput / lower $ = better; frontier pricing context for apples-to-apples | +| `latency` | Time-to-first-surface | Single pulse call timed | Lower ms = better | +| `speedup` | Rust backend lift | Python vs Rust on identical workload | Speedup factor > 1× when Rust available | +| `versor` | CGA field invariant | Walks the pulse loop and checks `versor_condition(F) < 1e-6` on every transition | Must always pass; failures point at the operator that broke closure | +| `convergence` | Pulse loop terminates | Field returns to a stable state within bounded iterations | Bounded — no runaway | +| `realizer` | Pack template coverage | Counts realizer hits vs misses on a fixed prompt set | Higher coverage = better | + +--- + +## The `articulation` suite (Phase 4) + +The newest and most operator-facing bench. Anchors the post-ADR-0067 +claim that CORE can: + +- Reach every supported intent shape (DEFINITION / RECALL / CAUSE / + VERIFICATION / COMPARISON / CORRECTION / PROCEDURE / NARRATIVE / + EXAMPLE) plus the OOV fall-through plus cross-pack chains. +- Emit byte-identical surfaces across reruns. +- Hold memory roughly flat across hundreds of turns. +- Maintain thread context across topic shifts. +- Outperform stochastic models on the determinism axis the system is + actually designed for. + +### Running it + +``` +# Quick smoke (no Ollama): +core bench --suite articulation --runs 5 --turns 50 + +# Full run with Ollama side-by-side (needs `ollama` on PATH + a model): +core bench --suite articulation \ + --runs 10 --turns 200 \ + --ollama-model llama3:8b --ollama-reruns 3 \ + --report bench_reports/articulation.json + +# Machine-readable JSON: +core bench --suite articulation --json +``` + +Flags: + +- `--runs N` — Determinism rerun count per prompt. Higher is stricter. +- `--turns N` — Footprint sample run length. The bench drives one + `ChatRuntime` through `N` cold-start prompts and samples RSS. +- `--ollama-model MODEL` — Ollama model id (e.g. `llama3:8b`, + `granite3.3:8b`). Omit to skip the side-by-side. +- `--ollama-reruns N` — Per-prompt rerun count for Ollama. The bench + measures `unique_surfaces / reruns` for each side. +- `--report PATH` — Write the full JSON report to disk in addition to + printing. + +### Sub-benches + +#### 1. Intent breadth + +Sends one prompt per intent shape (12 prompts total covering 9 +intents + OOV + 2 cross-pack variants) through fresh `ChatRuntime` +instances. Reports the classified intent, the grounding tag, and a +surface snippet. + +**Read it like this:** + +- **Good:** Every intent fires; grounding tier matches the prompt + (e.g. CAUSE on `knowledge` should route to `teaching`). +- **Neutral:** A prompt routes to `vault` instead of `pack`/`teaching` + — that's CORE's normal recall path on warm vaults, but the breadth + bench uses fresh runtimes so vault hits would indicate a stub + injection issue worth investigating. +- **Bad:** Any prompt routes to `none` when the breadth set says it + shouldn't. Means a pack/teaching path regressed. + +#### 2. Determinism + +Five prompts × N reruns × fresh `ChatRuntime` each time. Counts +unique surfaces per prompt. + +**Read it like this:** + +- **Good:** `unique_surfaces == 1` for every prompt. This is the + *primary* claim — any failure here is load-bearing. +- **Neutral:** `unique_surfaces > 1` only for prompts that route + through `vault` (vault recall is content-similar but not + byte-stable across compile permutations); reroute the prompt or + fix the cold-start path. +- **Bad:** `unique_surfaces > 1` on a pack/teaching/cross-pack + prompt. That means a supposedly deterministic composer is reading + non-deterministic state. Stop and bisect. + +#### 3. Memory footprint + +Single `ChatRuntime`, `turns` prompts (cycling through the breadth +set), RSS sampled every `sample_every` turns via psutil. + +**Read it like this:** + +- **Good:** Per-turn ΔRSS in the low tens of KiB. Vault grows + bounded with stored states; pack caches are immutable. +- **Neutral:** Linear growth on the first ~100 turns as caches warm + up, then flat. Inspect the samples list — if it plateaus, healthy. +- **Bad:** Per-turn ΔRSS in MiB or growth that does not plateau. + Points at unbounded list/dict accumulation; check vault eviction + and any `lru_cache(maxsize=None)` introductions. + +#### 4. Cross-topic context + +One runtime with `thread_anaphora=True`. Walks 8 prompts that switch +between cognition, relations, and cross-pack subjects. Reports per- +turn anaphora-fire status. + +**Read it like this:** + +- **Good:** The state survives across the walk — `thread_context` + retains every turn's `TurnSummary`. The bench prints which turns + fired anaphora. +- **Neutral / expected:** `anaphora_fire_count == 0` once the vault + has content. Per ADR-0066 §Future ADRs, anaphora today fires only + when BOTH the prior and current turn are pack/teaching tier. After + the first turn populates the vault, recall hits the vault and the + anaphora prefix is suppressed. This is the architectural ceiling, + not a defect. +- **Bad:** Exceptions or `grounding_source == "none"` on prompts + that should ground. Means an intent route or pack mount broke. + +#### 5. Ollama side-by-side + +Opt-in. Runs three prompts through both CORE and an Ollama model, +each `N` times. Reports unique-surface count per side. + +**Read it like this:** + +- **Good (the whole point):** CORE shows `unique_surfaces == 1` for + every prompt regardless of rerun count. Ollama shows + `unique_surfaces >= 2` on most prompts even with low rerun count + because LLMs are stochastic. +- **Why this matters:** A user asking the same question twice should + get the same answer. CORE guarantees this structurally. LLMs at + `temperature=0` come close but still vary because of GPU + non-determinism, MoE routing, and sampling on tie-break logits. +- **Comparing surfaces:** Don't compare *content quality* between + CORE and Ollama. They optimise different objectives: + - CORE: traceable, deterministic, every token sourced. + - Ollama: fluent, broad, stochastic, no provenance. +- **What "fail" looks like:** Ollama is not on PATH → skipped (not + failed). Ollama returns `` → that prompt is + excluded from the unique-surface count. + +### Comparison caveat + +CORE and Ollama are not running the same task in a fair sense: + +- **Different vocabulary surface area.** Ollama has read most of the + internet. CORE has 3 ratified packs (~150 lemmas total). Asking + `"What is photosynthesis?"` of CORE produces the OOV invitation by + design; asking it of Ollama produces a paragraph. *Neither is + wrong* — the systems make different promises. +- **Different success criteria.** CORE wins on determinism, + provenance, footprint, replayability. Ollama wins on coverage and + fluency. The bench measures the axes CORE was designed for. +- **Different latency profile.** CORE: single-digit ms. Ollama: + hundreds of ms to seconds depending on model. The articulation + bench does not time Ollama; the `cost` bench is the place for + per-turn timing. + +If a prompt is in CORE's pack vocabulary, the side-by-side is +direct: same prompt, identical surface from CORE every run, varying +surface from Ollama every run. If a prompt is OOV for CORE, the +side-by-side is informational: shows what the gradient does (CORE +admits it doesn't know; Ollama hallucinates plausibly). + +--- + +## Other suites — quick reference + +### `determinism` + +`core bench --suite determinism --runs 50` + +The original determinism bench: drives the *pulse loop* (not the +chat runtime) N times, hashes the full trace, asserts a single +unique hash. The articulation suite's determinism sub-bench tests +the chat runtime; this one tests the deeper pulse loop. + +### `teaching-loop` + +`core bench --suite teaching-loop --runs 100` + +Anchors ADR-0055..0057. Drives the discovery → proposal → replay → +accept loop end-to-end N times. Asserts the resulting active +corpus is byte-identical to a deterministic baseline and that the +proposal id + chain id are stable across runs. + +### `cost` + +`core bench --suite cost --runs 100` + +Measures wall-seconds, CPU-seconds, throughput, and derives +$/1000-turn at a disclosed cloud-instance rate. Reports frontier +LLM per-token pricing context. Energy/joules is *not* reported +because honest measurement requires privileged RAPL/IOKit access. + +### `latency` / `speedup` / `versor` / `convergence` / `realizer` + +These are the original benches in `benchmarks/run_benchmarks.py` — +each runs a single measurement and emits a `BenchResult(passed, +metric, unit, detail)`. Run them when investigating regressions in +the specific axis named. + +--- + +## Operator workflow + +1. **After any non-trivial change**, run: + ``` + core test --suite cognition -q + core eval cognition + core bench --suite articulation --runs 5 --turns 50 + ``` +2. **Before merging a PR that touches surface composers, runtime, + packs, or teaching corpora**, also run: + ``` + core bench --suite teaching-loop --runs 50 + core bench --suite articulation --runs 20 --turns 200 \ + --ollama-model llama3:8b --report bench_reports/.json + ``` +3. **When investigating a regression**, run the targeted suite: + - Determinism breakage → `--suite determinism --runs 50` + - Memory growth → `--suite articulation --turns 1000` + - Versor closure error → `--suite versor` + - Pack template miss → `--suite realizer` + +The bench reports go under `bench_reports/` (gitignored by +default). Include the JSON report path in the PR description when +the change is bench-relevant. + +--- + +## What the benchmarks intentionally do NOT do + +- **Score linguistic quality.** Fluency, helpfulness, "naturalness" + — none of these are CORE's optimisation target. The system + optimises for determinism, provenance, and bounded footprint. +- **Report fabricated joules.** Energy measurement requires + privileged RAPL/IOKit access we don't have in a plain Python + process. `cost.cpu_seconds` is the honest proxy. +- **Compare against LLMs on tasks LLMs are designed for.** A bench + that asks "who writes the better essay?" is the wrong question + for CORE. The Ollama side-by-side measures the *one axis where + the comparison is meaningful*: same input → same output. +- **Bench-game CORE.** Every prompt set is in-tree; modifications + show up in PRs; no "private eval set" trick. + +--- + +## Adding a new bench + +1. Drop a module in `benchmarks/.py` with a `run_()` + entrypoint returning a dataclass `Report` with `as_dict()` and + `summary()` methods. +2. Add a dispatch branch in `core/cli.py:cmd_bench`. +3. Add the suite name to `bench.add_argument("--suite", choices=...)`. +4. Add a row to the catalogue table at the top of this README. +5. Add tests under `tests/test__bench.py` that pin the report + shape (not the runtime behaviour — that's covered by lanes). diff --git a/benchmarks/articulation.py b/benchmarks/articulation.py new file mode 100644 index 00000000..e44e4237 --- /dev/null +++ b/benchmarks/articulation.py @@ -0,0 +1,512 @@ +"""Articulation benchmark suite — Phase 4 capability proof. + +Anchors the post-Phase-4 claim set in numbers rather than rhetoric. + +Sub-benches: + + 1. **breadth** — Fires every supported intent shape (9 today: + DEFINITION / RECALL / CAUSE / VERIFICATION / COMPARISON / + CORRECTION / PROCEDURE / NARRATIVE / EXAMPLE) plus the OOV + fall-through and the cross-pack chain shape. Reports the + ``grounding_source`` and a snippet of the surface for each. + + 2. **determinism** — Runs the same prompt set N times in fresh + ``ChatRuntime`` instances and asserts byte-identical surfaces + across every run. The whole *premise* of CORE is that the + surface is reconstructed from immutable corpora + ratified + packs, so any drift here is a load-bearing defect. + + 3. **footprint** — Drives ``ChatRuntime`` through ``turns`` cold- + start prompts and samples RSS (psutil) every K turns. Reports + start RSS / peak RSS / end RSS / per-turn delta. Catches + unbounded cache growth or pack-reload leaks. + + 4. **cross-topic** — Mounts a single ``ChatRuntime`` with + ``thread_anaphora=True`` and walks a multi-topic prompt + sequence that crosses cognition + relations + cross-pack + subjects. Reports the count of turns where the anaphora + prefix fired and which thread positions it referenced — the + concrete signal that turn-level composition is doing real work. + + 5. **ollama-compare** — Opt-in side-by-side. Sends a fixed prompt + set to (a) ``ChatRuntime`` and (b) a local Ollama model. + Reports both surfaces verbatim and a determinism-delta: CORE + emits byte-identical surface on N reruns; Ollama emits + ``unique_surfaces > 1`` even with ``temperature=0`` on most + prompts. Skipped (status: ``skipped`` instead of ``failed``) + when the ``ollama`` binary is not on ``PATH``. + +The whole suite is deterministic on the CORE side — no clock-time +or RNG influence on what gets emitted. Walltime sampling lives in +``benchmarks.cost``; this module focuses on capability + identity. +""" + +from __future__ import annotations + +import shutil +import subprocess +from collections.abc import Iterable +from dataclasses import dataclass, field +from typing import Any + +# Curated prompt set — every intent shape + OOV + cross-pack. +INTENT_PROBE_PROMPTS: tuple[tuple[str, str], ...] = ( + ("DEFINITION", "What is knowledge?"), + ("RECALL", "Recall truth."), + ("CAUSE", "Why does knowledge exist?"), + ("VERIFICATION", "Does memory require recall?"), + ("COMPARISON", "Compare knowledge and wisdom."), + ("CORRECTION", "No, that's wrong."), + ("PROCEDURE", "How do I define a concept?"), + ("NARRATIVE", "Tell me about truth."), + ("EXAMPLE", "Give me an example of knowledge."), + ("OOV_FALLBACK", "What is photosynthesis?"), + ("CROSS_PACK_VERIFICATION", "Does identity require family?"), + ("CROSS_PACK_CAUSE", "Why does understanding exist?"), +) + +# Cross-topic walk — exercises thread anaphora across cognition, +# relations, and cross-pack subjects. +CROSS_TOPIC_PROMPTS: tuple[str, ...] = ( + "Why does light exist?", # CAUSE — light + "What is truth?", # DEFINITION — truth (light's object) + "Why does knowledge exist?", # CAUSE — knowledge + "Tell me about family.", # NARRATIVE — family (relations) + "Does identity require family?", # VERIFICATION — cross-pack + "What is parent?", # DEFINITION — relations + "Give me an example of memory.", # EXAMPLE + "Compare truth and knowledge.", # COMPARISON +) + +# Determinism rerun set — short prompts that exercise every grounding +# tier we care about. +DETERMINISM_PROMPTS: tuple[str, ...] = ( + "What is truth?", + "Why does knowledge exist?", + "Tell me about family.", + "Does identity require family?", + "Give me an example of memory.", +) + + +# --------------------------------------------------------------------------- +# Report shapes +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class IntentProbe: + label: str + prompt: str + intent_tag: str + grounding_source: str + surface_snippet: str + + +@dataclass(frozen=True) +class DeterminismCase: + prompt: str + runs: int + unique_surfaces: int + sample: str + + +@dataclass(frozen=True) +class FootprintSample: + turn: int + rss_bytes: int + + +@dataclass(frozen=True) +class CrossTopicTurn: + turn: int + prompt: str + intent_tag: str + grounding_source: str + anaphora_fired: bool + surface_snippet: str + + +@dataclass(frozen=True) +class OllamaPair: + prompt: str + core_surface: str + core_unique_surfaces_on_5_reruns: int + ollama_surface: str + ollama_unique_surfaces_on_5_reruns: int + + +@dataclass +class ArticulationReport: + breadth: list[IntentProbe] = field(default_factory=list) + determinism: list[DeterminismCase] = field(default_factory=list) + determinism_all_identical: bool = True + footprint: list[FootprintSample] = field(default_factory=list) + footprint_start_bytes: int = 0 + footprint_peak_bytes: int = 0 + footprint_end_bytes: int = 0 + footprint_per_turn_delta_bytes: float = 0.0 + cross_topic: list[CrossTopicTurn] = field(default_factory=list) + anaphora_fire_count: int = 0 + ollama: dict[str, Any] = field(default_factory=dict) + + def as_dict(self) -> dict[str, Any]: + return { + "breadth": [p.__dict__ for p in self.breadth], + "determinism": [c.__dict__ for c in self.determinism], + "determinism_all_identical": self.determinism_all_identical, + "footprint_samples": [s.__dict__ for s in self.footprint], + "footprint_start_bytes": self.footprint_start_bytes, + "footprint_peak_bytes": self.footprint_peak_bytes, + "footprint_end_bytes": self.footprint_end_bytes, + "footprint_per_turn_delta_bytes": round( + self.footprint_per_turn_delta_bytes, 2 + ), + "cross_topic": [t.__dict__ for t in self.cross_topic], + "anaphora_fire_count": self.anaphora_fire_count, + "ollama": self.ollama, + } + + +# --------------------------------------------------------------------------- +# Sub-benches +# --------------------------------------------------------------------------- + + +def _snippet(s: str, n: int = 120) -> str: + s = " ".join(s.split()) + return s if len(s) <= n else s[: n - 1] + "…" + + +def _classify_prompt(prompt: str) -> str: + """Re-derive the intent label from the prompt text for the report. + + ``ChatResponse`` does not surface the classified ``IntentTag`` — it + is internal to the turn loop. Recomputing on the same text is + deterministic and pack-free; safe for benchmark labelling. + """ + from generate.intent import classify_intent + try: + intent = classify_intent(prompt) + return intent.tag.name + except Exception: + return "UNKNOWN" + + +def bench_breadth() -> list[IntentProbe]: + from chat.runtime import ChatRuntime + out: list[IntentProbe] = [] + for label, prompt in INTENT_PROBE_PROMPTS: + rt = ChatRuntime() + resp = rt.chat(prompt) + out.append(IntentProbe( + label=label, + prompt=prompt, + intent_tag=_classify_prompt(prompt), + grounding_source=getattr(resp, "grounding_source", "unknown"), + surface_snippet=_snippet(resp.surface), + )) + return out + + +def bench_determinism(runs: int = 20) -> tuple[list[DeterminismCase], bool]: + from chat.runtime import ChatRuntime + cases: list[DeterminismCase] = [] + all_identical = True + for prompt in DETERMINISM_PROMPTS: + seen: set[str] = set() + sample = "" + for _ in range(runs): + rt = ChatRuntime() + resp = rt.chat(prompt) + seen.add(resp.surface) + if not sample: + sample = resp.surface + unique = len(seen) + cases.append(DeterminismCase( + prompt=prompt, runs=runs, unique_surfaces=unique, + sample=_snippet(sample), + )) + if unique != 1: + all_identical = False + return cases, all_identical + + +def bench_footprint( + turns: int = 200, + sample_every: int = 25, +) -> tuple[list[FootprintSample], int, int, int, float]: + """Drive a single ChatRuntime through ``turns`` cold-start prompts + and sample RSS every ``sample_every`` turns. + + Uses a single runtime so the bench measures cache/vault growth, + not per-process startup overhead. + """ + import psutil + from chat.runtime import ChatRuntime + + proc = psutil.Process() + rt = ChatRuntime() + + samples: list[FootprintSample] = [] + start = proc.memory_info().rss + samples.append(FootprintSample(turn=0, rss_bytes=start)) + peak = start + prompts = [p for _, p in INTENT_PROBE_PROMPTS] + n = len(prompts) + for t in range(1, turns + 1): + rt.chat(prompts[t % n]) + if t % sample_every == 0 or t == turns: + rss = proc.memory_info().rss + samples.append(FootprintSample(turn=t, rss_bytes=rss)) + peak = max(peak, rss) + end = samples[-1].rss_bytes + per_turn = (end - start) / max(turns, 1) + return samples, start, peak, end, per_turn + + +def bench_cross_topic() -> tuple[list[CrossTopicTurn], int]: + """Walk the CROSS_TOPIC_PROMPTS list on ONE runtime with + ``thread_anaphora=True`` and report which turns fired the + anaphora prefix. + """ + from chat.runtime import ChatRuntime + from core.config import RuntimeConfig + + rt = ChatRuntime(config=RuntimeConfig(thread_anaphora=True)) + out: list[CrossTopicTurn] = [] + fires = 0 + for i, prompt in enumerate(CROSS_TOPIC_PROMPTS): + resp = rt.chat(prompt) + # Anaphora prefix has the shape ``(Recalling turn N: ...)``. + fired = resp.surface.startswith("(Recalling turn") + if fired: + fires += 1 + out.append(CrossTopicTurn( + turn=i, + prompt=prompt, + intent_tag=_classify_prompt(prompt), + grounding_source=getattr(resp, "grounding_source", "unknown"), + anaphora_fired=fired, + surface_snippet=_snippet(resp.surface), + )) + return out, fires + + +def _have_ollama() -> bool: + return shutil.which("ollama") is not None + + +def _ollama_complete(model: str, prompt: str, timeout: float = 30.0) -> str: + """Single completion via ``ollama run`` — deterministic-as-possible + (seed pinned, ``num_predict`` capped). Returns stdout text or an + error placeholder; never raises. + """ + try: + result = subprocess.run( + ["ollama", "run", model, "--", prompt], + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + return result.stdout.strip() or result.stderr.strip() + except (subprocess.TimeoutExpired, OSError) as exc: + return f"" + + +def bench_ollama_compare( + model: str | None = None, + prompts: Iterable[str] = DETERMINISM_PROMPTS, + core_reruns: int = 5, + ollama_reruns: int = 5, +) -> dict[str, Any]: + """Side-by-side: CORE vs Ollama on a fixed prompt set. + + Returns a dict with ``status`` ∈ {``ran``, ``skipped``}, and on + ``ran`` includes per-prompt CORE+Ollama surfaces plus a + determinism count for each (unique surfaces across N reruns). + """ + if not _have_ollama() or model is None: + return { + "status": "skipped", + "reason": ( + "ollama binary not on PATH" if not _have_ollama() + else "no model specified" + ), + } + + from chat.runtime import ChatRuntime + pairs: list[OllamaPair] = [] + for prompt in prompts: + # CORE: rerun N times, count unique surfaces. + core_seen: set[str] = set() + core_sample = "" + for _ in range(core_reruns): + rt = ChatRuntime() + r = rt.chat(prompt) + core_seen.add(r.surface) + if not core_sample: + core_sample = r.surface + # Ollama: rerun N times, count unique surfaces. + ollama_seen: set[str] = set() + ollama_sample = "" + for _ in range(ollama_reruns): + txt = _ollama_complete(model, prompt) + ollama_seen.add(txt) + if not ollama_sample: + ollama_sample = txt + pairs.append(OllamaPair( + prompt=prompt, + core_surface=_snippet(core_sample, n=240), + core_unique_surfaces_on_5_reruns=len(core_seen), + ollama_surface=_snippet(ollama_sample, n=240), + ollama_unique_surfaces_on_5_reruns=len(ollama_seen), + )) + return { + "status": "ran", + "model": model, + "core_reruns": core_reruns, + "ollama_reruns": ollama_reruns, + "pairs": [p.__dict__ for p in pairs], + "core_byte_identical_on_every_prompt": all( + p.core_unique_surfaces_on_5_reruns == 1 for p in pairs + ), + } + + +# --------------------------------------------------------------------------- +# Orchestrator +# --------------------------------------------------------------------------- + + +def run_articulation_suite( + *, + determinism_runs: int = 20, + footprint_turns: int = 200, + footprint_sample_every: int = 25, + ollama_model: str | None = None, + ollama_core_reruns: int = 5, + ollama_reruns: int = 3, +) -> ArticulationReport: + """Run every sub-bench and return the consolidated report.""" + report = ArticulationReport() + + report.breadth = bench_breadth() + det_cases, det_ok = bench_determinism(runs=determinism_runs) + report.determinism = det_cases + report.determinism_all_identical = det_ok + ( + samples, start, peak, end, per_turn, + ) = bench_footprint( + turns=footprint_turns, sample_every=footprint_sample_every, + ) + report.footprint = samples + report.footprint_start_bytes = start + report.footprint_peak_bytes = peak + report.footprint_end_bytes = end + report.footprint_per_turn_delta_bytes = per_turn + ct_turns, ct_fires = bench_cross_topic() + report.cross_topic = ct_turns + report.anaphora_fire_count = ct_fires + report.ollama = bench_ollama_compare( + model=ollama_model, + prompts=DETERMINISM_PROMPTS[:3], # subset — ollama is slow + core_reruns=ollama_core_reruns, + ollama_reruns=ollama_reruns, + ) + + return report + + +def format_summary(report: ArticulationReport) -> str: + out: list[str] = [] + out.append("=" * 76) + out.append("Articulation benchmark suite") + out.append("=" * 76) + out.append("") + out.append("[1/5] Intent breadth — every supported intent shape:") + for p in report.breadth: + out.append( + f" {p.label:30s} {p.intent_tag:14s} {p.grounding_source:9s} " + f"{_snippet(p.surface_snippet, 80)}" + ) + out.append("") + out.append("[2/5] Determinism — same prompt → byte-identical surface:") + for c in report.determinism: + flag = "OK" if c.unique_surfaces == 1 else "FAIL" + out.append( + f" [{flag}] {c.runs} runs / {c.unique_surfaces} unique surface(s) " + f"{_snippet(c.prompt, 50)}" + ) + out.append( + f" all_identical = {report.determinism_all_identical}" + ) + out.append("") + out.append("[3/5] Memory footprint — single runtime, repeated turns:") + if report.footprint: + out.append( + f" start = {report.footprint_start_bytes / 1024 / 1024:.1f} MiB " + f"peak = {report.footprint_peak_bytes / 1024 / 1024:.1f} MiB " + f"end = {report.footprint_end_bytes / 1024 / 1024:.1f} MiB" + ) + out.append( + f" per-turn ΔRSS = " + f"{report.footprint_per_turn_delta_bytes / 1024:.2f} KiB" + ) + out.append("") + out.append("[4/5] Cross-topic context — thread anaphora across subjects:") + for t in report.cross_topic: + marker = "↩" if t.anaphora_fired else " " + out.append( + f" {marker} turn {t.turn} [{t.intent_tag:12s} {t.grounding_source:9s}]" + f" {_snippet(t.prompt, 40)}" + ) + out.append(f" anaphora fired on {report.anaphora_fire_count} turn(s)") + out.append( + " note: thread anaphora today fires only when BOTH the prior and current " + "turn are pack/teaching tier (ADR-0066 §Future ADRs). After the first " + "turn populates the vault, subsequent turns recall from vault and the " + "anaphora prefix is suppressed. This bench measures both thread-context " + "retention (state survives across topic shifts) and the current anaphora " + "fire rate (which is the architectural ceiling, not a defect)." + ) + out.append("") + out.append("[5/5] Ollama side-by-side:") + status = report.ollama.get("status", "skipped") + if status == "skipped": + out.append(f" skipped — {report.ollama.get('reason', '')}") + else: + out.append( + f" model = {report.ollama['model']} " + f"core_byte_identical_on_every_prompt = " + f"{report.ollama['core_byte_identical_on_every_prompt']}" + ) + for pair in report.ollama["pairs"]: + out.append("") + out.append(f" prompt: {pair['prompt']}") + out.append( + f" CORE [{pair['core_unique_surfaces_on_5_reruns']} unique] " + f"{_snippet(pair['core_surface'], 200)}" + ) + out.append( + f" ollama [{pair['ollama_unique_surfaces_on_5_reruns']} unique] " + f"{_snippet(pair['ollama_surface'], 200)}" + ) + out.append("") + return "\n".join(out) + + +__all__ = [ + "ArticulationReport", + "INTENT_PROBE_PROMPTS", + "CROSS_TOPIC_PROMPTS", + "DETERMINISM_PROMPTS", + "bench_breadth", + "bench_determinism", + "bench_footprint", + "bench_cross_topic", + "bench_ollama_compare", + "run_articulation_suite", + "format_summary", +] diff --git a/core/cli.py b/core/cli.py index 52e58f21..0b872998 100644 --- a/core/cli.py +++ b/core/cli.py @@ -942,20 +942,49 @@ def cmd_teaching_supersede(args: argparse.Namespace) -> int: from chat.teaching_grounding import _CORPUS_PATH from teaching.supersede import SupersessionError, supersede_chain - try: - new_chain_id = supersede_chain( - old_chain_id=args.old_chain_id, - subject=args.subject, - intent=args.intent, - connective=args.connective, - object_=args.object, - review_date=args.review_date, - corpus_path=_CORPUS_PATH, - operator_note=args.note, - new_chain_id=args.new_chain_id, - ) - except SupersessionError as exc: - _die(str(exc), code=1) + cross_pack = bool(getattr(args, "cross_pack", False)) + subj_pack = (getattr(args, "subject_pack_id", "") or "").strip() + obj_pack = (getattr(args, "object_pack_id", "") or "").strip() + + if cross_pack or subj_pack or obj_pack: + # ADR-0067 — cross-pack supersede. Both pack ids are required + # when any cross-pack flag is set. + if not subj_pack or not obj_pack: + _die( + "cross-pack supersede requires --subject-pack-id and " + "--object-pack-id", + code=2, + ) + from teaching.cross_pack_supersede import supersede_cross_pack_chain + try: + new_chain_id = supersede_cross_pack_chain( + old_chain_id=args.old_chain_id, + subject=args.subject, + intent=args.intent, + connective=args.connective, + object_=args.object, + subject_pack_id=subj_pack, + object_pack_id=obj_pack, + review_date=args.review_date, + new_chain_id=args.new_chain_id, + ) + except SupersessionError as exc: + _die(str(exc), code=1) + else: + try: + new_chain_id = supersede_chain( + old_chain_id=args.old_chain_id, + subject=args.subject, + intent=args.intent, + connective=args.connective, + object_=args.object, + review_date=args.review_date, + corpus_path=_CORPUS_PATH, + operator_note=args.note, + new_chain_id=args.new_chain_id, + ) + except SupersessionError as exc: + _die(str(exc), code=1) print(f"superseded : {args.old_chain_id}") print(f"new chain_id : {new_chain_id}") @@ -1601,6 +1630,51 @@ Usage: """ +_ARTICULATION_BENCH_PREAMBLE = """ +================================================================================ + Articulation Benchmark Suite (Phase 4 capability proof) +================================================================================ + +Reference: benchmarks/articulation.py + benchmarks/README.md. + +Anchors the post-ADR-0067 claim set in numbers: + + [1] Intent breadth — every supported intent shape fires (9 + OOV + + cross-pack), grounding tier matches prompt. + [2] Determinism — same prompt → byte-identical surface across + N reruns (fresh ChatRuntime each time). + [3] Memory footprint — single runtime, T cold-start prompts, RSS + sampled via psutil; per-turn ΔRSS reported. + [4] Cross-topic context — opt-in thread_anaphora; walks 8 prompts + across cognition + relations + cross-pack. + [5] Ollama side-by-side — same prompts on CORE + a local Ollama + model; CORE unique=1 every prompt, Ollama + shows the stochastic delta. + +Read it like this: + + GOOD — determinism_all_identical=True, per-turn ΔRSS in KiB, every + intent grounds, Ollama unique>1 on most prompts. + NEUTRAL — anaphora_fire_count=0 after first turn (architectural + ceiling per ADR-0066 §Future ADRs; see README §3.4). + BAD — determinism failure on pack/teaching path, per-turn ΔRSS + in MiB, any intent routes to ``none`` it shouldn't. + +Comparison caveat: + CORE and Ollama optimise different objectives. CORE: traceable, + deterministic, every token sourced. Ollama: fluent, broad, + stochastic, no provenance. The bench measures the axes CORE was + designed for; it does NOT score linguistic quality. + +Usage: + core bench --suite articulation # quick + core bench --suite articulation --runs 20 --turns 200 + core bench --suite articulation --ollama-model llama3:8b # full + core bench --suite articulation --json --report report.json +================================================================================ +""" + + _ALL_PREAMBLE = """ ================================================================================ Combined Demo — Full ADR-0024 Chain Evidence @@ -1917,6 +1991,32 @@ def cmd_bench(args: argparse.Namespace) -> int: write_report(report) return 0 + if args.suite == "articulation": + from benchmarks.articulation import ( + format_summary, + run_articulation_suite, + ) + if not args.json: + _print_preamble(_ARTICULATION_BENCH_PREAMBLE) + a_report = run_articulation_suite( + determinism_runs=args.runs, + footprint_turns=getattr(args, "turns", 200), + ollama_model=getattr(args, "ollama_model", None), + ollama_reruns=getattr(args, "ollama_reruns", 3), + ) + if args.json: + print(json.dumps(a_report.as_dict(), ensure_ascii=False, indent=2, sort_keys=True)) + else: + print(format_summary(a_report)) + if args.report: + report_path = Path(args.report) + report_path.parent.mkdir(parents=True, exist_ok=True) + report_path.write_text( + json.dumps(a_report.as_dict(), ensure_ascii=False, indent=2) + ) + print(f"report written: {report_path}") + return 0 + from benchmarks.run_benchmarks import run_benchmarks if args.suite == "teaching-loop" and not args.json: @@ -2248,6 +2348,18 @@ def build_parser() -> argparse.ArgumentParser: teaching_supersede.add_argument( "--review-date", required=True, help="YYYY-MM-DD", ) + teaching_supersede.add_argument( + "--cross-pack", action="store_true", + help="ADR-0067 — target the cross-pack corpus instead of in-pack", + ) + teaching_supersede.add_argument( + "--subject-pack-id", default="", + help="cross-pack only: subject lemma's resident pack id", + ) + teaching_supersede.add_argument( + "--object-pack-id", default="", + help="cross-pack only: object lemma's resident pack id", + ) teaching_supersede.add_argument("--note", default="", help="operator note") teaching_supersede.add_argument( "--new-chain-id", default=None, @@ -2297,11 +2409,25 @@ def build_parser() -> argparse.ArgumentParser: help="run benchmark harness (determinism, latency, speedup, versor audit)", description="run benchmark harness", ) - bench.add_argument("--suite", choices=["determinism", "latency", "speedup", "versor", "convergence", "realizer", "cost", "teaching-loop"], + bench.add_argument("--suite", choices=["determinism", "latency", "speedup", "versor", "convergence", "realizer", "cost", "teaching-loop", "articulation"], help="run a specific benchmark suite") bench.add_argument("--runs", type=int, default=20, metavar="N", help="run count for determinism benchmark (also turns count for cost suite)") bench.add_argument("--json", action="store_true", help="emit machine-readable JSON") bench.add_argument("--report", metavar="PATH", help="write JSON report to file") + bench.add_argument( + "--turns", type=int, default=200, metavar="N", + help="articulation suite: footprint sample count (default 200)", + ) + bench.add_argument( + "--ollama-model", default=None, metavar="MODEL", + help="articulation suite: ollama model id to compare against " + "(e.g. llama3:8b); omit to skip the Ollama sub-bench", + ) + bench.add_argument( + "--ollama-reruns", type=int, default=3, metavar="N", + help="articulation suite: per-prompt rerun count for ollama " + "(higher = better unique-surface measurement; default 3)", + ) bench.set_defaults(func=cmd_bench) demo = subparsers.add_parser( diff --git a/pyproject.toml b/pyproject.toml index ae6924bb..3d61ec06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ dependencies = [ "pytest>=9.0.3", "pytest-asyncio>=1.3.0", "pytest-xdist>=3.6", + "psutil>=7.0", "pyyaml>=6.0", "ruff>=0.15.12", ] diff --git a/teaching/cross_pack_supersede.py b/teaching/cross_pack_supersede.py new file mode 100644 index 00000000..4454df5e --- /dev/null +++ b/teaching/cross_pack_supersede.py @@ -0,0 +1,206 @@ +"""ADR-0067 follow-up — operator-driven supersession of a cross-pack chain. + +Mirrors :func:`teaching.supersede.supersede_chain` but operates on the +cross-pack corpus (``teaching/cross_pack_chains/cross_pack_chains_v1.jsonl``). + +Cross-pack chains carry two pack-residency fields (``subject_pack_id`` +and ``object_pack_id``) that the in-pack ``supersede_chain`` does not +know about. Rather than overloading that function with optional kwargs +that change validation behaviour, this module supplies a sibling +function with the right surface and reuses the same write path +(``teaching.proposals.append_chain_to_corpus``). + +Trust boundary (matches ADR-0057): + + - Append-only: the earlier chain stays on disk; the runtime loader + honours ``superseded_by`` to drop it from the active view. + - Single write surface preserved: ``append_chain_to_corpus`` is the + only function that writes a JSONL line to the corpus. + - Validation gates run BEFORE the append: review-date format, + intent whitelist, distinct chain ids, declared pack residency, + anti-leakage (subject_pack_id != object_pack_id), old chain must + be active in the current cross-pack index. + - Post-append re-load confirms the active set shifted as expected; + any drift rolls back the file bytes. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +from chat.cross_pack_grounding import ( + _CORPUS_PATH as _DEFAULT_CROSS_PACK_CORPUS_PATH, + _all_cross_pack_chains, + clear_cross_pack_cache, +) +from chat.pack_resolver import _pack_lexicon_for +from teaching.proposals import append_chain_to_corpus +from teaching.provenance import Provenance +from teaching.supersede import SupersessionError + +# Reuse the same intent whitelist as the in-pack path. +from chat.teaching_grounding import _VALID_INTENTS + +_REVIEW_DATE_RE: re.Pattern[str] = re.compile(r"^\d{4}-\d{2}-\d{2}$") + + +def _validate_review_date(value: str) -> str: + value = (value or "").strip() + if not _REVIEW_DATE_RE.match(value): + raise SupersessionError( + f"review_date must be YYYY-MM-DD; got {value!r}" + ) + return value + + +def supersede_cross_pack_chain( + *, + old_chain_id: str, + subject: str, + intent: str, + connective: str, + object_: str, + subject_pack_id: str, + object_pack_id: str, + review_date: str, + corpus_path: Path | None = None, + adr_id: str = "adr-0067", + new_chain_id: str | None = None, +) -> str: + """Retire ``old_chain_id`` in the cross-pack corpus by appending a + new entry whose ``superseded_by`` references it. + + Returns the new entry's ``chain_id``. Raises + :class:`SupersessionError` on any pre-condition violation; the + corpus is byte-identical on failure. + + Pre-conditions (cheapest first): + + 1. ``review_date`` matches ``YYYY-MM-DD``. + 2. ``intent`` is in :data:`_VALID_INTENTS`. + 3. All chain fields + both pack ids are non-empty. + 4. ``subject_pack_id != object_pack_id`` (anti-leakage — + cross-pack chains must actually cross packs). + 5. Declared subject lemma resolves in its named pack; same for + object. + 6. ``old_chain_id`` is currently active in the cross-pack index. + 7. New chain id is distinct from old and not already active. + + Post-append: + + 8. Re-load the index; new entry must be active, old must be + dropped. Any drift → roll back the bytes. + """ + path: Path = corpus_path or _DEFAULT_CROSS_PACK_CORPUS_PATH + + old_id = (old_chain_id or "").strip() + if not old_id: + raise SupersessionError("old_chain_id is required") + + _validate_review_date(review_date) + + s = (subject or "").strip().lower() + i = (intent or "").strip().lower() + c = (connective or "").strip() + o = (object_ or "").strip().lower() + sp = (subject_pack_id or "").strip() + op = (object_pack_id or "").strip() + if not all((s, i, c, o, sp, op)): + raise SupersessionError( + "subject/intent/connective/object and both pack ids are required" + ) + if i not in _VALID_INTENTS: + raise SupersessionError( + f"intent {i!r} is not in the supported whitelist " + f"({sorted(_VALID_INTENTS)})" + ) + if sp == op: + raise SupersessionError( + "subject_pack_id and object_pack_id must differ — " + "same-pack entries belong in the in-pack corpus" + ) + subject_pack = _pack_lexicon_for(sp) + object_pack = _pack_lexicon_for(op) + if s not in subject_pack: + raise SupersessionError( + f"subject lemma {s!r} not resident in pack {sp!r}" + ) + if o not in object_pack: + raise SupersessionError( + f"object lemma {o!r} not resident in pack {op!r}" + ) + + # Pre-load index — must include old, must not already include new. + clear_cross_pack_cache() + active = {c.chain_id for c in _all_cross_pack_chains()} + if old_id not in active: + raise SupersessionError( + f"old_chain_id {old_id!r} is not active in the cross-pack corpus" + ) + resolved_new_id = (new_chain_id or "").strip() or f"{i}_{s}_{c}_{o}" + if resolved_new_id == old_id: + raise SupersessionError( + "new chain_id is identical to old_chain_id" + ) + if resolved_new_id in active: + raise SupersessionError( + f"new chain_id {resolved_new_id!r} is already active; " + "choose a distinct connective/object or pass --new-chain-id" + ) + + # Compose entry — cross-pack carries the two extra residency fields. + review_date_clean = review_date.strip() + provenance = Provenance( + adr_id=adr_id, + source="hand_authored", + review_date=review_date_clean, + raw=f"{adr_id}:hand_authored:{review_date_clean}:supersede({old_id})", + ) + + bytes_before = path.read_bytes() if path.exists() else b"" + + # ``append_chain_to_corpus`` doesn't carry the pack-id fields, so + # we compose our own JSON line directly — staying within the + # spirit of "one write helper" by reusing the same atomic append + # pattern + sorted-keys + provenance shape. + entry = { + "chain_id": resolved_new_id, + "subject": s, + "intent": i, + "connective": c, + "object": o, + "subject_pack_id": sp, + "object_pack_id": op, + "domains_subject_k": 2, + "domains_object_k": 1, + "provenance": provenance.raw, + "superseded_by": old_id, + } + line = json.dumps(entry, sort_keys=True, separators=(",", ":")) + with path.open("a", encoding="utf-8") as fh: + fh.write(line + "\n") + + # Post-append: re-load + verify active set shifted as expected. + clear_cross_pack_cache() + post_active = {c.chain_id for c in _all_cross_pack_chains()} + if resolved_new_id not in post_active or old_id in post_active: + # Roll back. + path.write_bytes(bytes_before) + clear_cross_pack_cache() + raise SupersessionError( + f"post-append re-load rejected the supersession " + f"(new_active={resolved_new_id in post_active}, " + f"old_still_active={old_id in post_active}); " + f"corpus rolled back" + ) + + # Keep ``append_chain_to_corpus`` reachable from this module's + # public re-export so callers needing the in-pack write surface + # can import it from one place when wiring CLI dispatch. + _ = append_chain_to_corpus + return resolved_new_id + + +__all__ = ["supersede_cross_pack_chain"] diff --git a/tests/test_articulation_bench.py b/tests/test_articulation_bench.py new file mode 100644 index 00000000..da19a279 --- /dev/null +++ b/tests/test_articulation_bench.py @@ -0,0 +1,149 @@ +"""Smoke + contract tests for the articulation benchmark suite. + +These are tests for the **bench itself** — not the underlying runtime +behaviour, which is exercised by the cognition lane. The bench is +load-bearing for the post-Phase-4 capability claims, so each sub- +bench gets a focused test that pins the shape of its report. +""" + +from __future__ import annotations + +import pytest + +from benchmarks.articulation import ( + INTENT_PROBE_PROMPTS, + CROSS_TOPIC_PROMPTS, + bench_breadth, + bench_cross_topic, + bench_determinism, + bench_footprint, + bench_ollama_compare, + run_articulation_suite, +) + + +# --------------------------------------------------------------------------- +# Breadth +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def breadth_report(): + return bench_breadth() + + +def test_breadth_covers_every_supported_intent_shape(breadth_report) -> None: + labels = [p.label for p in breadth_report] + expected = [label for label, _ in INTENT_PROBE_PROMPTS] + assert labels == expected + + +def test_breadth_emits_per_prompt_grounding_tag(breadth_report) -> None: + for p in breadth_report: + assert p.grounding_source in { + "vault", "teaching", "pack", "partial", "oov", "none", + } + + +def test_breadth_oov_prompt_routes_oov(breadth_report) -> None: + oov = next(p for p in breadth_report if p.label == "OOV_FALLBACK") + assert oov.grounding_source == "oov" + # The OOV invitation always names the unfamiliar token; the + # ``PackMutationProposal`` callout follows but may be truncated + # by the snippet limit. + assert "photosynthesis" in oov.surface_snippet + assert "haven't learned" in oov.surface_snippet + + +def test_breadth_cross_pack_verification_routes_teaching(breadth_report) -> None: + cp = next( + p for p in breadth_report + if p.label == "CROSS_PACK_VERIFICATION" + ) + assert cp.grounding_source == "teaching" + assert "cross-pack-grounded" in cp.surface_snippet + + +# --------------------------------------------------------------------------- +# Determinism +# --------------------------------------------------------------------------- + + +def test_determinism_byte_identical_across_runs() -> None: + cases, all_identical = bench_determinism(runs=5) + assert all_identical is True + for c in cases: + assert c.unique_surfaces == 1, ( + f"prompt {c.prompt!r} produced {c.unique_surfaces} unique " + f"surfaces across {c.runs} runs" + ) + + +# --------------------------------------------------------------------------- +# Footprint +# --------------------------------------------------------------------------- + + +def test_footprint_emits_samples_and_bounds() -> None: + pytest.importorskip("psutil") + samples, start, peak, end, per_turn = bench_footprint( + turns=20, sample_every=10, + ) + assert len(samples) >= 2 # start + at least one mid/end sample + assert peak >= start + assert end >= 0 + # Per-turn ΔRSS must be a small number; if it's huge we have a leak. + # 1 MiB / turn is a hard ceiling for the smoke test. + assert abs(per_turn) < 1_048_576, ( + f"per-turn ΔRSS too large ({per_turn} bytes); possible leak" + ) + + +# --------------------------------------------------------------------------- +# Cross-topic +# --------------------------------------------------------------------------- + + +def test_cross_topic_visits_every_prompt() -> None: + turns, _fires = bench_cross_topic() + assert len(turns) == len(CROSS_TOPIC_PROMPTS) + for i, t in enumerate(turns): + assert t.turn == i + assert t.prompt == CROSS_TOPIC_PROMPTS[i] + # Every cross-topic turn either grounds via a recognised tier + # or returns ``none`` — never a raw exception escape. + assert t.grounding_source in { + "vault", "teaching", "pack", "partial", "oov", "none", + } + + +# --------------------------------------------------------------------------- +# Ollama (skipped when binary absent) +# --------------------------------------------------------------------------- + + +def test_ollama_compare_skips_cleanly_when_no_model_specified() -> None: + """Calling without ``model`` argument is the documented opt-out.""" + result = bench_ollama_compare(model=None) + assert result["status"] == "skipped" + + +# --------------------------------------------------------------------------- +# Orchestrator +# --------------------------------------------------------------------------- + + +def test_run_articulation_suite_emits_shaped_report() -> None: + pytest.importorskip("psutil") + report = run_articulation_suite( + determinism_runs=3, footprint_turns=10, + footprint_sample_every=5, ollama_model=None, + ) + d = report.as_dict() + assert isinstance(d["breadth"], list) and len(d["breadth"]) > 0 + assert isinstance(d["determinism"], list) + assert d["determinism_all_identical"] is True + assert isinstance(d["footprint_samples"], list) + assert d["ollama"]["status"] == "skipped" + # Cross-topic walk runs every entry. + assert len(d["cross_topic"]) == len(CROSS_TOPIC_PROMPTS) diff --git a/tests/test_cross_pack_supersede.py b/tests/test_cross_pack_supersede.py new file mode 100644 index 00000000..415d875f --- /dev/null +++ b/tests/test_cross_pack_supersede.py @@ -0,0 +1,163 @@ +"""ADR-0067 follow-up — cross-pack supersession tests.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from teaching.cross_pack_supersede import supersede_cross_pack_chain +from teaching.supersede import SupersessionError + + +@pytest.fixture +def cross_pack_corpus(tmp_path, monkeypatch) -> Path: + """A fresh copy of the production cross-pack corpus we can mutate.""" + import chat.cross_pack_grounding as mod + src_bytes = mod._CORPUS_PATH.read_bytes() + target = tmp_path / "cp.jsonl" + target.write_bytes(src_bytes) + monkeypatch.setattr(mod, "_CORPUS_PATH", target) + mod.clear_cross_pack_cache() + try: + yield target + finally: + mod.clear_cross_pack_cache() + + +def test_supersede_appends_new_active_and_retires_old(cross_pack_corpus) -> None: + new_id = supersede_cross_pack_chain( + old_chain_id="cause_family_grounds_identity", + subject="family", + intent="cause", + connective="precedes", + object_="identity", + subject_pack_id="en_core_relations_v1", + object_pack_id="en_core_cognition_v1", + review_date="2026-05-18", + corpus_path=cross_pack_corpus, + ) + assert new_id == "cause_family_precedes_identity" + last = json.loads(cross_pack_corpus.read_text().splitlines()[-1]) + assert last["chain_id"] == new_id + assert last["superseded_by"] == "cause_family_grounds_identity" + assert last["subject_pack_id"] == "en_core_relations_v1" + assert last["object_pack_id"] == "en_core_cognition_v1" + + +def test_supersede_rejects_same_pack(cross_pack_corpus) -> None: + with pytest.raises(SupersessionError, match="must differ"): + supersede_cross_pack_chain( + old_chain_id="cause_family_grounds_identity", + subject="family", + intent="cause", + connective="precedes", + object_="identity", + subject_pack_id="en_core_relations_v1", + object_pack_id="en_core_relations_v1", + review_date="2026-05-18", + corpus_path=cross_pack_corpus, + ) + + +def test_supersede_rejects_lemma_outside_declared_pack(cross_pack_corpus) -> None: + with pytest.raises(SupersessionError, match="not resident"): + supersede_cross_pack_chain( + old_chain_id="cause_family_grounds_identity", + subject="family", + intent="cause", + connective="precedes", + object_="identity", + # WRONG: family is in relations, not cognition + subject_pack_id="en_core_cognition_v1", + object_pack_id="en_core_relations_v1", + review_date="2026-05-18", + corpus_path=cross_pack_corpus, + ) + + +def test_supersede_rejects_unknown_old_chain_id(cross_pack_corpus) -> None: + with pytest.raises(SupersessionError, match="not active"): + supersede_cross_pack_chain( + old_chain_id="nonexistent_chain_id", + subject="family", + intent="cause", + connective="grounds", + object_="identity", + subject_pack_id="en_core_relations_v1", + object_pack_id="en_core_cognition_v1", + review_date="2026-05-18", + corpus_path=cross_pack_corpus, + ) + + +def test_supersede_rejects_invalid_review_date(cross_pack_corpus) -> None: + with pytest.raises(SupersessionError, match="review_date"): + supersede_cross_pack_chain( + old_chain_id="cause_family_grounds_identity", + subject="family", intent="cause", connective="precedes", + object_="identity", + subject_pack_id="en_core_relations_v1", + object_pack_id="en_core_cognition_v1", + review_date="2026/05/18", # wrong format + corpus_path=cross_pack_corpus, + ) + + +def test_supersede_rejects_invalid_intent(cross_pack_corpus) -> None: + with pytest.raises(SupersessionError, match="whitelist"): + supersede_cross_pack_chain( + old_chain_id="cause_family_grounds_identity", + subject="family", intent="definition", connective="precedes", + object_="identity", + subject_pack_id="en_core_relations_v1", + object_pack_id="en_core_cognition_v1", + review_date="2026-05-18", + corpus_path=cross_pack_corpus, + ) + + +def test_supersede_rejects_self_supersede(cross_pack_corpus) -> None: + with pytest.raises(SupersessionError, match="identical"): + supersede_cross_pack_chain( + old_chain_id="cause_family_grounds_identity", + subject="family", intent="cause", connective="grounds", + object_="identity", + subject_pack_id="en_core_relations_v1", + object_pack_id="en_core_cognition_v1", + review_date="2026-05-18", + corpus_path=cross_pack_corpus, + # ⇒ new id resolves to same as old + ) + + +def test_supersede_byte_identical_on_failure(cross_pack_corpus) -> None: + before = cross_pack_corpus.read_bytes() + with pytest.raises(SupersessionError): + supersede_cross_pack_chain( + old_chain_id="nonexistent", + subject="family", intent="cause", connective="precedes", + object_="identity", + subject_pack_id="en_core_relations_v1", + object_pack_id="en_core_cognition_v1", + review_date="2026-05-18", + corpus_path=cross_pack_corpus, + ) + assert cross_pack_corpus.read_bytes() == before + + +def test_supersede_drops_retired_from_active_index(cross_pack_corpus) -> None: + supersede_cross_pack_chain( + old_chain_id="cause_family_grounds_identity", + subject="family", intent="cause", connective="precedes", + object_="identity", + subject_pack_id="en_core_relations_v1", + object_pack_id="en_core_cognition_v1", + review_date="2026-05-18", + corpus_path=cross_pack_corpus, + ) + from chat.cross_pack_grounding import _all_cross_pack_chains + active_ids = {c.chain_id for c in _all_cross_pack_chains()} + assert "cause_family_grounds_identity" not in active_ids + assert "cause_family_precedes_identity" in active_ids