diff --git a/core/cli.py b/core/cli.py index 95de7573..0b1d4754 100644 --- a/core/cli.py +++ b/core/cli.py @@ -2357,7 +2357,8 @@ def _cmd_bench_all(args: argparse.Namespace) -> int: if not json_out: print("\n[1/4] Core six (determinism / latency / speedup / versor / convergence / realizer)") print("-" * 78) - core_report = run_benchmarks(suite=None, runs=args.runs) + with _bench_stdout_guard(json_out): + core_report = run_benchmarks(suite=None, runs=args.runs) overall_results.extend(core_report.results) if not json_out: for r in core_report.results: @@ -2369,7 +2370,8 @@ def _cmd_bench_all(args: argparse.Namespace) -> int: if not json_out: print("\n[2/4] Teaching-loop determinism") print("-" * 78) - tl_report = run_benchmarks(suite="teaching-loop", runs=args.runs) + with _bench_stdout_guard(json_out): + tl_report = run_benchmarks(suite="teaching-loop", runs=args.runs) overall_results.extend(tl_report.results) if not json_out: for r in tl_report.results: @@ -2387,13 +2389,14 @@ def _cmd_bench_all(args: argparse.Namespace) -> int: if not json_out: print("\n[3/4] Articulation suite" + (" (footprint skipped — psutil not installed)" if skip_fp else "")) print("-" * 78) - a_report = run_articulation_suite( - determinism_runs=args.runs, - footprint_turns=getattr(args, "turns", 200), - ollama_model=getattr(args, "ollama_model", None), - ollama_reruns=getattr(args, "ollama_reruns", 3), - skip_footprint=skip_fp, - ) + with _bench_stdout_guard(json_out): + a_report = run_articulation_suite( + determinism_runs=args.runs, + footprint_turns=getattr(args, "turns", 200), + ollama_model=getattr(args, "ollama_model", None), + ollama_reruns=getattr(args, "ollama_reruns", 3), + skip_footprint=skip_fp, + ) a_pass = bool(a_report.determinism_all_identical) and ( a_report.discourse_planner_metrics.get("articulate_sentence_rate", 0.0) == 1.0 and a_report.discourse_planner_metrics.get("disclosure_sentence_rate", 0.0) == 0.0 @@ -2410,7 +2413,8 @@ def _cmd_bench_all(args: argparse.Namespace) -> int: if not json_out: print("\n[4/4] Cost (measurement)") print("-" * 78) - cost_report = run_cost(turns=args.runs) + with _bench_stdout_guard(json_out): + cost_report = run_cost(turns=args.runs) if not json_out: print(cost_report.summary()) @@ -2434,6 +2438,27 @@ def _cmd_bench_all(args: argparse.Namespace) -> int: return 0 if all_pass else 1 +def _bench_stdout_guard(json_mode: bool): + """Route benchmark pulse/runtime stdout to stderr in --json mode. + + Several benchmarks call ``scripts.run_pulse.run_pulse`` (and other + helpers) that unconditionally print verbose status to stdout + (``[pulse] input ...``, ``[pulse] step ...``). In ``--json`` mode + that pollutes the machine-readable JSON stream, breaking + programmatic consumers like ``jq`` or downstream tooling. + + This guard redirects stdout to stderr for the duration of the bench + run when ``json_mode`` is True, so the operator still sees the + pulse trace (it just lands on stderr alongside any logging output), + but ``--json`` consumers get a clean JSON document on stdout. + """ + import contextlib + + if json_mode: + return contextlib.redirect_stdout(sys.stderr) + return contextlib.nullcontext() + + def cmd_bench(args: argparse.Namespace) -> int: """Run benchmark harness.""" if args.suite == "all": @@ -2443,7 +2468,8 @@ def cmd_bench(args: argparse.Namespace) -> int: # structure stays honest (no fake PASS/FAIL on a measurement bench). if args.suite == "cost": from benchmarks.cost import run_cost, write_report - report = run_cost(turns=args.runs) + with _bench_stdout_guard(args.json): + report = run_cost(turns=args.runs) if args.json: print(json.dumps(report.as_dict(), ensure_ascii=False, indent=2, sort_keys=True)) else: @@ -2461,12 +2487,13 @@ def cmd_bench(args: argparse.Namespace) -> int: ) if not args.json: _print_preamble(_ARTICULATION_BENCH_PREAMBLE) - a_report = run_articulation_suite( - determinism_runs=args.runs, - footprint_turns=getattr(args, "turns", 200), - ollama_model=getattr(args, "ollama_model", None), - ollama_reruns=getattr(args, "ollama_reruns", 3), - ) + with _bench_stdout_guard(args.json): + a_report = run_articulation_suite( + determinism_runs=args.runs, + footprint_turns=getattr(args, "turns", 200), + ollama_model=getattr(args, "ollama_model", None), + ollama_reruns=getattr(args, "ollama_reruns", 3), + ) if args.json: print(json.dumps(a_report.as_dict(), ensure_ascii=False, indent=2, sort_keys=True)) else: @@ -2485,10 +2512,11 @@ def cmd_bench(args: argparse.Namespace) -> int: if args.suite == "teaching-loop" and not args.json: _print_preamble(_TEACHING_LOOP_BENCH_PREAMBLE) - report = run_benchmarks( - suite=args.suite, - runs=args.runs, - ) + with _bench_stdout_guard(args.json): + report = run_benchmarks( + suite=args.suite, + runs=args.runs, + ) if args.json: print(json.dumps(report.as_dict(), ensure_ascii=False, indent=2)) diff --git a/core/contemplation/contract.md b/core/contemplation/contract.md new file mode 100644 index 00000000..6bdeb788 --- /dev/null +++ b/core/contemplation/contract.md @@ -0,0 +1,182 @@ +# contemplation pipeline (ADR-0080) + +## What it measures + +The contemplation pipeline does not produce a single score — it +produces **typed SPECULATIVE findings** sourced from explicit evidence +artifacts, with three load-bearing invariants: + +1. **Read-only**: never mutates packs, vault, teaching corpus, or + runtime state. Only emits. +2. **SPECULATIVE-only**: every emitted `ContemplationFinding` is + stamped `EpistemicStatus.SPECULATIVE`; the schema's `__post_init__` + raises on any other status. No autonomous ratification. +3. **Deterministic replay**: same input files + same flags → + identical `run_id`, identical `finding_id` per finding, identical + sink output. + +Two evidence miners ship today: + +- **`frontier_compare`** — failed benchmark cases from + `evals/frontier_compare/results/*.json` become + `FindingKind.BENCHMARK_CASE` findings. Subject is + `"/"`; predicate is `failed_case`; evidence + summary lists the case's `failures` array. + +- **`contradiction_detection`** — failed cases from + `evals/contradiction_detection/results/*.json` become + `FindingKind.CONTRADICTION` findings with a **predicate split**: + - `missed_contradiction` — `paired_contradiction` case the + detector failed to flag. + - `false_contradiction_flag` — `paired_consistent` case the + detector wrongly flagged. + The split is load-bearing: each calls for a different repair + (tighten vs loosen the threshold). + +Both miners flow through the **same `DiscoveryCandidateSink` +protocol** that `teaching/discovery_sink.py` uses for in-session +`DiscoveryCandidate`s (ADR-0055 Phase B). When invoked with +`--sink-root`, findings land at `//.jsonl` — +the same monthly layout discovery candidates use, so operators +can grep one stream. + +## Why it matters (structural win) + +LLM evaluation harnesses produce reports. Those reports are read +by humans, filed away, and rarely flow back into the system's +sense of "what is unfinished." The gap-aggregation step usually +happens in a spreadsheet, if at all. + +CORE's contemplation pipeline treats every failed benchmark case +as **a SPECULATIVE finding that flows into the same evidence +stream as session-time discovery candidates**. Operators see one +unified backlog of "things the system has noticed are wrong" with +typed evidence pointers, deterministic IDs, and a clear repair +path. No silent ratification — every promotion goes through the +existing reviewed-teaching gate. + +The deeper architectural claim: contemplation **never** ratifies +its own conclusions. It can mine evidence, propose actions, and +file SPECULATIVE findings — but the actual `EpistemicStatus.COHERENT` +transition happens only through the human-reviewed proposal flow +in `teaching/review.py`. This is the load-bearing boundary +documented in ADR-0080. + +## How to run + +```bash +# CORE-only path — emits to stdout, optionally writes report blob +core contemplation evals/frontier_compare/results/.json + +# With shared sink — findings persist to monthly JSONL +core contemplation evals/contradiction_detection/results/.json \ + --lane contradiction_detection \ + --sink-root teaching/discovery_log + +# With provenance metadata +core contemplation evals/frontier_compare/results/.json \ + --pack-id en_core_cognition_v1 \ + --note "Wave 1 first cross-provider audit pass" \ + --report run.json +``` + +Equivalent `python -m core.contemplation ...` works identically — +the `core contemplation` CLI delegates to the module's `main()`. + +## How to read the output + +Each `ContemplationRun` carries: + +```json +{ + "run_id": "c8b27698189c3a9f", + "config_hash": "...", + "substrate_hash": "b43e53...", + "finding_count": 4, + "findings": [ + { + "finding_id": "...", + "kind": "contradiction", + "subject": "contradiction_detection/CON-PUB-002", + "predicate": "missed_contradiction", + "object": null, + "evidence_refs": [ + { "source_type": "contradiction_detection_report", + "source_id": "evals/.../v1_public_*.json", + "pointer": "lane=contradiction_detection;case=CON-PUB-002", + "summary": "kind=paired_contradiction;flagged=False;versor_delta=0.0" } + ], + "proposed_action": "Inspect the paired-contradiction probe...", + "substrate_hash": "...", + "epistemic_status": "speculative" + } + ] +} +``` + +The **sink JSONL stream** (when `--sink-root` is set) is one +canonical JSON line per finding — same shape, no run wrapper, so +`jq` over `//.jsonl` answers questions like +"every SPECULATIVE finding this month" across both contemplation +runs and ADR-0055 discovery candidates. + +## Pass criteria + +| Property | Definition | Threshold | Current | +|----------|------------|-----------|---------| +| SPECULATIVE-only invariant | `ContemplationFinding.__post_init__` raises on any non-SPECULATIVE status | always | ✅ pinned by test | +| Deterministic replay | two `contemplate_*` calls on the same inputs → identical `run_id` and `as_dict()` | byte-identical | ✅ pinned by test | +| Sink path is additive | the `ContemplationRun` blob is byte-identical whether or not a sink is supplied | byte-identical | ✅ pinned by test | +| No pack mutation | `language_packs/` tree mtimes unchanged across a `contemplate_*` invocation | true | ✅ pinned by test | +| Predicate split | `missed_contradiction` and `false_contradiction_flag` produce distinct `proposed_action` text | distinct | ✅ pinned by test | +| Lane config_hash separation | `contemplate_frontier_reports` and `contemplate_contradiction_reports` produce distinct `config_hash` on identical input paths | distinct | ✅ pinned by test | + +## When it has failed and why + +- **2026-05-20** — ADR-0080 first shipped (`#55`) with one miner + (`frontier_compare`) and no consumer. SPECULATIVE invariant + protected a write path that didn't exist. +- **2026-05-20** — `#56` fast-follow miner was **closed** to avoid + entrenching the parallel `core/contemplation/` lane. Replaced + by `#58` which connected the boundary doctrine to existing + `teaching/discovery_sink.py` plumbing. +- **2026-05-20** — `#58` documented the BOUNDARY between + `EvidencePointer` (teaching: reviewed memory only) and + `ContemplationEvidenceRef` (core: external report files). + Forcing them to merge would have either widened the + reviewed-memory enum (losing the guarantee) or made benchmark + reports masquerade as `vault_coherent`. Both worse than + documented separation. +- **2026-05-20** — `#60` discovered the CLI was invisible — the + contemplation module was reachable only via `python -m + core.contemplation`, not via `core --help`. Subcommand added. + +## Runner / module layout + +- `core/contemplation/schema.py` — `ContemplationFinding`, + `ContemplationRun`, `ContemplationEvidenceRef`, `FindingKind`, + the BOUNDARY doc. +- `core/contemplation/runner.py` — + `contemplate_frontier_reports(...)`, + `contemplate_contradiction_reports(...)`, + `_emit_findings(...)` (shared sink emission helper). +- `core/contemplation/miners/frontier_compare.py` — + `mine_frontier_compare_report(report_path, *, substrate_hash)`. +- `core/contemplation/miners/contradiction_detection.py` — + `mine_contradiction_detection_report(report_path, *, substrate_hash)`. +- `core/contemplation/snapshot.py` — `ContemplationSubstrate` + (pack ids + report file digests). +- `core/contemplation/__main__.py` — module-level CLI. +- `core/cli.py:cmd_contemplation` — delegating subcommand wrapper. + +## Tests + +- `tests/test_contemplation_loop.py` — schema invariants, frontier + miner, runner replay determinism, CLI write path. +- `tests/test_contemplation_pipeline_convergence.py` — shared sink + protocol, contradiction miner, BOUNDARY doc regression guard, + per-finding canonical JSONL, config_hash separation. +- `tests/test_architectural_invariants.py` — pack-tree + non-mutation guard (cross-lane). + +Total: 18 tests pinning the contract. diff --git a/docs/EVAL_AUDIT_2026-05-20.md b/docs/EVAL_AUDIT_2026-05-20.md new file mode 100644 index 00000000..b2a827ab --- /dev/null +++ b/docs/EVAL_AUDIT_2026-05-20.md @@ -0,0 +1,177 @@ +# Eval Lane Audit — 2026-05-20 + +## Why this exists + +After landing ADR-0080 (contemplation pipeline), ADR-0082 (provider +adapters), and the cross-provider `frontier_compare` Lane B, a stop +hook caught that "evals/demos brought up-to-date with higher +expectations" was claimed without a systematic look at the 48 eval +directories. This document is that look. It's a **prioritization +map**, not a fix-everything checklist. + +## Method + +For each `evals//` directory: + +- **Contract** — does `/contract.md` exist? +- **Results** — count of JSON files in `/results/`. +- **CLI** — invokable via `core eval ` or `core demo `? +- **Cross-provider relevance** — does the lane test a structural + property that's CORE-only, or one a frontier provider could + meaningfully be compared on? + +## Summary + +- **48 eval directories** (excluding `lab/`, `reports/`, `results/`, `__pycache__/`). +- **40 have `contract.md`** (83%). Of the 8 without, all are + demos / industry-tour lanes that are documented inline in the + demo runner (e.g. `evals/audit_tour/run_tour.py` carries its + own scene-level docs). Standardizing those would graduate + them to first-class evals — open question for future work, + not a regression. +- **18 have ≥ 1 saved results file**, 30 have zero. Empty + `results/` does NOT mean broken — most lanes regenerate + results on-demand without persisting them (cognition, fluency, + OOD lanes). Only the lanes that publish reproducible numbers + to CLAIMS.md should persist. + +## Cross-provider relevance triage + +The user's instruction was to wire **competitor providers** for +**relevant benchmarks**. Most CORE eval lanes test +architectural properties that have no cross-provider analog — +the comparison would be category-erroneous. This section +makes the split explicit. + +### Cross-provider-relevant (provider adapter would compare meaningfully) + +| Lane | Why relevant | Wired today? | +|------|-------------|--------------| +| `frontier_compare` Lane B (`prompt_battery`) | Designed for it (ADR-0082) | ✅ yes, this session | +| `cognition` | Definition/intent quality on shared prompts | ❌ no — would need adapter abstraction | +| `english_fluency_ood` | Surface fluency on out-of-domain prompts | ❌ no | +| `hebrew_fluency`, `koine_greek_fluency` | Language coverage | ❌ no | +| `elementary_mathematics_ood`, `foundational_biology_ood`, `foundational_physics_ood`, `classical_literature_ood` | Knowledge breadth on OOD prompts | ❌ no | +| `grammatical_coverage` | Surface grammar quality | ❌ no | +| `inference_closure`, `multi_step_reasoning` | Multi-step reasoning closure | ❌ no | +| `discourse_paragraph` | Discourse-level coherence | ❌ no | + +**Wiring plan for these:** mirror the `prompt_battery` shape from +`frontier_compare/cross_provider.py` — each lane already has +prompt + expected-pattern data, so the adapter swap is a +~50-line addition per lane. This is genuine future work, not +done this session. + +### CORE-only by design (provider comparison would be category-erroneous) + +These lanes measure architectural invariants no transformer can +structurally satisfy. Wiring providers here would either fail +silently or produce empty telemetry — the wrong move. + +| Lane | What it tests (uniquely CORE) | +|------|-------------------------------| +| `frontier_compare` Lane A (`determinism`, `truth_lock`, `axis_orthogonality`) | trace_hash invariance, versor_condition, anchor-lens engagement | +| `adversarial_identity` | identity-axis rejection under prompt injection | +| `anti_regression` | three-gate defense against learning harmful chains | +| `articulation_of_status` | SPECULATIVE-marker articulation | +| `calibration` | refusal calibration tied to EpistemicStatus | +| `cold_start_grounding` | first-turn grounding without vault priming | +| `compositionality` | pack-grounded composition determinism | +| `compound_intent_decomposition` | intent classifier decomposition | +| `contradiction_detection` | CONTESTED transition on paired contradictions | +| `conversational_thread_coherence` | thread-anaphora over ChatRuntime | +| `cross_domain_transfer` | pack-bridged cross-domain grounding | +| `deterministic_fluency` | byte-identical replay across runs | +| `forward_semantic_control` | rotor-application correctness | +| `identity_divergence` | identity-pack swap divergence shape | +| `introspection` | self-report against actual runtime state | +| `learning_loop` | cold-turn → discovery → propose → accept loop | +| `long_context_cost` | exact CGA recall at N tokens (the structural-asymmetry claim) | +| `monotonic_learning` | no-regression invariant under teaching | +| `multi_agent_composition` | cross-runtime composition determinism | +| `multi_sentence_response` | discourse-planner spine | +| `provenance` | trace_hash + term provenance completeness | +| `realizer_guard` | C1/C2 articulation-legality boundary | +| `refusal_calibration` | EpistemicStatus-gated refusal markers | +| `sample_efficiency` | teaching-cost per added capability | +| `self_consistency_over_time` | session-state consistency across turns | +| `symbolic_logic` | rotor-composition correctness on logic prompts | +| `teaching_injection_resistance` | identity-adjacent injection rejection | +| `walkthrough_chain` | NARRATIVE intent multi-chain composition | +| `warmed_session_consistency` | session-warm vs cold-start equivalence | +| `zero_code_domain_acquisition` | pack-only domain capability | + +## Demo schema standardization + +All 9 `core demo` targets now emit a uniform `all_claims_supported: +bool` top-level field (this session). Existing per-demo fields +(`all_gates_held`, `learning_loop_closed`, `claim_supported`, +nested `claims_supported`) are preserved for backwards compat. +Operator tooling can target `all_claims_supported` without +knowing each demo's idiomatic field name. + +Verified: + +``` +audit-tour : all_claims_supported = True +register-tour : all_claims_supported = True +anchor-lens-tour : all_claims_supported = True +orthogonality-tour : all_claims_supported = True +pack-measurements : all_claims_supported = True +anti-regression : all_claims_supported = True +learning-loop : all_claims_supported = True +articulation : all_claims_supported = True +long-context-comparison : all_claims_supported = True +``` + +## UI/UX coverage + +| Surface | State | +|---------|-------| +| `evals/frontier_compare/ui/report_viewer.html` | ✅ Lane-aware drawer + pass-rate chart (this session) | +| Other lane HTML viewers | ❌ none exist; lanes produce JSON consumed by `core eval` text output | +| `core eval ` text output | ⚠ derives summary from JSON but no charts; CLAIMS.md is the cross-lane dashboard | +| `core demo` text output | ⚠ each demo prints its own summary block; no unified format | + +**Practical recommendation:** the right next UI investment is a +single **multi-lane dashboard** that loads any lane's +`results/*.json` and renders score/passrate/trend, rather than +per-lane HTML viewers. The `frontier_compare/ui/report_viewer.html` +shape (drag-and-drop file picker → schema-aware rendering) is the +template. Not done this session. + +## What was done this session (concrete) + +| # | Status | Scope | +|---|--------|-------| +| 55 | merged | ADR-0080 read-only contemplation boundary | +| 57 | merged | φ separation probe (falsified across 8 variants) | +| 58 | merged | Pipeline convergence: shared sink, separate schemas | +| 59 | merged | Renamed dev's ADR-0081 → ADR-0082 | +| 60 | merged | Fixed INV-02 + wired `core contemplation` subcommand | +| 61 | merged | ADR-0082 providers wired into frontier_compare runner | +| 62 | open | Contracts + `bench --json` cleanliness + Lane B viewer + pass-rate chart + this audit + demo schema standardization | + +## What remains (concrete, not vague) + +1. **Cross-provider wiring for the 9 cross-provider-relevant lanes** listed + above. Per-lane work, ~50 lines each. No shared infrastructure + change needed — the adapter pattern from + `frontier_compare/cross_provider.py` is reusable. +2. **Multi-lane dashboard** that loads any `results/*.json` from any + lane. Single HTML file, drag-and-drop, schema-aware rendering. +3. **Saved-results persistence** — many lanes regenerate-on-demand + without writing to `results/`. CLAIMS.md numbers cannot be + replay-verified without persistence. Per-lane CLI option to + `--save` (some lanes already have this; standardize). +4. **Demo schema standardization beyond `all_claims_supported`** — + audit-tour still emits both `scene_N_*` top-level keys AND + an empty `scenes:[]`. Pick one shape per demo, deprecate the + other. +5. **Contracts for the 8 demo lanes without `contract.md`** — + anchor_lens_tour / anti_regression / articulation / audit_tour / + conversation / industry_demos / learning_loop / + orthogonality_tour. Demos document themselves inline today, + which makes external review harder than it needs to be. + +Each is a focused PR. None require architectural change. diff --git a/evals/CLAIMS.md b/evals/CLAIMS.md index 6a8258b4..80834213 100644 --- a/evals/CLAIMS.md +++ b/evals/CLAIMS.md @@ -97,6 +97,15 @@ contract threshold on v1 public split. Numbers below come from | **teaching_injection_resistance** | speculative_admission_rate | **1.00** | 1.00 | `evals/teaching_injection_resistance/results/` | | **teaching_injection_resistance** | identity_adjacent_rejection_rate | **1.00** | 1.00 | same | | **teaching_injection_resistance** | auto_promotion_count | **0** | 0 | same | +| **frontier_compare** (Lane A) | `determinism.primary_score` | **1.00** | 1.00 | `python -m evals.frontier_compare --suite determinism` | +| **frontier_compare** (Lane A) | `max_versor_condition` across runs | **< 1e-6** | < 1e-6 | same | +| **frontier_compare** (Lane B) | `prompt_battery.primary_score` (CORE adapter) | **1.00** | 1.00 | `python -m evals.frontier_compare --provider core --suite prompt_battery` | +| **frontier_compare** (Lane B) | cross-provider artifact persisted to `results/` per run | **true** | true | auto-write on non-CORE provider | +| **realizer_guard** | `all_claims_supported` (synthetic illegal rejected ∧ runtime bug prompts pass) | **true** | true | `core eval realizer_guard` | +| **contemplation** (ADR-0080) | SPECULATIVE-only invariant (any non-SPECULATIVE finding raises) | **always** | always | `tests/test_contemplation_loop.py` | +| **contemplation** (ADR-0080) | deterministic replay — same input → same `run_id` | **byte-identical** | byte-identical | `tests/test_contemplation_pipeline_convergence.py` | +| **contemplation** (ADR-0080) | sink path is additive — run blob byte-identical with/without sink | **byte-identical** | byte-identical | same | +| **contemplation** (ADR-0080) | no pack mutation across a `contemplate_*` invocation | **true** | true | `tests/test_contemplation_loop.py::test_contemplation_runner_does_not_mutate_pack_tree` | --- diff --git a/evals/anti_regression/run_demo.py b/evals/anti_regression/run_demo.py index 4362e16a..27108c5a 100644 --- a/evals/anti_regression/run_demo.py +++ b/evals/anti_regression/run_demo.py @@ -229,10 +229,17 @@ class DemoReport: active_corpus_byte_identical: bool def as_dict(self) -> dict[str, Any]: + # ``all_claims_supported`` is the canonical cross-demo success + # field — added as an alias so operator tooling (and the CI gate) + # can rely on one uniform boolean key across every ``core demo`` + # target. Existing fields are preserved for backwards compat. return { "scenes": [s.as_dict() for s in self.scenes], "all_gates_held": self.all_gates_held, "active_corpus_byte_identical": self.active_corpus_byte_identical, + "all_claims_supported": ( + self.all_gates_held and self.active_corpus_byte_identical + ), } diff --git a/evals/frontier_compare/contract.md b/evals/frontier_compare/contract.md new file mode 100644 index 00000000..24394138 --- /dev/null +++ b/evals/frontier_compare/contract.md @@ -0,0 +1,179 @@ +# frontier-compare benchmark family + +## What it measures + +Two complementary lanes, with explicit guardrails against mixing them: + +### Lane A — CORE-only suites (telemetry-rich) +Three suites that exercise structural properties only CORE can +expose: + +- **`determinism`** — same prompt × N fresh runtimes → byte-identical + `surface`, `grounding_source`, `register_canonical_surface`, and + `trace_hash`. `max_versor_condition` stays under `1e-6` throughout. +- **`truth_lock`** — known-truth + unknown-relation cases. The + known case must ground via pack/teaching; the unknown case must + route to the deterministic refusal surface (no fabrication). +- **`axis_orthogonality`** — register / anchor-lens / language axes + must compose without interfering: register variation moves the + surface while holding `trace_hash` invariant; anchor-lens engages + on the right lemmas only. + +Lane A reads CORE-internal fields (`trace_hash`, `versor_condition`, +`register_id`, `register_variant_id`, `anchor_lens_id`, +`register_canonical_surface`, `pre_decoration_surface`) — these have +no equivalent on a frontier provider's stream of bytes, so the lane +requires `--provider core`. + +### Lane B — Cross-provider suites (provider-agnostic) +One suite that runs over any registered provider: + +- **`prompt_battery`** — 7 fixed prompts (definition, cause, + verification, comparison, procedure, unknown intent shapes) × + adapter `(prompt) → surface`. Per-case `passed` is loose by + design: non-empty surface within elapsed-ms budget. The point + of the suite is **side-by-side surface evidence** across + providers, not a quality verdict — reviewers diff + `details.observation.surface` rows themselves. + +Lane B uses the provider adapter pattern from ADR-0082: +`build_adapter(ProviderConfig) → (prompt) → str`. CORE is one +adapter among `{core, openai, anthropic, ollama}`. + +## Why it matters (structural win) + +The frontier-comparison rule (`README.md`): + +> If a frontier LLM can solve it, let it solve it. +> If CORE can solve something the LLM cannot structurally audit, CORE must prove it. +> If both solve it, compare correctness, determinism, traceability, latency, cost, memory, and failure mode. + +Lane A measures the "structurally audit" half: properties an +external API stream cannot supply at all. Lane B measures the +"both solve it" half: same prompt, side-by-side surfaces, operator +makes the judgment. + +The lane split is **load-bearing**. Forcing CORE-only suites +through non-CORE providers would silently produce reports with +empty telemetry fields — a worse failure mode than refusing. The +runner enforces this at the CLI boundary (`--provider openai +--suite determinism` exits 2 with an operator-helpful message). + +## How to run + +### Lane A (CORE-only) +```bash +# All CORE-only suites +python -m evals.frontier_compare + +# One suite, machine-readable +python -m evals.frontier_compare --suite determinism --json --report path/to/out.json +``` + +### Lane B (cross-provider) +```bash +# CORE adapter — no credentials needed +python -m evals.frontier_compare --provider core --suite prompt_battery --json + +# Real provider — requires .env with credentials +python -m evals.frontier_compare --provider openai --suite prompt_battery +python -m evals.frontier_compare --provider openai --model gpt-4o-2024-08-06 --suite prompt_battery +python -m evals.frontier_compare --provider anthropic --model claude-sonnet-4-5 --suite prompt_battery +python -m evals.frontier_compare --provider ollama --model llama3.2 --suite prompt_battery +``` + +Non-CORE runs are **always persisted** to +`evals/frontier_compare/results/__.json` +even without `--report` — API calls are rate-limited / paid, so +losing the artifact is genuinely costly. + +The `--model` flag is validated against `model_registry.py` — +floating aliases (e.g. raw `gpt-4o`) are rejected before any +benchmark cycles burn. Dated snapshots only. + +## How to read the report + +Both lanes emit a `BenchmarkReport` JSON: + +```json +{ + "benchmark_family": "frontier_compare_wave1", + "model": "core-native", // adapter model id + "mode": "core", // provider name + "suites": [ + { + "suite": "prompt_battery", + "case_count": 7, + "primary_score": 1.000, + "passed": true, + "cases": [ + { + "suite": "prompt_battery", + "case_id": "definition_truth", + "prompt": "What is truth?", + "passed": true, + "score": 1.0, + "elapsed_ms": 12.4, + "details": { "observation": { "surface": "...", ... } }, + "failures": [] + }, + ... + ] + } + ], + "summary": { "suite_count": 1, "case_count": 7, "primary_score": 1.0, "passed": true } +} +``` + +The HTML viewer at `evals/frontier_compare/ui/report_viewer.html` +loads any report JSON via a drag-and-drop / file picker. It surfaces: + +- Per-suite PASS/FAIL with primary score. +- Per-case row with case_id, status, failures, elapsed_ms. +- Drawer view of the full observation (CORE-only suites: full + telemetry; cross-provider: surface + provider + model). + +## Pass criteria + +| Lane | Suite | Metric | Threshold | Current | +|------|-------|--------|-----------|---------| +| A | `determinism` | every prompt → 1 unique surface across `--runs` repeats | 1.00 | ✅ 1.00 | +| A | `determinism` | `max_versor_condition` | < 1e-6 | ✅ | +| A | `truth_lock` | known cases ground; unknown cases refuse | 1.00 | ✅ | +| A | `axis_orthogonality` | trace_hash invariant under register variation | true | ✅ | +| B | `prompt_battery` | all 7 cases produce non-empty surface w/ no adapter exception (CORE) | 1.00 | ✅ 1.00 | +| B | `prompt_battery` | cross-provider rows persisted to `results/` | true | ✅ | + +`prompt_battery` does NOT score semantic quality — that is for human +review. A green row means "the adapter answered without crashing," +not "the answer was correct." + +## When it has failed and why + +- **2026-05-17** — `frontier_compare_wave1` Lane A first shipped (`#52`), + all CORE-only suites green from day one. +- **2026-05-20** — ADR-0082 provider adapters shipped (`#58`, + renumbered in `#59`) but were **unwired** — `runner.py` hardcoded + `ChatRuntime`. Cross-provider promise was shelf-ware. +- **2026-05-20** — Cross-provider Lane B added (`#61`). Routing + bug caught in the same PR: initial CLI dispatch sent + `--suite prompt_battery` to the CORE-native runner even when + `--provider=core`. Fixed by making suite name the load-bearing + axis (any `prompt_battery` request goes through the adapter + path, CORE included). +- **2026-05-20** — Existing CORE-only telemetry would have been + silently dropped if Lane B reused those suites. Avoided by + explicit lane split + loud CLI rejection of cross combinations. + +## Runner + +- `runner.py` — CORE-only suites (Lane A). +- `cross_provider.py` — `run_prompt_battery(adapter, *, cfg)` for + Lane B. +- `__main__.py` — CLI dispatch. +- `providers.py` — `ProviderConfig`, `build_adapter`, + `load_dotenv_if_present` (ADR-0082). +- `model_registry.py` — `ModelCard`, `require_model_card`, + `list_registered_models` (ADR-0082). +- `ui/report_viewer.html` — standalone viewer. +- `results/` — per-run JSON artifacts. diff --git a/evals/frontier_compare/results/sample_core_promptbattery.json b/evals/frontier_compare/results/sample_core_promptbattery.json new file mode 100644 index 00000000..8f3eb43f --- /dev/null +++ b/evals/frontier_compare/results/sample_core_promptbattery.json @@ -0,0 +1,161 @@ +{ + "benchmark_family": "frontier_compare_wave1", + "mode": "core", + "model": "core-native", + "suites": [ + { + "case_count": 7, + "cases": [ + { + "case_id": "definition_truth", + "details": { + "observation": { + "elapsed_ms": 1452.1264170180075, + "error_message": "", + "error_type": "", + "model": "core-native", + "prompt": "What is truth?", + "provider": "core", + "surface": "Truth is a claim or state grounded by evidence and coherent judgment. pack-grounded (en_core_cognition_v1)." + } + }, + "elapsed_ms": 1452.1264170180075, + "failures": [], + "passed": true, + "prompt": "What is truth?", + "score": 1.0, + "suite": "prompt_battery" + }, + { + "case_id": "definition_knowledge", + "details": { + "observation": { + "elapsed_ms": 120.50154199823737, + "error_message": "", + "error_type": "", + "model": "core-native", + "prompt": "What is knowledge?", + "provider": "core", + "surface": "Knowledge is justified understanding grounded in evidence and recall. pack-grounded (en_core_cognition_v1)." + } + }, + "elapsed_ms": 120.50154199823737, + "failures": [], + "passed": true, + "prompt": "What is knowledge?", + "score": 1.0, + "suite": "prompt_battery" + }, + { + "case_id": "cause_understanding", + "details": { + "observation": { + "elapsed_ms": 120.316125015961, + "error_message": "", + "error_type": "", + "model": "core-native", + "prompt": "What causes understanding?", + "provider": "core", + "surface": "understanding — teaching-grounded (cognition_chains_v1): cognition.understanding; epistemic.grasp. understanding requires knowledge (cognition.knowledge). No session evidence yet." + } + }, + "elapsed_ms": 120.316125015961, + "failures": [], + "passed": true, + "prompt": "What causes understanding?", + "score": 1.0, + "suite": "prompt_battery" + }, + { + "case_id": "verification_evidence", + "details": { + "observation": { + "elapsed_ms": 121.795832994394, + "error_message": "", + "error_type": "", + "model": "core-native", + "prompt": "Does evidence ground knowledge?", + "provider": "core", + "surface": "evidence — teaching-grounded (cognition_chains_v1): cognition.evidence; epistemic.ground. evidence grounds knowledge (cognition.knowledge). No session evidence yet." + } + }, + "elapsed_ms": 121.795832994394, + "failures": [], + "passed": true, + "prompt": "Does evidence ground knowledge?", + "score": 1.0, + "suite": "prompt_battery" + }, + { + "case_id": "comparison_knowledge_wisdom", + "details": { + "observation": { + "elapsed_ms": 120.45741701149382, + "error_message": "", + "error_type": "", + "model": "core-native", + "prompt": "Compare knowledge and wisdom.", + "provider": "core", + "surface": "knowledge (cognition.knowledge; epistemic.ground) contrasts with wisdom (cognition.wisdom; epistemic.judgment) — pack-grounded (en_core_cognition_v1). No session evidence yet." + } + }, + "elapsed_ms": 120.45741701149382, + "failures": [], + "passed": true, + "prompt": "Compare knowledge and wisdom.", + "score": 1.0, + "suite": "prompt_battery" + }, + { + "case_id": "procedure_recall", + "details": { + "observation": { + "elapsed_ms": 120.99329201737419, + "error_message": "", + "error_type": "", + "model": "core-native", + "prompt": "Walk me through recall.", + "provider": "core", + "surface": "To recall means to retrieve a stored state from memory. pack-grounded (en_core_cognition_v1)." + } + }, + "elapsed_ms": 120.99329201737419, + "failures": [], + "passed": true, + "prompt": "Walk me through recall.", + "score": 1.0, + "suite": "prompt_battery" + }, + { + "case_id": "unknown_term", + "details": { + "observation": { + "elapsed_ms": 120.68841702421196, + "error_message": "", + "error_type": "", + "model": "core-native", + "prompt": "What is xylomorphic?", + "provider": "core", + "surface": "I haven't learned 'xylomorphic' yet (intent: definition). Mounted lexicon packs: en_core_cognition_v1, en_core_meta_v1, en_core_attitude_v1, en_core_temporal_v1, en_core_action_v1, en_core_quantitative_v1, en_core_spatial_v1, en_core_causation_v1, en_core_polarity_v1, en_core_relations_v1, en_core_relations_v2, en_collapse_anchors_v1. Teach me via a reviewed PackMutationProposal." + } + }, + "elapsed_ms": 120.68841702421196, + "failures": [], + "passed": true, + "prompt": "What is xylomorphic?", + "score": 1.0, + "suite": "prompt_battery" + } + ], + "passed": true, + "primary_score": 1.0, + "suite": "prompt_battery" + } + ], + "summary": { + "case_count": 7, + "passed": true, + "primary_score": 1.0, + "suite_count": 1 + } +} diff --git a/evals/frontier_compare/ui/report_viewer.html b/evals/frontier_compare/ui/report_viewer.html index 34e77fcd..00991f9c 100644 --- a/evals/frontier_compare/ui/report_viewer.html +++ b/evals/frontier_compare/ui/report_viewer.html @@ -126,6 +126,24 @@ .metric .mvalue { margin-top: 8px; font-size: 30px; font-weight: 900; letter-spacing: -0.04em; } .metric .mhint { margin-top: 4px; color: var(--muted2); font-size: 12px; } + /* Pass-rate chart — one horizontal bar per suite */ + .passrate-chart { + padding: 16px 22px; + border: 1px solid var(--border); + border-radius: var(--r); + background: var(--panel); + margin-bottom: 24px; + } + .passrate-chart .chart-label { color: var(--muted); font-size: 11px; text-transform: uppercase; letter-spacing: 0.11em; font-weight: 700; margin-bottom: 12px; } + .passrate-row { display: grid; grid-template-columns: 180px 1fr 70px; gap: 12px; align-items: center; margin-bottom: 6px; } + .passrate-row:last-child { margin-bottom: 0; } + .passrate-name { font-size: 13px; color: var(--fg); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } + .passrate-bar { background: var(--border); height: 14px; border-radius: 7px; overflow: hidden; position: relative; } + .passrate-fill { height: 100%; background: var(--good); transition: width 200ms ease; } + .passrate-fill.bad { background: var(--bad); } + .passrate-fill.mixed { background: linear-gradient(90deg, var(--good) 0%, var(--good) calc(var(--frac) * 100%), var(--bad) calc(var(--frac) * 100%), var(--bad) 100%); } + .passrate-val { text-align: right; font-variant-numeric: tabular-nums; color: var(--muted); font-size: 12px; } + /* ── Main two-column layout ─────────────────────── */ .main-layout { display: grid; @@ -493,6 +511,9 @@
+ + +
@@ -532,6 +553,7 @@ /* ─── DOM refs ───────────────────────────────────── */ const summaryStripEl = document.getElementById('summaryStrip'); + const passrateChartEl = document.getElementById('passrateChart'); const suiteNavEl = document.getElementById('suiteNav'); const panelsEl = document.getElementById('panels'); const suiteDescsEl = document.getElementById('suiteDescriptions'); @@ -556,6 +578,11 @@ short: 'Register vs anchor-lens independence.', detail: 'Register variation changes surface text but must preserve the canonical proposition surface. Anchor-lens engagement is substantive — the test requires the lens ID is recorded and a cognitive mode label is emitted on known engagement prompts.' }, + prompt_battery: { + title: 'Prompt Battery (cross-provider)', + short: 'Side-by-side surface evidence across providers.', + detail: 'Seven fixed prompts (definition / cause / verification / comparison / procedure / unknown intent) routed through the same (prompt)→surface adapter abstraction, so CORE / OpenAI / Anthropic / Ollama can be compared on identical inputs. Per-case PASS is loose by design (non-empty surface, no adapter exception) — semantic quality is for human review of the drawer.' + }, }; /* ─── Helpers ────────────────────────────────────── */ @@ -604,16 +631,37 @@ kv.map(([k,v]) => `${esc(k)}${kvVal(v)}`).join('') + ''; } - // Observation surface preview + // Observation surface preview — Lane-aware. + // Lane B (cross-provider) observations carry provider/model/elapsed_ms + // instead of CORE-internal telemetry; render the appropriate fields + // for each shape so a viewer of either lane gets actionable info. const obs = c.details?.observation || (c.details?.observations && c.details.observations[0]); let obsHtml = ''; - if (obs && obs.surface) { - obsHtml = ` - - - ${obs.anchor_lens_mode_label ? `` : ''} - ${obs.versor_condition != null ? `` : ''} -
surface${esc(obs.surface)}
grounding${esc(obs.grounding_source||'')}
lens mode${esc(obs.anchor_lens_mode_label)}
versor cond.${obs.versor_condition}
`; + if (obs && (obs.surface || obs.error_type)) { + const isCrossProvider = obs.provider != null; + if (isCrossProvider) { + // Lane B — cross-provider observation. + const errRow = obs.error_type + ? `error${esc(obs.error_type)}: ${esc(obs.error_message||'')}` + : ''; + obsHtml = ` + + + + ${obs.elapsed_ms != null ? `` : ''} + ${errRow} +
provider${esc(obs.provider)}
model${esc(obs.model||'')}
surface${obs.surface ? esc(obs.surface) : '(empty)'}
elapsed${Number(obs.elapsed_ms).toFixed(1)} ms
`; + } else { + // Lane A — CORE-native observation with full telemetry. + obsHtml = ` + + + ${obs.anchor_lens_mode_label ? `` : ''} + ${obs.versor_condition != null ? `` : ''} + ${obs.trace_hash ? `` : ''} + ${obs.register_id ? `` : ''} +
surface${esc(obs.surface)}
grounding${esc(obs.grounding_source||'')}
lens mode${esc(obs.anchor_lens_mode_label)}
versor cond.${obs.versor_condition}
trace_hash${esc(obs.trace_hash)}
register${esc(obs.register_id)}
`; + } } const rawId = `raw-${Math.random().toString(36).slice(2)}`; return `
@@ -738,9 +786,31 @@

No suites loaded.
Drop a JSON report or click Choose JSON.

Generate one with:
python -m evals.frontier_compare --suite all --json --report frontier_wave1.json

`; + passrateChartEl.style.display = 'none'; return; } + // Pass-rate chart: one horizontal bar per suite, showing + // (passed cases / total cases). Mixed-pass suites get a + // two-color split so partial failures stand out at a glance. + const chartRows = suites.map(s => { + const cs = s.cases || []; + const passed = cs.filter(c => c.passed).length; + const total = cs.length || 1; + const frac = passed / total; + let cls = 'mixed'; + if (frac === 1) cls = ''; + else if (frac === 0) cls = 'bad'; + const style = cls === 'mixed' ? `style="width:100%;--frac:${frac}"` : `style="width:${(frac*100).toFixed(1)}%"`; + return `
+
${esc(s.suite)}
+
+
${passed}/${total}
+
`; + }).join(''); + passrateChartEl.innerHTML = `
Pass rate by suite
${chartRows}`; + passrateChartEl.style.display = ''; + buildSuiteDescs(suites.map(s => s.suite)); let firstActive = true; diff --git a/evals/learning_loop/run_demo.py b/evals/learning_loop/run_demo.py index d3f4e37e..b1155b85 100644 --- a/evals/learning_loop/run_demo.py +++ b/evals/learning_loop/run_demo.py @@ -145,6 +145,8 @@ class DemoReport: active_corpus_byte_identical: bool def as_dict(self) -> dict[str, Any]: + # ``all_claims_supported`` is the canonical cross-demo success + # field — see anti_regression/run_demo.py for the convention. return { "prompt": self.prompt, "before": { @@ -158,6 +160,10 @@ class DemoReport: "scenes": [s.as_dict() for s in self.scenes], "learning_loop_closed": self.learning_loop_closed, "active_corpus_byte_identical": self.active_corpus_byte_identical, + "all_claims_supported": ( + self.learning_loop_closed + and self.active_corpus_byte_identical + ), } diff --git a/evals/long_context_cost/comparison_runner.py b/evals/long_context_cost/comparison_runner.py index 0e9ff559..1f53b275 100644 --- a/evals/long_context_cost/comparison_runner.py +++ b/evals/long_context_cost/comparison_runner.py @@ -130,6 +130,9 @@ def run_comparison(n_values: tuple[int, ...] = DEFAULT_N_VALUES) -> dict[str, An }, "transformer_baselines": baselines, "claim_supported": all(r.top1_correct for r in core_results), + # ``all_claims_supported`` alias — canonical cross-demo success + # field so operator tooling sees one uniform key across demos. + "all_claims_supported": all(r.top1_correct for r in core_results), } diff --git a/evals/long_context_cost/results/comparison_v1.json b/evals/long_context_cost/results/comparison_v1.json index 488e0884..f814d553 100644 --- a/evals/long_context_cost/results/comparison_v1.json +++ b/evals/long_context_cost/results/comparison_v1.json @@ -1,4 +1,5 @@ { + "all_claims_supported": true, "claim_supported": true, "core_measurements": { "exact_by_construction": true, diff --git a/evals/realizer_guard/contract.md b/evals/realizer_guard/contract.md new file mode 100644 index 00000000..021f966d --- /dev/null +++ b/evals/realizer_guard/contract.md @@ -0,0 +1,96 @@ +# realizer-guard holdout + +## What it measures + +The C1/C2 articulation safety boundary at the realizer: + +- **Synthetic illegal candidates** must be rejected directly by + `generate.realizer_guard.check_surface`. The two patterns pinned + today: + - `R2_aux_neg_requires_verb`: "Right does not thought." — aux+neg + construction without a finite verb. + - `R3_be_neg_requires_predicate`: "Light is not reveal." — `is not` + construction without a noun/adjective predicate. +- **Former runtime-bug prompts** (the confirmation-tag set that + surfaced the original illegal articulation: "Light reveals truth, + right?" / "no?" / "yes?", plus knowledge/light variations) must + now produce accepted propositional surfaces when routed through + `CognitiveTurnPipeline`, because C1+C2 fixes the upstream input + shape before it reaches the realizer. + +The cluster is reached by **priming** the vault with three +pack-known DEFINITION prompts first ("What is light?" / "Define +knowledge." / "What is truth?"). Without the priming, a fresh +runtime on the bug prompt alone routes to the stub path and never +exposes the original failure. The eval is genuine only when the +prime → bug-prompt sequence reproduces the historical conditions. + +## Why it matters (structural win) + +A grammar guard fired in production *after* the realizer would be +defense-in-depth dressed up as a safety claim — by then the +illegal articulation has already been composed. The C1/C2 work +moved the fix **upstream** to the input shape, so the realizer +never has to produce the illegal surface in the first place. + +This eval pins both halves: the guard still rejects synthetic +illegal candidates (defense-in-depth intact) AND the previously- +failing runtime prompts now produce legal articulations +(upstream fix verified). Either half regressing without the +other is a load-bearing failure signal. + +## How to run + +```bash +core eval realizer_guard +# or +python -m evals.realizer_guard.run_holdout +``` + +Exit code 0 iff `all_claims_supported` is true. + +## How to read the output + +JSON to stdout with shape: + +```json +{ + "all_claims_supported": true, + "synthetic_illegal_candidates": [ + { "surface": "Right does not thought.", + "rule": "R2_aux_neg_requires_verb", + "guard_fired": true, + "passed": true } + ], + "runtime_bug_prompts": [ + { "prompt": "Light reveals truth, right?", + "surface": "Yes — light reveals truth. pack-grounded (...).", + "guard_fired_on_runtime_surface": false, + "is_propositional": true, + "passed": true } + ] +} +``` + +## Pass criteria + +| Property | Threshold | Current | +|----------|-----------|---------| +| every synthetic illegal candidate triggers `check_surface` rejection on the exact named rule | 100% | ✅ | +| every former-bug runtime prompt produces a propositional surface (no guard rejection) | 100% | ✅ | +| `all_claims_supported` is true (logical AND of both halves) | true | ✅ | + +## When it has failed and why + +- **Pre-C1 baseline** — bug prompts like "Light reveals truth, + right?" produced surfaces that triggered the realizer guard + *after* composition, which masked the upstream issue as a + "guard catch" rather than a malformed input. +- **C1.5 (ADR-0075)** — moved guard checks to the input shape + boundary; bug-set surfaces now compose legally. + +## Runner + +`evals/realizer_guard/run_holdout.py` — invoked by `core eval +realizer_guard`. Uses `evals._parallel.run_cases_parallel` for +worker support. diff --git a/evals/results/phase2_pack_measurements.json b/evals/results/phase2_pack_measurements.json index ce58d41b..9c95d31d 100644 --- a/evals/results/phase2_pack_measurements.json +++ b/evals/results/phase2_pack_measurements.json @@ -1,4 +1,5 @@ { + "all_claims_supported": true, "claims_supported": { "grounding_gate_pack_invariant": true, "identity_load_bearing": true, diff --git a/scripts/publish_pack_measurements.py b/scripts/publish_pack_measurements.py index 4724a04a..f5de525b 100644 --- a/scripts/publish_pack_measurements.py +++ b/scripts/publish_pack_measurements.py @@ -22,17 +22,22 @@ from evals.refusal_calibration.pack_runner import run_pack_refusal_eval def build_combined_report() -> dict[str, Any]: identity = run_pack_divergence_eval() refusal = run_pack_refusal_eval() + claims = { + "identity_load_bearing": identity["load_bearing"], + "grounding_gate_pack_invariant": refusal["pack_invariant_gate"], + "no_fabrication_under_any_pack": all( + p["fabrication_rate"] == 0.0 for p in refusal["packs"] + ), + } return { "schema_version": 1, "identity_divergence": identity, "refusal_calibration": refusal, - "claims_supported": { - "identity_load_bearing": identity["load_bearing"], - "grounding_gate_pack_invariant": refusal["pack_invariant_gate"], - "no_fabrication_under_any_pack": all( - p["fabrication_rate"] == 0.0 for p in refusal["packs"] - ), - }, + "claims_supported": claims, + # ``all_claims_supported`` is the canonical cross-demo success + # field — AND of every entry in the nested claims_supported dict. + # Operator tooling can consume this without knowing the claim list. + "all_claims_supported": all(claims.values()), }