core-labs/core
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

feat: ADR-0112 runnable expert-demo showcase (core demo expert --domain <id>)

Browse source 
Closes the asymmetry between the `expert-demo` ledger status (audit
artifact only) and the actual `core demo` surface (runnable
walkthroughs producing HTML + JSON). Until this commit the word
"demo" in `expert-demo` was aspirational; now it corresponds to
something a reader can open.

What it does

- Reads the signed expert_demo_claims entry from docs/reviewers.yaml
- Loads latest on-disk result files for each attached lane × split
- Re-derives the evidence-bundle digest and asserts byte-for-byte
  match against the signed claim_digest — this is the load-bearing
  audit step, now exercised at two independent enforcement points
  (ledger gate + showcase)
- Runs each lane's metrics through the ADR-0109 lane-shape registry
  and surfaces the verdict
- Picks the first three cases from each split verbatim (deterministic
  by file order) and renders them as HTML for inspection
- Emits expert_demo.json (canonical bytes, deterministic) + expert_demo.html

Surface

  core demo expert --domain mathematics_logic
  core demo expert --domain physics
  # → evals/expert_demos/<domain>/latest/expert_demo.{json,html}

Read-only by construction: cannot mutate docs/reviewers.yaml or any
lane result file. Tested. Unpromoted domains raise ValueError —
no silent fallback, no "preview" mode that fakes a showcase.

Generated artifacts are gitignored — the inputs they derive from are
already committed, so duplicating the renders would just churn the
tree.

Tests: 16 new cases pinning all five ADR-0112 invariants. Smoke suite
still 67/67 green.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Shay 2026-05-22 14:59:27 -07:00
parent 4e66af644a
commit bd7005c786
7 changed files with 768 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.gitignore vendored
View file

@ -24,3 +24,8 @@ frontier_wave1.json
# Claude Code local session artifacts (per-developer, never tracked)
.claude/
# Runnable expert-demo showcases (ADR-0112) are generated on demand from
# the signed claim + on-disk lane result files. The inputs are committed;
# the renders are not.
evals/expert_demos/

10
README.md
View file

@ -229,6 +229,16 @@ CORE distinguishes *contract-passing* from *demonstrated*. A pack that satisfies
The contract has now demonstrated its load-bearing behavior end-to-end: refused one promotion attempt honestly ([ADR-0107](docs/decisions/ADR-0107-mathematics-logic-expert-demo-deferred.md)), amended its threshold rules once cleanly (ADR-0109), succeeded against `mathematics_logic` (ADR-0110), and succeeded against a second distinct domain `physics` without further contract change (ADR-0111). External readers can distinguish the two ceilings at a glance; the "math-only" objection is retired.
**See the actual demonstration ([ADR-0112](docs/decisions/ADR-0112-runnable-expert-demo-showcase.md)):**
```bash
core demo expert --domain mathematics_logic
core demo expert --domain physics
# → evals/expert_demos/<domain>/latest/expert_demo.html
```
Each run re-derives the signed evidence-bundle digest from on-disk lane result files, asserts byte-for-byte match against `docs/reviewers.yaml`, and renders an HTML showcase with per-lane shape-check verdicts plus the first three sample cases from each split. The composer is read-only and byte-deterministic (same inputs → same SHA-256). An unpromoted domain produces a typed refusal, not a fake showcase.
Full ADR index, frontier, and chain notes: [`docs/decisions/README.md`](docs/decisions/README.md).
---

57
core/cli.py
View file

@ -2107,6 +2107,47 @@ def cmd_demo(args: argparse.Namespace) -> int:
print(json.dumps(result, indent=2, sort_keys=True, default=str))
return 0 if result.get("all_claims_supported", False) else 1
if target == "expert":
from core.demos.expert_demo import run_expert_demo
domain_id = getattr(args, "domain", None)
if not domain_id:
print(
"core demo expert <domain>: --domain (or positional) required",
file=sys.stderr,
)
return 2
out_dir = args.output_dir
if out_dir is None:
out_dir = Path("evals/expert_demos") / domain_id / "latest"
try:
result = run_expert_demo(domain_id=domain_id, output_dir=out_dir)
except (FileNotFoundError, ValueError) as exc:
print(f"core demo expert: {exc}", file=sys.stderr)
return 1
if args.json:
print(json.dumps(result, indent=2, sort_keys=True, default=str))
else:
print(f"expert-demo: {out_dir / 'expert_demo.json'}")
print(f" html: {out_dir / 'expert_demo.html'}")
dv = result["digest_verification"]
mark = "✓" if dv["matches"] else "✗"
print(
f" digest_match: {mark} "
f"signed={dv['signed'][:16]}… derived={dv['derived'][:16]}…"
)
for lane in result["lanes"]:
for split_name, split in lane["splits"].items():
sc = split["shape_check"]
smark = "✓" if sc["passed"] else "✗"
print(
f" {smark} {lane['lane_id']:32s} {split_name:8s} "
f"({sc['shape']}): {sc['reason']}"
)
print(f"all_claims_supported: {result['all_claims_supported']}")
return 0 if result["all_claims_supported"] else 1
if target == "showcase":
from core.demos.showcase import render_html, run_showcase
@ -3097,6 +3138,7 @@ def build_parser() -> argparse.ArgumentParser:
"articulation",
"conversation",
"showcase",
"expert",
"all",
"list-results",
],
@ -3130,6 +3172,11 @@ def build_parser() -> argparse.ArgumentParser:
"WALKTHROUGH multi-sentence articulation + determinism gate. "
"conversation: layperson-facing chat transcript with live "
"word-by-word streaming and plain-English captions. "
"expert <domain>: per-domain runnable expert-demo showcase "
"(ADR-0112). Reads the signed expert_demo_claims entry, "
"re-derives the digest from on-disk lane result files, "
"asserts byte-for-byte match, surfaces sample cases per "
"attached lane × split. Pair with --domain <id>. "
"list-results: index every JSON report in the results directory."
),
)
@ -3155,6 +3202,16 @@ def build_parser() -> argparse.ArgumentParser:
"(default: evals/public_demo/results/<sha>/)"
),
)
demo.add_argument(
"--domain",
type=str,
default=None,
metavar="ID",
help=(
"for `expert` target: domain id whose signed expert_demo "
"claim should be rendered (e.g. `mathematics_logic`, `physics`)"
),
)
demo.add_argument(
"--no-stream",
dest="no_stream",

326
core/demos/expert_demo.py Normal file
View file

@ -0,0 +1,326 @@
"""ADR-0112 Runnable Expert-Demo Showcase composer.
Renders a per-domain runnable demonstration of an ``expert-demo``
ledger row. Reads the signed ``expert_demo_claims`` entry, re-derives
the evidence-bundle digest from on-disk lane result files, asserts
byte-for-byte match, then composes a JSON + HTML showcase exposing:
- the signed claim metadata (reviewer, evidence_revision, lanes)
- the recomputed digest alongside the signed one
- per-lane shape-check verdicts on public + holdout splits
- a small sample of cases (first N) drawn verbatim from each result
file so an external reader can inspect what the domain actually
produced
The composer does NOT re-run the lanes. The lane result files are the
artifact the digest covers; replaying them would not strengthen the
claim. Per-domain "watch CORE answer X" is achieved by surfacing the
already-shipped case records, with the digest match as the load-bearing
audit step.
Determinism: same on-disk lane result files + same signed claim →
byte-identical JSON output (modulo wall-clock timestamps which are
excluded from the canonical JSON exactly as ADR-0099's showcase does).
"""
from __future__ import annotations
import glob
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Mapping
from core.capability.expert_demo import (
SHAPE_CHECKERS,
derive_evidence_digest,
resolve_lane_shape,
)
from core.capability.reviewers import (
ExpertDemoClaim,
ReviewerRegistry,
load_reviewer_registry,
)
from core.capability.sources import LEDGER_SOURCES
from core.demos.contract import CLAIM_CONTRACT_VERSION, canonical_json
EXPERT_DEMO_VERSION: int = 1
SAMPLE_CASES_PER_SPLIT: int = 3
_REPO_ROOT = Path(__file__).resolve().parent.parent.parent
@dataclass(frozen=True, slots=True)
class _LaneResultBundle:
metrics: Mapping[str, Any]
cases: tuple[Mapping[str, Any], ...]
result_path: Path
def _latest_result_path(lane: str, split: str) -> Path:
pattern = _REPO_ROOT / "evals" / lane / "results" / f"v1_{split}_*.json"
matches = sorted(glob.glob(str(pattern)))
if matches:
return Path(matches[-1])
exact = _REPO_ROOT / "evals" / lane / "results" / f"v1_{split}.json"
return exact
def _load_lane_split(lane: str, split: str) -> _LaneResultBundle:
path = _latest_result_path(lane, split)
if not path.exists():
raise FileNotFoundError(
f"Expert-demo composer cannot find on-disk result for "
f"lane {lane!r} split {split!r}: {path}"
)
payload = json.loads(path.read_text(encoding="utf-8"))
metrics = dict(payload.get("metrics", {}) or {})
if "by_class" not in metrics and "by_class" in payload:
metrics["by_class"] = payload["by_class"]
cases_raw = payload.get("cases") or []
cases = tuple(dict(c) for c in cases_raw)
return _LaneResultBundle(metrics=metrics, cases=cases, result_path=path)
def _shape_check(lane_id: str, metrics: Mapping[str, Any]) -> dict[str, Any]:
shape_id = resolve_lane_shape(lane_id)
if shape_id is None:
return {"shape": None, "passed": False, "reason": "unregistered lane"}
checker = SHAPE_CHECKERS.get(shape_id)
if checker is None:
return {"shape": shape_id, "passed": False, "reason": "no checker"}
passed, reason = checker(lane_id, metrics)
return {"shape": shape_id, "passed": passed, "reason": reason}
def _sample_cases(cases: tuple[Mapping[str, Any], ...]) -> list[dict[str, Any]]:
"""Pick the first N cases verbatim — deterministic by file order.
Adapters do not see this list; it is rendered for human inspection
only. We deliberately preserve raw case keys (case_id / id / surface
/ passed) without normalization so a reader can see exactly what
the lane runner produced.
"""
sampled = list(cases[:SAMPLE_CASES_PER_SPLIT])
return [dict(sorted(c.items())) for c in sampled]
def _resolve_claim(domain_id: str) -> ExpertDemoClaim:
registry_path = _REPO_ROOT / LEDGER_SOURCES.reviewers
registry: ReviewerRegistry = load_reviewer_registry(registry_path)
claim = registry.expert_demo_claim_for(domain_id)
if claim is None:
raise ValueError(
f"No expert_demo_claims entry for domain {domain_id!r} — "
f"a runnable expert-demo requires a signed claim in "
f"docs/reviewers.yaml. Domain may be reasoning-capable but "
f"not promoted."
)
return claim
def build_expert_demo(domain_id: str) -> dict[str, Any]:
"""Compose the showcase payload for ``domain_id``.
Raises if ``domain_id`` lacks a signed claim, if any attached
lane result file is missing, or if the recomputed digest does
not match the signed claim_digest.
"""
claim = _resolve_claim(domain_id)
lane_bundles: dict[str, dict[str, _LaneResultBundle]] = {}
for lane in claim.evidence_lanes:
lane_bundles[lane] = {
"public": _load_lane_split(lane, "public"),
"holdout": _load_lane_split(lane, "holdout"),
}
lane_metrics_for_digest = {
lane: {
"public": dict(lane_bundles[lane]["public"].metrics),
"holdout": dict(lane_bundles[lane]["holdout"].metrics),
}
for lane in claim.evidence_lanes
}
derived = derive_evidence_digest(
domain_id=domain_id,
evidence_revision=claim.evidence_revision,
evidence_lanes=claim.evidence_lanes,
lane_results=lane_metrics_for_digest,
)
digests_match = derived == claim.claim_digest
lane_payloads: list[dict[str, Any]] = []
all_lanes_pass = True
for lane in claim.evidence_lanes:
splits: dict[str, Any] = {}
for split in ("public", "holdout"):
bundle = lane_bundles[lane][split]
verdict = _shape_check(lane, bundle.metrics)
if not verdict["passed"]:
all_lanes_pass = False
splits[split] = {
"metrics": dict(sorted(bundle.metrics.items())),
"shape_check": verdict,
"case_count": len(bundle.cases),
"sample_cases": _sample_cases(bundle.cases),
"result_path": str(bundle.result_path.relative_to(_REPO_ROOT)),
}
lane_payloads.append(
{
"lane_id": lane,
"shape": resolve_lane_shape(lane),
"splits": splits,
}
)
all_claims_supported = digests_match and all_lanes_pass
return {
"expert_demo_version": EXPERT_DEMO_VERSION,
"claim_contract_version": CLAIM_CONTRACT_VERSION,
"domain_id": domain_id,
"claim": {
"evidence_lanes": list(claim.evidence_lanes),
"evidence_revision": claim.evidence_revision,
"signed_by": claim.signed_by,
"claim_digest": claim.claim_digest,
},
"digest_verification": {
"signed": claim.claim_digest,
"derived": derived,
"matches": digests_match,
},
"lanes": lane_payloads,
"all_lanes_pass": all_lanes_pass,
"all_digests_match": digests_match,
"all_claims_supported": all_claims_supported,
}
def run_expert_demo(*, domain_id: str, output_dir: Path) -> dict[str, Any]:
"""Build, write JSON + HTML, return the payload.
``output_dir`` is created if absent. Two outputs land there:
``expert_demo.json`` (byte-deterministic via :func:`canonical_json`)
and ``expert_demo.html`` (presentation-only).
"""
output_dir.mkdir(parents=True, exist_ok=True)
payload = build_expert_demo(domain_id)
json_path = output_dir / "expert_demo.json"
json_path.write_bytes(canonical_json(payload))
html_path = output_dir / "expert_demo.html"
html_path.write_text(render_html(payload), encoding="utf-8")
return payload
def render_html(payload: dict[str, Any]) -> str:
"""Render the expert-demo payload as a static HTML document."""
import html
def esc(value: Any) -> str:
return html.escape(str(value))
def case_block(case: Mapping[str, Any]) -> str:
case_id = case.get("case_id") or case.get("id") or "?"
passed = case.get("passed")
if passed is None:
# fabrication_control uses outcome_matches_expected
passed = case.get("outcome_matches_expected")
mark = "✓" if passed else ("✗" if passed is False else "·")
surface = case.get("surface") or case.get("prompt") or ""
extras = []
for k in (
"construction_name",
"pattern",
"class",
"grounding_source",
"trace_hash",
):
if k in case and case[k] is not None:
extras.append(f"<code>{esc(k)}={esc(case[k])}</code>")
extras_html = " ".join(extras)
return (
f'<li class="case {("ok" if passed else "fail")}">'
f'<span class="mark">{mark}</span> '
f"<code>{esc(case_id)}</code> "
f"{extras_html}"
f'<div class="surface">{esc(surface)}</div>'
f"</li>"
)
lane_sections: list[str] = []
for lane in payload["lanes"]:
split_blocks: list[str] = []
for split_name, split in lane["splits"].items():
verdict = split["shape_check"]
mark = "✓" if verdict["passed"] else "✗"
cases_html = "".join(case_block(c) for c in split["sample_cases"])
metrics_pairs = ", ".join(
f"{esc(k)}={esc(v)}"
for k, v in split["metrics"].items()
if not isinstance(v, (dict, list))
)
split_blocks.append(
f"<section><h3>{mark} {esc(split_name)} split "
f"<small>(shape={esc(verdict['shape'])}, "
f"{esc(split['case_count'])} cases)</small></h3>"
f"<p class='metrics'><strong>metrics:</strong> "
f"{metrics_pairs}</p>"
f"<p class='verdict'><strong>shape-check:</strong> "
f"{esc(verdict['reason'])}</p>"
f"<p><strong>sample cases (first "
f"{esc(len(split['sample_cases']))}):</strong></p>"
f"<ul class='cases'>{cases_html}</ul></section>"
)
lane_sections.append(
f"<article><h2>{esc(lane['lane_id'])} "
f"<small>({esc(lane['shape'])})</small></h2>"
f"{''.join(split_blocks)}</article>"
)
dv = payload["digest_verification"]
digest_mark = "✓" if dv["matches"] else "✗"
overall_mark = "✓" if payload["all_claims_supported"] else "✗"
return (
"<!doctype html><html><head>"
"<meta charset='utf-8'>"
f"<title>CORE Expert-Demo: {esc(payload['domain_id'])}</title>"
"<style>"
"body{font-family:system-ui;max-width:980px;margin:2rem auto;padding:0 1rem;}"
"h1{font-size:1.6rem;}h2{font-size:1.2rem;margin-top:2rem;}"
"h3{font-size:1.0rem;margin-top:1rem;}"
".case{margin:.35rem 0;list-style:none;padding:.3rem;border-left:3px solid #ddd;}"
".case.ok{border-left-color:#4a8;}.case.fail{border-left-color:#a44;}"
".case .surface{font-family:ui-monospace,monospace;color:#333;margin-top:.2rem;padding-left:1.6rem;}"
".mark{font-weight:bold;display:inline-block;width:1.2rem;}"
".metrics{color:#444;font-size:.9rem;}"
".verdict{color:#666;font-size:.85rem;}"
"code{background:#f3f3f3;padding:.05rem .25rem;border-radius:3px;font-size:.85rem;}"
".claim-card{background:#f8f8f8;padding:.75rem 1rem;border-radius:6px;margin:1rem 0;}"
".digest{font-family:ui-monospace,monospace;font-size:.78rem;word-break:break-all;}"
".digest.match{color:#283;}.digest.mismatch{color:#a33;}"
"</style></head><body>"
f"<h1>{overall_mark} Expert-Demo: <code>{esc(payload['domain_id'])}</code></h1>"
f"<p>Per-domain runnable demonstration of the "
f"<code>expert-demo</code> ledger status. The digest below "
f"reproduces from on-disk lane result files; the sample cases "
f"are drawn verbatim from those files.</p>"
"<section class='claim-card'>"
"<h2>Signed claim (docs/reviewers.yaml)</h2>"
f"<p><strong>signed_by:</strong> <code>{esc(payload['claim']['signed_by'])}</code></p>"
f"<p><strong>evidence_revision:</strong> "
f"<code>{esc(payload['claim']['evidence_revision'])}</code></p>"
f"<p><strong>evidence_lanes:</strong> "
f"{', '.join(f'<code>{esc(l)}</code>' for l in payload['claim']['evidence_lanes'])}</p>"
f"<p><strong>signed digest:</strong> "
f"<span class='digest'>{esc(dv['signed'])}</span></p>"
f"<p><strong>derived digest:</strong> "
f"<span class='digest {'match' if dv['matches'] else 'mismatch'}'>"
f"{esc(dv['derived'])}</span> {digest_mark}</p>"
"</section>"
+ "".join(lane_sections)
+ "</body></html>"
)

216
docs/decisions/ADR-0112-runnable-expert-demo-showcase.md Normal file
View file

@ -0,0 +1,216 @@
# ADR-0112 — Runnable Expert-Demo Showcase
**Status:** Accepted
**Date:** 2026-05-22
**Author:** CORE agents + reviewers
**Depends on:** ADR-0091, ADR-0092, ADR-0098, ADR-0099, ADR-0106, ADR-0109, ADR-0110, ADR-0111
---
## Context
ADR-0106 introduced the `expert-demo` ledger status as a contract-gated
promotion above `reasoning-capable`. ADR-0110 and ADR-0111 promoted
`mathematics_logic` and `physics` respectively under that contract.
The status name carries the word "demo." Until this ADR, the artifact
backing a promotion was a signed evidence-bundle digest in
`docs/reviewers.yaml` plus a set of on-disk lane result files —
*audit evidence*, not a runnable demonstration. An external reader
clone-and-run could verify the digest but had no per-domain HTML
artifact to open and see what the domain actually produces.
The asymmetry was real. `core demo audit-tour`,
`core demo register-tour`, `core demo learning-loop`,
`core demo showcase` all emit inspectable JSON + HTML walkthroughs.
The `expert-demo` *status* had no equivalent surface. The name implied
a demonstration that did not exist.
ADR-0112 closes the gap.
---
## Decision
Add a new demo target: `core demo expert --domain <id>`.
For a domain whose ledger row carries `expert_demo=true`, the composer
produces a per-domain runnable showcase:
1. Reads the signed `expert_demo_claims` entry from
`docs/reviewers.yaml`.
2. Loads the latest on-disk result file for each attached lane on
both `public` and `holdout` splits.
3. Re-derives the evidence-bundle digest from those files and asserts
byte-for-byte equality with the signed `claim_digest`. This is the
load-bearing audit step.
4. Runs each lane's metrics through the ADR-0109 lane-shape registry
and surfaces the shape-check verdict.
5. Picks the first N cases (N = 3) from each split's `cases` array
verbatim — same bytes the digest already covers, rendered for
inspection.
6. Emits `expert_demo.json` (canonical-serialized, byte-deterministic
via `core.demos.contract.canonical_json`) and `expert_demo.html`
(presentation-only) under
`evals/expert_demos/<domain>/latest/` by default.
The composer does **not** re-run the lanes. The lane result files are
the artifact the digest covers; replaying them would not strengthen
the claim and would introduce non-determinism (timestamp churn,
ordering). The "watch CORE answer X" experience is achieved by
surfacing the already-shipped case records — `surface`, `passed`,
`grounding_source`, `trace_hash` — directly from disk.
The composer is **read-only**. It writes only to its `output_dir`;
it does not mutate `docs/reviewers.yaml`, any lane result file, or
any pack manifest.
An unpromoted domain (no signed claim) raises `ValueError` with a
message naming the missing claim. The CLI surfaces a non-zero exit.
---
## Surface
```
core demo expert --domain mathematics_logic
core demo expert --domain physics
# default output dir: evals/expert_demos/<domain>/latest/
# override: core demo expert --domain physics --output-dir <path>
```
Output structure:
```
evals/expert_demos/<domain>/latest/
expert_demo.json # canonical bytes; same on-disk inputs → same SHA
expert_demo.html # presentation surface; opens in a browser
```
JSON shape (excerpt):
```text
{
"expert_demo_version": 1,
"claim_contract_version": 1,
"domain_id": "physics",
"claim": {
"signed_by": "shay-j",
"evidence_revision": "adr-0111:reviewed:2026-05-22",
"evidence_lanes": ["foundational_physics_ood", "inference_closure", "fabrication_control"],
"claim_digest": "a104cad1..."
},
"digest_verification": {
"signed": "a104cad1...",
"derived": "a104cad1...",
"matches": true
},
"lanes": [
{
"lane_id": "foundational_physics_ood",
"shape": "accuracy_shape",
"splits": {
"public": { "metrics": {...}, "shape_check": {...}, "case_count": 117, "sample_cases": [...] },
"holdout": { "metrics": {...}, "shape_check": {...}, "case_count": 39, "sample_cases": [...] }
}
},
...
],
"all_lanes_pass": true,
"all_digests_match": true,
"all_claims_supported": true
}
```
---
## Invariants
### `adr_0112_promoted_domain_renders`
`build_expert_demo(domain_id)` for every domain whose ledger row
carries `expert_demo=true` returns a payload with
`all_claims_supported=True`. Tested by
`tests/test_expert_demo_runnable.py::TestPromotedDomainsBuildSuccessfully`.
### `adr_0112_digest_recompute_byte_equal`
The recomputed digest equals the signed `claim_digest`. This is the
load-bearing audit invariant: if any byte of any attached lane result
file changes, the digest changes and the showcase declares
`all_claims_supported=False`. Tested in the same module.
### `adr_0112_unpromoted_domain_refused`
A domain without a signed claim raises `ValueError`. There is no
silent fallback, no "preview" mode that would emit an unsigned
showcase.
### `adr_0112_byte_determinism`
Two consecutive `run_expert_demo` calls with identical on-disk inputs
produce byte-identical `expert_demo.json`. Tested by SHA-256 of the
output bytes.
### `adr_0112_read_only`
`run_expert_demo` does not mutate `docs/reviewers.yaml` or any
`evals/<lane>/results/v1_*.json` file. Tested by capturing the bytes
of the source files before and after a run.
---
## Acceptance evidence
Accepted when:
- `core/demos/expert_demo.py` exists with `build_expert_demo`,
`run_expert_demo`, and `render_html` exported
- `core demo expert --domain <id>` works for `mathematics_logic` and
`physics`; both produce `all_claims_supported=True` and digest match
- `tests/test_expert_demo_runnable.py` pins the five invariants
- README + docs/decisions/README.md updated to point readers at the
new runnable surface
---
## Consequences
- The word "demo" in `expert-demo` now corresponds to something a
reader can open. The name is no longer aspirational.
- Each future expert-demo promotion automatically gains a runnable
surface — no per-domain composer work required. Adding a new
promoted domain (e.g. `systems_software`) needs only its signed
claim and on-disk lane results; the showcase composer handles the
rest by following the lane-shape registry.
- The digest-recompute step is now exercised in two places: the
ledger gate (`evaluate_expert_demo` at report time) and the
runnable showcase (every time a reader runs it). Same load-bearing
invariant, two independent enforcement points.
- An `expert_demo.html` is now a stable artifact a third party can
ask for. Future PRs that touch any of the three attached lane
result files for a promoted domain will, by definition, change
that domain's showcase output — `expert_demo.json` SHA stability is
a useful signal that the promotion claim still reproduces.
---
## Out of scope
- This ADR does not change the digest-derivation algorithm
(`derive_evidence_digest`), the lane-shape registry (ADR-0109),
or the contract gate (ADR-0106). The showcase is *consumer of*
those contracts, not an amendment.
- This ADR does not commit the per-domain `expert_demo.{json,html}`
artifacts to the repo. They are generated on demand. A future
ADR may pin selected showcases under `evals/expert_demos/<domain>/`
the way ADR-0099 pinned `evals/public_demo/results/latest/`; that
is a separate decision involving review of the artifact churn
cost.
- This ADR does not introduce live re-running of lane runners under
the showcase. The lane result files are authoritative; the showcase
is a renderer.
- Sample-case selection is the deterministic first-N. Replacing this
with a stratified or quoted-claim sample (e.g. include one case
per construction class) would be a future amendment.

4
docs/decisions/README.md
View file

@ -32,12 +32,13 @@ ADRs record significant architectural decisions: what was decided, why, what alt
| [ADR-0109](ADR-0109-lane-shape-aware-thresholds.md) | Lane-Shape-Aware Thresholds (ADR-0106 Amendment) | Accepted (2026-05-22) |
| [ADR-0110](ADR-0110-mathematics-logic-expert-demo-promotion.md) | `mathematics_logic` Expert-Demo Promotion | Accepted (2026-05-22) |
| [ADR-0111](ADR-0111-physics-expert-demo-promotion.md) | `physics` Expert-Demo Promotion | Accepted (2026-05-22) |
| [ADR-0112](ADR-0112-runnable-expert-demo-showcase.md) | Runnable Expert-Demo Showcase | Accepted (2026-05-22) |
---
## Current frontier
The ADR-0091..0111 slate is fully accepted and mechanically evidenced:
The ADR-0091..0112 slate is fully accepted and mechanically evidenced:
- Domain Pack Contract v1 — ADR-0091
- Reviewer Registry v1 — ADR-0092
@ -60,6 +61,7 @@ The ADR-0091..0111 slate is fully accepted and mechanically evidenced:
- Lane-Shape-Aware Thresholds (ADR-0106 amendment) — ADR-0109
- `mathematics_logic` Expert-Demo Promotion (first successful) — ADR-0110
- `physics` Expert-Demo Promotion (second successful) — ADR-0111
- Runnable Expert-Demo Showcase (`core demo expert --domain <id>`) — ADR-0112
ADR-0080 has also landed: Contemplation Loop Phase 1 adds a read-only frontier-compare miner that emits `SPECULATIVE` findings only.

151
tests/test_expert_demo_runnable.py Normal file
View file

@ -0,0 +1,151 @@
"""ADR-0112 — Runnable Expert-Demo Showcase invariants.
Pins six load-bearing invariants:
1. Building an expert-demo for a promoted domain (`mathematics_logic`,
`physics`) succeeds and reports ``all_claims_supported=True``.
2. The recomputed digest matches the signed ``claim_digest`` byte-for-byte.
3. The composer refuses an unpromoted domain (no signed claim).
4. Output JSON is byte-identical across two consecutive runs (same
on-disk lane result files → same showcase bytes).
5. Per-lane shape-check verdicts are present for every attached lane on
both public and holdout splits.
6. The showcase does not mutate any source artifact (lane result files,
reviewers.yaml).
"""
from __future__ import annotations
import hashlib
import tempfile
from pathlib import Path
import pytest
from core.demos.expert_demo import (
SAMPLE_CASES_PER_SPLIT,
build_expert_demo,
run_expert_demo,
)
_REPO_ROOT = Path(__file__).resolve().parent.parent
_PROMOTED_DOMAINS = ("mathematics_logic", "physics")
@pytest.mark.parametrize("domain_id", _PROMOTED_DOMAINS)
class TestPromotedDomainsBuildSuccessfully:
def test_build_returns_all_claims_supported(self, domain_id: str) -> None:
payload = build_expert_demo(domain_id)
assert payload["all_claims_supported"] is True
assert payload["all_lanes_pass"] is True
assert payload["all_digests_match"] is True
def test_derived_digest_matches_signed(self, domain_id: str) -> None:
payload = build_expert_demo(domain_id)
dv = payload["digest_verification"]
assert dv["matches"] is True
assert dv["derived"] == dv["signed"]
# 64-char lowercase hex (SHA-256)
assert len(dv["signed"]) == 64
assert all(c in "0123456789abcdef" for c in dv["signed"])
def test_every_lane_split_has_shape_check(self, domain_id: str) -> None:
payload = build_expert_demo(domain_id)
assert len(payload["lanes"]) == 3, (
"every promoted domain in this slate attaches exactly three lanes"
)
for lane in payload["lanes"]:
assert set(lane["splits"].keys()) == {"public", "holdout"}
for split_name, split in lane["splits"].items():
assert split["shape_check"]["passed"] is True, (
f"{lane['lane_id']}/{split_name} failed shape check: "
f"{split['shape_check']}"
)
assert split["shape_check"]["shape"] is not None
def test_sample_cases_are_capped(self, domain_id: str) -> None:
payload = build_expert_demo(domain_id)
for lane in payload["lanes"]:
for split_name, split in lane["splits"].items():
assert len(split["sample_cases"]) <= SAMPLE_CASES_PER_SPLIT
assert len(split["sample_cases"]) >= 1, (
f"{lane['lane_id']}/{split_name} has zero sample cases"
)
class TestUnpromotedDomainRefused:
def test_unpromoted_domain_raises_value_error(self) -> None:
with pytest.raises(ValueError, match="No expert_demo_claims entry"):
build_expert_demo("systems_software")
def test_unknown_domain_raises_value_error(self) -> None:
with pytest.raises(ValueError, match="No expert_demo_claims entry"):
build_expert_demo("not_a_real_domain")
class TestByteDeterminism:
@pytest.mark.parametrize("domain_id", _PROMOTED_DOMAINS)
def test_two_runs_produce_byte_identical_json(self, domain_id: str) -> None:
with tempfile.TemporaryDirectory() as tmp:
out_a = Path(tmp) / "a"
out_b = Path(tmp) / "b"
run_expert_demo(domain_id=domain_id, output_dir=out_a)
run_expert_demo(domain_id=domain_id, output_dir=out_b)
bytes_a = (out_a / "expert_demo.json").read_bytes()
bytes_b = (out_b / "expert_demo.json").read_bytes()
assert bytes_a == bytes_b
sha_a = hashlib.sha256(bytes_a).hexdigest()
sha_b = hashlib.sha256(bytes_b).hexdigest()
assert sha_a == sha_b
class TestComposerIsReadOnly:
def test_run_does_not_mutate_reviewers_yaml(self) -> None:
path = _REPO_ROOT / "docs" / "reviewers.yaml"
before = path.read_bytes()
with tempfile.TemporaryDirectory() as tmp:
run_expert_demo(domain_id="physics", output_dir=Path(tmp))
after = path.read_bytes()
assert before == after
def test_run_does_not_mutate_lane_result_files(self) -> None:
results_dir = _REPO_ROOT / "evals" / "foundational_physics_ood" / "results"
before = {p.name: p.read_bytes() for p in results_dir.glob("v1_*.json")}
with tempfile.TemporaryDirectory() as tmp:
run_expert_demo(domain_id="physics", output_dir=Path(tmp))
after = {p.name: p.read_bytes() for p in results_dir.glob("v1_*.json")}
assert before == after
class TestOutputArtifacts:
def test_json_and_html_both_written(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
out = Path(tmp)
run_expert_demo(domain_id="physics", output_dir=out)
assert (out / "expert_demo.json").is_file()
assert (out / "expert_demo.html").is_file()
html = (out / "expert_demo.html").read_text(encoding="utf-8")
assert "<title>CORE Expert-Demo: physics</title>" in html
assert "a104cad136f3219df05dc7ce6a78437c02f7b5827cd3cdce568db3acda6a43ed" in html
def test_html_contains_per_lane_sections(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
out = Path(tmp)
run_expert_demo(domain_id="physics", output_dir=out)
html = (out / "expert_demo.html").read_text(encoding="utf-8")
for lane_id in (
"foundational_physics_ood",
"inference_closure",
"fabrication_control",
):
assert lane_id in html
assert "public split" in html
assert "holdout split" in html