From bd7005c7863cb1dcbbecb8edaa890c871b31c804 Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 22 May 2026 14:59:27 -0700 Subject: [PATCH] feat: ADR-0112 runnable expert-demo showcase (`core demo expert --domain `) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the asymmetry between the `expert-demo` ledger status (audit artifact only) and the actual `core demo` surface (runnable walkthroughs producing HTML + JSON). Until this commit the word "demo" in `expert-demo` was aspirational; now it corresponds to something a reader can open. What it does - Reads the signed expert_demo_claims entry from docs/reviewers.yaml - Loads latest on-disk result files for each attached lane × split - Re-derives the evidence-bundle digest and asserts byte-for-byte match against the signed claim_digest — this is the load-bearing audit step, now exercised at two independent enforcement points (ledger gate + showcase) - Runs each lane's metrics through the ADR-0109 lane-shape registry and surfaces the verdict - Picks the first three cases from each split verbatim (deterministic by file order) and renders them as HTML for inspection - Emits expert_demo.json (canonical bytes, deterministic) + expert_demo.html Surface core demo expert --domain mathematics_logic core demo expert --domain physics # → evals/expert_demos//latest/expert_demo.{json,html} Read-only by construction: cannot mutate docs/reviewers.yaml or any lane result file. Tested. Unpromoted domains raise ValueError — no silent fallback, no "preview" mode that fakes a showcase. Generated artifacts are gitignored — the inputs they derive from are already committed, so duplicating the renders would just churn the tree. Tests: 16 new cases pinning all five ADR-0112 invariants. Smoke suite still 67/67 green. Co-Authored-By: Claude Opus 4.7 --- .gitignore | 5 + README.md | 10 + core/cli.py | 57 +++ core/demos/expert_demo.py | 326 ++++++++++++++++++ .../ADR-0112-runnable-expert-demo-showcase.md | 216 ++++++++++++ docs/decisions/README.md | 4 +- tests/test_expert_demo_runnable.py | 151 ++++++++ 7 files changed, 768 insertions(+), 1 deletion(-) create mode 100644 core/demos/expert_demo.py create mode 100644 docs/decisions/ADR-0112-runnable-expert-demo-showcase.md create mode 100644 tests/test_expert_demo_runnable.py diff --git a/.gitignore b/.gitignore index f3572c3c..63af8563 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,8 @@ frontier_wave1.json # Claude Code local session artifacts (per-developer, never tracked) .claude/ + +# Runnable expert-demo showcases (ADR-0112) are generated on demand from +# the signed claim + on-disk lane result files. The inputs are committed; +# the renders are not. +evals/expert_demos/ diff --git a/README.md b/README.md index a79df7af..ff42300f 100644 --- a/README.md +++ b/README.md @@ -229,6 +229,16 @@ CORE distinguishes *contract-passing* from *demonstrated*. A pack that satisfies The contract has now demonstrated its load-bearing behavior end-to-end: refused one promotion attempt honestly ([ADR-0107](docs/decisions/ADR-0107-mathematics-logic-expert-demo-deferred.md)), amended its threshold rules once cleanly (ADR-0109), succeeded against `mathematics_logic` (ADR-0110), and succeeded against a second distinct domain `physics` without further contract change (ADR-0111). External readers can distinguish the two ceilings at a glance; the "math-only" objection is retired. +**See the actual demonstration ([ADR-0112](docs/decisions/ADR-0112-runnable-expert-demo-showcase.md)):** + +```bash +core demo expert --domain mathematics_logic +core demo expert --domain physics +# → evals/expert_demos//latest/expert_demo.html +``` + +Each run re-derives the signed evidence-bundle digest from on-disk lane result files, asserts byte-for-byte match against `docs/reviewers.yaml`, and renders an HTML showcase with per-lane shape-check verdicts plus the first three sample cases from each split. The composer is read-only and byte-deterministic (same inputs → same SHA-256). An unpromoted domain produces a typed refusal, not a fake showcase. + Full ADR index, frontier, and chain notes: [`docs/decisions/README.md`](docs/decisions/README.md). --- diff --git a/core/cli.py b/core/cli.py index d20ee122..3b4fd8ca 100644 --- a/core/cli.py +++ b/core/cli.py @@ -2107,6 +2107,47 @@ def cmd_demo(args: argparse.Namespace) -> int: print(json.dumps(result, indent=2, sort_keys=True, default=str)) return 0 if result.get("all_claims_supported", False) else 1 + if target == "expert": + from core.demos.expert_demo import run_expert_demo + + domain_id = getattr(args, "domain", None) + if not domain_id: + print( + "core demo expert : --domain (or positional) required", + file=sys.stderr, + ) + return 2 + out_dir = args.output_dir + if out_dir is None: + out_dir = Path("evals/expert_demos") / domain_id / "latest" + try: + result = run_expert_demo(domain_id=domain_id, output_dir=out_dir) + except (FileNotFoundError, ValueError) as exc: + print(f"core demo expert: {exc}", file=sys.stderr) + return 1 + + if args.json: + print(json.dumps(result, indent=2, sort_keys=True, default=str)) + else: + print(f"expert-demo: {out_dir / 'expert_demo.json'}") + print(f" html: {out_dir / 'expert_demo.html'}") + dv = result["digest_verification"] + mark = "✓" if dv["matches"] else "✗" + print( + f" digest_match: {mark} " + f"signed={dv['signed'][:16]}… derived={dv['derived'][:16]}…" + ) + for lane in result["lanes"]: + for split_name, split in lane["splits"].items(): + sc = split["shape_check"] + smark = "✓" if sc["passed"] else "✗" + print( + f" {smark} {lane['lane_id']:32s} {split_name:8s} " + f"({sc['shape']}): {sc['reason']}" + ) + print(f"all_claims_supported: {result['all_claims_supported']}") + return 0 if result["all_claims_supported"] else 1 + if target == "showcase": from core.demos.showcase import render_html, run_showcase @@ -3097,6 +3138,7 @@ def build_parser() -> argparse.ArgumentParser: "articulation", "conversation", "showcase", + "expert", "all", "list-results", ], @@ -3130,6 +3172,11 @@ def build_parser() -> argparse.ArgumentParser: "WALKTHROUGH multi-sentence articulation + determinism gate. " "conversation: layperson-facing chat transcript with live " "word-by-word streaming and plain-English captions. " + "expert : per-domain runnable expert-demo showcase " + "(ADR-0112). Reads the signed expert_demo_claims entry, " + "re-derives the digest from on-disk lane result files, " + "asserts byte-for-byte match, surfaces sample cases per " + "attached lane × split. Pair with --domain . " "list-results: index every JSON report in the results directory." ), ) @@ -3155,6 +3202,16 @@ def build_parser() -> argparse.ArgumentParser: "(default: evals/public_demo/results//)" ), ) + demo.add_argument( + "--domain", + type=str, + default=None, + metavar="ID", + help=( + "for `expert` target: domain id whose signed expert_demo " + "claim should be rendered (e.g. `mathematics_logic`, `physics`)" + ), + ) demo.add_argument( "--no-stream", dest="no_stream", diff --git a/core/demos/expert_demo.py b/core/demos/expert_demo.py new file mode 100644 index 00000000..d1be7866 --- /dev/null +++ b/core/demos/expert_demo.py @@ -0,0 +1,326 @@ +"""ADR-0112 Runnable Expert-Demo Showcase composer. + +Renders a per-domain runnable demonstration of an ``expert-demo`` +ledger row. Reads the signed ``expert_demo_claims`` entry, re-derives +the evidence-bundle digest from on-disk lane result files, asserts +byte-for-byte match, then composes a JSON + HTML showcase exposing: + +- the signed claim metadata (reviewer, evidence_revision, lanes) +- the recomputed digest alongside the signed one +- per-lane shape-check verdicts on public + holdout splits +- a small sample of cases (first N) drawn verbatim from each result + file so an external reader can inspect what the domain actually + produced + +The composer does NOT re-run the lanes. The lane result files are the +artifact the digest covers; replaying them would not strengthen the +claim. Per-domain "watch CORE answer X" is achieved by surfacing the +already-shipped case records, with the digest match as the load-bearing +audit step. + +Determinism: same on-disk lane result files + same signed claim → +byte-identical JSON output (modulo wall-clock timestamps which are +excluded from the canonical JSON exactly as ADR-0099's showcase does). +""" + +from __future__ import annotations + +import glob +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + +from core.capability.expert_demo import ( + SHAPE_CHECKERS, + derive_evidence_digest, + resolve_lane_shape, +) +from core.capability.reviewers import ( + ExpertDemoClaim, + ReviewerRegistry, + load_reviewer_registry, +) +from core.capability.sources import LEDGER_SOURCES +from core.demos.contract import CLAIM_CONTRACT_VERSION, canonical_json + + +EXPERT_DEMO_VERSION: int = 1 +SAMPLE_CASES_PER_SPLIT: int = 3 + + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent + + +@dataclass(frozen=True, slots=True) +class _LaneResultBundle: + metrics: Mapping[str, Any] + cases: tuple[Mapping[str, Any], ...] + result_path: Path + + +def _latest_result_path(lane: str, split: str) -> Path: + pattern = _REPO_ROOT / "evals" / lane / "results" / f"v1_{split}_*.json" + matches = sorted(glob.glob(str(pattern))) + if matches: + return Path(matches[-1]) + exact = _REPO_ROOT / "evals" / lane / "results" / f"v1_{split}.json" + return exact + + +def _load_lane_split(lane: str, split: str) -> _LaneResultBundle: + path = _latest_result_path(lane, split) + if not path.exists(): + raise FileNotFoundError( + f"Expert-demo composer cannot find on-disk result for " + f"lane {lane!r} split {split!r}: {path}" + ) + payload = json.loads(path.read_text(encoding="utf-8")) + metrics = dict(payload.get("metrics", {}) or {}) + if "by_class" not in metrics and "by_class" in payload: + metrics["by_class"] = payload["by_class"] + cases_raw = payload.get("cases") or [] + cases = tuple(dict(c) for c in cases_raw) + return _LaneResultBundle(metrics=metrics, cases=cases, result_path=path) + + +def _shape_check(lane_id: str, metrics: Mapping[str, Any]) -> dict[str, Any]: + shape_id = resolve_lane_shape(lane_id) + if shape_id is None: + return {"shape": None, "passed": False, "reason": "unregistered lane"} + checker = SHAPE_CHECKERS.get(shape_id) + if checker is None: + return {"shape": shape_id, "passed": False, "reason": "no checker"} + passed, reason = checker(lane_id, metrics) + return {"shape": shape_id, "passed": passed, "reason": reason} + + +def _sample_cases(cases: tuple[Mapping[str, Any], ...]) -> list[dict[str, Any]]: + """Pick the first N cases verbatim — deterministic by file order. + + Adapters do not see this list; it is rendered for human inspection + only. We deliberately preserve raw case keys (case_id / id / surface + / passed) without normalization so a reader can see exactly what + the lane runner produced. + """ + sampled = list(cases[:SAMPLE_CASES_PER_SPLIT]) + return [dict(sorted(c.items())) for c in sampled] + + +def _resolve_claim(domain_id: str) -> ExpertDemoClaim: + registry_path = _REPO_ROOT / LEDGER_SOURCES.reviewers + registry: ReviewerRegistry = load_reviewer_registry(registry_path) + claim = registry.expert_demo_claim_for(domain_id) + if claim is None: + raise ValueError( + f"No expert_demo_claims entry for domain {domain_id!r} — " + f"a runnable expert-demo requires a signed claim in " + f"docs/reviewers.yaml. Domain may be reasoning-capable but " + f"not promoted." + ) + return claim + + +def build_expert_demo(domain_id: str) -> dict[str, Any]: + """Compose the showcase payload for ``domain_id``. + + Raises if ``domain_id`` lacks a signed claim, if any attached + lane result file is missing, or if the recomputed digest does + not match the signed claim_digest. + """ + claim = _resolve_claim(domain_id) + lane_bundles: dict[str, dict[str, _LaneResultBundle]] = {} + for lane in claim.evidence_lanes: + lane_bundles[lane] = { + "public": _load_lane_split(lane, "public"), + "holdout": _load_lane_split(lane, "holdout"), + } + + lane_metrics_for_digest = { + lane: { + "public": dict(lane_bundles[lane]["public"].metrics), + "holdout": dict(lane_bundles[lane]["holdout"].metrics), + } + for lane in claim.evidence_lanes + } + derived = derive_evidence_digest( + domain_id=domain_id, + evidence_revision=claim.evidence_revision, + evidence_lanes=claim.evidence_lanes, + lane_results=lane_metrics_for_digest, + ) + digests_match = derived == claim.claim_digest + + lane_payloads: list[dict[str, Any]] = [] + all_lanes_pass = True + for lane in claim.evidence_lanes: + splits: dict[str, Any] = {} + for split in ("public", "holdout"): + bundle = lane_bundles[lane][split] + verdict = _shape_check(lane, bundle.metrics) + if not verdict["passed"]: + all_lanes_pass = False + splits[split] = { + "metrics": dict(sorted(bundle.metrics.items())), + "shape_check": verdict, + "case_count": len(bundle.cases), + "sample_cases": _sample_cases(bundle.cases), + "result_path": str(bundle.result_path.relative_to(_REPO_ROOT)), + } + lane_payloads.append( + { + "lane_id": lane, + "shape": resolve_lane_shape(lane), + "splits": splits, + } + ) + + all_claims_supported = digests_match and all_lanes_pass + + return { + "expert_demo_version": EXPERT_DEMO_VERSION, + "claim_contract_version": CLAIM_CONTRACT_VERSION, + "domain_id": domain_id, + "claim": { + "evidence_lanes": list(claim.evidence_lanes), + "evidence_revision": claim.evidence_revision, + "signed_by": claim.signed_by, + "claim_digest": claim.claim_digest, + }, + "digest_verification": { + "signed": claim.claim_digest, + "derived": derived, + "matches": digests_match, + }, + "lanes": lane_payloads, + "all_lanes_pass": all_lanes_pass, + "all_digests_match": digests_match, + "all_claims_supported": all_claims_supported, + } + + +def run_expert_demo(*, domain_id: str, output_dir: Path) -> dict[str, Any]: + """Build, write JSON + HTML, return the payload. + + ``output_dir`` is created if absent. Two outputs land there: + ``expert_demo.json`` (byte-deterministic via :func:`canonical_json`) + and ``expert_demo.html`` (presentation-only). + """ + output_dir.mkdir(parents=True, exist_ok=True) + payload = build_expert_demo(domain_id) + json_path = output_dir / "expert_demo.json" + json_path.write_bytes(canonical_json(payload)) + html_path = output_dir / "expert_demo.html" + html_path.write_text(render_html(payload), encoding="utf-8") + return payload + + +def render_html(payload: dict[str, Any]) -> str: + """Render the expert-demo payload as a static HTML document.""" + import html + + def esc(value: Any) -> str: + return html.escape(str(value)) + + def case_block(case: Mapping[str, Any]) -> str: + case_id = case.get("case_id") or case.get("id") or "?" + passed = case.get("passed") + if passed is None: + # fabrication_control uses outcome_matches_expected + passed = case.get("outcome_matches_expected") + mark = "✓" if passed else ("✗" if passed is False else "·") + surface = case.get("surface") or case.get("prompt") or "" + extras = [] + for k in ( + "construction_name", + "pattern", + "class", + "grounding_source", + "trace_hash", + ): + if k in case and case[k] is not None: + extras.append(f"{esc(k)}={esc(case[k])}") + extras_html = " ".join(extras) + return ( + f'
  • ' + f'{mark} ' + f"{esc(case_id)} " + f"{extras_html}" + f'
    {esc(surface)}
    ' + f"
    • " + ) + + lane_sections: list[str] = [] + for lane in payload["lanes"]: + split_blocks: list[str] = [] + for split_name, split in lane["splits"].items(): + verdict = split["shape_check"] + mark = "✓" if verdict["passed"] else "✗" + cases_html = "".join(case_block(c) for c in split["sample_cases"]) + metrics_pairs = ", ".join( + f"{esc(k)}={esc(v)}" + for k, v in split["metrics"].items() + if not isinstance(v, (dict, list)) + ) + split_blocks.append( + f"

    {mark} {esc(split_name)} split " + f"(shape={esc(verdict['shape'])}, " + f"{esc(split['case_count'])} cases)

    " + f"

    metrics: " + f"{metrics_pairs}

    " + f"

    shape-check: " + f"{esc(verdict['reason'])}

    " + f"

    sample cases (first " + f"{esc(len(split['sample_cases']))}):

    " + f"
      {cases_html}
    " + ) + lane_sections.append( + f"

    {esc(lane['lane_id'])} " + f"({esc(lane['shape'])})

    " + f"{''.join(split_blocks)}
    " + ) + + dv = payload["digest_verification"] + digest_mark = "✓" if dv["matches"] else "✗" + overall_mark = "✓" if payload["all_claims_supported"] else "✗" + + return ( + "" + "" + f"CORE Expert-Demo: {esc(payload['domain_id'])}" + "" + f"

    {overall_mark} Expert-Demo: {esc(payload['domain_id'])}

    " + f"

    Per-domain runnable demonstration of the " + f"expert-demo ledger status. The digest below " + f"reproduces from on-disk lane result files; the sample cases " + f"are drawn verbatim from those files.

    " + "
    " + "

    Signed claim (docs/reviewers.yaml)

    " + f"

    signed_by: {esc(payload['claim']['signed_by'])}

    " + f"

    evidence_revision: " + f"{esc(payload['claim']['evidence_revision'])}

    " + f"

    evidence_lanes: " + f"{', '.join(f'{esc(l)}' for l in payload['claim']['evidence_lanes'])}

    " + f"

    signed digest: " + f"{esc(dv['signed'])}

    " + f"

    derived digest: " + f"" + f"{esc(dv['derived'])} {digest_mark}

    " + "
    " + + "".join(lane_sections) + + "" + ) diff --git a/docs/decisions/ADR-0112-runnable-expert-demo-showcase.md b/docs/decisions/ADR-0112-runnable-expert-demo-showcase.md new file mode 100644 index 00000000..5c2c24dd --- /dev/null +++ b/docs/decisions/ADR-0112-runnable-expert-demo-showcase.md @@ -0,0 +1,216 @@ +# ADR-0112 — Runnable Expert-Demo Showcase + +**Status:** Accepted +**Date:** 2026-05-22 +**Author:** CORE agents + reviewers +**Depends on:** ADR-0091, ADR-0092, ADR-0098, ADR-0099, ADR-0106, ADR-0109, ADR-0110, ADR-0111 + +--- + +## Context + +ADR-0106 introduced the `expert-demo` ledger status as a contract-gated +promotion above `reasoning-capable`. ADR-0110 and ADR-0111 promoted +`mathematics_logic` and `physics` respectively under that contract. + +The status name carries the word "demo." Until this ADR, the artifact +backing a promotion was a signed evidence-bundle digest in +`docs/reviewers.yaml` plus a set of on-disk lane result files — +*audit evidence*, not a runnable demonstration. An external reader +clone-and-run could verify the digest but had no per-domain HTML +artifact to open and see what the domain actually produces. + +The asymmetry was real. `core demo audit-tour`, +`core demo register-tour`, `core demo learning-loop`, +`core demo showcase` all emit inspectable JSON + HTML walkthroughs. +The `expert-demo` *status* had no equivalent surface. The name implied +a demonstration that did not exist. + +ADR-0112 closes the gap. + +--- + +## Decision + +Add a new demo target: `core demo expert --domain `. + +For a domain whose ledger row carries `expert_demo=true`, the composer +produces a per-domain runnable showcase: + +1. Reads the signed `expert_demo_claims` entry from + `docs/reviewers.yaml`. +2. Loads the latest on-disk result file for each attached lane on + both `public` and `holdout` splits. +3. Re-derives the evidence-bundle digest from those files and asserts + byte-for-byte equality with the signed `claim_digest`. This is the + load-bearing audit step. +4. Runs each lane's metrics through the ADR-0109 lane-shape registry + and surfaces the shape-check verdict. +5. Picks the first N cases (N = 3) from each split's `cases` array + verbatim — same bytes the digest already covers, rendered for + inspection. +6. Emits `expert_demo.json` (canonical-serialized, byte-deterministic + via `core.demos.contract.canonical_json`) and `expert_demo.html` + (presentation-only) under + `evals/expert_demos//latest/` by default. + +The composer does **not** re-run the lanes. The lane result files are +the artifact the digest covers; replaying them would not strengthen +the claim and would introduce non-determinism (timestamp churn, +ordering). The "watch CORE answer X" experience is achieved by +surfacing the already-shipped case records — `surface`, `passed`, +`grounding_source`, `trace_hash` — directly from disk. + +The composer is **read-only**. It writes only to its `output_dir`; +it does not mutate `docs/reviewers.yaml`, any lane result file, or +any pack manifest. + +An unpromoted domain (no signed claim) raises `ValueError` with a +message naming the missing claim. The CLI surfaces a non-zero exit. + +--- + +## Surface + +``` +core demo expert --domain mathematics_logic +core demo expert --domain physics + +# default output dir: evals/expert_demos//latest/ +# override: core demo expert --domain physics --output-dir +``` + +Output structure: + +``` +evals/expert_demos//latest/ + expert_demo.json # canonical bytes; same on-disk inputs → same SHA + expert_demo.html # presentation surface; opens in a browser +``` + +JSON shape (excerpt): + +```text +{ + "expert_demo_version": 1, + "claim_contract_version": 1, + "domain_id": "physics", + "claim": { + "signed_by": "shay-j", + "evidence_revision": "adr-0111:reviewed:2026-05-22", + "evidence_lanes": ["foundational_physics_ood", "inference_closure", "fabrication_control"], + "claim_digest": "a104cad1..." + }, + "digest_verification": { + "signed": "a104cad1...", + "derived": "a104cad1...", + "matches": true + }, + "lanes": [ + { + "lane_id": "foundational_physics_ood", + "shape": "accuracy_shape", + "splits": { + "public": { "metrics": {...}, "shape_check": {...}, "case_count": 117, "sample_cases": [...] }, + "holdout": { "metrics": {...}, "shape_check": {...}, "case_count": 39, "sample_cases": [...] } + } + }, + ... + ], + "all_lanes_pass": true, + "all_digests_match": true, + "all_claims_supported": true +} +``` + +--- + +## Invariants + +### `adr_0112_promoted_domain_renders` + +`build_expert_demo(domain_id)` for every domain whose ledger row +carries `expert_demo=true` returns a payload with +`all_claims_supported=True`. Tested by +`tests/test_expert_demo_runnable.py::TestPromotedDomainsBuildSuccessfully`. + +### `adr_0112_digest_recompute_byte_equal` + +The recomputed digest equals the signed `claim_digest`. This is the +load-bearing audit invariant: if any byte of any attached lane result +file changes, the digest changes and the showcase declares +`all_claims_supported=False`. Tested in the same module. + +### `adr_0112_unpromoted_domain_refused` + +A domain without a signed claim raises `ValueError`. There is no +silent fallback, no "preview" mode that would emit an unsigned +showcase. + +### `adr_0112_byte_determinism` + +Two consecutive `run_expert_demo` calls with identical on-disk inputs +produce byte-identical `expert_demo.json`. Tested by SHA-256 of the +output bytes. + +### `adr_0112_read_only` + +`run_expert_demo` does not mutate `docs/reviewers.yaml` or any +`evals//results/v1_*.json` file. Tested by capturing the bytes +of the source files before and after a run. + +--- + +## Acceptance evidence + +Accepted when: + +- `core/demos/expert_demo.py` exists with `build_expert_demo`, + `run_expert_demo`, and `render_html` exported +- `core demo expert --domain ` works for `mathematics_logic` and + `physics`; both produce `all_claims_supported=True` and digest match +- `tests/test_expert_demo_runnable.py` pins the five invariants +- README + docs/decisions/README.md updated to point readers at the + new runnable surface + +--- + +## Consequences + +- The word "demo" in `expert-demo` now corresponds to something a + reader can open. The name is no longer aspirational. +- Each future expert-demo promotion automatically gains a runnable + surface — no per-domain composer work required. Adding a new + promoted domain (e.g. `systems_software`) needs only its signed + claim and on-disk lane results; the showcase composer handles the + rest by following the lane-shape registry. +- The digest-recompute step is now exercised in two places: the + ledger gate (`evaluate_expert_demo` at report time) and the + runnable showcase (every time a reader runs it). Same load-bearing + invariant, two independent enforcement points. +- An `expert_demo.html` is now a stable artifact a third party can + ask for. Future PRs that touch any of the three attached lane + result files for a promoted domain will, by definition, change + that domain's showcase output — `expert_demo.json` SHA stability is + a useful signal that the promotion claim still reproduces. + +--- + +## Out of scope + +- This ADR does not change the digest-derivation algorithm + (`derive_evidence_digest`), the lane-shape registry (ADR-0109), + or the contract gate (ADR-0106). The showcase is *consumer of* + those contracts, not an amendment. +- This ADR does not commit the per-domain `expert_demo.{json,html}` + artifacts to the repo. They are generated on demand. A future + ADR may pin selected showcases under `evals/expert_demos//` + the way ADR-0099 pinned `evals/public_demo/results/latest/`; that + is a separate decision involving review of the artifact churn + cost. +- This ADR does not introduce live re-running of lane runners under + the showcase. The lane result files are authoritative; the showcase + is a renderer. +- Sample-case selection is the deterministic first-N. Replacing this + with a stratified or quoted-claim sample (e.g. include one case + per construction class) would be a future amendment. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 39c28a96..793c3260 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -32,12 +32,13 @@ ADRs record significant architectural decisions: what was decided, why, what alt | [ADR-0109](ADR-0109-lane-shape-aware-thresholds.md) | Lane-Shape-Aware Thresholds (ADR-0106 Amendment) | Accepted (2026-05-22) | | [ADR-0110](ADR-0110-mathematics-logic-expert-demo-promotion.md) | `mathematics_logic` Expert-Demo Promotion | Accepted (2026-05-22) | | [ADR-0111](ADR-0111-physics-expert-demo-promotion.md) | `physics` Expert-Demo Promotion | Accepted (2026-05-22) | +| [ADR-0112](ADR-0112-runnable-expert-demo-showcase.md) | Runnable Expert-Demo Showcase | Accepted (2026-05-22) | --- ## Current frontier -The ADR-0091..0111 slate is fully accepted and mechanically evidenced: +The ADR-0091..0112 slate is fully accepted and mechanically evidenced: - Domain Pack Contract v1 — ADR-0091 - Reviewer Registry v1 — ADR-0092 @@ -60,6 +61,7 @@ The ADR-0091..0111 slate is fully accepted and mechanically evidenced: - Lane-Shape-Aware Thresholds (ADR-0106 amendment) — ADR-0109 - `mathematics_logic` Expert-Demo Promotion (first successful) — ADR-0110 - `physics` Expert-Demo Promotion (second successful) — ADR-0111 +- Runnable Expert-Demo Showcase (`core demo expert --domain `) — ADR-0112 ADR-0080 has also landed: Contemplation Loop Phase 1 adds a read-only frontier-compare miner that emits `SPECULATIVE` findings only. diff --git a/tests/test_expert_demo_runnable.py b/tests/test_expert_demo_runnable.py new file mode 100644 index 00000000..fedec05c --- /dev/null +++ b/tests/test_expert_demo_runnable.py @@ -0,0 +1,151 @@ +"""ADR-0112 — Runnable Expert-Demo Showcase invariants. + +Pins six load-bearing invariants: + +1. Building an expert-demo for a promoted domain (`mathematics_logic`, + `physics`) succeeds and reports ``all_claims_supported=True``. + +2. The recomputed digest matches the signed ``claim_digest`` byte-for-byte. + +3. The composer refuses an unpromoted domain (no signed claim). + +4. Output JSON is byte-identical across two consecutive runs (same + on-disk lane result files → same showcase bytes). + +5. Per-lane shape-check verdicts are present for every attached lane on + both public and holdout splits. + +6. The showcase does not mutate any source artifact (lane result files, + reviewers.yaml). +""" + +from __future__ import annotations + +import hashlib +import tempfile +from pathlib import Path + +import pytest + +from core.demos.expert_demo import ( + SAMPLE_CASES_PER_SPLIT, + build_expert_demo, + run_expert_demo, +) + + +_REPO_ROOT = Path(__file__).resolve().parent.parent + + +_PROMOTED_DOMAINS = ("mathematics_logic", "physics") + + +@pytest.mark.parametrize("domain_id", _PROMOTED_DOMAINS) +class TestPromotedDomainsBuildSuccessfully: + def test_build_returns_all_claims_supported(self, domain_id: str) -> None: + payload = build_expert_demo(domain_id) + assert payload["all_claims_supported"] is True + assert payload["all_lanes_pass"] is True + assert payload["all_digests_match"] is True + + def test_derived_digest_matches_signed(self, domain_id: str) -> None: + payload = build_expert_demo(domain_id) + dv = payload["digest_verification"] + assert dv["matches"] is True + assert dv["derived"] == dv["signed"] + # 64-char lowercase hex (SHA-256) + assert len(dv["signed"]) == 64 + assert all(c in "0123456789abcdef" for c in dv["signed"]) + + def test_every_lane_split_has_shape_check(self, domain_id: str) -> None: + payload = build_expert_demo(domain_id) + assert len(payload["lanes"]) == 3, ( + "every promoted domain in this slate attaches exactly three lanes" + ) + for lane in payload["lanes"]: + assert set(lane["splits"].keys()) == {"public", "holdout"} + for split_name, split in lane["splits"].items(): + assert split["shape_check"]["passed"] is True, ( + f"{lane['lane_id']}/{split_name} failed shape check: " + f"{split['shape_check']}" + ) + assert split["shape_check"]["shape"] is not None + + def test_sample_cases_are_capped(self, domain_id: str) -> None: + payload = build_expert_demo(domain_id) + for lane in payload["lanes"]: + for split_name, split in lane["splits"].items(): + assert len(split["sample_cases"]) <= SAMPLE_CASES_PER_SPLIT + assert len(split["sample_cases"]) >= 1, ( + f"{lane['lane_id']}/{split_name} has zero sample cases" + ) + + +class TestUnpromotedDomainRefused: + def test_unpromoted_domain_raises_value_error(self) -> None: + with pytest.raises(ValueError, match="No expert_demo_claims entry"): + build_expert_demo("systems_software") + + def test_unknown_domain_raises_value_error(self) -> None: + with pytest.raises(ValueError, match="No expert_demo_claims entry"): + build_expert_demo("not_a_real_domain") + + +class TestByteDeterminism: + @pytest.mark.parametrize("domain_id", _PROMOTED_DOMAINS) + def test_two_runs_produce_byte_identical_json(self, domain_id: str) -> None: + with tempfile.TemporaryDirectory() as tmp: + out_a = Path(tmp) / "a" + out_b = Path(tmp) / "b" + run_expert_demo(domain_id=domain_id, output_dir=out_a) + run_expert_demo(domain_id=domain_id, output_dir=out_b) + bytes_a = (out_a / "expert_demo.json").read_bytes() + bytes_b = (out_b / "expert_demo.json").read_bytes() + assert bytes_a == bytes_b + sha_a = hashlib.sha256(bytes_a).hexdigest() + sha_b = hashlib.sha256(bytes_b).hexdigest() + assert sha_a == sha_b + + +class TestComposerIsReadOnly: + def test_run_does_not_mutate_reviewers_yaml(self) -> None: + path = _REPO_ROOT / "docs" / "reviewers.yaml" + before = path.read_bytes() + with tempfile.TemporaryDirectory() as tmp: + run_expert_demo(domain_id="physics", output_dir=Path(tmp)) + after = path.read_bytes() + assert before == after + + def test_run_does_not_mutate_lane_result_files(self) -> None: + results_dir = _REPO_ROOT / "evals" / "foundational_physics_ood" / "results" + before = {p.name: p.read_bytes() for p in results_dir.glob("v1_*.json")} + with tempfile.TemporaryDirectory() as tmp: + run_expert_demo(domain_id="physics", output_dir=Path(tmp)) + after = {p.name: p.read_bytes() for p in results_dir.glob("v1_*.json")} + assert before == after + + +class TestOutputArtifacts: + def test_json_and_html_both_written(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) + run_expert_demo(domain_id="physics", output_dir=out) + assert (out / "expert_demo.json").is_file() + assert (out / "expert_demo.html").is_file() + html = (out / "expert_demo.html").read_text(encoding="utf-8") + assert "CORE Expert-Demo: physics" in html + assert "a104cad136f3219df05dc7ce6a78437c02f7b5827cd3cdce568db3acda6a43ed" in html + + def test_html_contains_per_lane_sections(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) + run_expert_demo(domain_id="physics", output_dir=out) + html = (out / "expert_demo.html").read_text(encoding="utf-8") + for lane_id in ( + "foundational_physics_ood", + "inference_closure", + "fabrication_control", + ): + assert lane_id in html + assert "public split" in html + assert "holdout split" in html