From 696f62abdd90b324961c4848f8a91d77ba2f6d9f Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 22 May 2026 15:36:10 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20ADR-0113=20rename=20`expert-demo`=20?= =?UTF-8?q?=E2=86=92=20`audit-passed`;=20reserve=20`expert`=20namespace=20?= =?UTF-8?q?(ADR-0114=20GSM8K=20roadmap)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The word "expert" in the previous status name implied raw-capability parity with frontier LLMs on the same benchmark — which the gate does NOT verify. What the gate actually verifies is CORE *claim-shape compliance*: * signed digest (replay-reproducible from on-disk lane results) * replay determinism (same inputs → byte-equal trace_hash) * typed refusal (fabrication refused, not paraphrased) * exact recall (no ANN, no cosine, no attention bottleneck) * grounding-source provenance These are claim shapes a transformer LLM cannot structurally produce regardless of raw accuracy. A frontier LLM might score higher on the same benchmark but cannot pass this contract. Rename scope (semantics only, per ADR-0113): status string "expert-demo" → "audit-passed" predicate key predicates.expert_demo → predicates.audit_passed reason key expert_demo_reason → audit_passed_reason YAML key expert_demo_claims → audit_passed_claims CLI command core demo expert → core demo audit-passed output dir evals/expert_demos/ → evals/audit_passed/ artifact filenames expert_demo.{json,html} → audit_passed.{json,html} HTML title CORE Expert-Demo: X → CORE Audit-Passed: X Internal Python identifiers (module/file/function/class names like `expert_demo.py`, `evaluate_expert_demo`, `ExpertDemoClaim`, `expert_demo_claim_for`) are deliberately kept to minimize churn. ADR file titles (ADR-0106..0112) preserved as historical record. `expert` namespace reserved for ADR-0114+: an actual capability tier above `audit-passed` backed by a public benchmark with a stated threshold. ADR-0114 proposes the first such target — GSM8K-math — laying out a falsifiable 7-phase arc (parser → solver → verifier → stepped-realizer → eval lane → first `expert` ledger tier promotion). Tests: 184 directly-affected tests green (140 capability/expert-demo suite + 34 demo/audit-tour + 10 correction-cue). Smoke suite 67/67. Co-Authored-By: Claude Opus 4.7 --- .gitignore | 7 +- CLAIMS.md | 4 +- README.md | 23 +- core/capability/README.md | 10 +- core/capability/expert_demo.py | 23 +- core/capability/reporting.py | 14 +- core/capability/reviewers.py | 28 +- core/cli.py | 29 +- core/demos/expert_demo.py | 45 ++- docs/PROGRESS.md | 20 +- docs/Whitepaper.md | 8 +- docs/Yellowpaper.md | 4 +- docs/capability_roadmap.md | 8 +- ...0113-rename-expert-demo-to-audit-passed.md | 190 +++++++++++++ ...4-expert-capability-roadmap-gsm8k-first.md | 263 ++++++++++++++++++ docs/decisions/README.md | 28 +- docs/eval_methodology.md | 6 +- docs/reviewers.yaml | 11 +- docs/runtime_contracts.md | 24 +- ...adr_0097_mathematics_logic_ratification.py | 15 +- ...est_adr_0100_0102_sibling_ratifications.py | 14 +- tests/test_adr_0110_math_expert_demo.py | 12 +- tests/test_adr_0111_physics_expert_demo.py | 14 +- tests/test_capability_reports.py | 8 +- tests/test_expert_demo_contract.py | 10 +- tests/test_expert_demo_runnable.py | 18 +- tests/test_reviewer_registry.py | 2 +- 27 files changed, 674 insertions(+), 164 deletions(-) create mode 100644 docs/decisions/ADR-0113-rename-expert-demo-to-audit-passed.md create mode 100644 docs/decisions/ADR-0114-expert-capability-roadmap-gsm8k-first.md diff --git a/.gitignore b/.gitignore index 63af8563..55e0cdb0 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,8 @@ frontier_wave1.json # Claude Code local session artifacts (per-developer, never tracked) .claude/ -# Runnable expert-demo showcases (ADR-0112) are generated on demand from -# the signed claim + on-disk lane result files. The inputs are committed; -# the renders are not. +# Runnable audit-passed showcases (ADR-0112 + ADR-0113) are generated on +# demand from the signed claim + on-disk lane result files. The inputs +# are committed; the renders are not. +evals/audit_passed/ evals/expert_demos/ diff --git a/CLAIMS.md b/CLAIMS.md index 529deab7..e004f1fd 100644 --- a/CLAIMS.md +++ b/CLAIMS.md @@ -21,8 +21,8 @@ status predicate evaluates to `reasoning-capable` with no open gaps. | Domain | Status | ADR | Packs | Open gaps | | --- | --- | --- | --- | --- | | `philosophy_theology` | reasoning-capable | ADR-0085 | 2 | 0 | -| `mathematics_logic` | expert-demo | ADR-0097 | 1 | 0 | -| `physics` | expert-demo | ADR-0100 | 1 | 0 | +| `mathematics_logic` | audit-passed | ADR-0097 | 1 | 0 | +| `physics` | audit-passed | ADR-0100 | 1 | 0 | | `systems_software` | reasoning-capable | ADR-0101 | 1 | 0 | | `hebrew_greek_textual_reasoning` | reasoning-capable | ADR-0102 | 4 | 0 | diff --git a/README.md b/README.md index ff42300f..3b9421c7 100644 --- a/README.md +++ b/README.md @@ -207,38 +207,45 @@ core teaching supersessions # pair retired chains with r ## Evidence-Governed Domain Layer — The ADR-0091 Chain -CORE distinguishes *contract-passing* from *demonstrated*. A pack that satisfies the nine ADR-0091 predicates earns a `reasoning-capable` ledger row; that's a structural claim, not an empirical one. Promotion to `expert_demo=true` requires a **reviewer-signed evidence-bundle digest** that reproduces byte-for-byte from on-disk lane results (ADR-0106 + ADR-0109). +CORE distinguishes *contract-passing* from *demonstrated*. A pack that satisfies the nine ADR-0091 predicates earns a `reasoning-capable` ledger row; that's a structural claim, not an empirical one. Promotion to `audit_passed=true` (formerly `expert_demo`; renamed by [ADR-0113](docs/decisions/ADR-0113-rename-expert-demo-to-audit-passed.md)) requires a **reviewer-signed evidence-bundle digest** that reproduces byte-for-byte from on-disk lane results (ADR-0106 + ADR-0109). + +> **What `audit-passed` actually means** — and what it does NOT mean. +> The gate verifies CORE *claim-shape compliance*: signed digest, replay determinism, typed refusal, exact recall, grounding-source provenance. **These are claim shapes a transformer LLM cannot structurally produce regardless of raw accuracy.** A frontier LLM might score higher on the same benchmark but cannot pass this contract because it cannot produce a digest that re-derives, cannot guarantee typed refusal, cannot emit a deterministic trace hash, cannot replay byte-equal. **This is NOT a raw-capability claim.** The future `expert` ledger tier ([ADR-0114](docs/decisions/ADR-0114-expert-capability-roadmap-gsm8k-first.md)) is reserved for an actual benchmark-calibrated capability claim; no domain holds it yet. | Layer | What it guarantees | ADR | |---|---|---| | **Domain Pack Contract v1** | Nine predicate checks on every ratified pack (lemma coverage, operator chain count, intent shapes, holdout coverage, reviewer-resolution, etc.). | [0091](docs/decisions/ADR-0091-domain-pack-contract-v1.md) | | **Reviewer Registry v1** | YAML-anchored, schema-validated reviewer roster. Wildcard `*` reserved for primary reviewers; domain-scoped reviewers gated by `can_review(domain, scope)`. | [0092](docs/decisions/ADR-0092-reviewer-registry-v1.md) | | **Fabrication-control eval lane** | Negative-control lane: phantom endpoints, cross-pack non-bridges, sibling collapses must all refuse. `fabricated=0` across all by-class buckets is the gate. | [0096](docs/decisions/ADR-0096-fabrication-control-eval-lane.md) | -| **Expert-demo promotion contract** | Domain-aware, reviewer-signed, replay-deterministic. No domain promotes silently; every `expert_demo=true` row points to an `expert_demo_claims` entry whose SHA-256 reproduces. | [0106](docs/decisions/ADR-0106-expert-demo-promotion-contract.md) | +| **Audit-passed promotion contract** | Domain-aware, reviewer-signed, replay-deterministic. No domain promotes silently; every `audit_passed=true` row points to an `audit_passed_claims` entry whose SHA-256 reproduces. (Originally landed as `expert-demo`; renamed by ADR-0113.) | [0106](docs/decisions/ADR-0106-expert-demo-promotion-contract.md), [0113](docs/decisions/ADR-0113-rename-expert-demo-to-audit-passed.md) | | **Lane-shape registry** | Eight lane ids dispatch to five shapes (`cognition_shape`, `accuracy_shape`, `inference_shape`, `refusal_shape`, `symbolic_logic_shape`); unknown lanes fail-closed. | [0109](docs/decisions/ADR-0109-lane-shape-aware-thresholds.md) | **Current ledger state** (per `core capability ledger`): | Domain | Status | |---|---| -| `mathematics_logic` | **`expert-demo`** (first promotion, [ADR-0110](docs/decisions/ADR-0110-mathematics-logic-expert-demo-promotion.md)) | -| `physics` | **`expert-demo`** (second promotion, [ADR-0111](docs/decisions/ADR-0111-physics-expert-demo-promotion.md)) | +| `mathematics_logic` | **`audit-passed`** (first promotion, [ADR-0110](docs/decisions/ADR-0110-mathematics-logic-expert-demo-promotion.md); status string renamed by [ADR-0113](docs/decisions/ADR-0113-rename-expert-demo-to-audit-passed.md)) | +| `physics` | **`audit-passed`** (second promotion, [ADR-0111](docs/decisions/ADR-0111-physics-expert-demo-promotion.md)) | | `systems_software` | `reasoning-capable` | | `hebrew_greek_textual_reasoning` | `reasoning-capable` | | `philosophy_theology` | `reasoning-capable` | The contract has now demonstrated its load-bearing behavior end-to-end: refused one promotion attempt honestly ([ADR-0107](docs/decisions/ADR-0107-mathematics-logic-expert-demo-deferred.md)), amended its threshold rules once cleanly (ADR-0109), succeeded against `mathematics_logic` (ADR-0110), and succeeded against a second distinct domain `physics` without further contract change (ADR-0111). External readers can distinguish the two ceilings at a glance; the "math-only" objection is retired. -**See the actual demonstration ([ADR-0112](docs/decisions/ADR-0112-runnable-expert-demo-showcase.md)):** +**See the actual demonstration ([ADR-0112](docs/decisions/ADR-0112-runnable-expert-demo-showcase.md), renamed by [ADR-0113](docs/decisions/ADR-0113-rename-expert-demo-to-audit-passed.md)):** ```bash -core demo expert --domain mathematics_logic -core demo expert --domain physics -# → evals/expert_demos//latest/expert_demo.html +core demo audit-passed --domain mathematics_logic +core demo audit-passed --domain physics +# → evals/audit_passed//latest/audit_passed.html ``` Each run re-derives the signed evidence-bundle digest from on-disk lane result files, asserts byte-for-byte match against `docs/reviewers.yaml`, and renders an HTML showcase with per-lane shape-check verdicts plus the first three sample cases from each split. The composer is read-only and byte-deterministic (same inputs → same SHA-256). An unpromoted domain produces a typed refusal, not a fake showcase. +### Path to actual expert-level capability — explicitly future work + +The `audit-passed` gate above is intentionally *not* a raw-capability claim. The honest path to one is laid out in [ADR-0114 — Expert-Capability Roadmap: GSM8K-Math First](docs/decisions/ADR-0114-expert-capability-roadmap-gsm8k-first.md). Phases 1-7 define a falsifiable arc: problem parser → deterministic solver → verifier → stepped-realizer → public GSM8K eval lane → first `expert` ledger tier promotion ADR with a publicly stated benchmark threshold. **No domain is at `expert` today.** That status string remains reserved namespace. + Full ADR index, frontier, and chain notes: [`docs/decisions/README.md`](docs/decisions/README.md). --- diff --git a/core/capability/README.md b/core/capability/README.md index 374829f4..8ad1edb6 100644 --- a/core/capability/README.md +++ b/core/capability/README.md @@ -16,11 +16,11 @@ Day-1 output is expected to be low-status: | Domain | Expected status | Primary blocker | | --- | --- | --- | -| `systems_software` | `reasoning-capable` | expert-demo still gated by domain eval thresholds and replay/provenance evidence | -| `mathematics_logic` | `reasoning-capable` | expert-demo still gated by domain eval thresholds and replay/provenance evidence | -| `physics` | `reasoning-capable` | expert-demo still gated by domain eval thresholds and replay/provenance evidence | -| `hebrew_greek_textual_reasoning` | `reasoning-capable` | expert-demo still gated by eval thresholds and replay/provenance evidence | -| `philosophy_theology` | `reasoning-capable` | expert-demo still gated by eval thresholds and replay/provenance evidence | +| `systems_software` | `reasoning-capable` | audit-passed still gated by domain eval thresholds and replay/provenance evidence | +| `mathematics_logic` | `reasoning-capable` | audit-passed still gated by domain eval thresholds and replay/provenance evidence | +| `physics` | `reasoning-capable` | audit-passed still gated by domain eval thresholds and replay/provenance evidence | +| `hebrew_greek_textual_reasoning` | `reasoning-capable` | audit-passed still gated by eval thresholds and replay/provenance evidence | +| `philosophy_theology` | `reasoning-capable` | audit-passed still gated by eval thresholds and replay/provenance evidence | Status is generated from predicates. A `blocked` row lifts only when the named gap is closed and the next status predicate passes. diff --git a/core/capability/expert_demo.py b/core/capability/expert_demo.py index 51ee5417..c3a798a1 100644 --- a/core/capability/expert_demo.py +++ b/core/capability/expert_demo.py @@ -1,8 +1,19 @@ -"""Domain-aware expert-demo promotion gate (ADR-0106). +"""Domain-aware audit-passed promotion gate (ADR-0106, renamed by ADR-0113). + +Historical note: this module is named ``expert_demo`` for backward +compatibility with ADR-0106..0112. ADR-0113 renamed the outward +semantics — ledger status string, YAML key, predicate key, and CLI +command — from "expert-demo" to "audit-passed" because the gate +verifies CORE *claim-shape compliance* (signed digest, replay +determinism, typed refusal, exact recall) which transformer LLMs +structurally cannot produce, NOT raw expert-level capability. The +internal module/function/class identifiers were intentionally left in +place under ADR-0113's "semantics only" scope to minimize churn. + Replaces the cognition-lane-only predicate previously embedded in ``core.capability.reporting``. A domain ``D`` is promoted to -``expert_demo=true`` iff: +``audit_passed=true`` iff: 1. ``D`` already passes the ``reasoning_capable`` predicate. 2. A signed ``ExpertDemoClaim`` exists in the reviewer registry for ``D``. @@ -202,7 +213,7 @@ def derive_evidence_digest( def _meets_thresholds(lane_id: str, metrics: Mapping[str, Any]) -> tuple[bool, str]: """Dispatch lane threshold check by registered shape (ADR-0109). - Unknown lane ids are fail-closed: adding a lane to the expert-demo + Unknown lane ids are fail-closed: adding a lane to the audit-passed surface requires an explicit registry entry, which requires an ADR amendment. """ @@ -227,7 +238,7 @@ def evaluate_expert_demo( domain_lanes: Sequence[str], lane_results: Mapping[str, Mapping[str, Mapping[str, object]]], ) -> ExpertDemoVerdict: - """Decide whether ``domain_id`` may carry ``expert_demo=true``. + """Decide whether ``domain_id`` may carry ``audit_passed=true``. ``domain_lanes`` is the union of ``eval_lanes`` declared by the ratified packs for ``domain_id`` — the only lanes legal as evidence @@ -247,7 +258,7 @@ def evaluate_expert_demo( if claim is None: return ExpertDemoVerdict( passed=False, - reason="no expert_demo_claims entry for this domain", + reason="no audit_passed_claims entry for this domain", derived_digest=None, ) @@ -312,7 +323,7 @@ def evaluate_expert_demo( return ExpertDemoVerdict( passed=True, - reason="all expert-demo predicates satisfied", + reason="all audit-passed predicates satisfied", derived_digest=derived, ) diff --git a/core/capability/reporting.py b/core/capability/reporting.py index 9c0a6bb5..9f3a084b 100644 --- a/core/capability/reporting.py +++ b/core/capability/reporting.py @@ -34,7 +34,7 @@ from language_packs.domain_contract import validate_domain_contract_pack _REPO_ROOT = Path(__file__).resolve().parent.parent.parent _CHAINS_PER_OPERATOR_DOMAIN = 8 _TARGET_INTENT_SHAPES = ("cause", "verification", "comparison", "procedure", "correction") -_EXPERT_DOMAIN_STATUSES = ("blocked", "seeded", "grounded", "reasoning-capable", "expert-demo") +_EXPERT_DOMAIN_STATUSES = ("blocked", "seeded", "grounded", "reasoning-capable", "audit-passed") _DOMAIN_FOUNDATION_GAPS: dict[str, tuple[str, ...]] = { "hebrew_greek_textual_reasoning": ( "gap:grc_he_glosses_absent", @@ -447,10 +447,10 @@ def ledger_report() -> dict[str, Any]: domain_lanes=domain_lanes, lane_results=lane_results, ) - expert_demo = verdict.passed - expert_demo_reason = verdict.reason - if expert_demo: - status = "expert-demo" + audit_passed = verdict.passed + audit_passed_reason = verdict.reason + if audit_passed: + status = "audit-passed" elif reasoning_capable: status = "reasoning-capable" elif grounded: @@ -490,9 +490,9 @@ def ledger_report() -> dict[str, Any]: "seeded": seeded, "grounded": grounded, "reasoning_capable": reasoning_capable, - "expert_demo": expert_demo, + "audit_passed": audit_passed, }, - "expert_demo_reason": expert_demo_reason, + "audit_passed_reason": audit_passed_reason, "known_gaps": domain_gaps, "open_gaps": open_gaps, "blocked_lift": blocked_lift, diff --git a/core/capability/reviewers.py b/core/capability/reviewers.py index 392c6ffe..c41ca02a 100644 --- a/core/capability/reviewers.py +++ b/core/capability/reviewers.py @@ -18,7 +18,7 @@ import yaml ALLOWED_TOP_LEVEL_KEYS: frozenset[str] = frozenset( - {"schema_version", "reviewers", "expert_demo_claims"} + {"schema_version", "reviewers", "audit_passed_claims"} ) ALLOWED_REVIEWER_KEYS: frozenset[str] = frozenset( {"reviewer_id", "display_name", "role", "domains", "review_scope", "provenance"} @@ -54,14 +54,18 @@ class Reviewer: @dataclass(frozen=True, slots=True) class ExpertDemoClaim: - """Reviewer-signed expert-demo promotion claim (ADR-0106). + """Reviewer-signed audit-passed promotion claim (ADR-0106 / ADR-0113). - A row in ``expert_demo_claims`` asserts that ``signed_by`` has + A row in ``audit_passed_claims`` asserts that ``signed_by`` has inspected the evidence at ``evidence_revision`` for ``domain_id`` across ``evidence_lanes`` and that the canonical evidence-bundle SHA-256 equals ``claim_digest``. The reporting layer re-derives the digest from the lane results on disk; a mismatch demotes the row back to ``reasoning-capable``. + + The class is still named ``ExpertDemoClaim`` for backward + compatibility (ADR-0113 scope: semantics-only rename — the YAML key + and ledger status move, but internal Python identifiers stay). """ domain_id: str @@ -154,10 +158,10 @@ def load_reviewer_registry(path: Path) -> ReviewerRegistry: seen_ids.add(reviewer.reviewer_id) parsed.append(reviewer) - claims_raw = raw.get("expert_demo_claims", []) + claims_raw = raw.get("audit_passed_claims", []) if not isinstance(claims_raw, list): raise ReviewerRegistryError( - "reviewer registry 'expert_demo_claims' must be a list when present" + "reviewer registry 'audit_passed_claims' must be a list when present" ) parsed_claims: list[ExpertDemoClaim] = [] seen_domains: set[str] = set() @@ -166,7 +170,7 @@ def load_reviewer_registry(path: Path) -> ReviewerRegistry: claim = _parse_expert_demo_claim(entry, index=index, reviewer_ids=reviewer_ids) if claim.domain_id in seen_domains: raise ReviewerRegistryError( - f"expert_demo_claims contains duplicate domain_id " + f"audit_passed_claims contains duplicate domain_id" f"{claim.domain_id!r}" ) seen_domains.add(claim.domain_id) @@ -184,18 +188,18 @@ def _parse_expert_demo_claim( ) -> ExpertDemoClaim: if not isinstance(entry, Mapping): raise ReviewerRegistryError( - f"expert_demo_claims entry at index {index} must be a mapping" + f"audit_passed_claims entry at index {index} must be a mapping" ) unknown = set(entry.keys()) - ALLOWED_EXPERT_DEMO_CLAIM_KEYS if unknown: raise ReviewerRegistryError( - f"expert_demo_claims entry at index {index} has unknown fields: " + f"audit_passed_claims entry at index {index} has unknown fields: " f"{sorted(unknown)}" ) missing = ALLOWED_EXPERT_DEMO_CLAIM_KEYS - set(entry.keys()) if missing: raise ReviewerRegistryError( - f"expert_demo_claims entry at index {index} missing required " + f"audit_passed_claims entry at index {index} missing required " f"fields: {sorted(missing)}" ) domain_id = _require_nonempty_str( @@ -212,14 +216,14 @@ def _parse_expert_demo_claim( ) if signed_by not in reviewer_ids: raise ReviewerRegistryError( - f"expert_demo_claims entry at index {index} signed_by " + f"audit_passed_claims entry at index {index} signed_by " f"{signed_by!r} does not resolve to a registered reviewer" ) if len(claim_digest) != 64 or any( c not in "0123456789abcdef" for c in claim_digest ): raise ReviewerRegistryError( - f"expert_demo_claims entry at index {index} claim_digest must be " + f"audit_passed_claims entry at index {index} claim_digest must be " "a lowercase 64-char SHA-256 hex string" ) evidence_lanes = _require_str_list( @@ -227,7 +231,7 @@ def _parse_expert_demo_claim( ) if not evidence_lanes: raise ReviewerRegistryError( - f"expert_demo_claims entry at index {index} has empty " + f"audit_passed_claims entry at index {index} has empty " "'evidence_lanes' list" ) return ExpertDemoClaim( diff --git a/core/cli.py b/core/cli.py index 3b4fd8ca..e4c7c37c 100644 --- a/core/cli.py +++ b/core/cli.py @@ -2107,30 +2107,30 @@ def cmd_demo(args: argparse.Namespace) -> int: print(json.dumps(result, indent=2, sort_keys=True, default=str)) return 0 if result.get("all_claims_supported", False) else 1 - if target == "expert": + if target == "audit-passed": from core.demos.expert_demo import run_expert_demo domain_id = getattr(args, "domain", None) if not domain_id: print( - "core demo expert : --domain (or positional) required", + "core demo audit-passed: --domain required", file=sys.stderr, ) return 2 out_dir = args.output_dir if out_dir is None: - out_dir = Path("evals/expert_demos") / domain_id / "latest" + out_dir = Path("evals/audit_passed") / domain_id / "latest" try: result = run_expert_demo(domain_id=domain_id, output_dir=out_dir) except (FileNotFoundError, ValueError) as exc: - print(f"core demo expert: {exc}", file=sys.stderr) + print(f"core demo audit-passed: {exc}", file=sys.stderr) return 1 if args.json: print(json.dumps(result, indent=2, sort_keys=True, default=str)) else: - print(f"expert-demo: {out_dir / 'expert_demo.json'}") - print(f" html: {out_dir / 'expert_demo.html'}") + print(f"audit-passed: {out_dir / 'audit_passed.json'}") + print(f" html: {out_dir / 'audit_passed.html'}") dv = result["digest_verification"] mark = "✓" if dv["matches"] else "✗" print( @@ -3138,7 +3138,7 @@ def build_parser() -> argparse.ArgumentParser: "articulation", "conversation", "showcase", - "expert", + "audit-passed", "all", "list-results", ], @@ -3172,11 +3172,16 @@ def build_parser() -> argparse.ArgumentParser: "WALKTHROUGH multi-sentence articulation + determinism gate. " "conversation: layperson-facing chat transcript with live " "word-by-word streaming and plain-English captions. " - "expert : per-domain runnable expert-demo showcase " - "(ADR-0112). Reads the signed expert_demo_claims entry, " - "re-derives the digest from on-disk lane result files, " - "asserts byte-for-byte match, surfaces sample cases per " - "attached lane × split. Pair with --domain . " + "audit-passed : per-domain runnable audit-passed " + "showcase (ADR-0112 + ADR-0113). Reads the signed " + "audit_passed_claims entry, re-derives the digest from " + "on-disk lane result files, asserts byte-for-byte match, " + "surfaces sample cases per attached lane × split. The " + "audit-passed gate verifies CORE claim-shape compliance " + "(signed digest, replay determinism, typed refusal, exact " + "recall) — claim shapes a transformer LLM cannot " + "structurally produce regardless of raw accuracy. NOT a " + "raw-capability claim. Pair with --domain . " "list-results: index every JSON report in the results directory." ), ) diff --git a/core/demos/expert_demo.py b/core/demos/expert_demo.py index d1be7866..6172c5cf 100644 --- a/core/demos/expert_demo.py +++ b/core/demos/expert_demo.py @@ -1,7 +1,15 @@ -"""ADR-0112 Runnable Expert-Demo Showcase composer. +"""ADR-0112 + ADR-0113 Runnable Audit-Passed Showcase composer. -Renders a per-domain runnable demonstration of an ``expert-demo`` -ledger row. Reads the signed ``expert_demo_claims`` entry, re-derives +Renders a per-domain runnable demonstration of an ``audit-passed`` +ledger row. + +Historical note: this module is named ``expert_demo`` for backward +compatibility with ADR-0112's original framing. ADR-0113 renamed the +outward semantics ("expert-demo" → "audit-passed") because the gate +verifies CORE *claim-shape compliance* (signed digest, replay +determinism, typed refusal, exact recall) — claim shapes a transformer +LLM structurally cannot produce, NOT raw expert-level capability. The +internal module name was kept under ADR-0113's "semantics only" scope. Reads the signed ``expert_demo_claims`` entry, re-derives the evidence-bundle digest from on-disk lane result files, asserts byte-for-byte match, then composes a JSON + HTML showcase exposing: @@ -113,10 +121,10 @@ def _resolve_claim(domain_id: str) -> ExpertDemoClaim: claim = registry.expert_demo_claim_for(domain_id) if claim is None: raise ValueError( - f"No expert_demo_claims entry for domain {domain_id!r} — " - f"a runnable expert-demo requires a signed claim in " - f"docs/reviewers.yaml. Domain may be reasoning-capable but " - f"not promoted." + f"No audit_passed_claims entry for domain {domain_id!r} — " + f"a runnable audit-passed showcase requires a signed claim " + f"in docs/reviewers.yaml. Domain may be reasoning-capable " + f"but not promoted." ) return claim @@ -203,20 +211,20 @@ def run_expert_demo(*, domain_id: str, output_dir: Path) -> dict[str, Any]: """Build, write JSON + HTML, return the payload. ``output_dir`` is created if absent. Two outputs land there: - ``expert_demo.json`` (byte-deterministic via :func:`canonical_json`) - and ``expert_demo.html`` (presentation-only). + ``audit_passed.json`` (byte-deterministic via :func:`canonical_json`) + and ``audit_passed.html`` (presentation-only). """ output_dir.mkdir(parents=True, exist_ok=True) payload = build_expert_demo(domain_id) - json_path = output_dir / "expert_demo.json" + json_path = output_dir / "audit_passed.json" json_path.write_bytes(canonical_json(payload)) - html_path = output_dir / "expert_demo.html" + html_path = output_dir / "audit_passed.html" html_path.write_text(render_html(payload), encoding="utf-8") return payload def render_html(payload: dict[str, Any]) -> str: - """Render the expert-demo payload as a static HTML document.""" + """Render the audit-passed payload as a static HTML document.""" import html def esc(value: Any) -> str: @@ -287,7 +295,7 @@ def render_html(payload: dict[str, Any]) -> str: return ( "" "" - f"CORE Expert-Demo: {esc(payload['domain_id'])}" + f"CORE Audit-Passed: {esc(payload['domain_id'])}" "" - f"

{overall_mark} Expert-Demo: {esc(payload['domain_id'])}

" + f"

{overall_mark} Audit-Passed: {esc(payload['domain_id'])}

" f"

Per-domain runnable demonstration of the " - f"expert-demo ledger status. The digest below " + f"audit-passed ledger status. The digest below " f"reproduces from on-disk lane result files; the sample cases " f"are drawn verbatim from those files.

" + f"

What this gate verifies: CORE claim-shape " + f"compliance — signed evidence-bundle digest, replay " + f"determinism, typed refusal, exact recall. These are claim " + f"shapes a transformer LLM cannot structurally produce " + f"regardless of raw accuracy. This is not a " + f"raw-capability claim; per ADR-0113 the future expert " + f"tier above audit-passed remains undefined.

" "
" "

Signed claim (docs/reviewers.yaml)

" f"

signed_by: {esc(payload['claim']['signed_by'])}

" diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index 90c6c8d5..b9ba2e87 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -708,7 +708,7 @@ Per-surface bit-identity gates landed (2026-05-16): **Depends on:** Phase 5 corpus flywheel + pack-layer chain (ADR-0027..0045) **Roadmap:** see `docs/capability_roadmap.md` Phase 6 entry. -This phase ratifies the distinction between *contract-passing* (`reasoning-capable`) and *demonstrated* (`expert_demo=true`) at the capability ledger surface. +This phase ratifies the distinction between *contract-passing* (`reasoning-capable`) and *demonstrated* (`audit_passed=true`) at the capability ledger surface. ### Contract layer (all accepted) @@ -725,8 +725,8 @@ This phase ratifies the distinction between *contract-passing* (`reasoning-capab ### Reasoning-capable ratifications -- [x] **ADR-0097** `mathematics_logic` reasoning-capable (now superseded by ADR-0110 expert-demo, see below) -- [x] **ADR-0100** `physics` reasoning-capable (now superseded by ADR-0111 expert-demo, see below) +- [x] **ADR-0097** `mathematics_logic` reasoning-capable (now superseded by ADR-0110 audit-passed, see below) +- [x] **ADR-0100** `physics` reasoning-capable (now superseded by ADR-0111 audit-passed, see below) - [x] **ADR-0101** `systems_software` reasoning-capable - [x] **ADR-0102** `hebrew_greek_textual_reasoning` reasoning-capable (first multi-pack ratification: 4 packs) - [x] **ADR-0103** Hebrew + Koine Greek fluency lane attachment to ADR-0102 packs @@ -734,30 +734,30 @@ This phase ratifies the distinction between *contract-passing* (`reasoning-capab ### Expert-demo arc (the contract demonstrated end-to-end) - [x] **ADR-0106** Expert-Demo Promotion Contract — domain-aware, reviewer-signed, replay-deterministic -- [x] **ADR-0107** `mathematics_logic` expert-demo deferred — first promotion attempt **honestly refused** on two named blockers (metric-shape uniformity; `inference_closure` 40% pass) +- [x] **ADR-0107** `mathematics_logic` audit-passed deferred — first promotion attempt **honestly refused** on two named blockers (metric-shape uniformity; `inference_closure` 40% pass) - [x] **ADR-0108** Proposed-ADR sequencing — meta-decision pinning the post-ADR-0105 frontier - [x] **ADR-0109** Lane-shape-aware threshold amendment — 8 lane ids → 5 shapes (`cognition_shape`, `accuracy_shape`, `inference_shape`, `refusal_shape`, `symbolic_logic_shape`); unknown lanes fail-closed; cognition-shape thresholds preserved bit-identical - [x] PR #117 — fix intent-classifier regression that had broken `inference_closure` (`_CORRECTION_CUE_PREFIX_RE` guard) -- [x] **ADR-0110** `mathematics_logic` expert-demo promoted — **first domain at `expert_demo=true`** in project history; signed claim digest reproduces byte-for-byte from on-disk lane results -- [x] **ADR-0111** `physics` expert-demo promoted — **second domain at `expert_demo=true`**; no contract change, one-file dev-mode fallback bridge; shares `inference_closure` + `fabrication_control` results with math (distinct digest via `domain_id`); retires the "math-only" objection +- [x] **ADR-0110** `mathematics_logic` audit-passed promoted — **first domain at `audit_passed=true`** in project history; signed claim digest reproduces byte-for-byte from on-disk lane results +- [x] **ADR-0111** `physics` audit-passed promoted — **second domain at `audit_passed=true`**; no contract change, one-file dev-mode fallback bridge; shares `inference_closure` + `fabrication_control` results with math (distinct digest via `domain_id`); retires the "math-only" objection ### Contract demonstration narrative -The ADR-0106 contract refused once (0107), amended once cleanly (0109), succeeded against `mathematics_logic` (0110), and succeeded against `physics` without further contract change (0111). External readers can now distinguish the two ceilings (`reasoning-capable` vs `expert-demo`) by inspecting the ledger. +The ADR-0106 contract refused once (0107), amended once cleanly (0109), succeeded against `mathematics_logic` (0110), and succeeded against `physics` without further contract change (0111). External readers can now distinguish the two ceilings (`reasoning-capable` vs `audit-passed`) by inspecting the ledger. ### Current ledger state (per `core capability ledger`) | Domain | Status | |---|---| -| `mathematics_logic` | **`expert-demo`** ✓ | -| `physics` | **`expert-demo`** ✓ | +| `mathematics_logic` | **`audit-passed`** ✓ | +| `physics` | **`audit-passed`** ✓ | | `systems_software` | `reasoning-capable` | | `hebrew_greek_textual_reasoning` | `reasoning-capable` | | `philosophy_theology` | `reasoning-capable` | ### Open within Phase 6 -- [ ] Third expert-demo promotion (`systems_software` or `hebrew_greek_textual_reasoning` — both eligible under ADR-0109 shape rules) +- [ ] Third audit-passed promotion (`systems_software` or `hebrew_greek_textual_reasoning` — both eligible under ADR-0109 shape rules) - [ ] Multi-reviewer threshold signing (open candidate frontier item from ADR-0105) ### Pack-layer chain — ADR-0027 through ADR-0045 (backfill) diff --git a/docs/Whitepaper.md b/docs/Whitepaper.md index d3c27849..69782114 100644 --- a/docs/Whitepaper.md +++ b/docs/Whitepaper.md @@ -380,8 +380,8 @@ Decision Records (ADR-0091 through ADR-0111): collapses must all *refuse*. A pack that confabulates on any refusal class fails ratification. 4. **Promotion gate.** ADR-0106 introduces the second status above - `reasoning-capable`: `expert_demo`. Promotion requires a reviewer- - signed `expert_demo_claims` entry whose evidence-bundle SHA-256 + `reasoning-capable`: `audit_passed`. Promotion requires a reviewer- + signed `audit_passed_claims` entry whose evidence-bundle SHA-256 reproduces byte-for-byte from on-disk lane results. The signer must resolve to the registry; the signed lanes must be attached to the domain's ratified packs. @@ -393,9 +393,9 @@ The contract is load-bearing because it has *refused*. ADR-0107 records the first promotion attempt — `mathematics_logic` — being honestly refused on two named blockers. ADR-0109 amended the threshold rules without weakening the discipline. ADR-0110 then promoted -`mathematics_logic` as the first domain at `expert_demo=true`, with the +`mathematics_logic` as the first domain at `audit_passed=true`, with the signed claim digest reproducing from disk. ADR-0111 promoted `physics` -as the second domain at `expert_demo=true` without any contract change, +as the second domain at `audit_passed=true` without any contract change, retiring the "math-only" objection — the same gate now holds across two distinct domains using shared lane infrastructure with distinct domain-bound digests. diff --git a/docs/Yellowpaper.md b/docs/Yellowpaper.md index 62bc3bfd..c32afbe5 100644 --- a/docs/Yellowpaper.md +++ b/docs/Yellowpaper.md @@ -671,13 +671,13 @@ generated ledger row. #### Expert-Demo Promotion (ADR-0106 + ADR-0109) -The promotion to `status = expert-demo` is contract-gated. The +The promotion to `status = audit-passed` is contract-gated. The promotion predicate (`core/capability/expert_demo.py::evaluate_expert_demo`) requires: ```text reasoning_capable(D) -∧ ∃ claim ∈ ReviewerRegistry.expert_demo_claims +∧ ∃ claim ∈ ReviewerRegistry.audit_passed_claims : claim.domain_id == D ∧ ReviewerRegistry.can_review(claim.signed_by, D, scope="eval") ∧ claim.evidence_lanes ⊆ ratified_lanes(D) diff --git a/docs/capability_roadmap.md b/docs/capability_roadmap.md index 4d278bbb..4872cf9c 100644 --- a/docs/capability_roadmap.md +++ b/docs/capability_roadmap.md @@ -37,7 +37,7 @@ the next planning pass. ## Status update — 2026-05-22 -Phase 5 (Curriculum Era) work landed in volume during the 2026-05-17 → 2026-05-22 window. The roadmap gains a new Part-II phase (**Phase 6 — Evidence-Governed Domain Layer**, see below) that did not exist in the 2026-05-15 draft because the substrate it ratifies (multi-domain packs, signed reviewer registry, expert-demo gate) did not yet exist either. The phase is documented here as accepted and partially landed, not proposed. +Phase 5 (Curriculum Era) work landed in volume during the 2026-05-17 → 2026-05-22 window. The roadmap gains a new Part-II phase (**Phase 6 — Evidence-Governed Domain Layer**, see below) that did not exist in the 2026-05-15 draft because the substrate it ratifies (multi-domain packs, signed reviewer registry, audit-passed gate) did not yet exist either. The phase is documented here as accepted and partially landed, not proposed. Major chains accepted since 2026-05-17: @@ -342,7 +342,7 @@ The phase has no single exit criterion. Instead, each domain becomes its own sub **Entry criteria:** Phase 5 corpus flywheel operational (curriculum + miner sourcing actively producing reviewed proposals); pack-layer chain (ADR-0027..0045) closed; forward-graph-constraint and surface-composer chains (ADR-0046..0089) shipping. -**Goal.** Distinguish *contract-passing* from *demonstrated* at the ledger surface. A pack that satisfies the nine ADR-0091 predicates earns a `reasoning-capable` ledger row; promotion to `expert_demo=true` requires a reviewer-signed evidence-bundle digest that reproduces byte-for-byte from on-disk lane results. +**Goal.** Distinguish *contract-passing* from *demonstrated* at the ledger surface. A pack that satisfies the nine ADR-0091 predicates earns a `reasoning-capable` ledger row; promotion to `audit_passed=true` requires a reviewer-signed evidence-bundle digest that reproduces byte-for-byte from on-disk lane results. **Substrate (all accepted as of 2026-05-22):** @@ -368,8 +368,8 @@ The phase has no single exit criterion. Instead, each domain becomes its own sub 1. **ADR-0107** — first promotion attempt (`mathematics_logic`) honestly refused by the contract on two named blockers (metric-shape uniformity assumption; `inference_closure` substantively failing at 40% pass). 2. **ADR-0109** — threshold rules amended with explicit lane-shape registry; cognition-shape thresholds preserved bit-identical; four new shapes added (`accuracy_shape`, `inference_shape`, `refusal_shape`, `symbolic_logic_shape`); unknown lanes fail-closed. 3. PR #117 fixed the intent-classifier regression that had broken `inference_closure`. -4. **ADR-0110** — `mathematics_logic` promoted to `expert_demo=true` under the amended contract. Signed claim digest reproduces from on-disk lane results; first domain at expert-demo. -5. **ADR-0111** — `physics` promoted to `expert_demo=true` without any contract change. Shares `inference_closure` + `fabrication_control` results with math; the digest distinguishes via `domain_id` + `evidence_revision`. Retires the "math-only" objection; confirms the bridges ADR-0110 landed were correctly scoped. +4. **ADR-0110** — `mathematics_logic` promoted to `audit_passed=true` under the amended contract. Signed claim digest reproduces from on-disk lane results; first domain at audit-passed. +5. **ADR-0111** — `physics` promoted to `audit_passed=true` without any contract change. Shares `inference_closure` + `fabrication_control` results with math; the digest distinguishes via `domain_id` + `evidence_revision`. Retires the "math-only" objection; confirms the bridges ADR-0110 landed were correctly scoped. **Exit criteria (cumulative; each can land independently of the others):** diff --git a/docs/decisions/ADR-0113-rename-expert-demo-to-audit-passed.md b/docs/decisions/ADR-0113-rename-expert-demo-to-audit-passed.md new file mode 100644 index 00000000..2409f887 --- /dev/null +++ b/docs/decisions/ADR-0113-rename-expert-demo-to-audit-passed.md @@ -0,0 +1,190 @@ +# ADR-0113 — Rename `expert-demo` → `audit-passed`; Reserve `expert` for Future Capability Tier + +**Status:** Accepted +**Date:** 2026-05-22 +**Author:** CORE agents + reviewers +**Depends on:** ADR-0106, ADR-0109, ADR-0110, ADR-0111, ADR-0112 + +--- + +## Context + +ADR-0106 introduced the `expert-demo` ledger status as a contract-gated +promotion above `reasoning-capable`. ADR-0110 / ADR-0111 promoted +`mathematics_logic` and `physics` to that status. ADR-0112 added a +runnable per-domain showcase (`core demo expert --domain `). + +The word "expert" carried an unintended implication: that a domain at +`expert-demo` had been demonstrated at *expert-level capability* (i.e. +raw task performance comparable to a human domain expert). The gate +**does not** verify that. The gate verifies that a domain has cleared +the **CORE claim contract**: + +1. Domain Pack Contract v1 — all nine ADR-0091 predicates pass. +2. A reviewer-signed evidence-bundle SHA-256 reproduces byte-for-byte + from on-disk lane result files (ADR-0106 §1.5). +3. The lane outputs satisfy CORE-specific claim shapes: + - **signed digest** — every claim reproduces from disk + - **replay determinism** — same inputs → byte-equal trace_hash + - **typed refusal** — fabrication is refused, not paraphrased + - **exact recall** — no ANN, no cosine, no attention-bottleneck + - **grounding-source provenance** — every surface tags its origin + +These are claim shapes a transformer-based LLM **structurally cannot +produce**, regardless of how high its raw accuracy is on the same +benchmark. A frontier LLM might score higher on the raw answers but +cannot pass this contract because it cannot produce a digest that +re-derives, cannot guarantee typed refusal, cannot emit a trace hash +bound to a deterministic execution, cannot replay byte-equal. + +That is the load-bearing claim. "Expert-demo" obscured it. "Audit-passed" +names it. + +The framing is due to Codex's review (see PR #125 thread): *the real +story is not "this domain is expert-level" — it is "this domain's +results have been audited against the CORE claim contract, signed, +and are replay-reproducible."* + +--- + +## Decision + +### Rename (semantics-only scope) + +The following **user-visible** identifiers are renamed: + +| Surface | Before | After | +|---|---|---| +| Ledger status string | `"expert-demo"` | `"audit-passed"` | +| Predicate key on ledger row | `predicates.expert_demo` | `predicates.audit_passed` | +| Reason key on ledger row | `expert_demo_reason` | `audit_passed_reason` | +| `docs/reviewers.yaml` top-level key | `expert_demo_claims` | `audit_passed_claims` | +| CLI demo target | `core demo expert --domain X` | `core demo audit-passed --domain X` | +| Generated artifact JSON name | `expert_demo.json` | `audit_passed.json` | +| Generated artifact HTML name | `expert_demo.html` | `audit_passed.html` | +| Default output directory | `evals/expert_demos//` | `evals/audit_passed//` | +| HTML title | `CORE Expert-Demo: ` | `CORE Audit-Passed: ` | + +### Kept (internal Python identifiers, not user-facing) + +The following internal names are **deliberately unchanged** to minimize +churn under this ADR's "semantics-only" scope: + +- Module names: `core/capability/expert_demo.py`, + `core/demos/expert_demo.py` +- Function names: `evaluate_expert_demo`, `derive_evidence_digest`, + `build_expert_demo`, `run_expert_demo`, `_load_registry_for_expert_demo` +- Class names: `ExpertDemoClaim`, `ExpertDemoVerdict` +- Method names on `ReviewerRegistry`: `expert_demo_claim_for` +- Dataclass field names: `ReviewerRegistry.expert_demo_claims` +- ADR file titles: ADR-0106 / ADR-0107 / ADR-0110 / ADR-0111 / ADR-0112 + retain their "expert-demo" titles as historical records +- Test class/method names that include "expert_demo": Python identifiers, + retained + +A future ADR may rename these internal identifiers if desired; that is +explicitly out of scope here. + +### Future `expert` namespace reserved + +The word `expert` (and any `"expert"` ledger status string above +`"audit-passed"`) is **reserved** for an actual raw-capability claim +backed by a domain-specific capability lane with a human-expert-calibrated +threshold. The current ADR establishes no such tier and no such gate. +ADR-0114+ may define it when there is evidence to gate on. + +The status ordering remains a 5-tuple: + +```text +blocked → seeded → grounded → reasoning-capable → audit-passed +``` + +A future `expert` tier would extend this to 6 entries; that extension +requires its own ADR and is out of scope here. + +--- + +## Invariants + +### `adr_0113_ledger_status_string_is_audit_passed` + +`ledger_report()` reports `"audit-passed"` (not `"expert-demo"`) for +every domain whose contract gate passes. Pinned by every existing +ADR-0110 / ADR-0111 / capability-reports test that was updated to the +new vocabulary. + +### `adr_0113_yaml_key_is_audit_passed_claims` + +`load_reviewer_registry` reads from the `audit_passed_claims` YAML key. +The legacy `expert_demo_claims` key is **not** accepted (hard cut, no +backwards-compat read). Tested by `test_reviewer_registry.py`. + +### `adr_0113_audit_passed_does_not_imply_capability` + +The `audit-passed` status string carries no claim about raw task +performance vs. external benchmarks. It is a CORE-claim-contract +compliance status. This invariant is documentary (enforced by the +README + ADR text, not by code). + +### `adr_0113_expert_namespace_reserved` + +No ledger row carries `predicates.expert` and no `"expert"` status +string is emitted by `ledger_report()`. Tested by the existence of +exactly five statuses in `_EXPERT_DOMAIN_STATUSES`. + +--- + +## Acceptance evidence + +Accepted when: + +- `_EXPERT_DOMAIN_STATUSES[-1] == "audit-passed"` in + `core/capability/reporting.py` +- `predicates.audit_passed` (not `expert_demo`) is the key on every + ledger row +- `docs/reviewers.yaml` top-level key is `audit_passed_claims` +- `core demo audit-passed --domain mathematics_logic` and + `--domain physics` both produce `all_claims_supported=True` with + digest match +- All test suites that previously asserted the old strings now assert + the new strings (or accept both in transitional cases like the + historical ratification tests) +- README, ADR-0091..0112 narrative refs, runtime_contracts, + capability_roadmap updated to the new vocabulary (with explicit + gloss on what `audit-passed` actually means) +- Generated artifacts are renamed (`audit_passed.{json,html}`); the + default output directory is `evals/audit_passed//` + +--- + +## Consequences + +- The ledger now tells the honest story. `audit-passed` accurately + names the load-bearing CORE-vs-LLM claim (audit-shape compliance) and + reserves "expert" for an actual capability claim if/when one ever + lands. +- External readers can no longer infer "this domain is expert-level" + from the status string. They must read the gloss: audit-passed = + signed digest + replay determinism + typed refusal + exact recall — + claim shapes a transformer cannot structurally produce. +- The future `expert` tier has clean namespace. ADR-0114 may define it + paired with a domain-specific capability lane (e.g. GSM8K for math). +- ADR-0106 + ADR-0109 contract bodies are unchanged. The gate's + mechanics are identical; only the names move. +- Generated artifacts under `evals/audit_passed//` are still + gitignored per ADR-0112. + +--- + +## Out of scope + +- Renaming internal Python identifiers (module / function / class + names). A separate ADR may do that. +- Defining the `expert` tier above `audit-passed`. ADR-0114+. +- Backward-compatibility reading of the old `expert_demo_claims` YAML + key. Hard cut. The repo is the single source of truth and ships with + the new key. +- Re-rendering the ADR-0099 public showcase. That demo composes + pre-existing scene adapters and is unaffected. +- The `core demo expert` CLI command. Replaced by `core demo + audit-passed`. No alias. diff --git a/docs/decisions/ADR-0114-expert-capability-roadmap-gsm8k-first.md b/docs/decisions/ADR-0114-expert-capability-roadmap-gsm8k-first.md new file mode 100644 index 00000000..3a4521af --- /dev/null +++ b/docs/decisions/ADR-0114-expert-capability-roadmap-gsm8k-first.md @@ -0,0 +1,263 @@ +# ADR-0114 — Expert-Capability Roadmap: GSM8K-Math First + +**Status:** Proposed +**Date:** 2026-05-22 +**Author:** CORE agents + reviewers +**Depends on:** ADR-0091, ADR-0106, ADR-0109, ADR-0110, ADR-0111, ADR-0112, ADR-0113 +**Supersedes:** none (greenfield) + +--- + +## Context + +ADR-0113 reserved the `expert` namespace for a future ledger tier +above `audit-passed`. `audit-passed` verifies CORE *claim-shape +compliance* (signed digest, replay determinism, typed refusal, exact +recall) — a real, transformer-unreachable property. It does **not** +verify raw task performance against external benchmarks. + +This ADR proposes the first concrete path toward an `expert` ledger +status. It is greenfield: no engine for it exists today. + +### Honest framing of distance + +The current architecture is impressive at the substrate level +(determinism, exact CGA recall, reviewed teaching, audit trail). It +is **far from expert-level on generative tasks**: + +- The deterministic realizer composes pack lemmas via fixed templates + and reviewed teaching chains. There is no symbolic solver, no proof + search, no multi-step problem decomposition. +- `en_mathematics_logic_v1` pack has 16 lemmas. Solving an + AMC-10 problem requires a working pipeline that does not exist. +- "Expert prose" is fuzzily defined and would need ~30k+ lemma + lexicon plus discourse-level planning beyond current capability. + +The honest current state is: **no domain is at expert-level capability +by any external measure**. The `audit-passed` rows for math and +physics describe CORE-specific claim shapes, not raw-task supremacy. + +This ADR proposes building toward that — starting with the lowest- +hanging credible target: **grade-school math word problems on GSM8K**. + +### Why GSM8K first + +- **Public benchmark with established frontier baselines.** Frontier + LLMs report 92-95% on GSM8K. Smaller open models report 30-70%. The + comparison space is well-mapped. +- **Checkable answers.** Each problem has a single integer answer; no + fuzzy grading required. +- **Smallest architectural delta.** The math pack already names the + right operators (`adds`, `subtracts`, `multiplies`, `divides`); the + realizer already composes deterministic surfaces. The missing piece + is a real solver loop, not a fundamentally different substrate. +- **Honest first step.** If CORE scores at 6th-grader level on GSM8K, + the result is *still load-bearing* — because the score would be + replayable, traceable, and grounded in a way frontier LLM scores + structurally cannot be. We can claim "Nth-percentile GSM8K, with + trace-verifiable solutions" — a claim no LLM can make. +- **Compositional fit.** GSM8K's reasoning depth (typically 2-8 steps) + fits CORE's proposition-graph + teaching-chain substrate without + requiring an unbounded search. + +--- + +## Decision + +Establish a phased path to a first `expert` ledger tier claim, scoped +to **`mathematics_logic`** as the first domain and **GSM8K** as the +first benchmark. Each phase is its own ADR; this ADR proposes the +sequence and defines exit criteria. + +### Phase 1 — Problem Parser (ADR-0115, future) + +Build `generate/math_parser.py`: turns a natural-language word problem +into a typed proposition graph. + +- Input: GSM8K problem string. +- Output: a `MathProblemGraph` with typed nodes: `entity`, `quantity`, + `unit`, `operation`, `unknown`, `relation`. +- Constraint: deterministic. Same input → byte-identical graph. +- No solver yet — just structured parse. + +Exit criterion: on a curated dev set of 50 GSM8K-style problems +(authored, not from GSM8K to avoid contamination), parse correctness +≥ 0.90 measured by human review against a published rubric. + +### Phase 2 — Deterministic Solver (ADR-0116, future) + +Build `generate/math_solver.py`: a tiny term-rewriting system over the +`MathProblemGraph` using the existing math-pack operator vocabulary. + +- Input: `MathProblemGraph`. +- Output: a `SolutionTrace` — ordered list of operation applications + ending at a numeric answer (or a typed refusal if the graph is + under-determined). +- Constraint: pure function. No sampling. Trace is byte-deterministic + from input. + +Exit criterion: on the Phase 1 curated dev set, solver yields correct +final answer on ≥ 80% of graphs the parser produces correctly. The +solver does **not** need to solve every problem; it needs to be honest +about which it can. + +### Phase 3 — Verifier (ADR-0117, future) + +The verifier re-derives the answer from the `SolutionTrace` and emits +a typed verdict. This is the easy phase — CORE's substrate already +has replayability. The verifier just enforces it on this new artifact. + +Exit criterion: replay determinism = 1.0 on all Phase 2 outputs. + +### Phase 4 — Stepped-Realizer Extension (ADR-0118, future) + +Extend the existing realizer to emit show-your-work prose from a +`SolutionTrace` — one sentence per operation, with pack-grounded +operator vocabulary. + +Exit criterion: every Phase 2 success produces a stepped explanation +of length proportional to trace length, with each sentence +pack-grounded. + +### Phase 5 — GSM8K Eval Lane (ADR-0119, future) + +Author `evals/gsm8k/`: + +- `dev/cases.jsonl` — curated subset of GSM8K train (300 problems). +- `public/v1/cases.jsonl` — curated subset of GSM8K train (1500 + problems; disjoint from dev). +- `holdouts/v1/cases_plaintext.jsonl` — curated subset of GSM8K + *test* (300 problems; never read during development). +- `runner.py` — drives parser → solver → verifier → realizer and + scores against the integer answer. +- `contract.md` — lane contract; lane shape `gsm8k_capability_shape` + (new shape; introduced by ADR-0119 amendment to ADR-0109). + +Exit criterion: lane runner produces deterministic results. Honest +first number reported (whatever it is). + +### Phase 6 — First `expert` Promotion Contract (ADR-0120, future) + +Define the `expert` ledger status: + +- `expert=true` iff: + 1. `audit_passed=true` predicates pass (ADR-0106 + ADR-0109) + 2. At least one **capability lane** attached to the domain meets a + **human-expert-calibrated threshold** declared in the lane's + `contract.md` + 3. Reviewer-signed `expert_claims` entry whose evidence-bundle + digest reproduces byte-for-byte (mirrors ADR-0106 §1.5 exactly) + 4. The capability lane's threshold is declared **publicly** in the + ADR, not buried in a config — so external readers can debate the + calibration. + +For GSM8K specifically, candidate human-expert-calibrated thresholds: + +| Threshold | Interpretation | +|---|---| +| ≥ 0.40 | "competent" — beats average 14-year-old human | +| ≥ 0.60 | "advanced" — competitive open-source LLM territory | +| ≥ 0.85 | "expert" — frontier-LLM territory; published-paper bar | + +ADR-0120 must pick one and justify the choice. The chosen number is +*falsifiable*: if CORE scores below it, no `expert` row. + +### Phase 7 — Second Capability Domain (ADR-0121+, future) + +Pick a second capability domain only after Phase 6 lands. Likely +candidates: symbolic logic with quantifiers (closest cousin to math), +or DSL code generation (checkable, lexicon-bounded). Defer until +math expertise is real. + +### Writing / open-prose capability — explicitly deferred + +Open prose ("expert essay" / "expert article") is **not** on this +roadmap. Reasons: + +- "Correct prose" is fuzzily defined; no GSM8K-equivalent benchmark + exists with sharp scoring. +- Required lexicon (~30k+ working English lemmas) is two orders of + magnitude larger than the current cognition pack. +- Discourse-level planning beyond current capability would need a + separate architectural arc. + +Writing capability becomes appropriate once at least two symbolic +domains have landed at `expert` and the substrate has been stress-tested +against checkable benchmarks. + +--- + +## Non-Decisions + +This ADR explicitly does **not**: + +- Commit to a phase timeline. Each phase is its own ADR with its own + scope. The ADR sequence is the durable artifact; the schedule is not. +- Promise CORE will reach 85% GSM8K. We commit to *honest scoring*, + not to a target. If the architecture caps out at 35%, we report 35%. +- Pretend the audit-passed gate is a capability claim. The two tiers + are distinct and remain distinct. +- Re-render existing audit-passed claims as expert claims. Math and + physics audit-passed promotions stand independently of any future + expert promotion on the same domain. + +--- + +## Invariants + +### `adr_0114_expert_namespace_undefined_until_adr_0120` + +No code ships an `expert` ledger status before ADR-0120 lands. The +`_EXPERT_DOMAIN_STATUSES` tuple in `core.capability.reporting` stays +at 5 entries. Tested by the existing reporting tests. + +### `adr_0114_gsm8k_is_first_capability_target` + +The first capability lane authored under this roadmap is GSM8K +(ADR-0119). No other capability lane lands first. + +### `adr_0114_expert_requires_explicit_threshold` + +The first `expert` promotion contract (ADR-0120) must declare a +public threshold number for the underlying capability lane. No +hidden calibration. + +--- + +## Acceptance evidence (for this proposed ADR) + +ADR-0114 is accepted when: + +- The ADR file exists in `docs/decisions/` and is linked from + `docs/decisions/README.md` +- No code changes — this is a roadmap ADR only +- README updated to point at this sequence as the path-to-expert + +No tests need to be added by this ADR. Tests are scoped to each +implementation ADR. + +--- + +## Consequences + +- The repo has a public, dated commitment to a first expert-capability + target. The "what are you actually claiming is expert-level?" + question now has a written answer: nothing yet; the path is ADR-0115 + through ADR-0120. +- The first `expert` claim, when it lands, will be *falsifiable*: tied + to a public benchmark with a stated threshold. If CORE underperforms, + the row stays at `audit-passed`. +- The `audit-passed` status remains the load-bearing CORE-vs-LLM claim + in the interim. Nothing about the expert roadmap diminishes it; the + two tiers measure orthogonal properties. + +--- + +## Out of scope + +- Implementation of any of Phases 1-7. Each is its own ADR. +- Writing / open-prose roadmap. Deferred until at least one symbolic + domain lands at `expert`. +- Specific GSM8K threshold choice. ADR-0120's job. +- Alternative first benchmarks (MATH, MMLU-math, AIME). All are + candidates after GSM8K; none replace it as Phase 1. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 793c3260..91b6f69f 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -32,13 +32,15 @@ ADRs record significant architectural decisions: what was decided, why, what alt | [ADR-0109](ADR-0109-lane-shape-aware-thresholds.md) | Lane-Shape-Aware Thresholds (ADR-0106 Amendment) | Accepted (2026-05-22) | | [ADR-0110](ADR-0110-mathematics-logic-expert-demo-promotion.md) | `mathematics_logic` Expert-Demo Promotion | Accepted (2026-05-22) | | [ADR-0111](ADR-0111-physics-expert-demo-promotion.md) | `physics` Expert-Demo Promotion | Accepted (2026-05-22) | -| [ADR-0112](ADR-0112-runnable-expert-demo-showcase.md) | Runnable Expert-Demo Showcase | Accepted (2026-05-22) | +| [ADR-0112](ADR-0112-runnable-expert-demo-showcase.md) | Runnable Audit-Passed Showcase (originally "Expert-Demo") | Accepted (2026-05-22) | +| [ADR-0113](ADR-0113-rename-expert-demo-to-audit-passed.md) | Rename `expert-demo` → `audit-passed`; Reserve `expert` for Future Capability Tier | Accepted (2026-05-22) | +| [ADR-0114](ADR-0114-expert-capability-roadmap-gsm8k-first.md) | Expert-Capability Roadmap: GSM8K-Math First | Proposed (2026-05-22) | --- ## Current frontier -The ADR-0091..0112 slate is fully accepted and mechanically evidenced: +The ADR-0091..0114 slate is fully accepted (0091..0113) plus one proposed-roadmap entry (0114) and mechanically evidenced: - Domain Pack Contract v1 — ADR-0091 - Reviewer Registry v1 — ADR-0092 @@ -61,7 +63,9 @@ The ADR-0091..0112 slate is fully accepted and mechanically evidenced: - Lane-Shape-Aware Thresholds (ADR-0106 amendment) — ADR-0109 - `mathematics_logic` Expert-Demo Promotion (first successful) — ADR-0110 - `physics` Expert-Demo Promotion (second successful) — ADR-0111 -- Runnable Expert-Demo Showcase (`core demo expert --domain `) — ADR-0112 +- Runnable Audit-Passed Showcase (originally "Expert-Demo"; renamed) — ADR-0112 + ADR-0113 +- Rename `expert-demo` → `audit-passed`; reserve `expert` namespace — ADR-0113 +- Expert-Capability Roadmap (GSM8K-Math first); proposed — ADR-0114 ADR-0080 has also landed: Contemplation Loop Phase 1 adds a read-only frontier-compare miner that emits `SPECULATIVE` findings only. @@ -82,7 +86,7 @@ Sequencing per ADR-0108. Listed in priority order: 1. **[ADR-0084](ADR-0084-definitional-layer.md) — Definitional Layer for Lexicon Packs.** Optional per-entry definitional block. Deferred — value surfaces during a worked expert promotion that needs definitional depth. 2. **[ADR-0087](ADR-0087-rhetorical-style-axis.md) — Rhetorical Style Axis.** A third substantive selection axis sibling to anchor-lens. Lowest current priority — no active downstream consumer; register + anchor-lens already demonstrate the orthogonality pattern. -ADR-0080 (Contemplation Loop, Phase 1), ADR-0110 (math expert-demo), and ADR-0111 (physics expert-demo) have all landed — `mathematics_logic` and `physics` are at `expert_demo=true`; the contemplation loop emits read-only `SPECULATIVE` findings from `frontier_compare` reports. The remaining two ratified domains (`systems_software`, `hebrew_greek_textual_reasoning`) need their own promotion ADRs. +ADR-0080 (Contemplation Loop, Phase 1), ADR-0110 (math audit-passed), and ADR-0111 (physics audit-passed) have all landed — `mathematics_logic` and `physics` are at `audit_passed=true`; the contemplation loop emits read-only `SPECULATIVE` findings from `frontier_compare` reports. The remaining two ratified domains (`systems_software`, `hebrew_greek_textual_reasoning`) need their own promotion ADRs. ### Open candidate directions (no ADR yet) @@ -92,12 +96,12 @@ ADR-0080 (Contemplation Loop, Phase 1), ADR-0110 (math expert-demo), and ADR-011 ## Accepted reasoning-capable domains -Per ADR-0106, `expert_demo` is **contract-gated**, not threshold-only: a domain row may carry `expert_demo=true` only when a reviewer-signed `expert_demo_claims` entry exists whose evidence-bundle digest reproduces byte-for-byte. ADR-0107 attempted the first worked promotion and the contract refused; ADR-0109 amended the threshold rules; ADR-0110 then successfully promoted `mathematics_logic` (the first domain at `expert_demo=true`); ADR-0111 promoted `physics` second without further contract change, retiring the "math-only" objection. The other two ratified domains remain at `reasoning-capable` pending their own promotion ADRs. +Per ADR-0106, `audit_passed` is **contract-gated**, not threshold-only: a domain row may carry `audit_passed=true` only when a reviewer-signed `audit_passed_claims` entry exists whose evidence-bundle digest reproduces byte-for-byte. ADR-0107 attempted the first worked promotion and the contract refused; ADR-0109 amended the threshold rules; ADR-0110 then successfully promoted `mathematics_logic` (the first domain at `audit_passed=true`); ADR-0111 promoted `physics` second without further contract change, retiring the "math-only" objection. The other two ratified domains remain at `reasoning-capable` pending their own promotion ADRs. | Domain | Ratification ADR | Pack(s) | Evidence summary | |---|---|---|---| -| `mathematics_logic` | ADR-0097 + ADR-0110 | `en_mathematics_logic_v1` | All nine ADR-0091 predicates pass; ledger row is **`expert-demo`** (first such promotion, ADR-0110); all three attached lanes meet ADR-0109 shape thresholds on public + holdout. | -| `physics` | ADR-0100 + ADR-0111 | `en_physics_v1` | All nine predicates pass; causal/modal operator coverage meets threshold; ledger row is **`expert-demo`** (second such promotion, ADR-0111); `foundational_physics_ood` 117/117 public + 39/39 holdout; shares `inference_closure` + `fabrication_control` results with math (distinct digest via `domain_id`). | +| `mathematics_logic` | ADR-0097 + ADR-0110 | `en_mathematics_logic_v1` | All nine ADR-0091 predicates pass; ledger row is **`audit-passed`** (first such promotion, ADR-0110); all three attached lanes meet ADR-0109 shape thresholds on public + holdout. | +| `physics` | ADR-0100 + ADR-0111 | `en_physics_v1` | All nine predicates pass; causal/modal operator coverage meets threshold; ledger row is **`audit-passed`** (second such promotion, ADR-0111); `foundational_physics_ood` 117/117 public + 39/39 holdout; shares `inference_closure` + `fabrication_control` results with math (distinct digest via `domain_id`). | | `systems_software` | ADR-0101 | `en_systems_software_v1` | All nine predicates pass; transitive/causal operator coverage meets threshold; ledger row is `reasoning-capable`; `symbolic_logic` is the v1 closest-fit eval lane. | | `hebrew_greek_textual_reasoning` | ADR-0102 + ADR-0103 | `grc_logos_micro_v1`, `grc_logos_cognition_v1`, `he_logos_micro_v1`, `he_core_cognition_v1` | First multi-pack ratification; all four packs carry uniform contract fields; causal/contradiction operator coverage meets threshold; ADR-0103 attaches Hebrew and Koine Greek fluency lanes with `dev/public/holdout` coverage. | @@ -123,7 +127,7 @@ ADR-0027 through ADR-0045 establish the identity / safety / ethics pack architec ### Evidence-governed domain chain — ADR-0091 through ADR-0111 -ADR-0091 through ADR-0111 establish the current domain-ratification substrate and the expert-demo promotion gate that distinguishes contract-passing from demonstrated: +ADR-0091 through ADR-0111 establish the current domain-ratification substrate and the audit-passed promotion gate that distinguishes contract-passing from demonstrated: ```text contract definition (0091) @@ -138,14 +142,14 @@ reasoning-capable domain ratification (0097 / 0100 / 0101 / 0102) ↓ language-specific fluency lane attachment (0103) ↓ -expert-demo promotion contract (0106 + 0109 amendment) +audit-passed promotion contract (0106 + 0109 amendment) ↓ -worked expert-demo promotion (0110 — mathematics_logic, first) +worked audit-passed promotion (0110 — mathematics_logic, first) ↓ -worked expert-demo promotion (0111 — physics, second; no contract change) +worked audit-passed promotion (0111 — physics, second; no contract change) ``` -No domain claim should be treated as mature merely because a pack exists. `reasoning-capable` means the nine ADR-0091 predicates pass; `expert-demo` requires a reviewer-signed evidence-bundle digest that reproduces byte-for-byte from on-disk lane results. +No domain claim should be treated as mature merely because a pack exists. `reasoning-capable` means the nine ADR-0091 predicates pass; `audit-passed` requires a reviewer-signed evidence-bundle digest that reproduces byte-for-byte from on-disk lane results. The contract has been demonstrated end-to-end: refused once honestly (ADR-0107), amended once cleanly (ADR-0109), succeeded against `mathematics_logic` (ADR-0110), and succeeded against `physics` without further contract change (ADR-0111). diff --git a/docs/eval_methodology.md b/docs/eval_methodology.md index a71b0f7d..928e3c2e 100644 --- a/docs/eval_methodology.md +++ b/docs/eval_methodology.md @@ -109,7 +109,7 @@ A lane without a `contract.md` does not run. ADR-0091's Domain Pack Contract v1 introduced a `dev/public/holdout` discipline that every ratified pack must declare. ADR-0106 added a -reviewer-signed expert-demo promotion gate that consults those same +reviewer-signed audit-passed promotion gate that consults those same lane outputs at the ledger level. ADR-0109 then formalized the rule that **threshold dispatch is lane-shape-aware**, not lane-uniform. @@ -134,10 +134,10 @@ broadened**. The path is: 3. Add the lane → shape mapping to `LANE_SHAPE_REGISTRY`. A lane id absent from the registry is **fail-closed** at the -expert-demo gate (reason: +audit-passed gate (reason: `lane has no registered shape — introduce via ADR amendment`). Unregistered lanes can still run as exploration; they just cannot -contribute evidence to a `reviewer-signed expert_demo` promotion. +contribute evidence to a `reviewer-signed audit_passed` promotion. ### Holdout-runner gating (ADR-0105) diff --git a/docs/reviewers.yaml b/docs/reviewers.yaml index be3b7504..0550b568 100644 --- a/docs/reviewers.yaml +++ b/docs/reviewers.yaml @@ -17,11 +17,16 @@ reviewers: review_scope: ["pack", "proposal", "chain", "eval"] provenance: "adr-0092:bootstrap:2026-05-21" -# Reviewer-signed expert-demo promotion claims (ADR-0106 / ADR-0109). +# Reviewer-signed audit-passed promotion claims +# (ADR-0106 / ADR-0109 / renamed by ADR-0113). # Each entry carries the canonical evidence-bundle SHA-256 hashed from # the on-disk lane result files; re-derivation must reproduce -# claim_digest byte-for-byte (ADR-0106 §1.5). -expert_demo_claims: +# claim_digest byte-for-byte (ADR-0106 §1.5). The audit-passed gate +# verifies CORE claim-shape compliance (signed digest, replay +# determinism, typed refusal, exact recall) — claim shapes a +# transformer LLM cannot structurally produce regardless of raw +# accuracy. This is NOT a raw-capability claim. +audit_passed_claims: - domain_id: mathematics_logic evidence_lanes: - elementary_mathematics_ood diff --git a/docs/runtime_contracts.md b/docs/runtime_contracts.md index da6e3871..775ebed6 100644 --- a/docs/runtime_contracts.md +++ b/docs/runtime_contracts.md @@ -361,12 +361,12 @@ without further contract change (ADR-0111). `ledger_report()` returns a `domains` list. Each row carries: ```text -status ∈ {blocked, seeded, grounded, reasoning-capable, expert-demo} -predicates.expert_demo bool -expert_demo_reason str (one-line legibility for operators) +status ∈ {blocked, seeded, grounded, reasoning-capable, audit-passed} +predicates.audit_passed bool +audit_passed_reason str (one-line legibility for operators) ``` -A row carries `expert_demo=True` iff **all** of: +A row carries `audit_passed=True` iff **all** of: 1. `reasoning_capable == True` (the ADR-0091 nine-predicate gate). 2. A signed `ExpertDemoClaim` exists in `docs/reviewers.yaml` for the @@ -382,14 +382,14 @@ A row carries `expert_demo=True` iff **all** of: byte-for-byte. Any failure leaves the row at `reasoning-capable` with -`expert_demo_reason` populated. +`audit_passed_reason` populated. ### Schema -`docs/reviewers.yaml` additively gains an `expert_demo_claims` block: +`docs/reviewers.yaml` additively gains an `audit_passed_claims` block: ```yaml -expert_demo_claims: +audit_passed_claims: - domain_id: mathematics_logic evidence_lanes: - elementary_mathematics_ood @@ -423,12 +423,12 @@ Threshold rules dispatch by lane shape, not uniformly. Registry in **Unknown lane ids fail closed** with reason `lane has no registered shape — introduce via ADR amendment`. -Adding a lane to the expert-demo surface requires an explicit registry +Adding a lane to the audit-passed surface requires an explicit registry entry, which requires an ADR amendment. ### Replay invariant -`core.capability.expert_demo.derive_evidence_digest` is deterministic +`core.capability.audit_passed.derive_evidence_digest` is deterministic in field order (sorted keys, compact separators). Re-running it against the on-disk lane results at the same `evidence_revision` reproduces `claim_digest` byte-for-byte. Drift in any lane result @@ -439,13 +439,13 @@ demotes the row back to `reasoning-capable` until re-signed. If `docs/reviewers.yaml` fails to parse, `reporting._load_registry_for_expert_demo` returns an empty registry (zero reviewers, zero claims) rather than raising. Every domain row -falls back to `expert_demo=false`. A broken registry must never -silently grant `expert_demo=true`. +falls back to `audit_passed=false`. A broken registry must never +silently grant `audit_passed=true`. ### Trust boundary - Pack mutation remains proposal-only (ADR-0029/0064 discipline). -- A reviewer signature in `expert_demo_claims` does not authorize +- A reviewer signature in `audit_passed_claims` does not authorize pack mutation — only a ledger-row promotion. The two paths remain separate. - `evidence_revision` may be a labeled string (e.g. diff --git a/tests/test_adr_0097_mathematics_logic_ratification.py b/tests/test_adr_0097_mathematics_logic_ratification.py index 68de9f5e..2aa19036 100644 --- a/tests/test_adr_0097_mathematics_logic_ratification.py +++ b/tests/test_adr_0097_mathematics_logic_ratification.py @@ -55,15 +55,20 @@ class TestRatificationPredicates: class TestLedgerStatus: """ADR-0097 invariant: ``mathematics_logic_reasoning_capable_ledger_row``.""" - def test_status_is_reasoning_capable(self) -> None: + def test_status_meets_reasoning_capable_at_minimum(self) -> None: + """ADR-0097 ratified math at reasoning-capable. ADR-0110 later + promoted it to audit-passed (renamed from expert-demo by + ADR-0113). The load-bearing invariant for ADR-0097 is that + reasoning_capable holds; the status string moves with later + promotions.""" row = _ledger_row(DOMAIN_ID) - assert row["status"] == "reasoning-capable" + assert row["status"] in ("reasoning-capable", "audit-passed") + assert row["predicates"]["reasoning_capable"] is True - def test_expert_demo_predicate_is_false(self) -> None: - """ADR-0097 explicitly defers expert-demo to a follow-up ADR.""" + def test_reasoning_capable_predicate_holds(self) -> None: + """ADR-0097's load-bearing invariant: reasoning_capable=True.""" row = _ledger_row(DOMAIN_ID) assert row["predicates"]["reasoning_capable"] is True - assert row["predicates"]["expert_demo"] is False def test_no_open_gaps(self) -> None: row = _ledger_row(DOMAIN_ID) diff --git a/tests/test_adr_0100_0102_sibling_ratifications.py b/tests/test_adr_0100_0102_sibling_ratifications.py index 8198cc69..4e42c8c4 100644 --- a/tests/test_adr_0100_0102_sibling_ratifications.py +++ b/tests/test_adr_0100_0102_sibling_ratifications.py @@ -109,19 +109,19 @@ def test_pack_passes_all_nine_predicates( @pytest.mark.parametrize("domain_id", list(_DOMAINS.keys())) class TestLedgerStatus: - def test_status_is_reasoning_capable(self, domain_id: str) -> None: + def test_status_meets_reasoning_capable_at_minimum(self, domain_id: str) -> None: + """Each ratification ADR established reasoning-capable as the + floor. ADR-0111 later promoted physics to audit-passed (renamed + from expert-demo by ADR-0113); the load-bearing invariant for + the ratification ADRs is that reasoning_capable holds, not + that the status string never moves up.""" row = _ledger_row(domain_id) - assert row["status"] == "reasoning-capable" + assert row["status"] in ("reasoning-capable", "audit-passed") def test_reasoning_capable_predicate_true(self, domain_id: str) -> None: row = _ledger_row(domain_id) assert row["predicates"]["reasoning_capable"] is True - def test_expert_demo_predicate_false(self, domain_id: str) -> None: - """Each ratification ADR explicitly defers expert-demo.""" - row = _ledger_row(domain_id) - assert row["predicates"]["expert_demo"] is False - def test_no_open_gaps(self, domain_id: str) -> None: row = _ledger_row(domain_id) assert row["open_gaps"] == [] diff --git a/tests/test_adr_0110_math_expert_demo.py b/tests/test_adr_0110_math_expert_demo.py index 2a3caa92..93411d18 100644 --- a/tests/test_adr_0110_math_expert_demo.py +++ b/tests/test_adr_0110_math_expert_demo.py @@ -3,15 +3,15 @@ Pins three load-bearing invariants: 1. ``adr_0110_math_expert_demo_holds`` — ``ledger_report()`` reports - ``mathematics_logic`` at ``status="expert-demo"`` with - ``predicates.expert_demo == True``. + ``mathematics_logic`` at ``status="audit-passed"`` with + ``predicates.audit_passed == True``. 2. ``adr_0110_replay_digest_byte_equality`` — re-deriving the evidence-bundle digest from the on-disk lane result files reproduces the signed ``claim_digest`` byte-for-byte (ADR-0106 §1.5). 3. ``adr_0110_other_domains_unaffected`` — ADR-0110 promotes exactly - one domain. Every other domain row stays at ``expert_demo=false`` + one domain. Every other domain row stays at ``audit_passed=false`` under its own (absent) ``expert_demo_claims`` entry. """ @@ -66,8 +66,8 @@ def _math_claim(): class TestAdr0110MathExpertDemoHolds: def test_math_row_is_expert_demo(self) -> None: row = _math_row() - assert row["status"] == "expert-demo" - assert row["predicates"]["expert_demo"] is True + assert row["status"] == "audit-passed" + assert row["predicates"]["audit_passed"] is True def test_signed_claim_is_present(self) -> None: claim = _math_claim() @@ -103,7 +103,7 @@ class TestAdr0110OtherDomainsUnaffected: promoted = [ row["domain"] for row in ledger_report()["domains"] - if row["predicates"]["expert_demo"] + if row["predicates"]["audit_passed"] ] assert "mathematics_logic" in promoted, ( f"ADR-0110 promotion of mathematics_logic must persist; got: {promoted}" diff --git a/tests/test_adr_0111_physics_expert_demo.py b/tests/test_adr_0111_physics_expert_demo.py index c37f8067..f0c034bf 100644 --- a/tests/test_adr_0111_physics_expert_demo.py +++ b/tests/test_adr_0111_physics_expert_demo.py @@ -3,8 +3,8 @@ Pins four load-bearing invariants: 1. ``adr_0111_physics_expert_demo_holds`` — ``ledger_report()`` reports - ``physics`` at ``status="expert-demo"`` with - ``predicates.expert_demo == True``. + ``physics`` at ``status="audit-passed"`` with + ``predicates.audit_passed == True``. 2. ``adr_0111_replay_digest_byte_equality`` — re-deriving the evidence-bundle digest from the on-disk lane result files reproduces @@ -13,7 +13,7 @@ Pins four load-bearing invariants: 3. ``adr_0111_other_domains_unaffected`` — ADR-0111 promotes exactly one new domain. ``mathematics_logic`` (ADR-0110) must remain promoted; ``systems_software``, ``hebrew_greek_textual_reasoning``, - and ``philosophy_theology`` must remain at ``expert_demo=false``. + and ``philosophy_theology`` must remain at ``audit_passed=false``. 4. ``adr_0111_distinct_digest_from_adr_0110`` — physics digest and math digest must differ, demonstrating the bundle's @@ -74,8 +74,8 @@ def _physics_claim(): class TestAdr0111PhysicsExpertDemoHolds: def test_physics_row_is_expert_demo(self) -> None: row = _physics_row() - assert row["status"] == "expert-demo" - assert row["predicates"]["expert_demo"] is True + assert row["status"] == "audit-passed" + assert row["predicates"]["audit_passed"] is True def test_signed_claim_is_present(self) -> None: claim = _physics_claim() @@ -104,7 +104,7 @@ class TestAdr0111OtherDomainsUnaffected: promoted = { row["domain"] for row in ledger_report()["domains"] - if row["predicates"]["expert_demo"] + if row["predicates"]["audit_passed"] } assert "mathematics_logic" in promoted assert "physics" in promoted @@ -122,7 +122,7 @@ class TestAdr0111OtherDomainsUnaffected: f"{row['domain']} expected reasoning-capable, " f"got {row['status']}" ) - assert row["predicates"]["expert_demo"] is False + assert row["predicates"]["audit_passed"] is False class TestAdr0111DistinctDigestFromAdr0110: diff --git a/tests/test_capability_reports.py b/tests/test_capability_reports.py index 0dac64fb..af898456 100644 --- a/tests/test_capability_reports.py +++ b/tests/test_capability_reports.py @@ -146,16 +146,16 @@ def test_ledger_status_is_predicate_derived() -> None: math = rows["mathematics_logic"] # ADR-0110 — first expert-demo promotion lands on math. - assert math["status"] == "expert-demo" + assert math["status"] == "audit-passed" assert math["predicates"]["reasoning_capable"] is True - assert math["predicates"]["expert_demo"] is True + assert math["predicates"]["audit_passed"] is True assert math["open_gaps"] == [] physics = rows["physics"] # ADR-0111 — second expert-demo promotion lands on physics. - assert physics["status"] == "expert-demo" + assert physics["status"] == "audit-passed" assert physics["predicates"]["reasoning_capable"] is True - assert physics["predicates"]["expert_demo"] is True + assert physics["predicates"]["audit_passed"] is True assert physics["open_gaps"] == [] he_grc = rows["hebrew_greek_textual_reasoning"] diff --git a/tests/test_expert_demo_contract.py b/tests/test_expert_demo_contract.py index 85d99b08..8078fbcd 100644 --- a/tests/test_expert_demo_contract.py +++ b/tests/test_expert_demo_contract.py @@ -3,7 +3,7 @@ Pins three load-bearing invariants: 1. ``expert_demo_requires_signature`` — no domain row may carry - ``expert_demo=true`` without a corresponding ``expert_demo_claims`` + ``audit_passed=true`` without a corresponding ``expert_demo_claims`` entry whose digest reproduces the on-disk evidence bundle. 2. ``expert_demo_domain_aware`` — the reporting layer must consult only @@ -17,7 +17,7 @@ Pins three load-bearing invariants: ``reasoning-capable``. The current main-branch state of the ledger has zero domains with -``expert_demo=true``; these tests synthesize fixtures to prove the gate +``audit_passed=true``; these tests synthesize fixtures to prove the gate behaves as ADR-0106 specifies, without flipping any production row. """ @@ -119,7 +119,7 @@ class TestExpertDemoRequiresSignature: ), ) assert verdict.passed is False - assert "no expert_demo_claims entry" in verdict.reason + assert "no audit_passed_claims entry" in verdict.reason def test_unsigned_lanes_refuse_promotion(self) -> None: registry = _build_registry(()) @@ -374,7 +374,7 @@ class TestExpertDemoThresholds: class TestProductionLedgerPromotionsAreSignedOnly: """ADR-0106 §Acceptance preserved post-ADR-0110. - Originally tested that NO domain row carried ``expert_demo=true`` + Originally tested that NO domain row carried ``audit_passed=true`` because no signed claims existed yet. After ADR-0110, the math domain carries a signed claim. The load-bearing invariant remains: every promoted domain must trace back to a signed claim in the @@ -397,6 +397,6 @@ class TestProductionLedgerPromotionsAreSignedOnly: domain = row["domain"] claim = registry.expert_demo_claim_for(domain) assert claim is not None, ( - f"domain {domain!r} reports expert_demo=true but has no " + f"domain {domain!r} reports audit_passed=true but has no " f"signed expert_demo_claims entry" ) diff --git a/tests/test_expert_demo_runnable.py b/tests/test_expert_demo_runnable.py index fedec05c..90bb3bd1 100644 --- a/tests/test_expert_demo_runnable.py +++ b/tests/test_expert_demo_runnable.py @@ -83,11 +83,11 @@ class TestPromotedDomainsBuildSuccessfully: class TestUnpromotedDomainRefused: def test_unpromoted_domain_raises_value_error(self) -> None: - with pytest.raises(ValueError, match="No expert_demo_claims entry"): + with pytest.raises(ValueError, match="No audit_passed_claims entry"): build_expert_demo("systems_software") def test_unknown_domain_raises_value_error(self) -> None: - with pytest.raises(ValueError, match="No expert_demo_claims entry"): + with pytest.raises(ValueError, match="No audit_passed_claims entry"): build_expert_demo("not_a_real_domain") @@ -99,8 +99,8 @@ class TestByteDeterminism: out_b = Path(tmp) / "b" run_expert_demo(domain_id=domain_id, output_dir=out_a) run_expert_demo(domain_id=domain_id, output_dir=out_b) - bytes_a = (out_a / "expert_demo.json").read_bytes() - bytes_b = (out_b / "expert_demo.json").read_bytes() + bytes_a = (out_a / "audit_passed.json").read_bytes() + bytes_b = (out_b / "audit_passed.json").read_bytes() assert bytes_a == bytes_b sha_a = hashlib.sha256(bytes_a).hexdigest() sha_b = hashlib.sha256(bytes_b).hexdigest() @@ -130,17 +130,17 @@ class TestOutputArtifacts: with tempfile.TemporaryDirectory() as tmp: out = Path(tmp) run_expert_demo(domain_id="physics", output_dir=out) - assert (out / "expert_demo.json").is_file() - assert (out / "expert_demo.html").is_file() - html = (out / "expert_demo.html").read_text(encoding="utf-8") - assert "CORE Expert-Demo: physics" in html + assert (out / "audit_passed.json").is_file() + assert (out / "audit_passed.html").is_file() + html = (out / "audit_passed.html").read_text(encoding="utf-8") + assert "CORE Audit-Passed: physics" in html assert "a104cad136f3219df05dc7ce6a78437c02f7b5827cd3cdce568db3acda6a43ed" in html def test_html_contains_per_lane_sections(self) -> None: with tempfile.TemporaryDirectory() as tmp: out = Path(tmp) run_expert_demo(domain_id="physics", output_dir=out) - html = (out / "expert_demo.html").read_text(encoding="utf-8") + html = (out / "audit_passed.html").read_text(encoding="utf-8") for lane_id in ( "foundational_physics_ood", "inference_closure", diff --git a/tests/test_reviewer_registry.py b/tests/test_reviewer_registry.py index 279d3949..b9a6af0d 100644 --- a/tests/test_reviewer_registry.py +++ b/tests/test_reviewer_registry.py @@ -301,7 +301,7 @@ reviewers: domains: ["*"] review_scope: ["pack", "proposal", "chain", "eval"] provenance: "adr-0092:bootstrap:2026-05-21" -expert_demo_claims: +audit_passed_claims: - domain_id: mathematics_logic evidence_lanes: ["inference_closure", "fabrication_control"] evidence_revision: "abc1234"