0493808215
Closes ADR-0106 acceptance evidence: - ExpertDemoClaim dataclass + additive expert_demo_claims block on ReviewerRegistry (schema_version stays at 1; backward-compatible). - New core/capability/expert_demo.py with derive_evidence_digest, evaluate_expert_demo, collect_domain_lanes, materialise_lane_results. - core/capability/reporting.py: replaces the cognition-lane-only predicate (previous lines 418-433) with a domain-aware, reviewer-signed gate; ledger rows now also carry expert_demo_reason for operator legibility. Reviewer registry is fail-closed: an unloadable registry yields zero claims, so a broken registry never silently grants expert_demo=true. - tests/test_expert_demo_contract.py covers all three ADR-0106 invariants: requires_signature, domain_aware, replay_byte_equality; plus threshold + production-ledger-untouched gates. 12 new tests. - tests/test_reviewer_registry.py extended with TestExpertDemoClaimsSchema covering omitted block, valid parse, unknown signer rejection, malformed digest rejection, duplicate domain rejection. 5 new tests. - README index row + table preface updated to note expert_demo is contract-gated. Frontier list trimmed (ADR-0106 has landed). - ADR-0106 Status flipped Proposed -> Accepted. No domain row's expert_demo field flips by this PR -- only the contract changes. Promotion of any ratified domain requires a follow-up ADR (ADR-0107 reserved for mathematics_logic) plus a signed claim.
352 lines
12 KiB
Python
352 lines
12 KiB
Python
"""ADR-0106 — expert-demo promotion contract invariants.
|
|
|
|
Pins three load-bearing invariants:
|
|
|
|
1. ``expert_demo_requires_signature`` — no domain row may carry
|
|
``expert_demo=true`` without a corresponding ``expert_demo_claims``
|
|
entry whose digest reproduces the on-disk evidence bundle.
|
|
|
|
2. ``expert_demo_domain_aware`` — the reporting layer must consult only
|
|
lanes attached to a domain's ratified packs when computing
|
|
``expert_demo``. Cross-domain lane bleed (e.g. cognition-lane metrics
|
|
deciding a math promotion) is refused.
|
|
|
|
3. ``expert_demo_replay_byte_equality`` — re-running every consulted
|
|
lane at ``evidence_revision`` must reproduce the exact JSON bytes
|
|
hashed into ``claim_digest``. Drift demotes the row back to
|
|
``reasoning-capable``.
|
|
|
|
The current main-branch state of the ledger has zero domains with
|
|
``expert_demo=true``; these tests synthesize fixtures to prove the gate
|
|
behaves as ADR-0106 specifies, without flipping any production row.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from core.capability.expert_demo import (
|
|
derive_evidence_digest,
|
|
evaluate_expert_demo,
|
|
)
|
|
from core.capability.reviewers import (
|
|
ExpertDemoClaim,
|
|
Reviewer,
|
|
ReviewerRegistry,
|
|
)
|
|
|
|
|
|
_GOOD_METRICS = {
|
|
"surface_groundedness": 0.97,
|
|
"term_capture_rate": 0.90,
|
|
"intent_accuracy": 0.96,
|
|
"versor_closure_rate": 1.0,
|
|
}
|
|
_FAB_METRICS = {"passed_rate": 1.0}
|
|
|
|
|
|
def _primary_reviewer() -> Reviewer:
|
|
return Reviewer(
|
|
reviewer_id="shay-j",
|
|
display_name="Joshua Shay",
|
|
role="primary",
|
|
domains=("*",),
|
|
review_scope=("pack", "proposal", "chain", "eval"),
|
|
provenance="adr-0092:bootstrap:2026-05-21",
|
|
)
|
|
|
|
|
|
def _build_registry(
|
|
reviewers: tuple[Reviewer, ...],
|
|
claims: tuple[ExpertDemoClaim, ...] = (),
|
|
) -> ReviewerRegistry:
|
|
return ReviewerRegistry(
|
|
schema_version=1, reviewers=reviewers, expert_demo_claims=claims
|
|
)
|
|
|
|
|
|
def _good_lane_results(lanes: tuple[str, ...]) -> dict[str, dict[str, dict]]:
|
|
out: dict[str, dict[str, dict]] = {}
|
|
for lane in lanes:
|
|
metrics = _FAB_METRICS if lane == "fabrication_control" else _GOOD_METRICS
|
|
out[lane] = {"public": dict(metrics), "holdout": dict(metrics)}
|
|
return out
|
|
|
|
|
|
class TestExpertDemoRequiresSignature:
|
|
def test_no_claim_refuses_promotion(self) -> None:
|
|
registry = _build_registry((_primary_reviewer(),))
|
|
verdict = evaluate_expert_demo(
|
|
domain_id="mathematics_logic",
|
|
reasoning_capable=True,
|
|
registry=registry,
|
|
domain_lanes=("inference_closure", "fabrication_control"),
|
|
lane_results=_good_lane_results(
|
|
("inference_closure", "fabrication_control")
|
|
),
|
|
)
|
|
assert verdict.passed is False
|
|
assert "no expert_demo_claims entry" in verdict.reason
|
|
|
|
def test_unsigned_lanes_refuse_promotion(self) -> None:
|
|
registry = _build_registry(())
|
|
verdict = evaluate_expert_demo(
|
|
domain_id="mathematics_logic",
|
|
reasoning_capable=True,
|
|
registry=registry,
|
|
domain_lanes=("inference_closure",),
|
|
lane_results=_good_lane_results(("inference_closure",)),
|
|
)
|
|
assert verdict.passed is False
|
|
|
|
def test_not_reasoning_capable_refuses_promotion(self) -> None:
|
|
registry = _build_registry((_primary_reviewer(),))
|
|
verdict = evaluate_expert_demo(
|
|
domain_id="mathematics_logic",
|
|
reasoning_capable=False,
|
|
registry=registry,
|
|
domain_lanes=("inference_closure",),
|
|
lane_results=_good_lane_results(("inference_closure",)),
|
|
)
|
|
assert verdict.passed is False
|
|
assert "not yet reasoning-capable" in verdict.reason
|
|
|
|
def test_signer_without_eval_scope_refuses_promotion(self) -> None:
|
|
domain = "mathematics_logic"
|
|
reviewer = Reviewer(
|
|
reviewer_id="math-pack-reviewer",
|
|
display_name="Math Pack Reviewer",
|
|
role="domain",
|
|
domains=(domain,),
|
|
review_scope=("pack", "chain"),
|
|
provenance="adr-0092:bootstrap:2026-05-21",
|
|
)
|
|
lanes = ("inference_closure", "fabrication_control")
|
|
results = _good_lane_results(lanes)
|
|
digest = derive_evidence_digest(
|
|
domain_id=domain,
|
|
evidence_revision="abc123",
|
|
evidence_lanes=lanes,
|
|
lane_results=results,
|
|
)
|
|
claim = ExpertDemoClaim(
|
|
domain_id=domain,
|
|
evidence_lanes=lanes,
|
|
evidence_revision="abc123",
|
|
signed_by="math-pack-reviewer",
|
|
claim_digest=digest,
|
|
)
|
|
registry = _build_registry((reviewer,), (claim,))
|
|
verdict = evaluate_expert_demo(
|
|
domain_id=domain,
|
|
reasoning_capable=True,
|
|
registry=registry,
|
|
domain_lanes=lanes,
|
|
lane_results=results,
|
|
)
|
|
assert verdict.passed is False
|
|
assert "cannot review eval-scope" in verdict.reason
|
|
|
|
|
|
class TestExpertDemoDomainAware:
|
|
def test_cross_domain_lane_bleed_refused(self) -> None:
|
|
"""A claim that cites a cognition lane for a math promotion fails."""
|
|
domain = "mathematics_logic"
|
|
math_lanes = ("inference_closure", "fabrication_control")
|
|
bad_claim_lanes = ("cognition",)
|
|
results = _good_lane_results(bad_claim_lanes + math_lanes)
|
|
digest = derive_evidence_digest(
|
|
domain_id=domain,
|
|
evidence_revision="rev1",
|
|
evidence_lanes=bad_claim_lanes,
|
|
lane_results=results,
|
|
)
|
|
claim = ExpertDemoClaim(
|
|
domain_id=domain,
|
|
evidence_lanes=bad_claim_lanes,
|
|
evidence_revision="rev1",
|
|
signed_by="shay-j",
|
|
claim_digest=digest,
|
|
)
|
|
registry = _build_registry((_primary_reviewer(),), (claim,))
|
|
verdict = evaluate_expert_demo(
|
|
domain_id=domain,
|
|
reasoning_capable=True,
|
|
registry=registry,
|
|
domain_lanes=math_lanes,
|
|
lane_results=results,
|
|
)
|
|
assert verdict.passed is False
|
|
assert "lanes not attached to domain" in verdict.reason
|
|
|
|
def test_in_domain_lanes_accepted(self) -> None:
|
|
domain = "mathematics_logic"
|
|
lanes = ("inference_closure", "fabrication_control")
|
|
results = _good_lane_results(lanes)
|
|
digest = derive_evidence_digest(
|
|
domain_id=domain,
|
|
evidence_revision="rev1",
|
|
evidence_lanes=lanes,
|
|
lane_results=results,
|
|
)
|
|
claim = ExpertDemoClaim(
|
|
domain_id=domain,
|
|
evidence_lanes=lanes,
|
|
evidence_revision="rev1",
|
|
signed_by="shay-j",
|
|
claim_digest=digest,
|
|
)
|
|
registry = _build_registry((_primary_reviewer(),), (claim,))
|
|
verdict = evaluate_expert_demo(
|
|
domain_id=domain,
|
|
reasoning_capable=True,
|
|
registry=registry,
|
|
domain_lanes=lanes,
|
|
lane_results=results,
|
|
)
|
|
assert verdict.passed is True
|
|
assert verdict.derived_digest == digest
|
|
|
|
|
|
class TestExpertDemoReplayByteEquality:
|
|
def test_digest_is_stable_across_invocations(self) -> None:
|
|
lanes = ("inference_closure", "fabrication_control")
|
|
results = _good_lane_results(lanes)
|
|
first = derive_evidence_digest(
|
|
domain_id="mathematics_logic",
|
|
evidence_revision="rev1",
|
|
evidence_lanes=lanes,
|
|
lane_results=results,
|
|
)
|
|
second = derive_evidence_digest(
|
|
domain_id="mathematics_logic",
|
|
evidence_revision="rev1",
|
|
evidence_lanes=lanes,
|
|
lane_results=results,
|
|
)
|
|
assert first == second
|
|
|
|
def test_digest_is_order_independent_on_lanes(self) -> None:
|
|
results = _good_lane_results(("a", "b", "c"))
|
|
d1 = derive_evidence_digest(
|
|
domain_id="d",
|
|
evidence_revision="rev1",
|
|
evidence_lanes=("a", "b", "c"),
|
|
lane_results=results,
|
|
)
|
|
d2 = derive_evidence_digest(
|
|
domain_id="d",
|
|
evidence_revision="rev1",
|
|
evidence_lanes=("c", "a", "b"),
|
|
lane_results=results,
|
|
)
|
|
assert d1 == d2
|
|
|
|
def test_drift_in_results_demotes_row(self) -> None:
|
|
domain = "mathematics_logic"
|
|
lanes = ("inference_closure", "fabrication_control")
|
|
original = _good_lane_results(lanes)
|
|
original_digest = derive_evidence_digest(
|
|
domain_id=domain,
|
|
evidence_revision="rev1",
|
|
evidence_lanes=lanes,
|
|
lane_results=original,
|
|
)
|
|
claim = ExpertDemoClaim(
|
|
domain_id=domain,
|
|
evidence_lanes=lanes,
|
|
evidence_revision="rev1",
|
|
signed_by="shay-j",
|
|
claim_digest=original_digest,
|
|
)
|
|
drifted = _good_lane_results(lanes)
|
|
drifted["inference_closure"]["public"]["intent_accuracy"] = 0.99
|
|
registry = _build_registry((_primary_reviewer(),), (claim,))
|
|
verdict = evaluate_expert_demo(
|
|
domain_id=domain,
|
|
reasoning_capable=True,
|
|
registry=registry,
|
|
domain_lanes=lanes,
|
|
lane_results=drifted,
|
|
)
|
|
assert verdict.passed is False
|
|
assert "replay drift" in verdict.reason
|
|
assert verdict.derived_digest is not None
|
|
assert verdict.derived_digest != original_digest
|
|
|
|
|
|
class TestExpertDemoThresholds:
|
|
def test_below_threshold_metric_refuses(self) -> None:
|
|
domain = "mathematics_logic"
|
|
lanes = ("inference_closure", "fabrication_control")
|
|
results = _good_lane_results(lanes)
|
|
results["inference_closure"]["holdout"]["surface_groundedness"] = 0.50
|
|
digest = derive_evidence_digest(
|
|
domain_id=domain,
|
|
evidence_revision="rev1",
|
|
evidence_lanes=lanes,
|
|
lane_results=results,
|
|
)
|
|
claim = ExpertDemoClaim(
|
|
domain_id=domain,
|
|
evidence_lanes=lanes,
|
|
evidence_revision="rev1",
|
|
signed_by="shay-j",
|
|
claim_digest=digest,
|
|
)
|
|
registry = _build_registry((_primary_reviewer(),), (claim,))
|
|
verdict = evaluate_expert_demo(
|
|
domain_id=domain,
|
|
reasoning_capable=True,
|
|
registry=registry,
|
|
domain_lanes=lanes,
|
|
lane_results=results,
|
|
)
|
|
assert verdict.passed is False
|
|
assert "surface_groundedness" in verdict.reason
|
|
assert "below threshold" in verdict.reason
|
|
|
|
def test_fabrication_control_failure_refuses(self) -> None:
|
|
domain = "mathematics_logic"
|
|
lanes = ("inference_closure", "fabrication_control")
|
|
results = _good_lane_results(lanes)
|
|
results["fabrication_control"]["holdout"]["passed_rate"] = 0.8
|
|
digest = derive_evidence_digest(
|
|
domain_id=domain,
|
|
evidence_revision="rev1",
|
|
evidence_lanes=lanes,
|
|
lane_results=results,
|
|
)
|
|
claim = ExpertDemoClaim(
|
|
domain_id=domain,
|
|
evidence_lanes=lanes,
|
|
evidence_revision="rev1",
|
|
signed_by="shay-j",
|
|
claim_digest=digest,
|
|
)
|
|
registry = _build_registry((_primary_reviewer(),), (claim,))
|
|
verdict = evaluate_expert_demo(
|
|
domain_id=domain,
|
|
reasoning_capable=True,
|
|
registry=registry,
|
|
domain_lanes=lanes,
|
|
lane_results=results,
|
|
)
|
|
assert verdict.passed is False
|
|
assert "fabrication_control" in verdict.reason
|
|
|
|
|
|
class TestProductionLedgerUntouched:
|
|
"""ADR-0106 §Acceptance: no domain row's expert_demo field flips."""
|
|
|
|
def test_no_production_domain_promoted_by_this_adr(self) -> None:
|
|
from core.capability.reporting import ledger_report
|
|
|
|
report = ledger_report()
|
|
promoted = [
|
|
row["domain"]
|
|
for row in report.get("domains", [])
|
|
if row.get("predicates", {}).get("expert_demo")
|
|
]
|
|
assert promoted == [], (
|
|
f"ADR-0106 must not promote any domain by code change alone; "
|
|
f"got: {promoted}"
|
|
)
|