696f62abdd
The word "expert" in the previous status name implied raw-capability parity
with frontier LLMs on the same benchmark — which the gate does NOT verify.
What the gate actually verifies is CORE *claim-shape compliance*:
* signed digest (replay-reproducible from on-disk lane results)
* replay determinism (same inputs → byte-equal trace_hash)
* typed refusal (fabrication refused, not paraphrased)
* exact recall (no ANN, no cosine, no attention bottleneck)
* grounding-source provenance
These are claim shapes a transformer LLM cannot structurally produce
regardless of raw accuracy. A frontier LLM might score higher on the
same benchmark but cannot pass this contract.
Rename scope (semantics only, per ADR-0113):
status string "expert-demo" → "audit-passed"
predicate key predicates.expert_demo → predicates.audit_passed
reason key expert_demo_reason → audit_passed_reason
YAML key expert_demo_claims → audit_passed_claims
CLI command core demo expert → core demo audit-passed
output dir evals/expert_demos/ → evals/audit_passed/
artifact filenames expert_demo.{json,html} → audit_passed.{json,html}
HTML title CORE Expert-Demo: X → CORE Audit-Passed: X
Internal Python identifiers (module/file/function/class names like
`expert_demo.py`, `evaluate_expert_demo`, `ExpertDemoClaim`,
`expert_demo_claim_for`) are deliberately kept to minimize churn. ADR
file titles (ADR-0106..0112) preserved as historical record.
`expert` namespace reserved for ADR-0114+: an actual capability tier
above `audit-passed` backed by a public benchmark with a stated
threshold. ADR-0114 proposes the first such target — GSM8K-math —
laying out a falsifiable 7-phase arc (parser → solver → verifier →
stepped-realizer → eval lane → first `expert` ledger tier promotion).
Tests: 184 directly-affected tests green (140 capability/expert-demo
suite + 34 demo/audit-tour + 10 correction-cue). Smoke suite 67/67.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
350 lines
13 KiB
Python
350 lines
13 KiB
Python
"""Unit tests for the Reviewer Registry v1 loader (ADR-0092)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from core.capability.reviewers import (
|
|
ExpertDemoClaim,
|
|
ReviewerRegistry,
|
|
ReviewerRegistryError,
|
|
SCHEMA_VERSION,
|
|
load_reviewer_registry,
|
|
)
|
|
|
|
|
|
_VALID_DIGEST = "a" * 64
|
|
|
|
|
|
def _write_yaml(tmp_path: Path, content: str) -> Path:
|
|
path = tmp_path / "reviewers.yaml"
|
|
path.write_text(content, encoding="utf-8")
|
|
return path
|
|
|
|
|
|
def _valid_primary_yaml() -> str:
|
|
return """\
|
|
schema_version: 1
|
|
reviewers:
|
|
- reviewer_id: shay-j
|
|
display_name: "Joshua Shay"
|
|
role: primary
|
|
domains: ["*"]
|
|
review_scope: ["pack", "proposal", "chain", "eval"]
|
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
|
"""
|
|
|
|
|
|
def _valid_domain_yaml() -> str:
|
|
return """\
|
|
schema_version: 1
|
|
reviewers:
|
|
- reviewer_id: math-reviewer
|
|
display_name: "Math Domain Reviewer"
|
|
role: domain
|
|
domains: ["mathematics_logic"]
|
|
review_scope: ["pack", "chain"]
|
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
|
"""
|
|
|
|
|
|
class TestPositiveLoad:
|
|
def test_loads_primary_reviewer(self, tmp_path: Path) -> None:
|
|
path = _write_yaml(tmp_path, _valid_primary_yaml())
|
|
registry = load_reviewer_registry(path)
|
|
assert isinstance(registry, ReviewerRegistry)
|
|
assert registry.schema_version == SCHEMA_VERSION
|
|
assert len(registry.reviewers) == 1
|
|
reviewer = registry.reviewers[0]
|
|
assert reviewer.reviewer_id == "shay-j"
|
|
assert reviewer.role == "primary"
|
|
assert reviewer.domains == ("*",)
|
|
assert reviewer.review_scope == ("pack", "proposal", "chain", "eval")
|
|
|
|
def test_loads_domain_reviewer(self, tmp_path: Path) -> None:
|
|
path = _write_yaml(tmp_path, _valid_domain_yaml())
|
|
registry = load_reviewer_registry(path)
|
|
assert registry.reviewers[0].role == "domain"
|
|
assert registry.reviewers[0].domains == ("mathematics_logic",)
|
|
|
|
def test_registry_is_frozen(self, tmp_path: Path) -> None:
|
|
path = _write_yaml(tmp_path, _valid_primary_yaml())
|
|
registry = load_reviewer_registry(path)
|
|
with pytest.raises((AttributeError, TypeError)):
|
|
registry.reviewers = () # type: ignore[misc]
|
|
|
|
def test_reviewer_is_frozen(self, tmp_path: Path) -> None:
|
|
path = _write_yaml(tmp_path, _valid_primary_yaml())
|
|
registry = load_reviewer_registry(path)
|
|
with pytest.raises((AttributeError, TypeError)):
|
|
registry.reviewers[0].role = "domain" # type: ignore[misc]
|
|
|
|
|
|
class TestResolution:
|
|
def test_resolve_known_reviewer(self, tmp_path: Path) -> None:
|
|
registry = load_reviewer_registry(_write_yaml(tmp_path, _valid_primary_yaml()))
|
|
reviewer = registry.resolve("shay-j")
|
|
assert reviewer is not None
|
|
assert reviewer.reviewer_id == "shay-j"
|
|
|
|
def test_resolve_unknown_reviewer(self, tmp_path: Path) -> None:
|
|
registry = load_reviewer_registry(_write_yaml(tmp_path, _valid_primary_yaml()))
|
|
assert registry.resolve("ghost") is None
|
|
|
|
def test_can_review_primary_covers_any_domain(self, tmp_path: Path) -> None:
|
|
registry = load_reviewer_registry(_write_yaml(tmp_path, _valid_primary_yaml()))
|
|
assert registry.can_review("shay-j", domain_id="mathematics_logic", scope="pack")
|
|
assert registry.can_review("shay-j", domain_id="physics", scope="proposal")
|
|
|
|
def test_can_review_domain_only_covers_declared(self, tmp_path: Path) -> None:
|
|
registry = load_reviewer_registry(_write_yaml(tmp_path, _valid_domain_yaml()))
|
|
assert registry.can_review("math-reviewer", domain_id="mathematics_logic", scope="pack")
|
|
assert not registry.can_review("math-reviewer", domain_id="physics", scope="pack")
|
|
|
|
def test_can_review_respects_scope(self, tmp_path: Path) -> None:
|
|
registry = load_reviewer_registry(_write_yaml(tmp_path, _valid_domain_yaml()))
|
|
assert registry.can_review("math-reviewer", domain_id="mathematics_logic", scope="chain")
|
|
assert not registry.can_review(
|
|
"math-reviewer", domain_id="mathematics_logic", scope="proposal"
|
|
)
|
|
|
|
def test_can_review_unknown_reviewer_returns_false(self, tmp_path: Path) -> None:
|
|
registry = load_reviewer_registry(_write_yaml(tmp_path, _valid_primary_yaml()))
|
|
assert not registry.can_review("ghost", domain_id="mathematics_logic", scope="pack")
|
|
|
|
|
|
class TestNegativeSchema:
|
|
def test_missing_file(self, tmp_path: Path) -> None:
|
|
with pytest.raises(ReviewerRegistryError, match="not found"):
|
|
load_reviewer_registry(tmp_path / "absent.yaml")
|
|
|
|
def test_wrong_schema_version(self, tmp_path: Path) -> None:
|
|
path = _write_yaml(tmp_path, "schema_version: 2\nreviewers: []\n")
|
|
with pytest.raises(ReviewerRegistryError, match="schema_version"):
|
|
load_reviewer_registry(path)
|
|
|
|
def test_missing_schema_version(self, tmp_path: Path) -> None:
|
|
path = _write_yaml(tmp_path, "reviewers: []\n")
|
|
with pytest.raises(ReviewerRegistryError, match="schema_version"):
|
|
load_reviewer_registry(path)
|
|
|
|
def test_unknown_top_level_key(self, tmp_path: Path) -> None:
|
|
path = _write_yaml(
|
|
tmp_path,
|
|
"schema_version: 1\nreviewers: []\nextra_field: nope\n",
|
|
)
|
|
with pytest.raises(ReviewerRegistryError, match="unknown top-level"):
|
|
load_reviewer_registry(path)
|
|
|
|
def test_reviewers_must_be_list(self, tmp_path: Path) -> None:
|
|
path = _write_yaml(tmp_path, "schema_version: 1\nreviewers: {}\n")
|
|
with pytest.raises(ReviewerRegistryError, match="must be a list"):
|
|
load_reviewer_registry(path)
|
|
|
|
def test_unknown_reviewer_field(self, tmp_path: Path) -> None:
|
|
yaml = """\
|
|
schema_version: 1
|
|
reviewers:
|
|
- reviewer_id: shay-j
|
|
display_name: "Joshua Shay"
|
|
role: primary
|
|
domains: ["*"]
|
|
review_scope: ["pack"]
|
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
|
secret_powers: ["unlimited"]
|
|
"""
|
|
with pytest.raises(ReviewerRegistryError, match="unknown fields"):
|
|
load_reviewer_registry(_write_yaml(tmp_path, yaml))
|
|
|
|
def test_missing_required_field(self, tmp_path: Path) -> None:
|
|
yaml = """\
|
|
schema_version: 1
|
|
reviewers:
|
|
- reviewer_id: shay-j
|
|
display_name: "Joshua Shay"
|
|
role: primary
|
|
domains: ["*"]
|
|
review_scope: ["pack"]
|
|
"""
|
|
with pytest.raises(ReviewerRegistryError, match="missing required fields"):
|
|
load_reviewer_registry(_write_yaml(tmp_path, yaml))
|
|
|
|
def test_invalid_role(self, tmp_path: Path) -> None:
|
|
yaml = """\
|
|
schema_version: 1
|
|
reviewers:
|
|
- reviewer_id: shay-j
|
|
display_name: "Joshua Shay"
|
|
role: emperor
|
|
domains: ["*"]
|
|
review_scope: ["pack"]
|
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
|
"""
|
|
with pytest.raises(ReviewerRegistryError, match="invalid role"):
|
|
load_reviewer_registry(_write_yaml(tmp_path, yaml))
|
|
|
|
def test_domain_reviewer_cannot_claim_wildcard(self, tmp_path: Path) -> None:
|
|
yaml = """\
|
|
schema_version: 1
|
|
reviewers:
|
|
- reviewer_id: math-reviewer
|
|
display_name: "Math Reviewer"
|
|
role: domain
|
|
domains: ["*"]
|
|
review_scope: ["pack"]
|
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
|
"""
|
|
with pytest.raises(ReviewerRegistryError, match="wildcard"):
|
|
load_reviewer_registry(_write_yaml(tmp_path, yaml))
|
|
|
|
def test_primary_reviewer_must_claim_wildcard(self, tmp_path: Path) -> None:
|
|
yaml = """\
|
|
schema_version: 1
|
|
reviewers:
|
|
- reviewer_id: shay-j
|
|
display_name: "Joshua Shay"
|
|
role: primary
|
|
domains: ["mathematics_logic"]
|
|
review_scope: ["pack"]
|
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
|
"""
|
|
with pytest.raises(ReviewerRegistryError, match="wildcard"):
|
|
load_reviewer_registry(_write_yaml(tmp_path, yaml))
|
|
|
|
def test_unknown_review_scope_value(self, tmp_path: Path) -> None:
|
|
yaml = """\
|
|
schema_version: 1
|
|
reviewers:
|
|
- reviewer_id: shay-j
|
|
display_name: "Joshua Shay"
|
|
role: primary
|
|
domains: ["*"]
|
|
review_scope: ["pack", "deploy"]
|
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
|
"""
|
|
with pytest.raises(ReviewerRegistryError, match="unknown review_scope"):
|
|
load_reviewer_registry(_write_yaml(tmp_path, yaml))
|
|
|
|
def test_empty_review_scope(self, tmp_path: Path) -> None:
|
|
yaml = """\
|
|
schema_version: 1
|
|
reviewers:
|
|
- reviewer_id: shay-j
|
|
display_name: "Joshua Shay"
|
|
role: primary
|
|
domains: ["*"]
|
|
review_scope: []
|
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
|
"""
|
|
with pytest.raises(ReviewerRegistryError, match="empty 'review_scope'"):
|
|
load_reviewer_registry(_write_yaml(tmp_path, yaml))
|
|
|
|
def test_duplicate_reviewer_id(self, tmp_path: Path) -> None:
|
|
yaml = """\
|
|
schema_version: 1
|
|
reviewers:
|
|
- reviewer_id: shay-j
|
|
display_name: "Joshua Shay"
|
|
role: primary
|
|
domains: ["*"]
|
|
review_scope: ["pack"]
|
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
|
- reviewer_id: shay-j
|
|
display_name: "Imposter"
|
|
role: primary
|
|
domains: ["*"]
|
|
review_scope: ["pack"]
|
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
|
"""
|
|
with pytest.raises(ReviewerRegistryError, match="duplicate reviewer_id"):
|
|
load_reviewer_registry(_write_yaml(tmp_path, yaml))
|
|
|
|
def test_malformed_yaml(self, tmp_path: Path) -> None:
|
|
path = _write_yaml(tmp_path, "schema_version: 1\nreviewers: [\n")
|
|
with pytest.raises(ReviewerRegistryError, match="YAML parse error"):
|
|
load_reviewer_registry(path)
|
|
|
|
|
|
class TestProductionRegistry:
|
|
"""Verify the in-tree docs/reviewers.yaml passes ADR-0092 v1 schema."""
|
|
|
|
def test_production_registry_loads(self) -> None:
|
|
repo_root = Path(__file__).resolve().parent.parent
|
|
registry = load_reviewer_registry(repo_root / "docs" / "reviewers.yaml")
|
|
assert registry.schema_version == SCHEMA_VERSION
|
|
assert len(registry.reviewers) >= 1
|
|
ids = [r.reviewer_id for r in registry.reviewers]
|
|
assert "shay-j" in ids
|
|
|
|
def test_production_bootstrap_reviewer_is_primary(self) -> None:
|
|
repo_root = Path(__file__).resolve().parent.parent
|
|
registry = load_reviewer_registry(repo_root / "docs" / "reviewers.yaml")
|
|
shay = registry.resolve("shay-j")
|
|
assert shay is not None
|
|
assert shay.role == "primary"
|
|
assert shay.domains == ("*",)
|
|
assert shay.provenance.startswith("adr-0092:bootstrap:")
|
|
|
|
|
|
class TestExpertDemoClaimsSchema:
|
|
"""ADR-0106 — additive ``expert_demo_claims`` block on the registry."""
|
|
|
|
def _yaml_with_claim(self, *, signed_by: str, digest: str = _VALID_DIGEST) -> str:
|
|
return f"""\
|
|
schema_version: 1
|
|
reviewers:
|
|
- reviewer_id: shay-j
|
|
display_name: "Joshua Shay"
|
|
role: primary
|
|
domains: ["*"]
|
|
review_scope: ["pack", "proposal", "chain", "eval"]
|
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
|
audit_passed_claims:
|
|
- domain_id: mathematics_logic
|
|
evidence_lanes: ["inference_closure", "fabrication_control"]
|
|
evidence_revision: "abc1234"
|
|
signed_by: "{signed_by}"
|
|
claim_digest: "{digest}"
|
|
"""
|
|
|
|
def test_omitted_block_yields_empty_tuple(self, tmp_path: Path) -> None:
|
|
path = _write_yaml(tmp_path, _valid_primary_yaml())
|
|
registry = load_reviewer_registry(path)
|
|
assert registry.expert_demo_claims == ()
|
|
assert registry.expert_demo_claim_for("mathematics_logic") is None
|
|
|
|
def test_valid_claim_parses(self, tmp_path: Path) -> None:
|
|
path = _write_yaml(tmp_path, self._yaml_with_claim(signed_by="shay-j"))
|
|
registry = load_reviewer_registry(path)
|
|
assert len(registry.expert_demo_claims) == 1
|
|
claim = registry.expert_demo_claim_for("mathematics_logic")
|
|
assert isinstance(claim, ExpertDemoClaim)
|
|
assert claim.signed_by == "shay-j"
|
|
assert claim.claim_digest == _VALID_DIGEST
|
|
assert claim.evidence_lanes == ("inference_closure", "fabrication_control")
|
|
|
|
def test_signer_unknown_reviewer_id_rejected(self, tmp_path: Path) -> None:
|
|
path = _write_yaml(tmp_path, self._yaml_with_claim(signed_by="ghost"))
|
|
with pytest.raises(ReviewerRegistryError, match="does not resolve"):
|
|
load_reviewer_registry(path)
|
|
|
|
def test_malformed_digest_rejected(self, tmp_path: Path) -> None:
|
|
path = _write_yaml(
|
|
tmp_path, self._yaml_with_claim(signed_by="shay-j", digest="short")
|
|
)
|
|
with pytest.raises(ReviewerRegistryError, match="claim_digest"):
|
|
load_reviewer_registry(path)
|
|
|
|
def test_duplicate_domain_rejected(self, tmp_path: Path) -> None:
|
|
yaml = self._yaml_with_claim(signed_by="shay-j")
|
|
yaml += """ - domain_id: mathematics_logic
|
|
evidence_lanes: ["inference_closure"]
|
|
evidence_revision: "abc1234"
|
|
signed_by: "shay-j"
|
|
claim_digest: "{digest}"
|
|
""".format(digest=_VALID_DIGEST)
|
|
path = _write_yaml(tmp_path, yaml)
|
|
with pytest.raises(ReviewerRegistryError, match="duplicate domain_id"):
|
|
load_reviewer_registry(path)
|