feat(capability): implement ADR-0092 — Reviewer Registry v1
Closes the load-bearing gap blocking every reasoning-capable claim
under ADR-0091: docs/reviewers.yaml was previously `reviewers: []` and
unparsed. Now schema-validated at v1, with a bootstrap shay-j entry
self-sealed via provenance.
- new core.capability.reviewers module: frozen Reviewer/ReviewerRegistry
dataclasses, strict load_reviewer_registry parser, ReviewerRegistryError
- enforces ADR-0092 schema rules: schema_version==1, no unknown
top-level keys, no unknown reviewer fields, role∈{primary,domain},
primary must claim ["*"], domain must NOT claim "*", review_scope
subset of {pack,proposal,chain,eval}, no duplicate reviewer_ids
- can_review(reviewer_id, domain_id, scope) helper implements
ADR-0092 rules 2-4 for downstream use by ADR-0093 validator
- docs/reviewers.yaml updated to v1 schema with shay-j bootstrap
- ledger_report() evidence_counts now exposes structured
reviewer_registry status (valid, schema_version, reviewer_count,
reviewer_ids, error) alongside the legacy reviewers_present bool
- new evals/reviewer_registry/ lane: 6 cases (2 positive + 4 negative)
covering empty-registry, wrong-version, domain-wildcard rejection,
and unknown-field rejection
- runner emits deterministic JSON report; two runs produce byte-identical
output (sha256 verified)
- 26 unit tests in tests/test_reviewer_registry.py
- capability ledger test extended to assert new reviewer_registry block
- smoke suite green (67/67); lane passes 6/6
The pre-existing test_flag_report_tracks_default_off_flags failure is
unrelated (discourse_planner flag default) and not introduced here.
This commit is contained in:
parent
10457bec69
commit
afdd2ee413
15 changed files with 935 additions and 1 deletions
|
|
@ -8,12 +8,32 @@ from .reporting import (
|
||||||
flag_report,
|
flag_report,
|
||||||
ledger_report,
|
ledger_report,
|
||||||
)
|
)
|
||||||
|
from .reviewers import (
|
||||||
|
ALLOWED_REVIEWER_KEYS,
|
||||||
|
ALLOWED_ROLES,
|
||||||
|
ALLOWED_SCOPES,
|
||||||
|
ALLOWED_TOP_LEVEL_KEYS,
|
||||||
|
Reviewer,
|
||||||
|
ReviewerRegistry,
|
||||||
|
ReviewerRegistryError,
|
||||||
|
SCHEMA_VERSION as REVIEWER_REGISTRY_SCHEMA_VERSION,
|
||||||
|
load_reviewer_registry,
|
||||||
|
)
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
|
"ALLOWED_REVIEWER_KEYS",
|
||||||
|
"ALLOWED_ROLES",
|
||||||
|
"ALLOWED_SCOPES",
|
||||||
|
"ALLOWED_TOP_LEVEL_KEYS",
|
||||||
"CapabilityArtifactQuery",
|
"CapabilityArtifactQuery",
|
||||||
|
"REVIEWER_REGISTRY_SCHEMA_VERSION",
|
||||||
|
"Reviewer",
|
||||||
|
"ReviewerRegistry",
|
||||||
|
"ReviewerRegistryError",
|
||||||
"artifact_report",
|
"artifact_report",
|
||||||
"chain_report",
|
"chain_report",
|
||||||
"evidence_plan_report",
|
"evidence_plan_report",
|
||||||
"flag_report",
|
"flag_report",
|
||||||
"ledger_report",
|
"ledger_report",
|
||||||
|
"load_reviewer_registry",
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,10 @@ from core.capability.domains import (
|
||||||
DOMAIN_OPERATOR_CLAIMS,
|
DOMAIN_OPERATOR_CLAIMS,
|
||||||
DOMAIN_PACKS,
|
DOMAIN_PACKS,
|
||||||
)
|
)
|
||||||
|
from core.capability.reviewers import (
|
||||||
|
ReviewerRegistryError,
|
||||||
|
load_reviewer_registry,
|
||||||
|
)
|
||||||
from core.capability.sources import LEDGER_SOURCES
|
from core.capability.sources import LEDGER_SOURCES
|
||||||
from core.config import DEFAULT_CONFIG
|
from core.config import DEFAULT_CONFIG
|
||||||
from language_packs.domain_contract import validate_domain_contract_pack
|
from language_packs.domain_contract import validate_domain_contract_pack
|
||||||
|
|
@ -498,6 +502,7 @@ def ledger_report() -> dict[str, Any]:
|
||||||
"audit_tour": len(audit_files),
|
"audit_tour": len(audit_files),
|
||||||
"pack_measurements_present": (_REPO_ROOT / LEDGER_SOURCES.pack_measurements).exists(),
|
"pack_measurements_present": (_REPO_ROOT / LEDGER_SOURCES.pack_measurements).exists(),
|
||||||
"reviewers_present": (_REPO_ROOT / LEDGER_SOURCES.reviewers).exists(),
|
"reviewers_present": (_REPO_ROOT / LEDGER_SOURCES.reviewers).exists(),
|
||||||
|
"reviewer_registry": _reviewer_registry_status(),
|
||||||
"gaps_present": (_REPO_ROOT / LEDGER_SOURCES.gaps).exists(),
|
"gaps_present": (_REPO_ROOT / LEDGER_SOURCES.gaps).exists(),
|
||||||
},
|
},
|
||||||
"domains": domain_rows,
|
"domains": domain_rows,
|
||||||
|
|
@ -583,3 +588,40 @@ def evidence_plan_report() -> dict[str, Any]:
|
||||||
"workers_promote_packs": False,
|
"workers_promote_packs": False,
|
||||||
"jobs": jobs,
|
"jobs": jobs,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _reviewer_registry_status() -> dict[str, Any]:
|
||||||
|
"""Report Reviewer Registry v1 validity for the capability ledger (ADR-0092).
|
||||||
|
|
||||||
|
Returns a structured object containing the parsed schema version,
|
||||||
|
reviewer count, and a list of registered reviewer ids. If the
|
||||||
|
registry is missing or fails ADR-0092 schema validation, the object
|
||||||
|
reports ``valid: False`` and names the error rather than raising,
|
||||||
|
so the ledger surface remains generatable on a broken registry.
|
||||||
|
"""
|
||||||
|
path = _REPO_ROOT / LEDGER_SOURCES.reviewers
|
||||||
|
if not path.exists():
|
||||||
|
return {
|
||||||
|
"valid": False,
|
||||||
|
"error": "reviewers.yaml not found",
|
||||||
|
"schema_version": None,
|
||||||
|
"reviewer_count": 0,
|
||||||
|
"reviewer_ids": [],
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
registry = load_reviewer_registry(path)
|
||||||
|
except ReviewerRegistryError as exc:
|
||||||
|
return {
|
||||||
|
"valid": False,
|
||||||
|
"error": str(exc),
|
||||||
|
"schema_version": None,
|
||||||
|
"reviewer_count": 0,
|
||||||
|
"reviewer_ids": [],
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"valid": True,
|
||||||
|
"error": None,
|
||||||
|
"schema_version": registry.schema_version,
|
||||||
|
"reviewer_count": len(registry.reviewers),
|
||||||
|
"reviewer_ids": sorted(r.reviewer_id for r in registry.reviewers),
|
||||||
|
}
|
||||||
|
|
|
||||||
207
core/capability/reviewers.py
Normal file
207
core/capability/reviewers.py
Normal file
|
|
@ -0,0 +1,207 @@
|
||||||
|
"""Reviewer Registry v1 loader and validator (ADR-0092).
|
||||||
|
|
||||||
|
Parses ``docs/reviewers.yaml`` into a typed, immutable :class:`ReviewerRegistry`.
|
||||||
|
Schema rejection is loud: unknown keys, malformed entries, duplicate ids, and
|
||||||
|
out-of-scope wildcards all raise :class:`ReviewerRegistryError`.
|
||||||
|
|
||||||
|
The registry is consulted by the Domain Pack Contract v1 validator
|
||||||
|
(ADR-0093) to enforce ADR-0091 predicate #8 (reviewer resolution).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
|
||||||
|
ALLOWED_TOP_LEVEL_KEYS: frozenset[str] = frozenset({"schema_version", "reviewers"})
|
||||||
|
ALLOWED_REVIEWER_KEYS: frozenset[str] = frozenset(
|
||||||
|
{"reviewer_id", "display_name", "role", "domains", "review_scope", "provenance"}
|
||||||
|
)
|
||||||
|
ALLOWED_ROLES: frozenset[str] = frozenset({"primary", "domain"})
|
||||||
|
ALLOWED_SCOPES: frozenset[str] = frozenset({"pack", "proposal", "chain", "eval"})
|
||||||
|
SCHEMA_VERSION: int = 1
|
||||||
|
WILDCARD_DOMAIN: str = "*"
|
||||||
|
|
||||||
|
|
||||||
|
class ReviewerRegistryError(ValueError):
|
||||||
|
"""Raised when ``docs/reviewers.yaml`` fails ADR-0092 v1 schema validation."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class Reviewer:
|
||||||
|
reviewer_id: str
|
||||||
|
display_name: str
|
||||||
|
role: str
|
||||||
|
domains: tuple[str, ...]
|
||||||
|
review_scope: tuple[str, ...]
|
||||||
|
provenance: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ReviewerRegistry:
|
||||||
|
schema_version: int
|
||||||
|
reviewers: tuple[Reviewer, ...]
|
||||||
|
|
||||||
|
def resolve(self, reviewer_id: str) -> Reviewer | None:
|
||||||
|
for reviewer in self.reviewers:
|
||||||
|
if reviewer.reviewer_id == reviewer_id:
|
||||||
|
return reviewer
|
||||||
|
return None
|
||||||
|
|
||||||
|
def can_review(self, reviewer_id: str, *, domain_id: str, scope: str) -> bool:
|
||||||
|
"""Return True iff this reviewer may ratify ``scope`` artifacts for ``domain_id``.
|
||||||
|
|
||||||
|
Implements ADR-0091 predicate #8 plus ADR-0092 rules 2, 3, and 4:
|
||||||
|
- rule 2: unknown ``reviewer_id`` → False
|
||||||
|
- rule 3: ``role: domain`` reviewer whose ``domains`` do not include
|
||||||
|
``domain_id`` → False
|
||||||
|
- rule 4: reviewer whose ``review_scope`` does not cover ``scope`` → False
|
||||||
|
"""
|
||||||
|
reviewer = self.resolve(reviewer_id)
|
||||||
|
if reviewer is None:
|
||||||
|
return False
|
||||||
|
if scope not in reviewer.review_scope:
|
||||||
|
return False
|
||||||
|
if reviewer.role == "primary":
|
||||||
|
return True
|
||||||
|
return domain_id in reviewer.domains
|
||||||
|
|
||||||
|
|
||||||
|
def load_reviewer_registry(path: Path) -> ReviewerRegistry:
|
||||||
|
"""Parse ``path`` as a Reviewer Registry v1 file.
|
||||||
|
|
||||||
|
Raises :class:`ReviewerRegistryError` on any schema violation. No
|
||||||
|
side effects beyond reading ``path``.
|
||||||
|
"""
|
||||||
|
if not path.exists():
|
||||||
|
raise ReviewerRegistryError(f"reviewer registry not found at {path}")
|
||||||
|
|
||||||
|
try:
|
||||||
|
raw = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||||
|
except yaml.YAMLError as exc:
|
||||||
|
raise ReviewerRegistryError(f"reviewer registry YAML parse error: {exc}") from exc
|
||||||
|
|
||||||
|
if not isinstance(raw, Mapping):
|
||||||
|
raise ReviewerRegistryError("reviewer registry root must be a mapping")
|
||||||
|
|
||||||
|
unknown_top = set(raw.keys()) - ALLOWED_TOP_LEVEL_KEYS
|
||||||
|
if unknown_top:
|
||||||
|
raise ReviewerRegistryError(
|
||||||
|
f"reviewer registry has unknown top-level keys: {sorted(unknown_top)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
schema_version = raw.get("schema_version")
|
||||||
|
if schema_version != SCHEMA_VERSION:
|
||||||
|
raise ReviewerRegistryError(
|
||||||
|
f"reviewer registry schema_version must be {SCHEMA_VERSION}; got {schema_version!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
reviewers_raw = raw.get("reviewers")
|
||||||
|
if reviewers_raw is None:
|
||||||
|
raise ReviewerRegistryError("reviewer registry missing 'reviewers' field")
|
||||||
|
if not isinstance(reviewers_raw, list):
|
||||||
|
raise ReviewerRegistryError("reviewer registry 'reviewers' must be a list")
|
||||||
|
|
||||||
|
parsed: list[Reviewer] = []
|
||||||
|
seen_ids: set[str] = set()
|
||||||
|
for index, entry in enumerate(reviewers_raw):
|
||||||
|
reviewer = _parse_reviewer(entry, index=index)
|
||||||
|
if reviewer.reviewer_id in seen_ids:
|
||||||
|
raise ReviewerRegistryError(
|
||||||
|
f"reviewer registry contains duplicate reviewer_id {reviewer.reviewer_id!r}"
|
||||||
|
)
|
||||||
|
seen_ids.add(reviewer.reviewer_id)
|
||||||
|
parsed.append(reviewer)
|
||||||
|
|
||||||
|
return ReviewerRegistry(schema_version=SCHEMA_VERSION, reviewers=tuple(parsed))
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_reviewer(entry: Any, *, index: int) -> Reviewer:
|
||||||
|
if not isinstance(entry, Mapping):
|
||||||
|
raise ReviewerRegistryError(f"reviewer entry at index {index} must be a mapping")
|
||||||
|
|
||||||
|
unknown = set(entry.keys()) - ALLOWED_REVIEWER_KEYS
|
||||||
|
if unknown:
|
||||||
|
raise ReviewerRegistryError(
|
||||||
|
f"reviewer entry at index {index} has unknown fields: {sorted(unknown)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
missing = ALLOWED_REVIEWER_KEYS - set(entry.keys())
|
||||||
|
if missing:
|
||||||
|
raise ReviewerRegistryError(
|
||||||
|
f"reviewer entry at index {index} missing required fields: {sorted(missing)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
reviewer_id = _require_nonempty_str(entry["reviewer_id"], field="reviewer_id", index=index)
|
||||||
|
display_name = _require_nonempty_str(entry["display_name"], field="display_name", index=index)
|
||||||
|
role = _require_nonempty_str(entry["role"], field="role", index=index)
|
||||||
|
provenance = _require_nonempty_str(entry["provenance"], field="provenance", index=index)
|
||||||
|
|
||||||
|
if role not in ALLOWED_ROLES:
|
||||||
|
raise ReviewerRegistryError(
|
||||||
|
f"reviewer entry at index {index} has invalid role {role!r}; "
|
||||||
|
f"must be one of {sorted(ALLOWED_ROLES)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
domains = _require_str_list(entry["domains"], field="domains", index=index)
|
||||||
|
if not domains:
|
||||||
|
raise ReviewerRegistryError(
|
||||||
|
f"reviewer entry at index {index} has empty 'domains' list"
|
||||||
|
)
|
||||||
|
if role == "domain" and WILDCARD_DOMAIN in domains:
|
||||||
|
raise ReviewerRegistryError(
|
||||||
|
f"reviewer entry at index {index} has role='domain' but claims wildcard "
|
||||||
|
f"'{WILDCARD_DOMAIN}'; only role='primary' may claim wildcard domains"
|
||||||
|
)
|
||||||
|
if role == "primary" and domains != (WILDCARD_DOMAIN,):
|
||||||
|
raise ReviewerRegistryError(
|
||||||
|
f"reviewer entry at index {index} has role='primary' but domains "
|
||||||
|
f"is not exactly [{WILDCARD_DOMAIN!r}]; primary reviewers must claim wildcard"
|
||||||
|
)
|
||||||
|
|
||||||
|
review_scope = _require_str_list(entry["review_scope"], field="review_scope", index=index)
|
||||||
|
if not review_scope:
|
||||||
|
raise ReviewerRegistryError(
|
||||||
|
f"reviewer entry at index {index} has empty 'review_scope' list"
|
||||||
|
)
|
||||||
|
unknown_scopes = set(review_scope) - ALLOWED_SCOPES
|
||||||
|
if unknown_scopes:
|
||||||
|
raise ReviewerRegistryError(
|
||||||
|
f"reviewer entry at index {index} has unknown review_scope values: "
|
||||||
|
f"{sorted(unknown_scopes)}; allowed: {sorted(ALLOWED_SCOPES)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
return Reviewer(
|
||||||
|
reviewer_id=reviewer_id,
|
||||||
|
display_name=display_name,
|
||||||
|
role=role,
|
||||||
|
domains=domains,
|
||||||
|
review_scope=review_scope,
|
||||||
|
provenance=provenance,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _require_nonempty_str(value: Any, *, field: str, index: int) -> str:
|
||||||
|
if not isinstance(value, str) or not value.strip():
|
||||||
|
raise ReviewerRegistryError(
|
||||||
|
f"reviewer entry at index {index} field {field!r} must be a non-empty string"
|
||||||
|
)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _require_str_list(value: Any, *, field: str, index: int) -> tuple[str, ...]:
|
||||||
|
if not isinstance(value, list):
|
||||||
|
raise ReviewerRegistryError(
|
||||||
|
f"reviewer entry at index {index} field {field!r} must be a list of strings"
|
||||||
|
)
|
||||||
|
for item in value:
|
||||||
|
if not isinstance(item, str) or not item.strip():
|
||||||
|
raise ReviewerRegistryError(
|
||||||
|
f"reviewer entry at index {index} field {field!r} contains a "
|
||||||
|
"non-string or empty value"
|
||||||
|
)
|
||||||
|
return tuple(value)
|
||||||
|
|
@ -1 +1,18 @@
|
||||||
reviewers: []
|
# Reviewer Registry v1 (ADR-0092).
|
||||||
|
#
|
||||||
|
# This file is parsed by core.capability.reviewers.load_reviewer_registry
|
||||||
|
# and consulted by the Domain Pack Contract v1 validator (ADR-0093) to
|
||||||
|
# enforce ADR-0091 predicate #8 (reviewer resolution).
|
||||||
|
#
|
||||||
|
# Adding reviewers is a reviewed proposal flow. Do not edit by hand
|
||||||
|
# outside that flow. The bootstrap entry below is self-sealed via its
|
||||||
|
# provenance field.
|
||||||
|
|
||||||
|
schema_version: 1
|
||||||
|
reviewers:
|
||||||
|
- reviewer_id: shay-j
|
||||||
|
display_name: "Joshua Shay"
|
||||||
|
role: primary
|
||||||
|
domains: ["*"]
|
||||||
|
review_scope: ["pack", "proposal", "chain", "eval"]
|
||||||
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
schema_version: 1
|
||||||
|
reviewers:
|
||||||
|
- reviewer_id: math-reviewer
|
||||||
|
display_name: "Mathematics Domain Reviewer"
|
||||||
|
role: domain
|
||||||
|
domains: ["*"]
|
||||||
|
review_scope: ["pack"]
|
||||||
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
||||||
2
evals/reviewer_registry/cases/negative_empty.yaml
Normal file
2
evals/reviewer_registry/cases/negative_empty.yaml
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
schema_version: 1
|
||||||
|
reviewers: []
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
schema_version: 1
|
||||||
|
reviewers:
|
||||||
|
- reviewer_id: shay-j
|
||||||
|
display_name: "Joshua Shay"
|
||||||
|
role: primary
|
||||||
|
domains: ["*"]
|
||||||
|
review_scope: ["pack"]
|
||||||
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
||||||
|
secret_powers: ["unlimited"]
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
schema_version: 2
|
||||||
|
reviewers:
|
||||||
|
- reviewer_id: shay-j
|
||||||
|
display_name: "Joshua Shay"
|
||||||
|
role: primary
|
||||||
|
domains: ["*"]
|
||||||
|
review_scope: ["pack"]
|
||||||
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
||||||
8
evals/reviewer_registry/cases/positive_domain.yaml
Normal file
8
evals/reviewer_registry/cases/positive_domain.yaml
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
schema_version: 1
|
||||||
|
reviewers:
|
||||||
|
- reviewer_id: math-reviewer
|
||||||
|
display_name: "Mathematics Domain Reviewer"
|
||||||
|
role: domain
|
||||||
|
domains: ["mathematics_logic"]
|
||||||
|
review_scope: ["pack", "chain"]
|
||||||
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
||||||
8
evals/reviewer_registry/cases/positive_primary.yaml
Normal file
8
evals/reviewer_registry/cases/positive_primary.yaml
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
schema_version: 1
|
||||||
|
reviewers:
|
||||||
|
- reviewer_id: shay-j
|
||||||
|
display_name: "Joshua Shay"
|
||||||
|
role: primary
|
||||||
|
domains: ["*"]
|
||||||
|
review_scope: ["pack", "proposal", "chain", "eval"]
|
||||||
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
||||||
44
evals/reviewer_registry/contract.md
Normal file
44
evals/reviewer_registry/contract.md
Normal file
|
|
@ -0,0 +1,44 @@
|
||||||
|
# evals/reviewer_registry — Lane Contract
|
||||||
|
|
||||||
|
**ADR:** ADR-0092
|
||||||
|
**Invariant:** `reviewer_registry_schema_v1`
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Prove that ADR-0092's Reviewer Registry v1 schema is enforced. The lane
|
||||||
|
asserts:
|
||||||
|
|
||||||
|
1. A schema-valid registry loads without error and exposes the expected
|
||||||
|
reviewer count and ids.
|
||||||
|
2. Each ADR-0092 schema violation produces a typed
|
||||||
|
:class:`ReviewerRegistryError` whose message names the failing field.
|
||||||
|
3. An empty registry (`reviewers: []` with no entries) blocks all
|
||||||
|
`reasoning-capable` claims under ADR-0091 by producing zero
|
||||||
|
resolvable reviewer ids.
|
||||||
|
4. A registry without `schema_version: 1` is rejected before any
|
||||||
|
reviewer entry is parsed.
|
||||||
|
|
||||||
|
## Cases
|
||||||
|
|
||||||
|
The runner iterates over the case fixtures in this directory:
|
||||||
|
|
||||||
|
- `cases/positive_primary.yaml` — single primary reviewer, must load.
|
||||||
|
- `cases/positive_domain.yaml` — single domain reviewer, must load.
|
||||||
|
- `cases/negative_empty.yaml` — `reviewers: []`, must load with zero
|
||||||
|
reviewers (no error, but no resolvable id either).
|
||||||
|
- `cases/negative_wrong_version.yaml` — `schema_version: 2`, must reject.
|
||||||
|
- `cases/negative_domain_wildcard.yaml` — domain reviewer claims
|
||||||
|
`["*"]`, must reject.
|
||||||
|
- `cases/negative_unknown_field.yaml` — reviewer entry has an
|
||||||
|
unrecognized field, must reject.
|
||||||
|
|
||||||
|
## Determinism
|
||||||
|
|
||||||
|
The runner emits a `results/v1_dev.json` containing per-case outcomes
|
||||||
|
and a SHA-256 of the report bytes. Two consecutive runs against the
|
||||||
|
same fixtures must produce identical bytes.
|
||||||
|
|
||||||
|
## Exit code
|
||||||
|
|
||||||
|
The runner exits non-zero on any case whose actual outcome diverges
|
||||||
|
from the case spec.
|
||||||
83
evals/reviewer_registry/results/v1_dev.json
Normal file
83
evals/reviewer_registry/results/v1_dev.json
Normal file
|
|
@ -0,0 +1,83 @@
|
||||||
|
{
|
||||||
|
"adr": "ADR-0092",
|
||||||
|
"all_passed": true,
|
||||||
|
"cases": [
|
||||||
|
{
|
||||||
|
"actual_error": "reviewer entry at index 0 has role='domain' but claims wildcard '*'; only role='primary' may claim wildcard domains",
|
||||||
|
"actual_outcome": "reject",
|
||||||
|
"case_id": "negative_domain_wildcard.yaml",
|
||||||
|
"divergence": null,
|
||||||
|
"passed": true,
|
||||||
|
"purpose": "domain reviewer claiming '*' rejected",
|
||||||
|
"registry": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"actual_error": null,
|
||||||
|
"actual_outcome": "load_ok",
|
||||||
|
"case_id": "negative_empty.yaml",
|
||||||
|
"divergence": null,
|
||||||
|
"passed": true,
|
||||||
|
"purpose": "empty registry loads but blocks reasoning-capable claims",
|
||||||
|
"registry": {
|
||||||
|
"reviewer_count": 0,
|
||||||
|
"reviewer_ids": [],
|
||||||
|
"schema_version": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"actual_error": "reviewer entry at index 0 has unknown fields: ['secret_powers']",
|
||||||
|
"actual_outcome": "reject",
|
||||||
|
"case_id": "negative_unknown_field.yaml",
|
||||||
|
"divergence": null,
|
||||||
|
"passed": true,
|
||||||
|
"purpose": "unknown reviewer field rejected",
|
||||||
|
"registry": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"actual_error": "reviewer registry schema_version must be 1; got 2",
|
||||||
|
"actual_outcome": "reject",
|
||||||
|
"case_id": "negative_wrong_version.yaml",
|
||||||
|
"divergence": null,
|
||||||
|
"passed": true,
|
||||||
|
"purpose": "non-v1 schema rejected before any reviewer parsed",
|
||||||
|
"registry": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"actual_error": null,
|
||||||
|
"actual_outcome": "load_ok",
|
||||||
|
"case_id": "positive_domain.yaml",
|
||||||
|
"divergence": null,
|
||||||
|
"passed": true,
|
||||||
|
"purpose": "single domain reviewer loads cleanly",
|
||||||
|
"registry": {
|
||||||
|
"reviewer_count": 1,
|
||||||
|
"reviewer_ids": [
|
||||||
|
"math-reviewer"
|
||||||
|
],
|
||||||
|
"schema_version": 1
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"actual_error": null,
|
||||||
|
"actual_outcome": "load_ok",
|
||||||
|
"case_id": "positive_primary.yaml",
|
||||||
|
"divergence": null,
|
||||||
|
"passed": true,
|
||||||
|
"purpose": "single primary reviewer loads cleanly",
|
||||||
|
"registry": {
|
||||||
|
"reviewer_count": 1,
|
||||||
|
"reviewer_ids": [
|
||||||
|
"shay-j"
|
||||||
|
],
|
||||||
|
"schema_version": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"failed_cases": 0,
|
||||||
|
"invariant": "reviewer_registry_schema_v1",
|
||||||
|
"lane": "reviewer_registry",
|
||||||
|
"lane_version": "v1",
|
||||||
|
"passed_cases": 6,
|
||||||
|
"split": "dev",
|
||||||
|
"total_cases": 6
|
||||||
|
}
|
||||||
187
evals/reviewer_registry/runner.py
Normal file
187
evals/reviewer_registry/runner.py
Normal file
|
|
@ -0,0 +1,187 @@
|
||||||
|
"""Runner for evals/reviewer_registry/ (ADR-0092).
|
||||||
|
|
||||||
|
Iterates over the case fixtures under ``cases/`` and asserts each one's
|
||||||
|
load outcome matches its declared expectation. Emits a deterministic
|
||||||
|
JSON report to ``results/v1_dev.json``.
|
||||||
|
|
||||||
|
Exit code is non-zero on any divergence.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from core.capability.reviewers import (
|
||||||
|
ReviewerRegistryError,
|
||||||
|
load_reviewer_registry,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
CASE_EXPECTATIONS: dict[str, dict[str, Any]] = {
|
||||||
|
"positive_primary.yaml": {
|
||||||
|
"expected_outcome": "load_ok",
|
||||||
|
"expected_reviewer_count": 1,
|
||||||
|
"expected_reviewer_ids": ["shay-j"],
|
||||||
|
"purpose": "single primary reviewer loads cleanly",
|
||||||
|
},
|
||||||
|
"positive_domain.yaml": {
|
||||||
|
"expected_outcome": "load_ok",
|
||||||
|
"expected_reviewer_count": 1,
|
||||||
|
"expected_reviewer_ids": ["math-reviewer"],
|
||||||
|
"purpose": "single domain reviewer loads cleanly",
|
||||||
|
},
|
||||||
|
"negative_empty.yaml": {
|
||||||
|
"expected_outcome": "load_ok",
|
||||||
|
"expected_reviewer_count": 0,
|
||||||
|
"expected_reviewer_ids": [],
|
||||||
|
"purpose": "empty registry loads but blocks reasoning-capable claims",
|
||||||
|
},
|
||||||
|
"negative_wrong_version.yaml": {
|
||||||
|
"expected_outcome": "reject",
|
||||||
|
"expected_error_substring": "schema_version",
|
||||||
|
"purpose": "non-v1 schema rejected before any reviewer parsed",
|
||||||
|
},
|
||||||
|
"negative_domain_wildcard.yaml": {
|
||||||
|
"expected_outcome": "reject",
|
||||||
|
"expected_error_substring": "wildcard",
|
||||||
|
"purpose": "domain reviewer claiming '*' rejected",
|
||||||
|
},
|
||||||
|
"negative_unknown_field.yaml": {
|
||||||
|
"expected_outcome": "reject",
|
||||||
|
"expected_error_substring": "unknown fields",
|
||||||
|
"purpose": "unknown reviewer field rejected",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _evaluate_case(path: Path, expectation: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
actual: dict[str, Any] = {"case_id": path.name, "purpose": expectation["purpose"]}
|
||||||
|
|
||||||
|
try:
|
||||||
|
registry = load_reviewer_registry(path)
|
||||||
|
except ReviewerRegistryError as exc:
|
||||||
|
actual["actual_outcome"] = "reject"
|
||||||
|
actual["actual_error"] = str(exc)
|
||||||
|
actual["registry"] = None
|
||||||
|
else:
|
||||||
|
actual["actual_outcome"] = "load_ok"
|
||||||
|
actual["actual_error"] = None
|
||||||
|
actual["registry"] = {
|
||||||
|
"schema_version": registry.schema_version,
|
||||||
|
"reviewer_count": len(registry.reviewers),
|
||||||
|
"reviewer_ids": sorted(r.reviewer_id for r in registry.reviewers),
|
||||||
|
}
|
||||||
|
|
||||||
|
if actual["actual_outcome"] != expectation["expected_outcome"]:
|
||||||
|
actual["passed"] = False
|
||||||
|
actual["divergence"] = (
|
||||||
|
f"expected {expectation['expected_outcome']!r} got "
|
||||||
|
f"{actual['actual_outcome']!r}"
|
||||||
|
)
|
||||||
|
return actual
|
||||||
|
|
||||||
|
if expectation["expected_outcome"] == "load_ok":
|
||||||
|
registry_data = actual["registry"]
|
||||||
|
assert registry_data is not None # narrowed by branch
|
||||||
|
if registry_data["reviewer_count"] != expectation["expected_reviewer_count"]:
|
||||||
|
actual["passed"] = False
|
||||||
|
actual["divergence"] = (
|
||||||
|
f"expected reviewer_count={expectation['expected_reviewer_count']} got "
|
||||||
|
f"{registry_data['reviewer_count']}"
|
||||||
|
)
|
||||||
|
return actual
|
||||||
|
if registry_data["reviewer_ids"] != expectation["expected_reviewer_ids"]:
|
||||||
|
actual["passed"] = False
|
||||||
|
actual["divergence"] = (
|
||||||
|
f"expected reviewer_ids={expectation['expected_reviewer_ids']} got "
|
||||||
|
f"{registry_data['reviewer_ids']}"
|
||||||
|
)
|
||||||
|
return actual
|
||||||
|
else:
|
||||||
|
actual_error = actual["actual_error"] or ""
|
||||||
|
if expectation["expected_error_substring"] not in actual_error:
|
||||||
|
actual["passed"] = False
|
||||||
|
actual["divergence"] = (
|
||||||
|
f"expected error to contain "
|
||||||
|
f"{expectation['expected_error_substring']!r} got "
|
||||||
|
f"{actual_error!r}"
|
||||||
|
)
|
||||||
|
return actual
|
||||||
|
|
||||||
|
actual["passed"] = True
|
||||||
|
actual["divergence"] = None
|
||||||
|
return actual
|
||||||
|
|
||||||
|
|
||||||
|
def run(*, lane_dir: Path) -> dict[str, Any]:
|
||||||
|
cases_dir = lane_dir / "cases"
|
||||||
|
case_results: list[dict[str, Any]] = []
|
||||||
|
for case_name in sorted(CASE_EXPECTATIONS.keys()):
|
||||||
|
case_path = cases_dir / case_name
|
||||||
|
if not case_path.exists():
|
||||||
|
case_results.append(
|
||||||
|
{
|
||||||
|
"case_id": case_name,
|
||||||
|
"passed": False,
|
||||||
|
"divergence": f"case fixture missing: {case_path}",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
case_results.append(_evaluate_case(case_path, CASE_EXPECTATIONS[case_name]))
|
||||||
|
|
||||||
|
summary = {
|
||||||
|
"lane": "reviewer_registry",
|
||||||
|
"lane_version": "v1",
|
||||||
|
"split": "dev",
|
||||||
|
"adr": "ADR-0092",
|
||||||
|
"invariant": "reviewer_registry_schema_v1",
|
||||||
|
"total_cases": len(case_results),
|
||||||
|
"passed_cases": sum(1 for r in case_results if r["passed"]),
|
||||||
|
"failed_cases": sum(1 for r in case_results if not r["passed"]),
|
||||||
|
"all_passed": all(r["passed"] for r in case_results),
|
||||||
|
"cases": case_results,
|
||||||
|
}
|
||||||
|
return summary
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical_json(payload: dict[str, Any]) -> bytes:
|
||||||
|
return json.dumps(payload, sort_keys=True, indent=2).encode("utf-8") + b"\n"
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
parser = argparse.ArgumentParser(description="reviewer_registry lane runner")
|
||||||
|
parser.add_argument(
|
||||||
|
"--lane-dir",
|
||||||
|
type=Path,
|
||||||
|
default=Path(__file__).resolve().parent,
|
||||||
|
help="lane root directory (defaults to this file's directory)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--report",
|
||||||
|
type=Path,
|
||||||
|
default=None,
|
||||||
|
help="path to write JSON report (defaults to <lane>/results/v1_dev.json)",
|
||||||
|
)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
|
||||||
|
summary = run(lane_dir=args.lane_dir)
|
||||||
|
report_path = args.report or (args.lane_dir / "results" / "v1_dev.json")
|
||||||
|
report_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
payload_bytes = _canonical_json(summary)
|
||||||
|
report_path.write_bytes(payload_bytes)
|
||||||
|
|
||||||
|
sha = hashlib.sha256(payload_bytes).hexdigest()
|
||||||
|
print(f"report: {report_path}")
|
||||||
|
print(f"sha256: {sha}")
|
||||||
|
print(f"passed: {summary['passed_cases']}/{summary['total_cases']}")
|
||||||
|
|
||||||
|
return 0 if summary["all_passed"] else 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
|
|
@ -77,6 +77,13 @@ def test_ledger_status_is_predicate_derived() -> None:
|
||||||
assert report["evidence_counts"]["reviewers_present"] is True
|
assert report["evidence_counts"]["reviewers_present"] is True
|
||||||
assert report["evidence_counts"]["gaps_present"] is True
|
assert report["evidence_counts"]["gaps_present"] is True
|
||||||
|
|
||||||
|
registry_status = report["evidence_counts"]["reviewer_registry"]
|
||||||
|
assert registry_status["valid"] is True
|
||||||
|
assert registry_status["schema_version"] == 1
|
||||||
|
assert registry_status["reviewer_count"] >= 1
|
||||||
|
assert "shay-j" in registry_status["reviewer_ids"]
|
||||||
|
assert registry_status["error"] is None
|
||||||
|
|
||||||
systems = rows["systems_software"]
|
systems = rows["systems_software"]
|
||||||
assert systems["status"] == "reasoning-capable"
|
assert systems["status"] == "reasoning-capable"
|
||||||
assert systems["predicates"]["reasoning_capable"] is True
|
assert systems["predicates"]["reasoning_capable"] is True
|
||||||
|
|
|
||||||
284
tests/test_reviewer_registry.py
Normal file
284
tests/test_reviewer_registry.py
Normal file
|
|
@ -0,0 +1,284 @@
|
||||||
|
"""Unit tests for the Reviewer Registry v1 loader (ADR-0092)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from core.capability.reviewers import (
|
||||||
|
ReviewerRegistry,
|
||||||
|
ReviewerRegistryError,
|
||||||
|
SCHEMA_VERSION,
|
||||||
|
load_reviewer_registry,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _write_yaml(tmp_path: Path, content: str) -> Path:
|
||||||
|
path = tmp_path / "reviewers.yaml"
|
||||||
|
path.write_text(content, encoding="utf-8")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _valid_primary_yaml() -> str:
|
||||||
|
return """\
|
||||||
|
schema_version: 1
|
||||||
|
reviewers:
|
||||||
|
- reviewer_id: shay-j
|
||||||
|
display_name: "Joshua Shay"
|
||||||
|
role: primary
|
||||||
|
domains: ["*"]
|
||||||
|
review_scope: ["pack", "proposal", "chain", "eval"]
|
||||||
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _valid_domain_yaml() -> str:
|
||||||
|
return """\
|
||||||
|
schema_version: 1
|
||||||
|
reviewers:
|
||||||
|
- reviewer_id: math-reviewer
|
||||||
|
display_name: "Math Domain Reviewer"
|
||||||
|
role: domain
|
||||||
|
domains: ["mathematics_logic"]
|
||||||
|
review_scope: ["pack", "chain"]
|
||||||
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class TestPositiveLoad:
|
||||||
|
def test_loads_primary_reviewer(self, tmp_path: Path) -> None:
|
||||||
|
path = _write_yaml(tmp_path, _valid_primary_yaml())
|
||||||
|
registry = load_reviewer_registry(path)
|
||||||
|
assert isinstance(registry, ReviewerRegistry)
|
||||||
|
assert registry.schema_version == SCHEMA_VERSION
|
||||||
|
assert len(registry.reviewers) == 1
|
||||||
|
reviewer = registry.reviewers[0]
|
||||||
|
assert reviewer.reviewer_id == "shay-j"
|
||||||
|
assert reviewer.role == "primary"
|
||||||
|
assert reviewer.domains == ("*",)
|
||||||
|
assert reviewer.review_scope == ("pack", "proposal", "chain", "eval")
|
||||||
|
|
||||||
|
def test_loads_domain_reviewer(self, tmp_path: Path) -> None:
|
||||||
|
path = _write_yaml(tmp_path, _valid_domain_yaml())
|
||||||
|
registry = load_reviewer_registry(path)
|
||||||
|
assert registry.reviewers[0].role == "domain"
|
||||||
|
assert registry.reviewers[0].domains == ("mathematics_logic",)
|
||||||
|
|
||||||
|
def test_registry_is_frozen(self, tmp_path: Path) -> None:
|
||||||
|
path = _write_yaml(tmp_path, _valid_primary_yaml())
|
||||||
|
registry = load_reviewer_registry(path)
|
||||||
|
with pytest.raises((AttributeError, TypeError)):
|
||||||
|
registry.reviewers = () # type: ignore[misc]
|
||||||
|
|
||||||
|
def test_reviewer_is_frozen(self, tmp_path: Path) -> None:
|
||||||
|
path = _write_yaml(tmp_path, _valid_primary_yaml())
|
||||||
|
registry = load_reviewer_registry(path)
|
||||||
|
with pytest.raises((AttributeError, TypeError)):
|
||||||
|
registry.reviewers[0].role = "domain" # type: ignore[misc]
|
||||||
|
|
||||||
|
|
||||||
|
class TestResolution:
|
||||||
|
def test_resolve_known_reviewer(self, tmp_path: Path) -> None:
|
||||||
|
registry = load_reviewer_registry(_write_yaml(tmp_path, _valid_primary_yaml()))
|
||||||
|
reviewer = registry.resolve("shay-j")
|
||||||
|
assert reviewer is not None
|
||||||
|
assert reviewer.reviewer_id == "shay-j"
|
||||||
|
|
||||||
|
def test_resolve_unknown_reviewer(self, tmp_path: Path) -> None:
|
||||||
|
registry = load_reviewer_registry(_write_yaml(tmp_path, _valid_primary_yaml()))
|
||||||
|
assert registry.resolve("ghost") is None
|
||||||
|
|
||||||
|
def test_can_review_primary_covers_any_domain(self, tmp_path: Path) -> None:
|
||||||
|
registry = load_reviewer_registry(_write_yaml(tmp_path, _valid_primary_yaml()))
|
||||||
|
assert registry.can_review("shay-j", domain_id="mathematics_logic", scope="pack")
|
||||||
|
assert registry.can_review("shay-j", domain_id="physics", scope="proposal")
|
||||||
|
|
||||||
|
def test_can_review_domain_only_covers_declared(self, tmp_path: Path) -> None:
|
||||||
|
registry = load_reviewer_registry(_write_yaml(tmp_path, _valid_domain_yaml()))
|
||||||
|
assert registry.can_review("math-reviewer", domain_id="mathematics_logic", scope="pack")
|
||||||
|
assert not registry.can_review("math-reviewer", domain_id="physics", scope="pack")
|
||||||
|
|
||||||
|
def test_can_review_respects_scope(self, tmp_path: Path) -> None:
|
||||||
|
registry = load_reviewer_registry(_write_yaml(tmp_path, _valid_domain_yaml()))
|
||||||
|
assert registry.can_review("math-reviewer", domain_id="mathematics_logic", scope="chain")
|
||||||
|
assert not registry.can_review(
|
||||||
|
"math-reviewer", domain_id="mathematics_logic", scope="proposal"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_can_review_unknown_reviewer_returns_false(self, tmp_path: Path) -> None:
|
||||||
|
registry = load_reviewer_registry(_write_yaml(tmp_path, _valid_primary_yaml()))
|
||||||
|
assert not registry.can_review("ghost", domain_id="mathematics_logic", scope="pack")
|
||||||
|
|
||||||
|
|
||||||
|
class TestNegativeSchema:
|
||||||
|
def test_missing_file(self, tmp_path: Path) -> None:
|
||||||
|
with pytest.raises(ReviewerRegistryError, match="not found"):
|
||||||
|
load_reviewer_registry(tmp_path / "absent.yaml")
|
||||||
|
|
||||||
|
def test_wrong_schema_version(self, tmp_path: Path) -> None:
|
||||||
|
path = _write_yaml(tmp_path, "schema_version: 2\nreviewers: []\n")
|
||||||
|
with pytest.raises(ReviewerRegistryError, match="schema_version"):
|
||||||
|
load_reviewer_registry(path)
|
||||||
|
|
||||||
|
def test_missing_schema_version(self, tmp_path: Path) -> None:
|
||||||
|
path = _write_yaml(tmp_path, "reviewers: []\n")
|
||||||
|
with pytest.raises(ReviewerRegistryError, match="schema_version"):
|
||||||
|
load_reviewer_registry(path)
|
||||||
|
|
||||||
|
def test_unknown_top_level_key(self, tmp_path: Path) -> None:
|
||||||
|
path = _write_yaml(
|
||||||
|
tmp_path,
|
||||||
|
"schema_version: 1\nreviewers: []\nextra_field: nope\n",
|
||||||
|
)
|
||||||
|
with pytest.raises(ReviewerRegistryError, match="unknown top-level"):
|
||||||
|
load_reviewer_registry(path)
|
||||||
|
|
||||||
|
def test_reviewers_must_be_list(self, tmp_path: Path) -> None:
|
||||||
|
path = _write_yaml(tmp_path, "schema_version: 1\nreviewers: {}\n")
|
||||||
|
with pytest.raises(ReviewerRegistryError, match="must be a list"):
|
||||||
|
load_reviewer_registry(path)
|
||||||
|
|
||||||
|
def test_unknown_reviewer_field(self, tmp_path: Path) -> None:
|
||||||
|
yaml = """\
|
||||||
|
schema_version: 1
|
||||||
|
reviewers:
|
||||||
|
- reviewer_id: shay-j
|
||||||
|
display_name: "Joshua Shay"
|
||||||
|
role: primary
|
||||||
|
domains: ["*"]
|
||||||
|
review_scope: ["pack"]
|
||||||
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
||||||
|
secret_powers: ["unlimited"]
|
||||||
|
"""
|
||||||
|
with pytest.raises(ReviewerRegistryError, match="unknown fields"):
|
||||||
|
load_reviewer_registry(_write_yaml(tmp_path, yaml))
|
||||||
|
|
||||||
|
def test_missing_required_field(self, tmp_path: Path) -> None:
|
||||||
|
yaml = """\
|
||||||
|
schema_version: 1
|
||||||
|
reviewers:
|
||||||
|
- reviewer_id: shay-j
|
||||||
|
display_name: "Joshua Shay"
|
||||||
|
role: primary
|
||||||
|
domains: ["*"]
|
||||||
|
review_scope: ["pack"]
|
||||||
|
"""
|
||||||
|
with pytest.raises(ReviewerRegistryError, match="missing required fields"):
|
||||||
|
load_reviewer_registry(_write_yaml(tmp_path, yaml))
|
||||||
|
|
||||||
|
def test_invalid_role(self, tmp_path: Path) -> None:
|
||||||
|
yaml = """\
|
||||||
|
schema_version: 1
|
||||||
|
reviewers:
|
||||||
|
- reviewer_id: shay-j
|
||||||
|
display_name: "Joshua Shay"
|
||||||
|
role: emperor
|
||||||
|
domains: ["*"]
|
||||||
|
review_scope: ["pack"]
|
||||||
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
||||||
|
"""
|
||||||
|
with pytest.raises(ReviewerRegistryError, match="invalid role"):
|
||||||
|
load_reviewer_registry(_write_yaml(tmp_path, yaml))
|
||||||
|
|
||||||
|
def test_domain_reviewer_cannot_claim_wildcard(self, tmp_path: Path) -> None:
|
||||||
|
yaml = """\
|
||||||
|
schema_version: 1
|
||||||
|
reviewers:
|
||||||
|
- reviewer_id: math-reviewer
|
||||||
|
display_name: "Math Reviewer"
|
||||||
|
role: domain
|
||||||
|
domains: ["*"]
|
||||||
|
review_scope: ["pack"]
|
||||||
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
||||||
|
"""
|
||||||
|
with pytest.raises(ReviewerRegistryError, match="wildcard"):
|
||||||
|
load_reviewer_registry(_write_yaml(tmp_path, yaml))
|
||||||
|
|
||||||
|
def test_primary_reviewer_must_claim_wildcard(self, tmp_path: Path) -> None:
|
||||||
|
yaml = """\
|
||||||
|
schema_version: 1
|
||||||
|
reviewers:
|
||||||
|
- reviewer_id: shay-j
|
||||||
|
display_name: "Joshua Shay"
|
||||||
|
role: primary
|
||||||
|
domains: ["mathematics_logic"]
|
||||||
|
review_scope: ["pack"]
|
||||||
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
||||||
|
"""
|
||||||
|
with pytest.raises(ReviewerRegistryError, match="wildcard"):
|
||||||
|
load_reviewer_registry(_write_yaml(tmp_path, yaml))
|
||||||
|
|
||||||
|
def test_unknown_review_scope_value(self, tmp_path: Path) -> None:
|
||||||
|
yaml = """\
|
||||||
|
schema_version: 1
|
||||||
|
reviewers:
|
||||||
|
- reviewer_id: shay-j
|
||||||
|
display_name: "Joshua Shay"
|
||||||
|
role: primary
|
||||||
|
domains: ["*"]
|
||||||
|
review_scope: ["pack", "deploy"]
|
||||||
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
||||||
|
"""
|
||||||
|
with pytest.raises(ReviewerRegistryError, match="unknown review_scope"):
|
||||||
|
load_reviewer_registry(_write_yaml(tmp_path, yaml))
|
||||||
|
|
||||||
|
def test_empty_review_scope(self, tmp_path: Path) -> None:
|
||||||
|
yaml = """\
|
||||||
|
schema_version: 1
|
||||||
|
reviewers:
|
||||||
|
- reviewer_id: shay-j
|
||||||
|
display_name: "Joshua Shay"
|
||||||
|
role: primary
|
||||||
|
domains: ["*"]
|
||||||
|
review_scope: []
|
||||||
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
||||||
|
"""
|
||||||
|
with pytest.raises(ReviewerRegistryError, match="empty 'review_scope'"):
|
||||||
|
load_reviewer_registry(_write_yaml(tmp_path, yaml))
|
||||||
|
|
||||||
|
def test_duplicate_reviewer_id(self, tmp_path: Path) -> None:
|
||||||
|
yaml = """\
|
||||||
|
schema_version: 1
|
||||||
|
reviewers:
|
||||||
|
- reviewer_id: shay-j
|
||||||
|
display_name: "Joshua Shay"
|
||||||
|
role: primary
|
||||||
|
domains: ["*"]
|
||||||
|
review_scope: ["pack"]
|
||||||
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
||||||
|
- reviewer_id: shay-j
|
||||||
|
display_name: "Imposter"
|
||||||
|
role: primary
|
||||||
|
domains: ["*"]
|
||||||
|
review_scope: ["pack"]
|
||||||
|
provenance: "adr-0092:bootstrap:2026-05-21"
|
||||||
|
"""
|
||||||
|
with pytest.raises(ReviewerRegistryError, match="duplicate reviewer_id"):
|
||||||
|
load_reviewer_registry(_write_yaml(tmp_path, yaml))
|
||||||
|
|
||||||
|
def test_malformed_yaml(self, tmp_path: Path) -> None:
|
||||||
|
path = _write_yaml(tmp_path, "schema_version: 1\nreviewers: [\n")
|
||||||
|
with pytest.raises(ReviewerRegistryError, match="YAML parse error"):
|
||||||
|
load_reviewer_registry(path)
|
||||||
|
|
||||||
|
|
||||||
|
class TestProductionRegistry:
|
||||||
|
"""Verify the in-tree docs/reviewers.yaml passes ADR-0092 v1 schema."""
|
||||||
|
|
||||||
|
def test_production_registry_loads(self) -> None:
|
||||||
|
repo_root = Path(__file__).resolve().parent.parent
|
||||||
|
registry = load_reviewer_registry(repo_root / "docs" / "reviewers.yaml")
|
||||||
|
assert registry.schema_version == SCHEMA_VERSION
|
||||||
|
assert len(registry.reviewers) >= 1
|
||||||
|
ids = [r.reviewer_id for r in registry.reviewers]
|
||||||
|
assert "shay-j" in ids
|
||||||
|
|
||||||
|
def test_production_bootstrap_reviewer_is_primary(self) -> None:
|
||||||
|
repo_root = Path(__file__).resolve().parent.parent
|
||||||
|
registry = load_reviewer_registry(repo_root / "docs" / "reviewers.yaml")
|
||||||
|
shay = registry.resolve("shay-j")
|
||||||
|
assert shay is not None
|
||||||
|
assert shay.role == "primary"
|
||||||
|
assert shay.domains == ("*",)
|
||||||
|
assert shay.provenance.startswith("adr-0092:bootstrap:")
|
||||||
Loading…
Reference in a new issue