From afdd2ee413df5f4f16ea5bc668b154b65889f8aa Mon Sep 17 00:00:00 2001 From: Shay Date: Thu, 21 May 2026 18:01:24 -0700 Subject: [PATCH] =?UTF-8?q?feat(capability):=20implement=20ADR-0092=20?= =?UTF-8?q?=E2=80=94=20Reviewer=20Registry=20v1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the load-bearing gap blocking every reasoning-capable claim under ADR-0091: docs/reviewers.yaml was previously `reviewers: []` and unparsed. Now schema-validated at v1, with a bootstrap shay-j entry self-sealed via provenance. - new core.capability.reviewers module: frozen Reviewer/ReviewerRegistry dataclasses, strict load_reviewer_registry parser, ReviewerRegistryError - enforces ADR-0092 schema rules: schema_version==1, no unknown top-level keys, no unknown reviewer fields, role∈{primary,domain}, primary must claim ["*"], domain must NOT claim "*", review_scope subset of {pack,proposal,chain,eval}, no duplicate reviewer_ids - can_review(reviewer_id, domain_id, scope) helper implements ADR-0092 rules 2-4 for downstream use by ADR-0093 validator - docs/reviewers.yaml updated to v1 schema with shay-j bootstrap - ledger_report() evidence_counts now exposes structured reviewer_registry status (valid, schema_version, reviewer_count, reviewer_ids, error) alongside the legacy reviewers_present bool - new evals/reviewer_registry/ lane: 6 cases (2 positive + 4 negative) covering empty-registry, wrong-version, domain-wildcard rejection, and unknown-field rejection - runner emits deterministic JSON report; two runs produce byte-identical output (sha256 verified) - 26 unit tests in tests/test_reviewer_registry.py - capability ledger test extended to assert new reviewer_registry block - smoke suite green (67/67); lane passes 6/6 The pre-existing test_flag_report_tracks_default_off_flags failure is unrelated (discourse_planner flag default) and not introduced here. --- core/capability/__init__.py | 20 ++ core/capability/reporting.py | 42 +++ core/capability/reviewers.py | 207 +++++++++++++ docs/reviewers.yaml | 19 +- .../cases/negative_domain_wildcard.yaml | 8 + .../cases/negative_empty.yaml | 2 + .../cases/negative_unknown_field.yaml | 9 + .../cases/negative_wrong_version.yaml | 8 + .../cases/positive_domain.yaml | 8 + .../cases/positive_primary.yaml | 8 + evals/reviewer_registry/contract.md | 44 +++ evals/reviewer_registry/results/v1_dev.json | 83 +++++ evals/reviewer_registry/runner.py | 187 ++++++++++++ tests/test_capability_reports.py | 7 + tests/test_reviewer_registry.py | 284 ++++++++++++++++++ 15 files changed, 935 insertions(+), 1 deletion(-) create mode 100644 core/capability/reviewers.py create mode 100644 evals/reviewer_registry/cases/negative_domain_wildcard.yaml create mode 100644 evals/reviewer_registry/cases/negative_empty.yaml create mode 100644 evals/reviewer_registry/cases/negative_unknown_field.yaml create mode 100644 evals/reviewer_registry/cases/negative_wrong_version.yaml create mode 100644 evals/reviewer_registry/cases/positive_domain.yaml create mode 100644 evals/reviewer_registry/cases/positive_primary.yaml create mode 100644 evals/reviewer_registry/contract.md create mode 100644 evals/reviewer_registry/results/v1_dev.json create mode 100644 evals/reviewer_registry/runner.py create mode 100644 tests/test_reviewer_registry.py diff --git a/core/capability/__init__.py b/core/capability/__init__.py index 04dfeeba..d579c796 100644 --- a/core/capability/__init__.py +++ b/core/capability/__init__.py @@ -8,12 +8,32 @@ from .reporting import ( flag_report, ledger_report, ) +from .reviewers import ( + ALLOWED_REVIEWER_KEYS, + ALLOWED_ROLES, + ALLOWED_SCOPES, + ALLOWED_TOP_LEVEL_KEYS, + Reviewer, + ReviewerRegistry, + ReviewerRegistryError, + SCHEMA_VERSION as REVIEWER_REGISTRY_SCHEMA_VERSION, + load_reviewer_registry, +) __all__ = [ + "ALLOWED_REVIEWER_KEYS", + "ALLOWED_ROLES", + "ALLOWED_SCOPES", + "ALLOWED_TOP_LEVEL_KEYS", "CapabilityArtifactQuery", + "REVIEWER_REGISTRY_SCHEMA_VERSION", + "Reviewer", + "ReviewerRegistry", + "ReviewerRegistryError", "artifact_report", "chain_report", "evidence_plan_report", "flag_report", "ledger_report", + "load_reviewer_registry", ] diff --git a/core/capability/reporting.py b/core/capability/reporting.py index c06353a9..9bd30d6e 100644 --- a/core/capability/reporting.py +++ b/core/capability/reporting.py @@ -17,6 +17,10 @@ from core.capability.domains import ( DOMAIN_OPERATOR_CLAIMS, DOMAIN_PACKS, ) +from core.capability.reviewers import ( + ReviewerRegistryError, + load_reviewer_registry, +) from core.capability.sources import LEDGER_SOURCES from core.config import DEFAULT_CONFIG from language_packs.domain_contract import validate_domain_contract_pack @@ -498,6 +502,7 @@ def ledger_report() -> dict[str, Any]: "audit_tour": len(audit_files), "pack_measurements_present": (_REPO_ROOT / LEDGER_SOURCES.pack_measurements).exists(), "reviewers_present": (_REPO_ROOT / LEDGER_SOURCES.reviewers).exists(), + "reviewer_registry": _reviewer_registry_status(), "gaps_present": (_REPO_ROOT / LEDGER_SOURCES.gaps).exists(), }, "domains": domain_rows, @@ -583,3 +588,40 @@ def evidence_plan_report() -> dict[str, Any]: "workers_promote_packs": False, "jobs": jobs, } + + +def _reviewer_registry_status() -> dict[str, Any]: + """Report Reviewer Registry v1 validity for the capability ledger (ADR-0092). + + Returns a structured object containing the parsed schema version, + reviewer count, and a list of registered reviewer ids. If the + registry is missing or fails ADR-0092 schema validation, the object + reports ``valid: False`` and names the error rather than raising, + so the ledger surface remains generatable on a broken registry. + """ + path = _REPO_ROOT / LEDGER_SOURCES.reviewers + if not path.exists(): + return { + "valid": False, + "error": "reviewers.yaml not found", + "schema_version": None, + "reviewer_count": 0, + "reviewer_ids": [], + } + try: + registry = load_reviewer_registry(path) + except ReviewerRegistryError as exc: + return { + "valid": False, + "error": str(exc), + "schema_version": None, + "reviewer_count": 0, + "reviewer_ids": [], + } + return { + "valid": True, + "error": None, + "schema_version": registry.schema_version, + "reviewer_count": len(registry.reviewers), + "reviewer_ids": sorted(r.reviewer_id for r in registry.reviewers), + } diff --git a/core/capability/reviewers.py b/core/capability/reviewers.py new file mode 100644 index 00000000..3736d84d --- /dev/null +++ b/core/capability/reviewers.py @@ -0,0 +1,207 @@ +"""Reviewer Registry v1 loader and validator (ADR-0092). + +Parses ``docs/reviewers.yaml`` into a typed, immutable :class:`ReviewerRegistry`. +Schema rejection is loud: unknown keys, malformed entries, duplicate ids, and +out-of-scope wildcards all raise :class:`ReviewerRegistryError`. + +The registry is consulted by the Domain Pack Contract v1 validator +(ADR-0093) to enforce ADR-0091 predicate #8 (reviewer resolution). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + +import yaml + + +ALLOWED_TOP_LEVEL_KEYS: frozenset[str] = frozenset({"schema_version", "reviewers"}) +ALLOWED_REVIEWER_KEYS: frozenset[str] = frozenset( + {"reviewer_id", "display_name", "role", "domains", "review_scope", "provenance"} +) +ALLOWED_ROLES: frozenset[str] = frozenset({"primary", "domain"}) +ALLOWED_SCOPES: frozenset[str] = frozenset({"pack", "proposal", "chain", "eval"}) +SCHEMA_VERSION: int = 1 +WILDCARD_DOMAIN: str = "*" + + +class ReviewerRegistryError(ValueError): + """Raised when ``docs/reviewers.yaml`` fails ADR-0092 v1 schema validation.""" + + +@dataclass(frozen=True, slots=True) +class Reviewer: + reviewer_id: str + display_name: str + role: str + domains: tuple[str, ...] + review_scope: tuple[str, ...] + provenance: str + + +@dataclass(frozen=True, slots=True) +class ReviewerRegistry: + schema_version: int + reviewers: tuple[Reviewer, ...] + + def resolve(self, reviewer_id: str) -> Reviewer | None: + for reviewer in self.reviewers: + if reviewer.reviewer_id == reviewer_id: + return reviewer + return None + + def can_review(self, reviewer_id: str, *, domain_id: str, scope: str) -> bool: + """Return True iff this reviewer may ratify ``scope`` artifacts for ``domain_id``. + + Implements ADR-0091 predicate #8 plus ADR-0092 rules 2, 3, and 4: + - rule 2: unknown ``reviewer_id`` → False + - rule 3: ``role: domain`` reviewer whose ``domains`` do not include + ``domain_id`` → False + - rule 4: reviewer whose ``review_scope`` does not cover ``scope`` → False + """ + reviewer = self.resolve(reviewer_id) + if reviewer is None: + return False + if scope not in reviewer.review_scope: + return False + if reviewer.role == "primary": + return True + return domain_id in reviewer.domains + + +def load_reviewer_registry(path: Path) -> ReviewerRegistry: + """Parse ``path`` as a Reviewer Registry v1 file. + + Raises :class:`ReviewerRegistryError` on any schema violation. No + side effects beyond reading ``path``. + """ + if not path.exists(): + raise ReviewerRegistryError(f"reviewer registry not found at {path}") + + try: + raw = yaml.safe_load(path.read_text(encoding="utf-8")) + except yaml.YAMLError as exc: + raise ReviewerRegistryError(f"reviewer registry YAML parse error: {exc}") from exc + + if not isinstance(raw, Mapping): + raise ReviewerRegistryError("reviewer registry root must be a mapping") + + unknown_top = set(raw.keys()) - ALLOWED_TOP_LEVEL_KEYS + if unknown_top: + raise ReviewerRegistryError( + f"reviewer registry has unknown top-level keys: {sorted(unknown_top)}" + ) + + schema_version = raw.get("schema_version") + if schema_version != SCHEMA_VERSION: + raise ReviewerRegistryError( + f"reviewer registry schema_version must be {SCHEMA_VERSION}; got {schema_version!r}" + ) + + reviewers_raw = raw.get("reviewers") + if reviewers_raw is None: + raise ReviewerRegistryError("reviewer registry missing 'reviewers' field") + if not isinstance(reviewers_raw, list): + raise ReviewerRegistryError("reviewer registry 'reviewers' must be a list") + + parsed: list[Reviewer] = [] + seen_ids: set[str] = set() + for index, entry in enumerate(reviewers_raw): + reviewer = _parse_reviewer(entry, index=index) + if reviewer.reviewer_id in seen_ids: + raise ReviewerRegistryError( + f"reviewer registry contains duplicate reviewer_id {reviewer.reviewer_id!r}" + ) + seen_ids.add(reviewer.reviewer_id) + parsed.append(reviewer) + + return ReviewerRegistry(schema_version=SCHEMA_VERSION, reviewers=tuple(parsed)) + + +def _parse_reviewer(entry: Any, *, index: int) -> Reviewer: + if not isinstance(entry, Mapping): + raise ReviewerRegistryError(f"reviewer entry at index {index} must be a mapping") + + unknown = set(entry.keys()) - ALLOWED_REVIEWER_KEYS + if unknown: + raise ReviewerRegistryError( + f"reviewer entry at index {index} has unknown fields: {sorted(unknown)}" + ) + + missing = ALLOWED_REVIEWER_KEYS - set(entry.keys()) + if missing: + raise ReviewerRegistryError( + f"reviewer entry at index {index} missing required fields: {sorted(missing)}" + ) + + reviewer_id = _require_nonempty_str(entry["reviewer_id"], field="reviewer_id", index=index) + display_name = _require_nonempty_str(entry["display_name"], field="display_name", index=index) + role = _require_nonempty_str(entry["role"], field="role", index=index) + provenance = _require_nonempty_str(entry["provenance"], field="provenance", index=index) + + if role not in ALLOWED_ROLES: + raise ReviewerRegistryError( + f"reviewer entry at index {index} has invalid role {role!r}; " + f"must be one of {sorted(ALLOWED_ROLES)}" + ) + + domains = _require_str_list(entry["domains"], field="domains", index=index) + if not domains: + raise ReviewerRegistryError( + f"reviewer entry at index {index} has empty 'domains' list" + ) + if role == "domain" and WILDCARD_DOMAIN in domains: + raise ReviewerRegistryError( + f"reviewer entry at index {index} has role='domain' but claims wildcard " + f"'{WILDCARD_DOMAIN}'; only role='primary' may claim wildcard domains" + ) + if role == "primary" and domains != (WILDCARD_DOMAIN,): + raise ReviewerRegistryError( + f"reviewer entry at index {index} has role='primary' but domains " + f"is not exactly [{WILDCARD_DOMAIN!r}]; primary reviewers must claim wildcard" + ) + + review_scope = _require_str_list(entry["review_scope"], field="review_scope", index=index) + if not review_scope: + raise ReviewerRegistryError( + f"reviewer entry at index {index} has empty 'review_scope' list" + ) + unknown_scopes = set(review_scope) - ALLOWED_SCOPES + if unknown_scopes: + raise ReviewerRegistryError( + f"reviewer entry at index {index} has unknown review_scope values: " + f"{sorted(unknown_scopes)}; allowed: {sorted(ALLOWED_SCOPES)}" + ) + + return Reviewer( + reviewer_id=reviewer_id, + display_name=display_name, + role=role, + domains=domains, + review_scope=review_scope, + provenance=provenance, + ) + + +def _require_nonempty_str(value: Any, *, field: str, index: int) -> str: + if not isinstance(value, str) or not value.strip(): + raise ReviewerRegistryError( + f"reviewer entry at index {index} field {field!r} must be a non-empty string" + ) + return value + + +def _require_str_list(value: Any, *, field: str, index: int) -> tuple[str, ...]: + if not isinstance(value, list): + raise ReviewerRegistryError( + f"reviewer entry at index {index} field {field!r} must be a list of strings" + ) + for item in value: + if not isinstance(item, str) or not item.strip(): + raise ReviewerRegistryError( + f"reviewer entry at index {index} field {field!r} contains a " + "non-string or empty value" + ) + return tuple(value) diff --git a/docs/reviewers.yaml b/docs/reviewers.yaml index 602b682a..f9082b52 100644 --- a/docs/reviewers.yaml +++ b/docs/reviewers.yaml @@ -1 +1,18 @@ -reviewers: [] +# Reviewer Registry v1 (ADR-0092). +# +# This file is parsed by core.capability.reviewers.load_reviewer_registry +# and consulted by the Domain Pack Contract v1 validator (ADR-0093) to +# enforce ADR-0091 predicate #8 (reviewer resolution). +# +# Adding reviewers is a reviewed proposal flow. Do not edit by hand +# outside that flow. The bootstrap entry below is self-sealed via its +# provenance field. + +schema_version: 1 +reviewers: + - reviewer_id: shay-j + display_name: "Joshua Shay" + role: primary + domains: ["*"] + review_scope: ["pack", "proposal", "chain", "eval"] + provenance: "adr-0092:bootstrap:2026-05-21" diff --git a/evals/reviewer_registry/cases/negative_domain_wildcard.yaml b/evals/reviewer_registry/cases/negative_domain_wildcard.yaml new file mode 100644 index 00000000..e34188b2 --- /dev/null +++ b/evals/reviewer_registry/cases/negative_domain_wildcard.yaml @@ -0,0 +1,8 @@ +schema_version: 1 +reviewers: + - reviewer_id: math-reviewer + display_name: "Mathematics Domain Reviewer" + role: domain + domains: ["*"] + review_scope: ["pack"] + provenance: "adr-0092:bootstrap:2026-05-21" diff --git a/evals/reviewer_registry/cases/negative_empty.yaml b/evals/reviewer_registry/cases/negative_empty.yaml new file mode 100644 index 00000000..c5d90732 --- /dev/null +++ b/evals/reviewer_registry/cases/negative_empty.yaml @@ -0,0 +1,2 @@ +schema_version: 1 +reviewers: [] diff --git a/evals/reviewer_registry/cases/negative_unknown_field.yaml b/evals/reviewer_registry/cases/negative_unknown_field.yaml new file mode 100644 index 00000000..c1c5740a --- /dev/null +++ b/evals/reviewer_registry/cases/negative_unknown_field.yaml @@ -0,0 +1,9 @@ +schema_version: 1 +reviewers: + - reviewer_id: shay-j + display_name: "Joshua Shay" + role: primary + domains: ["*"] + review_scope: ["pack"] + provenance: "adr-0092:bootstrap:2026-05-21" + secret_powers: ["unlimited"] diff --git a/evals/reviewer_registry/cases/negative_wrong_version.yaml b/evals/reviewer_registry/cases/negative_wrong_version.yaml new file mode 100644 index 00000000..6113a946 --- /dev/null +++ b/evals/reviewer_registry/cases/negative_wrong_version.yaml @@ -0,0 +1,8 @@ +schema_version: 2 +reviewers: + - reviewer_id: shay-j + display_name: "Joshua Shay" + role: primary + domains: ["*"] + review_scope: ["pack"] + provenance: "adr-0092:bootstrap:2026-05-21" diff --git a/evals/reviewer_registry/cases/positive_domain.yaml b/evals/reviewer_registry/cases/positive_domain.yaml new file mode 100644 index 00000000..c1e0a57f --- /dev/null +++ b/evals/reviewer_registry/cases/positive_domain.yaml @@ -0,0 +1,8 @@ +schema_version: 1 +reviewers: + - reviewer_id: math-reviewer + display_name: "Mathematics Domain Reviewer" + role: domain + domains: ["mathematics_logic"] + review_scope: ["pack", "chain"] + provenance: "adr-0092:bootstrap:2026-05-21" diff --git a/evals/reviewer_registry/cases/positive_primary.yaml b/evals/reviewer_registry/cases/positive_primary.yaml new file mode 100644 index 00000000..80c4f77d --- /dev/null +++ b/evals/reviewer_registry/cases/positive_primary.yaml @@ -0,0 +1,8 @@ +schema_version: 1 +reviewers: + - reviewer_id: shay-j + display_name: "Joshua Shay" + role: primary + domains: ["*"] + review_scope: ["pack", "proposal", "chain", "eval"] + provenance: "adr-0092:bootstrap:2026-05-21" diff --git a/evals/reviewer_registry/contract.md b/evals/reviewer_registry/contract.md new file mode 100644 index 00000000..3e12cf6a --- /dev/null +++ b/evals/reviewer_registry/contract.md @@ -0,0 +1,44 @@ +# evals/reviewer_registry — Lane Contract + +**ADR:** ADR-0092 +**Invariant:** `reviewer_registry_schema_v1` + +## Purpose + +Prove that ADR-0092's Reviewer Registry v1 schema is enforced. The lane +asserts: + +1. A schema-valid registry loads without error and exposes the expected + reviewer count and ids. +2. Each ADR-0092 schema violation produces a typed + :class:`ReviewerRegistryError` whose message names the failing field. +3. An empty registry (`reviewers: []` with no entries) blocks all + `reasoning-capable` claims under ADR-0091 by producing zero + resolvable reviewer ids. +4. A registry without `schema_version: 1` is rejected before any + reviewer entry is parsed. + +## Cases + +The runner iterates over the case fixtures in this directory: + +- `cases/positive_primary.yaml` — single primary reviewer, must load. +- `cases/positive_domain.yaml` — single domain reviewer, must load. +- `cases/negative_empty.yaml` — `reviewers: []`, must load with zero + reviewers (no error, but no resolvable id either). +- `cases/negative_wrong_version.yaml` — `schema_version: 2`, must reject. +- `cases/negative_domain_wildcard.yaml` — domain reviewer claims + `["*"]`, must reject. +- `cases/negative_unknown_field.yaml` — reviewer entry has an + unrecognized field, must reject. + +## Determinism + +The runner emits a `results/v1_dev.json` containing per-case outcomes +and a SHA-256 of the report bytes. Two consecutive runs against the +same fixtures must produce identical bytes. + +## Exit code + +The runner exits non-zero on any case whose actual outcome diverges +from the case spec. diff --git a/evals/reviewer_registry/results/v1_dev.json b/evals/reviewer_registry/results/v1_dev.json new file mode 100644 index 00000000..b396cf01 --- /dev/null +++ b/evals/reviewer_registry/results/v1_dev.json @@ -0,0 +1,83 @@ +{ + "adr": "ADR-0092", + "all_passed": true, + "cases": [ + { + "actual_error": "reviewer entry at index 0 has role='domain' but claims wildcard '*'; only role='primary' may claim wildcard domains", + "actual_outcome": "reject", + "case_id": "negative_domain_wildcard.yaml", + "divergence": null, + "passed": true, + "purpose": "domain reviewer claiming '*' rejected", + "registry": null + }, + { + "actual_error": null, + "actual_outcome": "load_ok", + "case_id": "negative_empty.yaml", + "divergence": null, + "passed": true, + "purpose": "empty registry loads but blocks reasoning-capable claims", + "registry": { + "reviewer_count": 0, + "reviewer_ids": [], + "schema_version": 1 + } + }, + { + "actual_error": "reviewer entry at index 0 has unknown fields: ['secret_powers']", + "actual_outcome": "reject", + "case_id": "negative_unknown_field.yaml", + "divergence": null, + "passed": true, + "purpose": "unknown reviewer field rejected", + "registry": null + }, + { + "actual_error": "reviewer registry schema_version must be 1; got 2", + "actual_outcome": "reject", + "case_id": "negative_wrong_version.yaml", + "divergence": null, + "passed": true, + "purpose": "non-v1 schema rejected before any reviewer parsed", + "registry": null + }, + { + "actual_error": null, + "actual_outcome": "load_ok", + "case_id": "positive_domain.yaml", + "divergence": null, + "passed": true, + "purpose": "single domain reviewer loads cleanly", + "registry": { + "reviewer_count": 1, + "reviewer_ids": [ + "math-reviewer" + ], + "schema_version": 1 + } + }, + { + "actual_error": null, + "actual_outcome": "load_ok", + "case_id": "positive_primary.yaml", + "divergence": null, + "passed": true, + "purpose": "single primary reviewer loads cleanly", + "registry": { + "reviewer_count": 1, + "reviewer_ids": [ + "shay-j" + ], + "schema_version": 1 + } + } + ], + "failed_cases": 0, + "invariant": "reviewer_registry_schema_v1", + "lane": "reviewer_registry", + "lane_version": "v1", + "passed_cases": 6, + "split": "dev", + "total_cases": 6 +} diff --git a/evals/reviewer_registry/runner.py b/evals/reviewer_registry/runner.py new file mode 100644 index 00000000..811ae471 --- /dev/null +++ b/evals/reviewer_registry/runner.py @@ -0,0 +1,187 @@ +"""Runner for evals/reviewer_registry/ (ADR-0092). + +Iterates over the case fixtures under ``cases/`` and asserts each one's +load outcome matches its declared expectation. Emits a deterministic +JSON report to ``results/v1_dev.json``. + +Exit code is non-zero on any divergence. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path +from typing import Any + +from core.capability.reviewers import ( + ReviewerRegistryError, + load_reviewer_registry, +) + + +CASE_EXPECTATIONS: dict[str, dict[str, Any]] = { + "positive_primary.yaml": { + "expected_outcome": "load_ok", + "expected_reviewer_count": 1, + "expected_reviewer_ids": ["shay-j"], + "purpose": "single primary reviewer loads cleanly", + }, + "positive_domain.yaml": { + "expected_outcome": "load_ok", + "expected_reviewer_count": 1, + "expected_reviewer_ids": ["math-reviewer"], + "purpose": "single domain reviewer loads cleanly", + }, + "negative_empty.yaml": { + "expected_outcome": "load_ok", + "expected_reviewer_count": 0, + "expected_reviewer_ids": [], + "purpose": "empty registry loads but blocks reasoning-capable claims", + }, + "negative_wrong_version.yaml": { + "expected_outcome": "reject", + "expected_error_substring": "schema_version", + "purpose": "non-v1 schema rejected before any reviewer parsed", + }, + "negative_domain_wildcard.yaml": { + "expected_outcome": "reject", + "expected_error_substring": "wildcard", + "purpose": "domain reviewer claiming '*' rejected", + }, + "negative_unknown_field.yaml": { + "expected_outcome": "reject", + "expected_error_substring": "unknown fields", + "purpose": "unknown reviewer field rejected", + }, +} + + +def _evaluate_case(path: Path, expectation: dict[str, Any]) -> dict[str, Any]: + actual: dict[str, Any] = {"case_id": path.name, "purpose": expectation["purpose"]} + + try: + registry = load_reviewer_registry(path) + except ReviewerRegistryError as exc: + actual["actual_outcome"] = "reject" + actual["actual_error"] = str(exc) + actual["registry"] = None + else: + actual["actual_outcome"] = "load_ok" + actual["actual_error"] = None + actual["registry"] = { + "schema_version": registry.schema_version, + "reviewer_count": len(registry.reviewers), + "reviewer_ids": sorted(r.reviewer_id for r in registry.reviewers), + } + + if actual["actual_outcome"] != expectation["expected_outcome"]: + actual["passed"] = False + actual["divergence"] = ( + f"expected {expectation['expected_outcome']!r} got " + f"{actual['actual_outcome']!r}" + ) + return actual + + if expectation["expected_outcome"] == "load_ok": + registry_data = actual["registry"] + assert registry_data is not None # narrowed by branch + if registry_data["reviewer_count"] != expectation["expected_reviewer_count"]: + actual["passed"] = False + actual["divergence"] = ( + f"expected reviewer_count={expectation['expected_reviewer_count']} got " + f"{registry_data['reviewer_count']}" + ) + return actual + if registry_data["reviewer_ids"] != expectation["expected_reviewer_ids"]: + actual["passed"] = False + actual["divergence"] = ( + f"expected reviewer_ids={expectation['expected_reviewer_ids']} got " + f"{registry_data['reviewer_ids']}" + ) + return actual + else: + actual_error = actual["actual_error"] or "" + if expectation["expected_error_substring"] not in actual_error: + actual["passed"] = False + actual["divergence"] = ( + f"expected error to contain " + f"{expectation['expected_error_substring']!r} got " + f"{actual_error!r}" + ) + return actual + + actual["passed"] = True + actual["divergence"] = None + return actual + + +def run(*, lane_dir: Path) -> dict[str, Any]: + cases_dir = lane_dir / "cases" + case_results: list[dict[str, Any]] = [] + for case_name in sorted(CASE_EXPECTATIONS.keys()): + case_path = cases_dir / case_name + if not case_path.exists(): + case_results.append( + { + "case_id": case_name, + "passed": False, + "divergence": f"case fixture missing: {case_path}", + } + ) + continue + case_results.append(_evaluate_case(case_path, CASE_EXPECTATIONS[case_name])) + + summary = { + "lane": "reviewer_registry", + "lane_version": "v1", + "split": "dev", + "adr": "ADR-0092", + "invariant": "reviewer_registry_schema_v1", + "total_cases": len(case_results), + "passed_cases": sum(1 for r in case_results if r["passed"]), + "failed_cases": sum(1 for r in case_results if not r["passed"]), + "all_passed": all(r["passed"] for r in case_results), + "cases": case_results, + } + return summary + + +def _canonical_json(payload: dict[str, Any]) -> bytes: + return json.dumps(payload, sort_keys=True, indent=2).encode("utf-8") + b"\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="reviewer_registry lane runner") + parser.add_argument( + "--lane-dir", + type=Path, + default=Path(__file__).resolve().parent, + help="lane root directory (defaults to this file's directory)", + ) + parser.add_argument( + "--report", + type=Path, + default=None, + help="path to write JSON report (defaults to /results/v1_dev.json)", + ) + args = parser.parse_args(argv) + + summary = run(lane_dir=args.lane_dir) + report_path = args.report or (args.lane_dir / "results" / "v1_dev.json") + report_path.parent.mkdir(parents=True, exist_ok=True) + payload_bytes = _canonical_json(summary) + report_path.write_bytes(payload_bytes) + + sha = hashlib.sha256(payload_bytes).hexdigest() + print(f"report: {report_path}") + print(f"sha256: {sha}") + print(f"passed: {summary['passed_cases']}/{summary['total_cases']}") + + return 0 if summary["all_passed"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_capability_reports.py b/tests/test_capability_reports.py index 32cac33d..0d058073 100644 --- a/tests/test_capability_reports.py +++ b/tests/test_capability_reports.py @@ -77,6 +77,13 @@ def test_ledger_status_is_predicate_derived() -> None: assert report["evidence_counts"]["reviewers_present"] is True assert report["evidence_counts"]["gaps_present"] is True + registry_status = report["evidence_counts"]["reviewer_registry"] + assert registry_status["valid"] is True + assert registry_status["schema_version"] == 1 + assert registry_status["reviewer_count"] >= 1 + assert "shay-j" in registry_status["reviewer_ids"] + assert registry_status["error"] is None + systems = rows["systems_software"] assert systems["status"] == "reasoning-capable" assert systems["predicates"]["reasoning_capable"] is True diff --git a/tests/test_reviewer_registry.py b/tests/test_reviewer_registry.py new file mode 100644 index 00000000..4136656d --- /dev/null +++ b/tests/test_reviewer_registry.py @@ -0,0 +1,284 @@ +"""Unit tests for the Reviewer Registry v1 loader (ADR-0092).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from core.capability.reviewers import ( + ReviewerRegistry, + ReviewerRegistryError, + SCHEMA_VERSION, + load_reviewer_registry, +) + + +def _write_yaml(tmp_path: Path, content: str) -> Path: + path = tmp_path / "reviewers.yaml" + path.write_text(content, encoding="utf-8") + return path + + +def _valid_primary_yaml() -> str: + return """\ +schema_version: 1 +reviewers: + - reviewer_id: shay-j + display_name: "Joshua Shay" + role: primary + domains: ["*"] + review_scope: ["pack", "proposal", "chain", "eval"] + provenance: "adr-0092:bootstrap:2026-05-21" +""" + + +def _valid_domain_yaml() -> str: + return """\ +schema_version: 1 +reviewers: + - reviewer_id: math-reviewer + display_name: "Math Domain Reviewer" + role: domain + domains: ["mathematics_logic"] + review_scope: ["pack", "chain"] + provenance: "adr-0092:bootstrap:2026-05-21" +""" + + +class TestPositiveLoad: + def test_loads_primary_reviewer(self, tmp_path: Path) -> None: + path = _write_yaml(tmp_path, _valid_primary_yaml()) + registry = load_reviewer_registry(path) + assert isinstance(registry, ReviewerRegistry) + assert registry.schema_version == SCHEMA_VERSION + assert len(registry.reviewers) == 1 + reviewer = registry.reviewers[0] + assert reviewer.reviewer_id == "shay-j" + assert reviewer.role == "primary" + assert reviewer.domains == ("*",) + assert reviewer.review_scope == ("pack", "proposal", "chain", "eval") + + def test_loads_domain_reviewer(self, tmp_path: Path) -> None: + path = _write_yaml(tmp_path, _valid_domain_yaml()) + registry = load_reviewer_registry(path) + assert registry.reviewers[0].role == "domain" + assert registry.reviewers[0].domains == ("mathematics_logic",) + + def test_registry_is_frozen(self, tmp_path: Path) -> None: + path = _write_yaml(tmp_path, _valid_primary_yaml()) + registry = load_reviewer_registry(path) + with pytest.raises((AttributeError, TypeError)): + registry.reviewers = () # type: ignore[misc] + + def test_reviewer_is_frozen(self, tmp_path: Path) -> None: + path = _write_yaml(tmp_path, _valid_primary_yaml()) + registry = load_reviewer_registry(path) + with pytest.raises((AttributeError, TypeError)): + registry.reviewers[0].role = "domain" # type: ignore[misc] + + +class TestResolution: + def test_resolve_known_reviewer(self, tmp_path: Path) -> None: + registry = load_reviewer_registry(_write_yaml(tmp_path, _valid_primary_yaml())) + reviewer = registry.resolve("shay-j") + assert reviewer is not None + assert reviewer.reviewer_id == "shay-j" + + def test_resolve_unknown_reviewer(self, tmp_path: Path) -> None: + registry = load_reviewer_registry(_write_yaml(tmp_path, _valid_primary_yaml())) + assert registry.resolve("ghost") is None + + def test_can_review_primary_covers_any_domain(self, tmp_path: Path) -> None: + registry = load_reviewer_registry(_write_yaml(tmp_path, _valid_primary_yaml())) + assert registry.can_review("shay-j", domain_id="mathematics_logic", scope="pack") + assert registry.can_review("shay-j", domain_id="physics", scope="proposal") + + def test_can_review_domain_only_covers_declared(self, tmp_path: Path) -> None: + registry = load_reviewer_registry(_write_yaml(tmp_path, _valid_domain_yaml())) + assert registry.can_review("math-reviewer", domain_id="mathematics_logic", scope="pack") + assert not registry.can_review("math-reviewer", domain_id="physics", scope="pack") + + def test_can_review_respects_scope(self, tmp_path: Path) -> None: + registry = load_reviewer_registry(_write_yaml(tmp_path, _valid_domain_yaml())) + assert registry.can_review("math-reviewer", domain_id="mathematics_logic", scope="chain") + assert not registry.can_review( + "math-reviewer", domain_id="mathematics_logic", scope="proposal" + ) + + def test_can_review_unknown_reviewer_returns_false(self, tmp_path: Path) -> None: + registry = load_reviewer_registry(_write_yaml(tmp_path, _valid_primary_yaml())) + assert not registry.can_review("ghost", domain_id="mathematics_logic", scope="pack") + + +class TestNegativeSchema: + def test_missing_file(self, tmp_path: Path) -> None: + with pytest.raises(ReviewerRegistryError, match="not found"): + load_reviewer_registry(tmp_path / "absent.yaml") + + def test_wrong_schema_version(self, tmp_path: Path) -> None: + path = _write_yaml(tmp_path, "schema_version: 2\nreviewers: []\n") + with pytest.raises(ReviewerRegistryError, match="schema_version"): + load_reviewer_registry(path) + + def test_missing_schema_version(self, tmp_path: Path) -> None: + path = _write_yaml(tmp_path, "reviewers: []\n") + with pytest.raises(ReviewerRegistryError, match="schema_version"): + load_reviewer_registry(path) + + def test_unknown_top_level_key(self, tmp_path: Path) -> None: + path = _write_yaml( + tmp_path, + "schema_version: 1\nreviewers: []\nextra_field: nope\n", + ) + with pytest.raises(ReviewerRegistryError, match="unknown top-level"): + load_reviewer_registry(path) + + def test_reviewers_must_be_list(self, tmp_path: Path) -> None: + path = _write_yaml(tmp_path, "schema_version: 1\nreviewers: {}\n") + with pytest.raises(ReviewerRegistryError, match="must be a list"): + load_reviewer_registry(path) + + def test_unknown_reviewer_field(self, tmp_path: Path) -> None: + yaml = """\ +schema_version: 1 +reviewers: + - reviewer_id: shay-j + display_name: "Joshua Shay" + role: primary + domains: ["*"] + review_scope: ["pack"] + provenance: "adr-0092:bootstrap:2026-05-21" + secret_powers: ["unlimited"] +""" + with pytest.raises(ReviewerRegistryError, match="unknown fields"): + load_reviewer_registry(_write_yaml(tmp_path, yaml)) + + def test_missing_required_field(self, tmp_path: Path) -> None: + yaml = """\ +schema_version: 1 +reviewers: + - reviewer_id: shay-j + display_name: "Joshua Shay" + role: primary + domains: ["*"] + review_scope: ["pack"] +""" + with pytest.raises(ReviewerRegistryError, match="missing required fields"): + load_reviewer_registry(_write_yaml(tmp_path, yaml)) + + def test_invalid_role(self, tmp_path: Path) -> None: + yaml = """\ +schema_version: 1 +reviewers: + - reviewer_id: shay-j + display_name: "Joshua Shay" + role: emperor + domains: ["*"] + review_scope: ["pack"] + provenance: "adr-0092:bootstrap:2026-05-21" +""" + with pytest.raises(ReviewerRegistryError, match="invalid role"): + load_reviewer_registry(_write_yaml(tmp_path, yaml)) + + def test_domain_reviewer_cannot_claim_wildcard(self, tmp_path: Path) -> None: + yaml = """\ +schema_version: 1 +reviewers: + - reviewer_id: math-reviewer + display_name: "Math Reviewer" + role: domain + domains: ["*"] + review_scope: ["pack"] + provenance: "adr-0092:bootstrap:2026-05-21" +""" + with pytest.raises(ReviewerRegistryError, match="wildcard"): + load_reviewer_registry(_write_yaml(tmp_path, yaml)) + + def test_primary_reviewer_must_claim_wildcard(self, tmp_path: Path) -> None: + yaml = """\ +schema_version: 1 +reviewers: + - reviewer_id: shay-j + display_name: "Joshua Shay" + role: primary + domains: ["mathematics_logic"] + review_scope: ["pack"] + provenance: "adr-0092:bootstrap:2026-05-21" +""" + with pytest.raises(ReviewerRegistryError, match="wildcard"): + load_reviewer_registry(_write_yaml(tmp_path, yaml)) + + def test_unknown_review_scope_value(self, tmp_path: Path) -> None: + yaml = """\ +schema_version: 1 +reviewers: + - reviewer_id: shay-j + display_name: "Joshua Shay" + role: primary + domains: ["*"] + review_scope: ["pack", "deploy"] + provenance: "adr-0092:bootstrap:2026-05-21" +""" + with pytest.raises(ReviewerRegistryError, match="unknown review_scope"): + load_reviewer_registry(_write_yaml(tmp_path, yaml)) + + def test_empty_review_scope(self, tmp_path: Path) -> None: + yaml = """\ +schema_version: 1 +reviewers: + - reviewer_id: shay-j + display_name: "Joshua Shay" + role: primary + domains: ["*"] + review_scope: [] + provenance: "adr-0092:bootstrap:2026-05-21" +""" + with pytest.raises(ReviewerRegistryError, match="empty 'review_scope'"): + load_reviewer_registry(_write_yaml(tmp_path, yaml)) + + def test_duplicate_reviewer_id(self, tmp_path: Path) -> None: + yaml = """\ +schema_version: 1 +reviewers: + - reviewer_id: shay-j + display_name: "Joshua Shay" + role: primary + domains: ["*"] + review_scope: ["pack"] + provenance: "adr-0092:bootstrap:2026-05-21" + - reviewer_id: shay-j + display_name: "Imposter" + role: primary + domains: ["*"] + review_scope: ["pack"] + provenance: "adr-0092:bootstrap:2026-05-21" +""" + with pytest.raises(ReviewerRegistryError, match="duplicate reviewer_id"): + load_reviewer_registry(_write_yaml(tmp_path, yaml)) + + def test_malformed_yaml(self, tmp_path: Path) -> None: + path = _write_yaml(tmp_path, "schema_version: 1\nreviewers: [\n") + with pytest.raises(ReviewerRegistryError, match="YAML parse error"): + load_reviewer_registry(path) + + +class TestProductionRegistry: + """Verify the in-tree docs/reviewers.yaml passes ADR-0092 v1 schema.""" + + def test_production_registry_loads(self) -> None: + repo_root = Path(__file__).resolve().parent.parent + registry = load_reviewer_registry(repo_root / "docs" / "reviewers.yaml") + assert registry.schema_version == SCHEMA_VERSION + assert len(registry.reviewers) >= 1 + ids = [r.reviewer_id for r in registry.reviewers] + assert "shay-j" in ids + + def test_production_bootstrap_reviewer_is_primary(self) -> None: + repo_root = Path(__file__).resolve().parent.parent + registry = load_reviewer_registry(repo_root / "docs" / "reviewers.yaml") + shay = registry.resolve("shay-j") + assert shay is not None + assert shay.role == "primary" + assert shay.domains == ("*",) + assert shay.provenance.startswith("adr-0092:bootstrap:")