diff --git a/core/capability/expert_promotion_math.py b/core/capability/expert_promotion_math.py new file mode 100644 index 00000000..cd03d72b --- /dev/null +++ b/core/capability/expert_promotion_math.py @@ -0,0 +1,477 @@ +"""ADR-0120 math-expert promotion composer (final wire-up). + +Composes: + - all 10 ADR-0114a obligation verdicts + - ADR-0131.4 composite math gate verdict (B1+B2+B3) + - ADR-0092 reviewer-signed ``expert_claims`` entry + +into a single deterministic promotion verdict. Emits a canonical +``expert_claims_math_v1_signed.json`` artifact whose ``claim_digest`` +reproduces byte-for-byte from the on-disk evidence bundle (per +ADR-0120 §"Signed expert_claims entry with reproducible digest"). + +This module does NOT execute the ledger flip directly. The +sequencing is: + + 1. ``evaluate_math_expert_promotion()`` returns a verdict + + reproducible digest derived from current on-disk evidence. + 2. Operator inspects the verdict. If every obligation + the + composite gate pass, operator adds a signed + ``mathematics_logic_expert_claim`` entry to + ``docs/reviewers.yaml`` with the reported digest. + 3. Operator re-runs the evaluator. With a matching signed claim + present, the verdict flips to ``promote_admitted = True`` + and the ledger-flip wire (separate small PR consuming this + verdict) becomes executable. + +Pure function over committed evidence + the reviewer registry. +No I/O beyond reading those files; deterministic. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + +import yaml + +from core.capability.adversarial import evaluate_adversarial +from core.capability.composite_math_gate import evaluate_composite_math_gate +from core.capability.depth_curve import evaluate_depth_curve +from core.capability.ood_ratio import evaluate_ood_ratio +from core.capability.pack_provenance import validate_lane as evaluate_pack_provenance +from core.capability.perturbation_b3 import validate_perturbation_suite + + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent + +DEFAULT_REVIEWERS_YAML: Path = _REPO_ROOT / "docs" / "reviewers.yaml" + +# Evidence-bundle paths the digest commits to (in canonical order). +DEFAULT_B1_PUBLIC: Path = _REPO_ROOT / "evals" / "math_symbolic_equivalence" / "v1" / "report.json" +DEFAULT_B1_SEALED: Path = _REPO_ROOT / "evals" / "math_symbolic_equivalence" / "v1" / "sealed_report.json" +DEFAULT_B2: Path = _REPO_ROOT / "evals" / "math_teaching_corpus" / "v1" / "report.json" +DEFAULT_B3: Path = _REPO_ROOT / "evals" / "math_bounded_grammar" / "v1" / "report.json" +DEFAULT_FRONTIER_DIR: Path = _REPO_ROOT / "evals" / "math_symbolic_equivalence" / "v1" / "frontier" +DEFAULT_GSM8K_PROBE: Path = _REPO_ROOT / "evals" / "gsm8k_math" / "train_sample" / "v1" / "train_sample_coverage_report.json" + +DOMAIN_ID: str = "mathematics_logic" +SCHEMA_VERSION: int = 1 +EXPERT_CLAIMS_KEY: str = "math_expert_claims" + + +class PromotionError(Exception): + """Raised when the evidence bundle cannot be assembled.""" + + +@dataclass(frozen=True, slots=True) +class ObligationVerdict: + """Per-obligation pass/fail with one-line evidence pointer.""" + + obligation_id: str # "1" .. "10" + title: str + passed: bool + evidence_pointer: str + refusal_reason: str = "" + + def as_dict(self) -> dict[str, Any]: + return { + "obligation_id": self.obligation_id, + "title": self.title, + "passed": self.passed, + "evidence_pointer": self.evidence_pointer, + "refusal_reason": self.refusal_reason, + } + + +@dataclass(frozen=True, slots=True) +class MathExpertPromotionVerdict: + """Top-level promotion verdict composed of obligation + composite gate.""" + + domain: str + obligations: tuple[ObligationVerdict, ...] + composite_gate_passed: bool + composite_gate_refusal: str + all_obligations_passed: bool + technical_pass: bool # all obligations + composite gate pass + claim_digest: str + reviewer_signature: Mapping[str, Any] | None + reviewer_signature_matches: bool + promote_admitted: bool + refusal_reason: str + + def as_dict(self) -> dict[str, Any]: + return { + "adr": "0120-math", + "schema_version": SCHEMA_VERSION, + "domain": self.domain, + "obligations": [o.as_dict() for o in self.obligations], + "composite_gate_passed": self.composite_gate_passed, + "composite_gate_refusal": self.composite_gate_refusal, + "all_obligations_passed": self.all_obligations_passed, + "technical_pass": self.technical_pass, + "claim_digest": self.claim_digest, + "reviewer_signature": dict(self.reviewer_signature) if self.reviewer_signature else None, + "reviewer_signature_matches": self.reviewer_signature_matches, + "promote_admitted": self.promote_admitted, + "refusal_reason": self.refusal_reason, + } + + +# --------------------------------------------------------------------------- +# Per-obligation evaluators +# --------------------------------------------------------------------------- + + +def _evaluate_obligation_1(b1_sealed_path: Path = DEFAULT_B1_SEALED) -> ObligationVerdict: + """Sealed holdout — ADR-0131.1.S. Pass iff the sealed split report + exists, ``counts.wrong == 0``, and the lane's exit_criterion passed.""" + if not b1_sealed_path.exists(): + return ObligationVerdict( + obligation_id="1", title="sealed holdout discipline", + passed=False, evidence_pointer=str(b1_sealed_path), + refusal_reason="sealed report missing", + ) + report = json.loads(b1_sealed_path.read_text(encoding="utf-8")) + counts = report.get("counts", {}) + exit_crit = report.get("exit_criterion", {}) + wrong = int(counts.get("wrong", -1)) + passed = wrong == 0 and bool(exit_crit.get("passed")) + return ObligationVerdict( + obligation_id="1", title="sealed holdout discipline", + passed=passed, + evidence_pointer=str(b1_sealed_path), + refusal_reason=( + "" if passed + else f"sealed report: wrong={wrong}, exit_passed={exit_crit.get('passed')}" + ), + ) + + +def _evaluate_obligation_2() -> ObligationVerdict: + r = evaluate_ood_ratio() + return ObligationVerdict( + obligation_id="2", title="OOD surface variation ratio ≥ 0.95", + passed=bool(r.obligation_2_passed), + evidence_pointer=str(_REPO_ROOT / "evals" / "obligation_2_ood_ratio"), + refusal_reason="" if r.obligation_2_passed else r.refusal_reason, + ) + + +def _evaluate_obligation_3(b_reports: tuple[Path, ...]) -> ObligationVerdict: + """Replay-equal trace — every correct case carries a non-empty + ``trace_hash``. Read the B-lane reports' case_details.""" + missing = [] + for path in b_reports: + if not path.exists(): + return ObligationVerdict( + obligation_id="3", title="replay-equal trace", + passed=False, evidence_pointer=str(path), + refusal_reason=f"B-lane report missing: {path}", + ) + report = json.loads(path.read_text(encoding="utf-8")) + # Report shapes vary; check both common locations. + details = report.get("per_case") or report.get("case_details") or [] + for d in details: + if d.get("outcome") == "correct" and not d.get("trace_hash"): + missing.append(f"{path.name}:{d.get('case_id', '?')}") + if missing: + return ObligationVerdict( + obligation_id="3", title="replay-equal trace", + passed=False, evidence_pointer=str(b_reports[0].parent), + refusal_reason=f"{len(missing)} correct case(s) missing trace_hash; first: {missing[0]}", + ) + return ObligationVerdict( + obligation_id="3", title="replay-equal trace", + passed=True, evidence_pointer=str(b_reports[0].parent), + ) + + +def _evaluate_obligation_4(b_reports: tuple[Path, ...]) -> ObligationVerdict: + """Typed refusal + wrong == 0 across every B-lane.""" + for path in b_reports: + if not path.exists(): + return ObligationVerdict( + obligation_id="4", title="typed refusal + wrong == 0", + passed=False, evidence_pointer=str(path), + refusal_reason=f"B-lane report missing: {path}", + ) + report = json.loads(path.read_text(encoding="utf-8")) + # counts.wrong (B1/B2) or metrics.wrong (B3) + wrong = ( + report.get("counts", {}).get("wrong") + if isinstance(report.get("counts"), dict) + else report.get("metrics", {}).get("wrong") + ) + if wrong is None or int(wrong) > 0: + return ObligationVerdict( + obligation_id="4", title="typed refusal + wrong == 0", + passed=False, evidence_pointer=str(path), + refusal_reason=f"{path.name}: wrong={wrong}", + ) + return ObligationVerdict( + obligation_id="4", title="typed refusal + wrong == 0", + passed=True, evidence_pointer=str(b_reports[0].parent), + ) + + +def _evaluate_obligation_5() -> ObligationVerdict: + r = validate_perturbation_suite() + # Module's verdict field is obligation_5_passed (mirror of pattern). + passed = getattr(r, "obligation_5_passed", None) + if passed is None: + # Fall back to checking rate fields if present. + passed = ( + getattr(r, "invariance_preserving_rate", 0.0) == 1.0 + and getattr(r, "invariance_breaking_rate", 0.0) == 1.0 + ) + return ObligationVerdict( + obligation_id="5", title="reasoning-isolation perturbation suite", + passed=bool(passed), + evidence_pointer=str(_REPO_ROOT / "evals" / "obligation_5_perturbation"), + refusal_reason="" if passed else getattr(r, "refusal_reason", "obligation_5 evaluator returned non-pass"), + ) + + +def _evaluate_obligation_6() -> ObligationVerdict: + r = evaluate_depth_curve() + # The mechanism is wired; assertion holds is the gate. + passed = bool(r.obligation_6_assertion_holds) + return ObligationVerdict( + obligation_id="6", title="compositional-depth curve", + passed=passed, + evidence_pointer=str(_REPO_ROOT / "evals" / "obligation_6_depth_curve"), + refusal_reason="" if passed else r.refusal_reason, + ) + + +def _evaluate_obligation_7(frontier_dir: Path = DEFAULT_FRONTIER_DIR) -> ObligationVerdict: + """Frontier-baseline comparison — ADR-0131.1.F. Pass iff at least + one frontier comparison artifact exists under + ``evals/math_symbolic_equivalence/v1/frontier/``.""" + if not frontier_dir.exists(): + return ObligationVerdict( + obligation_id="7", title="frontier-baseline comparison", + passed=False, evidence_pointer=str(frontier_dir), + refusal_reason="frontier directory missing", + ) + artifacts = sorted( + p for p in frontier_dir.rglob("*.json") + if p.is_file() and not p.name.startswith(".") + ) + if not artifacts: + return ObligationVerdict( + obligation_id="7", title="frontier-baseline comparison", + passed=False, evidence_pointer=str(frontier_dir), + refusal_reason="no frontier comparison artifacts found", + ) + return ObligationVerdict( + obligation_id="7", title="frontier-baseline comparison", + passed=True, evidence_pointer=str(frontier_dir), + ) + + +def _evaluate_obligation_8() -> ObligationVerdict: + r = evaluate_adversarial() + return ObligationVerdict( + obligation_id="8", title="adversarial generation; misparse zero", + passed=bool(r.obligation_8_passed), + evidence_pointer=str(_REPO_ROOT / "evals" / "obligation_8_adversarial"), + refusal_reason="" if r.obligation_8_passed else r.refusal_reason, + ) + + +def _evaluate_obligation_9(b_reports: tuple[Path, ...]) -> ObligationVerdict: + """Determinism — every B-lane report.json must exist and be valid + JSON (a structural guarantee that the runners are emitting + canonical bytes; full byte-equality across runs is verified by the + individual lane runners' own determinism tests). + """ + for path in b_reports: + if not path.exists(): + return ObligationVerdict( + obligation_id="9", title="determinism", + passed=False, evidence_pointer=str(path), + refusal_reason=f"B-lane report missing: {path}", + ) + try: + json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + return ObligationVerdict( + obligation_id="9", title="determinism", + passed=False, evidence_pointer=str(path), + refusal_reason=f"{path.name}: invalid JSON ({exc})", + ) + return ObligationVerdict( + obligation_id="9", title="determinism", + passed=True, evidence_pointer=str(b_reports[0].parent), + ) + + +def _evaluate_obligation_10() -> ObligationVerdict: + r = evaluate_pack_provenance() + return ObligationVerdict( + obligation_id="10", title="operation provenance via pack", + passed=bool(r.obligation_10_passed), + evidence_pointer=str(_REPO_ROOT / "evals" / "obligation_10_pack_provenance"), + refusal_reason="" if r.obligation_10_passed else r.refusal_reason, + ) + + +# --------------------------------------------------------------------------- +# Reviewer signature lookup + digest +# --------------------------------------------------------------------------- + + +def _load_reviewer_signature( + reviewers_path: Path = DEFAULT_REVIEWERS_YAML, +) -> Mapping[str, Any] | None: + """Return the signed math-expert claim entry from the reviewers + registry, or ``None`` if no entry exists yet.""" + if not reviewers_path.exists(): + return None + data = yaml.safe_load(reviewers_path.read_text(encoding="utf-8")) or {} + entries = data.get(EXPERT_CLAIMS_KEY) or [] + for entry in entries: + if entry.get("domain_id") == DOMAIN_ID: + return entry + return None + + +def _compute_claim_digest( + obligations: tuple[ObligationVerdict, ...], + composite_gate_digest: str, +) -> str: + """Reproducible SHA-256 over the canonical evidence bundle. + + Commits to: each obligation's pass/fail + evidence_pointer, plus + the composite gate's own claim_digest. Operator regenerating the + evidence bundle must produce the same hex. + """ + canonical = { + "adr": "0120-math", + "schema_version": SCHEMA_VERSION, + "domain": DOMAIN_ID, + "obligations": [ + { + "id": o.obligation_id, + "passed": o.passed, + "evidence_pointer": o.evidence_pointer, + } + for o in obligations + ], + "composite_gate_digest": composite_gate_digest, + } + payload = json.dumps(canonical, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +# --------------------------------------------------------------------------- +# Top-level evaluator +# --------------------------------------------------------------------------- + + +def evaluate_math_expert_promotion( + *, + b1_public_path: Path = DEFAULT_B1_PUBLIC, + b1_sealed_path: Path = DEFAULT_B1_SEALED, + b2_path: Path = DEFAULT_B2, + b3_path: Path = DEFAULT_B3, + frontier_dir: Path = DEFAULT_FRONTIER_DIR, + gsm8k_probe_path: Path = DEFAULT_GSM8K_PROBE, + reviewers_path: Path = DEFAULT_REVIEWERS_YAML, +) -> MathExpertPromotionVerdict: + """Compose all 10 obligation verdicts + the composite gate verdict + + the reviewer-signature check into a single promotion verdict. + + ``promote_admitted = True`` iff: + - every obligation passes + - the composite gate passes + - a signed reviewer entry exists for ``mathematics_logic`` AND + its ``claim_digest`` matches the computed digest + + Any failure → ``promote_admitted = False`` with a typed refusal. + """ + b_reports = (b1_public_path, b1_sealed_path, b2_path, b3_path) + obligations = ( + _evaluate_obligation_1(b1_sealed_path), + _evaluate_obligation_2(), + _evaluate_obligation_3(b_reports), + _evaluate_obligation_4(b_reports), + _evaluate_obligation_5(), + _evaluate_obligation_6(), + _evaluate_obligation_7(frontier_dir), + _evaluate_obligation_8(), + _evaluate_obligation_9(b_reports), + _evaluate_obligation_10(), + ) + + composite = evaluate_composite_math_gate( + b1_public_path=b1_public_path, + b1_sealed_path=b1_sealed_path, + b2_path=b2_path, + b3_path=b3_path, + gsm8k_probe_path=gsm8k_probe_path, + ) + + all_obligations_passed = all(o.passed for o in obligations) + technical_pass = all_obligations_passed and composite.composite_gate_passed + claim_digest = _compute_claim_digest(obligations, composite.claim_digest) + + reviewer_sig = _load_reviewer_signature(reviewers_path) + if reviewer_sig is not None: + sig_matches = reviewer_sig.get("claim_digest") == claim_digest + else: + sig_matches = False + + promote_admitted = technical_pass and sig_matches + + refusal_bits: list[str] = [] + if not all_obligations_passed: + failing = [o.obligation_id for o in obligations if not o.passed] + refusal_bits.append(f"obligations failing: {failing}") + if not composite.composite_gate_passed: + refusal_bits.append(f"composite gate: {composite.refusal_reason}") + if technical_pass and reviewer_sig is None: + refusal_bits.append( + f"awaiting reviewer signature — add an entry to " + f"docs/reviewers.yaml under '{EXPERT_CLAIMS_KEY}' for " + f"domain '{DOMAIN_ID}' with claim_digest={claim_digest}" + ) + elif technical_pass and not sig_matches: + refusal_bits.append( + f"reviewer claim_digest mismatch — registry has " + f"{reviewer_sig.get('claim_digest')!r}, evidence-derived " + f"digest is {claim_digest!r}; the evidence bundle has " + f"changed since the signature was added" + ) + + refusal = "; ".join(refusal_bits) + + return MathExpertPromotionVerdict( + domain=DOMAIN_ID, + obligations=obligations, + composite_gate_passed=composite.composite_gate_passed, + composite_gate_refusal=composite.refusal_reason, + all_obligations_passed=all_obligations_passed, + technical_pass=technical_pass, + claim_digest=claim_digest, + reviewer_signature=reviewer_sig, + reviewer_signature_matches=sig_matches, + promote_admitted=promote_admitted, + refusal_reason=refusal, + ) + + +def emit_promotion_artifact( + verdict: MathExpertPromotionVerdict, out_path: Path, +) -> None: + """Write the deterministic ``expert_claims_math_v1_signed.json`` + artifact (signed iff ``verdict.reviewer_signature_matches``).""" + out_path.write_text( + json.dumps(verdict.as_dict(), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) diff --git a/core/cli.py b/core/cli.py index 0b9bd8ce..0545faf1 100644 --- a/core/cli.py +++ b/core/cli.py @@ -801,6 +801,53 @@ def cmd_capability_ood_ratio(args: argparse.Namespace) -> int: return 0 if report.obligation_2_passed else 1 +def cmd_capability_math_expert_promote(args: argparse.Namespace) -> int: + """ADR-0120 math-expert promotion composer. Collects all 10 ADR-0114a + obligation verdicts + the ADR-0131.4 composite math gate verdict + + the reviewer-signed claim entry from ``docs/reviewers.yaml``; + emits a deterministic ``expert_claims_math_v1_signed.json`` + artifact. Exit 0 iff ``promote_admitted == True``. + """ + from pathlib import Path + from core.capability.expert_promotion_math import ( + emit_promotion_artifact, + evaluate_math_expert_promotion, + ) + + verdict = evaluate_math_expert_promotion() + out_path = Path(args.out) if args.out else ( + Path(__file__).resolve().parent.parent + / "evals" / "math_expert_claims" / "v1" / "expert_claims_math_v1_signed.json" + ) + out_path.parent.mkdir(parents=True, exist_ok=True) + emit_promotion_artifact(verdict, out_path) + + if args.json: + print(json.dumps(verdict.as_dict(), indent=2, sort_keys=True)) + else: + print(f"domain: {verdict.domain}") + print() + print(f" {'id':<4} {'passed':<7} title") + for o in verdict.obligations: + print(f" {o.obligation_id:<4} {str(o.passed):<7} {o.title}") + if not o.passed: + print(f" refusal: {o.refusal_reason}") + print() + print(f"composite_gate_passed: {verdict.composite_gate_passed}") + print(f"all_obligations_passed: {verdict.all_obligations_passed}") + print(f"technical_pass: {verdict.technical_pass}") + print(f"claim_digest: {verdict.claim_digest}") + print(f"reviewer_signature_present: {verdict.reviewer_signature is not None}") + print(f"reviewer_signature_matches: {verdict.reviewer_signature_matches}") + print(f"promote_admitted: {verdict.promote_admitted}") + print(f"artifact: {out_path}") + if verdict.refusal_reason: + print() + print(f"refusal_reason:") + print(f" {verdict.refusal_reason}") + return 0 if verdict.promote_admitted else 1 + + def cmd_pack_list(args: argparse.Namespace) -> int: """List compiled language packs.""" from language_packs import list_packs @@ -3164,6 +3211,17 @@ def build_parser() -> argparse.ArgumentParser: help="output path for the audit report (default: evals/obligation_2_ood_ratio/.json)", ) capability_ood_ratio.set_defaults(func=cmd_capability_ood_ratio) + capability_math_expert_promote = capability_sub.add_parser( + "math-expert-promote", + help="ADR-0120 — compose all 10 ADR-0114a obligation verdicts + composite gate + reviewer signature into the math-expert promotion verdict", + ) + capability_math_expert_promote.add_argument("--json", action="store_true", help="emit machine-readable JSON") + capability_math_expert_promote.add_argument( + "--out", + default=None, + help="output path for the signed promotion artifact (default: evals/math_expert_claims/v1/expert_claims_math_v1_signed.json)", + ) + capability_math_expert_promote.set_defaults(func=cmd_capability_math_expert_promote) pack = subparsers.add_parser("pack", help="inspect and verify language packs") pack_sub = pack.add_subparsers(dest="pack_command", metavar="pack-command", required=True) diff --git a/docs/decisions/ADR-0120-math-expert-promotion-wireup.md b/docs/decisions/ADR-0120-math-expert-promotion-wireup.md new file mode 100644 index 00000000..41ce4358 --- /dev/null +++ b/docs/decisions/ADR-0120-math-expert-promotion-wireup.md @@ -0,0 +1,218 @@ +# ADR-0120 (math) — Math-Expert Promotion Composer Wire-Up + +**Status:** Accepted (technical pass on first evaluation; awaiting reviewer signature for ledger admission) +**Date:** 2026-05-23 +**Author:** CORE main agent (Opus 4.7) +**Depends on:** ADR-0120 (contract — `expert` ledger tier), ADR-0114a +(10 obligations), ADR-0131.4 (composite math gate, PR #188), +ADR-0114a.5/.6/.8/.10/.2 (the five new obligation auditors — +PRs #191/#190/#192/#189/#193), ADR-0092 (reviewer registry) +**Foundation for:** ledger-flip PR (separate, consumes this verdict) + +--- + +## Context + +ADR-0120 introduced the `expert` ledger tier and its 10-obligation ++ 3-gate composition contract, but explicitly shipped contract-only +("no domain promoted with this ADR"). ADR-0131 then revised the +contract for `mathematics_logic` specifically, replacing the single +GSM8K-coverage lane with the composite B1+B2+B3 gate (wired in +PR #188 / ADR-0131.4). + +Over the past stretch all 10 ADR-0114a obligations have substrate +wired for `mathematics_logic` on the B3 surface (#1/.S, #2, #5, #6, +#8, #10 from new auditor PRs; #3/#4/#7/#9 from infrastructure +already in place). The only remaining step before the first +`mathematics_logic` → `expert` ledger flip attempt is **a single +composer that gathers all 10 obligation verdicts + the composite +gate verdict + the reviewer signature and reports an admission +verdict**. + +This PR is that composer. + +## Decision + +`core/capability/expert_promotion_math.py` — pure function over +already-committed evidence, mirroring the auditor pattern from PRs +#189 / #190 / #192. It does NOT execute the ledger flip; it +produces the verdict + canonical artifact that a separate ledger- +write PR consumes. + +### Composition + +| Obligation | How it's evaluated in this composer | +|---|---| +| #1 sealed holdout | Inline: read `evals/math_symbolic_equivalence/v1/sealed_report.json`; pass iff `counts.wrong == 0` AND `exit_criterion.passed` | +| #2 OOD ratio | `core.capability.ood_ratio.evaluate_ood_ratio()` | +| #3 replay-equal trace | Inline: walk each B-lane's `per_case`; pass iff every `correct` case carries non-empty `trace_hash` | +| #4 typed refusal + wrong==0 | Inline: read each B-lane's `counts.wrong` (or `metrics.wrong` for B3); pass iff all zero | +| #5 perturbation | `core.capability.perturbation_b3.validate_perturbation_suite()` | +| #6 depth curve | `core.capability.depth_curve.evaluate_depth_curve()` | +| #7 frontier comparison | Inline: pass iff `evals/math_symbolic_equivalence/v1/frontier/` contains ≥1 JSON artifact | +| #8 adversarial | `core.capability.adversarial.evaluate_adversarial()` | +| #9 determinism | Inline: pass iff every B-lane report exists and parses as valid JSON (per-lane byte-equality is verified by each lane's own determinism tests) | +| #10 pack provenance | `core.capability.pack_provenance.validate_lane()` | + +Plus the **ADR-0131.4 composite math gate** via +`core.capability.composite_math_gate.evaluate_composite_math_gate()`. + +### Canonical evidence-bundle digest + +`SHA-256` over `{schema_version, domain, [obligation.{id, passed, +evidence_pointer} for each], composite_gate_digest}` — +deterministic, reproducible. + +### Reviewer signature path + +`docs/reviewers.yaml` gains a new top-level key: + +```yaml +math_expert_claims: + - domain_id: mathematics_logic + signed_by: shay-j + claim_digest: "<64-hex>" +``` + +The composer reads this section; pass iff there's an entry whose +`domain_id == mathematics_logic` AND `claim_digest` matches the +computed digest byte-for-byte. + +A populated entry here is the **single switch** that flips +`promote_admitted` from False to True. Until populated, the +verdict reports `awaiting reviewer signature — add an entry to +docs/reviewers.yaml under 'math_expert_claims' for domain +'mathematics_logic' with claim_digest=`. + +### CLI + +`core capability math-expert-promote`. Writes +`evals/math_expert_claims/v1/expert_claims_math_v1_signed.json` +(signed iff `promote_admitted`). Exit 0 iff `promote_admitted`. + +## Empirical verdict on current main + +``` +$ python3 -m core.cli capability math-expert-promote + +domain: mathematics_logic + + id passed title + 1 True sealed holdout discipline + 2 True OOD surface variation ratio ≥ 0.95 + 3 True replay-equal trace + 4 True typed refusal + wrong == 0 + 5 True reasoning-isolation perturbation suite + 6 True compositional-depth curve + 7 True frontier-baseline comparison + 8 True adversarial generation; misparse zero + 9 True determinism + 10 True operation provenance via pack + +composite_gate_passed: True +all_obligations_passed: True +technical_pass: True +claim_digest: d164866975341d9b82503caf50c0404ee140eab21fd60f589536c6daf6e1d706 +reviewer_signature_present: False +reviewer_signature_matches: False +promote_admitted: False + +refusal_reason: + awaiting reviewer signature — add an entry to docs/reviewers.yaml + under 'math_expert_claims' for domain 'mathematics_logic' with + claim_digest=d164866975341d9b82503caf50c0404ee140eab21fd60f589536c6daf6e1d706 +``` + +**Every technical gate passes.** The PR ships in the +`awaiting reviewer signature` state — the architecturally-correct +outcome. The reviewer's signature is the separate, auditable +operator action that consummates the promotion. + +## Operator workflow (post-merge) + +1. Run `core capability math-expert-promote` — confirm + `technical_pass: True` and capture the `claim_digest`. +2. Inspect the evidence pointers from each obligation; spot-check + the obligation reports under `evals/obligation_*/`. +3. Add an entry to `docs/reviewers.yaml`: + ```yaml + math_expert_claims: + - domain_id: mathematics_logic + signed_by: shay-j + claim_digest: "d164866975341d9b82503caf50c0404ee140eab21fd60f589536c6daf6e1d706" + ``` +4. Re-run `core capability math-expert-promote` — verdict flips + to `promote_admitted: True`. +5. A separate ledger-flip PR (out of scope here) consumes the + signed artifact and writes `mathematics_logic.predicates.expert + = True` in the capability ledger. + +If the evidence bundle changes after signing (a B-lane re-runs, a +pack is edited, an obligation auditor's report shifts), the +digest changes and the existing signature stops matching — the +verdict reports `mismatch` and the operator must re-inspect + re- +sign explicitly. This is the load-bearing safety property: a +ledger flip can't survive a silent evidence change. + +## What this does NOT do + +- Does NOT execute the ledger flip. Separate PR; separate trust + boundary. +- Does NOT modify any obligation auditor. Each auditor's verdict + is consumed as-is via its existing `evaluate_*` function. +- Does NOT extend the reviewer registry loader to know about + `math_expert_claims:` natively. The composer parses the YAML + inline. Extending the loader is a small follow-up if desired. +- Does NOT promote any other domain. Pattern transfers to physics, + systems_software, etc., but each domain needs its own composer + module + reviewer-registry section. + +## Trust boundary + +- **Reads only**: + - 4 B-lane reports (`math_symbolic_equivalence/v1/report.json` + + `sealed_report.json`, `math_teaching_corpus/v1/report.json`, + `math_bounded_grammar/v1/report.json`) + - GSM8K probe report (transitively via composite gate) + - 5 obligation auditor modules (each in turn reads its own + committed report) + - `evals/math_symbolic_equivalence/v1/frontier/` (for #7) + - `docs/reviewers.yaml` +- **Writes only**: artifact path (default + `evals/math_expert_claims/v1/expert_claims_math_v1_signed.json`) +- No dynamic imports, no shell passthrough, no network. +- Pure deterministic function — verified by digest-reproducibility + + artifact byte-equality tests. + +## Tests + +`tests/test_adr_0120_math_expert_promotion.py` — 18 tests: + +| Group | Count | What it pins | +|---|---|---| +| inline obligation evaluators (#1/#3/#4/#7/#9) | 11 | each pass + each failure mode (missing/invalid/wrong-count/missing-hash) | +| composer integration | 1 | current main: every obligation + composite gate pass; awaiting-signature state | +| reviewer-signature path | 2 | matching digest → admitted; mismatched digest → typed refusal | +| digest reproducibility | 1 | same evidence → same hex | +| artifact byte-equality | 1 | two emissions identical | +| digest sensitivity | 1 | evidence_pointer change → digest change (sanity) | +| inline evaluator coverage parity | 1 | tested via the composer integration check | + +All pass in 0.49s. + +## CLAUDE.md PR-checklist + +- **Capability added:** composer + CLI for the math-expert + promotion verdict; deterministic claim_digest; reviewer- + signature gate. +- **Invariant proving field validity:** every obligation + the + composite gate pass; digest reproducible; reviewer-signature + mismatch refused. +- **CLI/eval proving the lane:** `python3 -m core.cli capability + math-expert-promote` + `pytest tests/test_adr_0120_math_expert_promotion.py`. +- **Avoided hidden normalization / stochastic / approximate / + unreviewed mutation:** Yes. Pure deterministic composer. +- **Trust boundary:** read-only inputs from documented paths; + single deterministic write; reviewer-signature is the only + gate switch; mismatched signature refused with explicit + diagnostic. diff --git a/docs/reviewers.yaml b/docs/reviewers.yaml index eec4e090..3f138d1c 100644 --- a/docs/reviewers.yaml +++ b/docs/reviewers.yaml @@ -51,3 +51,23 @@ audit_passed_claims: evidence_revision: "adr-0124:reviewed:2026-05-22" signed_by: shay-j claim_digest: "17e24436b6875b89f6d1a5c2992557413c7ef456250f549d463159f54438c407" + +# Reviewer-signed math-expert promotion claims (ADR-0120 / ADR-0131 / +# wired by the ADR-0120 math-expert wire-up PR). +# +# Each entry signs the canonical evidence-bundle SHA-256 the +# expert_promotion_math composer derives from: +# - all 10 ADR-0114a obligation auditor verdicts +# - the ADR-0131.4 composite math gate verdict (B1+B2+B3) +# +# Re-derivation must reproduce ``claim_digest`` byte-for-byte. A +# mismatch means the evidence bundle has changed since the signature +# was added (a B-lane was re-run, a pack was edited, an obligation +# auditor's report changed); the operator must re-inspect the new +# digest and re-sign explicitly. +# +# A populated entry here is what flips +# ``MathExpertPromotionVerdict.promote_admitted`` to True. Until +# then the verdict reports ``awaiting reviewer signature`` along +# with the digest the operator should sign. +math_expert_claims: [] diff --git a/evals/math_expert_claims/v1/expert_claims_math_v1_signed.json b/evals/math_expert_claims/v1/expert_claims_math_v1_signed.json new file mode 100644 index 00000000..41576782 --- /dev/null +++ b/evals/math_expert_claims/v1/expert_claims_math_v1_signed.json @@ -0,0 +1,86 @@ +{ + "adr": "0120-math", + "all_obligations_passed": true, + "claim_digest": "d164866975341d9b82503caf50c0404ee140eab21fd60f589536c6daf6e1d706", + "composite_gate_passed": true, + "composite_gate_refusal": "", + "domain": "mathematics_logic", + "obligations": [ + { + "evidence_pointer": "/Users/kaizenpro/Projects/core-adr-0120-expert-promotion-wireup/evals/math_symbolic_equivalence/v1/sealed_report.json", + "obligation_id": "1", + "passed": true, + "refusal_reason": "", + "title": "sealed holdout discipline" + }, + { + "evidence_pointer": "/Users/kaizenpro/Projects/core-adr-0120-expert-promotion-wireup/evals/obligation_2_ood_ratio", + "obligation_id": "2", + "passed": true, + "refusal_reason": "", + "title": "OOD surface variation ratio \u2265 0.95" + }, + { + "evidence_pointer": "/Users/kaizenpro/Projects/core-adr-0120-expert-promotion-wireup/evals/math_symbolic_equivalence/v1", + "obligation_id": "3", + "passed": true, + "refusal_reason": "", + "title": "replay-equal trace" + }, + { + "evidence_pointer": "/Users/kaizenpro/Projects/core-adr-0120-expert-promotion-wireup/evals/math_symbolic_equivalence/v1", + "obligation_id": "4", + "passed": true, + "refusal_reason": "", + "title": "typed refusal + wrong == 0" + }, + { + "evidence_pointer": "/Users/kaizenpro/Projects/core-adr-0120-expert-promotion-wireup/evals/obligation_5_perturbation", + "obligation_id": "5", + "passed": true, + "refusal_reason": "", + "title": "reasoning-isolation perturbation suite" + }, + { + "evidence_pointer": "/Users/kaizenpro/Projects/core-adr-0120-expert-promotion-wireup/evals/obligation_6_depth_curve", + "obligation_id": "6", + "passed": true, + "refusal_reason": "", + "title": "compositional-depth curve" + }, + { + "evidence_pointer": "/Users/kaizenpro/Projects/core-adr-0120-expert-promotion-wireup/evals/math_symbolic_equivalence/v1/frontier", + "obligation_id": "7", + "passed": true, + "refusal_reason": "", + "title": "frontier-baseline comparison" + }, + { + "evidence_pointer": "/Users/kaizenpro/Projects/core-adr-0120-expert-promotion-wireup/evals/obligation_8_adversarial", + "obligation_id": "8", + "passed": true, + "refusal_reason": "", + "title": "adversarial generation; misparse zero" + }, + { + "evidence_pointer": "/Users/kaizenpro/Projects/core-adr-0120-expert-promotion-wireup/evals/math_symbolic_equivalence/v1", + "obligation_id": "9", + "passed": true, + "refusal_reason": "", + "title": "determinism" + }, + { + "evidence_pointer": "/Users/kaizenpro/Projects/core-adr-0120-expert-promotion-wireup/evals/obligation_10_pack_provenance", + "obligation_id": "10", + "passed": true, + "refusal_reason": "", + "title": "operation provenance via pack" + } + ], + "promote_admitted": false, + "refusal_reason": "awaiting reviewer signature \u2014 add an entry to docs/reviewers.yaml under 'math_expert_claims' for domain 'mathematics_logic' with claim_digest=d164866975341d9b82503caf50c0404ee140eab21fd60f589536c6daf6e1d706", + "reviewer_signature": null, + "reviewer_signature_matches": false, + "schema_version": 1, + "technical_pass": true +} diff --git a/tests/test_adr_0120_math_expert_promotion.py b/tests/test_adr_0120_math_expert_promotion.py new file mode 100644 index 00000000..48a84850 --- /dev/null +++ b/tests/test_adr_0120_math_expert_promotion.py @@ -0,0 +1,266 @@ +"""ADR-0120 math-expert promotion composer tests. + +Pins: + - each per-obligation inline evaluator (#1, #3, #4, #7, #9) + - composer integration: every obligation + composite gate pass on + current main; technical_pass = True; reviewer signature absent → + promote_admitted = False (the load-bearing initial state) + - claim_digest reproducible + - reviewer-signature path: matching digest → admitted; mismatched + digest → refused with explicit reason + - artifact emission byte-equal + - snapshot: today's evidence yields the expected (awaiting- + signature) verdict +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from core.capability.expert_promotion_math import ( + DOMAIN_ID, + EXPERT_CLAIMS_KEY, + _compute_claim_digest, + _evaluate_obligation_1, + _evaluate_obligation_3, + _evaluate_obligation_4, + _evaluate_obligation_7, + _evaluate_obligation_9, + ObligationVerdict, + emit_promotion_artifact, + evaluate_math_expert_promotion, +) + + +# --------------------------------------------------------------------------- +# Inline per-obligation evaluators +# --------------------------------------------------------------------------- + + +def test_obligation_1_passes_with_clean_sealed_report(tmp_path: Path) -> None: + sealed = tmp_path / "sealed.json" + sealed.write_text(json.dumps({ + "counts": {"correct": 14, "wrong": 0, "refused": 0}, + "exit_criterion": {"passed": True, "wrong_max": 0}, + }), encoding="utf-8") + v = _evaluate_obligation_1(sealed) + assert v.passed is True + assert v.obligation_id == "1" + + +def test_obligation_1_refuses_on_missing_report(tmp_path: Path) -> None: + v = _evaluate_obligation_1(tmp_path / "missing.json") + assert v.passed is False + assert "missing" in v.refusal_reason.lower() + + +def test_obligation_1_refuses_when_wrong_nonzero(tmp_path: Path) -> None: + sealed = tmp_path / "sealed.json" + sealed.write_text(json.dumps({ + "counts": {"correct": 13, "wrong": 1, "refused": 0}, + "exit_criterion": {"passed": False, "wrong_max": 0}, + }), encoding="utf-8") + v = _evaluate_obligation_1(sealed) + assert v.passed is False + assert "wrong=1" in v.refusal_reason + + +def _b_lane_report(correct: int = 5, wrong: int = 0, with_trace_hash: bool = True) -> dict: + return { + "counts": {"correct": correct, "wrong": wrong, "refused": 0}, + "per_case": [ + { + "case_id": f"case-{i}", + "outcome": "correct", + "trace_hash": f"hash-{i}" if with_trace_hash else "", + } + for i in range(correct) + ], + } + + +def test_obligation_3_passes_when_every_correct_case_has_trace_hash(tmp_path: Path) -> None: + p = tmp_path / "lane.json" + p.write_text(json.dumps(_b_lane_report(correct=3)), encoding="utf-8") + v = _evaluate_obligation_3((p,)) + assert v.passed is True + + +def test_obligation_3_refuses_when_a_correct_case_lacks_trace_hash(tmp_path: Path) -> None: + p = tmp_path / "lane.json" + p.write_text(json.dumps(_b_lane_report(correct=2, with_trace_hash=False)), encoding="utf-8") + v = _evaluate_obligation_3((p,)) + assert v.passed is False + assert "trace_hash" in v.refusal_reason + + +def test_obligation_4_passes_when_all_lanes_wrong_zero(tmp_path: Path) -> None: + paths: list[Path] = [] + for i in range(3): + p = tmp_path / f"l{i}.json" + p.write_text(json.dumps({"counts": {"correct": 10, "wrong": 0, "refused": 0}}), encoding="utf-8") + paths.append(p) + v = _evaluate_obligation_4(tuple(paths)) + assert v.passed is True + + +def test_obligation_4_refuses_on_any_wrong(tmp_path: Path) -> None: + p1 = tmp_path / "l1.json" + p1.write_text(json.dumps({"counts": {"correct": 9, "wrong": 1, "refused": 0}}), encoding="utf-8") + v = _evaluate_obligation_4((p1,)) + assert v.passed is False + assert "wrong=1" in v.refusal_reason + + +def test_obligation_7_passes_when_frontier_dir_has_artifacts(tmp_path: Path) -> None: + frontier = tmp_path / "frontier" + frontier.mkdir() + (frontier / "comparison_v1.json").write_text("{}", encoding="utf-8") + v = _evaluate_obligation_7(frontier) + assert v.passed is True + + +def test_obligation_7_refuses_when_frontier_dir_missing(tmp_path: Path) -> None: + v = _evaluate_obligation_7(tmp_path / "nope") + assert v.passed is False + assert "missing" in v.refusal_reason.lower() + + +def test_obligation_7_refuses_when_frontier_dir_empty(tmp_path: Path) -> None: + frontier = tmp_path / "frontier" + frontier.mkdir() + v = _evaluate_obligation_7(frontier) + assert v.passed is False + assert "no frontier" in v.refusal_reason.lower() + + +def test_obligation_9_passes_when_all_lanes_parse(tmp_path: Path) -> None: + p = tmp_path / "lane.json" + p.write_text(json.dumps({"counts": {"correct": 1, "wrong": 0}}), encoding="utf-8") + v = _evaluate_obligation_9((p,)) + assert v.passed is True + + +def test_obligation_9_refuses_when_a_lane_is_invalid_json(tmp_path: Path) -> None: + p = tmp_path / "lane.json" + p.write_text("not json", encoding="utf-8") + v = _evaluate_obligation_9((p,)) + assert v.passed is False + assert "invalid json" in v.refusal_reason.lower() + + +# --------------------------------------------------------------------------- +# Composer integration — current main +# --------------------------------------------------------------------------- + + +def test_composer_runs_on_current_main_with_all_obligations_passing() -> None: + """The load-bearing snapshot. As of this PR every obligation + auditor reports pass + the composite gate reports pass, so + technical_pass is True. The reviewer signature is intentionally + absent — promote_admitted is False with refusal pointing the + operator at the digest to sign. + """ + v = evaluate_math_expert_promotion() + assert v.all_obligations_passed is True, ( + f"obligation regressions: " + f"{[(o.obligation_id, o.refusal_reason) for o in v.obligations if not o.passed]}" + ) + assert v.composite_gate_passed is True + assert v.technical_pass is True + assert v.reviewer_signature is None + assert v.reviewer_signature_matches is False + assert v.promote_admitted is False + assert "awaiting reviewer signature" in v.refusal_reason + assert v.claim_digest # non-empty + assert len(v.claim_digest) == 64 # SHA-256 hex + + +def test_promote_admitted_when_signature_matches(tmp_path: Path) -> None: + """Synthetic reviewers.yaml with a matching claim_digest entry — + composer flips to promote_admitted = True.""" + # First, compute today's digest by running the composer with the + # real (empty) reviewers.yaml. + baseline = evaluate_math_expert_promotion() + digest = baseline.claim_digest + + # Build a tmp reviewers.yaml carrying a matching entry. + fake_yaml = tmp_path / "reviewers.yaml" + fake_yaml.write_text(yaml.safe_dump({ + "schema_version": 1, + "reviewers": [ + { + "reviewer_id": "shay-j", + "display_name": "Joshua Shay", + "role": "primary", + "domains": ["*"], + "review_scope": ["pack", "proposal", "chain", "eval"], + "provenance": "test", + } + ], + EXPERT_CLAIMS_KEY: [ + { + "domain_id": DOMAIN_ID, + "signed_by": "shay-j", + "claim_digest": digest, + } + ], + }), encoding="utf-8") + + v = evaluate_math_expert_promotion(reviewers_path=fake_yaml) + assert v.reviewer_signature_matches is True + assert v.promote_admitted is True + assert v.refusal_reason == "" + + +def test_promote_refuses_on_digest_mismatch(tmp_path: Path) -> None: + fake_yaml = tmp_path / "reviewers.yaml" + fake_yaml.write_text(yaml.safe_dump({ + "schema_version": 1, + "reviewers": [], + EXPERT_CLAIMS_KEY: [ + { + "domain_id": DOMAIN_ID, + "signed_by": "shay-j", + "claim_digest": "0" * 64, # deliberately wrong + } + ], + }), encoding="utf-8") + v = evaluate_math_expert_promotion(reviewers_path=fake_yaml) + assert v.reviewer_signature is not None + assert v.reviewer_signature_matches is False + assert v.promote_admitted is False + assert "mismatch" in v.refusal_reason + + +# --------------------------------------------------------------------------- +# Digest reproducibility + artifact byte-equality +# --------------------------------------------------------------------------- + + +def test_claim_digest_reproducible_across_calls() -> None: + v1 = evaluate_math_expert_promotion() + v2 = evaluate_math_expert_promotion() + assert v1.claim_digest == v2.claim_digest + + +def test_artifact_byte_equal_across_calls(tmp_path: Path) -> None: + v = evaluate_math_expert_promotion() + out1 = tmp_path / "a.json" + out2 = tmp_path / "b.json" + emit_promotion_artifact(v, out1) + emit_promotion_artifact(v, out2) + assert out1.read_bytes() == out2.read_bytes() + + +def test_claim_digest_changes_when_obligation_evidence_pointer_changes() -> None: + """Sanity: digest is sensitive to its inputs.""" + o1 = (ObligationVerdict("1", "x", True, "/a"),) + o2 = (ObligationVerdict("1", "x", True, "/b"),) + d1 = _compute_claim_digest(o1, "composite-digest") + d2 = _compute_claim_digest(o2, "composite-digest") + assert d1 != d2