diff --git a/core/capability/composite_math_gate.py b/core/capability/composite_math_gate.py new file mode 100644 index 00000000..f1b9b1b1 --- /dev/null +++ b/core/capability/composite_math_gate.py @@ -0,0 +1,346 @@ +"""ADR-0131.4 — Composite math-expert promotion gate (Benchmark 1+2+3). + +Implements the *math-specific* revision of the ``expert`` promotion +contract that ADR-0131 introduces. The full ADR-0120 contract (10 +ADR-0114a obligations + 3 composition gates) is NOT implemented +here — most of its substrate (perturbation, depth curve, adversarial, +OOD ratio) is not yet built for ``mathematics_logic``. This module +implements only the load-bearing piece ADR-0131 changes: + + ADR-0120 single-benchmark check + `correct_rate ≥ 0.60` on public AND sealed GSM8K + --------------------------------------------------- REPLACED BY ---- + ADR-0131 composite check (this module) + Benchmark 1 (math_symbolic_equivalence/v1): public + sealed, + each ``correct_rate ≥ 0.95 AND wrong == 0`` + Benchmark 2 (math_teaching_corpus/v1): + ``correct_rate ≥ 0.95 AND wrong == 0`` + Benchmark 3 (math_bounded_grammar/v1): + ``correct_rate ≥ 0.95 AND wrong == 0`` + +Plus honest disclosure: the GSM8K coverage probe's +``admitted_solved`` / ``admitted_wrong`` are reported in the +``expert_claims`` artifact under ``honest_disclosure`` — they do NOT +gate, per ADR-0131 ("GSM8K is retained as a stress-test lane that +the math expert runs but is not gated on. ... reported in the +expert-claims artifact as honest disclosure"). + +The gate is a pure function over already-committed lane report +JSON. No I/O beyond reading those reports. Deterministic — same +inputs produce byte-equal verdict (after sort_keys + indent=2 +serialization). + +This module composes with — but does not implement — the broader +ADR-0120 contract. A future ``core/capability/expert_promotion.py`` +that implements all 10 ADR-0114a obligations will consume +:func:`evaluate_composite_math_gate` as the math-specific +substitute for the single-lane coverage check. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Mapping + + +# Thresholds pinned by ADR-0131's "Composite expert-promotion gate" +# table. Changing them requires a new ADR amending ADR-0131. +CORRECT_RATE_MIN: float = 0.95 +WRONG_MAX: int = 0 + + +# Repository root inferred from this module's location: +# core/capability/composite_math_gate.py -> ../../. +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent + +# Default benchmark report paths. The evaluator accepts override +# paths so tests can point at fixture data without touching main's +# committed artifacts. +DEFAULT_B1_PUBLIC: Path = _REPO_ROOT / "evals" / "math_symbolic_equivalence" / "v1" / "report.json" +DEFAULT_B1_SEALED: Path = _REPO_ROOT / "evals" / "math_symbolic_equivalence" / "v1" / "sealed_report.json" +DEFAULT_B2: Path = _REPO_ROOT / "evals" / "math_teaching_corpus" / "v1" / "report.json" +DEFAULT_B3: Path = _REPO_ROOT / "evals" / "math_bounded_grammar" / "v1" / "report.json" +DEFAULT_GSM8K_PROBE: Path = _REPO_ROOT / "evals" / "gsm8k_math" / "train_sample" / "v1" / "train_sample_coverage_report.json" + + +class CompositeMathGateError(Exception): + """Raised when a benchmark report cannot be located, parsed, or + canonicalized into the shape this gate consumes.""" + + +@dataclass(frozen=True, slots=True) +class BenchmarkVerdict: + """Single-benchmark gate outcome.""" + + benchmark_id: str + report_path: str + correct: int + wrong: int + cases_total: int + correct_rate: float + correct_rate_passes: bool + wrong_count_is_zero: bool + passed: bool + refusal_reason: str = "" + + def as_dict(self) -> dict[str, Any]: + return { + "benchmark_id": self.benchmark_id, + "report_path": self.report_path, + "correct": self.correct, + "wrong": self.wrong, + "cases_total": self.cases_total, + "correct_rate": self.correct_rate, + "correct_rate_passes": self.correct_rate_passes, + "wrong_count_is_zero": self.wrong_count_is_zero, + "passed": self.passed, + "refusal_reason": self.refusal_reason, + } + + +@dataclass(frozen=True, slots=True) +class CompositeMathGateVerdict: + """Aggregate composite-gate outcome.""" + + domain: str + benchmarks: tuple[BenchmarkVerdict, ...] + all_benchmarks_passed: bool + composite_gate_passed: bool + honest_disclosure: Mapping[str, Any] + claim_digest: str + thresholds: Mapping[str, Any] = field(default_factory=dict) + refusal_reason: str = "" + + def as_dict(self) -> dict[str, Any]: + return { + "adr": "0131.4", + "schema_version": 1, + "domain": self.domain, + "thresholds": dict(self.thresholds), + "benchmarks": [b.as_dict() for b in self.benchmarks], + "all_benchmarks_passed": self.all_benchmarks_passed, + "composite_gate_passed": self.composite_gate_passed, + "honest_disclosure": dict(self.honest_disclosure), + "claim_digest": self.claim_digest, + "refusal_reason": self.refusal_reason, + } + + +def _read_report(path: Path) -> dict[str, Any]: + if not path.exists(): + raise CompositeMathGateError( + f"benchmark report missing: {path}" + ) + try: + return json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise CompositeMathGateError( + f"benchmark report not valid JSON: {path}: {exc}" + ) from exc + + +def _extract_counts(report: dict[str, Any]) -> tuple[int, int, int]: + """Return ``(correct, wrong, cases_total)`` from a heterogeneous + lane report. + + B1 (public + sealed) and B2 emit ``counts: {correct, wrong, refused}`` + and ``sample_count`` (B1 sealed + B2) or ``counts`` totalable + (B1 public). B3 emits ``metrics: {correct, wrong, cases_total}``. + Refuse to extract if the shape is neither — fail loudly rather than + guess. + """ + counts = report.get("counts") + metrics = report.get("metrics") + if isinstance(counts, dict) and {"correct", "wrong"}.issubset(counts.keys()): + correct = int(counts["correct"]) + wrong = int(counts["wrong"]) + # cases_total: sum of correct + wrong + refused, or sample_count + # if available. + refused = int(counts.get("refused", 0)) + cases_total = correct + wrong + refused + if "sample_count" in report and cases_total != int(report["sample_count"]): + # Trust sample_count when present (B1 sealed/B2 ship it). + cases_total = int(report["sample_count"]) + return correct, wrong, cases_total + if isinstance(metrics, dict) and {"correct", "wrong"}.issubset(metrics.keys()): + return ( + int(metrics["correct"]), + int(metrics["wrong"]), + int(metrics.get("cases_total", metrics["correct"] + metrics["wrong"])), + ) + raise CompositeMathGateError( + "benchmark report has neither counts{correct,wrong} nor " + "metrics{correct,wrong} at the top level" + ) + + +def _extract_correct_rate(report: dict[str, Any]) -> float: + """Top-level ``correct_rate`` first, then ``metrics.correct_rate``, + then derive from counts.""" + if "correct_rate" in report: + return float(report["correct_rate"]) + metrics = report.get("metrics") + if isinstance(metrics, dict) and "correct_rate" in metrics: + return float(metrics["correct_rate"]) + correct, wrong, total = _extract_counts(report) + if total == 0: + return 0.0 + return correct / total + + +def _evaluate_one(benchmark_id: str, path: Path) -> BenchmarkVerdict: + try: + report = _read_report(path) + correct, wrong, total = _extract_counts(report) + rate = _extract_correct_rate(report) + except CompositeMathGateError as exc: + return BenchmarkVerdict( + benchmark_id=benchmark_id, + report_path=str(path), + correct=0, + wrong=0, + cases_total=0, + correct_rate=0.0, + correct_rate_passes=False, + wrong_count_is_zero=False, + passed=False, + refusal_reason=str(exc), + ) + rate_passes = rate >= CORRECT_RATE_MIN + wrong_zero = wrong <= WRONG_MAX + return BenchmarkVerdict( + benchmark_id=benchmark_id, + report_path=str(path), + correct=correct, + wrong=wrong, + cases_total=total, + correct_rate=rate, + correct_rate_passes=rate_passes, + wrong_count_is_zero=wrong_zero, + passed=rate_passes and wrong_zero, + refusal_reason=( + "" if (rate_passes and wrong_zero) + else f"correct_rate={rate:.4f} (min {CORRECT_RATE_MIN}), wrong={wrong} (max {WRONG_MAX})" + ), + ) + + +def _gsm8k_honest_disclosure(path: Path) -> dict[str, Any]: + """Read the GSM8K coverage probe report and return the + honest-disclosure subset. Per ADR-0131, GSM8K is reported but + does NOT gate. + """ + if not path.exists(): + return { + "probe_path": str(path), + "available": False, + "note": "probe report missing; honest disclosure unavailable", + } + try: + report = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + return { + "probe_path": str(path), + "available": False, + "note": f"probe report unparseable: {exc}", + } + metrics = report.get("metrics", {}) + return { + "probe_path": str(path), + "available": True, + "admitted_solved": metrics.get("admitted_solved", 0), + "admitted_wrong": metrics.get("admitted_wrong", 0), + "refused": metrics.get("refused", 0), + "cases_total": metrics.get("cases_total", 0), + "admission_rate": metrics.get("admission_rate", 0.0), + "safety_rail_intact": metrics.get("safety_rail_intact", False), + "substrate": report.get("substrate", "legacy"), + } + + +def _compute_claim_digest( + benchmarks: tuple[BenchmarkVerdict, ...], + honest_disclosure: Mapping[str, Any], +) -> str: + """Reproducible SHA-256 over the canonical evidence bundle. + + Per ADR-0120's "Signed expert_claims entry with reproducible + digest" requirement — every reviewer should be able to compute + the same digest from the same lane reports. + """ + canonical = { + "schema_version": 1, + "adr": "0131.4", + "benchmarks": [b.as_dict() for b in benchmarks], + "honest_disclosure": dict(honest_disclosure), + "thresholds": { + "correct_rate_min": CORRECT_RATE_MIN, + "wrong_max": WRONG_MAX, + }, + } + payload = json.dumps(canonical, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def evaluate_composite_math_gate( + *, + b1_public_path: Path = DEFAULT_B1_PUBLIC, + b1_sealed_path: Path = DEFAULT_B1_SEALED, + b2_path: Path = DEFAULT_B2, + b3_path: Path = DEFAULT_B3, + gsm8k_probe_path: Path = DEFAULT_GSM8K_PROBE, +) -> CompositeMathGateVerdict: + """Evaluate the ADR-0131 composite math-expert gate. + + Returns a :class:`CompositeMathGateVerdict` with per-benchmark + breakdown, the composite verdict, GSM8K honest-disclosure, and + a reproducible claim digest. Pure function over file paths; no + side effects. + """ + benchmarks = ( + _evaluate_one("B1_public", b1_public_path), + _evaluate_one("B1_sealed", b1_sealed_path), + _evaluate_one("B2_teaching_corpus", b2_path), + _evaluate_one("B3_bounded_grammar", b3_path), + ) + all_passed = all(b.passed for b in benchmarks) + honest = _gsm8k_honest_disclosure(gsm8k_probe_path) + digest = _compute_claim_digest(benchmarks, honest) + refusal = "" + if not all_passed: + failing = [b.benchmark_id for b in benchmarks if not b.passed] + refusal = f"benchmarks failing the gate: {failing}" + return CompositeMathGateVerdict( + domain="mathematics_logic", + benchmarks=benchmarks, + all_benchmarks_passed=all_passed, + composite_gate_passed=all_passed, + honest_disclosure=honest, + claim_digest=digest, + thresholds={ + "correct_rate_min": CORRECT_RATE_MIN, + "wrong_max": WRONG_MAX, + }, + refusal_reason=refusal, + ) + + +def emit_expert_claims_artifact( + verdict: CompositeMathGateVerdict, + out_path: Path, +) -> None: + """Write the deterministic ``expert_claims`` artifact. + + Per ADR-0120: "Signed expert_claims entry with reproducible + digest". This emits the unsigned artifact; reviewer signature + is added by a separate ADR-0092 reviewer-registry path that + is out of scope for ADR-0131.4. + """ + payload = verdict.as_dict() + out_path.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) diff --git a/core/cli.py b/core/cli.py index 3aa6474c..6e96a7b9 100644 --- a/core/cli.py +++ b/core/cli.py @@ -538,6 +538,49 @@ def cmd_capability_evidence_plan(args: argparse.Namespace) -> int: return 0 +def cmd_capability_math_expert_gate(args: argparse.Namespace) -> int: + """ADR-0131.4 — evaluate the composite math-expert promotion gate + (Benchmark 1 + 2 + 3, ADR-0131's revision of ADR-0120's single-lane + coverage check). Emits ``expert_claims_math_v1.json`` to ``--out`` + (default: ``evals/math_expert_claims/v1/expert_claims_math_v1.json``). + Exit 0 iff every benchmark passes.""" + from pathlib import Path + from core.capability.composite_math_gate import ( + emit_expert_claims_artifact, + evaluate_composite_math_gate, + ) + + verdict = evaluate_composite_math_gate() + out_path = Path(args.out) if args.out else ( + Path(__file__).resolve().parent.parent + / "evals" / "math_expert_claims" / "v1" / "expert_claims_math_v1.json" + ) + out_path.parent.mkdir(parents=True, exist_ok=True) + emit_expert_claims_artifact(verdict, out_path) + + if args.json: + print(json.dumps(verdict.as_dict(), indent=2, sort_keys=True)) + else: + print(f"composite_gate_passed: {verdict.composite_gate_passed}") + print(f"claim_digest: {verdict.claim_digest}") + print(f"artifact: {out_path}") + for b in verdict.benchmarks: + print( + f" {b.benchmark_id:>20} passed={b.passed} " + f"correct={b.correct}/{b.cases_total} wrong={b.wrong} " + f"rate={b.correct_rate:.4f}" + ) + hd = verdict.honest_disclosure + print( + f"GSM8K honest disclosure: admission={hd.get('admitted_solved', 0)}/" + f"{hd.get('cases_total', 0)}, wrong={hd.get('admitted_wrong', 0)}, " + f"substrate={hd.get('substrate', '?')}" + ) + if not verdict.composite_gate_passed: + print(f"refusal_reason: {verdict.refusal_reason}") + return 0 if verdict.composite_gate_passed else 1 + + def cmd_pack_list(args: argparse.Namespace) -> int: """List compiled language packs.""" from language_packs import list_packs @@ -2832,6 +2875,17 @@ def build_parser() -> argparse.ArgumentParser: ) capability_evidence_plan.add_argument("--json", action="store_true", help="emit machine-readable JSON") capability_evidence_plan.set_defaults(func=cmd_capability_evidence_plan) + capability_math_expert_gate = capability_sub.add_parser( + "math-expert-gate", + help="ADR-0131.4 evaluate the composite math-expert promotion gate (B1+B2+B3)", + ) + capability_math_expert_gate.add_argument("--json", action="store_true", help="emit machine-readable JSON") + capability_math_expert_gate.add_argument( + "--out", + default=None, + help="output path for expert_claims artifact (default: evals/math_expert_claims/v1/expert_claims_math_v1.json)", + ) + capability_math_expert_gate.set_defaults(func=cmd_capability_math_expert_gate) pack = subparsers.add_parser("pack", help="inspect and verify language packs") pack_sub = pack.add_subparsers(dest="pack_command", metavar="pack-command", required=True) diff --git a/docs/decisions/ADR-0131.4-composite-math-gate.md b/docs/decisions/ADR-0131.4-composite-math-gate.md new file mode 100644 index 00000000..10e5aecb --- /dev/null +++ b/docs/decisions/ADR-0131.4-composite-math-gate.md @@ -0,0 +1,182 @@ +# ADR-0131.4 — Composite Math-Expert Promotion Gate (wired) + +**Status:** Accepted +**Date:** 2026-05-23 +**Author:** CORE main agent (Opus 4.7) +**Depends on:** ADR-0131 (composite gate framing), +ADR-0131.1 (B1 substrate), ADR-0131.1.S (B1 sealed holdout), +ADR-0131.2 (B2 substrate), ADR-0131.3 (B3 substrate), +ADR-0131.G.0 (probe substrate) +**Parent:** ADR-0131 +**Reserved follow-up:** the full ADR-0120 contract (9 ADR-0114a +obligations not implemented yet for `mathematics_logic`). + +--- + +## Context + +ADR-0131 introduced the composite math-expert promotion contract: + +> The `mathematics_logic` domain `expert` promotion contract is +> revised. The ADR-0120 single-benchmark check +> (`correct_rate ≥ 0.60` on GSM8K) is replaced by a composite +> requirement: B1 ≥ 0.95 AND B2 ≥ 0.95 AND B3 ≥ 0.95, each with +> `wrong == 0`. GSM8K is retained as a stress-test lane that the +> math expert runs but is not gated on — reported as honest +> disclosure. + +ADR-0131's implementation plan (sub-phase `0131.4`) named +`formation/ratify.py` + `formation/promote.py` as the wire-up +points. That was a misidentification: those modules govern the +SPECULATIVE → COHERENT bridge for individual teaching examples +(ADR-0021), not domain-tier expert promotion. The correct site is +`core/capability/`, where `expert_demo.py` (the audit-passed gate) +already lives and where ADR-0120 reserves `expert_promotion.py`. + +A full ADR-0120 implementation requires substrate for 10 ADR-0114a +obligations (sealed holdout, OOD ratio, replay determinism, typed +refusal + `wrong == 0`, perturbation, depth curve, frontier +comparison, adversarial, byte-equal lane runner, operation +provenance via pack). For `mathematics_logic`, only **5 of those +10 obligations** have substrate landed today: +- #1 sealed holdout (ADR-0131.1.S — B1 sealed) +- #3 replay-equal trace (B1/B2/B3 runners emit `trace_hash`) +- #4 typed refusal + `wrong == 0` (the load-bearing invariant + every G. and B-lane has been preserving) +- #7 frontier-baseline comparison (ADR-0131.1.F) +- #9 determinism (every B-lane report is byte-equal across runs) + +The other 5 (#2 OOD ratio, #5 perturbation, #6 depth curve, #8 +adversarial, #10 operation-provenance-via-pack) need +domain-specific substrate that isn't built yet. Implementing those +is sequencing-wise *after* ADR-0131.4, not bundled with it. + +## Decision + +Implement only the **ADR-0131-specific revision** as a focused +module: the composite B1+B2+B3 evaluator. Structure it so a future +`core/capability/expert_promotion.py` implementing the full +ADR-0120 contract can consume it as the math-specific substitute +for the single-lane coverage check. + +### What this ADR wires + +`core/capability/composite_math_gate.py`: + +```python +def evaluate_composite_math_gate( + *, b1_public_path, b1_sealed_path, b2_path, b3_path, gsm8k_probe_path, +) -> CompositeMathGateVerdict +``` + +- Reads each benchmark's already-committed `report.json` (no I/O + beyond that; no recomputation of lane verdicts). +- Handles the heterogeneous report shapes (B1/B2 use + `counts: {correct, wrong, refused}`; B3 uses `metrics: {correct, + wrong, cases_total, correct_rate}`). Refuses cleanly if the + shape is neither. +- Applies the pinned thresholds: `correct_rate ≥ 0.95` AND + `wrong == 0` per benchmark. +- Composes per-benchmark verdicts → composite verdict. +- Computes a reproducible SHA-256 claim digest over the canonical + evidence bundle (per ADR-0120 "Signed `expert_claims` entry with + reproducible digest" requirement). +- Emits GSM8K honest-disclosure (admission, wrong, refused, + substrate) but does NOT use it in the gate. + +CLI wiring at `core capability math-expert-gate` (added to +`core/cli.py`). Writes +`evals/math_expert_claims/v1/expert_claims_math_v1.json` (unsigned). +Reviewer signature via ADR-0092 reviewer-registry is reserved for +the broader ADR-0120 wire-up. + +### Empirical verdict + +Run against current `main` (post-PR #182/#183/#184/#185, pre-G.1): + +``` +composite_gate_passed: True +claim_digest: 2bfc7f6c5b06a4c5befef3a9a2629a023518ae97490f6ddf092c1852c966c275 + + B1_public passed=True correct=185/185 wrong=0 rate=1.0000 + B1_sealed passed=True correct=14/14 wrong=0 rate=1.0000 + B2_teaching_corpus passed=True correct=40/40 wrong=0 rate=1.0000 + B3_bounded_grammar passed=True correct=50/50 wrong=0 rate=1.0000 +GSM8K honest disclosure: admission=0/50, wrong=0, substrate=candidate_graph +``` + +**The math expert is gate-passing under ADR-0131's revised +composite contract.** The bet ADR-0131 placed — that the +architecture's structural strengths align with three benchmarks +that measure those strengths — has paid off on first evaluation. + +This does NOT mean the full ADR-0120 contract passes (5 of 10 +obligations still need substrate). It means the *math-specific +revision portion* of the contract passes today, and the +architecturally-aligned bet was correct. + +## What this does NOT do + +- Does NOT implement the broader ADR-0120 10-obligation contract. + That requires substrate for 5 missing obligations (OOD ratio, + perturbation, depth curve, adversarial, operation-provenance). +- Does NOT promote `mathematics_logic` to `expert` ledger status. + Ledger promotion is a separate ADR that consumes this gate's + output AND the broader ADR-0120 obligations. +- Does NOT sign the `expert_claims` artifact. Reviewer signature + via ADR-0092 registry is reserved for the full wire-up. +- Does NOT touch `formation/ratify.py` or `formation/promote.py` + (the teaching-example-tier modules ADR-0131's plan + misidentified). +- Does NOT recompute the lane verdicts. The committed B-lane + reports are the gate's input contract; the lane runners own + their verdicts and the gate trusts them. If a reviewer suspects + a lane report is stale, re-run the lane. + +## Trust boundary + +- Reads only: + - `evals/math_symbolic_equivalence/v1/report.json` + + `sealed_report.json` + - `evals/math_teaching_corpus/v1/report.json` + - `evals/math_bounded_grammar/v1/report.json` + - `evals/gsm8k_math/train_sample/v1/train_sample_coverage_report.json` + (disclosure-only) +- Writes only: the path passed to `emit_expert_claims_artifact` + (default: `evals/math_expert_claims/v1/expert_claims_math_v1.json`). +- No dynamic imports, no shell passthrough, no network. +- Pure function over already-committed JSON; deterministic + (verified by `test_claim_digest_reproducible` and + `test_artifact_emission_byte_equal`). + +## Tests + +`tests/test_adr_0131_4_composite_math_gate.py` — 12 tests: +- Threshold values are pinned (changing requires a new ADR) +- Both lane-report shapes parse correctly +- Gate passes iff all 4 benchmarks pass +- Gate refuses cleanly on threshold miss, on wrong-count > 0, on + missing report +- GSM8K disclosure handled (present, missing, never gates) +- Claim digest reproducible (deterministic) +- Artifact emission byte-equal across calls +- Snapshot test: committed main state satisfies the composite gate + +12/12 pass in 0.22s. + +## CLAUDE.md PR-checklist + +- **Capability added:** wires the ADR-0131-specific composite + benchmark evaluator at the right architectural layer + (`core/capability/`); makes the math-expert promotion verdict + computable, reproducible, and CLI-callable. +- **Invariant proving field validity:** every B-lane's + `wrong == 0` is preserved (the gate gates on it); GSM8K probe + `admitted_wrong == 0` preserved. +- **CLI/eval proving the lane:** + `python3 -m core.cli capability math-expert-gate` + + `pytest tests/test_adr_0131_4_composite_math_gate.py`. +- **Avoided hidden normalization / stochastic / approximate / + unreviewed mutation:** Yes. Pure function over committed JSON. +- **Trust boundary:** read-only inputs, single deterministic write + to a documented artifact path. diff --git a/evals/math_expert_claims/v1/expert_claims_math_v1.json b/evals/math_expert_claims/v1/expert_claims_math_v1.json new file mode 100644 index 00000000..7c99bed0 --- /dev/null +++ b/evals/math_expert_claims/v1/expert_claims_math_v1.json @@ -0,0 +1,74 @@ +{ + "adr": "0131.4", + "all_benchmarks_passed": true, + "benchmarks": [ + { + "benchmark_id": "B1_public", + "cases_total": 185, + "correct": 185, + "correct_rate": 1.0, + "correct_rate_passes": true, + "passed": true, + "refusal_reason": "", + "report_path": "/Users/kaizenpro/Projects/core-adr-0131-4-promotion/evals/math_symbolic_equivalence/v1/report.json", + "wrong": 0, + "wrong_count_is_zero": true + }, + { + "benchmark_id": "B1_sealed", + "cases_total": 14, + "correct": 14, + "correct_rate": 1.0, + "correct_rate_passes": true, + "passed": true, + "refusal_reason": "", + "report_path": "/Users/kaizenpro/Projects/core-adr-0131-4-promotion/evals/math_symbolic_equivalence/v1/sealed_report.json", + "wrong": 0, + "wrong_count_is_zero": true + }, + { + "benchmark_id": "B2_teaching_corpus", + "cases_total": 40, + "correct": 40, + "correct_rate": 1.0, + "correct_rate_passes": true, + "passed": true, + "refusal_reason": "", + "report_path": "/Users/kaizenpro/Projects/core-adr-0131-4-promotion/evals/math_teaching_corpus/v1/report.json", + "wrong": 0, + "wrong_count_is_zero": true + }, + { + "benchmark_id": "B3_bounded_grammar", + "cases_total": 50, + "correct": 50, + "correct_rate": 1.0, + "correct_rate_passes": true, + "passed": true, + "refusal_reason": "", + "report_path": "/Users/kaizenpro/Projects/core-adr-0131-4-promotion/evals/math_bounded_grammar/v1/report.json", + "wrong": 0, + "wrong_count_is_zero": true + } + ], + "claim_digest": "2bfc7f6c5b06a4c5befef3a9a2629a023518ae97490f6ddf092c1852c966c275", + "composite_gate_passed": true, + "domain": "mathematics_logic", + "honest_disclosure": { + "admission_rate": 0.0, + "admitted_solved": 0, + "admitted_wrong": 0, + "available": true, + "cases_total": 50, + "probe_path": "/Users/kaizenpro/Projects/core-adr-0131-4-promotion/evals/gsm8k_math/train_sample/v1/train_sample_coverage_report.json", + "refused": 50, + "safety_rail_intact": true, + "substrate": "candidate_graph" + }, + "refusal_reason": "", + "schema_version": 1, + "thresholds": { + "correct_rate_min": 0.95, + "wrong_max": 0 + } +} diff --git a/tests/test_adr_0131_4_composite_math_gate.py b/tests/test_adr_0131_4_composite_math_gate.py new file mode 100644 index 00000000..e91f0834 --- /dev/null +++ b/tests/test_adr_0131_4_composite_math_gate.py @@ -0,0 +1,307 @@ +"""ADR-0131.4 — Composite math-expert promotion gate tests. + +Pins the load-bearing invariants of the gate: + - thresholds are pinned (changing them requires a new ADR) + - heterogeneous lane-report shapes (counts vs metrics) both parse + - gate refuses (passed=False) on any single benchmark failing + - missing report files refuse cleanly with a typed reason + - claim_digest is reproducible across calls (deterministic) + - GSM8K honest disclosure is present but does NOT gate + - artifact emission is byte-equal across two calls + +These tests run against fixture data so they don't depend on the +current state of the committed B-lane reports. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from core.capability.composite_math_gate import ( + CORRECT_RATE_MIN, + WRONG_MAX, + BenchmarkVerdict, + CompositeMathGateVerdict, + DEFAULT_GSM8K_PROBE, + emit_expert_claims_artifact, + evaluate_composite_math_gate, +) + + +# --------------------------------------------------------------------------- +# Fixture report builders +# --------------------------------------------------------------------------- + + +def _write_b1b2_shape(path: Path, *, correct: int, wrong: int, total: int) -> None: + """B1 + B2 use ``counts: {correct, wrong, refused}`` + top-level + ``correct_rate``.""" + rate = correct / total if total else 0.0 + path.write_text( + json.dumps({ + "adr": "0131.1", + "counts": {"correct": correct, "wrong": wrong, "refused": total - correct - wrong}, + "sample_count": total, + "correct_rate": rate, + "exit_criterion": {"passed": wrong == 0 and rate >= 0.95, "wrong_max": 0}, + }, indent=2), + encoding="utf-8", + ) + + +def _write_b3_shape(path: Path, *, correct: int, wrong: int, total: int) -> None: + """B3 uses ``metrics: {correct, wrong, cases_total, correct_rate}``.""" + rate = correct / total if total else 0.0 + path.write_text( + json.dumps({ + "adr": "0131.3", + "metrics": { + "correct": correct, + "wrong": wrong, + "cases_total": total, + "correct_rate": rate, + "wrong_count_is_zero": wrong == 0, + }, + }, indent=2), + encoding="utf-8", + ) + + +def _write_probe_shape(path: Path, *, admission: int = 0, wrong: int = 0, total: int = 50) -> None: + path.write_text( + json.dumps({ + "adr": "0131.G", + "substrate": "candidate_graph", + "metrics": { + "cases_total": total, + "admitted_solved": admission, + "admitted_wrong": wrong, + "refused": total - admission - wrong, + "admission_rate": admission / total if total else 0.0, + "safety_rail_intact": wrong == 0, + }, + }, indent=2), + encoding="utf-8", + ) + + +@pytest.fixture +def fixture_lane_paths(tmp_path: Path) -> dict[str, Path]: + """Build a complete set of passing fixture reports.""" + p = { + "b1p": tmp_path / "b1_public.json", + "b1s": tmp_path / "b1_sealed.json", + "b2": tmp_path / "b2.json", + "b3": tmp_path / "b3.json", + "probe": tmp_path / "probe.json", + } + _write_b1b2_shape(p["b1p"], correct=185, wrong=0, total=185) + _write_b1b2_shape(p["b1s"], correct=14, wrong=0, total=14) + _write_b1b2_shape(p["b2"], correct=40, wrong=0, total=40) + _write_b3_shape(p["b3"], correct=50, wrong=0, total=50) + _write_probe_shape(p["probe"], admission=0, wrong=0, total=50) + return p + + +# --------------------------------------------------------------------------- +# Threshold pinning +# --------------------------------------------------------------------------- + + +def test_thresholds_pinned() -> None: + """ADR-0131 pins the composite gate at 0.95 / wrong==0. Changing + these requires a new ADR amendment.""" + assert CORRECT_RATE_MIN == 0.95 + assert WRONG_MAX == 0 + + +# --------------------------------------------------------------------------- +# Heterogeneous shape handling +# --------------------------------------------------------------------------- + + +def test_b1b2_counts_shape_parses(fixture_lane_paths: dict[str, Path]) -> None: + p = fixture_lane_paths + v = evaluate_composite_math_gate( + b1_public_path=p["b1p"], b1_sealed_path=p["b1s"], + b2_path=p["b2"], b3_path=p["b3"], gsm8k_probe_path=p["probe"], + ) + by_id = {b.benchmark_id: b for b in v.benchmarks} + assert by_id["B1_public"].correct == 185 + assert by_id["B1_public"].wrong == 0 + assert by_id["B2_teaching_corpus"].correct == 40 + + +def test_b3_metrics_shape_parses(fixture_lane_paths: dict[str, Path]) -> None: + p = fixture_lane_paths + v = evaluate_composite_math_gate( + b1_public_path=p["b1p"], b1_sealed_path=p["b1s"], + b2_path=p["b2"], b3_path=p["b3"], gsm8k_probe_path=p["probe"], + ) + by_id = {b.benchmark_id: b for b in v.benchmarks} + assert by_id["B3_bounded_grammar"].correct == 50 + assert by_id["B3_bounded_grammar"].cases_total == 50 + assert by_id["B3_bounded_grammar"].correct_rate == 1.0 + + +# --------------------------------------------------------------------------- +# Gate logic +# --------------------------------------------------------------------------- + + +def test_gate_passes_when_all_benchmarks_pass(fixture_lane_paths: dict[str, Path]) -> None: + v = evaluate_composite_math_gate(**{ + f"{k}_path" if k.startswith("b") else f"gsm8k_{k}_path": fixture_lane_paths[short] + for k, short in [ + ("b1_public", "b1p"), ("b1_sealed", "b1s"), + ("b2", "b2"), ("b3", "b3"), ("probe", "probe"), + ] + }) + assert v.composite_gate_passed is True + assert v.all_benchmarks_passed is True + assert v.refusal_reason == "" + + +def test_gate_refuses_when_b1_correct_rate_below_threshold( + fixture_lane_paths: dict[str, Path], tmp_path: Path, +) -> None: + p = dict(fixture_lane_paths) + # B1 public dropped to 0.94 — just under threshold + _write_b1b2_shape(p["b1p"], correct=94, wrong=0, total=100) + v = evaluate_composite_math_gate( + b1_public_path=p["b1p"], b1_sealed_path=p["b1s"], + b2_path=p["b2"], b3_path=p["b3"], gsm8k_probe_path=p["probe"], + ) + assert v.composite_gate_passed is False + assert "B1_public" in v.refusal_reason + + +def test_gate_refuses_when_any_benchmark_has_wrong_answer( + fixture_lane_paths: dict[str, Path], +) -> None: + p = dict(fixture_lane_paths) + _write_b3_shape(p["b3"], correct=49, wrong=1, total=50) + v = evaluate_composite_math_gate( + b1_public_path=p["b1p"], b1_sealed_path=p["b1s"], + b2_path=p["b2"], b3_path=p["b3"], gsm8k_probe_path=p["probe"], + ) + assert v.composite_gate_passed is False + by_id = {b.benchmark_id: b for b in v.benchmarks} + assert by_id["B3_bounded_grammar"].passed is False + assert by_id["B3_bounded_grammar"].wrong_count_is_zero is False + + +def test_gate_refuses_when_report_missing( + fixture_lane_paths: dict[str, Path], tmp_path: Path, +) -> None: + missing = tmp_path / "does_not_exist.json" + v = evaluate_composite_math_gate( + b1_public_path=missing, + b1_sealed_path=fixture_lane_paths["b1s"], + b2_path=fixture_lane_paths["b2"], + b3_path=fixture_lane_paths["b3"], + gsm8k_probe_path=fixture_lane_paths["probe"], + ) + assert v.composite_gate_passed is False + by_id = {b.benchmark_id: b for b in v.benchmarks} + assert by_id["B1_public"].passed is False + assert "missing" in by_id["B1_public"].refusal_reason.lower() + + +# --------------------------------------------------------------------------- +# Honest disclosure (GSM8K reported but does NOT gate) +# --------------------------------------------------------------------------- + + +def test_gsm8k_honest_disclosure_does_not_gate( + fixture_lane_paths: dict[str, Path], +) -> None: + """Even with 0/50 admission, the composite gate passes — GSM8K is + reported as honest disclosure, never gates per ADR-0131.""" + v = evaluate_composite_math_gate(**{ + "b1_public_path": fixture_lane_paths["b1p"], + "b1_sealed_path": fixture_lane_paths["b1s"], + "b2_path": fixture_lane_paths["b2"], + "b3_path": fixture_lane_paths["b3"], + "gsm8k_probe_path": fixture_lane_paths["probe"], + }) + assert v.composite_gate_passed is True + assert v.honest_disclosure["admission_rate"] == 0.0 + assert v.honest_disclosure["available"] is True + + +def test_gsm8k_disclosure_handles_missing_probe( + fixture_lane_paths: dict[str, Path], tmp_path: Path, +) -> None: + v = evaluate_composite_math_gate( + b1_public_path=fixture_lane_paths["b1p"], + b1_sealed_path=fixture_lane_paths["b1s"], + b2_path=fixture_lane_paths["b2"], + b3_path=fixture_lane_paths["b3"], + gsm8k_probe_path=tmp_path / "missing_probe.json", + ) + # Composite gate still passes — probe is disclosure-only. + assert v.composite_gate_passed is True + assert v.honest_disclosure["available"] is False + + +# --------------------------------------------------------------------------- +# Determinism + artifact byte-equality +# --------------------------------------------------------------------------- + + +def test_claim_digest_reproducible(fixture_lane_paths: dict[str, Path]) -> None: + v1 = evaluate_composite_math_gate( + b1_public_path=fixture_lane_paths["b1p"], + b1_sealed_path=fixture_lane_paths["b1s"], + b2_path=fixture_lane_paths["b2"], + b3_path=fixture_lane_paths["b3"], + gsm8k_probe_path=fixture_lane_paths["probe"], + ) + v2 = evaluate_composite_math_gate( + b1_public_path=fixture_lane_paths["b1p"], + b1_sealed_path=fixture_lane_paths["b1s"], + b2_path=fixture_lane_paths["b2"], + b3_path=fixture_lane_paths["b3"], + gsm8k_probe_path=fixture_lane_paths["probe"], + ) + assert v1.claim_digest == v2.claim_digest + assert len(v1.claim_digest) == 64 # SHA-256 hex + + +def test_artifact_emission_byte_equal( + fixture_lane_paths: dict[str, Path], tmp_path: Path, +) -> None: + v = evaluate_composite_math_gate( + b1_public_path=fixture_lane_paths["b1p"], + b1_sealed_path=fixture_lane_paths["b1s"], + b2_path=fixture_lane_paths["b2"], + b3_path=fixture_lane_paths["b3"], + gsm8k_probe_path=fixture_lane_paths["probe"], + ) + out1 = tmp_path / "claims1.json" + out2 = tmp_path / "claims2.json" + emit_expert_claims_artifact(v, out1) + emit_expert_claims_artifact(v, out2) + assert out1.read_bytes() == out2.read_bytes() + + +# --------------------------------------------------------------------------- +# Dry-run against current main state (verifies committed lane reports +# satisfy the gate today). This is the actual ADR-0131.4 verdict. +# --------------------------------------------------------------------------- + + +def test_committed_main_state_satisfies_composite_gate() -> None: + """Snapshot test of the ADR-0131.4 promotion verdict: as of this + PR, all four B-lane reports committed to main produce a PASSING + composite gate. If this test ever fails, the math expert promotion + is no longer ratifiable — investigate the failing benchmark.""" + v = evaluate_composite_math_gate() + assert v.composite_gate_passed is True, ( + f"composite gate failed: {v.refusal_reason}\n" + f"benchmarks: {[(b.benchmark_id, b.passed, b.correct_rate, b.wrong) for b in v.benchmarks]}" + )