core-labs/core
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Merge pull request #188 from AssetOverflow/feat/adr-0131-4-promotion

Browse source 
feat(ADR-0131.4): composite math-expert gate — PASSING on first evaluation (B1+B2+B3 all green, wrong==0)
This commit is contained in:
Shay 2026-05-23 15:41:43 -07:00 committed by GitHub
commit c996e39c98
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 963 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

346
core/capability/composite_math_gate.py Normal file
View file

@ -0,0 +1,346 @@
"""ADR-0131.4 — Composite math-expert promotion gate (Benchmark 1+2+3).
Implements the *math-specific* revision of the ``expert`` promotion
contract that ADR-0131 introduces. The full ADR-0120 contract (10
ADR-0114a obligations + 3 composition gates) is NOT implemented
here — most of its substrate (perturbation, depth curve, adversarial,
OOD ratio) is not yet built for ``mathematics_logic``. This module
implements only the load-bearing piece ADR-0131 changes:
ADR-0120 single-benchmark check
`correct_rate ≥ 0.60` on public AND sealed GSM8K
--------------------------------------------------- REPLACED BY ----
ADR-0131 composite check (this module)
Benchmark 1 (math_symbolic_equivalence/v1): public + sealed,
each ``correct_rate ≥ 0.95 AND wrong == 0``
Benchmark 2 (math_teaching_corpus/v1):
``correct_rate ≥ 0.95 AND wrong == 0``
Benchmark 3 (math_bounded_grammar/v1):
``correct_rate ≥ 0.95 AND wrong == 0``
Plus honest disclosure: the GSM8K coverage probe's
``admitted_solved`` / ``admitted_wrong`` are reported in the
``expert_claims`` artifact under ``honest_disclosure`` — they do NOT
gate, per ADR-0131 ("GSM8K is retained as a stress-test lane that
the math expert runs but is not gated on. ... reported in the
expert-claims artifact as honest disclosure").
The gate is a pure function over already-committed lane report
JSON. No I/O beyond reading those reports. Deterministic — same
inputs produce byte-equal verdict (after sort_keys + indent=2
serialization).
This module composes with — but does not implement — the broader
ADR-0120 contract. A future ``core/capability/expert_promotion.py``
that implements all 10 ADR-0114a obligations will consume
:func:`evaluate_composite_math_gate` as the math-specific
substitute for the single-lane coverage check.
"""
from __future__ import annotations
import hashlib
import json
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Mapping
# Thresholds pinned by ADR-0131's "Composite expert-promotion gate"
# table. Changing them requires a new ADR amending ADR-0131.
CORRECT_RATE_MIN: float = 0.95
WRONG_MAX: int = 0
# Repository root inferred from this module's location:
# core/capability/composite_math_gate.py -> ../../.
_REPO_ROOT = Path(__file__).resolve().parent.parent.parent
# Default benchmark report paths. The evaluator accepts override
# paths so tests can point at fixture data without touching main's
# committed artifacts.
DEFAULT_B1_PUBLIC: Path = _REPO_ROOT / "evals" / "math_symbolic_equivalence" / "v1" / "report.json"
DEFAULT_B1_SEALED: Path = _REPO_ROOT / "evals" / "math_symbolic_equivalence" / "v1" / "sealed_report.json"
DEFAULT_B2: Path = _REPO_ROOT / "evals" / "math_teaching_corpus" / "v1" / "report.json"
DEFAULT_B3: Path = _REPO_ROOT / "evals" / "math_bounded_grammar" / "v1" / "report.json"
DEFAULT_GSM8K_PROBE: Path = _REPO_ROOT / "evals" / "gsm8k_math" / "train_sample" / "v1" / "train_sample_coverage_report.json"
class CompositeMathGateError(Exception):
"""Raised when a benchmark report cannot be located, parsed, or
canonicalized into the shape this gate consumes."""
@dataclass(frozen=True, slots=True)
class BenchmarkVerdict:
"""Single-benchmark gate outcome."""
benchmark_id: str
report_path: str
correct: int
wrong: int
cases_total: int
correct_rate: float
correct_rate_passes: bool
wrong_count_is_zero: bool
passed: bool
refusal_reason: str = ""
def as_dict(self) -> dict[str, Any]:
return {
"benchmark_id": self.benchmark_id,
"report_path": self.report_path,
"correct": self.correct,
"wrong": self.wrong,
"cases_total": self.cases_total,
"correct_rate": self.correct_rate,
"correct_rate_passes": self.correct_rate_passes,
"wrong_count_is_zero": self.wrong_count_is_zero,
"passed": self.passed,
"refusal_reason": self.refusal_reason,
}
@dataclass(frozen=True, slots=True)
class CompositeMathGateVerdict:
"""Aggregate composite-gate outcome."""
domain: str
benchmarks: tuple[BenchmarkVerdict, ...]
all_benchmarks_passed: bool
composite_gate_passed: bool
honest_disclosure: Mapping[str, Any]
claim_digest: str
thresholds: Mapping[str, Any] = field(default_factory=dict)
refusal_reason: str = ""
def as_dict(self) -> dict[str, Any]:
return {
"adr": "0131.4",
"schema_version": 1,
"domain": self.domain,
"thresholds": dict(self.thresholds),
"benchmarks": [b.as_dict() for b in self.benchmarks],
"all_benchmarks_passed": self.all_benchmarks_passed,
"composite_gate_passed": self.composite_gate_passed,
"honest_disclosure": dict(self.honest_disclosure),
"claim_digest": self.claim_digest,
"refusal_reason": self.refusal_reason,
}
def _read_report(path: Path) -> dict[str, Any]:
if not path.exists():
raise CompositeMathGateError(
f"benchmark report missing: {path}"
)
try:
return json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
raise CompositeMathGateError(
f"benchmark report not valid JSON: {path}: {exc}"
) from exc
def _extract_counts(report: dict[str, Any]) -> tuple[int, int, int]:
"""Return ``(correct, wrong, cases_total)`` from a heterogeneous
lane report.
B1 (public + sealed) and B2 emit ``counts: {correct, wrong, refused}``
and ``sample_count`` (B1 sealed + B2) or ``counts`` totalable
(B1 public). B3 emits ``metrics: {correct, wrong, cases_total}``.
Refuse to extract if the shape is neither — fail loudly rather than
guess.
"""
counts = report.get("counts")
metrics = report.get("metrics")
if isinstance(counts, dict) and {"correct", "wrong"}.issubset(counts.keys()):
correct = int(counts["correct"])
wrong = int(counts["wrong"])
# cases_total: sum of correct + wrong + refused, or sample_count
# if available.
refused = int(counts.get("refused", 0))
cases_total = correct + wrong + refused
if "sample_count" in report and cases_total != int(report["sample_count"]):
# Trust sample_count when present (B1 sealed/B2 ship it).
cases_total = int(report["sample_count"])
return correct, wrong, cases_total
if isinstance(metrics, dict) and {"correct", "wrong"}.issubset(metrics.keys()):
return (
int(metrics["correct"]),
int(metrics["wrong"]),
int(metrics.get("cases_total", metrics["correct"] + metrics["wrong"])),
)
raise CompositeMathGateError(
"benchmark report has neither counts{correct,wrong} nor "
"metrics{correct,wrong} at the top level"
)
def _extract_correct_rate(report: dict[str, Any]) -> float:
"""Top-level ``correct_rate`` first, then ``metrics.correct_rate``,
then derive from counts."""
if "correct_rate" in report:
return float(report["correct_rate"])
metrics = report.get("metrics")
if isinstance(metrics, dict) and "correct_rate" in metrics:
return float(metrics["correct_rate"])
correct, wrong, total = _extract_counts(report)
if total == 0:
return 0.0
return correct / total
def _evaluate_one(benchmark_id: str, path: Path) -> BenchmarkVerdict:
try:
report = _read_report(path)
correct, wrong, total = _extract_counts(report)
rate = _extract_correct_rate(report)
except CompositeMathGateError as exc:
return BenchmarkVerdict(
benchmark_id=benchmark_id,
report_path=str(path),
correct=0,
wrong=0,
cases_total=0,
correct_rate=0.0,
correct_rate_passes=False,
wrong_count_is_zero=False,
passed=False,
refusal_reason=str(exc),
)
rate_passes = rate >= CORRECT_RATE_MIN
wrong_zero = wrong <= WRONG_MAX
return BenchmarkVerdict(
benchmark_id=benchmark_id,
report_path=str(path),
correct=correct,
wrong=wrong,
cases_total=total,
correct_rate=rate,
correct_rate_passes=rate_passes,
wrong_count_is_zero=wrong_zero,
passed=rate_passes and wrong_zero,
refusal_reason=(
"" if (rate_passes and wrong_zero)
else f"correct_rate={rate:.4f} (min {CORRECT_RATE_MIN}), wrong={wrong} (max {WRONG_MAX})"
),
)
def _gsm8k_honest_disclosure(path: Path) -> dict[str, Any]:
"""Read the GSM8K coverage probe report and return the
honest-disclosure subset. Per ADR-0131, GSM8K is reported but
does NOT gate.
"""
if not path.exists():
return {
"probe_path": str(path),
"available": False,
"note": "probe report missing; honest disclosure unavailable",
}
try:
report = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
return {
"probe_path": str(path),
"available": False,
"note": f"probe report unparseable: {exc}",
}
metrics = report.get("metrics", {})
return {
"probe_path": str(path),
"available": True,
"admitted_solved": metrics.get("admitted_solved", 0),
"admitted_wrong": metrics.get("admitted_wrong", 0),
"refused": metrics.get("refused", 0),
"cases_total": metrics.get("cases_total", 0),
"admission_rate": metrics.get("admission_rate", 0.0),
"safety_rail_intact": metrics.get("safety_rail_intact", False),
"substrate": report.get("substrate", "legacy"),
}
def _compute_claim_digest(
benchmarks: tuple[BenchmarkVerdict, ...],
honest_disclosure: Mapping[str, Any],
) -> str:
"""Reproducible SHA-256 over the canonical evidence bundle.
Per ADR-0120's "Signed expert_claims entry with reproducible
digest" requirement — every reviewer should be able to compute
the same digest from the same lane reports.
"""
canonical = {
"schema_version": 1,
"adr": "0131.4",
"benchmarks": [b.as_dict() for b in benchmarks],
"honest_disclosure": dict(honest_disclosure),
"thresholds": {
"correct_rate_min": CORRECT_RATE_MIN,
"wrong_max": WRONG_MAX,
},
}
payload = json.dumps(canonical, sort_keys=True, separators=(",", ":")).encode("utf-8")
return hashlib.sha256(payload).hexdigest()
def evaluate_composite_math_gate(
*,
b1_public_path: Path = DEFAULT_B1_PUBLIC,
b1_sealed_path: Path = DEFAULT_B1_SEALED,
b2_path: Path = DEFAULT_B2,
b3_path: Path = DEFAULT_B3,
gsm8k_probe_path: Path = DEFAULT_GSM8K_PROBE,
) -> CompositeMathGateVerdict:
"""Evaluate the ADR-0131 composite math-expert gate.
Returns a :class:`CompositeMathGateVerdict` with per-benchmark
breakdown, the composite verdict, GSM8K honest-disclosure, and
a reproducible claim digest. Pure function over file paths; no
side effects.
"""
benchmarks = (
_evaluate_one("B1_public", b1_public_path),
_evaluate_one("B1_sealed", b1_sealed_path),
_evaluate_one("B2_teaching_corpus", b2_path),
_evaluate_one("B3_bounded_grammar", b3_path),
)
all_passed = all(b.passed for b in benchmarks)
honest = _gsm8k_honest_disclosure(gsm8k_probe_path)
digest = _compute_claim_digest(benchmarks, honest)
refusal = ""
if not all_passed:
failing = [b.benchmark_id for b in benchmarks if not b.passed]
refusal = f"benchmarks failing the gate: {failing}"
return CompositeMathGateVerdict(
domain="mathematics_logic",
benchmarks=benchmarks,
all_benchmarks_passed=all_passed,
composite_gate_passed=all_passed,
honest_disclosure=honest,
claim_digest=digest,
thresholds={
"correct_rate_min": CORRECT_RATE_MIN,
"wrong_max": WRONG_MAX,
},
refusal_reason=refusal,
)
def emit_expert_claims_artifact(
verdict: CompositeMathGateVerdict,
out_path: Path,
) -> None:
"""Write the deterministic ``expert_claims`` artifact.
Per ADR-0120: "Signed expert_claims entry with reproducible
digest". This emits the unsigned artifact; reviewer signature
is added by a separate ADR-0092 reviewer-registry path that
is out of scope for ADR-0131.4.
"""
payload = verdict.as_dict()
out_path.write_text(
json.dumps(payload, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)

54
core/cli.py
View file

@ -538,6 +538,49 @@ def cmd_capability_evidence_plan(args: argparse.Namespace) -> int:
return 0
def cmd_capability_math_expert_gate(args: argparse.Namespace) -> int:
"""ADR-0131.4 — evaluate the composite math-expert promotion gate
(Benchmark 1 + 2 + 3, ADR-0131's revision of ADR-0120's single-lane
coverage check). Emits ``expert_claims_math_v1.json`` to ``--out``
(default: ``evals/math_expert_claims/v1/expert_claims_math_v1.json``).
Exit 0 iff every benchmark passes."""
from pathlib import Path
from core.capability.composite_math_gate import (
emit_expert_claims_artifact,
evaluate_composite_math_gate,
)
verdict = evaluate_composite_math_gate()
out_path = Path(args.out) if args.out else (
Path(__file__).resolve().parent.parent
/ "evals" / "math_expert_claims" / "v1" / "expert_claims_math_v1.json"
)
out_path.parent.mkdir(parents=True, exist_ok=True)
emit_expert_claims_artifact(verdict, out_path)
if args.json:
print(json.dumps(verdict.as_dict(), indent=2, sort_keys=True))
else:
print(f"composite_gate_passed: {verdict.composite_gate_passed}")
print(f"claim_digest: {verdict.claim_digest}")
print(f"artifact: {out_path}")
for b in verdict.benchmarks:
print(
f" {b.benchmark_id:>20} passed={b.passed} "
f"correct={b.correct}/{b.cases_total} wrong={b.wrong} "
f"rate={b.correct_rate:.4f}"
)
hd = verdict.honest_disclosure
print(
f"GSM8K honest disclosure: admission={hd.get('admitted_solved', 0)}/"
f"{hd.get('cases_total', 0)}, wrong={hd.get('admitted_wrong', 0)}, "
f"substrate={hd.get('substrate', '?')}"
)
if not verdict.composite_gate_passed:
print(f"refusal_reason: {verdict.refusal_reason}")
return 0 if verdict.composite_gate_passed else 1
def cmd_pack_list(args: argparse.Namespace) -> int:
"""List compiled language packs."""
from language_packs import list_packs
@ -2832,6 +2875,17 @@ def build_parser() -> argparse.ArgumentParser:
)
capability_evidence_plan.add_argument("--json", action="store_true", help="emit machine-readable JSON")
capability_evidence_plan.set_defaults(func=cmd_capability_evidence_plan)
capability_math_expert_gate = capability_sub.add_parser(
"math-expert-gate",
help="ADR-0131.4 evaluate the composite math-expert promotion gate (B1+B2+B3)",
)
capability_math_expert_gate.add_argument("--json", action="store_true", help="emit machine-readable JSON")
capability_math_expert_gate.add_argument(
"--out",
default=None,
help="output path for expert_claims artifact (default: evals/math_expert_claims/v1/expert_claims_math_v1.json)",
)
capability_math_expert_gate.set_defaults(func=cmd_capability_math_expert_gate)
pack = subparsers.add_parser("pack", help="inspect and verify language packs")
pack_sub = pack.add_subparsers(dest="pack_command", metavar="pack-command", required=True)

182
docs/decisions/ADR-0131.4-composite-math-gate.md Normal file
View file

@ -0,0 +1,182 @@
# ADR-0131.4 — Composite Math-Expert Promotion Gate (wired)
**Status:** Accepted
**Date:** 2026-05-23
**Author:** CORE main agent (Opus 4.7)
**Depends on:** ADR-0131 (composite gate framing),
ADR-0131.1 (B1 substrate), ADR-0131.1.S (B1 sealed holdout),
ADR-0131.2 (B2 substrate), ADR-0131.3 (B3 substrate),
ADR-0131.G.0 (probe substrate)
**Parent:** ADR-0131
**Reserved follow-up:** the full ADR-0120 contract (9 ADR-0114a
obligations not implemented yet for `mathematics_logic`).
---
## Context
ADR-0131 introduced the composite math-expert promotion contract:
> The `mathematics_logic` domain `expert` promotion contract is
> revised. The ADR-0120 single-benchmark check
> (`correct_rate ≥ 0.60` on GSM8K) is replaced by a composite
> requirement: B1 ≥ 0.95 AND B2 ≥ 0.95 AND B3 ≥ 0.95, each with
> `wrong == 0`. GSM8K is retained as a stress-test lane that the
> math expert runs but is not gated on — reported as honest
> disclosure.
ADR-0131's implementation plan (sub-phase `0131.4`) named
`formation/ratify.py` + `formation/promote.py` as the wire-up
points. That was a misidentification: those modules govern the
SPECULATIVE → COHERENT bridge for individual teaching examples
(ADR-0021), not domain-tier expert promotion. The correct site is
`core/capability/`, where `expert_demo.py` (the audit-passed gate)
already lives and where ADR-0120 reserves `expert_promotion.py`.
A full ADR-0120 implementation requires substrate for 10 ADR-0114a
obligations (sealed holdout, OOD ratio, replay determinism, typed
refusal + `wrong == 0`, perturbation, depth curve, frontier
comparison, adversarial, byte-equal lane runner, operation
provenance via pack). For `mathematics_logic`, only **5 of those
10 obligations** have substrate landed today:
- #1 sealed holdout (ADR-0131.1.S — B1 sealed)
- #3 replay-equal trace (B1/B2/B3 runners emit `trace_hash`)
- #4 typed refusal + `wrong == 0` (the load-bearing invariant
every G.<n> and B-lane has been preserving)
- #7 frontier-baseline comparison (ADR-0131.1.F)
- #9 determinism (every B-lane report is byte-equal across runs)
The other 5 (#2 OOD ratio, #5 perturbation, #6 depth curve, #8
adversarial, #10 operation-provenance-via-pack) need
domain-specific substrate that isn't built yet. Implementing those
is sequencing-wise *after* ADR-0131.4, not bundled with it.
## Decision
Implement only the **ADR-0131-specific revision** as a focused
module: the composite B1+B2+B3 evaluator. Structure it so a future
`core/capability/expert_promotion.py` implementing the full
ADR-0120 contract can consume it as the math-specific substitute
for the single-lane coverage check.
### What this ADR wires
`core/capability/composite_math_gate.py`:
```python
def evaluate_composite_math_gate(
*, b1_public_path, b1_sealed_path, b2_path, b3_path, gsm8k_probe_path,
) -> CompositeMathGateVerdict
```
- Reads each benchmark's already-committed `report.json` (no I/O
beyond that; no recomputation of lane verdicts).
- Handles the heterogeneous report shapes (B1/B2 use
`counts: {correct, wrong, refused}`; B3 uses `metrics: {correct,
wrong, cases_total, correct_rate}`). Refuses cleanly if the
shape is neither.
- Applies the pinned thresholds: `correct_rate ≥ 0.95` AND
`wrong == 0` per benchmark.
- Composes per-benchmark verdicts → composite verdict.
- Computes a reproducible SHA-256 claim digest over the canonical
evidence bundle (per ADR-0120 "Signed `expert_claims` entry with
reproducible digest" requirement).
- Emits GSM8K honest-disclosure (admission, wrong, refused,
substrate) but does NOT use it in the gate.
CLI wiring at `core capability math-expert-gate` (added to
`core/cli.py`). Writes
`evals/math_expert_claims/v1/expert_claims_math_v1.json` (unsigned).
Reviewer signature via ADR-0092 reviewer-registry is reserved for
the broader ADR-0120 wire-up.
### Empirical verdict
Run against current `main` (post-PR #182/#183/#184/#185, pre-G.1):
```
composite_gate_passed: True
claim_digest: 2bfc7f6c5b06a4c5befef3a9a2629a023518ae97490f6ddf092c1852c966c275
B1_public passed=True correct=185/185 wrong=0 rate=1.0000
B1_sealed passed=True correct=14/14 wrong=0 rate=1.0000
B2_teaching_corpus passed=True correct=40/40 wrong=0 rate=1.0000
B3_bounded_grammar passed=True correct=50/50 wrong=0 rate=1.0000
GSM8K honest disclosure: admission=0/50, wrong=0, substrate=candidate_graph
```
**The math expert is gate-passing under ADR-0131's revised
composite contract.** The bet ADR-0131 placed — that the
architecture's structural strengths align with three benchmarks
that measure those strengths — has paid off on first evaluation.
This does NOT mean the full ADR-0120 contract passes (5 of 10
obligations still need substrate). It means the *math-specific
revision portion* of the contract passes today, and the
architecturally-aligned bet was correct.
## What this does NOT do
- Does NOT implement the broader ADR-0120 10-obligation contract.
That requires substrate for 5 missing obligations (OOD ratio,
perturbation, depth curve, adversarial, operation-provenance).
- Does NOT promote `mathematics_logic` to `expert` ledger status.
Ledger promotion is a separate ADR that consumes this gate's
output AND the broader ADR-0120 obligations.
- Does NOT sign the `expert_claims` artifact. Reviewer signature
via ADR-0092 registry is reserved for the full wire-up.
- Does NOT touch `formation/ratify.py` or `formation/promote.py`
(the teaching-example-tier modules ADR-0131's plan
misidentified).
- Does NOT recompute the lane verdicts. The committed B-lane
reports are the gate's input contract; the lane runners own
their verdicts and the gate trusts them. If a reviewer suspects
a lane report is stale, re-run the lane.
## Trust boundary
- Reads only:
- `evals/math_symbolic_equivalence/v1/report.json` +
`sealed_report.json`
- `evals/math_teaching_corpus/v1/report.json`
- `evals/math_bounded_grammar/v1/report.json`
- `evals/gsm8k_math/train_sample/v1/train_sample_coverage_report.json`
(disclosure-only)
- Writes only: the path passed to `emit_expert_claims_artifact`
(default: `evals/math_expert_claims/v1/expert_claims_math_v1.json`).
- No dynamic imports, no shell passthrough, no network.
- Pure function over already-committed JSON; deterministic
(verified by `test_claim_digest_reproducible` and
`test_artifact_emission_byte_equal`).
## Tests
`tests/test_adr_0131_4_composite_math_gate.py` — 12 tests:
- Threshold values are pinned (changing requires a new ADR)
- Both lane-report shapes parse correctly
- Gate passes iff all 4 benchmarks pass
- Gate refuses cleanly on threshold miss, on wrong-count > 0, on
missing report
- GSM8K disclosure handled (present, missing, never gates)
- Claim digest reproducible (deterministic)
- Artifact emission byte-equal across calls
- Snapshot test: committed main state satisfies the composite gate
12/12 pass in 0.22s.
## CLAUDE.md PR-checklist
- **Capability added:** wires the ADR-0131-specific composite
benchmark evaluator at the right architectural layer
(`core/capability/`); makes the math-expert promotion verdict
computable, reproducible, and CLI-callable.
- **Invariant proving field validity:** every B-lane's
`wrong == 0` is preserved (the gate gates on it); GSM8K probe
`admitted_wrong == 0` preserved.
- **CLI/eval proving the lane:**
`python3 -m core.cli capability math-expert-gate` +
`pytest tests/test_adr_0131_4_composite_math_gate.py`.
- **Avoided hidden normalization / stochastic / approximate /
unreviewed mutation:** Yes. Pure function over committed JSON.
- **Trust boundary:** read-only inputs, single deterministic write
to a documented artifact path.

74
evals/math_expert_claims/v1/expert_claims_math_v1.json Normal file
View file

@ -0,0 +1,74 @@
{
"adr": "0131.4",
"all_benchmarks_passed": true,
"benchmarks": [
{
"benchmark_id": "B1_public",
"cases_total": 185,
"correct": 185,
"correct_rate": 1.0,
"correct_rate_passes": true,
"passed": true,
"refusal_reason": "",
"report_path": "/Users/kaizenpro/Projects/core-adr-0131-4-promotion/evals/math_symbolic_equivalence/v1/report.json",
"wrong": 0,
"wrong_count_is_zero": true
},
{
"benchmark_id": "B1_sealed",
"cases_total": 14,
"correct": 14,
"correct_rate": 1.0,
"correct_rate_passes": true,
"passed": true,
"refusal_reason": "",
"report_path": "/Users/kaizenpro/Projects/core-adr-0131-4-promotion/evals/math_symbolic_equivalence/v1/sealed_report.json",
"wrong": 0,
"wrong_count_is_zero": true
},
{
"benchmark_id": "B2_teaching_corpus",
"cases_total": 40,
"correct": 40,
"correct_rate": 1.0,
"correct_rate_passes": true,
"passed": true,
"refusal_reason": "",
"report_path": "/Users/kaizenpro/Projects/core-adr-0131-4-promotion/evals/math_teaching_corpus/v1/report.json",
"wrong": 0,
"wrong_count_is_zero": true
},
{
"benchmark_id": "B3_bounded_grammar",
"cases_total": 50,
"correct": 50,
"correct_rate": 1.0,
"correct_rate_passes": true,
"passed": true,
"refusal_reason": "",
"report_path": "/Users/kaizenpro/Projects/core-adr-0131-4-promotion/evals/math_bounded_grammar/v1/report.json",
"wrong": 0,
"wrong_count_is_zero": true
}
],
"claim_digest": "2bfc7f6c5b06a4c5befef3a9a2629a023518ae97490f6ddf092c1852c966c275",
"composite_gate_passed": true,
"domain": "mathematics_logic",
"honest_disclosure": {
"admission_rate": 0.0,
"admitted_solved": 0,
"admitted_wrong": 0,
"available": true,
"cases_total": 50,
"probe_path": "/Users/kaizenpro/Projects/core-adr-0131-4-promotion/evals/gsm8k_math/train_sample/v1/train_sample_coverage_report.json",
"refused": 50,
"safety_rail_intact": true,
"substrate": "candidate_graph"
},
"refusal_reason": "",
"schema_version": 1,
"thresholds": {
"correct_rate_min": 0.95,
"wrong_max": 0
}
}

307
tests/test_adr_0131_4_composite_math_gate.py Normal file
View file

@ -0,0 +1,307 @@
"""ADR-0131.4 — Composite math-expert promotion gate tests.
Pins the load-bearing invariants of the gate:
- thresholds are pinned (changing them requires a new ADR)
- heterogeneous lane-report shapes (counts vs metrics) both parse
- gate refuses (passed=False) on any single benchmark failing
- missing report files refuse cleanly with a typed reason
- claim_digest is reproducible across calls (deterministic)
- GSM8K honest disclosure is present but does NOT gate
- artifact emission is byte-equal across two calls
These tests run against fixture data so they don't depend on the
current state of the committed B-lane reports.
"""
from __future__ import annotations
import json
from pathlib import Path
import pytest
from core.capability.composite_math_gate import (
CORRECT_RATE_MIN,
WRONG_MAX,
BenchmarkVerdict,
CompositeMathGateVerdict,
DEFAULT_GSM8K_PROBE,
emit_expert_claims_artifact,
evaluate_composite_math_gate,
)
# ---------------------------------------------------------------------------
# Fixture report builders
# ---------------------------------------------------------------------------
def _write_b1b2_shape(path: Path, *, correct: int, wrong: int, total: int) -> None:
"""B1 + B2 use ``counts: {correct, wrong, refused}`` + top-level
``correct_rate``."""
rate = correct / total if total else 0.0
path.write_text(
json.dumps({
"adr": "0131.1",
"counts": {"correct": correct, "wrong": wrong, "refused": total - correct - wrong},
"sample_count": total,
"correct_rate": rate,
"exit_criterion": {"passed": wrong == 0 and rate >= 0.95, "wrong_max": 0},
}, indent=2),
encoding="utf-8",
)
def _write_b3_shape(path: Path, *, correct: int, wrong: int, total: int) -> None:
"""B3 uses ``metrics: {correct, wrong, cases_total, correct_rate}``."""
rate = correct / total if total else 0.0
path.write_text(
json.dumps({
"adr": "0131.3",
"metrics": {
"correct": correct,
"wrong": wrong,
"cases_total": total,
"correct_rate": rate,
"wrong_count_is_zero": wrong == 0,
},
}, indent=2),
encoding="utf-8",
)
def _write_probe_shape(path: Path, *, admission: int = 0, wrong: int = 0, total: int = 50) -> None:
path.write_text(
json.dumps({
"adr": "0131.G",
"substrate": "candidate_graph",
"metrics": {
"cases_total": total,
"admitted_solved": admission,
"admitted_wrong": wrong,
"refused": total - admission - wrong,
"admission_rate": admission / total if total else 0.0,
"safety_rail_intact": wrong == 0,
},
}, indent=2),
encoding="utf-8",
)
@pytest.fixture
def fixture_lane_paths(tmp_path: Path) -> dict[str, Path]:
"""Build a complete set of passing fixture reports."""
p = {
"b1p": tmp_path / "b1_public.json",
"b1s": tmp_path / "b1_sealed.json",
"b2": tmp_path / "b2.json",
"b3": tmp_path / "b3.json",
"probe": tmp_path / "probe.json",
}
_write_b1b2_shape(p["b1p"], correct=185, wrong=0, total=185)
_write_b1b2_shape(p["b1s"], correct=14, wrong=0, total=14)
_write_b1b2_shape(p["b2"], correct=40, wrong=0, total=40)
_write_b3_shape(p["b3"], correct=50, wrong=0, total=50)
_write_probe_shape(p["probe"], admission=0, wrong=0, total=50)
return p
# ---------------------------------------------------------------------------
# Threshold pinning
# ---------------------------------------------------------------------------
def test_thresholds_pinned() -> None:
"""ADR-0131 pins the composite gate at 0.95 / wrong==0. Changing
these requires a new ADR amendment."""
assert CORRECT_RATE_MIN == 0.95
assert WRONG_MAX == 0
# ---------------------------------------------------------------------------
# Heterogeneous shape handling
# ---------------------------------------------------------------------------
def test_b1b2_counts_shape_parses(fixture_lane_paths: dict[str, Path]) -> None:
p = fixture_lane_paths
v = evaluate_composite_math_gate(
b1_public_path=p["b1p"], b1_sealed_path=p["b1s"],
b2_path=p["b2"], b3_path=p["b3"], gsm8k_probe_path=p["probe"],
)
by_id = {b.benchmark_id: b for b in v.benchmarks}
assert by_id["B1_public"].correct == 185
assert by_id["B1_public"].wrong == 0
assert by_id["B2_teaching_corpus"].correct == 40
def test_b3_metrics_shape_parses(fixture_lane_paths: dict[str, Path]) -> None:
p = fixture_lane_paths
v = evaluate_composite_math_gate(
b1_public_path=p["b1p"], b1_sealed_path=p["b1s"],
b2_path=p["b2"], b3_path=p["b3"], gsm8k_probe_path=p["probe"],
)
by_id = {b.benchmark_id: b for b in v.benchmarks}
assert by_id["B3_bounded_grammar"].correct == 50
assert by_id["B3_bounded_grammar"].cases_total == 50
assert by_id["B3_bounded_grammar"].correct_rate == 1.0
# ---------------------------------------------------------------------------
# Gate logic
# ---------------------------------------------------------------------------
def test_gate_passes_when_all_benchmarks_pass(fixture_lane_paths: dict[str, Path]) -> None:
v = evaluate_composite_math_gate(**{
f"{k}_path" if k.startswith("b") else f"gsm8k_{k}_path": fixture_lane_paths[short]
for k, short in [
("b1_public", "b1p"), ("b1_sealed", "b1s"),
("b2", "b2"), ("b3", "b3"), ("probe", "probe"),
]
})
assert v.composite_gate_passed is True
assert v.all_benchmarks_passed is True
assert v.refusal_reason == ""
def test_gate_refuses_when_b1_correct_rate_below_threshold(
fixture_lane_paths: dict[str, Path], tmp_path: Path,
) -> None:
p = dict(fixture_lane_paths)
# B1 public dropped to 0.94 — just under threshold
_write_b1b2_shape(p["b1p"], correct=94, wrong=0, total=100)
v = evaluate_composite_math_gate(
b1_public_path=p["b1p"], b1_sealed_path=p["b1s"],
b2_path=p["b2"], b3_path=p["b3"], gsm8k_probe_path=p["probe"],
)
assert v.composite_gate_passed is False
assert "B1_public" in v.refusal_reason
def test_gate_refuses_when_any_benchmark_has_wrong_answer(
fixture_lane_paths: dict[str, Path],
) -> None:
p = dict(fixture_lane_paths)
_write_b3_shape(p["b3"], correct=49, wrong=1, total=50)
v = evaluate_composite_math_gate(
b1_public_path=p["b1p"], b1_sealed_path=p["b1s"],
b2_path=p["b2"], b3_path=p["b3"], gsm8k_probe_path=p["probe"],
)
assert v.composite_gate_passed is False
by_id = {b.benchmark_id: b for b in v.benchmarks}
assert by_id["B3_bounded_grammar"].passed is False
assert by_id["B3_bounded_grammar"].wrong_count_is_zero is False
def test_gate_refuses_when_report_missing(
fixture_lane_paths: dict[str, Path], tmp_path: Path,
) -> None:
missing = tmp_path / "does_not_exist.json"
v = evaluate_composite_math_gate(
b1_public_path=missing,
b1_sealed_path=fixture_lane_paths["b1s"],
b2_path=fixture_lane_paths["b2"],
b3_path=fixture_lane_paths["b3"],
gsm8k_probe_path=fixture_lane_paths["probe"],
)
assert v.composite_gate_passed is False
by_id = {b.benchmark_id: b for b in v.benchmarks}
assert by_id["B1_public"].passed is False
assert "missing" in by_id["B1_public"].refusal_reason.lower()
# ---------------------------------------------------------------------------
# Honest disclosure (GSM8K reported but does NOT gate)
# ---------------------------------------------------------------------------
def test_gsm8k_honest_disclosure_does_not_gate(
fixture_lane_paths: dict[str, Path],
) -> None:
"""Even with 0/50 admission, the composite gate passes — GSM8K is
reported as honest disclosure, never gates per ADR-0131."""
v = evaluate_composite_math_gate(**{
"b1_public_path": fixture_lane_paths["b1p"],
"b1_sealed_path": fixture_lane_paths["b1s"],
"b2_path": fixture_lane_paths["b2"],
"b3_path": fixture_lane_paths["b3"],
"gsm8k_probe_path": fixture_lane_paths["probe"],
})
assert v.composite_gate_passed is True
assert v.honest_disclosure["admission_rate"] == 0.0
assert v.honest_disclosure["available"] is True
def test_gsm8k_disclosure_handles_missing_probe(
fixture_lane_paths: dict[str, Path], tmp_path: Path,
) -> None:
v = evaluate_composite_math_gate(
b1_public_path=fixture_lane_paths["b1p"],
b1_sealed_path=fixture_lane_paths["b1s"],
b2_path=fixture_lane_paths["b2"],
b3_path=fixture_lane_paths["b3"],
gsm8k_probe_path=tmp_path / "missing_probe.json",
)
# Composite gate still passes — probe is disclosure-only.
assert v.composite_gate_passed is True
assert v.honest_disclosure["available"] is False
# ---------------------------------------------------------------------------
# Determinism + artifact byte-equality
# ---------------------------------------------------------------------------
def test_claim_digest_reproducible(fixture_lane_paths: dict[str, Path]) -> None:
v1 = evaluate_composite_math_gate(
b1_public_path=fixture_lane_paths["b1p"],
b1_sealed_path=fixture_lane_paths["b1s"],
b2_path=fixture_lane_paths["b2"],
b3_path=fixture_lane_paths["b3"],
gsm8k_probe_path=fixture_lane_paths["probe"],
)
v2 = evaluate_composite_math_gate(
b1_public_path=fixture_lane_paths["b1p"],
b1_sealed_path=fixture_lane_paths["b1s"],
b2_path=fixture_lane_paths["b2"],
b3_path=fixture_lane_paths["b3"],
gsm8k_probe_path=fixture_lane_paths["probe"],
)
assert v1.claim_digest == v2.claim_digest
assert len(v1.claim_digest) == 64 # SHA-256 hex
def test_artifact_emission_byte_equal(
fixture_lane_paths: dict[str, Path], tmp_path: Path,
) -> None:
v = evaluate_composite_math_gate(
b1_public_path=fixture_lane_paths["b1p"],
b1_sealed_path=fixture_lane_paths["b1s"],
b2_path=fixture_lane_paths["b2"],
b3_path=fixture_lane_paths["b3"],
gsm8k_probe_path=fixture_lane_paths["probe"],
)
out1 = tmp_path / "claims1.json"
out2 = tmp_path / "claims2.json"
emit_expert_claims_artifact(v, out1)
emit_expert_claims_artifact(v, out2)
assert out1.read_bytes() == out2.read_bytes()
# ---------------------------------------------------------------------------
# Dry-run against current main state (verifies committed lane reports
# satisfy the gate today). This is the actual ADR-0131.4 verdict.
# ---------------------------------------------------------------------------
def test_committed_main_state_satisfies_composite_gate() -> None:
"""Snapshot test of the ADR-0131.4 promotion verdict: as of this
PR, all four B-lane reports committed to main produce a PASSING
composite gate. If this test ever fails, the math expert promotion
is no longer ratifiable — investigate the failing benchmark."""
v = evaluate_composite_math_gate()
assert v.composite_gate_passed is True, (
f"composite gate failed: {v.refusal_reason}\n"
f"benchmarks: {[(b.benchmark_id, b.passed, b.correct_rate, b.wrong) for b in v.benchmarks]}"
)