Merge pull request #149 from AssetOverflow/feat/adr-0119.8-lane-gate

feat: ADR-0119.8 — gsm8k_math overall lane gate (`gsm8k_capability_shape`)

core/capability/expert_demo.py

@@ -63,6 +63,10 @@ LANE_SHAPE_REGISTRY: dict[str, str] = {
     "koine_greek_fluency": "accuracy_shape",
     "inference_closure": "inference_shape",
     "fabrication_control": "refusal_shape",
+    # ADR-0119.8 — gsm8k_math capability lane. Distinct shape because
+    # the gate composes ``wrong == 0`` (Obligation #4) with
+    # ``correct + refused == total`` and ``overall_pass == True``.
+    "gsm8k_math": "gsm8k_capability_shape",
 }
 
 
@@ -160,11 +164,60 @@ def _check_refusal_shape(lane_id: str, metrics: Mapping[str, Any]) -> tuple[bool
     return True, ""
 
 
+def _check_gsm8k_capability_shape(
+    lane_id: str, metrics: Mapping[str, Any]
+) -> tuple[bool, str]:
+    """ADR-0119.8 — overall gsm8k_math lane gate.
+
+    The lane runner (ADR-0119.3) emits a per-split metrics block with:
+
+    - ``cases_total`` (int)
+    - ``correct`` (int)
+    - ``wrong`` (int)        — must be 0 per ADR-0114a Obligation #4
+    - ``refused`` (int)
+    - ``wrong_count_is_zero`` (bool)
+    - ``overall_pass`` (bool) — ``wrong == 0 AND correct + refused == total``
+
+    The checker validates all three load-bearing constraints. A missing
+    field, a nonzero wrong count, or an arithmetic discrepancy refuses
+    the lane.
+    """
+    for required in ("cases_total", "correct", "wrong", "refused"):
+        if required not in metrics:
+            return False, (
+                f"lane {lane_id!r} missing required metric {required!r}"
+            )
+    total = int(metrics["cases_total"] or 0)
+    correct = int(metrics["correct"] or 0)
+    wrong = int(metrics["wrong"] or 0)
+    refused = int(metrics["refused"] or 0)
+
+    if total <= 0:
+        return False, f"lane {lane_id!r} cases_total={total} (must be > 0)"
+    if wrong != 0:
+        return False, (
+            f"lane {lane_id!r} wrong={wrong} (must be 0 — ADR-0114a Obligation #4)"
+        )
+    if correct + refused != total:
+        return False, (
+            f"lane {lane_id!r} correct({correct}) + refused({refused}) "
+            f"!= cases_total({total}); outcome accounting incomplete"
+        )
+    overall_pass = metrics.get("overall_pass")
+    if overall_pass is not None and not bool(overall_pass):
+        return False, (
+            f"lane {lane_id!r} overall_pass is False despite "
+            f"wrong=0 and accounting balanced"
+        )
+    return True, ""
+
+
 SHAPE_CHECKERS: dict[str, Any] = {
     "cognition_shape": _check_cognition_shape,
     "accuracy_shape": _check_accuracy_shape,
     "inference_shape": _check_inference_shape,
     "refusal_shape": _check_refusal_shape,
+    "gsm8k_capability_shape": _check_gsm8k_capability_shape,
 }
 
 

docs/decisions/ADR-0119.8-lane-gate.md

@@ -0,0 +1,183 @@
+# ADR-0119.8 — gsm8k_math Overall Lane Gate (`gsm8k_capability_shape`)
+
+**Status:** Accepted
+**Date:** 2026-05-23
+**Author:** CORE agents + reviewers
+**Depends on:** ADR-0109 (lane-shape registry), ADR-0114a, ADR-0119, ADR-0119.1, ADR-0119.2, ADR-0119.3, ADR-0119.4, ADR-0119.5, ADR-0119.6
+
+---
+
+## Context
+
+Phase 5.8 of [ADR-0119](ADR-0119-gsm8k-eval-lane-roadmap.md). Composes
+the per-sub-phase work (5.1..5.6) into a single per-split lane verdict
+the eventual ADR-0120 `expert` promotion contract can consume.
+
+Per the ADR-0119 §5.8 brief:
+
+> "A new lane shape `gsm8k_capability_shape` is registered in
+> `LANE_SHAPE_REGISTRY` with the above thresholds. ADR-0119.8 ships
+> the shape; ADR-0120 invokes it."
+
+The shape's load-bearing constraints (per ADR-0114a Obligation #4):
+
+- **`wrong == 0`** — the misparse-discipline gate
+- **`correct + refused == cases_total`** — accounting completeness
+- **`cases_total > 0`** — non-empty input
+- **`overall_pass == True`** (when present) — runner self-consistency
+
+Substrate prerequisites (5.1..5.6 artifacts) are checked by the
+companion test module, not by the shape checker itself — that keeps
+shape checkers narrowly metrics-focused (consistent with ADR-0109's
+existing checker pattern).
+
+---
+
+## Decision
+
+### `LANE_SHAPE_REGISTRY` mapping
+
+```python
+LANE_SHAPE_REGISTRY["gsm8k_math"] = "gsm8k_capability_shape"
+```
+
+Distinct from the existing four shapes because the metrics keys
+differ (`cases_total` / `correct` / `wrong` / `refused` /
+`overall_pass`) and the composition rule
+(`correct + refused == total AND wrong == 0`) is unique to this
+lane's runner contract.
+
+### `_check_gsm8k_capability_shape` checker
+
+Refuses on any of:
+
+| Condition | Reason |
+|---|---|
+| Any of `cases_total` / `correct` / `wrong` / `refused` missing | "missing required metric X" |
+| `cases_total <= 0` | "cases_total=N (must be > 0)" |
+| `wrong != 0` | "wrong=N (must be 0 — ADR-0114a Obligation #4)" |
+| `correct + refused != cases_total` | "outcome accounting incomplete" |
+| `overall_pass` present and false | "overall_pass is False despite wrong=0 and accounting balanced" |
+
+Accepts on everything else. Edge case: **all-refused (0 correct,
+0 wrong, N refused) PASSES the shape gate.** Whether that's acceptable
+for an `expert` promotion is ADR-0120's job (it sets the
+correct-rate-minimum); this layer just verifies the runner's
+contract is internally consistent.
+
+### Current measurement on main
+
+Running the gate against the on-main `evals/gsm8k_math/` corpus:
+
+| Split | cases_total | correct | wrong | refused | gate |
+|---|---|---|---|---|---|
+| dev | 50 | 50 | 0 | 0 | ✓ |
+| public | 150 | 150 | 0 | 0 | ✓ |
+
+---
+
+## ADR-0114a obligation roll-up after Phase 5.8
+
+All 10 obligations are now mechanically gated on `main`:
+
+| # | Obligation | Discharged by |
+|---|---|---|
+| 1 | Sealed-holdout discipline | ADR-0119.1 (fab_control); ADR-0119.7 for the GSM8K test set |
+| 2 | OOD surface variation | ADR-0118a |
+| 3 | Replay-equal trace | ADR-0117 |
+| 4 | Typed refusal + wrong==0 | ADR-0116 + ADR-0119.3 + **gate enforces here** |
+| 5 | Reasoning-isolation perturbation suite | ADR-0125 |
+| 6 | Compositional-depth curve | ADR-0119.6 (ε threshold to ADR-0120) |
+| 7 | Frontier-baseline comparison | ADR-0119.4 |
+| 8 | Adversarial generation; wrong==0 | ADR-0119.5 |
+| 9 | Determinism | solver + verifier + realizer + runner + shape checker |
+| 10 | Operation provenance via pack | ADR-0116 |
+
+The only piece left before ADR-0120 (first `expert` promotion
+contract) is **Phase 5.7** — sealing the real GSM8K test set into
+`evals/gsm8k_math/holdouts/v1/cases.jsonl.age`. That requires a
+human to (a) acquire the GSM8K test set and (b) encrypt it against
+the existing fab_control recipient key (or a fresh GSM8K-specific
+recipient). All other machinery is in place.
+
+---
+
+## Invariants
+
+### `adr_0119_8_registry_mapping`
+
+`LANE_SHAPE_REGISTRY["gsm8k_math"] == "gsm8k_capability_shape"`;
+`SHAPE_CHECKERS["gsm8k_capability_shape"]` is callable.
+
+### `adr_0119_8_live_runner_passes_gate`
+
+Running the on-main lane runner against the dev or public split
+produces metrics the shape checker accepts. Today: dev 50/50,
+public 150/150, both green.
+
+### `adr_0119_8_nonzero_wrong_refuses`
+
+A metrics block with `wrong > 0` refuses the gate with a reason
+naming ADR-0114a Obligation #4.
+
+### `adr_0119_8_incomplete_accounting_refuses`
+
+A metrics block where `correct + refused != cases_total` refuses
+with a typed "outcome accounting incomplete" reason.
+
+### `adr_0119_8_missing_field_refuses`
+
+Removing any of `cases_total` / `correct` / `wrong` / `refused`
+from the metrics refuses with a "missing required metric" reason
+naming the field.
+
+### `adr_0119_8_substrate_artifacts_present`
+
+Phase 5.1..5.6 artifacts exist on disk (sealed holdout for
+fab_control; corpus + verify.py; runner; frontier + comparison;
+adversarial generator + score; depth-curve harness). Phase 5.7
+placeholder is also present (empty cases.jsonl files reserved for
+the eventual real-GSM8K-test seal).
+
+---
+
+## Acceptance evidence
+
+- `core/capability/expert_demo.py` registers `gsm8k_math →
+  gsm8k_capability_shape`; `SHAPE_CHECKERS` exports the checker
+- `tests/test_adr_0119_8_lane_gate.py` (21 cases) green; pins all
+  six invariants
+- Live runner on dev + public both pass the gate
+- 101 tests green across Phase 5 + lane-shape-thresholds +
+  expert-demo-contract + capability-reports
+- Smoke suite green (67/67)
+- ADR linked from `docs/decisions/README.md` index + frontier
+
+---
+
+## Consequences
+
+- The gsm8k_math lane is now first-class in the lane-shape registry.
+  Adding it to a domain pack's `eval_lanes` manifest and signing an
+  `audit_passed_claim` would route through the new gate cleanly.
+- ADR-0120 (first `expert` promotion contract) can now reference
+  `gsm8k_capability_shape` as the lane shape it requires. The only
+  remaining gap is the real-GSM8K-test seal (Phase 5.7) and the
+  numeric ε threshold for the depth curve (ADR-0120's call).
+- The "all-refused" edge case (where the lane refuses every input)
+  passes the shape gate intentionally. That makes the shape narrowly
+  about *runner self-consistency*, not capability. ADR-0120 sets the
+  capability bar.
+
+---
+
+## Out of scope
+
+- Sealing the real GSM8K test set (ADR-0119.7).
+- The numeric ε threshold for the depth curve (ADR-0120).
+- A minimum `correct_rate` for the lane to qualify as `expert`
+  evidence — that's a capability claim, lives in ADR-0120.
+- Cross-split gates (e.g. holdout must score within Δ of public).
+  Those are anti-overfitting properties already handled by
+  Obligation #2 (OOD) and Obligation #6 (depth curve); the shape
+  checker doesn't duplicate them.

docs/decisions/README.md

@@ -53,6 +53,7 @@ ADRs record significant architectural decisions: what was decided, why, what alt
 | [ADR-0119.4](ADR-0119.4-frontier-baseline-comparison.md) | GSM8K Math: Frontier-Baseline Comparison (ADR-0114a §Obligation #7) | Accepted (2026-05-22) |
 | [ADR-0119.5](ADR-0119.5-adversarial-generation.md) | GSM8K Math Adversarial Generation (ADR-0114a Obligation #8) | Accepted (2026-05-23) |
 | [ADR-0119.6](ADR-0119.6-depth-curve-harness.md) | GSM8K Math Depth-Curve Measurement Harness | Accepted (2026-05-23) |
+| [ADR-0119.8](ADR-0119.8-lane-gate.md) | gsm8k_math Overall Lane Gate (`gsm8k_capability_shape`) | Accepted (2026-05-23) |
 
 ---
 
@@ -101,6 +102,7 @@ The ADR-0091..0114 slate is fully accepted (0091..0113) plus one proposed-roadma
 - GSM8K Math: Frontier-Baseline Comparison (citations for Claude 3.5, GPT-4, Gemini 1.5; comparison_v1.json; discharges ADR-0114a §Obligation #7) — ADR-0119.4
 - GSM8K Math Depth-Curve Measurement Harness (discharges ADR-0114a Obligation #6 measurement-side) — ADR-0119.6
 - GSM8K Math Adversarial Generation (38 cases × 12 families; **closes ADR-0114a Obligation #8**; misparse rate 0/38; 10 of 10 obligations now discharged on main) — ADR-0119.5
+- gsm8k_math Overall Lane Gate (Phase 5.8; new `gsm8k_capability_shape` in `LANE_SHAPE_REGISTRY`; composes wrong==0 + correct+refused==total + overall_pass; live dev 50/50 + public 150/150 pass) — ADR-0119.8
 
 ADR-0080 has also landed: Contemplation Loop Phase 1 adds a read-only frontier-compare miner that emits `SPECULATIVE` findings only.
 

tests/test_adr_0119_8_lane_gate.py

@@ -0,0 +1,218 @@
+"""ADR-0119.8 — gsm8k_math overall lane gate invariants.
+
+Pins six load-bearing invariants:
+
+1. **Registry mapping.** ``LANE_SHAPE_REGISTRY["gsm8k_math"] ==
+   "gsm8k_capability_shape"``.
+
+2. **Shape checker registered.** ``SHAPE_CHECKERS`` contains the
+   ``gsm8k_capability_shape`` checker.
+
+3. **Live lane runner output passes the gate** on dev + public splits
+   (today: 50/50 + 150/150 correct, 0 wrong, 0 refused).
+
+4. **Nonzero wrong refuses the gate.** ADR-0114a Obligation #4 enforced.
+
+5. **Outcome-accounting incompleteness refuses the gate.** If
+   correct + refused != total, the gate refuses.
+
+6. **Substrate artifacts exist** for Phase 5.1..5.6 (sealed holdout,
+   corpus + verify.py, runner, frontier baseline, adversarial
+   generator, depth-curve harness). Phase 5.7 (sealed GSM8K test)
+   is documented as pending but not gated here — that's its own
+   future ADR's job to flip on.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from core.capability.expert_demo import (
+    LANE_SHAPE_REGISTRY,
+    SHAPE_CHECKERS,
+    _check_gsm8k_capability_shape,
+    resolve_lane_shape,
+)
+from evals.gsm8k_math.runner import run_lane
+
+
+_REPO_ROOT = Path(__file__).resolve().parent.parent
+
+
+def _load_jsonl(path: Path) -> list[dict]:
+    return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
+
+
+class TestRegistryMapping:
+    def test_gsm8k_math_maps_to_capability_shape(self) -> None:
+        assert LANE_SHAPE_REGISTRY["gsm8k_math"] == "gsm8k_capability_shape"
+
+    def test_resolve_lane_shape_returns_capability_shape(self) -> None:
+        assert resolve_lane_shape("gsm8k_math") == "gsm8k_capability_shape"
+
+    def test_shape_checkers_includes_capability_shape(self) -> None:
+        assert "gsm8k_capability_shape" in SHAPE_CHECKERS
+
+
+class TestLiveLaneRunnerPassesGate:
+    def test_dev_split_metrics_pass(self) -> None:
+        cases = _load_jsonl(_REPO_ROOT / "evals/gsm8k_math/dev/cases.jsonl")
+        report = run_lane(cases)
+        ok, reason = _check_gsm8k_capability_shape("gsm8k_math", report.metrics)
+        assert ok, f"dev split gate failed: {reason}"
+
+    def test_public_split_metrics_pass(self) -> None:
+        cases = _load_jsonl(_REPO_ROOT / "evals/gsm8k_math/public/v1/cases.jsonl")
+        report = run_lane(cases)
+        ok, reason = _check_gsm8k_capability_shape("gsm8k_math", report.metrics)
+        assert ok, f"public split gate failed: {reason}"
+
+
+class TestGateRefusesNonzeroWrong:
+    def test_wrong_one_refuses(self) -> None:
+        metrics = {
+            "cases_total": 10,
+            "correct": 9,
+            "wrong": 1,
+            "refused": 0,
+        }
+        ok, reason = _check_gsm8k_capability_shape("gsm8k_math", metrics)
+        assert ok is False
+        assert "wrong=1" in reason
+        assert "Obligation #4" in reason
+
+
+class TestGateRefusesIncompleteAccounting:
+    def test_correct_plus_refused_not_total(self) -> None:
+        metrics = {
+            "cases_total": 10,
+            "correct": 7,
+            "wrong": 0,
+            "refused": 2,  # 7 + 2 = 9, not 10
+        }
+        ok, reason = _check_gsm8k_capability_shape("gsm8k_math", metrics)
+        assert ok is False
+        assert "outcome accounting incomplete" in reason
+
+
+class TestGateRefusesMissingFields:
+    @pytest.mark.parametrize(
+        "missing", ["cases_total", "correct", "wrong", "refused"]
+    )
+    def test_missing_field_refuses(self, missing: str) -> None:
+        metrics = {
+            "cases_total": 10,
+            "correct": 10,
+            "wrong": 0,
+            "refused": 0,
+        }
+        del metrics[missing]
+        ok, reason = _check_gsm8k_capability_shape("gsm8k_math", metrics)
+        assert ok is False
+        assert missing in reason
+
+
+class TestGateRefusesZeroTotal:
+    def test_zero_total_refuses(self) -> None:
+        metrics = {
+            "cases_total": 0,
+            "correct": 0,
+            "wrong": 0,
+            "refused": 0,
+        }
+        ok, reason = _check_gsm8k_capability_shape("gsm8k_math", metrics)
+        assert ok is False
+        assert "cases_total=0" in reason
+
+
+class TestGateAcceptsCleanMetrics:
+    def test_clean_balanced_metrics_pass(self) -> None:
+        metrics = {
+            "cases_total": 100,
+            "correct": 95,
+            "wrong": 0,
+            "refused": 5,
+            "overall_pass": True,
+        }
+        ok, reason = _check_gsm8k_capability_shape("gsm8k_math", metrics)
+        assert ok is True
+        assert reason == ""
+
+    def test_all_refused_balanced_passes(self) -> None:
+        """Edge case: 0 correct, 0 wrong, all refused — still passes the
+        shape gate. (This means CORE refused every case; whether that's
+        acceptable for ``expert`` promotion is ADR-0120's job, not this
+        layer's.)"""
+        metrics = {
+            "cases_total": 50,
+            "correct": 0,
+            "wrong": 0,
+            "refused": 50,
+            "overall_pass": True,
+        }
+        ok, reason = _check_gsm8k_capability_shape("gsm8k_math", metrics)
+        assert ok is True
+
+
+class TestSubstrateArtifactsExist:
+    """Verify Phase 5.1..5.6 substrate is in place on disk."""
+
+    def test_phase_5_1_sealed_holdout_artifact(self) -> None:
+        """ADR-0119.1 — fabrication_control holdout is age-encrypted."""
+        sealed = _REPO_ROOT / "evals/fabrication_control/holdouts/v1/cases.jsonl.age"
+        assert sealed.exists() and sealed.is_file()
+        # age header starts with "age-encryption.org/"
+        assert sealed.read_bytes().startswith(b"age-encryption.org/")
+
+    def test_phase_5_2_corpus_and_verify(self) -> None:
+        """ADR-0119.2 — dev + public + verify.py."""
+        dev = _REPO_ROOT / "evals/gsm8k_math/dev/cases.jsonl"
+        public = _REPO_ROOT / "evals/gsm8k_math/public/v1/cases.jsonl"
+        verify_py = _REPO_ROOT / "evals/gsm8k_math/verify.py"
+        for p in (dev, public, verify_py):
+            assert p.exists(), f"missing Phase 5.2 artifact: {p}"
+        assert len(_load_jsonl(dev)) == 50
+        assert len(_load_jsonl(public)) == 150
+
+    def test_phase_5_3_runner(self) -> None:
+        runner_py = _REPO_ROOT / "evals/gsm8k_math/runner.py"
+        assert runner_py.exists()
+
+    def test_phase_5_4_frontier_baseline(self) -> None:
+        frontier = _REPO_ROOT / "evals/gsm8k_math/baselines/frontier.json"
+        comparison = _REPO_ROOT / "evals/gsm8k_math/baselines/comparison_v1.json"
+        assert frontier.exists()
+        assert comparison.exists()
+
+    def test_phase_5_5_adversarial(self) -> None:
+        gen = _REPO_ROOT / "evals/gsm8k_math/adversarial/generator.py"
+        score = _REPO_ROOT / "evals/gsm8k_math/adversarial/score.py"
+        assert gen.exists()
+        assert score.exists()
+
+    def test_phase_5_6_depth_curve(self) -> None:
+        harness = _REPO_ROOT / "evals/gsm8k_math/scoring/depth_curve.py"
+        assert harness.exists()
+
+
+class TestPhase5_7PendingDocumented:
+    """Phase 5.7 (sealed GSM8K test) is the one remaining sub-phase.
+
+    This test pins that it's NOT yet a hard gate — that's the future
+    ADR-0119.7's job. Here we just confirm the placeholder is in place
+    and the gate doesn't already fire on a nonexistent sealed test.
+    """
+
+    def test_placeholder_present_or_documented(self) -> None:
+        # The placeholder may be either an empty cases.jsonl or a missing
+        # cases.jsonl.age file; both shapes are acceptable at this stage.
+        sealed = _REPO_ROOT / "evals/gsm8k_math/holdouts/v1/cases.jsonl.age"
+        plaintext = _REPO_ROOT / "evals/gsm8k_math/holdouts/v1/cases.jsonl"
+        plaintext_fallback = _REPO_ROOT / "evals/gsm8k_math/holdouts/v1/cases_plaintext.jsonl"
+        assert sealed.exists() or plaintext.exists() or plaintext_fallback.exists(), (
+            "Phase 5.7 placeholder missing — expected at least one of "
+            "cases.jsonl.age / cases.jsonl / cases_plaintext.jsonl"
+        )