From a13df6f370711530b61642069218330c54c04853 Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 22 May 2026 19:45:44 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20ADR-0119.8=20=E2=80=94=20gsm8k=5Fmath?= =?UTF-8?q?=20overall=20lane=20gate=20(gsm8k=5Fcapability=5Fshape)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 5.8 of ADR-0119. Composes the per-sub-phase substrate (5.1..5.6) into a single per-split lane verdict the eventual ADR-0120 expert promotion contract can consume. LANE_SHAPE_REGISTRY adds: "gsm8k_math": "gsm8k_capability_shape" _check_gsm8k_capability_shape refuses on any of: - missing cases_total / correct / wrong / refused fields - cases_total <= 0 - wrong != 0 (ADR-0114a Obligation #4) - correct + refused != cases_total (accounting incomplete) - overall_pass present and false Accepts otherwise. Edge: all-refused passes the shape gate (runner self-consistency). Capability bar (min correct-rate, depth-curve ε) lives in ADR-0120. Live measurement on main: dev 50/50 correct, 0 wrong, 0 refused → gate ✓ public 150/150 correct, 0 wrong, 0 refused → gate ✓ 21 invariant tests pin: registry mapping, shape checker presence, live runner passes, nonzero wrong refuses, incomplete accounting refuses, missing field refuses, clean metrics pass, all-refused edge passes, all Phase 5.1..5.6 substrate artifacts exist on disk. Phase 5 status: 5.1..5.6 + 5.8 ✓. Only 5.7 (sealed real GSM8K test) remains before ADR-0120 (first expert promotion contract) becomes feasible. ADR-0114a roll-up unchanged: 10/10 obligations discharged on main (modulo Phase 5.7's lane-specific GSM8K test sealing). Tests: 21 new + 80 prior across Phase 5 + adjacent suites = 101 green; 67/67 smoke. Co-Authored-By: Claude Opus 4.7 --- core/capability/expert_demo.py | 53 ++++++ docs/decisions/ADR-0119.8-lane-gate.md | 183 +++++++++++++++++++++ docs/decisions/README.md | 2 + tests/test_adr_0119_8_lane_gate.py | 218 +++++++++++++++++++++++++ 4 files changed, 456 insertions(+) create mode 100644 docs/decisions/ADR-0119.8-lane-gate.md create mode 100644 tests/test_adr_0119_8_lane_gate.py diff --git a/core/capability/expert_demo.py b/core/capability/expert_demo.py index 2f225cc2..f1dbadf1 100644 --- a/core/capability/expert_demo.py +++ b/core/capability/expert_demo.py @@ -63,6 +63,10 @@ LANE_SHAPE_REGISTRY: dict[str, str] = { "koine_greek_fluency": "accuracy_shape", "inference_closure": "inference_shape", "fabrication_control": "refusal_shape", + # ADR-0119.8 — gsm8k_math capability lane. Distinct shape because + # the gate composes ``wrong == 0`` (Obligation #4) with + # ``correct + refused == total`` and ``overall_pass == True``. + "gsm8k_math": "gsm8k_capability_shape", } @@ -160,11 +164,60 @@ def _check_refusal_shape(lane_id: str, metrics: Mapping[str, Any]) -> tuple[bool return True, "" +def _check_gsm8k_capability_shape( + lane_id: str, metrics: Mapping[str, Any] +) -> tuple[bool, str]: + """ADR-0119.8 — overall gsm8k_math lane gate. + + The lane runner (ADR-0119.3) emits a per-split metrics block with: + + - ``cases_total`` (int) + - ``correct`` (int) + - ``wrong`` (int) — must be 0 per ADR-0114a Obligation #4 + - ``refused`` (int) + - ``wrong_count_is_zero`` (bool) + - ``overall_pass`` (bool) — ``wrong == 0 AND correct + refused == total`` + + The checker validates all three load-bearing constraints. A missing + field, a nonzero wrong count, or an arithmetic discrepancy refuses + the lane. + """ + for required in ("cases_total", "correct", "wrong", "refused"): + if required not in metrics: + return False, ( + f"lane {lane_id!r} missing required metric {required!r}" + ) + total = int(metrics["cases_total"] or 0) + correct = int(metrics["correct"] or 0) + wrong = int(metrics["wrong"] or 0) + refused = int(metrics["refused"] or 0) + + if total <= 0: + return False, f"lane {lane_id!r} cases_total={total} (must be > 0)" + if wrong != 0: + return False, ( + f"lane {lane_id!r} wrong={wrong} (must be 0 — ADR-0114a Obligation #4)" + ) + if correct + refused != total: + return False, ( + f"lane {lane_id!r} correct({correct}) + refused({refused}) " + f"!= cases_total({total}); outcome accounting incomplete" + ) + overall_pass = metrics.get("overall_pass") + if overall_pass is not None and not bool(overall_pass): + return False, ( + f"lane {lane_id!r} overall_pass is False despite " + f"wrong=0 and accounting balanced" + ) + return True, "" + + SHAPE_CHECKERS: dict[str, Any] = { "cognition_shape": _check_cognition_shape, "accuracy_shape": _check_accuracy_shape, "inference_shape": _check_inference_shape, "refusal_shape": _check_refusal_shape, + "gsm8k_capability_shape": _check_gsm8k_capability_shape, } diff --git a/docs/decisions/ADR-0119.8-lane-gate.md b/docs/decisions/ADR-0119.8-lane-gate.md new file mode 100644 index 00000000..88e8ec4b --- /dev/null +++ b/docs/decisions/ADR-0119.8-lane-gate.md @@ -0,0 +1,183 @@ +# ADR-0119.8 — gsm8k_math Overall Lane Gate (`gsm8k_capability_shape`) + +**Status:** Accepted +**Date:** 2026-05-23 +**Author:** CORE agents + reviewers +**Depends on:** ADR-0109 (lane-shape registry), ADR-0114a, ADR-0119, ADR-0119.1, ADR-0119.2, ADR-0119.3, ADR-0119.4, ADR-0119.5, ADR-0119.6 + +--- + +## Context + +Phase 5.8 of [ADR-0119](ADR-0119-gsm8k-eval-lane-roadmap.md). Composes +the per-sub-phase work (5.1..5.6) into a single per-split lane verdict +the eventual ADR-0120 `expert` promotion contract can consume. + +Per the ADR-0119 §5.8 brief: + +> "A new lane shape `gsm8k_capability_shape` is registered in +> `LANE_SHAPE_REGISTRY` with the above thresholds. ADR-0119.8 ships +> the shape; ADR-0120 invokes it." + +The shape's load-bearing constraints (per ADR-0114a Obligation #4): + +- **`wrong == 0`** — the misparse-discipline gate +- **`correct + refused == cases_total`** — accounting completeness +- **`cases_total > 0`** — non-empty input +- **`overall_pass == True`** (when present) — runner self-consistency + +Substrate prerequisites (5.1..5.6 artifacts) are checked by the +companion test module, not by the shape checker itself — that keeps +shape checkers narrowly metrics-focused (consistent with ADR-0109's +existing checker pattern). + +--- + +## Decision + +### `LANE_SHAPE_REGISTRY` mapping + +```python +LANE_SHAPE_REGISTRY["gsm8k_math"] = "gsm8k_capability_shape" +``` + +Distinct from the existing four shapes because the metrics keys +differ (`cases_total` / `correct` / `wrong` / `refused` / +`overall_pass`) and the composition rule +(`correct + refused == total AND wrong == 0`) is unique to this +lane's runner contract. + +### `_check_gsm8k_capability_shape` checker + +Refuses on any of: + +| Condition | Reason | +|---|---| +| Any of `cases_total` / `correct` / `wrong` / `refused` missing | "missing required metric X" | +| `cases_total <= 0` | "cases_total=N (must be > 0)" | +| `wrong != 0` | "wrong=N (must be 0 — ADR-0114a Obligation #4)" | +| `correct + refused != cases_total` | "outcome accounting incomplete" | +| `overall_pass` present and false | "overall_pass is False despite wrong=0 and accounting balanced" | + +Accepts on everything else. Edge case: **all-refused (0 correct, +0 wrong, N refused) PASSES the shape gate.** Whether that's acceptable +for an `expert` promotion is ADR-0120's job (it sets the +correct-rate-minimum); this layer just verifies the runner's +contract is internally consistent. + +### Current measurement on main + +Running the gate against the on-main `evals/gsm8k_math/` corpus: + +| Split | cases_total | correct | wrong | refused | gate | +|---|---|---|---|---|---| +| dev | 50 | 50 | 0 | 0 | ✓ | +| public | 150 | 150 | 0 | 0 | ✓ | + +--- + +## ADR-0114a obligation roll-up after Phase 5.8 + +All 10 obligations are now mechanically gated on `main`: + +| # | Obligation | Discharged by | +|---|---|---| +| 1 | Sealed-holdout discipline | ADR-0119.1 (fab_control); ADR-0119.7 for the GSM8K test set | +| 2 | OOD surface variation | ADR-0118a | +| 3 | Replay-equal trace | ADR-0117 | +| 4 | Typed refusal + wrong==0 | ADR-0116 + ADR-0119.3 + **gate enforces here** | +| 5 | Reasoning-isolation perturbation suite | ADR-0125 | +| 6 | Compositional-depth curve | ADR-0119.6 (ε threshold to ADR-0120) | +| 7 | Frontier-baseline comparison | ADR-0119.4 | +| 8 | Adversarial generation; wrong==0 | ADR-0119.5 | +| 9 | Determinism | solver + verifier + realizer + runner + shape checker | +| 10 | Operation provenance via pack | ADR-0116 | + +The only piece left before ADR-0120 (first `expert` promotion +contract) is **Phase 5.7** — sealing the real GSM8K test set into +`evals/gsm8k_math/holdouts/v1/cases.jsonl.age`. That requires a +human to (a) acquire the GSM8K test set and (b) encrypt it against +the existing fab_control recipient key (or a fresh GSM8K-specific +recipient). All other machinery is in place. + +--- + +## Invariants + +### `adr_0119_8_registry_mapping` + +`LANE_SHAPE_REGISTRY["gsm8k_math"] == "gsm8k_capability_shape"`; +`SHAPE_CHECKERS["gsm8k_capability_shape"]` is callable. + +### `adr_0119_8_live_runner_passes_gate` + +Running the on-main lane runner against the dev or public split +produces metrics the shape checker accepts. Today: dev 50/50, +public 150/150, both green. + +### `adr_0119_8_nonzero_wrong_refuses` + +A metrics block with `wrong > 0` refuses the gate with a reason +naming ADR-0114a Obligation #4. + +### `adr_0119_8_incomplete_accounting_refuses` + +A metrics block where `correct + refused != cases_total` refuses +with a typed "outcome accounting incomplete" reason. + +### `adr_0119_8_missing_field_refuses` + +Removing any of `cases_total` / `correct` / `wrong` / `refused` +from the metrics refuses with a "missing required metric" reason +naming the field. + +### `adr_0119_8_substrate_artifacts_present` + +Phase 5.1..5.6 artifacts exist on disk (sealed holdout for +fab_control; corpus + verify.py; runner; frontier + comparison; +adversarial generator + score; depth-curve harness). Phase 5.7 +placeholder is also present (empty cases.jsonl files reserved for +the eventual real-GSM8K-test seal). + +--- + +## Acceptance evidence + +- `core/capability/expert_demo.py` registers `gsm8k_math → + gsm8k_capability_shape`; `SHAPE_CHECKERS` exports the checker +- `tests/test_adr_0119_8_lane_gate.py` (21 cases) green; pins all + six invariants +- Live runner on dev + public both pass the gate +- 101 tests green across Phase 5 + lane-shape-thresholds + + expert-demo-contract + capability-reports +- Smoke suite green (67/67) +- ADR linked from `docs/decisions/README.md` index + frontier + +--- + +## Consequences + +- The gsm8k_math lane is now first-class in the lane-shape registry. + Adding it to a domain pack's `eval_lanes` manifest and signing an + `audit_passed_claim` would route through the new gate cleanly. +- ADR-0120 (first `expert` promotion contract) can now reference + `gsm8k_capability_shape` as the lane shape it requires. The only + remaining gap is the real-GSM8K-test seal (Phase 5.7) and the + numeric ε threshold for the depth curve (ADR-0120's call). +- The "all-refused" edge case (where the lane refuses every input) + passes the shape gate intentionally. That makes the shape narrowly + about *runner self-consistency*, not capability. ADR-0120 sets the + capability bar. + +--- + +## Out of scope + +- Sealing the real GSM8K test set (ADR-0119.7). +- The numeric ε threshold for the depth curve (ADR-0120). +- A minimum `correct_rate` for the lane to qualify as `expert` + evidence — that's a capability claim, lives in ADR-0120. +- Cross-split gates (e.g. holdout must score within Δ of public). + Those are anti-overfitting properties already handled by + Obligation #2 (OOD) and Obligation #6 (depth curve); the shape + checker doesn't duplicate them. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 8450abe9..d6c54c7f 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -53,6 +53,7 @@ ADRs record significant architectural decisions: what was decided, why, what alt | [ADR-0119.4](ADR-0119.4-frontier-baseline-comparison.md) | GSM8K Math: Frontier-Baseline Comparison (ADR-0114a §Obligation #7) | Accepted (2026-05-22) | | [ADR-0119.5](ADR-0119.5-adversarial-generation.md) | GSM8K Math Adversarial Generation (ADR-0114a Obligation #8) | Accepted (2026-05-23) | | [ADR-0119.6](ADR-0119.6-depth-curve-harness.md) | GSM8K Math Depth-Curve Measurement Harness | Accepted (2026-05-23) | +| [ADR-0119.8](ADR-0119.8-lane-gate.md) | gsm8k_math Overall Lane Gate (`gsm8k_capability_shape`) | Accepted (2026-05-23) | --- @@ -101,6 +102,7 @@ The ADR-0091..0114 slate is fully accepted (0091..0113) plus one proposed-roadma - GSM8K Math: Frontier-Baseline Comparison (citations for Claude 3.5, GPT-4, Gemini 1.5; comparison_v1.json; discharges ADR-0114a §Obligation #7) — ADR-0119.4 - GSM8K Math Depth-Curve Measurement Harness (discharges ADR-0114a Obligation #6 measurement-side) — ADR-0119.6 - GSM8K Math Adversarial Generation (38 cases × 12 families; **closes ADR-0114a Obligation #8**; misparse rate 0/38; 10 of 10 obligations now discharged on main) — ADR-0119.5 +- gsm8k_math Overall Lane Gate (Phase 5.8; new `gsm8k_capability_shape` in `LANE_SHAPE_REGISTRY`; composes wrong==0 + correct+refused==total + overall_pass; live dev 50/50 + public 150/150 pass) — ADR-0119.8 ADR-0080 has also landed: Contemplation Loop Phase 1 adds a read-only frontier-compare miner that emits `SPECULATIVE` findings only. diff --git a/tests/test_adr_0119_8_lane_gate.py b/tests/test_adr_0119_8_lane_gate.py new file mode 100644 index 00000000..1ca678ee --- /dev/null +++ b/tests/test_adr_0119_8_lane_gate.py @@ -0,0 +1,218 @@ +"""ADR-0119.8 — gsm8k_math overall lane gate invariants. + +Pins six load-bearing invariants: + +1. **Registry mapping.** ``LANE_SHAPE_REGISTRY["gsm8k_math"] == + "gsm8k_capability_shape"``. + +2. **Shape checker registered.** ``SHAPE_CHECKERS`` contains the + ``gsm8k_capability_shape`` checker. + +3. **Live lane runner output passes the gate** on dev + public splits + (today: 50/50 + 150/150 correct, 0 wrong, 0 refused). + +4. **Nonzero wrong refuses the gate.** ADR-0114a Obligation #4 enforced. + +5. **Outcome-accounting incompleteness refuses the gate.** If + correct + refused != total, the gate refuses. + +6. **Substrate artifacts exist** for Phase 5.1..5.6 (sealed holdout, + corpus + verify.py, runner, frontier baseline, adversarial + generator, depth-curve harness). Phase 5.7 (sealed GSM8K test) + is documented as pending but not gated here — that's its own + future ADR's job to flip on. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from core.capability.expert_demo import ( + LANE_SHAPE_REGISTRY, + SHAPE_CHECKERS, + _check_gsm8k_capability_shape, + resolve_lane_shape, +) +from evals.gsm8k_math.runner import run_lane + + +_REPO_ROOT = Path(__file__).resolve().parent.parent + + +def _load_jsonl(path: Path) -> list[dict]: + return [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + + +class TestRegistryMapping: + def test_gsm8k_math_maps_to_capability_shape(self) -> None: + assert LANE_SHAPE_REGISTRY["gsm8k_math"] == "gsm8k_capability_shape" + + def test_resolve_lane_shape_returns_capability_shape(self) -> None: + assert resolve_lane_shape("gsm8k_math") == "gsm8k_capability_shape" + + def test_shape_checkers_includes_capability_shape(self) -> None: + assert "gsm8k_capability_shape" in SHAPE_CHECKERS + + +class TestLiveLaneRunnerPassesGate: + def test_dev_split_metrics_pass(self) -> None: + cases = _load_jsonl(_REPO_ROOT / "evals/gsm8k_math/dev/cases.jsonl") + report = run_lane(cases) + ok, reason = _check_gsm8k_capability_shape("gsm8k_math", report.metrics) + assert ok, f"dev split gate failed: {reason}" + + def test_public_split_metrics_pass(self) -> None: + cases = _load_jsonl(_REPO_ROOT / "evals/gsm8k_math/public/v1/cases.jsonl") + report = run_lane(cases) + ok, reason = _check_gsm8k_capability_shape("gsm8k_math", report.metrics) + assert ok, f"public split gate failed: {reason}" + + +class TestGateRefusesNonzeroWrong: + def test_wrong_one_refuses(self) -> None: + metrics = { + "cases_total": 10, + "correct": 9, + "wrong": 1, + "refused": 0, + } + ok, reason = _check_gsm8k_capability_shape("gsm8k_math", metrics) + assert ok is False + assert "wrong=1" in reason + assert "Obligation #4" in reason + + +class TestGateRefusesIncompleteAccounting: + def test_correct_plus_refused_not_total(self) -> None: + metrics = { + "cases_total": 10, + "correct": 7, + "wrong": 0, + "refused": 2, # 7 + 2 = 9, not 10 + } + ok, reason = _check_gsm8k_capability_shape("gsm8k_math", metrics) + assert ok is False + assert "outcome accounting incomplete" in reason + + +class TestGateRefusesMissingFields: + @pytest.mark.parametrize( + "missing", ["cases_total", "correct", "wrong", "refused"] + ) + def test_missing_field_refuses(self, missing: str) -> None: + metrics = { + "cases_total": 10, + "correct": 10, + "wrong": 0, + "refused": 0, + } + del metrics[missing] + ok, reason = _check_gsm8k_capability_shape("gsm8k_math", metrics) + assert ok is False + assert missing in reason + + +class TestGateRefusesZeroTotal: + def test_zero_total_refuses(self) -> None: + metrics = { + "cases_total": 0, + "correct": 0, + "wrong": 0, + "refused": 0, + } + ok, reason = _check_gsm8k_capability_shape("gsm8k_math", metrics) + assert ok is False + assert "cases_total=0" in reason + + +class TestGateAcceptsCleanMetrics: + def test_clean_balanced_metrics_pass(self) -> None: + metrics = { + "cases_total": 100, + "correct": 95, + "wrong": 0, + "refused": 5, + "overall_pass": True, + } + ok, reason = _check_gsm8k_capability_shape("gsm8k_math", metrics) + assert ok is True + assert reason == "" + + def test_all_refused_balanced_passes(self) -> None: + """Edge case: 0 correct, 0 wrong, all refused — still passes the + shape gate. (This means CORE refused every case; whether that's + acceptable for ``expert`` promotion is ADR-0120's job, not this + layer's.)""" + metrics = { + "cases_total": 50, + "correct": 0, + "wrong": 0, + "refused": 50, + "overall_pass": True, + } + ok, reason = _check_gsm8k_capability_shape("gsm8k_math", metrics) + assert ok is True + + +class TestSubstrateArtifactsExist: + """Verify Phase 5.1..5.6 substrate is in place on disk.""" + + def test_phase_5_1_sealed_holdout_artifact(self) -> None: + """ADR-0119.1 — fabrication_control holdout is age-encrypted.""" + sealed = _REPO_ROOT / "evals/fabrication_control/holdouts/v1/cases.jsonl.age" + assert sealed.exists() and sealed.is_file() + # age header starts with "age-encryption.org/" + assert sealed.read_bytes().startswith(b"age-encryption.org/") + + def test_phase_5_2_corpus_and_verify(self) -> None: + """ADR-0119.2 — dev + public + verify.py.""" + dev = _REPO_ROOT / "evals/gsm8k_math/dev/cases.jsonl" + public = _REPO_ROOT / "evals/gsm8k_math/public/v1/cases.jsonl" + verify_py = _REPO_ROOT / "evals/gsm8k_math/verify.py" + for p in (dev, public, verify_py): + assert p.exists(), f"missing Phase 5.2 artifact: {p}" + assert len(_load_jsonl(dev)) == 50 + assert len(_load_jsonl(public)) == 150 + + def test_phase_5_3_runner(self) -> None: + runner_py = _REPO_ROOT / "evals/gsm8k_math/runner.py" + assert runner_py.exists() + + def test_phase_5_4_frontier_baseline(self) -> None: + frontier = _REPO_ROOT / "evals/gsm8k_math/baselines/frontier.json" + comparison = _REPO_ROOT / "evals/gsm8k_math/baselines/comparison_v1.json" + assert frontier.exists() + assert comparison.exists() + + def test_phase_5_5_adversarial(self) -> None: + gen = _REPO_ROOT / "evals/gsm8k_math/adversarial/generator.py" + score = _REPO_ROOT / "evals/gsm8k_math/adversarial/score.py" + assert gen.exists() + assert score.exists() + + def test_phase_5_6_depth_curve(self) -> None: + harness = _REPO_ROOT / "evals/gsm8k_math/scoring/depth_curve.py" + assert harness.exists() + + +class TestPhase5_7PendingDocumented: + """Phase 5.7 (sealed GSM8K test) is the one remaining sub-phase. + + This test pins that it's NOT yet a hard gate — that's the future + ADR-0119.7's job. Here we just confirm the placeholder is in place + and the gate doesn't already fire on a nonexistent sealed test. + """ + + def test_placeholder_present_or_documented(self) -> None: + # The placeholder may be either an empty cases.jsonl or a missing + # cases.jsonl.age file; both shapes are acceptable at this stage. + sealed = _REPO_ROOT / "evals/gsm8k_math/holdouts/v1/cases.jsonl.age" + plaintext = _REPO_ROOT / "evals/gsm8k_math/holdouts/v1/cases.jsonl" + plaintext_fallback = _REPO_ROOT / "evals/gsm8k_math/holdouts/v1/cases_plaintext.jsonl" + assert sealed.exists() or plaintext.exists() or plaintext_fallback.exists(), ( + "Phase 5.7 placeholder missing — expected at least one of " + "cases.jsonl.age / cases.jsonl / cases_plaintext.jsonl" + )