From 82f9b021fb9e4e03bebce54931bddfae0f1efcef Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 14 Jul 2026 21:02:43 -0700 Subject: [PATCH] ci: fast-lane main gate, nightly full suite, skip-safe lane-shas Unblock the single Act runner: post-merge full-pytest runs the fast marker (not quarantine and not slow); full non-quarantine suite moves to nightly + workflow_dispatch. lane-shas uses job-level path detection so docs-only PRs skip green without hanging required checks. Sync smoke/conftest/docs CI contracts to the PR/main/nightly policy. --- .github/workflows/full-pytest.yml | 34 ++++++---- .github/workflows/lane-shas.yml | 50 ++++++++++++++- .github/workflows/nightly-full-pytest.yml | 72 +++++++++++++++++++++ .github/workflows/smoke.yml | 7 ++- conftest.py | 21 ++++--- docs/ci-optimization.md | 77 +++++++++++++++++++++++ docs/testing-lanes.md | 42 +++++++++++-- 7 files changed, 269 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/nightly-full-pytest.yml create mode 100644 docs/ci-optimization.md diff --git a/.github/workflows/full-pytest.yml b/.github/workflows/full-pytest.yml index ae93955e..087ce13b 100644 --- a/.github/workflows/full-pytest.yml +++ b/.github/workflows/full-pytest.yml @@ -1,16 +1,25 @@ name: full-pytest -# Post-merge validation — runs the full pytest suite on every push to main. -# PRs are gated by the faster smoke workflow (smoke.yml); this catches -# anything outside the smoke suite within minutes of merge. +# Post-merge FAST lane — runs on every push to main. # -# Quarantined tests are excluded via the conftest.py QUARANTINE registry. -# The intent is a ratchet: once a test is removed from the registry it -# must keep passing on this gate. +# Marker: -m "not quarantine and not slow" +# ~9.5k unit/integration tests; excludes the slow registry in conftest.py +# (soak / bench / proof / register-matrix; ~912 tests including the 16 min +# phase2 fixture floor). +# +# Why "full-pytest" still names this file: +# Keep the workflow id stable for Forgejo required-check / history matching. +# The job display name and this comment state the true contract: FAST on main. +# The complete non-quarantine suite runs in nightly-full-pytest.yml. +# +# PR gate: smoke.yml (small critical subset). +# Lane pins: lane-shas.yml. +# Full soak: nightly-full-pytest.yml (schedule + workflow_dispatch). # # See: -# conftest.py — the QUARANTINE registry (one entry per quarantined test) -# docs/test-debt-quarantine.md — cluster diagnoses + removal policy +# conftest.py — QUARANTINE + SLOW_FILES / SLOW_TESTS registries +# docs/testing-lanes.md — lane commands and CI policy (SSoT) +# docs/ci-optimization.md — runner bottleneck + capacity notes on: push: @@ -25,9 +34,10 @@ concurrency: jobs: pytest: - name: full pytest (-m "not quarantine" -n 2) + name: fast pytest (-m "not quarantine and not slow" -n 2) runs-on: ubuntu-latest - timeout-minutes: 45 + # Headroom over ~9.5 min parallel on a 10-core host; Act 2-vCPU is slower. + timeout-minutes: 30 steps: - name: checkout @@ -45,12 +55,12 @@ jobs: run: | uv pip install -e ".[dev]" pyyaml - - name: pytest (parallel, quarantine excluded) + - name: pytest (parallel, quarantine and slow excluded) env: PYTHONPATH: ${{ github.workspace }} CORE_SHOWCASE_SKIP_BUDGET: "1" run: | - uv run pytest -m "not quarantine" -n 2 --tb=short -q --maxfail=10 + uv run pytest -m "not quarantine and not slow" -n 2 --tb=short -q --maxfail=10 - name: report quarantine size (informational) if: always() diff --git a/.github/workflows/lane-shas.yml b/.github/workflows/lane-shas.yml index 3486e817..b9c220a9 100644 --- a/.github/workflows/lane-shas.yml +++ b/.github/workflows/lane-shas.yml @@ -1,12 +1,20 @@ name: lane-shas -# Verify that every ADR-0092..0099 lane produces its pinned SHA-256 +# Verify that every ADR-0092..0104 lane produces its pinned SHA-256 # report. A failing job means a lane's deterministic output changed # without an explicit ADR-tracked pin update via: # # python scripts/verify_lane_shas.py --update # # Single source of truth for the pinned values is scripts/verify_lane_shas.py. +# +# PR path policy (job-level, not workflow-level): +# Pin bytes can move from Python, packs, eval fixtures/corpora, teaching +# corpora, dependency pins, CLAIMS.md, or this workflow. When none of those +# paths change on a PR, we skip the multi-minute runners and exit success so +# a required check never sits "Waiting" forever (workflow-level `paths:` +# would omit the job entirely and hang required status checks). +# Main pushes always verify. on: push: @@ -31,36 +39,72 @@ jobs: - name: checkout uses: actions/checkout@v4 with: - fetch-depth: 1 + # Need base..head for PR path detection. + fetch-depth: 0 + + - name: detect pin-relevant paths + id: paths + shell: bash + run: | + set -euo pipefail + # Always verify on main pushes. + if [ "${{ github.event_name }}" = "push" ]; then + echo "run=true" >> "$GITHUB_OUTPUT" + echo "Pin-relevant path check: always run on push" + exit 0 + fi + + BASE="${{ github.event.pull_request.base.sha }}" + HEAD="${{ github.event.pull_request.head.sha }}" + # Paths that can change pin bytes or CLAIMS generation inputs. + # Keep in sync with docs/testing-lanes.md § CI policy. + PATTERN='(\.py$|^packs/|^evals/|^teaching/|^CLAIMS\.md$|^pyproject\.toml$|^uv\.lock$|^\.github/workflows/lane-shas\.yml$)' + if git diff --name-only "$BASE" "$HEAD" | grep -E "$PATTERN" >/dev/null; then + echo "run=true" >> "$GITHUB_OUTPUT" + echo "Pin-relevant paths changed; running lane SHA verification" + git diff --name-only "$BASE" "$HEAD" | grep -E "$PATTERN" || true + else + echo "run=false" >> "$GITHUB_OUTPUT" + echo "No pin-relevant paths changed; skipping lane SHA verification (job still green)" + fi # setup-uv (not actions/setup-python) provisions Python on the aarch64 # self-hosted runner; actions/setup-python has no arm64 build for the # pinned 3.12.13. Matches smoke.yml / full-pytest.yml. - name: set up uv + if: steps.paths.outputs.run == 'true' uses: astral-sh/setup-uv@v5 with: python-version: '3.12.13' enable-cache: true - name: install dependencies + if: steps.paths.outputs.run == 'true' run: | uv pip install -e . pyyaml pytest - name: verify lane SHAs + if: steps.paths.outputs.run == 'true' env: PYTHONPATH: ${{ github.workspace }} run: | uv run python scripts/verify_lane_shas.py - name: verify CLAIMS.md is current + if: steps.paths.outputs.run == 'true' env: PYTHONPATH: ${{ github.workspace }} run: | uv run python scripts/generate_claims.py --check - name: emit machine-readable report (on failure) - if: failure() + if: failure() && steps.paths.outputs.run == 'true' env: PYTHONPATH: ${{ github.workspace }} run: | uv run python scripts/verify_lane_shas.py --json || true + + - name: skip notice + if: steps.paths.outputs.run != 'true' + run: | + echo "::notice title=lane-shas skipped::No pin-relevant paths in this PR; verification skipped (success)." diff --git a/.github/workflows/nightly-full-pytest.yml b/.github/workflows/nightly-full-pytest.yml new file mode 100644 index 00000000..1920e1fe --- /dev/null +++ b/.github/workflows/nightly-full-pytest.yml @@ -0,0 +1,72 @@ +name: nightly-full-pytest + +# Nightly FULL lane — complete non-quarantine suite including the slow registry. +# +# Marker: -m "not quarantine" +# Includes soak / bench / proof / register-matrix (SLOW_FILES + SLOW_TESTS). +# Intentionally off the PR and post-merge critical path so a single 2-vCPU +# Act runner is not held for 1–2h after every main push (see docs/ci-optimization.md). +# +# Risk owned here: a main merge can break slow tests until the next nightly +# (or a manual workflow_dispatch). Treat red nightlies as release-blocking +# debt; re-run via Actions → nightly-full-pytest → Run workflow after fixes. +# +# See docs/testing-lanes.md for the full CI policy. + +on: + schedule: + # 02:00 UTC daily — off peak for human PR iteration. + - cron: '0 2 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: nightly-full-pytest + cancel-in-progress: false + +jobs: + pytest: + name: full pytest (-m "not quarantine" -n 2) + runs-on: ubuntu-latest + # Full suite parallel floor includes ~16 min phase2 fixture; thrashing on a + # 2-vCPU Act host has been observed well past 60 min. Prefer a red timeout + # only after a genuine hang, not under normal soak load. + timeout-minutes: 120 + + steps: + - name: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + ref: main + + - name: set up uv + uses: astral-sh/setup-uv@v5 + with: + python-version: '3.12.13' + enable-cache: true + + - name: install dependencies + run: | + uv pip install -e ".[dev]" pyyaml + + - name: pytest (parallel, full suite, quarantine excluded) + env: + PYTHONPATH: ${{ github.workspace }} + CORE_SHOWCASE_SKIP_BUDGET: "1" + run: | + uv run pytest -m "not quarantine" -n 2 --tb=short -q --maxfail=10 + + - name: report quarantine size (informational) + if: always() + env: + PYTHONPATH: ${{ github.workspace }} + run: | + uv run python -c " + import sys + sys.path.insert(0, '.') + from conftest import QUARANTINE + print(f'::notice title=Quarantine size::{len(QUARANTINE)} tests currently quarantined. Goal: shrink this number.') + " diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 21668180..acde38c1 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -7,9 +7,10 @@ name: smoke # — ratified packs diverge directionally; pack-invariant refusal floor; no # fabrication). The falsifiability lane adds ~4 min but blocks-on-regression. # -# Full pytest runs post-merge to main (see full-pytest.yml). -# Regressions caught here block the PR; anything outside the smoke -# suite is caught on main within minutes of merge. +# Post-merge on main: full-pytest.yml runs the FAST lane +# (-m "not quarantine and not slow"). Soak / proof / register-matrix coverage +# is nightly-full-pytest.yml (not on the PR critical path). +# See docs/testing-lanes.md. on: pull_request: diff --git a/conftest.py b/conftest.py index 6e0f0743..5b4077c5 100644 --- a/conftest.py +++ b/conftest.py @@ -1,19 +1,18 @@ """Project-root conftest — test classification registries. The QUARANTINE set is the only allowed registry for known-failing tests. -It is currently empty. If it ever contains nodeids, the CI gate at -.github/workflows/full-pytest.yml runs ``pytest -m "not quarantine"`` -so those explicitly tracked failures do not block unrelated PRs. The -suite is a ratchet: a quarantined test removed from this set must pass +It is currently empty. If it ever contains nodeids, CI excludes them via +``-m "not quarantine"`` (smoke, full-pytest fast lane, nightly full). +The suite is a ratchet: a quarantined test removed from this set must pass on its own merits. See docs/test-debt-quarantine.md for current policy and historical cluster -diagnoses. +diagnoses. See docs/testing-lanes.md for CI lane policy (PR / main / nightly). To remove a test from quarantine: 1. Land a PR that makes the test pass. 2. Delete its entry from QUARANTINE in the same PR. - 3. The full-pytest CI gate will now require it to keep passing. + 3. The main fast gate and nightly full gate will both require it to pass. Adding a test to QUARANTINE is strongly discouraged. If a new failure surfaces, the right default is to fix it in the PR that @@ -80,12 +79,14 @@ QUARANTINE: frozenset[str] = frozenset() # a developer run a fast lane locally. Classification adds the ``slow`` marker # ONLY — it never skips — so ``-m slow`` SELECTS these tests. Choose a lane: # -# fast lane: pytest -m "not quarantine and not slow" (make test-fast) +# fast lane: pytest -m "not quarantine and not slow" (make test-fast; +# also full-pytest.yml on main) # slow lane: pytest -m "slow and not quarantine" (make test-slow) -# full lane: pytest -m "not quarantine" (make test-full; CI) +# full lane: pytest -m "not quarantine" (make test-full; +# also nightly-full-pytest.yml) # -# CI is unchanged: smoke.yml and full-pytest.yml run ``-m "not quarantine"``, -# which still includes slow tests. See docs/testing-lanes.md. +# CI policy: PR = smoke subset; main = fast lane; nightly = full including +# slow. See docs/testing-lanes.md. # --------------------------------------------------------------------------- # Whole-file: the cost is carried by a module/session-scoped fixture, so marking diff --git a/docs/ci-optimization.md b/docs/ci-optimization.md new file mode 100644 index 00000000..edb28d95 --- /dev/null +++ b/docs/ci-optimization.md @@ -0,0 +1,77 @@ +# CI optimization — runner bottleneck and lane policy + +Companion to [testing-lanes.md](./testing-lanes.md) (SSoT for markers and +workflow → lane mapping). This note records **why** the multi-lane CI split +exists and what capacity still limits PR queue time. + +## The bottleneck + +A single self-hosted Act runner (≈2 vCPU / limited RAM) serializes all workflows +that target `ubuntu-latest` for this repo. When post-merge CI ran the **full** +non-quarantine suite (`-m "not quarantine"`) with `-n 2`, the slow registry +dominated wall-clock: + +- Module fixture floor: `test_inner_loop_phase2` ≈16 min setup alone +- ~912 tests classified `slow` in `conftest.py` (including the register matrix) +- Thrashing under oversubscription can stretch the job toward **1–2 hours** + +While that job holds the runner, PR gates (`smoke`, `lane-shas`) sit in +**Waiting**. That is a queue problem first, a test-count problem second. + +## What we changed (in-repo) + +| Change | Effect | +|---|---| +| `full-pytest.yml` → fast marker | Main push runs `not quarantine and not slow`; frees the runner much sooner after merge | +| `nightly-full-pytest.yml` | Full suite (`not quarantine`) daily at 02:00 UTC + `workflow_dispatch`; timeout 120 min | +| `lane-shas.yml` job-level path skip | PR pin verify runs only when pin-relevant paths change; job still green when skipped (no required-check hang) | +| Contract comments | `smoke.yml`, `conftest.py`, `testing-lanes.md` match the real PR / main / nightly split | + +Workflow **id** `full-pytest` is kept on purpose so Forgejo history / required-check +names do not break; the job display name is `fast pytest (...)`. + +## What is out of band (ops, not git) + +Host cgroup limits, hung-container cleanup, and adding a second runner are +**VM/ops** actions. They are not encoded in this repository. Document them in +ops runbooks when you change the host; do not treat this file as a substitute. + +## Expected feedback loop after this change + +| Stage | Typical hold | +|---|---| +| Local | Targeted tests; `make test-fast` before push | +| PR | `smoke` (+ `lane-shas` when pin-relevant paths change) | +| Main | Fast `full-pytest` + always-on `lane-shas` | +| Nightly | Full soak including slow registry | + +This is roughly **hours of runner hold after merge → tens of minutes** on the +fast lane for the same host class — not infinite parallel capacity. Multiple +open PRs still queue on one runner. + +## Capacity still required + +1. **Second Act runner or larger VM** — only real fix for concurrent PR + main + execution. +2. **xdist hermeticity** — see follow-ups in `testing-lanes.md` before raising + `-n` further. +3. **Warm-runtime fixture** — long tail of `ChatRuntime` construction in the + fast lane. +4. **Nightly failure signal** — wire Forgejo notification / issue on red + `nightly-full-pytest` so ≤24h slow-break debt is not silent. +5. **Optional path-triggered slow job** — if main must catch soak breaks same-day + without nightly wait, add a non-blocking or path-filtered slow workflow later. + +## Validation + +```bash +# Local parity with main CI +make test-fast + +# Local parity with nightly +make test-full + +# Lane pins (same as lane-shas job body) +uv run python scripts/verify_lane_shas.py +uv run python scripts/generate_claims.py --check +``` diff --git a/docs/testing-lanes.md b/docs/testing-lanes.md index 6db04c4d..6574861c 100644 --- a/docs/testing-lanes.md +++ b/docs/testing-lanes.md @@ -2,26 +2,56 @@ The full pytest suite is ~10,600 tests and ~73 min serial. A small set of heavyweight tests dominates that wall-clock, so we classify them and offer a -**fast lane** for local development. Classification is empirical -test-infrastructure metadata, so it lives in one auditable place +**fast lane** for local development and post-merge CI. Classification is +empirical test-infrastructure metadata, so it lives in one auditable place (`conftest.py`), beside the `QUARANTINE` registry — not as `@pytest.mark.slow` decorators spread across ~24 files. +Runner capacity notes and the single-Act-runner queue story live in +[ci-optimization.md](./ci-optimization.md). **This file is the SSoT for lane +commands and which workflow runs which marker.** + ## Lanes | Lane | Command | What it runs | |---|---|---| | **fast** | `make test-fast` → `pytest -m "not quarantine and not slow"` | everything except the slow registry | | **slow** | `make test-slow` → `pytest -m "slow and not quarantine"` | only the heavyweight registry | -| **full** | `make test-full` → `pytest -m "not quarantine"` | everything (what CI runs) | +| **full** | `make test-full` → `pytest -m "not quarantine"` | everything non-quarantine (local + nightly CI) | The marker is **classification only** — it never skips. `-m slow` *selects* the slow tests; you choose a lane with an explicit marker expression. Plain `pytest` (no `-m`) still runs the full suite. -CI is unchanged: `.github/workflows/smoke.yml` and `full-pytest.yml` both run -`-m "not quarantine"`, which includes the slow tests — so the split costs no CI -coverage. +## CI policy + +| Surface | Workflow | Marker / scope | +|---|---|---| +| **PR** | `smoke.yml` | Fixed critical subset (`not quarantine` within those files) | +| **PR** | `lane-shas.yml` | Pinned ADR lane SHAs + `CLAIMS.md` check; **skipped green** when the PR does not touch pin-relevant paths | +| **main push** | `full-pytest.yml` | **Fast lane** (`not quarantine and not slow`); workflow *id* kept as `full-pytest` for check stability | +| **main push** | `lane-shas.yml` | Always runs (no path skip on push) | +| **nightly / manual** | `nightly-full-pytest.yml` | **Full lane** (`not quarantine`), includes slow registry | + +### Pin-relevant paths (`lane-shas` PR skip) + +On `pull_request`, verification runs only when the base…head diff touches any of: + +- `*.py` +- `packs/**`, `evals/**`, `teaching/**` +- `CLAIMS.md`, `pyproject.toml`, `uv.lock` +- `.github/workflows/lane-shas.yml` + +Otherwise the job still starts and exits **success** (skip notice) so a required +status check never sits on “Waiting” forever. Do **not** reintroduce workflow-level +`on.pull_request.paths` for this gate without a always-green companion job. + +### Owned risk + +Slow/soak/proof tests (including the ~16 min phase2 fixture and the register +matrix) are **not** on the PR or post-merge critical path. A merge can break them +until the next nightly (02:00 UTC) or a manual `workflow_dispatch`. Treat a red +nightly as release-blocking debt. ## Measured timings (10-core macOS, `CORE_BACKEND=numpy`)