From f93f756e4490b8d7044451bf4905df1fe557bf93 Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 14 Jul 2026 20:40:52 -0700 Subject: [PATCH] ci: order-of-magnitude CI optimization for single 2-vCPU Act runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of PR checks stuck "Waiting to run": full-pytest on main ran ~12k tests including the slow registry on a 1.5-CPU / 1.2 GiB job cgroup for hours, monopolizing the only runner. Workflows: - full-pytest.yml: fast lane only (`not quarantine and not slow`) - nightly-full-pytest.yml: full suite including slow (schedule + manual) - lane-shas.yml: PR path filters so pin verification is skipped when the change cannot affect lane report bytes Docs: docs/ci-optimization.md + testing-lanes CI mapping. Infra (VM, same change window): job cgroup 2 CPU / 1800 MiB; hung full-pytest task stopped so the queue can drain PR smoke/lane-shas. --- .github/workflows/full-pytest.yml | 41 +++++++++------ .github/workflows/lane-shas.yml | 23 +++++++++ .github/workflows/nightly-full-pytest.yml | 52 +++++++++++++++++++ conftest.py | 5 +- docs/ci-optimization.md | 63 +++++++++++++++++++++++ docs/testing-lanes.md | 16 ++++-- 6 files changed, 178 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/nightly-full-pytest.yml create mode 100644 docs/ci-optimization.md diff --git a/.github/workflows/full-pytest.yml b/.github/workflows/full-pytest.yml index ae93955e..57f3a230 100644 --- a/.github/workflows/full-pytest.yml +++ b/.github/workflows/full-pytest.yml @@ -1,16 +1,20 @@ name: full-pytest -# Post-merge validation — runs the full pytest suite on every push to main. -# PRs are gated by the faster smoke workflow (smoke.yml); this catches -# anything outside the smoke suite within minutes of merge. +# Post-merge gate — runs the **fast** lane on every push to main. # -# Quarantined tests are excluded via the conftest.py QUARANTINE registry. -# The intent is a ratchet: once a test is removed from the registry it -# must keep passing on this gate. +# Why not full+slow here: +# Host is a 2‑vCPU / ~4 GiB VM shared with Forgejo + Postgres + Traefik. +# Job containers were capped at 1.5 CPU / 1.2 GiB. Running the entire +# ~12k-test suite including the slow registry (one 16 min fixture floor) +# monopolized the single Act runner for hours and starved PR smoke / +# lane-shas ("Waiting to run"). # -# See: -# conftest.py — the QUARANTINE registry (one entry per quarantined test) -# docs/test-debt-quarantine.md — cluster diagnoses + removal policy +# Coverage contract: +# - This workflow: ``not quarantine and not slow`` (make test-fast). +# - Slow/soak/proof matrix: ``nightly-full-pytest.yml`` (schedule + manual). +# - PR gate remains smoke.yml (minutes) + path-filtered lane-shas.yml. +# +# See docs/testing-lanes.md and docs/ci-optimization.md. on: push: @@ -25,9 +29,10 @@ concurrency: jobs: pytest: - name: full pytest (-m "not quarantine" -n 2) + name: full pytest (fast lane, not quarantine and not slow) runs-on: ubuntu-latest - timeout-minutes: 45 + # Fast lane should finish well under this on a healthy runner. + timeout-minutes: 30 steps: - name: checkout @@ -45,14 +50,17 @@ jobs: run: | uv pip install -e ".[dev]" pyyaml - - name: pytest (parallel, quarantine excluded) + - name: pytest fast lane (parallel) env: PYTHONPATH: ${{ github.workspace }} CORE_SHOWCASE_SKIP_BUDGET: "1" + # Avoid uv hardlink warning noise across filesystems on Act. + UV_LINK_MODE: copy run: | - uv run pytest -m "not quarantine" -n 2 --tb=short -q --maxfail=10 + # -n 2 matches prior CI; host is 2 vCPU — do not raise without more cores. + uv run pytest -m "not quarantine and not slow" -n 2 --tb=short -q --maxfail=10 - - name: report quarantine size (informational) + - name: report quarantine + slow registry size (informational) if: always() env: PYTHONPATH: ${{ github.workspace }} @@ -60,6 +68,7 @@ jobs: uv run python -c " import sys sys.path.insert(0, '.') - from conftest import QUARANTINE - print(f'::notice title=Quarantine size::{len(QUARANTINE)} tests currently quarantined. Goal: shrink this number.') + from conftest import QUARANTINE, SLOW_FILES, SLOW_TESTS + print(f'::notice title=Quarantine size::{len(QUARANTINE)} tests quarantined.') + print(f'::notice title=Slow registry::{len(SLOW_FILES)} files + {len(SLOW_TESTS)} exact tests classified slow (nightly only).') " diff --git a/.github/workflows/lane-shas.yml b/.github/workflows/lane-shas.yml index aa3fb8de..a9c98bdf 100644 --- a/.github/workflows/lane-shas.yml +++ b/.github/workflows/lane-shas.yml @@ -7,12 +7,34 @@ name: lane-shas # python scripts/verify_lane_shas.py --update # # Single source of truth for the pinned values is scripts/verify_lane_shas.py. +# +# PR path filter: skip the multi-lane wall-clock cost when the change cannot +# affect pin bytes (e.g. docs-only, pure physics off the demo path). Main +# still always verifies after merge. on: push: branches: [main] pull_request: branches: [main] + paths: + - "scripts/verify_lane_shas.py" + - "scripts/generate_claims.py" + - "CLAIMS.md" + - "evals/**" + - "demos/**" + - "core/demos/**" + - "chat/**" + - "generate/**" + - "packs/**" + - "teaching/**" + - "vault/**" + - "algebra/**" + - "core/physics/**" + - "core/cognition/**" + - "core/capability/**" + - "pyproject.toml" + - ".github/workflows/lane-shas.yml" permissions: contents: read @@ -49,6 +71,7 @@ jobs: - name: verify lane SHAs env: PYTHONPATH: ${{ github.workspace }} + UV_LINK_MODE: copy # public_demo wall-clock is soft by default (see evals/public_demo/runner.py). # Do not set CORE_SHOWCASE_HARD_BUDGET here — cold Act runners exceed 60s. # Content cases (claims, determinism, pure composition) remain hard gates. diff --git a/.github/workflows/nightly-full-pytest.yml b/.github/workflows/nightly-full-pytest.yml new file mode 100644 index 00000000..60d5a14f --- /dev/null +++ b/.github/workflows/nightly-full-pytest.yml @@ -0,0 +1,52 @@ +name: nightly-full-pytest + +# Slow/soak lane — the heavyweight registry excluded from post-merge full-pytest. +# Runs on a schedule so it cannot starve PR smoke / lane-shas on the single +# self-hosted Act runner. +# +# Manual: Actions → nightly-full-pytest → Run workflow. + +on: + schedule: + # 06:00 UTC daily — adjust if runner is shared with other heavy jobs. + - cron: "0 6 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: nightly-full-pytest + cancel-in-progress: true + +jobs: + pytest-slow: + name: nightly full (not quarantine, includes slow) + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + ref: main + + - name: set up uv + uses: astral-sh/setup-uv@v5 + with: + python-version: '3.12.13' + enable-cache: true + + - name: install dependencies + run: | + uv pip install -e ".[dev]" pyyaml + + - name: pytest full including slow registry + env: + PYTHONPATH: ${{ github.workspace }} + CORE_SHOWCASE_SKIP_BUDGET: "1" + UV_LINK_MODE: copy + run: | + # Single worker on 2‑vCPU host avoids xdist thrash with 16 min fixtures. + uv run pytest -m "not quarantine" -n 1 --tb=short -q --maxfail=20 diff --git a/conftest.py b/conftest.py index 6e0f0743..2e3c7ad2 100644 --- a/conftest.py +++ b/conftest.py @@ -84,8 +84,9 @@ QUARANTINE: frozenset[str] = frozenset() # slow lane: pytest -m "slow and not quarantine" (make test-slow) # full lane: pytest -m "not quarantine" (make test-full; CI) # -# CI is unchanged: smoke.yml and full-pytest.yml run ``-m "not quarantine"``, -# which still includes slow tests. See docs/testing-lanes.md. +# CI: full-pytest.yml runs the fast lane (``not quarantine and not slow``). +# Nightly workflow runs full including slow. See docs/testing-lanes.md and +# docs/ci-optimization.md. # --------------------------------------------------------------------------- # Whole-file: the cost is carried by a module/session-scoped fixture, so marking diff --git a/docs/ci-optimization.md b/docs/ci-optimization.md new file mode 100644 index 00000000..9967ee1d --- /dev/null +++ b/docs/ci-optimization.md @@ -0,0 +1,63 @@ +# CI optimization (single self-hosted Act runner) + +## Problem (measured 2026-07-15) + +| Fact | Value | +|------|--------| +| Host | 2 vCPU, ~3.8 GiB RAM, shared Forgejo + Postgres + Traefik | +| Job cgroup (was) | `--cpus=1.5 --memory=1200m` | +| Concurrent capacity | 1 job | +| Suite size | ~12.7k tests (`not quarantine`) | +| Post-merge job | `full-pytest` with **slow included** + `-n 2` | +| Observed | One `full-pytest` ran **2+ hours** at ~25% progress; PR **smoke / lane-shas** stayed **Waiting to run** | + +That is not “slow tests” alone — it is **queue starvation**: one oversized job owns the only runner. + +## Order-of-magnitude strategy + +### 1. Right-size the default CI lane (workflow) — largest lever + +| Lane | When | Marker expression | Intent | +|------|------|-------------------|--------| +| **smoke** | every PR | fixed file list | minutes; merge gate | +| **lane-shas** | PR (path-filtered) + main | pin script | only when pin surface can change | +| **full-pytest (main)** | every push to main | `not quarantine and not slow` | fast post-merge ratchet | +| **nightly-full** | schedule + manual | `not quarantine` (includes slow) | soak / proof / eval-matrix | + +Local doc already measured **~7.7×** from “full serial” → “fast + parallel” on a laptop (`docs/testing-lanes.md`). On a 2‑vCPU VM the absolute times are worse, but **excluding slow** removes the 16‑minute parallel floor and hundreds of soak tests — typically multi‑× wall clock and frees the runner for PR gates. + +### 2. Path-filter expensive PR jobs + +`lane-shas` is multi‑minute (demos + showcase). Skip on PR when the diff cannot change pin bytes (docs-only, pure tooling, etc.). Main still always verifies after merge. + +### 3. Runner cgroup (infra) + +`infra/runner/config.yaml` job options should match host reality: + +- Prefer **2 CPUs** for the job when the host is 2‑vCPU (avoid 1.5 + xdist thrash). +- Memory **≤ available** after Forgejo (~800 MiB limit on host stack). +- Keep **capacity 1** until a second runner exists — concurrency >1 on 2 cores makes two jobs slower than one. + +### 4. What will *not* give 100× on this box + +- Throwing more xdist workers at a 2‑vCPU host. +- Running full+slow on every main push. +- Expecting MLX/Rust to speed pytest collection/runtime of the whole suite without suite redesign. + +True second order of magnitude needs either **more hardware** (second runner / larger VM) or **further suite partitioning** (shard by package, change-based test selection). + +## Local policy (solo branch work) + +1. Targeted tests for files you touched. +2. Dual-run only if pin surface changed. +3. Full smoke before merge (or trust green smoke if unchanged smoke paths). +4. Never rely on remote `full-pytest` for PR feedback — it is post-merge / nightly. + +## Related + +- `docs/testing-lanes.md` — fast / slow / full classification +- `conftest.py` — `SLOW_FILES` / `SLOW_TESTS` / `QUARANTINE` +- `.github/workflows/full-pytest.yml` +- `.github/workflows/nightly-full-pytest.yml` +- `.github/workflows/smoke.yml` +- `.github/workflows/lane-shas.yml` diff --git a/docs/testing-lanes.md b/docs/testing-lanes.md index 6db04c4d..23a52a30 100644 --- a/docs/testing-lanes.md +++ b/docs/testing-lanes.md @@ -13,15 +13,23 @@ decorators spread across ~24 files. |---|---|---| | **fast** | `make test-fast` → `pytest -m "not quarantine and not slow"` | everything except the slow registry | | **slow** | `make test-slow` → `pytest -m "slow and not quarantine"` | only the heavyweight registry | -| **full** | `make test-full` → `pytest -m "not quarantine"` | everything (what CI runs) | +| **full** | `make test-full` → `pytest -m "not quarantine"` | everything (local + **nightly** CI) | The marker is **classification only** — it never skips. `-m slow` *selects* the slow tests; you choose a lane with an explicit marker expression. Plain `pytest` (no `-m`) still runs the full suite. -CI is unchanged: `.github/workflows/smoke.yml` and `full-pytest.yml` both run -`-m "not quarantine"`, which includes the slow tests — so the split costs no CI -coverage. +**CI mapping (2026-07-15 optimization):** + +| Workflow | Expression | When | +|----------|------------|------| +| `smoke.yml` | fixed smoke file list | every PR | +| `full-pytest.yml` | `not quarantine and not slow` | every push to **main** | +| `nightly-full-pytest.yml` | `not quarantine` (includes slow) | daily schedule + manual | +| `lane-shas.yml` | pin script | main always; PR only if pin-affecting paths change | + +See `docs/ci-optimization.md` for why the single Act runner cannot run full+slow +on every merge without starving PR gates. ## Measured timings (10-core macOS, `CORE_BACKEND=numpy`)