From daa13684f8f1980f3a3a93c1b66dd0b984362069 Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 24 Jun 2026 12:56:57 -0700 Subject: [PATCH] feat(bench): Rust-enabled Apple UMA baseline report lane (PR B) (#906) Add rust_backend_status helper, backend_status report fields, and rust_backend_notes in claim safety audit. Improve diffusion_step skip reasons and markdown backend summary. Document core_rs install and CORE_BACKEND=rust activation in docs/benchmarks/apple-uma-rust-baseline.md. Regenerate seed report under honest Python fallback (core_rs unavailable locally). No scalar Rust binding changes. --- benchmarks/apple_uma_mechanical_sympathy.py | 141 +++++++++++++- docs/benchmarks/apple-uma-rust-baseline.md | 161 +++++++++++++++ .../apple_uma_mechanical_sympathy_latest.json | 184 +++++++++++------- .../apple_uma_mechanical_sympathy_latest.md | 31 ++- ...apple_uma_mechanical_sympathy_benchmark.py | 73 +++++++ 5 files changed, 500 insertions(+), 90 deletions(-) create mode 100644 docs/benchmarks/apple-uma-rust-baseline.md diff --git a/benchmarks/apple_uma_mechanical_sympathy.py b/benchmarks/apple_uma_mechanical_sympathy.py index ba816e47..e4f163ab 100644 --- a/benchmarks/apple_uma_mechanical_sympathy.py +++ b/benchmarks/apple_uma_mechanical_sympathy.py @@ -35,7 +35,7 @@ REPORT_JSON_NAME = "apple_uma_mechanical_sympathy_latest.json" REPORT_MD_NAME = "apple_uma_mechanical_sympathy_latest.md" BENCHMARK_NAME = "CORE Apple Silicon UMA Mechanical Sympathy Benchmark" -BENCHMARK_VERSION = "1.0.0" +BENCHMARK_VERSION = "1.0.1" N_COMPONENTS = 32 DEFAULT_WARMUP = 5 @@ -182,10 +182,79 @@ def _core_rs_import_status() -> dict[str, Any]: return {"import_succeeded": False, "reason": str(exc)} +_RUST_BACKEND_ALIASES = frozenset({"rust", "core_rs", "rs"}) + + +def rust_backend_status() -> dict[str, Any]: + """Summarize Rust backend availability for report consumers.""" + from algebra import backend as alg_backend + + requested_raw = os.environ.get("CORE_BACKEND", "").strip() + requested_norm = requested_raw.lower() + rust_requested = requested_norm in _RUST_BACKEND_ALIASES + core_rs_status = _core_rs_import_status() + import_succeeded = bool(core_rs_status.get("import_succeeded")) + using_rust = alg_backend.using_rust() + + if using_rust: + native_status = "rust_active" + activation_hint = None + elif rust_requested and not import_succeeded: + native_status = "rust_requested_unavailable" + activation_hint = ( + "CORE_BACKEND requests Rust but core_rs is not installed; " + "run `core rust build` then rerun with CORE_BACKEND=rust" + ) + elif import_succeeded and not rust_requested: + native_status = "rust_importable_python_fallback" + activation_hint = ( + "core_rs is importable but inactive; set CORE_BACKEND=rust to " + "activate the native baseline report" + ) + else: + native_status = "python_fallback" + activation_hint = ( + "Python semantic fallback active; install core_rs and set " + "CORE_BACKEND=rust for the native baseline report" + ) + + return { + "requested_backend": requested_raw or "(default python)", + "rust_backend_requested": rust_requested, + "core_rs_import_succeeded": import_succeeded, + "using_rust": using_rust, + "native_status": native_status, + "activation_hint": activation_hint, + "diffusion_step_eligible": using_rust, + "vault_recall_rust_zero_copy_eligible": using_rust, + "scalar_rust_copy_paths_remain": using_rust, + } + + +def _diffusion_skip_reason(*, using_rust: bool, status: dict[str, Any]) -> str: + if using_rust: + return "unexpected: diffusion_step should not be skipped when using_rust()" + if status["rust_backend_requested"] and not status["core_rs_import_succeeded"]: + return ( + "CORE_BACKEND requests Rust but core_rs is not installed " + "(run `core rust build`)" + ) + if status["core_rs_import_succeeded"] and not status["rust_backend_requested"]: + return ( + "core_rs is importable but CORE_BACKEND is not set to rust " + "(set CORE_BACKEND=rust)" + ) + return ( + "Rust backend not enabled (set CORE_BACKEND=rust and install core_rs " + "via `core rust build`)" + ) + + def collect_machine_metadata() -> dict[str, Any]: from algebra import backend as alg_backend rs_status = _core_rs_import_status() + backend_status = rust_backend_status() return { "platform": platform.platform(), "os": platform.system(), @@ -196,6 +265,7 @@ def collect_machine_metadata() -> dict[str, Any]: "CORE_BACKEND": os.environ.get("CORE_BACKEND", ""), "core_rs": rs_status, "using_rust": alg_backend.using_rust(), + "backend_status": backend_status, } @@ -204,12 +274,18 @@ def collect_machine_metadata() -> dict[str, Any]: # --------------------------------------------------------------------------- -def build_claim_safety_audit(*, using_rust: bool) -> dict[str, list[str]]: +def build_claim_safety_audit( + *, + using_rust: bool, + backend_status: dict[str, Any] | None = None, +) -> dict[str, list[str]]: + status = backend_status or rust_backend_status() safe = [ "array_codec is bit-exact deterministic persistence/replay support.", "FrameVerdict benchmark measures off-serving closed-world proof/verdict latency.", "Exact CGA recall via algebra.backend.vault_recall — no ANN or approximate search.", ] + rust_backend_notes: list[str] = [] if using_rust: safe.append( "vault_recall Rust binding consumes contiguous (N, 32) float32 NumPy " @@ -219,13 +295,42 @@ def build_claim_safety_audit(*, using_rust: bool) -> dict[str, list[str]]: "diffusion_step consumes contiguous input buffers via read-only views " "and returns owned output." ) + rust_backend_notes.append( + "Native Rust backend active (CORE_BACKEND=rust and core_rs loaded)." + ) + rust_backend_notes.append( + "Scalar Cl(4,1) Rust helpers still copy inputs via extract_f32_slice; " + "zero-copy scalar cleanup is future work (ADR-0235 Lane 2 / PR C)." + ) + rust_backend_notes.append( + "Batch inputs for vault_recall and diffusion_step may be zero-copy " + "eligible when contiguous float32." + ) else: safe.append( "Python vault_recall path uses vectorised exact scan when Rust is unavailable." ) + if status["rust_backend_requested"] and not status["core_rs_import_succeeded"]: + rust_backend_notes.append( + "CORE_BACKEND requests Rust but core_rs is not installed; " + "report reflects Python fallback and skipped Rust-only tracks." + ) + elif status["core_rs_import_succeeded"]: + rust_backend_notes.append( + "core_rs is importable but inactive; set CORE_BACKEND=rust for " + "native baseline measurements." + ) + else: + rust_backend_notes.append( + "Rust backend unavailable; report uses Python semantic fallback." + ) + rust_backend_notes.append( + "diffusion_step track skipped until CORE_BACKEND=rust and core_rs are active." + ) return { "safe_claims": safe, + "rust_backend_notes": rust_backend_notes, "unsafe_claims_not_made": [ "No CoreML acceleration claim.", "No Neural Engine acceleration claim.", @@ -507,12 +612,15 @@ def track_diffusion_step( from algebra import backend as alg_backend requested, actual, using_rust = _backend_labels() + status = rust_backend_status() if not using_rust: return { "track": "diffusion_step", "skipped": True, - "reason": "Rust backend not enabled (set CORE_BACKEND=rust and install core_rs)", - "rust_available": False, + "reason": _diffusion_skip_reason(using_rust=using_rust, status=status), + "native_status": status["native_status"], + "rust_available": status["core_rs_import_succeeded"], + "backend_status": status, } n_nodes = 128 @@ -654,6 +762,7 @@ def run_benchmark( ) -> dict[str, Any]: machine = collect_machine_metadata() using_rust = bool(machine["using_rust"]) + backend_status = machine["backend_status"] tracks = { "cl41_scalar_ops": track_cl41_scalar_ops(warmup=warmup, measured=measured), "exact_cga_recall": track_exact_cga_recall(warmup=warmup, measured=measured), @@ -665,8 +774,12 @@ def run_benchmark( "benchmark_name": BENCHMARK_NAME, "benchmark_version": BENCHMARK_VERSION, "machine": machine, + "backend_status": backend_status, "tracks": tracks, - "claim_safety_audit": build_claim_safety_audit(using_rust=using_rust), + "claim_safety_audit": build_claim_safety_audit( + using_rust=using_rust, + backend_status=backend_status, + ), "copy_zero_copy_truth_table": build_copy_zero_copy_truth_table( using_rust=using_rust ), @@ -719,6 +832,7 @@ def write_markdown_summary( base = root or PROJECT_ROOT / "evals" / "reports" base.mkdir(parents=True, exist_ok=True) machine = report["machine"] + backend_status = report.get("backend_status") or machine.get("backend_status", {}) tracks = report["tracks"] audit = report["claim_safety_audit"] truth = report["copy_zero_copy_truth_table"] @@ -743,10 +857,23 @@ def write_markdown_summary( f"- CORE_BACKEND: `{machine['CORE_BACKEND'] or '(default python)'}`", f"- core_rs import: {machine['core_rs'].get('import_succeeded')}", f"- using_rust(): {machine['using_rust']}", - "", - "## 3. Exact CGA recall", + f"- Native status: `{backend_status.get('native_status', 'unknown')}`", + f"- diffusion_step eligible: {backend_status.get('diffusion_step_eligible')}", + f"- vault_recall Rust zero-copy eligible: " + f"{backend_status.get('vault_recall_rust_zero_copy_eligible')}", "", ] + if backend_status.get("activation_hint"): + lines.append(f"- Activation hint: {backend_status['activation_hint']}") + lines.append("") + rust_notes = audit.get("rust_backend_notes", []) + if rust_notes: + lines.append("### Rust backend notes") + lines.append("") + for note in rust_notes: + lines.append(f"- {note}") + lines.append("") + lines.extend(["", "## 3. Exact CGA recall", ""]) recall = tracks["exact_cga_recall"] for case in recall.get("cases", []): lines.append( diff --git a/docs/benchmarks/apple-uma-rust-baseline.md b/docs/benchmarks/apple-uma-rust-baseline.md new file mode 100644 index 00000000..9fe76753 --- /dev/null +++ b/docs/benchmarks/apple-uma-rust-baseline.md @@ -0,0 +1,161 @@ +# Apple UMA Rust Baseline Report + +Engineering guide for generating a **Rust-enabled** Apple Silicon UMA mechanical +sympathy benchmark report. Python remains the semantic source of truth; Rust is +the incumbent native algebra backend activated only when explicitly requested. + +See also: + +- `benchmarks/apple_uma_mechanical_sympathy.py` — benchmark harness +- `docs/outreach/apple-silicon-support-brief.md` — claim-safe outreach framing +- `docs/adr/ADR-0235-apple-silicon-uma-acceleration-lanes.md` — staged acceleration roadmap + +## Prerequisites + +- macOS arm64 (Apple Silicon) recommended; benchmark runs on any platform +- Python environment with repo dependencies (`uv sync`) +- Rust toolchain (`cargo`, `rustc`) for building `core_rs` +- `maturin` (installed automatically by `core rust build`) + +## Install / build `core_rs` + +From the repository root: + +```bash +# Verify Rust toolchain +cargo --version + +# Build and install core_rs into the active Python environment +core rust build + +# Confirm activation (optional; exits nonzero if inactive) +core rust status +core rust status --require-active +``` + +Manual equivalent: + +```bash +uv pip install maturin +python -m maturin develop --release --manifest-path core-rs/Cargo.toml +``` + +The crate lives at `core-rs/` (`module-name = "core_rs"` in `core-rs/pyproject.toml`). + +## Activate the Rust backend + +Rust is **opt-in**. Importing `core_rs` alone does not activate native dispatch. +Set: + +```bash +export CORE_BACKEND=rust +``` + +Accepted aliases: `rust`, `core_rs`, `rs` (case-insensitive). + +Verify in Python: + +```python +from algebra.backend import using_rust +assert using_rust() is True +``` + +Or: + +```bash +CORE_BACKEND=rust core rust status --require-active +``` + +## Run the benchmark + +```bash +CORE_BACKEND=rust uv run python -m benchmarks.apple_uma_mechanical_sympathy --json +CORE_BACKEND=rust uv run python -m benchmarks.apple_uma_mechanical_sympathy --write-report + +CORE_BACKEND=rust uv run python -m core.cli bench --suite apple-uma --json +CORE_BACKEND=rust uv run python -m core.cli bench --suite apple-uma --write-report +``` + +Reports are written to: + +- `evals/reports/apple_uma_mechanical_sympathy_latest.json` +- `evals/reports/apple_uma_mechanical_sympathy_latest.md` + +## What changes when Rust is active + +When `CORE_BACKEND=rust`, `core_rs` is installed, and `using_rust()` is true, +the report should show: + +| Field | Rust-active expectation | +|---|---| +| `machine.using_rust` | `true` | +| `machine.core_rs.import_succeeded` | `true` | +| `backend_status.native_status` | `rust_active` | +| `backend_status.diffusion_step_eligible` | `true` | +| `backend_status.vault_recall_rust_zero_copy_eligible` | `true` | +| `tracks.diffusion_step.skipped` | `false` | +| `tracks.exact_cga_recall.cases[].rust_zero_copy_input_eligible` | `true` (contiguous f32) | +| `claim_safety_audit.rust_backend_notes` | Notes native active + scalar copy paths remain | +| `copy_zero_copy_truth_table` | Includes Rust `vault_recall` and `diffusion_step` zero-copy input rows | + +Scalar Cl(4,1) ops (`geometric_product`, `cga_inner`, `versor_condition`, +`versor_apply`) **still report copy paths** via `extract_f32_slice` until PR C +(scalar zero-copy boundary cleanup). This is intentional and honest. + +## Python fallback (Rust unavailable) + +When `core_rs` is not installed or `CORE_BACKEND` is unset: + +| Field | Python-fallback expectation | +|---|---| +| `machine.using_rust` | `false` | +| `backend_status.native_status` | `python_fallback` or `rust_requested_unavailable` | +| `tracks.diffusion_step.skipped` | `true` with explicit `reason` | +| `claim_safety_audit.rust_backend_notes` | Documents Python fallback / activation hint | +| `copy_zero_copy_truth_table` | `diffusion_step` row shows skipped — Rust unavailable | + +Do **not** publish a Rust-active report without actually running the commands above +on hardware where `using_rust()` is true. + +## Explicit non-claims + +This benchmark and report do **not** claim: + +- MLX as semantic backend or acceleration (future ADR lane) +- CoreML or Neural Engine acceleration +- "Zero-copy everywhere" +- Token-generation throughput +- ANN / approximate recall +- A fixed hardware speedup multiplier +- Apple endorsement, partnership, or product integration + +## Validation + +After generating a report: + +```bash +uv run python -m pytest -q tests/test_apple_uma_mechanical_sympathy_benchmark.py +``` + +When Rust is available: + +```bash +CORE_BACKEND=rust cargo test --manifest-path core-rs/Cargo.toml +CORE_BACKEND=rust uv run python -m pytest -q tests/test_cga_inner_rust_parity.py +``` + +## Troubleshooting + +| Symptom | Likely cause | Fix | +|---|---|---| +| `using_rust(): false` with `CORE_BACKEND=rust` | `core_rs` not built | `core rust build` | +| `using_rust(): false`, `core_rs` importable | `CORE_BACKEND` unset | `export CORE_BACKEND=rust` | +| `diffusion_step.skipped: true` | Rust not active | Follow activation steps above | +| `native_status: rust_requested_unavailable` | Import failed | Rebuild `core_rs`; check `cargo` toolchain | + +## Next lane (PR C) + +Scalar Rust zero-copy boundary cleanup (`feat/rust-scalar-cl41-zero-copy-boundary`) +targets `extract_f32_slice` copy tax on scalar hot paths. That work is separate +from this baseline report lane and requires parity gates before any copy-boundary +claims change. \ No newline at end of file diff --git a/evals/reports/apple_uma_mechanical_sympathy_latest.json b/evals/reports/apple_uma_mechanical_sympathy_latest.json index 496782c0..01312481 100644 --- a/evals/reports/apple_uma_mechanical_sympathy_latest.json +++ b/evals/reports/apple_uma_mechanical_sympathy_latest.json @@ -2,10 +2,21 @@ "_metadata": { "note": "Non-hash metadata section; excluded from deterministic claim payloads.", "report_path": "evals/reports/apple_uma_mechanical_sympathy_latest.json", - "written_at_unix": 1782329239.704502 + "written_at_unix": 1782330167.772608 + }, + "backend_status": { + "activation_hint": "Python semantic fallback active; install core_rs and set CORE_BACKEND=rust for the native baseline report", + "core_rs_import_succeeded": false, + "diffusion_step_eligible": false, + "native_status": "python_fallback", + "requested_backend": "(default python)", + "rust_backend_requested": false, + "scalar_rust_copy_paths_remain": false, + "using_rust": false, + "vault_recall_rust_zero_copy_eligible": false }, "benchmark_name": "CORE Apple Silicon UMA Mechanical Sympathy Benchmark", - "benchmark_version": "1.0.0", + "benchmark_version": "1.0.1", "claim_safety_audit": { "future_work": [ "MLX kernel experiment requires separate ADR/parity lane.", @@ -22,6 +33,10 @@ "diffusion_step returns owned output allocation even when inputs are zero-copy." ], "known_zero_copy_input_paths": [], + "rust_backend_notes": [ + "Rust backend unavailable; report uses Python semantic fallback.", + "diffusion_step track skipped until CORE_BACKEND=rust and core_rs are active." + ], "safe_claims": [ "array_codec is bit-exact deterministic persistence/replay support.", "FrameVerdict benchmark measures off-serving closed-world proof/verdict latency.", @@ -90,13 +105,24 @@ ], "machine": { "CORE_BACKEND": "", + "backend_status": { + "activation_hint": "Python semantic fallback active; install core_rs and set CORE_BACKEND=rust for the native baseline report", + "core_rs_import_succeeded": false, + "diffusion_step_eligible": false, + "native_status": "python_fallback", + "requested_backend": "(default python)", + "rust_backend_requested": false, + "scalar_rust_copy_paths_remain": false, + "using_rust": false, + "vault_recall_rust_zero_copy_eligible": false + }, "core_rs": { "import_succeeded": false, "reason": "No module named 'core_rs'" }, "machine": "arm64", "memory": { - "available_bytes": 4375969792, + "available_bytes": 4473192448, "source": "psutil.virtual_memory", "total_bytes": 17179869184 }, @@ -109,24 +135,24 @@ "tracks": { "array_codec_replay": { "decode_timing": { - "max_ms": 0.035041, - "mean_ms": 0.023529, + "max_ms": 0.025167, + "mean_ms": 0.023081, "measured_iterations": 50, - "min_ms": 0.022875, - "ops_per_sec": 42500.381, + "min_ms": 0.022916, + "ops_per_sec": 43326.204, "p50_ms": 0.023, - "p95_ms": 0.025875, + "p95_ms": 0.023208, "warmup_iterations": 5 }, "dtype": "float32", "encode_timing": { - "max_ms": 0.015417, - "mean_ms": 0.015281, + "max_ms": 0.017291, + "mean_ms": 0.015287, "measured_iterations": 50, - "min_ms": 0.015167, - "ops_per_sec": 65441.682, - "p50_ms": 0.015291, - "p95_ms": 0.015417, + "min_ms": 0.014042, + "ops_per_sec": 65413.257, + "p50_ms": 0.015292, + "p95_ms": 0.015458, "warmup_iterations": 5 }, "encoded_bytes": 10924, @@ -162,13 +188,13 @@ 32 ], "timing": { - "max_ms": 1.417042, - "mean_ms": 1.367876, + "max_ms": 1.497667, + "mean_ms": 1.396194, "measured_iterations": 50, - "min_ms": 1.353834, - "ops_per_sec": 731.06, - "p50_ms": 1.359729, - "p95_ms": 1.407416, + "min_ms": 1.379709, + "ops_per_sec": 716.233, + "p50_ms": 1.387396, + "p95_ms": 1.424292, "warmup_iterations": 5 } }, @@ -189,13 +215,13 @@ 32 ], "timing": { - "max_ms": 3.086375, - "mean_ms": 2.939975, + "max_ms": 3.028709, + "mean_ms": 2.935689, "measured_iterations": 50, - "min_ms": 2.886917, - "ops_per_sec": 340.139, - "p50_ms": 2.926645, - "p95_ms": 3.044667, + "min_ms": 2.88475, + "ops_per_sec": 340.636, + "p50_ms": 2.926646, + "p95_ms": 3.013958, "warmup_iterations": 5 } }, @@ -214,13 +240,13 @@ 32 ], "timing": { - "max_ms": 4.999417, - "mean_ms": 2.823273, + "max_ms": 5.186584, + "mean_ms": 2.855146, "measured_iterations": 50, - "min_ms": 2.685958, - "ops_per_sec": 354.199, - "p50_ms": 2.7085, - "p95_ms": 3.641125, + "min_ms": 2.724542, + "ops_per_sec": 350.245, + "p50_ms": 2.741645, + "p95_ms": 3.58975, "warmup_iterations": 5 } }, @@ -239,13 +265,13 @@ 32 ], "timing": { - "max_ms": 0.605458, - "mean_ms": 0.543464, + "max_ms": 0.602333, + "mean_ms": 0.546057, "measured_iterations": 50, - "min_ms": 0.530541, - "ops_per_sec": 1840.048, - "p50_ms": 0.536229, - "p95_ms": 0.598667, + "min_ms": 0.528708, + "ops_per_sec": 1831.309, + "p50_ms": 0.53575, + "p95_ms": 0.585375, "warmup_iterations": 5 } } @@ -254,7 +280,19 @@ "track": "cl41_scalar_ops" }, "diffusion_step": { - "reason": "Rust backend not enabled (set CORE_BACKEND=rust and install core_rs)", + "backend_status": { + "activation_hint": "Python semantic fallback active; install core_rs and set CORE_BACKEND=rust for the native baseline report", + "core_rs_import_succeeded": false, + "diffusion_step_eligible": false, + "native_status": "python_fallback", + "requested_backend": "(default python)", + "rust_backend_requested": false, + "scalar_rust_copy_paths_remain": false, + "using_rust": false, + "vault_recall_rust_zero_copy_eligible": false + }, + "native_status": "python_fallback", + "reason": "Rust backend not enabled (set CORE_BACKEND=rust and install core_rs via `core rust build`)", "rust_available": false, "skipped": true, "track": "diffusion_step" @@ -268,16 +306,16 @@ "contiguous": true, "dtype": "float32", "result_deterministic": true, - "rows_per_sec": 1840049.683, + "rows_per_sec": 1774127.463, "rust_zero_copy_input_eligible": false, "timing": { - "max_ms": 0.070416, - "mean_ms": 0.069563, + "max_ms": 0.118, + "mean_ms": 0.072148, "measured_iterations": 50, - "min_ms": 0.069125, - "ops_per_sec": 14375.388, - "p50_ms": 0.0695, - "p95_ms": 0.070209, + "min_ms": 0.0705, + "ops_per_sec": 13860.371, + "p50_ms": 0.071, + "p95_ms": 0.073208, "warmup_iterations": 5 }, "top_k": 5, @@ -303,16 +341,16 @@ "contiguous": true, "dtype": "float32", "result_deterministic": true, - "rows_per_sec": 8937898.275, + "rows_per_sec": 8765686.74, "rust_zero_copy_input_eligible": false, "timing": { - "max_ms": 0.151084, - "mean_ms": 0.114568, + "max_ms": 0.197, + "mean_ms": 0.116819, "measured_iterations": 50, - "min_ms": 0.112583, - "ops_per_sec": 8728.416, - "p50_ms": 0.113542, - "p95_ms": 0.117292, + "min_ms": 0.113459, + "ops_per_sec": 8560.241, + "p50_ms": 0.114812, + "p95_ms": 0.118958, "warmup_iterations": 5 }, "top_k": 5, @@ -338,16 +376,16 @@ "contiguous": true, "dtype": "float32", "result_deterministic": true, - "rows_per_sec": 17482143.653, + "rows_per_sec": 17346134.584, "rust_zero_copy_input_eligible": false, "timing": { - "max_ms": 0.576834, - "mean_ms": 0.468592, + "max_ms": 0.603167, + "mean_ms": 0.472267, "measured_iterations": 50, - "min_ms": 0.456166, - "ops_per_sec": 2134.051, - "p50_ms": 0.461208, - "p95_ms": 0.531042, + "min_ms": 0.4565, + "ops_per_sec": 2117.448, + "p50_ms": 0.459, + "p95_ms": 0.558666, "warmup_iterations": 5 }, "top_k": 5, @@ -373,16 +411,16 @@ "contiguous": true, "dtype": "float32", "result_deterministic": true, - "rows_per_sec": 22937268.738, + "rows_per_sec": 19851684.504, "rust_zero_copy_input_eligible": false, "timing": { - "max_ms": 3.492375, - "mean_ms": 2.857184, + "max_ms": 3.369292, + "mean_ms": 3.301282, "measured_iterations": 50, - "min_ms": 2.711583, - "ops_per_sec": 349.995, - "p50_ms": 2.783229, - "p95_ms": 3.422333, + "min_ms": 3.264083, + "ops_per_sec": 302.913, + "p50_ms": 3.299208, + "p95_ms": 3.331209, "warmup_iterations": 5 }, "top_k": 5, @@ -422,13 +460,13 @@ }, "skipped": false, "timing": { - "max_ms": 0.245625, - "mean_ms": 0.166753, + "max_ms": 0.447458, + "mean_ms": 0.169548, "measured_iterations": 50, - "min_ms": 0.149, - "ops_per_sec": 5996.886, - "p50_ms": 0.15325, - "p95_ms": 0.231166, + "min_ms": 0.148083, + "ops_per_sec": 5898.052, + "p50_ms": 0.151417, + "p95_ms": 0.224583, "warmup_iterations": 5 }, "trace_hash_present": true, diff --git a/evals/reports/apple_uma_mechanical_sympathy_latest.md b/evals/reports/apple_uma_mechanical_sympathy_latest.md index 8253e835..f78d0e52 100644 --- a/evals/reports/apple_uma_mechanical_sympathy_latest.md +++ b/evals/reports/apple_uma_mechanical_sympathy_latest.md @@ -1,6 +1,6 @@ # CORE Apple Silicon UMA Mechanical Sympathy Benchmark -Version: 1.0.0 +Version: 1.0.1 ## 1. What this measures @@ -17,24 +17,35 @@ memory boundaries. No token generation. No approximate recall. - CORE_BACKEND: `(default python)` - core_rs import: False - using_rust(): False +- Native status: `python_fallback` +- diffusion_step eligible: False +- vault_recall Rust zero-copy eligible: False + +- Activation hint: Python semantic fallback active; install core_rs and set CORE_BACKEND=rust for the native baseline report + +### Rust backend notes + +- Rust backend unavailable; report uses Python semantic fallback. +- diffusion_step track skipped until CORE_BACKEND=rust and core_rs are active. + ## 3. Exact CGA recall -- N=128: p50=0.070 ms, rows/sec=1840049.683, zero-copy eligible=False -- N=1024: p50=0.114 ms, rows/sec=8937898.275, zero-copy eligible=False -- N=8192: p50=0.461 ms, rows/sec=17482143.653, zero-copy eligible=False -- N=65536: p50=2.783 ms, rows/sec=22937268.738, zero-copy eligible=False +- N=128: p50=0.071 ms, rows/sec=1774127.463, zero-copy eligible=False +- N=1024: p50=0.115 ms, rows/sec=8765686.74, zero-copy eligible=False +- N=8192: p50=0.459 ms, rows/sec=17346134.584, zero-copy eligible=False +- N=65536: p50=3.299 ms, rows/sec=19851684.504, zero-copy eligible=False ## 4. Cl(4,1) scalar algebra -- geometric_product: p50=1.360 ms, ops/sec=731.06 -- versor_apply: p50=2.927 ms, ops/sec=340.139 -- cga_inner: p50=2.708 ms, ops/sec=354.199 -- versor_condition: p50=0.536 ms, ops/sec=1840.048 +- geometric_product: p50=1.387 ms, ops/sec=716.233 +- versor_apply: p50=2.927 ms, ops/sec=340.636 +- cga_inner: p50=2.742 ms, ops/sec=350.245 +- versor_condition: p50=0.536 ms, ops/sec=1831.309 ## 5. FrameVerdict TTFV -- Verdict: entailed_true, p50=0.153 ms, producer=proof_chain.entail +- Verdict: entailed_true, p50=0.151 ms, producer=proof_chain.entail ## 6. Deterministic replay/persistence diff --git a/tests/test_apple_uma_mechanical_sympathy_benchmark.py b/tests/test_apple_uma_mechanical_sympathy_benchmark.py index 05a58f79..f9d99852 100644 --- a/tests/test_apple_uma_mechanical_sympathy_benchmark.py +++ b/tests/test_apple_uma_mechanical_sympathy_benchmark.py @@ -14,8 +14,10 @@ from benchmarks.apple_uma_mechanical_sympathy import ( BENCHMARK_NAME, REPORT_JSON_NAME, build_claim_safety_audit, + build_copy_zero_copy_truth_table, deterministic_closed_frame, run_benchmark, + rust_backend_status, synthetic_matrix, track_array_codec_replay, track_frame_verdict_ttfv, @@ -32,6 +34,7 @@ REQUIRED_TOP_LEVEL_KEYS = frozenset( "benchmark_name", "benchmark_version", "machine", + "backend_status", "tracks", "claim_safety_audit", "copy_zero_copy_truth_table", @@ -76,6 +79,76 @@ def test_skipped_tracks_include_explicit_reasons(fast_bench_kwargs: dict[str, in if diffusion.get("skipped"): assert "reason" in diffusion assert diffusion["reason"] + assert diffusion.get("native_status") == "python_fallback" + assert "backend_status" in diffusion + + +def test_backend_status_present_and_python_fallback( + fast_bench_kwargs: dict[str, int], +) -> None: + with mock.patch.dict("os.environ", {}, clear=True): + report = run_benchmark(**fast_bench_kwargs) + status = report["backend_status"] + assert status["using_rust"] is False + assert status["native_status"] == "python_fallback" + assert status["diffusion_step_eligible"] is False + assert status["activation_hint"] + assert report["machine"]["backend_status"] == status + notes = report["claim_safety_audit"]["rust_backend_notes"] + assert notes + assert any("Python" in n or "diffusion_step" in n for n in notes) + + +def test_rust_backend_status_requested_unavailable(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CORE_BACKEND", "rust") + with mock.patch( + "benchmarks.apple_uma_mechanical_sympathy._core_rs_import_status", + return_value={"import_succeeded": False, "reason": "No module named 'core_rs'"}, + ): + with mock.patch("algebra.backend.using_rust", return_value=False): + status = rust_backend_status() + assert status["native_status"] == "rust_requested_unavailable" + assert status["rust_backend_requested"] is True + assert status["core_rs_import_succeeded"] is False + assert status["activation_hint"] + + +def test_rust_backend_status_active(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CORE_BACKEND", "rust") + with mock.patch( + "benchmarks.apple_uma_mechanical_sympathy._core_rs_import_status", + return_value={"import_succeeded": True}, + ): + with mock.patch("algebra.backend.using_rust", return_value=True): + status = rust_backend_status() + assert status["native_status"] == "rust_active" + assert status["using_rust"] is True + assert status["diffusion_step_eligible"] is True + assert status["activation_hint"] is None + + +def test_claim_safety_audit_rust_active_notes() -> None: + status = { + "rust_backend_requested": True, + "core_rs_import_succeeded": True, + "using_rust": True, + "native_status": "rust_active", + } + audit = build_claim_safety_audit(using_rust=True, backend_status=status) + assert audit["rust_backend_notes"] + assert any("Scalar" in n for n in audit["rust_backend_notes"]) + assert audit["known_zero_copy_input_paths"] + assert any("vault_recall" in n for n in audit["known_zero_copy_input_paths"]) + assert any("vault_recall" in n for n in audit["safe_claims"]) + + +def test_copy_truth_table_includes_rust_zero_copy_rows_when_active() -> None: + table = build_copy_zero_copy_truth_table(using_rust=True) + paths = {row["path"] for row in table} + assert "algebra.backend.vault_recall (Rust)" in paths + assert "algebra.backend.diffusion_step (Rust)" in paths + vault_row = next(r for r in table if r["path"] == "algebra.backend.vault_recall (Rust)") + assert vault_row["zero_copy_input"] == "yes (contiguous float32)" def test_deterministic_synthetic_sanity_checks_stable(