From c581eb407590ca624d2d53b748f14bccbd3a87ca Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 24 Jun 2026 12:27:51 -0700 Subject: [PATCH] fix(bench): patch apple-uma report paths, decode timing, CLI --report - Use repo-relative report_path in JSON metadata (no absolute paths) - Measure decode_array only; precompute encode payload before decode bench - core bench apple-uma --report writes exactly to PATH; --write-report for defaults - Add final newlines; regenerate seed report --- benchmarks/apple_uma_mechanical_sympathy.py | 31 +++- core/cli.py | 19 +-- docs/outreach/apple-silicon-support-brief.md | 2 +- .../apple_uma_mechanical_sympathy_latest.json | 146 +++++++++--------- .../apple_uma_mechanical_sympathy_latest.md | 18 +-- ...apple_uma_mechanical_sympathy_benchmark.py | 4 +- 6 files changed, 119 insertions(+), 101 deletions(-) diff --git a/benchmarks/apple_uma_mechanical_sympathy.py b/benchmarks/apple_uma_mechanical_sympathy.py index e9178342..ba816e47 100644 --- a/benchmarks/apple_uma_mechanical_sympathy.py +++ b/benchmarks/apple_uma_mechanical_sympathy.py @@ -617,12 +617,12 @@ def track_array_codec_replay( encode_timing = _measure_timing(_encode, warmup=warmup, measured=measured) + payload = encode_array(arr) + def _decode_only() -> np.ndarray: - payload = encode_array(arr) return decode_array(payload) decode_timing = _measure_timing(_decode_only, warmup=warmup, measured=measured) - payload = encode_array(arr) restored = decode_array(payload) encoded_bytes = len(payload["b64"]) return { @@ -673,22 +673,37 @@ def run_benchmark( } +def _repo_relative_path(path: Path) -> str | None: + try: + return str(path.resolve().relative_to(PROJECT_ROOT.resolve())) + except ValueError: + return None + + def write_json_report( report: dict[str, Any], *, root: Path | None = None, + dest: Path | None = None, include_metadata: bool = True, ) -> Path: - base = root or PROJECT_ROOT / "evals" / "reports" - base.mkdir(parents=True, exist_ok=True) + if dest is not None: + path = dest + path.parent.mkdir(parents=True, exist_ok=True) + else: + base = root or PROJECT_ROOT / "evals" / "reports" + base.mkdir(parents=True, exist_ok=True) + path = base / REPORT_JSON_NAME out = dict(report) if include_metadata: - out["_metadata"] = { + metadata: dict[str, Any] = { "written_at_unix": time.time(), - "report_path": str(base / REPORT_JSON_NAME), "note": "Non-hash metadata section; excluded from deterministic claim payloads.", } - path = base / REPORT_JSON_NAME + rel = _repo_relative_path(path) + if rel is not None: + metadata["report_path"] = rel + out["_metadata"] = metadata path.write_text( json.dumps(out, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8", @@ -841,4 +856,4 @@ def _cli_main(argv: list[str] | None = None) -> int: if __name__ == "__main__": - sys.exit(_cli_main()) \ No newline at end of file + sys.exit(_cli_main()) diff --git a/core/cli.py b/core/cli.py index bbf7dcc3..c343930f 100644 --- a/core/cli.py +++ b/core/cli.py @@ -4141,15 +4141,16 @@ def cmd_bench(args: argparse.Namespace) -> int: ) with _bench_stdout_guard(args.json): uma_report = run_apple_uma_benchmark() - if getattr(args, "write_report", False) or args.report: - if args.report: - root = Path(args.report).parent - write_apple_uma_reports(uma_report, root=root) - print(f"report written: {args.report}", file=sys.stderr) - else: - json_path, md_path = write_apple_uma_reports(uma_report) - print(f"report written: {json_path}", file=sys.stderr) - print(f"summary written: {md_path}", file=sys.stderr) + if args.report: + from benchmarks.apple_uma_mechanical_sympathy import write_json_report + + report_path = Path(args.report) + write_json_report(uma_report, dest=report_path) + print(f"report written: {report_path}", file=sys.stderr) + elif getattr(args, "write_report", False): + json_path, md_path = write_apple_uma_reports(uma_report) + print(f"report written: {json_path}", file=sys.stderr) + print(f"summary written: {md_path}", file=sys.stderr) if args.json: print(json.dumps(uma_report, ensure_ascii=False, indent=2, sort_keys=True)) else: diff --git a/docs/outreach/apple-silicon-support-brief.md b/docs/outreach/apple-silicon-support-brief.md index 8a5e95ac..447c7aa8 100644 --- a/docs/outreach/apple-silicon-support-brief.md +++ b/docs/outreach/apple-silicon-support-brief.md @@ -63,4 +63,4 @@ python -m benchmarks.apple_uma_mechanical_sympathy --write-report core bench --suite apple-uma --write-report ``` -Reports land under `evals/reports/`. No network access is required. \ No newline at end of file +Reports land under `evals/reports/`. No network access is required. diff --git a/evals/reports/apple_uma_mechanical_sympathy_latest.json b/evals/reports/apple_uma_mechanical_sympathy_latest.json index d68178e9..496782c0 100644 --- a/evals/reports/apple_uma_mechanical_sympathy_latest.json +++ b/evals/reports/apple_uma_mechanical_sympathy_latest.json @@ -1,8 +1,8 @@ { "_metadata": { "note": "Non-hash metadata section; excluded from deterministic claim payloads.", - "report_path": "/Users/kaizenpro/Projects/core/evals/reports/apple_uma_mechanical_sympathy_latest.json", - "written_at_unix": 1782329051.9986272 + "report_path": "evals/reports/apple_uma_mechanical_sympathy_latest.json", + "written_at_unix": 1782329239.704502 }, "benchmark_name": "CORE Apple Silicon UMA Mechanical Sympathy Benchmark", "benchmark_version": "1.0.0", @@ -96,7 +96,7 @@ }, "machine": "arm64", "memory": { - "available_bytes": 4247093248, + "available_bytes": 4375969792, "source": "psutil.virtual_memory", "total_bytes": 17179869184 }, @@ -109,24 +109,24 @@ "tracks": { "array_codec_replay": { "decode_timing": { - "max_ms": 0.087833, - "mean_ms": 0.040115, + "max_ms": 0.035041, + "mean_ms": 0.023529, "measured_iterations": 50, - "min_ms": 0.036959, - "ops_per_sec": 24928.306, - "p50_ms": 0.038542, - "p95_ms": 0.044542, + "min_ms": 0.022875, + "ops_per_sec": 42500.381, + "p50_ms": 0.023, + "p95_ms": 0.025875, "warmup_iterations": 5 }, "dtype": "float32", "encode_timing": { - "max_ms": 0.015584, - "mean_ms": 0.015312, + "max_ms": 0.015417, + "mean_ms": 0.015281, "measured_iterations": 50, - "min_ms": 0.013791, - "ops_per_sec": 65309.705, - "p50_ms": 0.015375, - "p95_ms": 0.015583, + "min_ms": 0.015167, + "ops_per_sec": 65441.682, + "p50_ms": 0.015291, + "p95_ms": 0.015417, "warmup_iterations": 5 }, "encoded_bytes": 10924, @@ -162,13 +162,13 @@ 32 ], "timing": { - "max_ms": 1.534875, - "mean_ms": 1.378704, + "max_ms": 1.417042, + "mean_ms": 1.367876, "measured_iterations": 50, - "min_ms": 1.354083, - "ops_per_sec": 725.319, - "p50_ms": 1.365563, - "p95_ms": 1.433042, + "min_ms": 1.353834, + "ops_per_sec": 731.06, + "p50_ms": 1.359729, + "p95_ms": 1.407416, "warmup_iterations": 5 } }, @@ -189,13 +189,13 @@ 32 ], "timing": { - "max_ms": 39.863625, - "mean_ms": 4.150485, + "max_ms": 3.086375, + "mean_ms": 2.939975, "measured_iterations": 50, - "min_ms": 2.849542, - "ops_per_sec": 240.936, - "p50_ms": 2.881021, - "p95_ms": 5.586291, + "min_ms": 2.886917, + "ops_per_sec": 340.139, + "p50_ms": 2.926645, + "p95_ms": 3.044667, "warmup_iterations": 5 } }, @@ -214,13 +214,13 @@ 32 ], "timing": { - "max_ms": 9.633625, - "mean_ms": 3.313125, + "max_ms": 4.999417, + "mean_ms": 2.823273, "measured_iterations": 50, - "min_ms": 2.689583, - "ops_per_sec": 301.83, - "p50_ms": 2.716479, - "p95_ms": 5.592333, + "min_ms": 2.685958, + "ops_per_sec": 354.199, + "p50_ms": 2.7085, + "p95_ms": 3.641125, "warmup_iterations": 5 } }, @@ -239,13 +239,13 @@ 32 ], "timing": { - "max_ms": 0.569708, - "mean_ms": 0.534274, + "max_ms": 0.605458, + "mean_ms": 0.543464, "measured_iterations": 50, - "min_ms": 0.525417, - "ops_per_sec": 1871.699, - "p50_ms": 0.531354, - "p95_ms": 0.562083, + "min_ms": 0.530541, + "ops_per_sec": 1840.048, + "p50_ms": 0.536229, + "p95_ms": 0.598667, "warmup_iterations": 5 } } @@ -268,16 +268,16 @@ "contiguous": true, "dtype": "float32", "result_deterministic": true, - "rows_per_sec": 1776009.18, + "rows_per_sec": 1840049.683, "rust_zero_copy_input_eligible": false, "timing": { - "max_ms": 0.146542, - "mean_ms": 0.072072, + "max_ms": 0.070416, + "mean_ms": 0.069563, "measured_iterations": 50, - "min_ms": 0.069291, - "ops_per_sec": 13875.072, - "p50_ms": 0.070208, - "p95_ms": 0.072666, + "min_ms": 0.069125, + "ops_per_sec": 14375.388, + "p50_ms": 0.0695, + "p95_ms": 0.070209, "warmup_iterations": 5 }, "top_k": 5, @@ -303,16 +303,16 @@ "contiguous": true, "dtype": "float32", "result_deterministic": true, - "rows_per_sec": 8883489.435, + "rows_per_sec": 8937898.275, "rust_zero_copy_input_eligible": false, "timing": { - "max_ms": 0.152416, - "mean_ms": 0.11527, + "max_ms": 0.151084, + "mean_ms": 0.114568, "measured_iterations": 50, - "min_ms": 0.111542, - "ops_per_sec": 8675.283, - "p50_ms": 0.113437, - "p95_ms": 0.128167, + "min_ms": 0.112583, + "ops_per_sec": 8728.416, + "p50_ms": 0.113542, + "p95_ms": 0.117292, "warmup_iterations": 5 }, "top_k": 5, @@ -338,16 +338,16 @@ "contiguous": true, "dtype": "float32", "result_deterministic": true, - "rows_per_sec": 14772738.586, + "rows_per_sec": 17482143.653, "rust_zero_copy_input_eligible": false, "timing": { - "max_ms": 0.648541, - "mean_ms": 0.554535, + "max_ms": 0.576834, + "mean_ms": 0.468592, "measured_iterations": 50, - "min_ms": 0.54, - "ops_per_sec": 1803.313, - "p50_ms": 0.544959, - "p95_ms": 0.608208, + "min_ms": 0.456166, + "ops_per_sec": 2134.051, + "p50_ms": 0.461208, + "p95_ms": 0.531042, "warmup_iterations": 5 }, "top_k": 5, @@ -373,16 +373,16 @@ "contiguous": true, "dtype": "float32", "result_deterministic": true, - "rows_per_sec": 17156750.778, + "rows_per_sec": 22937268.738, "rust_zero_copy_input_eligible": false, "timing": { - "max_ms": 4.390667, - "mean_ms": 3.819838, + "max_ms": 3.492375, + "mean_ms": 2.857184, "measured_iterations": 50, - "min_ms": 3.007459, - "ops_per_sec": 261.791, - "p50_ms": 3.808583, - "p95_ms": 4.3295, + "min_ms": 2.711583, + "ops_per_sec": 349.995, + "p50_ms": 2.783229, + "p95_ms": 3.422333, "warmup_iterations": 5 }, "top_k": 5, @@ -422,13 +422,13 @@ }, "skipped": false, "timing": { - "max_ms": 0.497917, - "mean_ms": 0.171413, + "max_ms": 0.245625, + "mean_ms": 0.166753, "measured_iterations": 50, - "min_ms": 0.148041, - "ops_per_sec": 5833.878, - "p50_ms": 0.152709, - "p95_ms": 0.237083, + "min_ms": 0.149, + "ops_per_sec": 5996.886, + "p50_ms": 0.15325, + "p95_ms": 0.231166, "warmup_iterations": 5 }, "trace_hash_present": true, diff --git a/evals/reports/apple_uma_mechanical_sympathy_latest.md b/evals/reports/apple_uma_mechanical_sympathy_latest.md index 2dae7ee3..8253e835 100644 --- a/evals/reports/apple_uma_mechanical_sympathy_latest.md +++ b/evals/reports/apple_uma_mechanical_sympathy_latest.md @@ -20,17 +20,17 @@ memory boundaries. No token generation. No approximate recall. ## 3. Exact CGA recall -- N=128: p50=0.070 ms, rows/sec=1776009.18, zero-copy eligible=False -- N=1024: p50=0.113 ms, rows/sec=8883489.435, zero-copy eligible=False -- N=8192: p50=0.545 ms, rows/sec=14772738.586, zero-copy eligible=False -- N=65536: p50=3.809 ms, rows/sec=17156750.778, zero-copy eligible=False +- N=128: p50=0.070 ms, rows/sec=1840049.683, zero-copy eligible=False +- N=1024: p50=0.114 ms, rows/sec=8937898.275, zero-copy eligible=False +- N=8192: p50=0.461 ms, rows/sec=17482143.653, zero-copy eligible=False +- N=65536: p50=2.783 ms, rows/sec=22937268.738, zero-copy eligible=False ## 4. Cl(4,1) scalar algebra -- geometric_product: p50=1.366 ms, ops/sec=725.319 -- versor_apply: p50=2.881 ms, ops/sec=240.936 -- cga_inner: p50=2.716 ms, ops/sec=301.83 -- versor_condition: p50=0.531 ms, ops/sec=1871.699 +- geometric_product: p50=1.360 ms, ops/sec=731.06 +- versor_apply: p50=2.927 ms, ops/sec=340.139 +- cga_inner: p50=2.708 ms, ops/sec=354.199 +- versor_condition: p50=0.536 ms, ops/sec=1840.048 ## 5. FrameVerdict TTFV @@ -38,7 +38,7 @@ memory boundaries. No token generation. No approximate recall. ## 6. Deterministic replay/persistence -- encode p50=0.015 ms, decode p50=0.039 ms, bytes=10924 +- encode p50=0.015 ms, decode p50=0.023 ms, bytes=10924 ## 7. Copy / zero-copy truth table diff --git a/tests/test_apple_uma_mechanical_sympathy_benchmark.py b/tests/test_apple_uma_mechanical_sympathy_benchmark.py index b99266cb..05a58f79 100644 --- a/tests/test_apple_uma_mechanical_sympathy_benchmark.py +++ b/tests/test_apple_uma_mechanical_sympathy_benchmark.py @@ -141,6 +141,8 @@ def test_report_writer_creates_json_under_evals_reports( assert path.parent == tmp_path loaded = json.loads(path.read_text(encoding="utf-8")) assert "_metadata" in loaded + if "report_path" in loaded["_metadata"]: + assert not loaded["_metadata"]["report_path"].startswith("/") assert loaded["benchmark_name"] == BENCHMARK_NAME @@ -165,4 +167,4 @@ def test_benchmark_module_makes_no_network_calls( raise AssertionError("network access attempted during benchmark") monkeypatch.setattr(socket.socket, "connect", _blocked) - run_benchmark(**fast_bench_kwargs) \ No newline at end of file + run_benchmark(**fast_bench_kwargs)