core-labs/core
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

fix(bench): patch apple-uma report paths, decode timing, CLI --report

Browse source 
- Use repo-relative report_path in JSON metadata (no absolute paths)
- Measure decode_array only; precompute encode payload before decode bench
- core bench apple-uma --report writes exactly to PATH; --write-report for defaults
- Add final newlines; regenerate seed report
This commit is contained in:
Shay 2026-06-24 12:27:51 -07:00
parent 941adcfd43
commit c581eb4075
6 changed files with 119 additions and 101 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
benchmarks/apple_uma_mechanical_sympathy.py
View file

@ -617,12 +617,12 @@ def track_array_codec_replay(
encode_timing = _measure_timing(_encode, warmup=warmup, measured=measured)
payload = encode_array(arr)
def _decode_only() -> np.ndarray:
payload = encode_array(arr)
return decode_array(payload)
decode_timing = _measure_timing(_decode_only, warmup=warmup, measured=measured)
payload = encode_array(arr)
restored = decode_array(payload)
encoded_bytes = len(payload["b64"])
return {
@ -673,22 +673,37 @@ def run_benchmark(
}
def _repo_relative_path(path: Path) -> str | None:
try:
return str(path.resolve().relative_to(PROJECT_ROOT.resolve()))
except ValueError:
return None
def write_json_report(
report: dict[str, Any],
*,
root: Path | None = None,
dest: Path | None = None,
include_metadata: bool = True,
) -> Path:
base = root or PROJECT_ROOT / "evals" / "reports"
base.mkdir(parents=True, exist_ok=True)
if dest is not None:
path = dest
path.parent.mkdir(parents=True, exist_ok=True)
else:
base = root or PROJECT_ROOT / "evals" / "reports"
base.mkdir(parents=True, exist_ok=True)
path = base / REPORT_JSON_NAME
out = dict(report)
if include_metadata:
out["_metadata"] = {
metadata: dict[str, Any] = {
"written_at_unix": time.time(),
"report_path": str(base / REPORT_JSON_NAME),
"note": "Non-hash metadata section; excluded from deterministic claim payloads.",
}
path = base / REPORT_JSON_NAME
rel = _repo_relative_path(path)
if rel is not None:
metadata["report_path"] = rel
out["_metadata"] = metadata
path.write_text(
json.dumps(out, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
@ -841,4 +856,4 @@ def _cli_main(argv: list[str] | None = None) -> int:
if __name__ == "__main__":
sys.exit(_cli_main())
sys.exit(_cli_main())

19
core/cli.py
View file

@ -4141,15 +4141,16 @@ def cmd_bench(args: argparse.Namespace) -> int:
)
with _bench_stdout_guard(args.json):
uma_report = run_apple_uma_benchmark()
if getattr(args, "write_report", False) or args.report:
if args.report:
root = Path(args.report).parent
write_apple_uma_reports(uma_report, root=root)
print(f"report written: {args.report}", file=sys.stderr)
else:
json_path, md_path = write_apple_uma_reports(uma_report)
print(f"report written: {json_path}", file=sys.stderr)
print(f"summary written: {md_path}", file=sys.stderr)
if args.report:
from benchmarks.apple_uma_mechanical_sympathy import write_json_report
report_path = Path(args.report)
write_json_report(uma_report, dest=report_path)
print(f"report written: {report_path}", file=sys.stderr)
elif getattr(args, "write_report", False):
json_path, md_path = write_apple_uma_reports(uma_report)
print(f"report written: {json_path}", file=sys.stderr)
print(f"summary written: {md_path}", file=sys.stderr)
if args.json:
print(json.dumps(uma_report, ensure_ascii=False, indent=2, sort_keys=True))
else:

2
docs/outreach/apple-silicon-support-brief.md
View file

@ -63,4 +63,4 @@ python -m benchmarks.apple_uma_mechanical_sympathy --write-report
core bench --suite apple-uma --write-report
```
Reports land under `evals/reports/`. No network access is required.
Reports land under `evals/reports/`. No network access is required.

146
evals/reports/apple_uma_mechanical_sympathy_latest.json
View file

@ -1,8 +1,8 @@
{
"_metadata": {
"note": "Non-hash metadata section; excluded from deterministic claim payloads.",
"report_path": "/Users/kaizenpro/Projects/core/evals/reports/apple_uma_mechanical_sympathy_latest.json",
"written_at_unix": 1782329051.9986272
"report_path": "evals/reports/apple_uma_mechanical_sympathy_latest.json",
"written_at_unix": 1782329239.704502
},
"benchmark_name": "CORE Apple Silicon UMA Mechanical Sympathy Benchmark",
"benchmark_version": "1.0.0",
@ -96,7 +96,7 @@
},
"machine": "arm64",
"memory": {
"available_bytes": 4247093248,
"available_bytes": 4375969792,
"source": "psutil.virtual_memory",
"total_bytes": 17179869184
},
@ -109,24 +109,24 @@
"tracks": {
"array_codec_replay": {
"decode_timing": {
"max_ms": 0.087833,
"mean_ms": 0.040115,
"max_ms": 0.035041,
"mean_ms": 0.023529,
"measured_iterations": 50,
"min_ms": 0.036959,
"ops_per_sec": 24928.306,
"p50_ms": 0.038542,
"p95_ms": 0.044542,
"min_ms": 0.022875,
"ops_per_sec": 42500.381,
"p50_ms": 0.023,
"p95_ms": 0.025875,
"warmup_iterations": 5
},
"dtype": "float32",
"encode_timing": {
"max_ms": 0.015584,
"mean_ms": 0.015312,
"max_ms": 0.015417,
"mean_ms": 0.015281,
"measured_iterations": 50,
"min_ms": 0.013791,
"ops_per_sec": 65309.705,
"p50_ms": 0.015375,
"p95_ms": 0.015583,
"min_ms": 0.015167,
"ops_per_sec": 65441.682,
"p50_ms": 0.015291,
"p95_ms": 0.015417,
"warmup_iterations": 5
},
"encoded_bytes": 10924,
@ -162,13 +162,13 @@
32
],
"timing": {
"max_ms": 1.534875,
"mean_ms": 1.378704,
"max_ms": 1.417042,
"mean_ms": 1.367876,
"measured_iterations": 50,
"min_ms": 1.354083,
"ops_per_sec": 725.319,
"p50_ms": 1.365563,
"p95_ms": 1.433042,
"min_ms": 1.353834,
"ops_per_sec": 731.06,
"p50_ms": 1.359729,
"p95_ms": 1.407416,
"warmup_iterations": 5
}
},
@ -189,13 +189,13 @@
32
],
"timing": {
"max_ms": 39.863625,
"mean_ms": 4.150485,
"max_ms": 3.086375,
"mean_ms": 2.939975,
"measured_iterations": 50,
"min_ms": 2.849542,
"ops_per_sec": 240.936,
"p50_ms": 2.881021,
"p95_ms": 5.586291,
"min_ms": 2.886917,
"ops_per_sec": 340.139,
"p50_ms": 2.926645,
"p95_ms": 3.044667,
"warmup_iterations": 5
}
},
@ -214,13 +214,13 @@
32
],
"timing": {
"max_ms": 9.633625,
"mean_ms": 3.313125,
"max_ms": 4.999417,
"mean_ms": 2.823273,
"measured_iterations": 50,
"min_ms": 2.689583,
"ops_per_sec": 301.83,
"p50_ms": 2.716479,
"p95_ms": 5.592333,
"min_ms": 2.685958,
"ops_per_sec": 354.199,
"p50_ms": 2.7085,
"p95_ms": 3.641125,
"warmup_iterations": 5
}
},
@ -239,13 +239,13 @@
32
],
"timing": {
"max_ms": 0.569708,
"mean_ms": 0.534274,
"max_ms": 0.605458,
"mean_ms": 0.543464,
"measured_iterations": 50,
"min_ms": 0.525417,
"ops_per_sec": 1871.699,
"p50_ms": 0.531354,
"p95_ms": 0.562083,
"min_ms": 0.530541,
"ops_per_sec": 1840.048,
"p50_ms": 0.536229,
"p95_ms": 0.598667,
"warmup_iterations": 5
}
}
@ -268,16 +268,16 @@
"contiguous": true,
"dtype": "float32",
"result_deterministic": true,
"rows_per_sec": 1776009.18,
"rows_per_sec": 1840049.683,
"rust_zero_copy_input_eligible": false,
"timing": {
"max_ms": 0.146542,
"mean_ms": 0.072072,
"max_ms": 0.070416,
"mean_ms": 0.069563,
"measured_iterations": 50,
"min_ms": 0.069291,
"ops_per_sec": 13875.072,
"p50_ms": 0.070208,
"p95_ms": 0.072666,
"min_ms": 0.069125,
"ops_per_sec": 14375.388,
"p50_ms": 0.0695,
"p95_ms": 0.070209,
"warmup_iterations": 5
},
"top_k": 5,
@ -303,16 +303,16 @@
"contiguous": true,
"dtype": "float32",
"result_deterministic": true,
"rows_per_sec": 8883489.435,
"rows_per_sec": 8937898.275,
"rust_zero_copy_input_eligible": false,
"timing": {
"max_ms": 0.152416,
"mean_ms": 0.11527,
"max_ms": 0.151084,
"mean_ms": 0.114568,
"measured_iterations": 50,
"min_ms": 0.111542,
"ops_per_sec": 8675.283,
"p50_ms": 0.113437,
"p95_ms": 0.128167,
"min_ms": 0.112583,
"ops_per_sec": 8728.416,
"p50_ms": 0.113542,
"p95_ms": 0.117292,
"warmup_iterations": 5
},
"top_k": 5,
@ -338,16 +338,16 @@
"contiguous": true,
"dtype": "float32",
"result_deterministic": true,
"rows_per_sec": 14772738.586,
"rows_per_sec": 17482143.653,
"rust_zero_copy_input_eligible": false,
"timing": {
"max_ms": 0.648541,
"mean_ms": 0.554535,
"max_ms": 0.576834,
"mean_ms": 0.468592,
"measured_iterations": 50,
"min_ms": 0.54,
"ops_per_sec": 1803.313,
"p50_ms": 0.544959,
"p95_ms": 0.608208,
"min_ms": 0.456166,
"ops_per_sec": 2134.051,
"p50_ms": 0.461208,
"p95_ms": 0.531042,
"warmup_iterations": 5
},
"top_k": 5,
@ -373,16 +373,16 @@
"contiguous": true,
"dtype": "float32",
"result_deterministic": true,
"rows_per_sec": 17156750.778,
"rows_per_sec": 22937268.738,
"rust_zero_copy_input_eligible": false,
"timing": {
"max_ms": 4.390667,
"mean_ms": 3.819838,
"max_ms": 3.492375,
"mean_ms": 2.857184,
"measured_iterations": 50,
"min_ms": 3.007459,
"ops_per_sec": 261.791,
"p50_ms": 3.808583,
"p95_ms": 4.3295,
"min_ms": 2.711583,
"ops_per_sec": 349.995,
"p50_ms": 2.783229,
"p95_ms": 3.422333,
"warmup_iterations": 5
},
"top_k": 5,
@ -422,13 +422,13 @@
},
"skipped": false,
"timing": {
"max_ms": 0.497917,
"mean_ms": 0.171413,
"max_ms": 0.245625,
"mean_ms": 0.166753,
"measured_iterations": 50,
"min_ms": 0.148041,
"ops_per_sec": 5833.878,
"p50_ms": 0.152709,
"p95_ms": 0.237083,
"min_ms": 0.149,
"ops_per_sec": 5996.886,
"p50_ms": 0.15325,
"p95_ms": 0.231166,
"warmup_iterations": 5
},
"trace_hash_present": true,

18
evals/reports/apple_uma_mechanical_sympathy_latest.md
View file

@ -20,17 +20,17 @@ memory boundaries. No token generation. No approximate recall.
## 3. Exact CGA recall
- N=128: p50=0.070 ms, rows/sec=1776009.18, zero-copy eligible=False
- N=1024: p50=0.113 ms, rows/sec=8883489.435, zero-copy eligible=False
- N=8192: p50=0.545 ms, rows/sec=14772738.586, zero-copy eligible=False
- N=65536: p50=3.809 ms, rows/sec=17156750.778, zero-copy eligible=False
- N=128: p50=0.070 ms, rows/sec=1840049.683, zero-copy eligible=False
- N=1024: p50=0.114 ms, rows/sec=8937898.275, zero-copy eligible=False
- N=8192: p50=0.461 ms, rows/sec=17482143.653, zero-copy eligible=False
- N=65536: p50=2.783 ms, rows/sec=22937268.738, zero-copy eligible=False
## 4. Cl(4,1) scalar algebra
- geometric_product: p50=1.366 ms, ops/sec=725.319
- versor_apply: p50=2.881 ms, ops/sec=240.936
- cga_inner: p50=2.716 ms, ops/sec=301.83
- versor_condition: p50=0.531 ms, ops/sec=1871.699
- geometric_product: p50=1.360 ms, ops/sec=731.06
- versor_apply: p50=2.927 ms, ops/sec=340.139
- cga_inner: p50=2.708 ms, ops/sec=354.199
- versor_condition: p50=0.536 ms, ops/sec=1840.048
## 5. FrameVerdict TTFV
@ -38,7 +38,7 @@ memory boundaries. No token generation. No approximate recall.
## 6. Deterministic replay/persistence
- encode p50=0.015 ms, decode p50=0.039 ms, bytes=10924
- encode p50=0.015 ms, decode p50=0.023 ms, bytes=10924
## 7. Copy / zero-copy truth table

4
tests/test_apple_uma_mechanical_sympathy_benchmark.py
View file

@ -141,6 +141,8 @@ def test_report_writer_creates_json_under_evals_reports(
assert path.parent == tmp_path
loaded = json.loads(path.read_text(encoding="utf-8"))
assert "_metadata" in loaded
if "report_path" in loaded["_metadata"]:
assert not loaded["_metadata"]["report_path"].startswith("/")
assert loaded["benchmark_name"] == BENCHMARK_NAME
@ -165,4 +167,4 @@ def test_benchmark_module_makes_no_network_calls(
raise AssertionError("network access attempted during benchmark")
monkeypatch.setattr(socket.socket, "connect", _blocked)
run_benchmark(**fast_bench_kwargs)
run_benchmark(**fast_bench_kwargs)