core-labs/core
core-labs/core
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fix(bench): patch apple-uma report paths, decode timing, CLI --report

Browse source 
- Use repo-relative report_path in JSON metadata (no absolute paths)
- Measure decode_array only; precompute encode payload before decode bench
- core bench apple-uma --report writes exactly to PATH; --write-report for defaults
- Add final newlines; regenerate seed report
This commit is contained in:
Shay 2026-06-24 12:27:51 -07:00
parent 941adcfd43
commit c581eb4075
6 changed files with 119 additions and 101 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
benchmarks/apple_uma_mechanical_sympathy.py
View file

@ -617,12 +617,12 @@ def track_array_codec_replay(
encode_timing = _measure_timing(_encode, warmup=warmup, measured=measured) encode_timing = _measure_timing(_encode, warmup=warmup, measured=measured)
def _decode_only() -> np.ndarray:
payload = encode_array(arr) payload = encode_array(arr)
def _decode_only() -> np.ndarray:
return decode_array(payload) return decode_array(payload)
decode_timing = _measure_timing(_decode_only, warmup=warmup, measured=measured) decode_timing = _measure_timing(_decode_only, warmup=warmup, measured=measured)
payload = encode_array(arr)
restored = decode_array(payload) restored = decode_array(payload)
encoded_bytes = len(payload["b64"]) encoded_bytes = len(payload["b64"])
return { return {
@ -673,22 +673,37 @@ def run_benchmark(
} }
def _repo_relative_path(path: Path) -> str | None:
try:
return str(path.resolve().relative_to(PROJECT_ROOT.resolve()))
except ValueError:
return None
def write_json_report( def write_json_report(
report: dict[str, Any], report: dict[str, Any],
*, *,
root: Path | None = None, root: Path | None = None,
dest: Path | None = None,
include_metadata: bool = True, include_metadata: bool = True,
) -> Path: ) -> Path:
if dest is not None:
path = dest
path.parent.mkdir(parents=True, exist_ok=True)
else:
base = root or PROJECT_ROOT / "evals" / "reports" base = root or PROJECT_ROOT / "evals" / "reports"
base.mkdir(parents=True, exist_ok=True) base.mkdir(parents=True, exist_ok=True)
path = base / REPORT_JSON_NAME
out = dict(report) out = dict(report)
if include_metadata: if include_metadata:
out["_metadata"] = { metadata: dict[str, Any] = {
"written_at_unix": time.time(), "written_at_unix": time.time(),
"report_path": str(base / REPORT_JSON_NAME),
"note": "Non-hash metadata section; excluded from deterministic claim payloads.", "note": "Non-hash metadata section; excluded from deterministic claim payloads.",
} }
path = base / REPORT_JSON_NAME rel = _repo_relative_path(path)
if rel is not None:
metadata["report_path"] = rel
out["_metadata"] = metadata
path.write_text( path.write_text(
json.dumps(out, ensure_ascii=False, indent=2, sort_keys=True) + "\n", json.dumps(out, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
encoding="utf-8", encoding="utf-8",

11
core/cli.py
View file

@ -4141,12 +4141,13 @@ def cmd_bench(args: argparse.Namespace) -> int:
) )
with _bench_stdout_guard(args.json): with _bench_stdout_guard(args.json):
uma_report = run_apple_uma_benchmark() uma_report = run_apple_uma_benchmark()
if getattr(args, "write_report", False) or args.report:
if args.report: if args.report:
root = Path(args.report).parent from benchmarks.apple_uma_mechanical_sympathy import write_json_report
write_apple_uma_reports(uma_report, root=root)
print(f"report written: {args.report}", file=sys.stderr) report_path = Path(args.report)
else: write_json_report(uma_report, dest=report_path)
print(f"report written: {report_path}", file=sys.stderr)
elif getattr(args, "write_report", False):
json_path, md_path = write_apple_uma_reports(uma_report) json_path, md_path = write_apple_uma_reports(uma_report)
print(f"report written: {json_path}", file=sys.stderr) print(f"report written: {json_path}", file=sys.stderr)
print(f"summary written: {md_path}", file=sys.stderr) print(f"summary written: {md_path}", file=sys.stderr)

146
evals/reports/apple_uma_mechanical_sympathy_latest.json
View file

@ -1,8 +1,8 @@
{ {
"_metadata": { "_metadata": {
"note": "Non-hash metadata section; excluded from deterministic claim payloads.", "note": "Non-hash metadata section; excluded from deterministic claim payloads.",
"report_path": "/Users/kaizenpro/Projects/core/evals/reports/apple_uma_mechanical_sympathy_latest.json", "report_path": "evals/reports/apple_uma_mechanical_sympathy_latest.json",
"written_at_unix": 1782329051.9986272 "written_at_unix": 1782329239.704502
}, },
"benchmark_name": "CORE Apple Silicon UMA Mechanical Sympathy Benchmark", "benchmark_name": "CORE Apple Silicon UMA Mechanical Sympathy Benchmark",
"benchmark_version": "1.0.0", "benchmark_version": "1.0.0",
@ -96,7 +96,7 @@
}, },
"machine": "arm64", "machine": "arm64",
"memory": { "memory": {
"available_bytes": 4247093248, "available_bytes": 4375969792,
"source": "psutil.virtual_memory", "source": "psutil.virtual_memory",
"total_bytes": 17179869184 "total_bytes": 17179869184
}, },
@ -109,24 +109,24 @@
"tracks": { "tracks": {
"array_codec_replay": { "array_codec_replay": {
"decode_timing": { "decode_timing": {
"max_ms": 0.087833, "max_ms": 0.035041,
"mean_ms": 0.040115, "mean_ms": 0.023529,
"measured_iterations": 50, "measured_iterations": 50,
"min_ms": 0.036959, "min_ms": 0.022875,
"ops_per_sec": 24928.306, "ops_per_sec": 42500.381,
"p50_ms": 0.038542, "p50_ms": 0.023,
"p95_ms": 0.044542, "p95_ms": 0.025875,
"warmup_iterations": 5 "warmup_iterations": 5
}, },
"dtype": "float32", "dtype": "float32",
"encode_timing": { "encode_timing": {
"max_ms": 0.015584, "max_ms": 0.015417,
"mean_ms": 0.015312, "mean_ms": 0.015281,
"measured_iterations": 50, "measured_iterations": 50,
"min_ms": 0.013791, "min_ms": 0.015167,
"ops_per_sec": 65309.705, "ops_per_sec": 65441.682,
"p50_ms": 0.015375, "p50_ms": 0.015291,
"p95_ms": 0.015583, "p95_ms": 0.015417,
"warmup_iterations": 5 "warmup_iterations": 5
}, },
"encoded_bytes": 10924, "encoded_bytes": 10924,
@ -162,13 +162,13 @@
32 32
], ],
"timing": { "timing": {
"max_ms": 1.534875, "max_ms": 1.417042,
"mean_ms": 1.378704, "mean_ms": 1.367876,
"measured_iterations": 50, "measured_iterations": 50,
"min_ms": 1.354083, "min_ms": 1.353834,
"ops_per_sec": 725.319, "ops_per_sec": 731.06,
"p50_ms": 1.365563, "p50_ms": 1.359729,
"p95_ms": 1.433042, "p95_ms": 1.407416,
"warmup_iterations": 5 "warmup_iterations": 5
} }
}, },
@ -189,13 +189,13 @@
32 32
], ],
"timing": { "timing": {
"max_ms": 39.863625, "max_ms": 3.086375,
"mean_ms": 4.150485, "mean_ms": 2.939975,
"measured_iterations": 50, "measured_iterations": 50,
"min_ms": 2.849542, "min_ms": 2.886917,
"ops_per_sec": 240.936, "ops_per_sec": 340.139,
"p50_ms": 2.881021, "p50_ms": 2.926645,
"p95_ms": 5.586291, "p95_ms": 3.044667,
"warmup_iterations": 5 "warmup_iterations": 5
} }
}, },
@ -214,13 +214,13 @@
32 32
], ],
"timing": { "timing": {
"max_ms": 9.633625, "max_ms": 4.999417,
"mean_ms": 3.313125, "mean_ms": 2.823273,
"measured_iterations": 50, "measured_iterations": 50,
"min_ms": 2.689583, "min_ms": 2.685958,
"ops_per_sec": 301.83, "ops_per_sec": 354.199,
"p50_ms": 2.716479, "p50_ms": 2.7085,
"p95_ms": 5.592333, "p95_ms": 3.641125,
"warmup_iterations": 5 "warmup_iterations": 5
} }
}, },
@ -239,13 +239,13 @@
32 32
], ],
"timing": { "timing": {
"max_ms": 0.569708, "max_ms": 0.605458,
"mean_ms": 0.534274, "mean_ms": 0.543464,
"measured_iterations": 50, "measured_iterations": 50,
"min_ms": 0.525417, "min_ms": 0.530541,
"ops_per_sec": 1871.699, "ops_per_sec": 1840.048,
"p50_ms": 0.531354, "p50_ms": 0.536229,
"p95_ms": 0.562083, "p95_ms": 0.598667,
"warmup_iterations": 5 "warmup_iterations": 5
} }
} }
@ -268,16 +268,16 @@
"contiguous": true, "contiguous": true,
"dtype": "float32", "dtype": "float32",
"result_deterministic": true, "result_deterministic": true,
"rows_per_sec": 1776009.18, "rows_per_sec": 1840049.683,
"rust_zero_copy_input_eligible": false, "rust_zero_copy_input_eligible": false,
"timing": { "timing": {
"max_ms": 0.146542, "max_ms": 0.070416,
"mean_ms": 0.072072, "mean_ms": 0.069563,
"measured_iterations": 50, "measured_iterations": 50,
"min_ms": 0.069291, "min_ms": 0.069125,
"ops_per_sec": 13875.072, "ops_per_sec": 14375.388,
"p50_ms": 0.070208, "p50_ms": 0.0695,
"p95_ms": 0.072666, "p95_ms": 0.070209,
"warmup_iterations": 5 "warmup_iterations": 5
}, },
"top_k": 5, "top_k": 5,
@ -303,16 +303,16 @@
"contiguous": true, "contiguous": true,
"dtype": "float32", "dtype": "float32",
"result_deterministic": true, "result_deterministic": true,
"rows_per_sec": 8883489.435, "rows_per_sec": 8937898.275,
"rust_zero_copy_input_eligible": false, "rust_zero_copy_input_eligible": false,
"timing": { "timing": {
"max_ms": 0.152416, "max_ms": 0.151084,
"mean_ms": 0.11527, "mean_ms": 0.114568,
"measured_iterations": 50, "measured_iterations": 50,
"min_ms": 0.111542, "min_ms": 0.112583,
"ops_per_sec": 8675.283, "ops_per_sec": 8728.416,
"p50_ms": 0.113437, "p50_ms": 0.113542,
"p95_ms": 0.128167, "p95_ms": 0.117292,
"warmup_iterations": 5 "warmup_iterations": 5
}, },
"top_k": 5, "top_k": 5,
@ -338,16 +338,16 @@
"contiguous": true, "contiguous": true,
"dtype": "float32", "dtype": "float32",
"result_deterministic": true, "result_deterministic": true,
"rows_per_sec": 14772738.586, "rows_per_sec": 17482143.653,
"rust_zero_copy_input_eligible": false, "rust_zero_copy_input_eligible": false,
"timing": { "timing": {
"max_ms": 0.648541, "max_ms": 0.576834,
"mean_ms": 0.554535, "mean_ms": 0.468592,
"measured_iterations": 50, "measured_iterations": 50,
"min_ms": 0.54, "min_ms": 0.456166,
"ops_per_sec": 1803.313, "ops_per_sec": 2134.051,
"p50_ms": 0.544959, "p50_ms": 0.461208,
"p95_ms": 0.608208, "p95_ms": 0.531042,
"warmup_iterations": 5 "warmup_iterations": 5
}, },
"top_k": 5, "top_k": 5,
@ -373,16 +373,16 @@
"contiguous": true, "contiguous": true,
"dtype": "float32", "dtype": "float32",
"result_deterministic": true, "result_deterministic": true,
"rows_per_sec": 17156750.778, "rows_per_sec": 22937268.738,
"rust_zero_copy_input_eligible": false, "rust_zero_copy_input_eligible": false,
"timing": { "timing": {
"max_ms": 4.390667, "max_ms": 3.492375,
"mean_ms": 3.819838, "mean_ms": 2.857184,
"measured_iterations": 50, "measured_iterations": 50,
"min_ms": 3.007459, "min_ms": 2.711583,
"ops_per_sec": 261.791, "ops_per_sec": 349.995,
"p50_ms": 3.808583, "p50_ms": 2.783229,
"p95_ms": 4.3295, "p95_ms": 3.422333,
"warmup_iterations": 5 "warmup_iterations": 5
}, },
"top_k": 5, "top_k": 5,
@ -422,13 +422,13 @@
}, },
"skipped": false, "skipped": false,
"timing": { "timing": {
"max_ms": 0.497917, "max_ms": 0.245625,
"mean_ms": 0.171413, "mean_ms": 0.166753,
"measured_iterations": 50, "measured_iterations": 50,
"min_ms": 0.148041, "min_ms": 0.149,
"ops_per_sec": 5833.878, "ops_per_sec": 5996.886,
"p50_ms": 0.152709, "p50_ms": 0.15325,
"p95_ms": 0.237083, "p95_ms": 0.231166,
"warmup_iterations": 5 "warmup_iterations": 5
}, },
"trace_hash_present": true, "trace_hash_present": true,

18
evals/reports/apple_uma_mechanical_sympathy_latest.md
View file

@ -20,17 +20,17 @@ memory boundaries. No token generation. No approximate recall.
## 3. Exact CGA recall ## 3. Exact CGA recall
- N=128: p50=0.070 ms, rows/sec=1776009.18, zero-copy eligible=False - N=128: p50=0.070 ms, rows/sec=1840049.683, zero-copy eligible=False
- N=1024: p50=0.113 ms, rows/sec=8883489.435, zero-copy eligible=False - N=1024: p50=0.114 ms, rows/sec=8937898.275, zero-copy eligible=False
- N=8192: p50=0.545 ms, rows/sec=14772738.586, zero-copy eligible=False - N=8192: p50=0.461 ms, rows/sec=17482143.653, zero-copy eligible=False
- N=65536: p50=3.809 ms, rows/sec=17156750.778, zero-copy eligible=False - N=65536: p50=2.783 ms, rows/sec=22937268.738, zero-copy eligible=False
## 4. Cl(4,1) scalar algebra ## 4. Cl(4,1) scalar algebra
- geometric_product: p50=1.366 ms, ops/sec=725.319 - geometric_product: p50=1.360 ms, ops/sec=731.06
- versor_apply: p50=2.881 ms, ops/sec=240.936 - versor_apply: p50=2.927 ms, ops/sec=340.139
- cga_inner: p50=2.716 ms, ops/sec=301.83 - cga_inner: p50=2.708 ms, ops/sec=354.199
- versor_condition: p50=0.531 ms, ops/sec=1871.699 - versor_condition: p50=0.536 ms, ops/sec=1840.048
## 5. FrameVerdict TTFV ## 5. FrameVerdict TTFV
@ -38,7 +38,7 @@ memory boundaries. No token generation. No approximate recall.
## 6. Deterministic replay/persistence ## 6. Deterministic replay/persistence
- encode p50=0.015 ms, decode p50=0.039 ms, bytes=10924 - encode p50=0.015 ms, decode p50=0.023 ms, bytes=10924
## 7. Copy / zero-copy truth table ## 7. Copy / zero-copy truth table

2
tests/test_apple_uma_mechanical_sympathy_benchmark.py
View file

@ -141,6 +141,8 @@ def test_report_writer_creates_json_under_evals_reports(
assert path.parent == tmp_path assert path.parent == tmp_path
loaded = json.loads(path.read_text(encoding="utf-8")) loaded = json.loads(path.read_text(encoding="utf-8"))
assert "_metadata" in loaded assert "_metadata" in loaded
if "report_path" in loaded["_metadata"]:
assert not loaded["_metadata"]["report_path"].startswith("/")
assert loaded["benchmark_name"] == BENCHMARK_NAME assert loaded["benchmark_name"] == BENCHMARK_NAME