core-labs/core
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

feat(adr-0181-p4): audio compiler eval gate lane (sensorium/audio) (#470)

Browse source 
PR-4 of ADR-0181. The acceptance-gate lane that decides whether audio_core_v1's
gate may open. Deterministic synthesis-spec fixtures (no .wav blobs) with
predicted parses, so the gates grade parser semantics as well as determinism.

evals/audio_sensorium/:
- synth.py            deterministic fixture synthesis (PCG64 + float32-at-boundary)
- fixtures.json       5 specs: silence, rising-pitch question, falling statement,
                      noise burst, speech-then-pause
- generate_expected.py reproducible pin generator (uv run -m ...)
- expected_ir.jsonl   frozen canonical_sha256 + ir_sha256 + event_type_counts
- expected_projection.json frozen projection_sha256 + reference versor

tests/test_audio_eval_gates.py (12): the gate table per fixture —
shape/dtype, versor_condition<1e-6, within-run replay, canonical-checksum
stability (hard int/cast-stable pin), IR-replay + frozen ir_sha256, semantic
event_type_counts (parser-accuracy gate), and cross-platform versor stability
within atol=1e-6 of the reference (float-safe per eval plan); plus trace
hygiene and gate-closure refusal.

Verified semantics: rise→prosody.rise, fall→prosody.fall, silence→pause.long+
turn.boundary, noise→nonspeech.noise, speech_then_pause→all three.

Cross-platform note: int/quantized-derived hashes are pinned hard; the float
versor is compared within tolerance rather than hash-pinned, since cos/sin/
geometric_product can differ by a ULP across arches. This is the eval-plan's
"equal within declared numeric tolerance" reading — keeps CI stable.

All audio 44 + arch-invariants 40 + smoke 67 green. No core mutation.
This commit is contained in:
Shay 2026-05-29 11:20:31 -07:00 committed by GitHub
parent d95f8a99e0
commit 53573263cb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 487 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
evals/audio_sensorium/__init__.py Normal file
View file

@ -0,0 +1 @@
"""evals.audio_sensorium — deterministic audio compiler eval lane (ADR-0181 PR-4)."""

5
evals/audio_sensorium/expected_ir.jsonl Normal file
View file

@ -0,0 +1,5 @@
{"canonical_sha256": "bb918147fe10391b43adeba4bd21b9ef32e5bd6c5076c3517733a05ed6dd0569", "event_type_counts": {"pause.long": 1, "turn.boundary": 1}, "id": "silence_500ms", "ir_sha256": "6c6839e91f0604f13eea53b63ab56baa995770522d003709bdb2fb3f4ee987e5"}
{"canonical_sha256": "e41e53d6ed3ecc272d99190c3eca7af1a476f52d5fc87124811ec139f4e741f9", "event_type_counts": {"prosody.rise": 1, "speech.voiced": 1}, "id": "rise_question", "ir_sha256": "ddf46e0b81bf128a0654560e9a0c8e4dd512831f50a3cc63cd3e403421bfeb7c"}
{"canonical_sha256": "e7a9f1c129d925ea8e85bcb594a0326dd84413873586ac3bc10bbf65a2e264da", "event_type_counts": {"prosody.fall": 1, "speech.voiced": 1}, "id": "fall_statement", "ir_sha256": "31e1121c536be22a8dfee12ef1c42b921ffea901e7df4a4040b8ea5cb00479b5"}
{"canonical_sha256": "be7f9a83417b7e3c9dc33afb94c2f5839b47629ec505a984a379d7d071657a4b", "event_type_counts": {"nonspeech.noise": 1}, "id": "noise_burst", "ir_sha256": "1eddf450392a2eead1eadf7cc08618f869e620a409f9fa1178a86999d5bb4f48"}
{"canonical_sha256": "d051b8c1bac51b1cd5eefec39afc8ac89cb9e1c80a0f37e85df2a65bf9f998ee", "event_type_counts": {"pause.long": 1, "speech.voiced": 1, "turn.boundary": 1}, "id": "speech_then_pause", "ir_sha256": "ebcee833ba15eb210d2e7e232761d84190367288b7c5dccc7d0e77a8861e979a"}

187
evals/audio_sensorium/expected_projection.json Normal file
View file

@ -0,0 +1,187 @@
{
"fall_statement": {
"projection_sha256": "a5a950781b0ba0c808d4b0be20010c20067b79c48cd85d934f3267c7c0ef1200",
"reference_versor": [
0.8503938913345337,
0.0,
0.0,
0.0,
0.0,
0.0,
-0.13039138913154602,
0.0,
0.24944056570529938,
0.0,
0.0,
0.4445309042930603,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
]
},
"noise_burst": {
"projection_sha256": "2c7196378d56c2145f347b7dce75220763aa8d31459af679eee09e29df3db06d",
"reference_versor": [
0.7027547359466553,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.7114322185516357,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
]
},
"rise_question": {
"projection_sha256": "2d9f678b3199263667ed8d90aa555938e6369e622f96e1ef2bea85642da5f9fc",
"reference_versor": [
0.8544492125511169,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.25063008069992065,
0.0,
0.4366852343082428,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.12809006869792938,
0.0,
0.0,
0.0,
0.0,
0.0
]
},
"silence_500ms": {
"projection_sha256": "3704165842070b666b94e66d035b3829affae5be807ff2f8a7c0a5119a4c1585",
"reference_versor": [
0.5819125771522522,
0.0,
0.0,
0.0,
0.0,
0.0,
0.5890979170799255,
0.3940075933933258,
0.0,
0.0,
0.3988727033138275,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
]
},
"speech_then_pause": {
"projection_sha256": "69c2d2de82950e0dab3837ac8065a086b81624e0c95abf31cc7887dcd50a0fdb",
"reference_versor": [
0.6149082183837891,
0.0,
0.0,
0.0,
0.0,
0.0,
0.5504857897758484,
0.3634520173072815,
0.18241775035858154,
0.0,
0.325374037027359,
0.16330629587173462,
0.0,
-0.10782112926244736,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
-0.09652497619390488,
0.0,
0.0,
0.0,
0.0,
0.0
]
}
}

20
evals/audio_sensorium/fixtures.json Normal file
View file

@ -0,0 +1,20 @@
{
"sample_rate": 24000,
"comment": "Deterministic synthesis specs. Each fixture is designed with a predictable parse so the gate grades parser semantics, not just determinism (ADR-0181 PR-4).",
"fixtures": [
{"id": "silence_500ms", "kind": "silence", "ms": 500,
"expect": "long pause + turn boundary, no speech"},
{"id": "rise_question", "kind": "tone", "ms": 300, "hz": 150, "sweep": 90, "amp": 0.5,
"expect": "voiced speech + rising final contour"},
{"id": "fall_statement", "kind": "tone", "ms": 300, "hz": 230, "sweep": -90, "amp": 0.5,
"expect": "voiced speech + falling final contour"},
{"id": "noise_burst", "kind": "noise", "ms": 300, "seed": 7, "amp": 0.3,
"expect": "non-speech broadband noise"},
{"id": "speech_then_pause", "kind": "concat",
"parts": [
{"kind": "tone", "ms": 300, "hz": 150, "amp": 0.5},
{"kind": "silence", "ms": 400}
],
"expect": "voiced speech then long pause + turn boundary"}
]
}

66
evals/audio_sensorium/generate_expected.py Normal file
View file

@ -0,0 +1,66 @@
"""
evals/audio_sensorium/generate_expected.py — (re)generate frozen expected artifacts.
Run from the repo root: ``uv run python -m evals.audio_sensorium.generate_expected``
Produces (committed, reviewed):
- expected_ir.jsonl per fixture: canonical_sha256, ir_sha256, event_type_counts
- expected_projection.json per fixture: projection_sha256 + reference_versor
The int/quantized-derived pins (ir_sha256, event_type_counts, canonical_sha256)
are platform-stable and asserted hard by the gate tests. The reference_versor is
compared within numeric tolerance (cross-platform float safety — eval plan).
"""
from __future__ import annotations
import json
from collections import Counter
from pathlib import Path
from evals.audio_sensorium.synth import synthesize
from sensorium.audio.canonical import canonicalize
from sensorium.audio.compiler import AudioCompiler
from sensorium.audio.types import AudioIR
_HERE = Path(__file__).resolve().parent
SR = 24_000
def event_type_counts(ir: AudioIR) -> dict[str, int]:
events = (
*ir.speech_spans, *ir.pause_spans, *ir.prosody_arcs,
*ir.turn_events, *ir.non_speech_events, *ir.content_anchors,
)
return dict(sorted(Counter(e.event_type for e in events).items()))
def main() -> None:
spec = json.loads((_HERE / "fixtures.json").read_text())
compiler = AudioCompiler()
ir_lines: list[str] = []
projection: dict[str, dict] = {}
for fx in spec["fixtures"]:
signal = canonicalize(synthesize(fx), SR)
unit = compiler.compile_signal(signal)
ir_lines.append(json.dumps({
"id": fx["id"],
"canonical_sha256": unit.canonical_sha256,
"ir_sha256": unit.ir_sha256,
"event_type_counts": event_type_counts(unit.audio_ir),
}, sort_keys=True))
projection[fx["id"]] = {
"projection_sha256": unit.projection_sha256,
"reference_versor": [float(x) for x in unit.versor.tolist()],
}
(_HERE / "expected_ir.jsonl").write_text("\n".join(ir_lines) + "\n")
(_HERE / "expected_projection.json").write_text(
json.dumps(projection, indent=2, sort_keys=True) + "\n"
)
print(f"wrote expected artifacts for {len(spec['fixtures'])} fixtures")
if __name__ == "__main__":
main()

59
evals/audio_sensorium/synth.py Normal file
View file

@ -0,0 +1,59 @@
"""
evals/audio_sensorium/synth.py — deterministic fixture synthesis (ADR-0181 PR-4).
Fixtures are described as synthesis specs (``fixtures.json``) rather than
committed .wav blobs: the parameters are diffable and the signal is a pure,
reproducible function of them. The same synthesiser is used by the expected-
artifact generator and by the gate tests, so what is pinned is exactly what is
checked.
`numpy.random.Generator(PCG64)` is bit-reproducible across platforms, and
sine/`standard_normal` results are cast to float32 at the boundary — the cast
absorbs cross-platform ULP noise in the float64 transcendentals, so the
canonical-signal hash is stable across builds (the versor itself is checked
within numeric tolerance — see the eval plan).
"""
from __future__ import annotations
import numpy as np
SAMPLE_RATE = 24_000
def _tone(ms: int, hz: float, amp: float, sweep: float) -> np.ndarray:
n = int(SAMPLE_RATE * ms / 1000)
t = np.arange(n, dtype=np.float64) / SAMPLE_RATE
span = t[-1] if t.size > 1 else 1.0
freq = hz + sweep * (t / span) # linear F0 sweep over the span
phase = 2 * np.pi * np.cumsum(freq) / SAMPLE_RATE
return (amp * np.sin(phase)).astype(np.float32)
def _silence(ms: int) -> np.ndarray:
return np.zeros(int(SAMPLE_RATE * ms / 1000), dtype=np.float32)
def _noise(ms: int, amp: float, seed: int) -> np.ndarray:
rng = np.random.default_rng(seed)
n = int(SAMPLE_RATE * ms / 1000)
return (amp * rng.standard_normal(n)).astype(np.float32)
def _part(spec: dict) -> np.ndarray:
kind = spec["kind"]
if kind == "silence":
return _silence(int(spec["ms"]))
if kind == "tone":
return _tone(int(spec["ms"]), float(spec["hz"]),
float(spec.get("amp", 0.5)), float(spec.get("sweep", 0.0)))
if kind == "noise":
return _noise(int(spec["ms"]), float(spec.get("amp", 0.3)), int(spec.get("seed", 0)))
raise ValueError(f"unknown synth kind: {kind!r}")
def synthesize(spec: dict) -> np.ndarray:
"""Return the canonical-rate float32 mono signal for a fixture spec."""
if spec["kind"] == "concat":
return np.concatenate([_part(p) for p in spec["parts"]]).astype(np.float32)
return _part(spec)

149
tests/test_audio_eval_gates.py Normal file
View file

@ -0,0 +1,149 @@
"""
ADR-0181 PR-4 — audio compiler eval gate table.
Implements the acceptance gates that lift `audio_core_v1` from gate-closed to
gate-engaged (eval plan §2). Each fixture is a deterministic synthesis spec
with a *predicted* parse, so the gates grade parser semantics as well as
determinism.
Gate → assertion:
projection shape -> exactly (32,)
projection dtype -> exactly float32
compiler replay -> bit-identical within a run
versor_condition -> < 1e-6
canonical checksum stability-> canonical_sha256 == frozen pin (int/cast-stable)
IR replay -> compile_ir versor equal + ir_sha256 == frozen
cross-platform stability -> versor within atol=1e-6 of reference (float-safe)
semantic structure -> event_type_counts == frozen pin (parser accuracy)
trace hygiene -> no PCM in the evidence trace
gate closure -> closed pack refuses projection
The int-derived pins (ir_sha256, event_type_counts, canonical_sha256) are
asserted hard cross-platform; the float versor is compared within tolerance
(eval plan: "cross-platform stability — equal within declared numeric
tolerance"). Regenerate pins with
``uv run python -m evals.audio_sensorium.generate_expected``.
"""
from __future__ import annotations
import json
from collections import Counter
from pathlib import Path
import numpy as np
import pytest
from evals.audio_sensorium.synth import synthesize
from sensorium.audio.canonical import canonicalize
from sensorium.audio.compiler import AudioCompiler
from sensorium.audio.trace import audio_evidence_trace
from sensorium.audio.types import AudioIR
from sensorium.adapters.audio import make_audio_pack
_EVAL_DIR = Path("evals/audio_sensorium")
SR = 24_000
TOL = 1e-6
def _load_fixtures() -> list[dict]:
return json.loads((_EVAL_DIR / "fixtures.json").read_text())["fixtures"]
def _load_expected_ir() -> dict[str, dict]:
out: dict[str, dict] = {}
for line in (_EVAL_DIR / "expected_ir.jsonl").read_text().splitlines():
if line.strip():
row = json.loads(line)
out[row["id"]] = row
return out
def _load_expected_projection() -> dict[str, dict]:
return json.loads((_EVAL_DIR / "expected_projection.json").read_text())
FIXTURES = _load_fixtures()
EXPECTED_IR = _load_expected_ir()
EXPECTED_PROJ = _load_expected_projection()
IDS = [fx["id"] for fx in FIXTURES]
def _event_type_counts(ir: AudioIR) -> dict[str, int]:
events = (
*ir.speech_spans, *ir.pause_spans, *ir.prosody_arcs,
*ir.turn_events, *ir.non_speech_events, *ir.content_anchors,
)
return dict(sorted(Counter(e.event_type for e in events).items()))
@pytest.fixture(scope="module")
def compiler() -> AudioCompiler:
return AudioCompiler()
@pytest.mark.parametrize("fx", FIXTURES, ids=IDS)
def test_gate_table(fx, compiler):
signal = canonicalize(synthesize(fx), SR)
unit = compiler.compile_signal(signal)
fid = fx["id"]
# shape / dtype
assert unit.versor.shape == (32,)
assert unit.versor.dtype == np.float32
# versor condition (A-5)
assert unit.versor_condition < TOL
# compiler replay — bit-identical within a run (A-1)
again = compiler.compile_signal(canonicalize(synthesize(fx), SR))
assert np.array_equal(unit.versor, again.versor)
assert unit.merge_key == again.merge_key
# canonical checksum stability (int/cast-stable pin)
assert unit.canonical_sha256 == EXPECTED_IR[fid]["canonical_sha256"]
# IR replay + frozen ir_sha256
replay = compiler.compile_ir(unit.audio_ir)
assert np.array_equal(unit.versor, replay.versor)
assert unit.ir_sha256 == replay.ir_sha256 == EXPECTED_IR[fid]["ir_sha256"]
# semantic structure — the parser-accuracy gate
assert _event_type_counts(unit.audio_ir) == EXPECTED_IR[fid]["event_type_counts"]
# cross-platform projection stability — within tolerance of the reference
reference = np.asarray(EXPECTED_PROJ[fid]["reference_versor"], dtype=np.float32)
assert np.allclose(unit.versor, reference, atol=TOL)
@pytest.mark.parametrize("fx", FIXTURES, ids=IDS)
def test_trace_hygiene_no_pcm(fx, compiler):
unit = compiler.compile_signal(canonicalize(synthesize(fx), SR))
trace = audio_evidence_trace(unit)
for value in trace.values():
assert not isinstance(value, (np.ndarray, bytes, bytearray))
assert "samples" not in trace
def test_gate_closure_refuses_projection():
"""A gate-closed pack refuses to project (eval plan: gate closure)."""
from sensorium.registry import ModalityRegistry
sig = canonicalize(synthesize(FIXTURES[1]), SR)
reg = ModalityRegistry()
reg.mount(make_audio_pack("audio_core_v1"), sample=sig)
with pytest.raises(RuntimeError, match="gate is not engaged"):
reg.project("audio_core_v1", sig)
def test_semantic_expectations_match_designed_fixtures():
"""Guard the fixture design intent: the frozen event_type_counts must match
what each fixture's 'expect' note describes (rise/fall/pause/noise)."""
must_contain = {
"silence_500ms": {"pause.long", "turn.boundary"},
"rise_question": {"speech.voiced", "prosody.rise"},
"fall_statement": {"speech.voiced", "prosody.fall"},
"noise_burst": {"nonspeech.noise"},
"speech_then_pause": {"speech.voiced", "pause.long", "turn.boundary"},
}
for fid, required in must_contain.items():
assert required <= set(EXPECTED_IR[fid]["event_type_counts"]), fid