diff --git a/evals/audio_sensorium/__init__.py b/evals/audio_sensorium/__init__.py new file mode 100644 index 00000000..be3d11a2 --- /dev/null +++ b/evals/audio_sensorium/__init__.py @@ -0,0 +1 @@ +"""evals.audio_sensorium — deterministic audio compiler eval lane (ADR-0181 PR-4).""" diff --git a/evals/audio_sensorium/expected_ir.jsonl b/evals/audio_sensorium/expected_ir.jsonl new file mode 100644 index 00000000..00e2d8a8 --- /dev/null +++ b/evals/audio_sensorium/expected_ir.jsonl @@ -0,0 +1,5 @@ +{"canonical_sha256": "bb918147fe10391b43adeba4bd21b9ef32e5bd6c5076c3517733a05ed6dd0569", "event_type_counts": {"pause.long": 1, "turn.boundary": 1}, "id": "silence_500ms", "ir_sha256": "6c6839e91f0604f13eea53b63ab56baa995770522d003709bdb2fb3f4ee987e5"} +{"canonical_sha256": "e41e53d6ed3ecc272d99190c3eca7af1a476f52d5fc87124811ec139f4e741f9", "event_type_counts": {"prosody.rise": 1, "speech.voiced": 1}, "id": "rise_question", "ir_sha256": "ddf46e0b81bf128a0654560e9a0c8e4dd512831f50a3cc63cd3e403421bfeb7c"} +{"canonical_sha256": "e7a9f1c129d925ea8e85bcb594a0326dd84413873586ac3bc10bbf65a2e264da", "event_type_counts": {"prosody.fall": 1, "speech.voiced": 1}, "id": "fall_statement", "ir_sha256": "31e1121c536be22a8dfee12ef1c42b921ffea901e7df4a4040b8ea5cb00479b5"} +{"canonical_sha256": "be7f9a83417b7e3c9dc33afb94c2f5839b47629ec505a984a379d7d071657a4b", "event_type_counts": {"nonspeech.noise": 1}, "id": "noise_burst", "ir_sha256": "1eddf450392a2eead1eadf7cc08618f869e620a409f9fa1178a86999d5bb4f48"} +{"canonical_sha256": "d051b8c1bac51b1cd5eefec39afc8ac89cb9e1c80a0f37e85df2a65bf9f998ee", "event_type_counts": {"pause.long": 1, "speech.voiced": 1, "turn.boundary": 1}, "id": "speech_then_pause", "ir_sha256": "ebcee833ba15eb210d2e7e232761d84190367288b7c5dccc7d0e77a8861e979a"} diff --git a/evals/audio_sensorium/expected_projection.json b/evals/audio_sensorium/expected_projection.json new file mode 100644 index 00000000..550c2df2 --- /dev/null +++ b/evals/audio_sensorium/expected_projection.json @@ -0,0 +1,187 @@ +{ + "fall_statement": { + "projection_sha256": "a5a950781b0ba0c808d4b0be20010c20067b79c48cd85d934f3267c7c0ef1200", + "reference_versor": [ + 0.8503938913345337, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + -0.13039138913154602, + 0.0, + 0.24944056570529938, + 0.0, + 0.0, + 0.4445309042930603, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "noise_burst": { + "projection_sha256": "2c7196378d56c2145f347b7dce75220763aa8d31459af679eee09e29df3db06d", + "reference_versor": [ + 0.7027547359466553, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.7114322185516357, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "rise_question": { + "projection_sha256": "2d9f678b3199263667ed8d90aa555938e6369e622f96e1ef2bea85642da5f9fc", + "reference_versor": [ + 0.8544492125511169, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.25063008069992065, + 0.0, + 0.4366852343082428, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.12809006869792938, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "silence_500ms": { + "projection_sha256": "3704165842070b666b94e66d035b3829affae5be807ff2f8a7c0a5119a4c1585", + "reference_versor": [ + 0.5819125771522522, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.5890979170799255, + 0.3940075933933258, + 0.0, + 0.0, + 0.3988727033138275, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "speech_then_pause": { + "projection_sha256": "69c2d2de82950e0dab3837ac8065a086b81624e0c95abf31cc7887dcd50a0fdb", + "reference_versor": [ + 0.6149082183837891, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.5504857897758484, + 0.3634520173072815, + 0.18241775035858154, + 0.0, + 0.325374037027359, + 0.16330629587173462, + 0.0, + -0.10782112926244736, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + -0.09652497619390488, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + } +} diff --git a/evals/audio_sensorium/fixtures.json b/evals/audio_sensorium/fixtures.json new file mode 100644 index 00000000..6d0357be --- /dev/null +++ b/evals/audio_sensorium/fixtures.json @@ -0,0 +1,20 @@ +{ + "sample_rate": 24000, + "comment": "Deterministic synthesis specs. Each fixture is designed with a predictable parse so the gate grades parser semantics, not just determinism (ADR-0181 PR-4).", + "fixtures": [ + {"id": "silence_500ms", "kind": "silence", "ms": 500, + "expect": "long pause + turn boundary, no speech"}, + {"id": "rise_question", "kind": "tone", "ms": 300, "hz": 150, "sweep": 90, "amp": 0.5, + "expect": "voiced speech + rising final contour"}, + {"id": "fall_statement", "kind": "tone", "ms": 300, "hz": 230, "sweep": -90, "amp": 0.5, + "expect": "voiced speech + falling final contour"}, + {"id": "noise_burst", "kind": "noise", "ms": 300, "seed": 7, "amp": 0.3, + "expect": "non-speech broadband noise"}, + {"id": "speech_then_pause", "kind": "concat", + "parts": [ + {"kind": "tone", "ms": 300, "hz": 150, "amp": 0.5}, + {"kind": "silence", "ms": 400} + ], + "expect": "voiced speech then long pause + turn boundary"} + ] +} diff --git a/evals/audio_sensorium/generate_expected.py b/evals/audio_sensorium/generate_expected.py new file mode 100644 index 00000000..fb0338bd --- /dev/null +++ b/evals/audio_sensorium/generate_expected.py @@ -0,0 +1,66 @@ +""" +evals/audio_sensorium/generate_expected.py — (re)generate frozen expected artifacts. + +Run from the repo root: ``uv run python -m evals.audio_sensorium.generate_expected`` + +Produces (committed, reviewed): + - expected_ir.jsonl per fixture: canonical_sha256, ir_sha256, event_type_counts + - expected_projection.json per fixture: projection_sha256 + reference_versor + +The int/quantized-derived pins (ir_sha256, event_type_counts, canonical_sha256) +are platform-stable and asserted hard by the gate tests. The reference_versor is +compared within numeric tolerance (cross-platform float safety — eval plan). +""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path + +from evals.audio_sensorium.synth import synthesize +from sensorium.audio.canonical import canonicalize +from sensorium.audio.compiler import AudioCompiler +from sensorium.audio.types import AudioIR + +_HERE = Path(__file__).resolve().parent +SR = 24_000 + + +def event_type_counts(ir: AudioIR) -> dict[str, int]: + events = ( + *ir.speech_spans, *ir.pause_spans, *ir.prosody_arcs, + *ir.turn_events, *ir.non_speech_events, *ir.content_anchors, + ) + return dict(sorted(Counter(e.event_type for e in events).items())) + + +def main() -> None: + spec = json.loads((_HERE / "fixtures.json").read_text()) + compiler = AudioCompiler() + ir_lines: list[str] = [] + projection: dict[str, dict] = {} + + for fx in spec["fixtures"]: + signal = canonicalize(synthesize(fx), SR) + unit = compiler.compile_signal(signal) + ir_lines.append(json.dumps({ + "id": fx["id"], + "canonical_sha256": unit.canonical_sha256, + "ir_sha256": unit.ir_sha256, + "event_type_counts": event_type_counts(unit.audio_ir), + }, sort_keys=True)) + projection[fx["id"]] = { + "projection_sha256": unit.projection_sha256, + "reference_versor": [float(x) for x in unit.versor.tolist()], + } + + (_HERE / "expected_ir.jsonl").write_text("\n".join(ir_lines) + "\n") + (_HERE / "expected_projection.json").write_text( + json.dumps(projection, indent=2, sort_keys=True) + "\n" + ) + print(f"wrote expected artifacts for {len(spec['fixtures'])} fixtures") + + +if __name__ == "__main__": + main() diff --git a/evals/audio_sensorium/synth.py b/evals/audio_sensorium/synth.py new file mode 100644 index 00000000..b93a701c --- /dev/null +++ b/evals/audio_sensorium/synth.py @@ -0,0 +1,59 @@ +""" +evals/audio_sensorium/synth.py — deterministic fixture synthesis (ADR-0181 PR-4). + +Fixtures are described as synthesis specs (``fixtures.json``) rather than +committed .wav blobs: the parameters are diffable and the signal is a pure, +reproducible function of them. The same synthesiser is used by the expected- +artifact generator and by the gate tests, so what is pinned is exactly what is +checked. + +`numpy.random.Generator(PCG64)` is bit-reproducible across platforms, and +sine/`standard_normal` results are cast to float32 at the boundary — the cast +absorbs cross-platform ULP noise in the float64 transcendentals, so the +canonical-signal hash is stable across builds (the versor itself is checked +within numeric tolerance — see the eval plan). +""" + +from __future__ import annotations + +import numpy as np + +SAMPLE_RATE = 24_000 + + +def _tone(ms: int, hz: float, amp: float, sweep: float) -> np.ndarray: + n = int(SAMPLE_RATE * ms / 1000) + t = np.arange(n, dtype=np.float64) / SAMPLE_RATE + span = t[-1] if t.size > 1 else 1.0 + freq = hz + sweep * (t / span) # linear F0 sweep over the span + phase = 2 * np.pi * np.cumsum(freq) / SAMPLE_RATE + return (amp * np.sin(phase)).astype(np.float32) + + +def _silence(ms: int) -> np.ndarray: + return np.zeros(int(SAMPLE_RATE * ms / 1000), dtype=np.float32) + + +def _noise(ms: int, amp: float, seed: int) -> np.ndarray: + rng = np.random.default_rng(seed) + n = int(SAMPLE_RATE * ms / 1000) + return (amp * rng.standard_normal(n)).astype(np.float32) + + +def _part(spec: dict) -> np.ndarray: + kind = spec["kind"] + if kind == "silence": + return _silence(int(spec["ms"])) + if kind == "tone": + return _tone(int(spec["ms"]), float(spec["hz"]), + float(spec.get("amp", 0.5)), float(spec.get("sweep", 0.0))) + if kind == "noise": + return _noise(int(spec["ms"]), float(spec.get("amp", 0.3)), int(spec.get("seed", 0))) + raise ValueError(f"unknown synth kind: {kind!r}") + + +def synthesize(spec: dict) -> np.ndarray: + """Return the canonical-rate float32 mono signal for a fixture spec.""" + if spec["kind"] == "concat": + return np.concatenate([_part(p) for p in spec["parts"]]).astype(np.float32) + return _part(spec) diff --git a/tests/test_audio_eval_gates.py b/tests/test_audio_eval_gates.py new file mode 100644 index 00000000..6c4b5b11 --- /dev/null +++ b/tests/test_audio_eval_gates.py @@ -0,0 +1,149 @@ +""" +ADR-0181 PR-4 — audio compiler eval gate table. + +Implements the acceptance gates that lift `audio_core_v1` from gate-closed to +gate-engaged (eval plan §2). Each fixture is a deterministic synthesis spec +with a *predicted* parse, so the gates grade parser semantics as well as +determinism. + +Gate → assertion: + projection shape -> exactly (32,) + projection dtype -> exactly float32 + compiler replay -> bit-identical within a run + versor_condition -> < 1e-6 + canonical checksum stability-> canonical_sha256 == frozen pin (int/cast-stable) + IR replay -> compile_ir versor equal + ir_sha256 == frozen + cross-platform stability -> versor within atol=1e-6 of reference (float-safe) + semantic structure -> event_type_counts == frozen pin (parser accuracy) + trace hygiene -> no PCM in the evidence trace + gate closure -> closed pack refuses projection + +The int-derived pins (ir_sha256, event_type_counts, canonical_sha256) are +asserted hard cross-platform; the float versor is compared within tolerance +(eval plan: "cross-platform stability — equal within declared numeric +tolerance"). Regenerate pins with +``uv run python -m evals.audio_sensorium.generate_expected``. +""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path + +import numpy as np +import pytest + +from evals.audio_sensorium.synth import synthesize +from sensorium.audio.canonical import canonicalize +from sensorium.audio.compiler import AudioCompiler +from sensorium.audio.trace import audio_evidence_trace +from sensorium.audio.types import AudioIR +from sensorium.adapters.audio import make_audio_pack + +_EVAL_DIR = Path("evals/audio_sensorium") +SR = 24_000 +TOL = 1e-6 + + +def _load_fixtures() -> list[dict]: + return json.loads((_EVAL_DIR / "fixtures.json").read_text())["fixtures"] + + +def _load_expected_ir() -> dict[str, dict]: + out: dict[str, dict] = {} + for line in (_EVAL_DIR / "expected_ir.jsonl").read_text().splitlines(): + if line.strip(): + row = json.loads(line) + out[row["id"]] = row + return out + + +def _load_expected_projection() -> dict[str, dict]: + return json.loads((_EVAL_DIR / "expected_projection.json").read_text()) + + +FIXTURES = _load_fixtures() +EXPECTED_IR = _load_expected_ir() +EXPECTED_PROJ = _load_expected_projection() +IDS = [fx["id"] for fx in FIXTURES] + + +def _event_type_counts(ir: AudioIR) -> dict[str, int]: + events = ( + *ir.speech_spans, *ir.pause_spans, *ir.prosody_arcs, + *ir.turn_events, *ir.non_speech_events, *ir.content_anchors, + ) + return dict(sorted(Counter(e.event_type for e in events).items())) + + +@pytest.fixture(scope="module") +def compiler() -> AudioCompiler: + return AudioCompiler() + + +@pytest.mark.parametrize("fx", FIXTURES, ids=IDS) +def test_gate_table(fx, compiler): + signal = canonicalize(synthesize(fx), SR) + unit = compiler.compile_signal(signal) + fid = fx["id"] + + # shape / dtype + assert unit.versor.shape == (32,) + assert unit.versor.dtype == np.float32 + + # versor condition (A-5) + assert unit.versor_condition < TOL + + # compiler replay — bit-identical within a run (A-1) + again = compiler.compile_signal(canonicalize(synthesize(fx), SR)) + assert np.array_equal(unit.versor, again.versor) + assert unit.merge_key == again.merge_key + + # canonical checksum stability (int/cast-stable pin) + assert unit.canonical_sha256 == EXPECTED_IR[fid]["canonical_sha256"] + + # IR replay + frozen ir_sha256 + replay = compiler.compile_ir(unit.audio_ir) + assert np.array_equal(unit.versor, replay.versor) + assert unit.ir_sha256 == replay.ir_sha256 == EXPECTED_IR[fid]["ir_sha256"] + + # semantic structure — the parser-accuracy gate + assert _event_type_counts(unit.audio_ir) == EXPECTED_IR[fid]["event_type_counts"] + + # cross-platform projection stability — within tolerance of the reference + reference = np.asarray(EXPECTED_PROJ[fid]["reference_versor"], dtype=np.float32) + assert np.allclose(unit.versor, reference, atol=TOL) + + +@pytest.mark.parametrize("fx", FIXTURES, ids=IDS) +def test_trace_hygiene_no_pcm(fx, compiler): + unit = compiler.compile_signal(canonicalize(synthesize(fx), SR)) + trace = audio_evidence_trace(unit) + for value in trace.values(): + assert not isinstance(value, (np.ndarray, bytes, bytearray)) + assert "samples" not in trace + + +def test_gate_closure_refuses_projection(): + """A gate-closed pack refuses to project (eval plan: gate closure).""" + from sensorium.registry import ModalityRegistry + sig = canonicalize(synthesize(FIXTURES[1]), SR) + reg = ModalityRegistry() + reg.mount(make_audio_pack("audio_core_v1"), sample=sig) + with pytest.raises(RuntimeError, match="gate is not engaged"): + reg.project("audio_core_v1", sig) + + +def test_semantic_expectations_match_designed_fixtures(): + """Guard the fixture design intent: the frozen event_type_counts must match + what each fixture's 'expect' note describes (rise/fall/pause/noise).""" + must_contain = { + "silence_500ms": {"pause.long", "turn.boundary"}, + "rise_question": {"speech.voiced", "prosody.rise"}, + "fall_statement": {"speech.voiced", "prosody.fall"}, + "noise_burst": {"nonspeech.noise"}, + "speech_then_pause": {"speech.voiced", "pause.long", "turn.boundary"}, + } + for fid, required in must_contain.items(): + assert required <= set(EXPECTED_IR[fid]["event_type_counts"]), fid