diff --git a/core/cli.py b/core/cli.py index 638acec2..8351f8e2 100644 --- a/core/cli.py +++ b/core/cli.py @@ -122,6 +122,7 @@ _TEST_SUITES: dict[str, tuple[str, ...]] = { "tests/test_audio_pack_manifest.py", "tests/test_audio_sensorium_mount.py", "tests/test_vision_compiler.py", + "tests/test_event_vision_compiler.py", "tests/test_vision_crdt_merge.py", "tests/test_vision_eval_gates.py", "tests/test_vision_sensorium_mount.py", @@ -4978,7 +4979,7 @@ def build_parser() -> argparse.ArgumentParser: eval_cmd.add_argument("--report", metavar="PATH", help="write JSON report to file") eval_cmd.add_argument( "--modality", - choices=["audio", "vision", "sensorimotor"], + choices=["audio", "vision", "event-vision", "sensorimotor"], default="vision", help="sensorium lane modality to evaluate (default: vision)", ) diff --git a/evals/event_vision_sensorium/__init__.py b/evals/event_vision_sensorium/__init__.py new file mode 100644 index 00000000..5794e735 --- /dev/null +++ b/evals/event_vision_sensorium/__init__.py @@ -0,0 +1,2 @@ +"""Deterministic synthetic event-vision eval lane.""" + diff --git a/evals/event_vision_sensorium/expected_ir.jsonl b/evals/event_vision_sensorium/expected_ir.jsonl new file mode 100644 index 00000000..ad687390 --- /dev/null +++ b/evals/event_vision_sensorium/expected_ir.jsonl @@ -0,0 +1,3 @@ +{"event_type_counts": {"event.decay": 2, "event.motion_delta": 2, "event.onset": 4}, "id": "edge_sweep_right", "ir_sha256": "ee71e43dfc0638ad95228dc938df58c24af3571aa67462088cbe8cfdca4310c2"} +{"event_type_counts": {"event.decay": 3, "event.motion_delta": 2, "event.onset": 4}, "id": "corner_blink", "ir_sha256": "3532d1a81a7d7358e0443f5b87bf2e596938f9fd1894edac6efc521b93291d7c"} +{"event_type_counts": {"event.decay": 4}, "id": "sparse_decay", "ir_sha256": "c62a2ac5ec3078fa5fdefa117ecbeaff8dec03973e53f2ffba89ebf6effba98a"} diff --git a/evals/event_vision_sensorium/expected_projection.json b/evals/event_vision_sensorium/expected_projection.json new file mode 100644 index 00000000..3e3022f3 --- /dev/null +++ b/evals/event_vision_sensorium/expected_projection.json @@ -0,0 +1,113 @@ +{ + "corner_blink": { + "projection_sha256": "e52cfcf722b19c987eddf405c3872e570dad2f56de3d83d2f5ab08350c0ce4e8", + "reference_versor": [ + 0.36488470435142517, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.5650413632392883, + 0.49697229266166687, + 0.22949537634849548, + 0.0, + -0.13096900284290314, + -0.3553845286369324, + 0.0, + -0.31257227063179016, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + -0.08237336575984955, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "edge_sweep_right": { + "projection_sha256": "9138236958b14ac5030573447624068f9d252bc28461cf5027dd2a1d6eee0bb0", + "reference_versor": [ + 0.5723526477813721, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.5173685550689697, + 0.2860219180583954, + 0.36986204981803894, + 0.0, + -0.1683901995420456, + -0.33433058857917786, + 0.0, + -0.18483126163482666, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + -0.10881602764129639, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "sparse_decay": { + "projection_sha256": "97be0f61ebf9695623297fb6050448a3fecc492aa29c8a4ce923dfb9882d4015", + "reference_versor": [ + 0.5324031114578247, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.8464909195899963, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + } +} diff --git a/evals/event_vision_sensorium/fixtures.json b/evals/event_vision_sensorium/fixtures.json new file mode 100644 index 00000000..b14a73ee --- /dev/null +++ b/evals/event_vision_sensorium/fixtures.json @@ -0,0 +1,45 @@ +{ + "fixtures": [ + { + "id": "edge_sweep_right", + "grid_w": 32, + "grid_h": 24, + "packet_tick": 10, + "events": [ + [4, 8, 1, 0], + [5, 8, 1, 1], + [6, 8, -1, 2], + [7, 8, 1, 3], + [8, 8, -1, 4], + [9, 8, 1, 5] + ] + }, + { + "id": "corner_blink", + "grid_w": 32, + "grid_h": 24, + "packet_tick": 11, + "events": [ + [14, 10, 1, 0], + [14, 11, 1, 1], + [15, 10, -1, 2], + [15, 11, -1, 3], + [16, 10, 1, 4], + [16, 11, -1, 5], + [17, 12, 1, 9] + ] + }, + { + "id": "sparse_decay", + "grid_w": 32, + "grid_h": 24, + "packet_tick": 12, + "events": [ + [2, 2, -1, 0], + [12, 6, -1, 8], + [20, 14, -1, 16], + [28, 20, -1, 24] + ] + } + ] +} diff --git a/evals/event_vision_sensorium/generate_expected.py b/evals/event_vision_sensorium/generate_expected.py new file mode 100644 index 00000000..16574c70 --- /dev/null +++ b/evals/event_vision_sensorium/generate_expected.py @@ -0,0 +1,51 @@ +"""Regenerate frozen expected artifacts for the event-vision eval lane.""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path + +from evals.event_vision_sensorium.synth import synthesize +from sensorium.vision_event import EventVisionCompiler + +_ROOT = Path(__file__).resolve().parent + + +def main() -> None: + fixtures = json.loads((_ROOT / "fixtures.json").read_text(encoding="utf-8"))["fixtures"] + compiler = EventVisionCompiler() + expected_ir: list[dict[str, object]] = [] + expected_projection: dict[str, object] = {} + for fixture in fixtures: + fid = fixture["id"] + unit = compiler.compile_packet(synthesize(fixture)) + counts = Counter( + event.event_type + for event in ( + *unit.event_ir.onset_events, + *unit.event_ir.decay_events, + *unit.event_ir.motion_bins, + ) + ) + expected_ir.append({ + "id": fid, + "ir_sha256": unit.ir_sha256, + "event_type_counts": dict(sorted(counts.items())), + }) + expected_projection[fid] = { + "projection_sha256": unit.projection_sha256, + "reference_versor": [float(x) for x in unit.versor.tolist()], + } + (_ROOT / "expected_ir.jsonl").write_text( + "\n".join(json.dumps(row, sort_keys=True) for row in expected_ir) + "\n", + encoding="utf-8", + ) + (_ROOT / "expected_projection.json").write_text( + json.dumps(expected_projection, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() diff --git a/evals/event_vision_sensorium/synth.py b/evals/event_vision_sensorium/synth.py new file mode 100644 index 00000000..48a83bec --- /dev/null +++ b/evals/event_vision_sensorium/synth.py @@ -0,0 +1,20 @@ +"""Deterministic synthetic EventPacket fixtures.""" + +from __future__ import annotations + +from typing import Any + +from sensorium.vision_event import EventPacket, build_event_packet + + +def synthesize(spec: dict[str, Any]) -> EventPacket: + return build_event_packet( + grid_w=int(spec["grid_w"]), + grid_h=int(spec["grid_h"]), + packet_tick=int(spec["packet_tick"]), + events=tuple(tuple(event) for event in spec["events"]), + source_id=str(spec["id"]), + ) + + +__all__ = ["synthesize"] diff --git a/evals/sensorium/report.py b/evals/sensorium/report.py index 3eb1b1a0..df30d2bc 100644 --- a/evals/sensorium/report.py +++ b/evals/sensorium/report.py @@ -10,11 +10,13 @@ from typing import Literal import numpy as np from evals.audio_sensorium.synth import synthesize as synthesize_audio +from evals.event_vision_sensorium.synth import synthesize as synthesize_event_vision from evals.sensorimotor_sensorium.synth import synthesize as synthesize_sensorimotor from evals.vision_sensorium.synth import synthesize as synthesize_vision from sensorium.adapters.audio import make_audio_pack from sensorium.adapters.sensorimotor import make_sensorimotor_pack from sensorium.adapters.vision import make_vision_pack +from sensorium.adapters.vision_event import make_event_vision_pack from sensorium.audio.canonical import canonicalize as canonicalize_audio from sensorium.audio.compiler import AudioCompiler from sensorium.audio.trace import audio_evidence_trace @@ -24,11 +26,13 @@ from sensorium.sensorimotor import SensorimotorCompiler, sensorimotor_evidence_t from sensorium.vision import VisionCompiler, canonicalize_image, vision_evidence_trace from sensorium.vision.grid import iter_tile_signals from sensorium.vision.types import VisionIR +from sensorium.vision_event import EventIR, EventVisionCompiler, event_vision_evidence_trace -ModalityName = Literal["audio", "vision", "sensorimotor"] +ModalityName = Literal["audio", "vision", "event-vision", "sensorimotor"] _ROOT = Path(__file__).resolve().parents[2] _AUDIO_DIR = _ROOT / "evals" / "audio_sensorium" _VISION_DIR = _ROOT / "evals" / "vision_sensorium" +_EVENT_VISION_DIR = _ROOT / "evals" / "event_vision_sensorium" _SENSORIMOTOR_DIR = _ROOT / "evals" / "sensorimotor_sensorium" _AUDIO_SR = 24_000 _TOL = 1e-6 @@ -71,6 +75,15 @@ def _vision_counts(ir: VisionIR) -> dict[str, int]: return dict(sorted(Counter(e.event_type for e in events).items())) +def _event_vision_counts(ir: EventIR) -> dict[str, int]: + events = ( + *ir.onset_events, + *ir.decay_events, + *ir.motion_bins, + ) + return dict(sorted(Counter(e.event_type for e in events).items())) + + def _trace_safe(trace: dict[str, object]) -> bool: return all(not isinstance(value, (np.ndarray, bytes, bytearray)) for value in trace.values()) @@ -155,6 +168,53 @@ def _vision_report() -> dict[str, object]: return _report("vision", "vision_core_v1", cases, gate_closed) +def _event_vision_report() -> dict[str, object]: + fixtures = _json(_EVENT_VISION_DIR / "fixtures.json")["fixtures"] + expected_ir = _jsonl_by_id(_EVENT_VISION_DIR / "expected_ir.jsonl") + expected_proj = _json(_EVENT_VISION_DIR / "expected_projection.json") + compiler = EventVisionCompiler() + cases: list[dict[str, object]] = [] + for fx in fixtures: + fid = fx["id"] + packet = synthesize_event_vision(fx) + unit = compiler.compile_packet(packet) + replay = compiler.compile_ir( + unit.event_ir, + canonical_sha256=packet.canonical_sha256, + packet_tick=packet.packet_tick, + grid_shape=(packet.grid_h, packet.grid_w), + ) + ref = np.asarray(expected_proj[fid]["reference_versor"], dtype=np.float32) + permuted = dict(fx) + permuted["events"] = list(reversed(fx["events"])) + permuted_unit = compiler.compile_packet(synthesize_event_vision(permuted)) + cases.append({ + "id": fid, + "canonical_sha256": unit.canonical_sha256, + "ir_sha256": unit.ir_sha256, + "projection_sha256": unit.projection_sha256, + "shape_ok": unit.versor.shape == (32,), + "dtype_ok": unit.versor.dtype == np.float32, + "replay_ok": bool(np.array_equal(unit.versor, replay.versor)), + "expected_ir_ok": unit.ir_sha256 == expected_ir[fid]["ir_sha256"], + "expected_projection_ok": bool(np.allclose(unit.versor, ref, atol=_TOL)), + "projection_hash_ok": unit.projection_sha256 == expected_proj[fid]["projection_sha256"], + "event_counts_ok": _event_vision_counts(unit.event_ir) == expected_ir[fid]["event_type_counts"], + "packet_order_invariant_ok": permuted_unit.merge_key == unit.merge_key, + "trace_hygiene_ok": _trace_safe(event_vision_evidence_trace(unit)), + "versor_condition": unit.versor_condition, + }) + reg = ModalityRegistry() + sample = synthesize_event_vision(fixtures[0]) + reg.mount(make_event_vision_pack("vision_event_core_v1"), sample=sample) + gate_closed = False + try: + reg.project("vision_event_core_v1", sample) + except RuntimeError: + gate_closed = True + return _report("event-vision", "vision_event_core_v1", cases, gate_closed) + + def _sensorimotor_report() -> dict[str, object]: fixtures = _json(_SENSORIMOTOR_DIR / "fixtures.json")["fixtures"] expected_ir = _jsonl_by_id(_SENSORIMOTOR_DIR / "expected_ir.jsonl") @@ -214,6 +274,8 @@ def build_sensorium_report(modality: ModalityName) -> dict[str, object]: return _audio_report() if modality == "vision": return _vision_report() + if modality == "event-vision": + return _event_vision_report() if modality == "sensorimotor": return _sensorimotor_report() raise ValueError(f"unsupported sensorium modality: {modality!r}") diff --git a/packs/vision/vision_event_core_v1/basis_map.json b/packs/vision/vision_event_core_v1/basis_map.json new file mode 100644 index 00000000..9d774737 --- /dev/null +++ b/packs/vision/vision_event_core_v1/basis_map.json @@ -0,0 +1,8 @@ +{ + "basis_version": "vision-event-basis-v1", + "elliptic_planes": { + "B_EVENT_ONSET": 6, + "B_EVENT_DECAY": 7, + "B_EVENT_MOTION": 8 + } +} diff --git a/packs/vision/vision_event_core_v1/checksums.json b/packs/vision/vision_event_core_v1/checksums.json new file mode 100644 index 00000000..84ec9201 --- /dev/null +++ b/packs/vision/vision_event_core_v1/checksums.json @@ -0,0 +1,7 @@ +{ + "files": { + "basis_map.json": "sha256:e9402742ad1707ecdeefb26213856d7152eb75b36589f41e44b6816ab8c1cdb9", + "manifest.json": "sha256:15775345f2c13905855d1df5ea8aeda67109b12cdb10bbaa794841b46cc0efc2", + "operators.jsonl": "sha256:4c77b2610ba00924629e24246d2eefa8606d730c4010bd6f908105c5e9c49447" + } +} diff --git a/packs/vision/vision_event_core_v1/manifest.json b/packs/vision/vision_event_core_v1/manifest.json new file mode 100644 index 00000000..f182c739 --- /dev/null +++ b/packs/vision/vision_event_core_v1/manifest.json @@ -0,0 +1,24 @@ +{ + "pack_id": "vision_event_core_v1", + "modality": "vision", + "sensorium_lane": "event-vision", + "cl41_dim": 32, + "compiler_version": "0.1.0", + "basis_version": "vision-event-basis-v1", + "canonical": { + "sensor_grid": "deterministic_synthetic_event_grid_v1", + "event_tuple": ["x", "y", "polarity", "t_q"], + "event_order": "canonical_sort_by_x_y_polarity_t", + "output_dtype": "float32", + "internal_dtype": "float64" + }, + "gating": { + "gate_engaged": false, + "checksum_verified": false, + "versor_condition_max": 1e-06 + }, + "ordering": { + "event_precedence": ["event.onset", "event.decay", "event.motion_delta"], + "fold_scope": "single_packet_serialization_barrier" + } +} diff --git a/packs/vision/vision_event_core_v1/operators.jsonl b/packs/vision/vision_event_core_v1/operators.jsonl new file mode 100644 index 00000000..fdd55a97 --- /dev/null +++ b/packs/vision/vision_event_core_v1/operators.jsonl @@ -0,0 +1,3 @@ +{"operator_id":"vision_event.onset.v1","event_type":"event.onset","blade_alias":"B_EVENT_ONSET","blade_index":6,"base_theta_q":36,"gain_rules":{"polarity_q":7,"t_bin":2,"x_q":1,"y_q":1},"theta_clip_q":768,"version":"1"} +{"operator_id":"vision_event.decay.v1","event_type":"event.decay","blade_alias":"B_EVENT_DECAY","blade_index":7,"base_theta_q":44,"gain_rules":{"polarity_q":-7,"t_bin":2,"x_q":1,"y_q":1},"theta_clip_q":768,"version":"1"} +{"operator_id":"vision_event.motion_delta.v1","event_type":"event.motion_delta","blade_alias":"B_EVENT_MOTION","blade_index":8,"base_theta_q":72,"gain_rules":{"balance_q":3,"count_q":5,"t_bin":2},"theta_clip_q":768,"version":"1"} diff --git a/sensorium/adapters/vision_event.py b/sensorium/adapters/vision_event.py new file mode 100644 index 00000000..725773b0 --- /dev/null +++ b/sensorium/adapters/vision_event.py @@ -0,0 +1,67 @@ +"""Event-stream vision modality adapter.""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np + +from sensorium.protocol import CL41_DIM, Modality, ModalityPack, ModalityVocabulary +from sensorium.vision_event.compiler import EventVisionCompiler +from sensorium.vision_event.types import EventPacket + + +@dataclass(frozen=True, slots=True) +class EventVisionProjectionHead: + """ProjectionHead for one synthetic EventPacket.""" + + compiler: EventVisionCompiler + modality: Modality = Modality.VISION + + @property + def embedding_dim(self) -> int: + return CL41_DIM + + def project(self, signal: EventPacket) -> np.ndarray: + out = self.compiler.compile_packet(signal).versor + if out.shape != (CL41_DIM,): + raise ValueError(f"expected ({CL41_DIM},), got {out.shape}") + if out.dtype != np.float32: + raise TypeError(f"expected float32, got {out.dtype}") + return out + + def project_batch(self, signals: list[EventPacket]) -> np.ndarray: + return np.stack([self.project(signal) for signal in signals], axis=0) + + def verify_unitarity(self, sample: EventPacket) -> bool: + try: + return self.compiler.compile_packet(sample).versor_condition < 1e-6 + except Exception: + return False + + +def make_event_vision_pack( + pack_id: str = "vision_event_core_v1", + *, + gate_engaged: bool = False, + checksum_verified: bool = False, + packs_root=None, +) -> ModalityPack: + from packs.vision.loader import load_vision_pack + + loaded = load_vision_pack(pack_id, packs_root=packs_root) + compiler = EventVisionCompiler(loaded.registry, pack_id=loaded.pack_id) + head = EventVisionProjectionHead(compiler) + return ModalityPack( + pack_id=loaded.pack_id, + modality_type=Modality.VISION, + projection=head, + decoder=None, + vocabulary=ModalityVocabulary(), + grammar_scaffold=None, + checksum_verified=checksum_verified, + gate_engaged=gate_engaged, + ) + + +__all__ = ["EventVisionProjectionHead", "make_event_vision_pack"] diff --git a/sensorium/vision_event/__init__.py b/sensorium/vision_event/__init__.py new file mode 100644 index 00000000..e473f5b7 --- /dev/null +++ b/sensorium/vision_event/__init__.py @@ -0,0 +1,29 @@ +"""Synthetic event-stream vision compiler surface.""" + +from sensorium.vision_event.compiler import ( + EventVisionCompiler, + canonical_event_atom_order, + compile_event_atoms, +) +from sensorium.vision_event.packet import build_event_packet +from sensorium.vision_event.trace import event_vision_evidence_trace +from sensorium.vision_event.types import ( + Event, + EventAtom, + EventCompilationUnit, + EventIR, + EventPacket, +) + +__all__ = [ + "Event", + "EventAtom", + "EventCompilationUnit", + "EventIR", + "EventPacket", + "EventVisionCompiler", + "build_event_packet", + "canonical_event_atom_order", + "compile_event_atoms", + "event_vision_evidence_trace", +] diff --git a/sensorium/vision_event/compiler.py b/sensorium/vision_event/compiler.py new file mode 100644 index 00000000..710de086 --- /dev/null +++ b/sensorium/vision_event/compiler.py @@ -0,0 +1,113 @@ +"""Python reference compiler for synthetic event-stream vision packets.""" + +from __future__ import annotations + +import numpy as np + +from algebra.cl41 import geometric_product +from algebra.versor import unitize_versor, versor_condition +from sensorium.vision.checksum import sha256_array +from sensorium.vision.operators import VisionOperatorRegistry, build_elliptic_rotor +from sensorium.vision_event.lexer import lex_event_packet +from sensorium.vision_event.operators import DEFAULT_EVENT_OPERATOR_REGISTRY +from sensorium.vision_event.parser import parse_event_atoms +from sensorium.vision_event.types import EventAtom, EventCompilationUnit, EventIR, EventPacket + +CL41_DIM = 32 +VERSOR_CONDITION_MAX = 1e-6 +_PRECEDENCE = {"event.onset": 0, "event.decay": 1, "event.motion_delta": 2} + + +def canonical_event_atom_order(ir: EventIR) -> list[EventAtom]: + atoms = [*ir.onset_events, *ir.decay_events, *ir.motion_bins] + return sorted( + atoms, + key=lambda atom: ( + atom.coord.scale_level, + atom.coord.morton, + _PRECEDENCE.get(atom.event_type, len(_PRECEDENCE)), + atom.event_type, + atom.attrs, + atom.evidence_ids, + ), + ) + + +def compile_event_atoms( + atoms: list[EventAtom], + registry: VisionOperatorRegistry, +) -> tuple[np.ndarray, float]: + """Serialization barrier: fold one packet's canonical deltas into one versor.""" + v = np.zeros(CL41_DIM, dtype=np.float64) + v[0] = 1.0 + for atom in atoms: + if atom.event_type not in registry: + continue + spec = registry[atom.event_type] + r = build_elliptic_rotor(spec.blade_index, spec.theta_q_from_event(atom)) + v = geometric_product(v, r) + v = unitize_versor(v) + vc = float(versor_condition(v)) + if vc >= VERSOR_CONDITION_MAX: + raise ValueError( + f"event vision compilation failed versor check: versor_condition={vc:.3e} " + f">= {VERSOR_CONDITION_MAX:.0e}" + ) + return v.astype(np.float32), vc + + +class EventVisionCompiler: + """Deterministic compiler from one EventPacket to one compilation unit.""" + + def __init__( + self, + registry: VisionOperatorRegistry = DEFAULT_EVENT_OPERATOR_REGISTRY, + pack_id: str = "vision_event_core_v1", + ) -> None: + self._registry = registry + self._pack_id = pack_id + self._manifest_sha256 = registry.manifest_sha256() + + def compile_packet(self, packet: EventPacket) -> EventCompilationUnit: + ir = parse_event_atoms(lex_event_packet(packet)) + return self.compile_ir( + ir, + canonical_sha256=packet.canonical_sha256, + packet_tick=packet.packet_tick, + grid_shape=(packet.grid_h, packet.grid_w), + ) + + def compile_signal(self, packet: EventPacket) -> EventCompilationUnit: + return self.compile_packet(packet) + + def compile_ir( + self, + ir: EventIR, + *, + canonical_sha256: str = "", + packet_tick: int = 0, + grid_shape: tuple[int, int] = (0, 0), + ) -> EventCompilationUnit: + atoms = canonical_event_atom_order(ir) + if not atoms: + raise ValueError("cannot replay EventIR with no event atoms") + versor, vc = compile_event_atoms(atoms, self._registry) + return EventCompilationUnit( + canonical_sha256=canonical_sha256, + ir_sha256=ir.ir_sha256, + pack_id=self._pack_id, + pack_manifest_sha256=self._manifest_sha256, + projection_sha256=sha256_array(versor), + packet_tick=int(packet_tick), + grid_shape=(int(grid_shape[0]), int(grid_shape[1])), + versor=versor, + versor_condition=vc, + event_ir=ir, + ) + + +__all__ = [ + "EventVisionCompiler", + "canonical_event_atom_order", + "compile_event_atoms", +] diff --git a/sensorium/vision_event/lexer.py b/sensorium/vision_event/lexer.py new file mode 100644 index 00000000..a538da09 --- /dev/null +++ b/sensorium/vision_event/lexer.py @@ -0,0 +1,92 @@ +"""Event-packet lexer for sparse asynchronous vision deltas.""" + +from __future__ import annotations + +from collections import defaultdict + +from sensorium.vision.checksum import sha256_json +from sensorium.vision.types import TileCoord +from sensorium.vision_event.types import EventAtom, EventPacket + +_TILE_PX = 8 +_TIME_BIN_Q = 8 + + +def _coord(packet: EventPacket, x: int, y: int) -> TileCoord: + max_row = max(0, (packet.grid_h - 1) // _TILE_PX) + max_col = max(0, (packet.grid_w - 1) // _TILE_PX) + return TileCoord( + scale_level=0, + tile_row=min(max_row, y // _TILE_PX), + tile_col=min(max_col, x // _TILE_PX), + ) + + +def _event_evidence_id(packet: EventPacket, *, x: int, y: int, polarity: int, t_q: int) -> str: + return sha256_json({ + "kind": "EventPacket.event", + "packet": packet.canonical_sha256, + "x": x, + "y": y, + "polarity": polarity, + "t_q": t_q, + }) + + +def lex_event_packet(packet: EventPacket) -> tuple[EventAtom, ...]: + atoms: list[EventAtom] = [] + bins: dict[tuple[int, int, int], list[int]] = defaultdict(list) + for event in packet.events: + coord = _coord(packet, event.x, event.y) + t_bin = event.t_q // _TIME_BIN_Q + event_type = "event.onset" if event.polarity > 0 else "event.decay" + evidence_id = _event_evidence_id( + packet, + x=event.x, + y=event.y, + polarity=event.polarity, + t_q=event.t_q, + ) + atoms.append(EventAtom( + event_type=event_type, + coord=coord, + attrs=( + ("polarity_q", event.polarity), + ("t_bin", t_bin), + ("x_q", event.x), + ("y_q", event.y), + ), + evidence_ids=(evidence_id,), + )) + bins[(coord.tile_row, coord.tile_col, t_bin)].append(event.polarity) + + for (tile_row, tile_col, t_bin), polarities in sorted(bins.items()): + count = len(polarities) + balance = sum(polarities) + if count < 2 and abs(balance) == count: + continue + atoms.append(EventAtom( + event_type="event.motion_delta", + coord=TileCoord(scale_level=1, tile_row=tile_row, tile_col=tile_col), + attrs=( + ("balance_q", balance), + ("count_q", count), + ("t_bin", t_bin), + ), + evidence_ids=tuple( + sha256_json({ + "kind": "EventPacket.motion_bin", + "packet": packet.canonical_sha256, + "tile_row": tile_row, + "tile_col": tile_col, + "t_bin": t_bin, + "index": idx, + "polarity": polarity, + }) + for idx, polarity in enumerate(sorted(polarities)) + ), + )) + return tuple(atoms) + + +__all__ = ["lex_event_packet"] diff --git a/sensorium/vision_event/operators.py b/sensorium/vision_event/operators.py new file mode 100644 index 00000000..937dd358 --- /dev/null +++ b/sensorium/vision_event/operators.py @@ -0,0 +1,43 @@ +"""Operator registry for vision_event_core_v1.""" + +from __future__ import annotations + +from sensorium.vision.operators import VisionOperatorRegistry, VisionOperatorSpec + + +def _spec(op_id, etype, alias, blade, base, gains, clip=768) -> VisionOperatorSpec: + return VisionOperatorSpec(op_id, etype, alias, blade, base, tuple(gains), clip) + + +DEFAULT_EVENT_OPERATOR_REGISTRY = VisionOperatorRegistry( + { + "event.onset": _spec( + "vision_event.onset.v1", + "event.onset", + "B_EVENT_ONSET", + 6, + 36, + [("polarity_q", 7), ("t_bin", 2), ("x_q", 1), ("y_q", 1)], + ), + "event.decay": _spec( + "vision_event.decay.v1", + "event.decay", + "B_EVENT_DECAY", + 7, + 44, + [("polarity_q", -7), ("t_bin", 2), ("x_q", 1), ("y_q", 1)], + ), + "event.motion_delta": _spec( + "vision_event.motion_delta.v1", + "event.motion_delta", + "B_EVENT_MOTION", + 8, + 72, + [("balance_q", 3), ("count_q", 5), ("t_bin", 2)], + ), + }, + basis_version="vision-event-basis-v1", +) + + +__all__ = ["DEFAULT_EVENT_OPERATOR_REGISTRY"] diff --git a/sensorium/vision_event/packet.py b/sensorium/vision_event/packet.py new file mode 100644 index 00000000..804395cd --- /dev/null +++ b/sensorium/vision_event/packet.py @@ -0,0 +1,67 @@ +"""Canonical construction for synthetic event-stream vision packets.""" + +from __future__ import annotations + +from collections.abc import Iterable + +from sensorium.vision.checksum import sha256_json +from sensorium.vision_event.types import Event, EventPacket + + +def _event_from_tuple(raw: Event | tuple[int, int, int, int]) -> Event: + if isinstance(raw, Event): + return raw + x, y, polarity, t_q = raw + return Event(int(x), int(y), int(polarity), int(t_q)) + + +def build_event_packet( + *, + grid_w: int, + grid_h: int, + packet_tick: int, + events: Iterable[Event | tuple[int, int, int, int]], + source_id: str = "synthetic-event-fixture", +) -> EventPacket: + if grid_w <= 0 or grid_h <= 0: + raise ValueError("event packet grid dimensions must be positive") + if packet_tick < 0: + raise ValueError("event packet tick must be non-negative") + canonical_events = tuple(sorted(_event_from_tuple(event) for event in events)) + if not canonical_events: + raise ValueError("event packet requires at least one event") + for event in canonical_events: + if not (0 <= event.x < grid_w and 0 <= event.y < grid_h): + raise ValueError("event coordinate outside canonical sensor grid") + if event.polarity not in {-1, 1}: + raise ValueError("event polarity must be -1 or 1") + if event.t_q < 0: + raise ValueError("event time quantum must be non-negative") + source_sha256 = sha256_json({ + "kind": "EventPacket.source", + "source_id": str(source_id), + "grid_w": int(grid_w), + "grid_h": int(grid_h), + }) + payload = { + "kind": "EventPacket", + "grid_w": int(grid_w), + "grid_h": int(grid_h), + "packet_tick": int(packet_tick), + "source_sha256": source_sha256, + "events": [ + {"x": e.x, "y": e.y, "polarity": e.polarity, "t_q": e.t_q} + for e in canonical_events + ], + } + return EventPacket( + grid_w=int(grid_w), + grid_h=int(grid_h), + packet_tick=int(packet_tick), + events=canonical_events, + source_sha256=source_sha256, + canonical_sha256=sha256_json(payload), + ) + + +__all__ = ["build_event_packet"] diff --git a/sensorium/vision_event/parser.py b/sensorium/vision_event/parser.py new file mode 100644 index 00000000..1014f07a --- /dev/null +++ b/sensorium/vision_event/parser.py @@ -0,0 +1,54 @@ +"""Parser from event atoms into deterministic EventIR.""" + +from __future__ import annotations + +from collections.abc import Iterable + +from sensorium.vision.checksum import sha256_json +from sensorium.vision_event.types import EventAtom, EventIR + + +def _atom_record(atom: EventAtom) -> dict[str, object]: + return { + "event_type": atom.event_type, + "coord": { + "scale_level": atom.coord.scale_level, + "tile_row": atom.coord.tile_row, + "tile_col": atom.coord.tile_col, + "morton": atom.coord.morton, + }, + "attrs": [list(attr) for attr in atom.attrs], + "evidence_ids": list(atom.evidence_ids), + } + + +def _stable_atom(atom: EventAtom) -> tuple[object, ...]: + return ( + atom.coord.scale_level, + atom.coord.morton, + atom.event_type, + atom.attrs, + atom.evidence_ids, + ) + + +def parse_event_atoms(atoms: Iterable[EventAtom]) -> EventIR: + ordered = tuple(sorted(atoms, key=_stable_atom)) + onset = tuple(atom for atom in ordered if atom.event_type == "event.onset") + decay = tuple(atom for atom in ordered if atom.event_type == "event.decay") + motion = tuple(atom for atom in ordered if atom.event_type == "event.motion_delta") + payload = { + "kind": "EventIR", + "onset_events": [_atom_record(atom) for atom in onset], + "decay_events": [_atom_record(atom) for atom in decay], + "motion_bins": [_atom_record(atom) for atom in motion], + } + return EventIR( + onset_events=onset, + decay_events=decay, + motion_bins=motion, + ir_sha256=sha256_json(payload), + ) + + +__all__ = ["parse_event_atoms"] diff --git a/sensorium/vision_event/trace.py b/sensorium/vision_event/trace.py new file mode 100644 index 00000000..d7688501 --- /dev/null +++ b/sensorium/vision_event/trace.py @@ -0,0 +1,24 @@ +"""Trace-safe evidence records for event-stream vision compilation units.""" + +from __future__ import annotations + +from sensorium.vision_event.types import EventCompilationUnit + + +def event_vision_evidence_trace(unit: EventCompilationUnit) -> dict[str, object]: + return { + "modality": "vision", + "sensorium_lane": "event-vision", + "pack_id": unit.pack_id, + "canonical_sha256": unit.canonical_sha256, + "ir_sha256": unit.ir_sha256, + "pack_manifest_sha256": unit.pack_manifest_sha256, + "projection_sha256": unit.projection_sha256, + "merge_key": list(unit.merge_key), + "packet_tick": unit.packet_tick, + "grid_shape": list(unit.grid_shape), + "versor_condition": unit.versor_condition, + } + + +__all__ = ["event_vision_evidence_trace"] diff --git a/sensorium/vision_event/types.py b/sensorium/vision_event/types.py new file mode 100644 index 00000000..2a3b0fcc --- /dev/null +++ b/sensorium/vision_event/types.py @@ -0,0 +1,70 @@ +"""Typed event-stream vision packets, IR, and compilation units.""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np + +from sensorium.vision.types import TileCoord + + +@dataclass(frozen=True, slots=True, order=True) +class Event: + x: int + y: int + polarity: int + t_q: int + + +@dataclass(frozen=True, slots=True) +class EventPacket: + grid_w: int + grid_h: int + packet_tick: int + events: tuple[Event, ...] + source_sha256: str + canonical_sha256: str + + +@dataclass(frozen=True, slots=True) +class EventAtom: + event_type: str + coord: TileCoord + attrs: tuple[tuple[str, int | str], ...] + evidence_ids: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class EventIR: + onset_events: tuple[EventAtom, ...] + decay_events: tuple[EventAtom, ...] + motion_bins: tuple[EventAtom, ...] + ir_sha256: str + + +@dataclass(frozen=True, slots=True) +class EventCompilationUnit: + canonical_sha256: str + ir_sha256: str + pack_id: str + pack_manifest_sha256: str + projection_sha256: str + packet_tick: int + grid_shape: tuple[int, int] + versor: np.ndarray + versor_condition: float + event_ir: EventIR + + @property + def merge_key(self) -> tuple[str, str, str]: + return (self.canonical_sha256, self.ir_sha256, self.projection_sha256) + + +__all__ = [ + "Event", + "EventAtom", + "EventCompilationUnit", + "EventIR", + "EventPacket", +] diff --git a/tests/test_event_vision_compiler.py b/tests/test_event_vision_compiler.py new file mode 100644 index 00000000..3942a0da --- /dev/null +++ b/tests/test_event_vision_compiler.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import numpy as np + +from sensorium.adapters.vision_event import make_event_vision_pack +from sensorium.protocol import Modality +from sensorium.vision_event import ( + EventVisionCompiler, + build_event_packet, + event_vision_evidence_trace, +) + + +def _packet(): + return build_event_packet( + grid_w=32, + grid_h=24, + packet_tick=1, + events=( + (4, 8, 1, 0), + (6, 8, -1, 2), + (5, 8, 1, 1), + (7, 8, 1, 3), + ), + source_id="test-event-vision", + ) + + +def test_event_packet_hash_is_order_invariant(): + p1 = _packet() + p2 = build_event_packet( + grid_w=32, + grid_h=24, + packet_tick=1, + events=tuple(reversed(tuple((e.x, e.y, e.polarity, e.t_q) for e in p1.events))), + source_id="test-event-vision", + ) + assert p1.events == p2.events + assert p1.canonical_sha256 == p2.canonical_sha256 + + +def test_event_vision_compiler_outputs_unit_contract(): + unit = EventVisionCompiler().compile_packet(_packet()) + replay = EventVisionCompiler().compile_ir( + unit.event_ir, + canonical_sha256=unit.canonical_sha256, + packet_tick=unit.packet_tick, + grid_shape=unit.grid_shape, + ) + assert unit.pack_id == "vision_event_core_v1" + assert unit.versor.shape == (32,) + assert unit.versor.dtype == np.float32 + assert unit.versor_condition < 1e-6 + assert unit.merge_key == (unit.canonical_sha256, unit.ir_sha256, unit.projection_sha256) + assert np.array_equal(unit.versor, replay.versor) + + +def test_event_vision_trace_has_no_raw_events(): + unit = EventVisionCompiler().compile_packet(_packet()) + trace = event_vision_evidence_trace(unit) + text = str(trace) + assert trace["modality"] == "vision" + assert trace["sensorium_lane"] == "event-vision" + assert "raw_events" not in text + assert "event_ir" not in text + assert "raw" not in text + assert "reference_versor" not in text + + +def test_event_vision_pack_uses_vision_modality(): + pack = make_event_vision_pack("vision_event_core_v1") + assert pack.modality_type is Modality.VISION + assert pack.projection is not None diff --git a/tests/test_sensorium_eval_cli.py b/tests/test_sensorium_eval_cli.py index 177cd496..3758465f 100644 --- a/tests/test_sensorium_eval_cli.py +++ b/tests/test_sensorium_eval_cli.py @@ -16,6 +16,16 @@ def test_core_eval_sensorium_json_reports_selected_modality(capsys): assert report["failed"] == 0 +def test_core_eval_sensorium_event_vision_json(capsys): + assert main(["eval", "sensorium", "--modality", "event-vision", "--json"]) == 0 + out = capsys.readouterr().out + report = json.loads(out) + assert report["lane"] == "sensorium" + assert report["modality"] == "event-vision" + assert report["pack_id"] == "vision_event_core_v1" + assert report["failed"] == 0 + + def test_core_eval_sensorium_text_summary(capsys): assert main(["eval", "sensorium", "--modality", "vision"]) == 0 out = capsys.readouterr().out