From b5f892bd95b1212039d3b8dcd828e0e5d755cc7e Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 6 Jun 2026 12:08:07 -0700 Subject: [PATCH] Add offline witness log importer --- core/cli.py | 1 + .../expected_hashes.json | 2 +- evals/environment_falsification/report.py | 57 +++- .../witness_log.jsonl | 3 + .../witness_payloads.json | 23 ++ sensorium/logs/__init__.py | 17 ++ sensorium/logs/importer.py | 269 ++++++++++++++++++ tests/test_witness_log_importer.py | 112 ++++++++ 8 files changed, 480 insertions(+), 4 deletions(-) create mode 100644 evals/environment_falsification/witness_log.jsonl create mode 100644 evals/environment_falsification/witness_payloads.json create mode 100644 sensorium/logs/__init__.py create mode 100644 sensorium/logs/importer.py create mode 100644 tests/test_witness_log_importer.py diff --git a/core/cli.py b/core/cli.py index 8351f8e2..9900a8bc 100644 --- a/core/cli.py +++ b/core/cli.py @@ -132,6 +132,7 @@ _TEST_SUITES: dict[str, tuple[str, ...]] = { "tests/test_observation_frame_harness.py", "tests/test_environment_falsification.py", "tests/test_environment_falsification_eval_cli.py", + "tests/test_witness_log_importer.py", "tests/test_sensorium_eval_cli.py", "tests/test_efferent_gate.py", ), diff --git a/evals/environment_falsification/expected_hashes.json b/evals/environment_falsification/expected_hashes.json index 22bdddff..770dcc94 100644 --- a/evals/environment_falsification/expected_hashes.json +++ b/evals/environment_falsification/expected_hashes.json @@ -1,4 +1,4 @@ { "frame_report_sha256": "c97b2dca7282d0231f1b448add87256cc2f59d39c5cec5ac1350231541e07d0b", - "report_sha256": "e0903e408f882bf7c93b7aadb6b3f8a064438ac09dc1cd4139f381bdc520b923" + "report_sha256": "7d683ec2ca43866a23daf13a24abcc6334627f5c68a0cc05d93b10e1f313ca12" } diff --git a/evals/environment_falsification/report.py b/evals/environment_falsification/report.py index 361aaf53..5d7580e2 100644 --- a/evals/environment_falsification/report.py +++ b/evals/environment_falsification/report.py @@ -23,6 +23,7 @@ from sensorium.environment import ( compare_expected_to_observation, run_falsification_scenario, ) +from sensorium.logs import import_witness_jsonl, import_witness_records from sensorium.sensorimotor.compiler import SensorimotorCompiler from sensorium.vision import VisionCompiler, canonicalize_image from sensorium.vision.grid import iter_tile_signals @@ -35,6 +36,14 @@ def _load_json(name: str) -> dict[str, Any]: return json.loads((_ROOT / name).read_text(encoding="utf-8")) +def _load_jsonl(name: str) -> list[dict[str, Any]]: + return [ + json.loads(line) + for line in (_ROOT / name).read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + def _trace_safe(value: object) -> bool: if isinstance(value, (np.ndarray, bytes, bytearray)): return False @@ -185,6 +194,37 @@ def _scenario_case_report(index: int, scenario: dict[str, Any]) -> dict[str, obj return row +def _witness_import_report() -> dict[str, object]: + payloads = _load_json("witness_payloads.json")["payloads"] + + def resolve(payload_ref: str): + return _compile_unit(payloads[payload_ref]) + + path = _ROOT / "witness_log.jsonl" + imported = import_witness_jsonl(path, resolve_payload_ref=resolve) + repeated = import_witness_jsonl(path, resolve_payload_ref=resolve) + rows = _load_jsonl("witness_log.jsonl") + permuted = import_witness_records(reversed(rows), resolve_payload_ref=resolve) + trace = imported.as_dict() + frame_trace_hashes = [frame.trace_hash for frame in imported.frames] + return { + "id": "jsonl_witness_import", + "record_count": imported.manifest.record_count, + "frame_count": len(imported.frames), + "trace_hash": imported.trace_hash, + "manifest_sha256": imported.manifest.manifest_sha256, + "frame_trace_hashes": frame_trace_hashes, + "deterministic_reimport_ok": imported.trace_hash == repeated.trace_hash, + "order_stability_ok": imported.trace_hash == permuted.trace_hash, + "trace_hygiene_ok": _trace_safe(trace) and "pixels" not in str(trace) and "action_trace" not in str(trace), + "no_actuation_ok": all( + not unit.pack_id.startswith("motor") + for frame in imported.frames + for unit in frame.units + ), + } + + def build_environment_falsification_report() -> dict[str, object]: fixtures = _load_json("fixtures.json")["fixtures"] cases = [_case_report(idx, case) for idx, case in enumerate(fixtures)] @@ -200,19 +240,30 @@ def build_environment_falsification_report() -> dict[str, object]: for case in scenario_cases if case["verdict_ok"] is True and case["trace_hygiene_ok"] is True ) + witness_import = _witness_import_report() + witness_ok = all( + witness_import[key] is True + for key in ( + "deterministic_reimport_ok", + "order_stability_ok", + "trace_hygiene_ok", + "no_actuation_ok", + ) + ) frame_passed = int(frame_report["passed"]) frame_failed = int(frame_report["failed"]) - total = len(cases) + len(scenario_cases) - passed = frame_passed + scenario_passed + total = len(cases) + len(scenario_cases) + 1 + passed = frame_passed + scenario_passed + (1 if witness_ok else 0) report = { "lane": "environment-falsification", "version": "v1", "total": total, "passed": passed, - "failed": frame_failed + (len(scenario_cases) - scenario_passed), + "failed": frame_failed + (len(scenario_cases) - scenario_passed) + (0 if witness_ok else 1), "cases": cases, "frame_report_sha256": frame_report_sha256, "scenario_cases": scenario_cases, + "witness_import": witness_import, } report["report_sha256"] = _report_hash(report) expected_hashes = _load_json("expected_hashes.json") diff --git a/evals/environment_falsification/witness_log.jsonl b/evals/environment_falsification/witness_log.jsonl new file mode 100644 index 00000000..3443c5ac --- /dev/null +++ b/evals/environment_falsification/witness_log.jsonl @@ -0,0 +1,3 @@ +{"modality":"audio","payload_ref":"audio_left_tone","slot_id":"audio:left_tone","source_clock":"witness-jsonl-fixture","tick":30} +{"modality":"vision","payload_ref":"vision_corner","slot_id":"vision:corner","source_clock":"witness-jsonl-fixture","tick":30} +{"modality":"sensorimotor","payload_ref":"sensorimotor_contact","slot_id":"sensorimotor:contact","source_clock":"witness-jsonl-fixture","tick":31} diff --git a/evals/environment_falsification/witness_payloads.json b/evals/environment_falsification/witness_payloads.json new file mode 100644 index 00000000..48ea756b --- /dev/null +++ b/evals/environment_falsification/witness_payloads.json @@ -0,0 +1,23 @@ +{ + "payloads": { + "audio_left_tone": { + "modality": "audio", + "signal": {"id": "witness_left_tone", "kind": "tone", "ms": 240, "hz": 180, "sweep": 0, "amp": 0.4} + }, + "vision_corner": { + "modality": "vision", + "signal": {"id": "witness_corner", "kind": "corner", "size": 32} + }, + "sensorimotor_contact": { + "modality": "sensorimotor", + "signal": { + "id": "witness_contact", + "pose_q": [10, -4, 3], + "velocity_q": [1, 0, -1], + "force_torque_q": [2, 3, 5], + "contact_q": [1, 0, 1, 0], + "actuator_state_q": [7, 8] + } + } + } +} diff --git a/sensorium/logs/__init__.py b/sensorium/logs/__init__.py new file mode 100644 index 00000000..c320995a --- /dev/null +++ b/sensorium/logs/__init__.py @@ -0,0 +1,17 @@ +"""Read-only witness log ingestion for offline afferent evidence.""" + +from sensorium.logs.importer import ( + ImportedObservationSequence, + WitnessLogManifest, + WitnessRecord, + import_witness_jsonl, + import_witness_records, +) + +__all__ = [ + "ImportedObservationSequence", + "WitnessLogManifest", + "WitnessRecord", + "import_witness_jsonl", + "import_witness_records", +] diff --git a/sensorium/logs/importer.py b/sensorium/logs/importer.py new file mode 100644 index 00000000..bc4eca27 --- /dev/null +++ b/sensorium/logs/importer.py @@ -0,0 +1,269 @@ +"""Offline JSONL witness-log importer. + +Witness logs are evidence transport, not truth. The importer accepts only +payload references and uses a caller-provided resolver to obtain already +bounded afferent compilation units. +""" + +from __future__ import annotations + +import json +from collections import defaultdict +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from sensorium.audio.checksum import sha256_json +from sensorium.compiler.protocol import CompilationUnitLike +from sensorium.environment import ObservationFrame, ObservationUnitRef, build_observation_frame + +_RAW_KEYS = {"pixels", "samples", "pcm", "waveform", "raw_bytes", "action_trace", "events"} +_LIVE_KEYS = {"device", "device_handle", "socket", "url", "network", "ros_node", "mcap_reader"} +_ALLOWED_MODALITIES = {"audio", "vision", "event-vision", "sensorimotor"} + + +@dataclass(frozen=True, slots=True) +class WitnessLogManifest: + source_kind: str + source_sha256: str + schema_version: str + record_count: int + manifest_sha256: str + + def as_dict(self) -> dict[str, object]: + return { + "source_kind": self.source_kind, + "source_sha256": self.source_sha256, + "schema_version": self.schema_version, + "record_count": self.record_count, + "manifest_sha256": self.manifest_sha256, + } + + +@dataclass(frozen=True, slots=True) +class WitnessRecord: + tick: int + source_clock: str + modality: str + slot_id: str + payload_ref: str + provenance_sha256: str + + def as_dict(self) -> dict[str, object]: + return { + "tick": self.tick, + "source_clock": self.source_clock, + "modality": self.modality, + "slot_id": self.slot_id, + "payload_ref": self.payload_ref, + "provenance_sha256": self.provenance_sha256, + } + + +@dataclass(frozen=True, slots=True) +class ImportedObservationSequence: + manifest: WitnessLogManifest + frames: tuple[ObservationFrame, ...] + frame_refs: tuple[tuple[str, tuple[ObservationUnitRef, ...]], ...] + trace_hash: str + + def as_dict(self) -> dict[str, object]: + return { + "manifest": self.manifest.as_dict(), + "frames": [ + { + "frame_id": frame.frame_id, + "monotonic_tick": frame.monotonic_tick, + "source_clock": frame.source_clock, + "environment_sha256": frame.environment_sha256, + "trace_hash": frame.trace_hash, + } + for frame in self.frames + ], + "frame_refs": [ + { + "frame_id": frame_id, + "slots": [ref.slot_id for ref in refs], + } + for frame_id, refs in self.frame_refs + ], + "trace_hash": self.trace_hash, + } + + +def _safe_ref(value: object, *, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{field} must be a non-empty string") + if ".." in value or "/" in value or "\\" in value: + raise ValueError(f"{field} must not contain path traversal") + for ch in value: + if not (ch.isascii() and (ch.isalnum() or ch in {"_", "-", ":", "."})): + raise ValueError(f"{field} contains an unsafe character") + return value + + +def _reject_raw_or_live(value: object) -> None: + if isinstance(value, Mapping): + for key, child in value.items(): + key_str = str(key) + if key_str in _RAW_KEYS: + raise ValueError(f"witness trace record contains raw payload field: {key_str}") + if key_str in _LIVE_KEYS: + raise ValueError(f"witness trace record contains live device field: {key_str}") + _reject_raw_or_live(child) + elif isinstance(value, (list, tuple)): + for child in value: + _reject_raw_or_live(child) + + +def _record_from_mapping(row: Mapping[str, Any]) -> WitnessRecord: + _reject_raw_or_live(row) + tick = int(row["tick"]) + if tick < 0: + raise ValueError("witness tick must be non-negative") + modality = _safe_ref(row["modality"], field="modality") + if modality not in _ALLOWED_MODALITIES: + raise ValueError(f"unsupported witness modality: {modality}") + slot_id = _safe_ref(row["slot_id"], field="slot_id") + payload_ref = _safe_ref(row["payload_ref"], field="payload_ref") + source_clock = _safe_ref(row.get("source_clock", "witness-jsonl"), field="source_clock") + payload = { + "kind": "WitnessRecord", + "tick": tick, + "source_clock": source_clock, + "modality": modality, + "slot_id": slot_id, + "payload_ref": payload_ref, + } + return WitnessRecord( + tick=tick, + source_clock=source_clock, + modality=modality, + slot_id=slot_id, + payload_ref=payload_ref, + provenance_sha256=sha256_json(payload), + ) + + +def _build_manifest( + records: tuple[WitnessRecord, ...], + *, + source_kind: str, + schema_version: str, +) -> WitnessLogManifest: + source_sha256 = sha256_json({ + "kind": "WitnessLog.source", + "records": [record.as_dict() for record in records], + }) + payload = { + "kind": "WitnessLogManifest", + "source_kind": source_kind, + "source_sha256": source_sha256, + "schema_version": schema_version, + "record_count": len(records), + } + return WitnessLogManifest( + source_kind=source_kind, + source_sha256=source_sha256, + schema_version=schema_version, + record_count=len(records), + manifest_sha256=sha256_json(payload), + ) + + +def import_witness_records( + rows: Iterable[Mapping[str, Any]], + *, + resolve_payload_ref: Callable[[str], CompilationUnitLike], + source_kind: str = "jsonl-witness-v1", + schema_version: str = "1", +) -> ImportedObservationSequence: + records = tuple(sorted( + (_record_from_mapping(row) for row in rows), + key=lambda record: ( + record.tick, + record.source_clock, + record.slot_id, + record.payload_ref, + record.provenance_sha256, + ), + )) + manifest = _build_manifest( + records, + source_kind=_safe_ref(source_kind, field="source_kind"), + schema_version=_safe_ref(schema_version, field="schema_version"), + ) + by_frame: dict[tuple[int, str], list[WitnessRecord]] = defaultdict(list) + for record in records: + by_frame[(record.tick, record.source_clock)].append(record) + + frames: list[ObservationFrame] = [] + frame_refs: list[tuple[str, tuple[ObservationUnitRef, ...]]] = [] + for (tick, source_clock), frame_records in sorted(by_frame.items()): + refs = tuple( + ObservationUnitRef(record.slot_id, resolve_payload_ref(record.payload_ref)) + for record in frame_records + ) + frame = build_observation_frame( + monotonic_tick=tick, + source_clock=source_clock, + units=tuple(ref.unit for ref in refs), + causal_parent_ids=(manifest.manifest_sha256,), + ) + frames.append(frame) + frame_refs.append((frame.frame_id, refs)) + + trace_hash = sha256_json({ + "kind": "ImportedObservationSequence", + "manifest_sha256": manifest.manifest_sha256, + "frame_trace_hashes": [frame.trace_hash for frame in frames], + "frame_ref_slots": [ + {"frame_id": frame_id, "slots": [ref.slot_id for ref in refs]} + for frame_id, refs in frame_refs + ], + }) + return ImportedObservationSequence( + manifest=manifest, + frames=tuple(frames), + frame_refs=tuple(frame_refs), + trace_hash=trace_hash, + ) + + +def _validate_offline_path(path: Path) -> Path: + if ".." in path.parts: + raise ValueError("witness log path must not contain path traversal") + resolved = path.resolve() + if not resolved.is_file(): + raise FileNotFoundError(resolved) + return resolved + + +def import_witness_jsonl( + path: str | Path, + *, + resolve_payload_ref: Callable[[str], CompilationUnitLike], + schema_version: str = "1", +) -> ImportedObservationSequence: + log_path = _validate_offline_path(Path(path)) + rows = [ + json.loads(line) + for line in log_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + return import_witness_records( + rows, + resolve_payload_ref=resolve_payload_ref, + source_kind="jsonl-witness-v1", + schema_version=schema_version, + ) + + +__all__ = [ + "ImportedObservationSequence", + "WitnessLogManifest", + "WitnessRecord", + "import_witness_jsonl", + "import_witness_records", +] diff --git a/tests/test_witness_log_importer.py b/tests/test_witness_log_importer.py new file mode 100644 index 00000000..f8acfdf9 --- /dev/null +++ b/tests/test_witness_log_importer.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import builtins +import json +from dataclasses import dataclass + +import numpy as np +import pytest + +from sensorium.logs import import_witness_jsonl, import_witness_records + + +@dataclass(frozen=True, slots=True) +class _Unit: + canonical_sha256: str + ir_sha256: str + pack_id: str + pack_manifest_sha256: str + projection_sha256: str + versor: np.ndarray + versor_condition: float = 0.0 + + @property + def merge_key(self) -> tuple[str, str, str]: + return (self.canonical_sha256, self.ir_sha256, self.projection_sha256) + + +def _unit(name: str, pack_id: str = "vision_core_v1") -> _Unit: + v = np.zeros(32, dtype=np.float32) + v[0] = 1.0 + return _Unit(name, f"ir-{name}", pack_id, "manifest", f"proj-{name}", v) + + +def _resolver(ref: str): + return _unit(ref) + + +def _rows(): + return [ + { + "tick": 2, + "source_clock": "fixture", + "modality": "vision", + "slot_id": "vision:anchor", + "payload_ref": "vision_anchor", + }, + { + "tick": 1, + "source_clock": "fixture", + "modality": "sensorimotor", + "slot_id": "sensorimotor:contact", + "payload_ref": "contact_anchor", + }, + ] + + +def test_witness_import_is_order_stable_and_trace_safe(): + first = import_witness_records(_rows(), resolve_payload_ref=_resolver) + second = import_witness_records(reversed(_rows()), resolve_payload_ref=_resolver) + assert first.trace_hash == second.trace_hash + assert [frame.monotonic_tick for frame in first.frames] == [1, 2] + payload = first.as_dict() + assert "vision_anchor" not in str(payload) + assert "versor" not in str(payload) + assert "pixels" not in str(payload) + assert "action_trace" not in str(payload) + + +def test_witness_jsonl_import_rejects_path_traversal(tmp_path): + with pytest.raises(ValueError, match="path traversal"): + import_witness_jsonl(tmp_path / ".." / "witness.jsonl", resolve_payload_ref=_resolver) + + +def test_witness_import_rejects_raw_payload_and_live_handles(): + bad_raw = dict(_rows()[0]) + bad_raw["pixels"] = [1, 2, 3] + with pytest.raises(ValueError, match="raw payload"): + import_witness_records([bad_raw], resolve_payload_ref=_resolver) + + bad_live = dict(_rows()[0]) + bad_live["socket"] = "localhost:11311" + with pytest.raises(ValueError, match="live device"): + import_witness_records([bad_live], resolve_payload_ref=_resolver) + + +def test_witness_jsonl_repeated_import_is_identical(tmp_path): + path = tmp_path / "witness.jsonl" + path.write_text("\n".join(json.dumps(row, sort_keys=True) for row in _rows()) + "\n") + first = import_witness_jsonl(path, resolve_payload_ref=_resolver) + second = import_witness_jsonl(path, resolve_payload_ref=_resolver) + assert first.trace_hash == second.trace_hash + assert first.manifest.manifest_sha256 == second.manifest.manifest_sha256 + + +def test_witness_importer_does_not_import_generate_or_call_decode(monkeypatch): + original_import = builtins.__import__ + + def guarded_import(name, *args, **kwargs): + if name.startswith("generate"): + raise AssertionError("witness importer must not import generate") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", guarded_import) + from sensorium.registry import ModalityRegistry + + monkeypatch.setattr( + ModalityRegistry, + "decode", + lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("decode called")), + ) + imported = import_witness_records(_rows(), resolve_payload_ref=_resolver) + assert len(imported.frames) == 2