Add offline witness log importer

core/cli.py

@@ -132,6 +132,7 @@ _TEST_SUITES: dict[str, tuple[str, ...]] = {
         "tests/test_observation_frame_harness.py",
         "tests/test_environment_falsification.py",
         "tests/test_environment_falsification_eval_cli.py",
+        "tests/test_witness_log_importer.py",
         "tests/test_sensorium_eval_cli.py",
         "tests/test_efferent_gate.py",
     ),

evals/environment_falsification/expected_hashes.json

@@ -1,4 +1,4 @@
 {
   "frame_report_sha256": "c97b2dca7282d0231f1b448add87256cc2f59d39c5cec5ac1350231541e07d0b",
-  "report_sha256": "e0903e408f882bf7c93b7aadb6b3f8a064438ac09dc1cd4139f381bdc520b923"
+  "report_sha256": "7d683ec2ca43866a23daf13a24abcc6334627f5c68a0cc05d93b10e1f313ca12"
 }

evals/environment_falsification/report.py

@@ -23,6 +23,7 @@ from sensorium.environment import (
     compare_expected_to_observation,
     run_falsification_scenario,
 )
+from sensorium.logs import import_witness_jsonl, import_witness_records
 from sensorium.sensorimotor.compiler import SensorimotorCompiler
 from sensorium.vision import VisionCompiler, canonicalize_image
 from sensorium.vision.grid import iter_tile_signals
@@ -35,6 +36,14 @@ def _load_json(name: str) -> dict[str, Any]:
     return json.loads((_ROOT / name).read_text(encoding="utf-8"))
 
 
+def _load_jsonl(name: str) -> list[dict[str, Any]]:
+    return [
+        json.loads(line)
+        for line in (_ROOT / name).read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+
+
 def _trace_safe(value: object) -> bool:
     if isinstance(value, (np.ndarray, bytes, bytearray)):
         return False
@@ -185,6 +194,37 @@ def _scenario_case_report(index: int, scenario: dict[str, Any]) -> dict[str, obj
     return row
 
 
+def _witness_import_report() -> dict[str, object]:
+    payloads = _load_json("witness_payloads.json")["payloads"]
+
+    def resolve(payload_ref: str):
+        return _compile_unit(payloads[payload_ref])
+
+    path = _ROOT / "witness_log.jsonl"
+    imported = import_witness_jsonl(path, resolve_payload_ref=resolve)
+    repeated = import_witness_jsonl(path, resolve_payload_ref=resolve)
+    rows = _load_jsonl("witness_log.jsonl")
+    permuted = import_witness_records(reversed(rows), resolve_payload_ref=resolve)
+    trace = imported.as_dict()
+    frame_trace_hashes = [frame.trace_hash for frame in imported.frames]
+    return {
+        "id": "jsonl_witness_import",
+        "record_count": imported.manifest.record_count,
+        "frame_count": len(imported.frames),
+        "trace_hash": imported.trace_hash,
+        "manifest_sha256": imported.manifest.manifest_sha256,
+        "frame_trace_hashes": frame_trace_hashes,
+        "deterministic_reimport_ok": imported.trace_hash == repeated.trace_hash,
+        "order_stability_ok": imported.trace_hash == permuted.trace_hash,
+        "trace_hygiene_ok": _trace_safe(trace) and "pixels" not in str(trace) and "action_trace" not in str(trace),
+        "no_actuation_ok": all(
+            not unit.pack_id.startswith("motor")
+            for frame in imported.frames
+            for unit in frame.units
+        ),
+    }
+
+
 def build_environment_falsification_report() -> dict[str, object]:
     fixtures = _load_json("fixtures.json")["fixtures"]
     cases = [_case_report(idx, case) for idx, case in enumerate(fixtures)]
@@ -200,19 +240,30 @@ def build_environment_falsification_report() -> dict[str, object]:
         for case in scenario_cases
         if case["verdict_ok"] is True and case["trace_hygiene_ok"] is True
     )
+    witness_import = _witness_import_report()
+    witness_ok = all(
+        witness_import[key] is True
+        for key in (
+            "deterministic_reimport_ok",
+            "order_stability_ok",
+            "trace_hygiene_ok",
+            "no_actuation_ok",
+        )
+    )
     frame_passed = int(frame_report["passed"])
     frame_failed = int(frame_report["failed"])
-    total = len(cases) + len(scenario_cases)
+    total = len(cases) + len(scenario_cases) + 1
-    passed = frame_passed + scenario_passed
+    passed = frame_passed + scenario_passed + (1 if witness_ok else 0)
     report = {
         "lane": "environment-falsification",
         "version": "v1",
         "total": total,
         "passed": passed,
-        "failed": frame_failed + (len(scenario_cases) - scenario_passed),
+        "failed": frame_failed + (len(scenario_cases) - scenario_passed) + (0 if witness_ok else 1),
         "cases": cases,
         "frame_report_sha256": frame_report_sha256,
         "scenario_cases": scenario_cases,
+        "witness_import": witness_import,
     }
     report["report_sha256"] = _report_hash(report)
     expected_hashes = _load_json("expected_hashes.json")

evals/environment_falsification/witness_log.jsonl

@@ -0,0 +1,3 @@
+{"modality":"audio","payload_ref":"audio_left_tone","slot_id":"audio:left_tone","source_clock":"witness-jsonl-fixture","tick":30}
+{"modality":"vision","payload_ref":"vision_corner","slot_id":"vision:corner","source_clock":"witness-jsonl-fixture","tick":30}
+{"modality":"sensorimotor","payload_ref":"sensorimotor_contact","slot_id":"sensorimotor:contact","source_clock":"witness-jsonl-fixture","tick":31}

evals/environment_falsification/witness_payloads.json

@@ -0,0 +1,23 @@
+{
+  "payloads": {
+    "audio_left_tone": {
+      "modality": "audio",
+      "signal": {"id": "witness_left_tone", "kind": "tone", "ms": 240, "hz": 180, "sweep": 0, "amp": 0.4}
+    },
+    "vision_corner": {
+      "modality": "vision",
+      "signal": {"id": "witness_corner", "kind": "corner", "size": 32}
+    },
+    "sensorimotor_contact": {
+      "modality": "sensorimotor",
+      "signal": {
+        "id": "witness_contact",
+        "pose_q": [10, -4, 3],
+        "velocity_q": [1, 0, -1],
+        "force_torque_q": [2, 3, 5],
+        "contact_q": [1, 0, 1, 0],
+        "actuator_state_q": [7, 8]
+      }
+    }
+  }
+}

sensorium/logs/__init__.py

@@ -0,0 +1,17 @@
+"""Read-only witness log ingestion for offline afferent evidence."""
+
+from sensorium.logs.importer import (
+    ImportedObservationSequence,
+    WitnessLogManifest,
+    WitnessRecord,
+    import_witness_jsonl,
+    import_witness_records,
+)
+
+__all__ = [
+    "ImportedObservationSequence",
+    "WitnessLogManifest",
+    "WitnessRecord",
+    "import_witness_jsonl",
+    "import_witness_records",
+]

sensorium/logs/importer.py

@@ -0,0 +1,269 @@
+"""Offline JSONL witness-log importer.
+
+Witness logs are evidence transport, not truth. The importer accepts only
+payload references and uses a caller-provided resolver to obtain already
+bounded afferent compilation units.
+"""
+
+from __future__ import annotations
+
+import json
+from collections import defaultdict
+from collections.abc import Callable, Iterable, Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from sensorium.audio.checksum import sha256_json
+from sensorium.compiler.protocol import CompilationUnitLike
+from sensorium.environment import ObservationFrame, ObservationUnitRef, build_observation_frame
+
+_RAW_KEYS = {"pixels", "samples", "pcm", "waveform", "raw_bytes", "action_trace", "events"}
+_LIVE_KEYS = {"device", "device_handle", "socket", "url", "network", "ros_node", "mcap_reader"}
+_ALLOWED_MODALITIES = {"audio", "vision", "event-vision", "sensorimotor"}
+
+
+@dataclass(frozen=True, slots=True)
+class WitnessLogManifest:
+    source_kind: str
+    source_sha256: str
+    schema_version: str
+    record_count: int
+    manifest_sha256: str
+
+    def as_dict(self) -> dict[str, object]:
+        return {
+            "source_kind": self.source_kind,
+            "source_sha256": self.source_sha256,
+            "schema_version": self.schema_version,
+            "record_count": self.record_count,
+            "manifest_sha256": self.manifest_sha256,
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class WitnessRecord:
+    tick: int
+    source_clock: str
+    modality: str
+    slot_id: str
+    payload_ref: str
+    provenance_sha256: str
+
+    def as_dict(self) -> dict[str, object]:
+        return {
+            "tick": self.tick,
+            "source_clock": self.source_clock,
+            "modality": self.modality,
+            "slot_id": self.slot_id,
+            "payload_ref": self.payload_ref,
+            "provenance_sha256": self.provenance_sha256,
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class ImportedObservationSequence:
+    manifest: WitnessLogManifest
+    frames: tuple[ObservationFrame, ...]
+    frame_refs: tuple[tuple[str, tuple[ObservationUnitRef, ...]], ...]
+    trace_hash: str
+
+    def as_dict(self) -> dict[str, object]:
+        return {
+            "manifest": self.manifest.as_dict(),
+            "frames": [
+                {
+                    "frame_id": frame.frame_id,
+                    "monotonic_tick": frame.monotonic_tick,
+                    "source_clock": frame.source_clock,
+                    "environment_sha256": frame.environment_sha256,
+                    "trace_hash": frame.trace_hash,
+                }
+                for frame in self.frames
+            ],
+            "frame_refs": [
+                {
+                    "frame_id": frame_id,
+                    "slots": [ref.slot_id for ref in refs],
+                }
+                for frame_id, refs in self.frame_refs
+            ],
+            "trace_hash": self.trace_hash,
+        }
+
+
+def _safe_ref(value: object, *, field: str) -> str:
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError(f"{field} must be a non-empty string")
+    if ".." in value or "/" in value or "\\" in value:
+        raise ValueError(f"{field} must not contain path traversal")
+    for ch in value:
+        if not (ch.isascii() and (ch.isalnum() or ch in {"_", "-", ":", "."})):
+            raise ValueError(f"{field} contains an unsafe character")
+    return value
+
+
+def _reject_raw_or_live(value: object) -> None:
+    if isinstance(value, Mapping):
+        for key, child in value.items():
+            key_str = str(key)
+            if key_str in _RAW_KEYS:
+                raise ValueError(f"witness trace record contains raw payload field: {key_str}")
+            if key_str in _LIVE_KEYS:
+                raise ValueError(f"witness trace record contains live device field: {key_str}")
+            _reject_raw_or_live(child)
+    elif isinstance(value, (list, tuple)):
+        for child in value:
+            _reject_raw_or_live(child)
+
+
+def _record_from_mapping(row: Mapping[str, Any]) -> WitnessRecord:
+    _reject_raw_or_live(row)
+    tick = int(row["tick"])
+    if tick < 0:
+        raise ValueError("witness tick must be non-negative")
+    modality = _safe_ref(row["modality"], field="modality")
+    if modality not in _ALLOWED_MODALITIES:
+        raise ValueError(f"unsupported witness modality: {modality}")
+    slot_id = _safe_ref(row["slot_id"], field="slot_id")
+    payload_ref = _safe_ref(row["payload_ref"], field="payload_ref")
+    source_clock = _safe_ref(row.get("source_clock", "witness-jsonl"), field="source_clock")
+    payload = {
+        "kind": "WitnessRecord",
+        "tick": tick,
+        "source_clock": source_clock,
+        "modality": modality,
+        "slot_id": slot_id,
+        "payload_ref": payload_ref,
+    }
+    return WitnessRecord(
+        tick=tick,
+        source_clock=source_clock,
+        modality=modality,
+        slot_id=slot_id,
+        payload_ref=payload_ref,
+        provenance_sha256=sha256_json(payload),
+    )
+
+
+def _build_manifest(
+    records: tuple[WitnessRecord, ...],
+    *,
+    source_kind: str,
+    schema_version: str,
+) -> WitnessLogManifest:
+    source_sha256 = sha256_json({
+        "kind": "WitnessLog.source",
+        "records": [record.as_dict() for record in records],
+    })
+    payload = {
+        "kind": "WitnessLogManifest",
+        "source_kind": source_kind,
+        "source_sha256": source_sha256,
+        "schema_version": schema_version,
+        "record_count": len(records),
+    }
+    return WitnessLogManifest(
+        source_kind=source_kind,
+        source_sha256=source_sha256,
+        schema_version=schema_version,
+        record_count=len(records),
+        manifest_sha256=sha256_json(payload),
+    )
+
+
+def import_witness_records(
+    rows: Iterable[Mapping[str, Any]],
+    *,
+    resolve_payload_ref: Callable[[str], CompilationUnitLike],
+    source_kind: str = "jsonl-witness-v1",
+    schema_version: str = "1",
+) -> ImportedObservationSequence:
+    records = tuple(sorted(
+        (_record_from_mapping(row) for row in rows),
+        key=lambda record: (
+            record.tick,
+            record.source_clock,
+            record.slot_id,
+            record.payload_ref,
+            record.provenance_sha256,
+        ),
+    ))
+    manifest = _build_manifest(
+        records,
+        source_kind=_safe_ref(source_kind, field="source_kind"),
+        schema_version=_safe_ref(schema_version, field="schema_version"),
+    )
+    by_frame: dict[tuple[int, str], list[WitnessRecord]] = defaultdict(list)
+    for record in records:
+        by_frame[(record.tick, record.source_clock)].append(record)
+
+    frames: list[ObservationFrame] = []
+    frame_refs: list[tuple[str, tuple[ObservationUnitRef, ...]]] = []
+    for (tick, source_clock), frame_records in sorted(by_frame.items()):
+        refs = tuple(
+            ObservationUnitRef(record.slot_id, resolve_payload_ref(record.payload_ref))
+            for record in frame_records
+        )
+        frame = build_observation_frame(
+            monotonic_tick=tick,
+            source_clock=source_clock,
+            units=tuple(ref.unit for ref in refs),
+            causal_parent_ids=(manifest.manifest_sha256,),
+        )
+        frames.append(frame)
+        frame_refs.append((frame.frame_id, refs))
+
+    trace_hash = sha256_json({
+        "kind": "ImportedObservationSequence",
+        "manifest_sha256": manifest.manifest_sha256,
+        "frame_trace_hashes": [frame.trace_hash for frame in frames],
+        "frame_ref_slots": [
+            {"frame_id": frame_id, "slots": [ref.slot_id for ref in refs]}
+            for frame_id, refs in frame_refs
+        ],
+    })
+    return ImportedObservationSequence(
+        manifest=manifest,
+        frames=tuple(frames),
+        frame_refs=tuple(frame_refs),
+        trace_hash=trace_hash,
+    )
+
+
+def _validate_offline_path(path: Path) -> Path:
+    if ".." in path.parts:
+        raise ValueError("witness log path must not contain path traversal")
+    resolved = path.resolve()
+    if not resolved.is_file():
+        raise FileNotFoundError(resolved)
+    return resolved
+
+
+def import_witness_jsonl(
+    path: str | Path,
+    *,
+    resolve_payload_ref: Callable[[str], CompilationUnitLike],
+    schema_version: str = "1",
+) -> ImportedObservationSequence:
+    log_path = _validate_offline_path(Path(path))
+    rows = [
+        json.loads(line)
+        for line in log_path.read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+    return import_witness_records(
+        rows,
+        resolve_payload_ref=resolve_payload_ref,
+        source_kind="jsonl-witness-v1",
+        schema_version=schema_version,
+    )
+
+
+__all__ = [
+    "ImportedObservationSequence",
+    "WitnessLogManifest",
+    "WitnessRecord",
+    "import_witness_jsonl",
+    "import_witness_records",
+]

tests/test_witness_log_importer.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+import builtins
+import json
+from dataclasses import dataclass
+
+import numpy as np
+import pytest
+
+from sensorium.logs import import_witness_jsonl, import_witness_records
+
+
+@dataclass(frozen=True, slots=True)
+class _Unit:
+    canonical_sha256: str
+    ir_sha256: str
+    pack_id: str
+    pack_manifest_sha256: str
+    projection_sha256: str
+    versor: np.ndarray
+    versor_condition: float = 0.0
+
+    @property
+    def merge_key(self) -> tuple[str, str, str]:
+        return (self.canonical_sha256, self.ir_sha256, self.projection_sha256)
+
+
+def _unit(name: str, pack_id: str = "vision_core_v1") -> _Unit:
+    v = np.zeros(32, dtype=np.float32)
+    v[0] = 1.0
+    return _Unit(name, f"ir-{name}", pack_id, "manifest", f"proj-{name}", v)
+
+
+def _resolver(ref: str):
+    return _unit(ref)
+
+
+def _rows():
+    return [
+        {
+            "tick": 2,
+            "source_clock": "fixture",
+            "modality": "vision",
+            "slot_id": "vision:anchor",
+            "payload_ref": "vision_anchor",
+        },
+        {
+            "tick": 1,
+            "source_clock": "fixture",
+            "modality": "sensorimotor",
+            "slot_id": "sensorimotor:contact",
+            "payload_ref": "contact_anchor",
+        },
+    ]
+
+
+def test_witness_import_is_order_stable_and_trace_safe():
+    first = import_witness_records(_rows(), resolve_payload_ref=_resolver)
+    second = import_witness_records(reversed(_rows()), resolve_payload_ref=_resolver)
+    assert first.trace_hash == second.trace_hash
+    assert [frame.monotonic_tick for frame in first.frames] == [1, 2]
+    payload = first.as_dict()
+    assert "vision_anchor" not in str(payload)
+    assert "versor" not in str(payload)
+    assert "pixels" not in str(payload)
+    assert "action_trace" not in str(payload)
+
+
+def test_witness_jsonl_import_rejects_path_traversal(tmp_path):
+    with pytest.raises(ValueError, match="path traversal"):
+        import_witness_jsonl(tmp_path / ".." / "witness.jsonl", resolve_payload_ref=_resolver)
+
+
+def test_witness_import_rejects_raw_payload_and_live_handles():
+    bad_raw = dict(_rows()[0])
+    bad_raw["pixels"] = [1, 2, 3]
+    with pytest.raises(ValueError, match="raw payload"):
+        import_witness_records([bad_raw], resolve_payload_ref=_resolver)
+
+    bad_live = dict(_rows()[0])
+    bad_live["socket"] = "localhost:11311"
+    with pytest.raises(ValueError, match="live device"):
+        import_witness_records([bad_live], resolve_payload_ref=_resolver)
+
+
+def test_witness_jsonl_repeated_import_is_identical(tmp_path):
+    path = tmp_path / "witness.jsonl"
+    path.write_text("\n".join(json.dumps(row, sort_keys=True) for row in _rows()) + "\n")
+    first = import_witness_jsonl(path, resolve_payload_ref=_resolver)
+    second = import_witness_jsonl(path, resolve_payload_ref=_resolver)
+    assert first.trace_hash == second.trace_hash
+    assert first.manifest.manifest_sha256 == second.manifest.manifest_sha256
+
+
+def test_witness_importer_does_not_import_generate_or_call_decode(monkeypatch):
+    original_import = builtins.__import__
+
+    def guarded_import(name, *args, **kwargs):
+        if name.startswith("generate"):
+            raise AssertionError("witness importer must not import generate")
+        return original_import(name, *args, **kwargs)
+
+    monkeypatch.setattr(builtins, "__import__", guarded_import)
+    from sensorium.registry import ModalityRegistry
+
+    monkeypatch.setattr(
+        ModalityRegistry,
+        "decode",
+        lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("decode called")),
+    )
+    imported = import_witness_records(_rows(), resolve_payload_ref=_resolver)
+    assert len(imported.frames) == 2