Add falsification scenario layer
This commit is contained in:
Shay
parent
f9205cc86e
commit
d9276eec31
7 changed files with 658 additions and 5 deletions
|
|
@ -1,3 +1,4 @@
|
|||
{
|
||||
"report_sha256": "c97b2dca7282d0231f1b448add87256cc2f59d39c5cec5ac1350231541e07d0b"
|
||||
"frame_report_sha256": "c97b2dca7282d0231f1b448add87256cc2f59d39c5cec5ac1350231541e07d0b",
|
||||
"report_sha256": "e0903e408f882bf7c93b7aadb6b3f8a064438ac09dc1cd4139f381bdc520b923"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,9 +16,12 @@ from sensorium.audio.compiler import AudioCompiler
|
|||
from sensorium.audio.checksum import sha256_json
|
||||
from sensorium.environment import (
|
||||
ObservationUnitRef,
|
||||
build_experiment_plan,
|
||||
build_expected_observation_frame,
|
||||
build_hypothesis_claim,
|
||||
build_observation_frame,
|
||||
compare_expected_to_observation,
|
||||
run_falsification_scenario,
|
||||
)
|
||||
from sensorium.sensorimotor.compiler import SensorimotorCompiler
|
||||
from sensorium.vision import VisionCompiler, canonicalize_image
|
||||
|
|
@ -107,15 +110,13 @@ def _report_hash(report_without_hash: dict[str, object]) -> str:
|
|||
return sha256_json(report_without_hash)
|
||||
|
||||
|
||||
def build_environment_falsification_report() -> dict[str, object]:
|
||||
fixtures = _load_json("fixtures.json")["fixtures"]
|
||||
cases = [_case_report(idx, case) for idx, case in enumerate(fixtures)]
|
||||
def _frame_report(cases: list[dict[str, object]]) -> dict[str, object]:
|
||||
passed = sum(
|
||||
1
|
||||
for case in cases
|
||||
if case["verdict_ok"] is True and case["trace_hygiene_ok"] is True
|
||||
)
|
||||
report = {
|
||||
return {
|
||||
"lane": "environment-falsification",
|
||||
"version": "v1",
|
||||
"total": len(cases),
|
||||
|
|
@ -123,10 +124,108 @@ def build_environment_falsification_report() -> dict[str, object]:
|
|||
"failed": len(cases) - passed,
|
||||
"cases": cases,
|
||||
}
|
||||
|
||||
|
||||
def _scenario_case_report(index: int, scenario: dict[str, Any]) -> dict[str, object]:
|
||||
hypothesis_spec = scenario["hypothesis"]
|
||||
hypothesis = build_hypothesis_claim(
|
||||
claim_id=str(hypothesis_spec["claim_id"]),
|
||||
claim_text=str(hypothesis_spec["claim_text"]),
|
||||
domain=str(hypothesis_spec["domain"]),
|
||||
basis_trace_hashes=tuple(hypothesis_spec.get("basis_trace_hashes", ())),
|
||||
)
|
||||
expected_frames = []
|
||||
actual_frames_by_expected_id = {}
|
||||
actual_refs_by_expected_id = {}
|
||||
for offset, frame_spec in enumerate(scenario["frames"]):
|
||||
tick = int(frame_spec.get("tick", index * 100 + offset))
|
||||
expected_refs = _refs(frame_spec["expected"])
|
||||
expected = build_expected_observation_frame(
|
||||
monotonic_tick=tick,
|
||||
source_clock="environment-falsification-scenario-fixture",
|
||||
unit_refs=expected_refs,
|
||||
causal_parent_ids=tuple(frame_spec.get("causal_parent_ids", ())),
|
||||
)
|
||||
expected_frames.append(expected)
|
||||
if frame_spec["actual"] == "missing":
|
||||
continue
|
||||
actual_spec = frame_spec["expected"] if frame_spec["actual"] == "same" else frame_spec["actual"]
|
||||
actual_refs = _refs(actual_spec)
|
||||
actual = build_observation_frame(
|
||||
monotonic_tick=tick,
|
||||
source_clock="environment-falsification-scenario-fixture",
|
||||
units=tuple(ref.unit for ref in actual_refs),
|
||||
causal_parent_ids=(expected.expected_id,),
|
||||
)
|
||||
actual_frames_by_expected_id[expected.expected_id] = actual
|
||||
actual_refs_by_expected_id[expected.expected_id] = actual_refs
|
||||
|
||||
plan = build_experiment_plan(hypothesis=hypothesis, expected_frames=expected_frames)
|
||||
report = run_falsification_scenario(
|
||||
plan,
|
||||
actual_frames_by_expected_id=actual_frames_by_expected_id,
|
||||
actual_refs_by_expected_id=actual_refs_by_expected_id,
|
||||
)
|
||||
expected_verdict = str(scenario["expected_verdict"])
|
||||
row = {
|
||||
"id": scenario["id"],
|
||||
"expected_verdict": expected_verdict,
|
||||
"actual_verdict": report.verdict,
|
||||
"verdict_ok": report.verdict == expected_verdict,
|
||||
"trace_hygiene_ok": _trace_safe(report.as_dict()),
|
||||
"hypothesis_sha256": hypothesis.hypothesis_sha256,
|
||||
"plan_sha256": plan.plan_sha256,
|
||||
"scenario_sha256": report.scenario_sha256,
|
||||
"scenario_report_sha256": report.report_sha256,
|
||||
"total_count": report.total_count,
|
||||
"supported_count": report.supported_count,
|
||||
"falsified_count": report.falsified_count,
|
||||
"run_trace_hashes": [run.trace_hash for run in report.runs],
|
||||
}
|
||||
return row
|
||||
|
||||
|
||||
def build_environment_falsification_report() -> dict[str, object]:
|
||||
fixtures = _load_json("fixtures.json")["fixtures"]
|
||||
cases = [_case_report(idx, case) for idx, case in enumerate(fixtures)]
|
||||
frame_report = _frame_report(cases)
|
||||
frame_report_sha256 = _report_hash(frame_report)
|
||||
scenario_fixtures = _load_json("scenario_fixtures.json")["scenarios"]
|
||||
scenario_cases = [
|
||||
_scenario_case_report(idx, scenario)
|
||||
for idx, scenario in enumerate(scenario_fixtures)
|
||||
]
|
||||
scenario_passed = sum(
|
||||
1
|
||||
for case in scenario_cases
|
||||
if case["verdict_ok"] is True and case["trace_hygiene_ok"] is True
|
||||
)
|
||||
frame_passed = int(frame_report["passed"])
|
||||
frame_failed = int(frame_report["failed"])
|
||||
total = len(cases) + len(scenario_cases)
|
||||
passed = frame_passed + scenario_passed
|
||||
report = {
|
||||
"lane": "environment-falsification",
|
||||
"version": "v1",
|
||||
"total": total,
|
||||
"passed": passed,
|
||||
"failed": frame_failed + (len(scenario_cases) - scenario_passed),
|
||||
"cases": cases,
|
||||
"frame_report_sha256": frame_report_sha256,
|
||||
"scenario_cases": scenario_cases,
|
||||
}
|
||||
report["report_sha256"] = _report_hash(report)
|
||||
expected_hashes = _load_json("expected_hashes.json")
|
||||
expected_frame_report_sha256 = expected_hashes.get(
|
||||
"frame_report_sha256",
|
||||
expected_hashes["report_sha256"],
|
||||
)
|
||||
report["expected_frame_report_sha256"] = expected_frame_report_sha256
|
||||
report["expected_frame_report_hash_ok"] = frame_report_sha256 == expected_frame_report_sha256
|
||||
report["expected_report_sha256"] = expected_hashes["report_sha256"]
|
||||
report["expected_report_hash_ok"] = report["report_sha256"] == expected_hashes["report_sha256"]
|
||||
if not report["expected_frame_report_hash_ok"]:
|
||||
report["failed"] = int(report["failed"]) + 1
|
||||
if not report["expected_report_hash_ok"]:
|
||||
report["failed"] = int(report["failed"]) + 1
|
||||
return report
|
||||
|
|
|
|||
88
evals/environment_falsification/scenario_fixtures.json
Normal file
88
evals/environment_falsification/scenario_fixtures.json
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
{
|
||||
"scenarios": [
|
||||
{
|
||||
"id": "supported_passive_contact_hypothesis",
|
||||
"expected_verdict": "SUPPORTED",
|
||||
"hypothesis": {
|
||||
"claim_id": "passive-contact-stays-anchored",
|
||||
"claim_text": "When the passive contact fixture remains anchored, audio, vision, and contact evidence repeat exactly.",
|
||||
"domain": "passive_tabletop",
|
||||
"basis_trace_hashes": ["basis-passive-contact", "basis-vision-anchor", "basis-passive-contact"]
|
||||
},
|
||||
"frames": [
|
||||
{
|
||||
"tick": 10,
|
||||
"expected": {
|
||||
"audio:left_tone": {
|
||||
"modality": "audio",
|
||||
"signal": {"id": "scenario_left_tone", "kind": "tone", "ms": 240, "hz": 180, "sweep": 0, "amp": 0.4}
|
||||
},
|
||||
"vision:corner": {
|
||||
"modality": "vision",
|
||||
"signal": {"id": "scenario_corner", "kind": "corner", "size": 32}
|
||||
}
|
||||
},
|
||||
"actual": "same"
|
||||
},
|
||||
{
|
||||
"tick": 11,
|
||||
"expected": {
|
||||
"sensorimotor:contact": {
|
||||
"modality": "sensorimotor",
|
||||
"signal": {
|
||||
"id": "scenario_contact",
|
||||
"pose_q": [10, -4, 3],
|
||||
"velocity_q": [1, 0, -1],
|
||||
"force_torque_q": [2, 3, 5],
|
||||
"contact_q": [1, 0, 1, 0],
|
||||
"actuator_state_q": [7, 8]
|
||||
}
|
||||
}
|
||||
},
|
||||
"actual": "same"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "falsified_passive_contact_hypothesis",
|
||||
"expected_verdict": "FALSIFIED",
|
||||
"hypothesis": {
|
||||
"claim_id": "passive-contact-breaks-on-release",
|
||||
"claim_text": "A released passive contact fixture should continue to emit the original contact evidence.",
|
||||
"domain": "passive_tabletop",
|
||||
"basis_trace_hashes": ["basis-contact-release"]
|
||||
},
|
||||
"frames": [
|
||||
{
|
||||
"tick": 20,
|
||||
"expected": {
|
||||
"sensorimotor:contact": {
|
||||
"modality": "sensorimotor",
|
||||
"signal": {
|
||||
"id": "scenario_contact_expected",
|
||||
"pose_q": [10, -4, 3],
|
||||
"velocity_q": [1, 0, -1],
|
||||
"force_torque_q": [2, 3, 5],
|
||||
"contact_q": [1, 0, 1, 0],
|
||||
"actuator_state_q": [7, 8]
|
||||
}
|
||||
}
|
||||
},
|
||||
"actual": {
|
||||
"sensorimotor:contact": {
|
||||
"modality": "sensorimotor",
|
||||
"signal": {
|
||||
"id": "scenario_contact_actual",
|
||||
"pose_q": [10, -4, 3],
|
||||
"velocity_q": [1, 0, -1],
|
||||
"force_torque_q": [2, 3, 5],
|
||||
"contact_q": [0, 0, 0, 0],
|
||||
"actuator_state_q": [7, 8]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -11,16 +11,36 @@ from sensorium.environment.falsification import (
|
|||
)
|
||||
from sensorium.environment.frame import ObservationFrame, build_observation_frame
|
||||
from sensorium.environment.harness import build_fixture_observation_frame
|
||||
from sensorium.environment.scenario import (
|
||||
ExperimentPlan,
|
||||
FalsificationScenario,
|
||||
HypothesisClaim,
|
||||
ScenarioActualFrame,
|
||||
ScenarioReport,
|
||||
build_experiment_plan,
|
||||
build_falsification_scenario,
|
||||
build_hypothesis_claim,
|
||||
run_falsification_scenario,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ChangedSlot",
|
||||
"ExperimentPlan",
|
||||
"ExpectedObservationFrame",
|
||||
"FalsificationScenario",
|
||||
"FalsificationResidual",
|
||||
"FalsificationRun",
|
||||
"HypothesisClaim",
|
||||
"ObservationFrame",
|
||||
"ObservationUnitRef",
|
||||
"ScenarioActualFrame",
|
||||
"ScenarioReport",
|
||||
"build_experiment_plan",
|
||||
"build_expected_observation_frame",
|
||||
"build_falsification_scenario",
|
||||
"build_fixture_observation_frame",
|
||||
"build_hypothesis_claim",
|
||||
"build_observation_frame",
|
||||
"compare_expected_to_observation",
|
||||
"run_falsification_scenario",
|
||||
]
|
||||
|
|
|
|||
338
sensorium/environment/scenario.py
Normal file
338
sensorium/environment/scenario.py
Normal file
|
|
@ -0,0 +1,338 @@
|
|||
"""Hypothesis-scoped environmental falsification scenarios."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Iterable, Mapping
|
||||
|
||||
from sensorium.audio.checksum import sha256_json
|
||||
from sensorium.environment.falsification import (
|
||||
ExpectedObservationFrame,
|
||||
FalsificationResidual,
|
||||
FalsificationRun,
|
||||
ObservationUnitRef,
|
||||
compare_expected_to_observation,
|
||||
)
|
||||
from sensorium.environment.frame import ObservationFrame
|
||||
|
||||
SPECULATIVE_STATUS = "SPECULATIVE"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class HypothesisClaim:
|
||||
"""Speculative claim that gives expected evidence a reason to exist."""
|
||||
|
||||
claim_id: str
|
||||
claim_text: str
|
||||
domain: str
|
||||
basis_trace_hashes: tuple[str, ...]
|
||||
hypothesis_sha256: str
|
||||
epistemic_status: str = SPECULATIVE_STATUS
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"claim_id": self.claim_id,
|
||||
"claim_text": self.claim_text,
|
||||
"domain": self.domain,
|
||||
"basis_trace_hashes": list(self.basis_trace_hashes),
|
||||
"hypothesis_sha256": self.hypothesis_sha256,
|
||||
"epistemic_status": self.epistemic_status,
|
||||
}
|
||||
|
||||
|
||||
def build_hypothesis_claim(
|
||||
*,
|
||||
claim_id: str,
|
||||
claim_text: str,
|
||||
domain: str,
|
||||
basis_trace_hashes: Iterable[str] = (),
|
||||
) -> HypothesisClaim:
|
||||
if not claim_id.strip():
|
||||
raise ValueError("HypothesisClaim.claim_id is required")
|
||||
if not claim_text.strip():
|
||||
raise ValueError("HypothesisClaim.claim_text is required")
|
||||
if not domain.strip():
|
||||
raise ValueError("HypothesisClaim.domain is required")
|
||||
basis = tuple(sorted(set(str(h) for h in basis_trace_hashes if str(h).strip())))
|
||||
payload = {
|
||||
"kind": "HypothesisClaim",
|
||||
"claim_id": claim_id,
|
||||
"claim_text": claim_text,
|
||||
"domain": domain,
|
||||
"basis_trace_hashes": list(basis),
|
||||
"epistemic_status": SPECULATIVE_STATUS,
|
||||
}
|
||||
return HypothesisClaim(
|
||||
claim_id=claim_id,
|
||||
claim_text=claim_text,
|
||||
domain=domain,
|
||||
basis_trace_hashes=basis,
|
||||
hypothesis_sha256=sha256_json(payload),
|
||||
)
|
||||
|
||||
|
||||
def _canonical_expected_frames(
|
||||
frames: Iterable[ExpectedObservationFrame],
|
||||
) -> tuple[ExpectedObservationFrame, ...]:
|
||||
ordered = sorted(tuple(frames), key=lambda f: (f.expected_id, f.expected_sha256))
|
||||
deduped: list[ExpectedObservationFrame] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
seen_ids: dict[str, str] = {}
|
||||
for frame in ordered:
|
||||
key = (frame.expected_id, frame.expected_sha256)
|
||||
if key in seen:
|
||||
continue
|
||||
if frame.expected_id in seen_ids and seen_ids[frame.expected_id] != frame.expected_sha256:
|
||||
raise ValueError(f"conflicting expected frame id: {frame.expected_id}")
|
||||
seen.add(key)
|
||||
seen_ids[frame.expected_id] = frame.expected_sha256
|
||||
deduped.append(frame)
|
||||
return tuple(deduped)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ExperimentPlan:
|
||||
"""One speculative hypothesis plus canonical expected evidence frames."""
|
||||
|
||||
hypothesis: HypothesisClaim
|
||||
expected_frames: tuple[ExpectedObservationFrame, ...]
|
||||
plan_sha256: str
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"hypothesis": self.hypothesis.as_dict(),
|
||||
"expected_frames": [frame.as_dict() for frame in self.expected_frames],
|
||||
"plan_sha256": self.plan_sha256,
|
||||
}
|
||||
|
||||
|
||||
def build_experiment_plan(
|
||||
*,
|
||||
hypothesis: HypothesisClaim,
|
||||
expected_frames: Iterable[ExpectedObservationFrame],
|
||||
) -> ExperimentPlan:
|
||||
frames = _canonical_expected_frames(expected_frames)
|
||||
if not frames:
|
||||
raise ValueError("ExperimentPlan requires at least one expected frame")
|
||||
payload = {
|
||||
"kind": "ExperimentPlan",
|
||||
"hypothesis": hypothesis.as_dict(),
|
||||
"expected_frames": [frame.as_dict() for frame in frames],
|
||||
}
|
||||
return ExperimentPlan(
|
||||
hypothesis=hypothesis,
|
||||
expected_frames=frames,
|
||||
plan_sha256=sha256_json(payload),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ScenarioActualFrame:
|
||||
"""Actual observation frame bound to one expected frame id."""
|
||||
|
||||
expected_id: str
|
||||
frame: ObservationFrame
|
||||
actual_refs: tuple[ObservationUnitRef, ...]
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"expected_id": self.expected_id,
|
||||
"actual_frame_id": self.frame.frame_id,
|
||||
"actual_trace_hash": self.frame.trace_hash,
|
||||
"actual_ref_slots": [ref.slot_id for ref in self.actual_refs],
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FalsificationScenario:
|
||||
"""Immutable scenario binding one plan to actual observation frames."""
|
||||
|
||||
plan: ExperimentPlan
|
||||
actual_frames: tuple[ScenarioActualFrame, ...]
|
||||
scenario_sha256: str
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"plan": self.plan.as_dict(),
|
||||
"actual_frames": [frame.as_dict() for frame in self.actual_frames],
|
||||
"scenario_sha256": self.scenario_sha256,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ScenarioReport:
|
||||
"""Aggregate report over ordered falsification runs."""
|
||||
|
||||
hypothesis_sha256: str
|
||||
plan_sha256: str
|
||||
scenario_sha256: str
|
||||
verdict: str
|
||||
total_count: int
|
||||
supported_count: int
|
||||
falsified_count: int
|
||||
runs: tuple[FalsificationRun, ...]
|
||||
report_sha256: str
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"hypothesis_sha256": self.hypothesis_sha256,
|
||||
"plan_sha256": self.plan_sha256,
|
||||
"scenario_sha256": self.scenario_sha256,
|
||||
"verdict": self.verdict,
|
||||
"total_count": self.total_count,
|
||||
"supported_count": self.supported_count,
|
||||
"falsified_count": self.falsified_count,
|
||||
"runs": [run.as_dict() for run in self.runs],
|
||||
"report_sha256": self.report_sha256,
|
||||
}
|
||||
|
||||
|
||||
def _actual_frame_items(
|
||||
plan: ExperimentPlan,
|
||||
actual_frames_by_expected_id: Mapping[str, ObservationFrame],
|
||||
actual_refs_by_expected_id: Mapping[str, Iterable[ObservationUnitRef]] | None,
|
||||
) -> tuple[ScenarioActualFrame, ...]:
|
||||
refs_by_id = actual_refs_by_expected_id or {}
|
||||
expected_ids = {expected.expected_id for expected in plan.expected_frames}
|
||||
unknown_actual = sorted(set(actual_frames_by_expected_id) - expected_ids)
|
||||
if unknown_actual:
|
||||
raise ValueError(f"actual frame supplied for unknown expected_id: {unknown_actual[0]}")
|
||||
unknown_refs = sorted(set(refs_by_id) - expected_ids)
|
||||
if unknown_refs:
|
||||
raise ValueError(f"actual refs supplied for unknown expected_id: {unknown_refs[0]}")
|
||||
items: list[ScenarioActualFrame] = []
|
||||
for expected in plan.expected_frames:
|
||||
actual = actual_frames_by_expected_id.get(expected.expected_id)
|
||||
if actual is None:
|
||||
continue
|
||||
refs = tuple(refs_by_id.get(expected.expected_id, ()))
|
||||
items.append(ScenarioActualFrame(expected.expected_id, actual, refs))
|
||||
return tuple(items)
|
||||
|
||||
|
||||
def build_falsification_scenario(
|
||||
*,
|
||||
plan: ExperimentPlan,
|
||||
actual_frames_by_expected_id: Mapping[str, ObservationFrame],
|
||||
actual_refs_by_expected_id: Mapping[str, Iterable[ObservationUnitRef]] | None = None,
|
||||
) -> FalsificationScenario:
|
||||
actual_frames = _actual_frame_items(
|
||||
plan,
|
||||
actual_frames_by_expected_id,
|
||||
actual_refs_by_expected_id,
|
||||
)
|
||||
payload = {
|
||||
"kind": "FalsificationScenario",
|
||||
"plan_sha256": plan.plan_sha256,
|
||||
"actual_frames": [frame.as_dict() for frame in actual_frames],
|
||||
}
|
||||
return FalsificationScenario(
|
||||
plan=plan,
|
||||
actual_frames=actual_frames,
|
||||
scenario_sha256=sha256_json(payload),
|
||||
)
|
||||
|
||||
|
||||
def _missing_actual_run(expected: ExpectedObservationFrame) -> FalsificationRun:
|
||||
missing = tuple(ref.slot_id for ref in expected.unit_refs)
|
||||
residual_payload = {
|
||||
"kind": "FalsificationResidual",
|
||||
"matched": [],
|
||||
"missing": list(missing),
|
||||
"unexpected": [],
|
||||
"changed": [],
|
||||
}
|
||||
residual = FalsificationResidual(
|
||||
matched=(),
|
||||
missing=missing,
|
||||
unexpected=(),
|
||||
changed=(),
|
||||
residual_sha256=sha256_json(residual_payload),
|
||||
)
|
||||
actual_trace_hash = sha256_json({
|
||||
"kind": "MissingActualObservationFrame",
|
||||
"expected_id": expected.expected_id,
|
||||
})
|
||||
trace_payload = {
|
||||
"kind": "FalsificationRun",
|
||||
"expected_id": expected.expected_id,
|
||||
"actual_frame_id": "__missing_observation_frame__",
|
||||
"verdict": "FALSIFIED",
|
||||
"residual_sha256": residual.residual_sha256,
|
||||
"expected_sha256": expected.expected_sha256,
|
||||
"actual_trace_hash": actual_trace_hash,
|
||||
}
|
||||
return FalsificationRun(
|
||||
expected_id=expected.expected_id,
|
||||
actual_frame_id="__missing_observation_frame__",
|
||||
verdict="FALSIFIED",
|
||||
residual=residual,
|
||||
expected_sha256=expected.expected_sha256,
|
||||
actual_trace_hash=actual_trace_hash,
|
||||
trace_hash=sha256_json(trace_payload),
|
||||
)
|
||||
|
||||
|
||||
def run_falsification_scenario(
|
||||
plan: ExperimentPlan,
|
||||
actual_frames_by_expected_id: Mapping[str, ObservationFrame],
|
||||
actual_refs_by_expected_id: Mapping[str, Iterable[ObservationUnitRef]] | None = None,
|
||||
) -> ScenarioReport:
|
||||
"""Run a plan through the existing expected-vs-actual comparator."""
|
||||
scenario = build_falsification_scenario(
|
||||
plan=plan,
|
||||
actual_frames_by_expected_id=actual_frames_by_expected_id,
|
||||
actual_refs_by_expected_id=actual_refs_by_expected_id,
|
||||
)
|
||||
actual_by_expected = {item.expected_id: item for item in scenario.actual_frames}
|
||||
runs: list[FalsificationRun] = []
|
||||
for expected in scenario.plan.expected_frames:
|
||||
actual_item = actual_by_expected.get(expected.expected_id)
|
||||
if actual_item is None:
|
||||
runs.append(_missing_actual_run(expected))
|
||||
else:
|
||||
runs.append(
|
||||
compare_expected_to_observation(
|
||||
expected,
|
||||
actual_item.frame,
|
||||
actual_refs=actual_item.actual_refs or None,
|
||||
)
|
||||
)
|
||||
supported = sum(1 for run in runs if run.verdict == "SUPPORTED")
|
||||
falsified = sum(1 for run in runs if run.verdict == "FALSIFIED")
|
||||
verdict = "FALSIFIED" if falsified else "SUPPORTED"
|
||||
payload = {
|
||||
"kind": "ScenarioReport",
|
||||
"hypothesis_sha256": scenario.plan.hypothesis.hypothesis_sha256,
|
||||
"plan_sha256": scenario.plan.plan_sha256,
|
||||
"scenario_sha256": scenario.scenario_sha256,
|
||||
"verdict": verdict,
|
||||
"total_count": len(runs),
|
||||
"supported_count": supported,
|
||||
"falsified_count": falsified,
|
||||
"run_trace_hashes": [run.trace_hash for run in runs],
|
||||
}
|
||||
return ScenarioReport(
|
||||
hypothesis_sha256=scenario.plan.hypothesis.hypothesis_sha256,
|
||||
plan_sha256=scenario.plan.plan_sha256,
|
||||
scenario_sha256=scenario.scenario_sha256,
|
||||
verdict=verdict,
|
||||
total_count=len(runs),
|
||||
supported_count=supported,
|
||||
falsified_count=falsified,
|
||||
runs=tuple(runs),
|
||||
report_sha256=sha256_json(payload),
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ExperimentPlan",
|
||||
"FalsificationScenario",
|
||||
"HypothesisClaim",
|
||||
"ScenarioActualFrame",
|
||||
"ScenarioReport",
|
||||
"build_experiment_plan",
|
||||
"build_falsification_scenario",
|
||||
"build_hypothesis_claim",
|
||||
"run_falsification_scenario",
|
||||
]
|
||||
|
|
@ -9,9 +9,12 @@ import pytest
|
|||
|
||||
from sensorium.environment import (
|
||||
ObservationUnitRef,
|
||||
build_experiment_plan,
|
||||
build_expected_observation_frame,
|
||||
build_hypothesis_claim,
|
||||
build_observation_frame,
|
||||
compare_expected_to_observation,
|
||||
run_falsification_scenario,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -71,6 +74,46 @@ def test_expected_frame_hash_is_order_invariant_and_duplicate_safe():
|
|||
assert len(f1.unit_refs) == 2
|
||||
|
||||
|
||||
def test_hypothesis_claim_is_speculative_and_basis_order_stable():
|
||||
h1 = build_hypothesis_claim(
|
||||
claim_id="claim-a",
|
||||
claim_text="The anchored object repeats its contact evidence.",
|
||||
domain="passive_tabletop",
|
||||
basis_trace_hashes=("b", "a", "b"),
|
||||
)
|
||||
h2 = build_hypothesis_claim(
|
||||
claim_id="claim-a",
|
||||
claim_text="The anchored object repeats its contact evidence.",
|
||||
domain="passive_tabletop",
|
||||
basis_trace_hashes=("a", "b"),
|
||||
)
|
||||
assert h1.epistemic_status == "SPECULATIVE"
|
||||
assert h1.basis_trace_hashes == ("a", "b")
|
||||
assert h1.hypothesis_sha256 == h2.hypothesis_sha256
|
||||
|
||||
|
||||
def test_experiment_plan_hash_is_order_invariant_and_duplicate_safe():
|
||||
hypothesis = build_hypothesis_claim(
|
||||
claim_id="claim-plan",
|
||||
claim_text="Expected frames define the evidence envelope.",
|
||||
domain="fixture",
|
||||
)
|
||||
first = build_expected_observation_frame(
|
||||
monotonic_tick=10,
|
||||
source_clock="fixture",
|
||||
unit_refs=(_ref("slot:a", "a"),),
|
||||
)
|
||||
second = build_expected_observation_frame(
|
||||
monotonic_tick=11,
|
||||
source_clock="fixture",
|
||||
unit_refs=(_ref("slot:b", "b"),),
|
||||
)
|
||||
p1 = build_experiment_plan(hypothesis=hypothesis, expected_frames=(first, second, first))
|
||||
p2 = build_experiment_plan(hypothesis=hypothesis, expected_frames=(second, first))
|
||||
assert p1.plan_sha256 == p2.plan_sha256
|
||||
assert len(p1.expected_frames) == 2
|
||||
|
||||
|
||||
def test_raw_payloads_are_rejected():
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BadUnit(_Unit):
|
||||
|
|
@ -147,6 +190,68 @@ def test_missing_unexpected_and_changed_slots_are_falsified():
|
|||
assert run.residual.changed[0].slot_id == "slot:b"
|
||||
|
||||
|
||||
def test_scenario_supported_and_falsified_verdicts():
|
||||
hypothesis = build_hypothesis_claim(
|
||||
claim_id="claim-scenario",
|
||||
claim_text="A scenario binds expected evidence to actual frames.",
|
||||
domain="fixture",
|
||||
)
|
||||
refs = (_ref("slot:a", "a"),)
|
||||
expected = build_expected_observation_frame(
|
||||
monotonic_tick=6,
|
||||
source_clock="fixture",
|
||||
unit_refs=refs,
|
||||
)
|
||||
plan = build_experiment_plan(hypothesis=hypothesis, expected_frames=(expected,))
|
||||
supported_actual = build_observation_frame(
|
||||
monotonic_tick=6,
|
||||
source_clock="fixture",
|
||||
units=[ref.unit for ref in refs],
|
||||
)
|
||||
supported = run_falsification_scenario(
|
||||
plan,
|
||||
actual_frames_by_expected_id={expected.expected_id: supported_actual},
|
||||
actual_refs_by_expected_id={expected.expected_id: refs},
|
||||
)
|
||||
assert supported.verdict == "SUPPORTED"
|
||||
assert supported.total_count == 1
|
||||
assert supported.supported_count == 1
|
||||
|
||||
changed_refs = (_ref("slot:a", "changed"),)
|
||||
changed_actual = build_observation_frame(
|
||||
monotonic_tick=6,
|
||||
source_clock="fixture",
|
||||
units=[ref.unit for ref in changed_refs],
|
||||
)
|
||||
falsified = run_falsification_scenario(
|
||||
plan,
|
||||
actual_frames_by_expected_id={expected.expected_id: changed_actual},
|
||||
actual_refs_by_expected_id={expected.expected_id: changed_refs},
|
||||
)
|
||||
assert falsified.verdict == "FALSIFIED"
|
||||
assert falsified.falsified_count == 1
|
||||
assert falsified.runs[0].residual.changed[0].slot_id == "slot:a"
|
||||
|
||||
|
||||
def test_scenario_missing_actual_frame_is_falsified():
|
||||
hypothesis = build_hypothesis_claim(
|
||||
claim_id="claim-missing",
|
||||
claim_text="Missing actual evidence falsifies the scenario.",
|
||||
domain="fixture",
|
||||
)
|
||||
expected = build_expected_observation_frame(
|
||||
monotonic_tick=7,
|
||||
source_clock="fixture",
|
||||
unit_refs=(_ref("slot:a", "a"),),
|
||||
)
|
||||
plan = build_experiment_plan(hypothesis=hypothesis, expected_frames=(expected,))
|
||||
report = run_falsification_scenario(plan, actual_frames_by_expected_id={})
|
||||
assert report.verdict == "FALSIFIED"
|
||||
assert report.falsified_count == 1
|
||||
assert report.runs[0].actual_frame_id == "__missing_observation_frame__"
|
||||
assert report.runs[0].residual.missing == ("slot:a",)
|
||||
|
||||
|
||||
def test_run_trace_is_hash_only():
|
||||
refs = (_ref("slot:a", "a"),)
|
||||
expected = build_expected_observation_frame(
|
||||
|
|
|
|||
|
|
@ -11,7 +11,9 @@ def test_environment_falsification_report_passes_expected_fixtures():
|
|||
assert report["lane"] == "environment-falsification"
|
||||
assert report["failed"] == 0
|
||||
assert report["expected_report_hash_ok"] is True
|
||||
assert report["expected_frame_report_hash_ok"] is True
|
||||
assert {case["actual_verdict"] for case in report["cases"]} == {"SUPPORTED", "FALSIFIED"}
|
||||
assert {case["actual_verdict"] for case in report["scenario_cases"]} == {"SUPPORTED", "FALSIFIED"}
|
||||
|
||||
|
||||
def test_core_eval_environment_falsification_json(capsys):
|
||||
|
|
|
|||
Loading…
Reference in a new issue