diff --git a/evals/environment_falsification/expected_hashes.json b/evals/environment_falsification/expected_hashes.json index 15fe8bd2..22bdddff 100644 --- a/evals/environment_falsification/expected_hashes.json +++ b/evals/environment_falsification/expected_hashes.json @@ -1,3 +1,4 @@ { - "report_sha256": "c97b2dca7282d0231f1b448add87256cc2f59d39c5cec5ac1350231541e07d0b" + "frame_report_sha256": "c97b2dca7282d0231f1b448add87256cc2f59d39c5cec5ac1350231541e07d0b", + "report_sha256": "e0903e408f882bf7c93b7aadb6b3f8a064438ac09dc1cd4139f381bdc520b923" } diff --git a/evals/environment_falsification/report.py b/evals/environment_falsification/report.py index 7814d4ac..361aaf53 100644 --- a/evals/environment_falsification/report.py +++ b/evals/environment_falsification/report.py @@ -16,9 +16,12 @@ from sensorium.audio.compiler import AudioCompiler from sensorium.audio.checksum import sha256_json from sensorium.environment import ( ObservationUnitRef, + build_experiment_plan, build_expected_observation_frame, + build_hypothesis_claim, build_observation_frame, compare_expected_to_observation, + run_falsification_scenario, ) from sensorium.sensorimotor.compiler import SensorimotorCompiler from sensorium.vision import VisionCompiler, canonicalize_image @@ -107,15 +110,13 @@ def _report_hash(report_without_hash: dict[str, object]) -> str: return sha256_json(report_without_hash) -def build_environment_falsification_report() -> dict[str, object]: - fixtures = _load_json("fixtures.json")["fixtures"] - cases = [_case_report(idx, case) for idx, case in enumerate(fixtures)] +def _frame_report(cases: list[dict[str, object]]) -> dict[str, object]: passed = sum( 1 for case in cases if case["verdict_ok"] is True and case["trace_hygiene_ok"] is True ) - report = { + return { "lane": "environment-falsification", "version": "v1", "total": len(cases), @@ -123,10 +124,108 @@ def build_environment_falsification_report() -> dict[str, object]: "failed": len(cases) - passed, "cases": cases, } + + +def _scenario_case_report(index: int, scenario: dict[str, Any]) -> dict[str, object]: + hypothesis_spec = scenario["hypothesis"] + hypothesis = build_hypothesis_claim( + claim_id=str(hypothesis_spec["claim_id"]), + claim_text=str(hypothesis_spec["claim_text"]), + domain=str(hypothesis_spec["domain"]), + basis_trace_hashes=tuple(hypothesis_spec.get("basis_trace_hashes", ())), + ) + expected_frames = [] + actual_frames_by_expected_id = {} + actual_refs_by_expected_id = {} + for offset, frame_spec in enumerate(scenario["frames"]): + tick = int(frame_spec.get("tick", index * 100 + offset)) + expected_refs = _refs(frame_spec["expected"]) + expected = build_expected_observation_frame( + monotonic_tick=tick, + source_clock="environment-falsification-scenario-fixture", + unit_refs=expected_refs, + causal_parent_ids=tuple(frame_spec.get("causal_parent_ids", ())), + ) + expected_frames.append(expected) + if frame_spec["actual"] == "missing": + continue + actual_spec = frame_spec["expected"] if frame_spec["actual"] == "same" else frame_spec["actual"] + actual_refs = _refs(actual_spec) + actual = build_observation_frame( + monotonic_tick=tick, + source_clock="environment-falsification-scenario-fixture", + units=tuple(ref.unit for ref in actual_refs), + causal_parent_ids=(expected.expected_id,), + ) + actual_frames_by_expected_id[expected.expected_id] = actual + actual_refs_by_expected_id[expected.expected_id] = actual_refs + + plan = build_experiment_plan(hypothesis=hypothesis, expected_frames=expected_frames) + report = run_falsification_scenario( + plan, + actual_frames_by_expected_id=actual_frames_by_expected_id, + actual_refs_by_expected_id=actual_refs_by_expected_id, + ) + expected_verdict = str(scenario["expected_verdict"]) + row = { + "id": scenario["id"], + "expected_verdict": expected_verdict, + "actual_verdict": report.verdict, + "verdict_ok": report.verdict == expected_verdict, + "trace_hygiene_ok": _trace_safe(report.as_dict()), + "hypothesis_sha256": hypothesis.hypothesis_sha256, + "plan_sha256": plan.plan_sha256, + "scenario_sha256": report.scenario_sha256, + "scenario_report_sha256": report.report_sha256, + "total_count": report.total_count, + "supported_count": report.supported_count, + "falsified_count": report.falsified_count, + "run_trace_hashes": [run.trace_hash for run in report.runs], + } + return row + + +def build_environment_falsification_report() -> dict[str, object]: + fixtures = _load_json("fixtures.json")["fixtures"] + cases = [_case_report(idx, case) for idx, case in enumerate(fixtures)] + frame_report = _frame_report(cases) + frame_report_sha256 = _report_hash(frame_report) + scenario_fixtures = _load_json("scenario_fixtures.json")["scenarios"] + scenario_cases = [ + _scenario_case_report(idx, scenario) + for idx, scenario in enumerate(scenario_fixtures) + ] + scenario_passed = sum( + 1 + for case in scenario_cases + if case["verdict_ok"] is True and case["trace_hygiene_ok"] is True + ) + frame_passed = int(frame_report["passed"]) + frame_failed = int(frame_report["failed"]) + total = len(cases) + len(scenario_cases) + passed = frame_passed + scenario_passed + report = { + "lane": "environment-falsification", + "version": "v1", + "total": total, + "passed": passed, + "failed": frame_failed + (len(scenario_cases) - scenario_passed), + "cases": cases, + "frame_report_sha256": frame_report_sha256, + "scenario_cases": scenario_cases, + } report["report_sha256"] = _report_hash(report) expected_hashes = _load_json("expected_hashes.json") + expected_frame_report_sha256 = expected_hashes.get( + "frame_report_sha256", + expected_hashes["report_sha256"], + ) + report["expected_frame_report_sha256"] = expected_frame_report_sha256 + report["expected_frame_report_hash_ok"] = frame_report_sha256 == expected_frame_report_sha256 report["expected_report_sha256"] = expected_hashes["report_sha256"] report["expected_report_hash_ok"] = report["report_sha256"] == expected_hashes["report_sha256"] + if not report["expected_frame_report_hash_ok"]: + report["failed"] = int(report["failed"]) + 1 if not report["expected_report_hash_ok"]: report["failed"] = int(report["failed"]) + 1 return report diff --git a/evals/environment_falsification/scenario_fixtures.json b/evals/environment_falsification/scenario_fixtures.json new file mode 100644 index 00000000..6cbe9bca --- /dev/null +++ b/evals/environment_falsification/scenario_fixtures.json @@ -0,0 +1,88 @@ +{ + "scenarios": [ + { + "id": "supported_passive_contact_hypothesis", + "expected_verdict": "SUPPORTED", + "hypothesis": { + "claim_id": "passive-contact-stays-anchored", + "claim_text": "When the passive contact fixture remains anchored, audio, vision, and contact evidence repeat exactly.", + "domain": "passive_tabletop", + "basis_trace_hashes": ["basis-passive-contact", "basis-vision-anchor", "basis-passive-contact"] + }, + "frames": [ + { + "tick": 10, + "expected": { + "audio:left_tone": { + "modality": "audio", + "signal": {"id": "scenario_left_tone", "kind": "tone", "ms": 240, "hz": 180, "sweep": 0, "amp": 0.4} + }, + "vision:corner": { + "modality": "vision", + "signal": {"id": "scenario_corner", "kind": "corner", "size": 32} + } + }, + "actual": "same" + }, + { + "tick": 11, + "expected": { + "sensorimotor:contact": { + "modality": "sensorimotor", + "signal": { + "id": "scenario_contact", + "pose_q": [10, -4, 3], + "velocity_q": [1, 0, -1], + "force_torque_q": [2, 3, 5], + "contact_q": [1, 0, 1, 0], + "actuator_state_q": [7, 8] + } + } + }, + "actual": "same" + } + ] + }, + { + "id": "falsified_passive_contact_hypothesis", + "expected_verdict": "FALSIFIED", + "hypothesis": { + "claim_id": "passive-contact-breaks-on-release", + "claim_text": "A released passive contact fixture should continue to emit the original contact evidence.", + "domain": "passive_tabletop", + "basis_trace_hashes": ["basis-contact-release"] + }, + "frames": [ + { + "tick": 20, + "expected": { + "sensorimotor:contact": { + "modality": "sensorimotor", + "signal": { + "id": "scenario_contact_expected", + "pose_q": [10, -4, 3], + "velocity_q": [1, 0, -1], + "force_torque_q": [2, 3, 5], + "contact_q": [1, 0, 1, 0], + "actuator_state_q": [7, 8] + } + } + }, + "actual": { + "sensorimotor:contact": { + "modality": "sensorimotor", + "signal": { + "id": "scenario_contact_actual", + "pose_q": [10, -4, 3], + "velocity_q": [1, 0, -1], + "force_torque_q": [2, 3, 5], + "contact_q": [0, 0, 0, 0], + "actuator_state_q": [7, 8] + } + } + } + } + ] + } + ] +} diff --git a/sensorium/environment/__init__.py b/sensorium/environment/__init__.py index 5291209c..ba8acae1 100644 --- a/sensorium/environment/__init__.py +++ b/sensorium/environment/__init__.py @@ -11,16 +11,36 @@ from sensorium.environment.falsification import ( ) from sensorium.environment.frame import ObservationFrame, build_observation_frame from sensorium.environment.harness import build_fixture_observation_frame +from sensorium.environment.scenario import ( + ExperimentPlan, + FalsificationScenario, + HypothesisClaim, + ScenarioActualFrame, + ScenarioReport, + build_experiment_plan, + build_falsification_scenario, + build_hypothesis_claim, + run_falsification_scenario, +) __all__ = [ "ChangedSlot", + "ExperimentPlan", "ExpectedObservationFrame", + "FalsificationScenario", "FalsificationResidual", "FalsificationRun", + "HypothesisClaim", "ObservationFrame", "ObservationUnitRef", + "ScenarioActualFrame", + "ScenarioReport", + "build_experiment_plan", "build_expected_observation_frame", + "build_falsification_scenario", "build_fixture_observation_frame", + "build_hypothesis_claim", "build_observation_frame", "compare_expected_to_observation", + "run_falsification_scenario", ] diff --git a/sensorium/environment/scenario.py b/sensorium/environment/scenario.py new file mode 100644 index 00000000..418798ad --- /dev/null +++ b/sensorium/environment/scenario.py @@ -0,0 +1,338 @@ +"""Hypothesis-scoped environmental falsification scenarios.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Iterable, Mapping + +from sensorium.audio.checksum import sha256_json +from sensorium.environment.falsification import ( + ExpectedObservationFrame, + FalsificationResidual, + FalsificationRun, + ObservationUnitRef, + compare_expected_to_observation, +) +from sensorium.environment.frame import ObservationFrame + +SPECULATIVE_STATUS = "SPECULATIVE" + + +@dataclass(frozen=True, slots=True) +class HypothesisClaim: + """Speculative claim that gives expected evidence a reason to exist.""" + + claim_id: str + claim_text: str + domain: str + basis_trace_hashes: tuple[str, ...] + hypothesis_sha256: str + epistemic_status: str = SPECULATIVE_STATUS + + def as_dict(self) -> dict[str, object]: + return { + "claim_id": self.claim_id, + "claim_text": self.claim_text, + "domain": self.domain, + "basis_trace_hashes": list(self.basis_trace_hashes), + "hypothesis_sha256": self.hypothesis_sha256, + "epistemic_status": self.epistemic_status, + } + + +def build_hypothesis_claim( + *, + claim_id: str, + claim_text: str, + domain: str, + basis_trace_hashes: Iterable[str] = (), +) -> HypothesisClaim: + if not claim_id.strip(): + raise ValueError("HypothesisClaim.claim_id is required") + if not claim_text.strip(): + raise ValueError("HypothesisClaim.claim_text is required") + if not domain.strip(): + raise ValueError("HypothesisClaim.domain is required") + basis = tuple(sorted(set(str(h) for h in basis_trace_hashes if str(h).strip()))) + payload = { + "kind": "HypothesisClaim", + "claim_id": claim_id, + "claim_text": claim_text, + "domain": domain, + "basis_trace_hashes": list(basis), + "epistemic_status": SPECULATIVE_STATUS, + } + return HypothesisClaim( + claim_id=claim_id, + claim_text=claim_text, + domain=domain, + basis_trace_hashes=basis, + hypothesis_sha256=sha256_json(payload), + ) + + +def _canonical_expected_frames( + frames: Iterable[ExpectedObservationFrame], +) -> tuple[ExpectedObservationFrame, ...]: + ordered = sorted(tuple(frames), key=lambda f: (f.expected_id, f.expected_sha256)) + deduped: list[ExpectedObservationFrame] = [] + seen: set[tuple[str, str]] = set() + seen_ids: dict[str, str] = {} + for frame in ordered: + key = (frame.expected_id, frame.expected_sha256) + if key in seen: + continue + if frame.expected_id in seen_ids and seen_ids[frame.expected_id] != frame.expected_sha256: + raise ValueError(f"conflicting expected frame id: {frame.expected_id}") + seen.add(key) + seen_ids[frame.expected_id] = frame.expected_sha256 + deduped.append(frame) + return tuple(deduped) + + +@dataclass(frozen=True, slots=True) +class ExperimentPlan: + """One speculative hypothesis plus canonical expected evidence frames.""" + + hypothesis: HypothesisClaim + expected_frames: tuple[ExpectedObservationFrame, ...] + plan_sha256: str + + def as_dict(self) -> dict[str, object]: + return { + "hypothesis": self.hypothesis.as_dict(), + "expected_frames": [frame.as_dict() for frame in self.expected_frames], + "plan_sha256": self.plan_sha256, + } + + +def build_experiment_plan( + *, + hypothesis: HypothesisClaim, + expected_frames: Iterable[ExpectedObservationFrame], +) -> ExperimentPlan: + frames = _canonical_expected_frames(expected_frames) + if not frames: + raise ValueError("ExperimentPlan requires at least one expected frame") + payload = { + "kind": "ExperimentPlan", + "hypothesis": hypothesis.as_dict(), + "expected_frames": [frame.as_dict() for frame in frames], + } + return ExperimentPlan( + hypothesis=hypothesis, + expected_frames=frames, + plan_sha256=sha256_json(payload), + ) + + +@dataclass(frozen=True, slots=True) +class ScenarioActualFrame: + """Actual observation frame bound to one expected frame id.""" + + expected_id: str + frame: ObservationFrame + actual_refs: tuple[ObservationUnitRef, ...] + + def as_dict(self) -> dict[str, object]: + return { + "expected_id": self.expected_id, + "actual_frame_id": self.frame.frame_id, + "actual_trace_hash": self.frame.trace_hash, + "actual_ref_slots": [ref.slot_id for ref in self.actual_refs], + } + + +@dataclass(frozen=True, slots=True) +class FalsificationScenario: + """Immutable scenario binding one plan to actual observation frames.""" + + plan: ExperimentPlan + actual_frames: tuple[ScenarioActualFrame, ...] + scenario_sha256: str + + def as_dict(self) -> dict[str, object]: + return { + "plan": self.plan.as_dict(), + "actual_frames": [frame.as_dict() for frame in self.actual_frames], + "scenario_sha256": self.scenario_sha256, + } + + +@dataclass(frozen=True, slots=True) +class ScenarioReport: + """Aggregate report over ordered falsification runs.""" + + hypothesis_sha256: str + plan_sha256: str + scenario_sha256: str + verdict: str + total_count: int + supported_count: int + falsified_count: int + runs: tuple[FalsificationRun, ...] + report_sha256: str + + def as_dict(self) -> dict[str, object]: + return { + "hypothesis_sha256": self.hypothesis_sha256, + "plan_sha256": self.plan_sha256, + "scenario_sha256": self.scenario_sha256, + "verdict": self.verdict, + "total_count": self.total_count, + "supported_count": self.supported_count, + "falsified_count": self.falsified_count, + "runs": [run.as_dict() for run in self.runs], + "report_sha256": self.report_sha256, + } + + +def _actual_frame_items( + plan: ExperimentPlan, + actual_frames_by_expected_id: Mapping[str, ObservationFrame], + actual_refs_by_expected_id: Mapping[str, Iterable[ObservationUnitRef]] | None, +) -> tuple[ScenarioActualFrame, ...]: + refs_by_id = actual_refs_by_expected_id or {} + expected_ids = {expected.expected_id for expected in plan.expected_frames} + unknown_actual = sorted(set(actual_frames_by_expected_id) - expected_ids) + if unknown_actual: + raise ValueError(f"actual frame supplied for unknown expected_id: {unknown_actual[0]}") + unknown_refs = sorted(set(refs_by_id) - expected_ids) + if unknown_refs: + raise ValueError(f"actual refs supplied for unknown expected_id: {unknown_refs[0]}") + items: list[ScenarioActualFrame] = [] + for expected in plan.expected_frames: + actual = actual_frames_by_expected_id.get(expected.expected_id) + if actual is None: + continue + refs = tuple(refs_by_id.get(expected.expected_id, ())) + items.append(ScenarioActualFrame(expected.expected_id, actual, refs)) + return tuple(items) + + +def build_falsification_scenario( + *, + plan: ExperimentPlan, + actual_frames_by_expected_id: Mapping[str, ObservationFrame], + actual_refs_by_expected_id: Mapping[str, Iterable[ObservationUnitRef]] | None = None, +) -> FalsificationScenario: + actual_frames = _actual_frame_items( + plan, + actual_frames_by_expected_id, + actual_refs_by_expected_id, + ) + payload = { + "kind": "FalsificationScenario", + "plan_sha256": plan.plan_sha256, + "actual_frames": [frame.as_dict() for frame in actual_frames], + } + return FalsificationScenario( + plan=plan, + actual_frames=actual_frames, + scenario_sha256=sha256_json(payload), + ) + + +def _missing_actual_run(expected: ExpectedObservationFrame) -> FalsificationRun: + missing = tuple(ref.slot_id for ref in expected.unit_refs) + residual_payload = { + "kind": "FalsificationResidual", + "matched": [], + "missing": list(missing), + "unexpected": [], + "changed": [], + } + residual = FalsificationResidual( + matched=(), + missing=missing, + unexpected=(), + changed=(), + residual_sha256=sha256_json(residual_payload), + ) + actual_trace_hash = sha256_json({ + "kind": "MissingActualObservationFrame", + "expected_id": expected.expected_id, + }) + trace_payload = { + "kind": "FalsificationRun", + "expected_id": expected.expected_id, + "actual_frame_id": "__missing_observation_frame__", + "verdict": "FALSIFIED", + "residual_sha256": residual.residual_sha256, + "expected_sha256": expected.expected_sha256, + "actual_trace_hash": actual_trace_hash, + } + return FalsificationRun( + expected_id=expected.expected_id, + actual_frame_id="__missing_observation_frame__", + verdict="FALSIFIED", + residual=residual, + expected_sha256=expected.expected_sha256, + actual_trace_hash=actual_trace_hash, + trace_hash=sha256_json(trace_payload), + ) + + +def run_falsification_scenario( + plan: ExperimentPlan, + actual_frames_by_expected_id: Mapping[str, ObservationFrame], + actual_refs_by_expected_id: Mapping[str, Iterable[ObservationUnitRef]] | None = None, +) -> ScenarioReport: + """Run a plan through the existing expected-vs-actual comparator.""" + scenario = build_falsification_scenario( + plan=plan, + actual_frames_by_expected_id=actual_frames_by_expected_id, + actual_refs_by_expected_id=actual_refs_by_expected_id, + ) + actual_by_expected = {item.expected_id: item for item in scenario.actual_frames} + runs: list[FalsificationRun] = [] + for expected in scenario.plan.expected_frames: + actual_item = actual_by_expected.get(expected.expected_id) + if actual_item is None: + runs.append(_missing_actual_run(expected)) + else: + runs.append( + compare_expected_to_observation( + expected, + actual_item.frame, + actual_refs=actual_item.actual_refs or None, + ) + ) + supported = sum(1 for run in runs if run.verdict == "SUPPORTED") + falsified = sum(1 for run in runs if run.verdict == "FALSIFIED") + verdict = "FALSIFIED" if falsified else "SUPPORTED" + payload = { + "kind": "ScenarioReport", + "hypothesis_sha256": scenario.plan.hypothesis.hypothesis_sha256, + "plan_sha256": scenario.plan.plan_sha256, + "scenario_sha256": scenario.scenario_sha256, + "verdict": verdict, + "total_count": len(runs), + "supported_count": supported, + "falsified_count": falsified, + "run_trace_hashes": [run.trace_hash for run in runs], + } + return ScenarioReport( + hypothesis_sha256=scenario.plan.hypothesis.hypothesis_sha256, + plan_sha256=scenario.plan.plan_sha256, + scenario_sha256=scenario.scenario_sha256, + verdict=verdict, + total_count=len(runs), + supported_count=supported, + falsified_count=falsified, + runs=tuple(runs), + report_sha256=sha256_json(payload), + ) + + +__all__ = [ + "ExperimentPlan", + "FalsificationScenario", + "HypothesisClaim", + "ScenarioActualFrame", + "ScenarioReport", + "build_experiment_plan", + "build_falsification_scenario", + "build_hypothesis_claim", + "run_falsification_scenario", +] diff --git a/tests/test_environment_falsification.py b/tests/test_environment_falsification.py index 7d3ab3ef..0caa1bec 100644 --- a/tests/test_environment_falsification.py +++ b/tests/test_environment_falsification.py @@ -9,9 +9,12 @@ import pytest from sensorium.environment import ( ObservationUnitRef, + build_experiment_plan, build_expected_observation_frame, + build_hypothesis_claim, build_observation_frame, compare_expected_to_observation, + run_falsification_scenario, ) @@ -71,6 +74,46 @@ def test_expected_frame_hash_is_order_invariant_and_duplicate_safe(): assert len(f1.unit_refs) == 2 +def test_hypothesis_claim_is_speculative_and_basis_order_stable(): + h1 = build_hypothesis_claim( + claim_id="claim-a", + claim_text="The anchored object repeats its contact evidence.", + domain="passive_tabletop", + basis_trace_hashes=("b", "a", "b"), + ) + h2 = build_hypothesis_claim( + claim_id="claim-a", + claim_text="The anchored object repeats its contact evidence.", + domain="passive_tabletop", + basis_trace_hashes=("a", "b"), + ) + assert h1.epistemic_status == "SPECULATIVE" + assert h1.basis_trace_hashes == ("a", "b") + assert h1.hypothesis_sha256 == h2.hypothesis_sha256 + + +def test_experiment_plan_hash_is_order_invariant_and_duplicate_safe(): + hypothesis = build_hypothesis_claim( + claim_id="claim-plan", + claim_text="Expected frames define the evidence envelope.", + domain="fixture", + ) + first = build_expected_observation_frame( + monotonic_tick=10, + source_clock="fixture", + unit_refs=(_ref("slot:a", "a"),), + ) + second = build_expected_observation_frame( + monotonic_tick=11, + source_clock="fixture", + unit_refs=(_ref("slot:b", "b"),), + ) + p1 = build_experiment_plan(hypothesis=hypothesis, expected_frames=(first, second, first)) + p2 = build_experiment_plan(hypothesis=hypothesis, expected_frames=(second, first)) + assert p1.plan_sha256 == p2.plan_sha256 + assert len(p1.expected_frames) == 2 + + def test_raw_payloads_are_rejected(): @dataclass(frozen=True, slots=True) class BadUnit(_Unit): @@ -147,6 +190,68 @@ def test_missing_unexpected_and_changed_slots_are_falsified(): assert run.residual.changed[0].slot_id == "slot:b" +def test_scenario_supported_and_falsified_verdicts(): + hypothesis = build_hypothesis_claim( + claim_id="claim-scenario", + claim_text="A scenario binds expected evidence to actual frames.", + domain="fixture", + ) + refs = (_ref("slot:a", "a"),) + expected = build_expected_observation_frame( + monotonic_tick=6, + source_clock="fixture", + unit_refs=refs, + ) + plan = build_experiment_plan(hypothesis=hypothesis, expected_frames=(expected,)) + supported_actual = build_observation_frame( + monotonic_tick=6, + source_clock="fixture", + units=[ref.unit for ref in refs], + ) + supported = run_falsification_scenario( + plan, + actual_frames_by_expected_id={expected.expected_id: supported_actual}, + actual_refs_by_expected_id={expected.expected_id: refs}, + ) + assert supported.verdict == "SUPPORTED" + assert supported.total_count == 1 + assert supported.supported_count == 1 + + changed_refs = (_ref("slot:a", "changed"),) + changed_actual = build_observation_frame( + monotonic_tick=6, + source_clock="fixture", + units=[ref.unit for ref in changed_refs], + ) + falsified = run_falsification_scenario( + plan, + actual_frames_by_expected_id={expected.expected_id: changed_actual}, + actual_refs_by_expected_id={expected.expected_id: changed_refs}, + ) + assert falsified.verdict == "FALSIFIED" + assert falsified.falsified_count == 1 + assert falsified.runs[0].residual.changed[0].slot_id == "slot:a" + + +def test_scenario_missing_actual_frame_is_falsified(): + hypothesis = build_hypothesis_claim( + claim_id="claim-missing", + claim_text="Missing actual evidence falsifies the scenario.", + domain="fixture", + ) + expected = build_expected_observation_frame( + monotonic_tick=7, + source_clock="fixture", + unit_refs=(_ref("slot:a", "a"),), + ) + plan = build_experiment_plan(hypothesis=hypothesis, expected_frames=(expected,)) + report = run_falsification_scenario(plan, actual_frames_by_expected_id={}) + assert report.verdict == "FALSIFIED" + assert report.falsified_count == 1 + assert report.runs[0].actual_frame_id == "__missing_observation_frame__" + assert report.runs[0].residual.missing == ("slot:a",) + + def test_run_trace_is_hash_only(): refs = (_ref("slot:a", "a"),) expected = build_expected_observation_frame( diff --git a/tests/test_environment_falsification_eval_cli.py b/tests/test_environment_falsification_eval_cli.py index b968ad17..936933ac 100644 --- a/tests/test_environment_falsification_eval_cli.py +++ b/tests/test_environment_falsification_eval_cli.py @@ -11,7 +11,9 @@ def test_environment_falsification_report_passes_expected_fixtures(): assert report["lane"] == "environment-falsification" assert report["failed"] == 0 assert report["expected_report_hash_ok"] is True + assert report["expected_frame_report_hash_ok"] is True assert {case["actual_verdict"] for case in report["cases"]} == {"SUPPORTED", "FALSIFIED"} + assert {case["actual_verdict"] for case in report["scenario_cases"]} == {"SUPPORTED", "FALSIFIED"} def test_core_eval_environment_falsification_json(capsys):