from __future__ import annotations import json from pathlib import Path import pytest from core.contemplation.__main__ import main from core.contemplation.miners.frontier_compare import mine_frontier_compare_report from core.contemplation.runner import ( contemplate_frontier_reports, run_contemplation, write_contemplation_run, ) from core.contemplation.schema import ( ContemplationEvidenceRef, ContemplationFinding, FindingKind, ) from core.contemplation.snapshot import ContemplationSubstrate from teaching.epistemic import EpistemicStatus from teaching.source import ProposalSource def _sample_frontier_report(path: Path) -> None: payload = { "benchmark_family": "frontier_compare_wave1", "model": "core", "mode": "native", "summary": {"suite_count": 1, "case_count": 2, "primary_score": 0.5, "passed": False}, "suites": [ { "suite": "truth_lock", "case_count": 2, "primary_score": 0.5, "passed": False, "cases": [ { "suite": "truth_lock", "case_id": "known_truth", "prompt": "What is truth?", "passed": True, "score": 1.0, "elapsed_ms": 1.0, "details": {}, "failures": [], }, { "suite": "truth_lock", "case_id": "unknown_relation", "prompt": "Why does xylomorphic matter?", "passed": False, "score": 0.0, "elapsed_ms": 1.0, "details": {}, "failures": ["unexpected_grounding_source:vault"], }, ], } ], } path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8") def test_contemplation_finding_rejects_non_speculative_status() -> None: evidence = ContemplationEvidenceRef( source_type="test", source_id="unit", pointer="case=1", ) with pytest.raises(ValueError, match="SPECULATIVE"): ContemplationFinding( kind=FindingKind.BENCHMARK_CASE, subject="suite/case", predicate="failed_case", object="prompt", evidence_refs=(evidence,), proposed_action="review failure", substrate_hash="substrate", epistemic_status=EpistemicStatus.COHERENT, ) def test_contemplation_finding_requires_evidence() -> None: with pytest.raises(ValueError, match="evidence"): ContemplationFinding( kind=FindingKind.BENCHMARK_CASE, subject="suite/case", predicate="failed_case", object="prompt", evidence_refs=(), proposed_action="review failure", substrate_hash="substrate", ) def test_frontier_report_miner_emits_failed_cases_only(tmp_path: Path) -> None: report = tmp_path / "frontier_wave1.json" _sample_frontier_report(report) substrate = ContemplationSubstrate.from_report_paths((report,)) findings = mine_frontier_compare_report(report, substrate_hash=substrate.substrate_hash) assert len(findings) == 1 finding = findings[0] assert finding.kind is FindingKind.BENCHMARK_CASE assert finding.subject == "truth_lock/unknown_relation" assert finding.predicate == "failed_case" assert finding.object == "Why does xylomorphic matter?" assert finding.epistemic_status is EpistemicStatus.SPECULATIVE assert finding.confidence is EpistemicStatus.SPECULATIVE assert finding.evidence == finding.evidence_refs assert finding.provenance["source"] == "contemplation" assert finding.evidence_refs[0].summary == "unexpected_grounding_source:vault" def test_contemplation_runner_is_replay_deterministic(tmp_path: Path) -> None: report = tmp_path / "frontier_wave1.json" _sample_frontier_report(report) first = contemplate_frontier_reports((report,), pack_ids=("en_core_cognition_v1",)) second = contemplate_frontier_reports((report,), pack_ids=("en_core_cognition_v1",)) assert first.run_id == second.run_id assert first.as_dict() == second.as_dict() assert all( finding.epistemic_status is EpistemicStatus.SPECULATIVE for finding in first.findings ) def test_contemplation_runner_does_not_mutate_pack_tree(tmp_path: Path) -> None: report = tmp_path / "frontier_wave1.json" _sample_frontier_report(report) pack_root = Path("language_packs") before = sorted( (p.relative_to(pack_root).as_posix(), p.stat().st_mtime_ns, p.stat().st_size) for p in pack_root.rglob("*") if p.is_file() ) contemplate_frontier_reports((report,)) after = sorted( (p.relative_to(pack_root).as_posix(), p.stat().st_mtime_ns, p.stat().st_size) for p in pack_root.rglob("*") if p.is_file() ) assert before == after def test_contemplation_runner_does_not_write_teaching_store_without_review( tmp_path: Path, ) -> None: report = tmp_path / "frontier_wave1.json" _sample_frontier_report(report) proposal_store = Path("teaching/proposals/proposals.jsonl") before = proposal_store.read_bytes() if proposal_store.exists() else b"" run = contemplate_frontier_reports((report,)) after = proposal_store.read_bytes() if proposal_store.exists() else b"" assert run.findings assert before == after def test_run_contemplation_default_entrypoint_is_deterministic() -> None: first = run_contemplation() second = run_contemplation() assert first.as_dict() == second.as_dict() assert all( finding.confidence is EpistemicStatus.SPECULATIVE for finding in first.findings ) def test_contemplation_proposal_source_parses() -> None: source = ProposalSource( kind="contemplation", source_id="frontier_compare", emitted_at_revision="abc123", ) assert source.serialize() == "contemplation:frontier_compare" assert ProposalSource.from_dict(source.as_dict()) == source def test_contemplation_write_and_cli(tmp_path: Path, capsys) -> None: source_report = tmp_path / "frontier_wave1.json" output_report = tmp_path / "contemplation.json" _sample_frontier_report(source_report) run = contemplate_frontier_reports((source_report,)) write_contemplation_run(run, output_report) written = json.loads(output_report.read_text(encoding="utf-8")) assert written["run_id"] == run.run_id assert written["finding_count"] == 1 code = main([str(source_report), "--report", str(output_report)]) out = capsys.readouterr().out printed = json.loads(out) assert code == 0 assert printed["finding_count"] == 1 assert json.loads(output_report.read_text(encoding="utf-8")) == printed