diff --git a/benchmarks/run_benchmarks.py b/benchmarks/run_benchmarks.py index 29c6266a..785c6bcd 100644 --- a/benchmarks/run_benchmarks.py +++ b/benchmarks/run_benchmarks.py @@ -316,6 +316,37 @@ def bench_realizer_coverage() -> BenchResult: # Runner # --------------------------------------------------------------------------- +def bench_teaching_loop_determinism(runs: int = 10) -> BenchResult: + """Run propose → replay → accept N times; assert byte-identical artifacts. + + This is the determinism benchmark for the *learning loop* itself + (ADR-0055..0057): per-fact provenance, replay-equivalence gate, + operator-gated corpus write — all replayable bit-identically. + The active corpus on disk is byte-identical pre/post. + """ + from benchmarks.teaching_loop import run_teaching_loop_determinism + + report = run_teaching_loop_determinism(runs=runs) + passed = report.deterministic and report.active_corpus_byte_identical + metric = 1.0 if passed else 0.0 + detail = ( + f"{report.runs} runs; unique(proposal_id)={report.unique_proposal_ids}, " + f"unique(baseline)={report.unique_replay_baselines}, " + f"unique(candidate)={report.unique_replay_candidates}, " + f"unique(chain_id)={report.unique_chain_ids}; " + f"mean={report.elapsed_mean_s:.3f}s p50={report.elapsed_p50_s:.3f}s " + f"p95={report.elapsed_p95_s:.3f}s; active_corpus_byte_eq=" + f"{report.active_corpus_byte_identical}" + ) + return BenchResult( + name="teaching_loop_determinism", + passed=passed, + metric=metric, + unit="byte_identity_ratio", + detail=detail, + ) + + _SUITES: dict[str, list] = { "determinism": [bench_determinism], "latency": [bench_latency], @@ -323,6 +354,7 @@ _SUITES: dict[str, list] = { "versor": [bench_versor_closure_audit], "convergence": [bench_convergence_proof], "realizer": [bench_realizer_coverage], + "teaching-loop": [bench_teaching_loop_determinism], } _ALL = [ @@ -349,6 +381,8 @@ def run_benchmarks( for func in funcs: if func is bench_determinism: result = func(runs=runs) + elif func is bench_teaching_loop_determinism: + result = func(runs=runs) else: result = func() report.results.append(result) diff --git a/benchmarks/teaching_loop.py b/benchmarks/teaching_loop.py new file mode 100644 index 00000000..c0dd9418 --- /dev/null +++ b/benchmarks/teaching_loop.py @@ -0,0 +1,222 @@ +"""Teaching-loop determinism benchmark. + +Run the full reviewed-corpus extension pipeline (propose → replay- +equivalence gate → operator accept) N times against the same input. +Assert byte-identical artifacts every run: + + - proposal_id (SHA-256 of canonical-JSON payload) + - replay baseline (cognition lane metrics on active corpus) + - replay candidate (cognition lane metrics on transient corpus) + - regressed_metrics (sorted tuple) + - corpus_append_chain_id + +Also report latency: + - per-iteration wall-time (mean / p50 / p95) + - total wall-time + +Trust boundary: the benchmark writes ONLY to tempdir-scoped paths. +The active teaching corpus on disk is byte-identical pre/post. +Asserted in the report and in the test. +""" + +from __future__ import annotations + +import shutil +import statistics +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from chat import teaching_grounding as _tg +from teaching.discovery import DiscoveryCandidate, EvidencePointer +from teaching.proposals import ( + ProposalLog, + accept_proposal, + propose_from_candidate, +) + + +# Canonical demo candidate — identical to the learning-loop demo's +# operator-augmented payload. Same input → same artifacts on every +# iteration; that's the entire benchmark thesis. +def _canonical_candidate() -> DiscoveryCandidate: + return DiscoveryCandidate( + candidate_id="bench_canonical_001", + proposed_chain={ + "subject": "thought", "intent": "cause", + "connective": "reveals", "object": "meaning", + }, + trigger="would_have_grounded", + source_turn_trace="", + pack_consistent=True, + boundary_clean=True, + polarity="affirms", + claim_domain="factual", + evidence=( + EvidencePointer( + source="corpus", + ref="cause_creation_reveals_meaning", + polarity="affirms", + epistemic_status="coherent", + ), + ), + ) + + +@dataclass(frozen=True, slots=True) +class _IterationArtifact: + proposal_id: str + replay_baseline: tuple[tuple[str, float], ...] + replay_candidate: tuple[tuple[str, float], ...] + regressed_metrics: tuple[str, ...] + chain_id_written: str + elapsed_s: float + + +@dataclass(frozen=True, slots=True) +class TeachingLoopBenchReport: + runs: int + unique_proposal_ids: int + unique_replay_baselines: int + unique_replay_candidates: int + unique_regressed_metrics: int + unique_chain_ids: int + deterministic: bool + active_corpus_byte_identical: bool + elapsed_mean_s: float + elapsed_p50_s: float + elapsed_p95_s: float + elapsed_total_s: float + sample_proposal_id: str + sample_chain_id: str + + def as_dict(self) -> dict[str, Any]: + return { + "runs": self.runs, + "unique_proposal_ids": self.unique_proposal_ids, + "unique_replay_baselines": self.unique_replay_baselines, + "unique_replay_candidates": self.unique_replay_candidates, + "unique_regressed_metrics": self.unique_regressed_metrics, + "unique_chain_ids": self.unique_chain_ids, + "deterministic": self.deterministic, + "active_corpus_byte_identical": self.active_corpus_byte_identical, + "elapsed_mean_s": round(self.elapsed_mean_s, 4), + "elapsed_p50_s": round(self.elapsed_p50_s, 4), + "elapsed_p95_s": round(self.elapsed_p95_s, 4), + "elapsed_total_s": round(self.elapsed_total_s, 4), + "sample_proposal_id": self.sample_proposal_id, + "sample_chain_id": self.sample_chain_id, + } + + +def _freeze_metrics(d: dict[str, float]) -> tuple[tuple[str, float], ...]: + """Convert a metrics dict to a sorted tuple-of-pairs (hashable, ordered).""" + return tuple(sorted((k, round(float(v), 6)) for k, v in d.items())) + + +def _percentile(values: list[float], pct: float) -> float: + """Inclusive percentile via linear interpolation. Pure stdlib so + the bench has no numpy dependency on this path.""" + if not values: + return 0.0 + s = sorted(values) + if len(s) == 1: + return s[0] + k = (len(s) - 1) * pct / 100.0 + lo, hi = int(k), min(int(k) + 1, len(s) - 1) + if lo == hi: + return s[lo] + return s[lo] + (s[hi] - s[lo]) * (k - lo) + + +def run_teaching_loop_determinism(runs: int = 10) -> TeachingLoopBenchReport: + """Execute the full propose → replay → accept loop ``runs`` times + against the same candidate, then assert byte-identical artifacts. + + Trust boundary: the active corpus is read once at the start and + once at the end; any byte difference is a defect. All writes + are confined to tempdirs created inside this function. + """ + active_path = _tg._CORPUS_PATH + active_bytes_before = active_path.read_bytes() if active_path.exists() else b"" + + artifacts: list[_IterationArtifact] = [] + total_t0 = time.perf_counter() + + for _ in range(runs): + with tempfile.TemporaryDirectory() as tmpdir: + log_path = Path(tmpdir) / "proposals.jsonl" + transient = Path(tmpdir) / "cognition_chains_v1.jsonl" + if active_path.exists(): + shutil.copyfile(active_path, transient) + else: + transient.write_text("", encoding="utf-8") + + log = ProposalLog(log_path) + candidate = _canonical_candidate() + + t0 = time.perf_counter() + proposal = propose_from_candidate(candidate, log=log) + rec = log.find(proposal.proposal_id) or {} + ev = rec.get("replay_evidence") or {} + + chain_id = accept_proposal( + proposal.proposal_id, log=log, + corpus_path=transient, + review_date="2026-05-18", + operator_note="bench", + ) + elapsed = time.perf_counter() - t0 + + artifacts.append(_IterationArtifact( + proposal_id=proposal.proposal_id, + replay_baseline=_freeze_metrics(ev.get("baseline", {})), + replay_candidate=_freeze_metrics(ev.get("candidate", {})), + regressed_metrics=tuple(ev.get("regressed_metrics") or ()), + chain_id_written=chain_id, + elapsed_s=elapsed, + )) + + elapsed_total = time.perf_counter() - total_t0 + elapsed_values = [a.elapsed_s for a in artifacts] + + active_bytes_after = active_path.read_bytes() if active_path.exists() else b"" + + unique_pids = len({a.proposal_id for a in artifacts}) + unique_baselines = len({a.replay_baseline for a in artifacts}) + unique_candidates = len({a.replay_candidate for a in artifacts}) + unique_regressed = len({a.regressed_metrics for a in artifacts}) + unique_chain_ids = len({a.chain_id_written for a in artifacts}) + + deterministic = ( + unique_pids == 1 + and unique_baselines == 1 + and unique_candidates == 1 + and unique_regressed == 1 + and unique_chain_ids == 1 + ) + + return TeachingLoopBenchReport( + runs=runs, + unique_proposal_ids=unique_pids, + unique_replay_baselines=unique_baselines, + unique_replay_candidates=unique_candidates, + unique_regressed_metrics=unique_regressed, + unique_chain_ids=unique_chain_ids, + deterministic=deterministic, + active_corpus_byte_identical=(active_bytes_before == active_bytes_after), + elapsed_mean_s=statistics.mean(elapsed_values) if elapsed_values else 0.0, + elapsed_p50_s=_percentile(elapsed_values, 50.0), + elapsed_p95_s=_percentile(elapsed_values, 95.0), + elapsed_total_s=elapsed_total, + sample_proposal_id=artifacts[0].proposal_id if artifacts else "", + sample_chain_id=artifacts[0].chain_id_written if artifacts else "", + ) + + +__all__ = [ + "TeachingLoopBenchReport", + "run_teaching_loop_determinism", +] diff --git a/core/cli.py b/core/cli.py index d79e2574..ed9255b5 100644 --- a/core/cli.py +++ b/core/cli.py @@ -1777,7 +1777,7 @@ def build_parser() -> argparse.ArgumentParser: help="run benchmark harness (determinism, latency, speedup, versor audit)", description="run benchmark harness", ) - bench.add_argument("--suite", choices=["determinism", "latency", "speedup", "versor", "convergence", "realizer", "cost"], + bench.add_argument("--suite", choices=["determinism", "latency", "speedup", "versor", "convergence", "realizer", "cost", "teaching-loop"], help="run a specific benchmark suite") bench.add_argument("--runs", type=int, default=20, metavar="N", help="run count for determinism benchmark (also turns count for cost suite)") bench.add_argument("--json", action="store_true", help="emit machine-readable JSON") diff --git a/tests/test_teaching_loop_bench.py b/tests/test_teaching_loop_bench.py new file mode 100644 index 00000000..65b9349b --- /dev/null +++ b/tests/test_teaching_loop_bench.py @@ -0,0 +1,57 @@ +"""Teaching-loop determinism benchmark — falsifiable claim test. + +The bench itself runs at any N ≥ 1; the test pins the headline claim +at a low N for fast CI. Headline claim: + + "N identical inputs produce N byte-identical proposal artifacts, + the active corpus is byte-identical pre/post, and the wall-time + distribution is well-formed." + +If determinism breaks anywhere in the pipeline (proposal_id hashing, +replay-equivalence gate, accept-side corpus_append, ProposalLog +replay), at least one of the ``unique_*`` counts in the bench report +will exceed 1 and this test fails. +""" + +from __future__ import annotations + +from benchmarks.teaching_loop import run_teaching_loop_determinism + + +def test_teaching_loop_is_deterministic_across_three_runs() -> None: + report = run_teaching_loop_determinism(runs=3) + assert report.deterministic is True + assert report.active_corpus_byte_identical is True + assert report.unique_proposal_ids == 1 + assert report.unique_replay_baselines == 1 + assert report.unique_replay_candidates == 1 + assert report.unique_regressed_metrics == 1 + assert report.unique_chain_ids == 1 + + +def test_proposal_id_is_a_sha256_prefix() -> None: + report = run_teaching_loop_determinism(runs=2) + pid = report.sample_proposal_id + assert len(pid) == 32 + assert all(c in "0123456789abcdef" for c in pid) + + +def test_chain_id_matches_canonical_layout() -> None: + report = run_teaching_loop_determinism(runs=2) + assert report.sample_chain_id == "cause_thought_reveals_meaning" + + +def test_latency_stats_are_well_formed() -> None: + report = run_teaching_loop_determinism(runs=3) + assert report.elapsed_mean_s > 0.0 + assert report.elapsed_p50_s > 0.0 + assert report.elapsed_p95_s >= report.elapsed_p50_s + assert report.elapsed_total_s >= report.elapsed_mean_s * report.runs * 0.9 + + +def test_report_serialises_to_json() -> None: + import json + report = run_teaching_loop_determinism(runs=2) + blob = json.dumps(report.as_dict(), sort_keys=True) + assert "deterministic" in blob + assert "active_corpus_byte_identical" in blob