6f4b2b7b2c
`core demo anti-regression` (+ `--json`) is a self-contained walkthrough of the three independent gates that every reviewed-corpus extension must pass. Designed for showcasing CORE's epistemic discipline to reviewers / industry observers — no LLM provider has a published equivalent. Scenes: - S1. Eligibility predicate refuses an undetermined-polarity candidate before any replay is invoked. ProposalError raised; no log row. - S2. Replay-equivalence gate auto-rejects a regressing candidate with the named regressed metrics in the operator note. Uses the documented `run_replay=` kwarg of `propose_from_candidate` to inject a controlled regression of the same `ReplayEvidence` shape the real gate produces. - S3. Real `teaching.replay.run_replay_equivalence` runs the cognition public lane. A replay-equivalent candidate reaches 'pending' — operator `--accept` is still required to write. Each scene asserts the active corpus is byte-identical pre/post. - evals/anti_regression/run_demo.py — `run_demo(emit_json=False)` returns a structured `DemoReport`; verbose human output by default, JSON on flag. - core/cli.py — `core demo anti-regression` target wired alongside audit-tour / pack-measurements / long-context-comparison. - tests/test_anti_regression_demo.py — 5 tests pin each scene's load-bearing claim + the corpus-byte-identical invariant. Lane state: anti-regression-demo 5 new — green. Demo runs in ~10s end-to-end.
63 lines
2.5 KiB
Python
63 lines
2.5 KiB
Python
"""Anti-regression demo — pins each scene's load-bearing claim.
|
|
|
|
These are the falsifiable assertions the demo would make to a viewer.
|
|
If any assertion fails, the demo's headline claim no longer holds.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from evals.anti_regression.run_demo import run_demo
|
|
|
|
|
|
def test_demo_runs_to_completion_with_all_three_gates_holding() -> None:
|
|
report = run_demo(emit_json=True)
|
|
assert report["all_gates_held"] is True
|
|
assert report["active_corpus_byte_identical"] is True
|
|
assert len(report["scenes"]) == 3
|
|
|
|
|
|
def test_s1_eligibility_gate_rejects_pre_replay() -> None:
|
|
report = run_demo(emit_json=True)
|
|
s1 = report["scenes"][0]
|
|
assert s1["scene"] == "S1_eligibility_gate"
|
|
assert s1["outcome"] == "rejected_pre_replay"
|
|
assert s1["proposal_id"] is None
|
|
assert s1["replay_evidence"] is None
|
|
assert "undetermined" in (s1["error"] or "")
|
|
assert s1["corpus_byte_identical"] is True
|
|
|
|
|
|
def test_s2_replay_gate_auto_rejects_with_named_metrics() -> None:
|
|
report = run_demo(emit_json=True)
|
|
s2 = report["scenes"][1]
|
|
assert s2["scene"] == "S2_replay_auto_reject"
|
|
assert s2["outcome"] == "auto_rejected_on_regression"
|
|
assert s2["review_state"] == "rejected"
|
|
assert s2["replay_evidence"]["replay_equivalent"] is False
|
|
assert "surface_groundedness" in s2["replay_evidence"]["regressed_metrics"]
|
|
assert "term_capture_rate" in s2["replay_evidence"]["regressed_metrics"]
|
|
# The operator note must name the regressed metrics.
|
|
assert "surface_groundedness" in s2["operator_note"]
|
|
assert "term_capture_rate" in s2["operator_note"]
|
|
assert s2["corpus_byte_identical"] is True
|
|
|
|
|
|
def test_s3_real_gate_passes_to_pending_not_accepted() -> None:
|
|
report = run_demo(emit_json=True)
|
|
s3 = report["scenes"][2]
|
|
assert s3["scene"] == "S3_real_gate_pass_through"
|
|
assert s3["outcome"] == "pending_awaiting_operator"
|
|
assert s3["review_state"] == "pending"
|
|
assert s3["replay_evidence"]["replay_equivalent"] is True
|
|
assert s3["replay_evidence"]["regressed_metrics"] == []
|
|
assert s3["corpus_byte_identical"] is True
|
|
|
|
|
|
def test_active_corpus_never_touched_across_full_demo() -> None:
|
|
"""Defence-in-depth: even though each scene asserts byte-identity,
|
|
re-confirm at the report level — the demo never writes to the
|
|
production teaching corpus regardless of scene outcomes."""
|
|
report = run_demo(emit_json=True)
|
|
assert report["active_corpus_byte_identical"] is True
|
|
for scene in report["scenes"]:
|
|
assert scene["corpus_byte_identical"] is True
|