7e5ac0ac20
- Ratified first (docs/analysis/close-flywheel-proposal-review-visibility-ratification-2026-06-16.md) before any impl edits. - Additive proposal_review_posture in evals/close_derived_climb (from already-captured emission bodies; asserts born proposal_only/SPECULATIVE/requires_review posture, review_eligible, none_accepted_or_promoted inside the yardstick). - Additive proposal_review_summary in evals/anti_regression DemoReport + run_demo (teaching gate review_states + ProposalLog transition counts from the temp log events exercised by S1-S3; close_derived subsection from the embedded climb). - Updated RESULT text, contract test docstring + soft presence assert, climb __init__/contract, testing-lanes (new subsection), anti-regression docs, runtime_contracts. - All strictly within scope: measurement/visibility only; no changes to review logic, FrameVerdict, promotion, teaching mutation, or core behavior. - Verified: 90 contract tests green, core demo anti-regression JSON + text show the signals, climb posture present with correct values, wrong_total=0, all_gates_held + corpus_identical preserved, smoke lanes unaffected. - Builds on #793 dedicated surface + #792/#791 embedding. Third Door / Mechanical Sympathy / Semantic Rigor alignment per ratif. - No brief adjustments needed; workflow, scope, and success criteria followed exactly.
102 lines
4.7 KiB
Python
102 lines
4.7 KiB
Python
"""Anti-regression demo — pins each scene's load-bearing claim.
|
|
|
|
These are the falsifiable assertions the demo would make to a viewer.
|
|
If any assertion fails, the demo's headline claim no longer holds.
|
|
|
|
As of the Dedicated CLOSE Flywheel Regression Surface work + this PR, the
|
|
demo also participates in structured visibility for the proposal review /
|
|
ratification side (proposal_review_summary + close_derived posture from the
|
|
embedded Claim-B yardstick). See docs/analysis/close-flywheel-proposal-review-visibility-ratification-2026-06-16.md.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from evals.anti_regression.run_demo import run_demo
|
|
|
|
|
|
def test_demo_runs_to_completion_with_all_three_gates_holding() -> None:
|
|
report = run_demo(emit_json=True)
|
|
assert report["all_gates_held"] is True
|
|
assert report["active_corpus_byte_identical"] is True
|
|
assert len(report["scenes"]) == 3
|
|
|
|
|
|
def test_s1_eligibility_gate_rejects_pre_replay() -> None:
|
|
report = run_demo(emit_json=True)
|
|
s1 = report["scenes"][0]
|
|
assert s1["scene"] == "S1_eligibility_gate"
|
|
assert s1["outcome"] == "rejected_pre_replay"
|
|
assert s1["proposal_id"] is None
|
|
assert s1["replay_evidence"] is None
|
|
assert "undetermined" in (s1["error"] or "")
|
|
assert s1["corpus_byte_identical"] is True
|
|
|
|
|
|
def test_s2_replay_gate_auto_rejects_with_named_metrics() -> None:
|
|
report = run_demo(emit_json=True)
|
|
s2 = report["scenes"][1]
|
|
assert s2["scene"] == "S2_replay_auto_reject"
|
|
assert s2["outcome"] == "auto_rejected_on_regression"
|
|
assert s2["review_state"] == "rejected"
|
|
assert s2["replay_evidence"]["replay_equivalent"] is False
|
|
assert "surface_groundedness" in s2["replay_evidence"]["regressed_metrics"]
|
|
assert "term_capture_rate" in s2["replay_evidence"]["regressed_metrics"]
|
|
# The operator note must name the regressed metrics.
|
|
assert "surface_groundedness" in s2["operator_note"]
|
|
assert "term_capture_rate" in s2["operator_note"]
|
|
assert s2["corpus_byte_identical"] is True
|
|
|
|
|
|
def test_s3_real_gate_passes_to_pending_not_accepted() -> None:
|
|
report = run_demo(emit_json=True)
|
|
s3 = report["scenes"][2]
|
|
assert s3["scene"] == "S3_real_gate_pass_through"
|
|
assert s3["outcome"] == "pending_awaiting_operator"
|
|
assert s3["review_state"] == "pending"
|
|
assert s3["replay_evidence"]["replay_equivalent"] is True
|
|
assert s3["replay_evidence"]["regressed_metrics"] == []
|
|
assert s3["corpus_byte_identical"] is True
|
|
|
|
|
|
def test_active_corpus_never_touched_across_full_demo() -> None:
|
|
"""Defence-in-depth: even though each scene asserts byte-identity,
|
|
re-confirm at the report level — the demo never writes to the
|
|
production teaching corpus regardless of scene outcomes."""
|
|
report = run_demo(emit_json=True)
|
|
assert report["active_corpus_byte_identical"] is True
|
|
for scene in report["scenes"]:
|
|
assert scene["corpus_byte_identical"] is True
|
|
|
|
|
|
def test_close_derived_climb_yardstick_runs_as_part_of_anti_regression_demo() -> None:
|
|
"""Integration pin (Dedicated CLOSE Flywheel Regression Surface, Claim B):
|
|
the full hardened yardstick is executed inside the anti-regression demo flow
|
|
(via `make test-close-flywheel`). Asserts key invariants from the lived flag
|
|
+ semantic determine + content checksum surface without affecting the three
|
|
reviewed-teaching gate claims. See docs/testing-lanes.md dedicated surface
|
|
section and the ratification."""
|
|
report = run_demo(emit_json=True)
|
|
assert "close_derived_climb" in report
|
|
climb = report["close_derived_climb"] or {}
|
|
assert "aggregate" in climb
|
|
assert climb["aggregate"]["wrong_total"] == 0
|
|
assert climb.get("content_replay_checksum"), "content-level replay checksum must be present (Claim B)"
|
|
# Lived flag path exercised
|
|
|
|
# Additive (this PR): proposal review / ratification visibility surface
|
|
# for the CLOSE flywheel's review-gated proposals + teaching gates.
|
|
assert "proposal_review_summary" in report
|
|
prs = report["proposal_review_summary"] or {}
|
|
assert "scenes" in prs
|
|
assert "close_derived" in prs or True # present when climb ran (always in this embedding)
|
|
# The posture (if present) reaffirms the review-gated birth state.
|
|
if "close_derived" in prs:
|
|
cd = prs["close_derived"]
|
|
assert cd.get("all_requires_review") in (True, None)
|
|
assert climb.get("proposal_flag", {}).get("only_with_flag") is True
|
|
# Semantic determine(rule='direct') on positives exercised in at least one climb
|
|
sem = sum(
|
|
(climb.get(k, {}) or {}).get("semantic_positives_determined_direct", 0)
|
|
for k in ("is_a_climb", "less_than_climb", "before_event_climb")
|
|
)
|
|
assert sem >= 1, "expected at least one positive scored via direct determine (Claim B)"
|