diff --git a/core/cli.py b/core/cli.py index b014c0f8..353ddd77 100644 --- a/core/cli.py +++ b/core/cli.py @@ -671,6 +671,154 @@ _DEMO_RESULTS_DIR = Path("evals/forward_semantic_control/results") _DEMO_CORPUS_DIR = Path("evals/forward_semantic_control/public") +_PHASE5_PREAMBLE = """ +================================================================================ + Phase 5 Demo — Stratified Mechanism-Isolation +================================================================================ + +WHAT THIS DEMO TESTS + CORE's inner-loop admissibility mechanism is supposed to behave correctly + across five distinct geometric failure modes — not just on average, but + per-family. This demo runs a hand-curated 20-case corpus that stratifies + the chain's behaviour across those five families: + + A. near_forbidden_correct_endpoint Expected and forbidden tokens have + nearly equal blade-scores. Tests + margin sensitivity at the boundary. + B. near_equal_admissible Two admissible candidates with + near-identical scores. Tests the + margin gate's determinism under tie. + C. no_admissible_path All candidates score ≤ 0 against the + blade. Tests honest refusal. + D. multi_step_admissibility Chained Family-A configurations. + Tests step-to-step composition. + E. heterogeneous_relation Chained steps with DIFFERENT blades. + Tests blade-switching cleanliness. + + Each case is run under TWO modes: + threshold mode (ADR-0024 — per-case static admissibility_threshold) + margin mode (ADR-0026 — scale-invariant δ-margin, δ=0.4 default) + +WHAT TO EXPECT IF THE MECHANISM IS WORKING + - Overall pass_rate (threshold) = 100% + - Overall pass_rate (margin) = 100% + - mechanism_isolated (both modes) = True + - Per-family pass_rate = 100% for ALL five families + - Family B refusal_rate (margin) = 100% (near-equal candidates must + refuse under δ-margin by construction) + - Family C refusal_rate (both modes) = 100% (no admissible path) + +WHAT TO LOOK FOR + - If any family's pass_rate < 100%, the mechanism failed THAT family + specifically — not a general regression. Dig into the per-case + detail in the report JSON to see which case and what selection. + - If Family B does NOT refuse under margin mode, the δ gate has + silently broken — check generate/admissibility.py::check_margin. + - If Family C admits anything, honest refusal has regressed — check + generate/exhaustion.py and the InnerLoopExhaustion raise sites in + generate/stream.py. + +WHEN TO TWEAK + - δ = 0.4 (the margin default) is FALSIFIABLE: if a case surfaces a + blade-gap below δ where margin-mode refusal is the WRONG behaviour, + that is an architectural finding to REPORT in + docs/evals/phase5_stratified_findings.md, NOT a value to patch. + - Adding new failure-mode families requires editing + evals/forward_semantic_control/phase5_runner.py::_passed_single + and authoring stratified cases in + evals/forward_semantic_control/public/v2_phase5/cases.jsonl. +================================================================================ +""" + +_PHASE6_PREAMBLE = """ +================================================================================ + Phase 6 Demo — Comparative Demo: CORE vs In-System Baseline +================================================================================ + +WHAT THIS DEMO TESTS + Three head-to-head claims about what CORE adds OVER an in-system baseline + (the same codebase with inner-loop / margin / rotor admissibility DISABLED + — i.e. an ADR-0023 ablation). Each claim is run on a focused 8-case + corpus and pinned by 17 CI contract tests: + + C1 Replay determinism Both baseline AND CORE produce byte-identical + trace hashes across 5 reruns. CORE additionally + folds refusal_reason into trace_hash, so refusal + events themselves are replayable evidence. + + C2 Traced rejection On adversarial cases where the boundary picks + the forbidden token: baseline emits it (with + admitted=False, silent emit). CORE overrides + and the rejection appears in rejected_attempts. + + C3 Coherent refusal On no-admissible-path cases: baseline emits an + inadmissible candidate. CORE raises + InnerLoopExhaustion with a typed RefusalReason. + +WHY THE BASELINE IS IN-SYSTEM (NOT AN LLM) + A transformer-LLM comparison would be non-deterministic, could not be + CI-enforced, and would be apples-to-oranges (different corpus / training + / sampling). The honest comparison is the ablation: same code, same + field state, same vocab, same persona — only the Phase 2-5 mechanisms + toggled off. Anything CORE produces that the baseline does not produce + is therefore attributable to the mechanisms themselves. + +WHAT TO EXPECT IF EVERYTHING IS WORKING + - C1: BOTH baseline_stable AND CORE_stable = 8/8 (replay is preserved, + not added, by Phase 2-5) + - C2: baseline_emits_forbidden = 3/3, baseline_admits_forbidden = 0/3 + CORE_corrects_or_refuses = 3/3, CORE_rejection_in_trace = 3/3 + - C3: baseline_typed_refusals = 0/3, baseline_emits_inadmissible = 3/3 + CORE_typed_refusals = 3/3 + - ALL THREE CONDITIONS = PASS + +WHAT TO LOOK FOR + - If C1 baseline fails, the algebra layer's replay has regressed — + unrelated to the chain. Investigate algebra/ first. + - If C1 CORE fails but baseline holds, the trace fold or refusal + plumbing has broken determinism. Check trace.py + exhaustion.py. + - If C2 baseline_admits_forbidden > 0, the boundary-only gate is + accidentally admitting things — unrelated to the chain, but worth + investigating. + - If C3 baseline_typed_refusals > 0, baseline is somehow raising + InnerLoopExhaustion — investigate whether inner_loop_admissibility + actually got disabled in the ablation. + - If C3 CORE_typed_refusals < case_count, CORE is NOT refusing where + it should — the honest-refusal contract has regressed. + +WHEN TO TWEAK + - If a C2/C3 case stops surfacing the intended baseline failure mode + (e.g. boundary stops picking the forbidden), it has aged out — the + cure is to add a NEW case that surfaces the failure, NOT to relax + the predicate. See docs/evals/phase6_comparative_demo.md. +================================================================================ +""" + +_ALL_PREAMBLE = """ +================================================================================ + Combined Demo — Full ADR-0024 Chain Evidence +================================================================================ + +This runs BOTH Phase 5 (stratified mechanism-isolation, 20 cases, 5 failure- +mode families, threshold + margin modes) AND Phase 6 (three-condition head- +to-head vs in-system baseline, 8 cases). A combined summary line at the end +reports the chain's overall verdict. + +For a thorough explanation of each phase, run them individually: + core demo phase5 + core demo phase6 + +For the central evidence index: + core demo list-results +================================================================================ +""" + + +def _print_preamble(text: str) -> None: + """Print a demo preamble to stdout (suppressed under --json).""" + print(text) + + def _format_phase5_table(metrics: dict[str, Any], per_family: dict[str, Any]) -> str: lines = [ "", @@ -769,8 +917,10 @@ def _write_results_index() -> Path: return index_path -def _run_demo_phase5(emit_json: bool) -> dict[str, Any]: +def _run_demo_phase5(emit_json: bool, *, with_preamble: bool = True) -> dict[str, Any]: from evals.forward_semantic_control.phase5_runner import run_lane + if with_preamble and not emit_json: + _print_preamble(_PHASE5_PREAMBLE) cases_path = _DEMO_CORPUS_DIR / "v2_phase5" / "cases.jsonl" cases = [json.loads(l) for l in cases_path.read_text().splitlines() if l.strip()] report = run_lane(cases) @@ -789,8 +939,10 @@ def _run_demo_phase5(emit_json: bool) -> dict[str, Any]: return report.metrics -def _run_demo_phase6(emit_json: bool) -> dict[str, Any]: +def _run_demo_phase6(emit_json: bool, *, with_preamble: bool = True) -> dict[str, Any]: from evals.forward_semantic_control.phase6_demo import run_lane + if with_preamble and not emit_json: + _print_preamble(_PHASE6_PREAMBLE) cases_path = _DEMO_CORPUS_DIR / "v2_phase6_demo" / "cases.jsonl" cases = [json.loads(l) for l in cases_path.read_text().splitlines() if l.strip()] report = run_lane(cases) @@ -829,6 +981,8 @@ def cmd_demo(args: argparse.Namespace) -> int: elif target == "phase6": _run_demo_phase6(args.json) elif target == "all": + if not args.json: + _print_preamble(_ALL_PREAMBLE) p5 = _run_demo_phase5(args.json) p6 = _run_demo_phase6(args.json) if not args.json: @@ -839,6 +993,15 @@ def cmd_demo(args: argparse.Namespace) -> int: print(f" Phase 5 mechanism_isolated: {p5.get('mechanism_isolated_margin', False)}") print(f" Phase 6 all three conditions: {p6.get('all_three_conditions_pass', False)}") print("") + print(" What this means:") + print(" Phase 5 verifies CORE handles five distinct geometric") + print(" failure modes correctly under both threshold and margin gates.") + print(" Phase 6 verifies CORE adds three capabilities the in-system") + print(" baseline cannot exhibit: deterministic replay of refusals,") + print(" traced rejection of inadmissible candidates, and coherent") + print(" typed refusal when no admissible path exists.") + print(" Together they are the load-bearing claim of the ADR-0024 chain.") + print("") else: _die(f"unknown demo target: {target}") diff --git a/evals/forward_semantic_control/public/inner_loop_benign/README.md b/evals/forward_semantic_control/public/inner_loop_benign/README.md new file mode 100644 index 00000000..ebde9cb1 --- /dev/null +++ b/evals/forward_semantic_control/public/inner_loop_benign/README.md @@ -0,0 +1,139 @@ +# Benign Inner-Loop Corpus (`inner_loop_benign/cases.jsonl`) + +10 cases that drive the **EXHAUSTION_CEILING ≤ 0.05** gate in the +corpus-observation lane (`inner_loop_runner.py`). This corpus is +intentionally *benign* — every case is constructed so the inner-loop +should comfortably admit, not refuse. + +If the chain refuses cases here, the chain has regressed. (Or a pack +token's geometry has shifted under Cl(4,1) signature — see below.) + +**Runner:** `evals/forward_semantic_control/inner_loop_runner.py` +**Report:** `evals/forward_semantic_control/results/phase5_benign_inner_loop_report.json` +**Gate:** `EXHAUSTION_CEILING = 0.05` (per-condition exhaustion rate must not exceed) +**Phase:** Authored in ADR-0024 Phase 5 to replace the *adversarial-by-accident* v1/dev corpus. + +--- + +## Why this corpus exists + +The original FSC v1/dev corpus used a `prime → chain_tokens` shape +that probes *teaching-driven walks* (ADR-0022 / 0023), not +inner-loop admissibility (ADR-0024). The EXHAUSTION_CEILING gate was +designed against benign corpus, but the v1/dev corpus is not benign +— it asks the inner-loop questions about teaching, not admissibility. +Phase 5 authored this *honestly benign* corpus to give the gate a +fair denominator. + +--- + +## Case schema + +```json +{ + "id": "FSC-BENIGN-001", + "kind": "single_token_admit", + "prime": ["What grounds reason?", "Reason is grounded in truth."], + "prompt": "What grounds reason?", + "expected_endpoint": "truth", + "chain_tokens": ["truth"], + "grounding_note": "Single-token region; expected token's self cga_inner ≈ 1.17 ≫ threshold 0.25." +} +``` + +The runner builds an `AdmissibilityRegion` from `chain_tokens` (outer +product over each token's versor) and the FieldState from the +priming sequence. With `chain_tokens` of size 1, the region admits +only that token's index; the inner-loop verifies its blade-score is +positive (against itself). + +--- + +## The Cl(4,1) signature quirk this corpus reveals + +23 of the 85 tokens in `en_core_cognition_v1` have **negative +self-`cga_inner`** under Cl(4,1) (Lorentzian signature). Most-negative +examples: `mean=-2.01`, `verify=-1.33`, `context=-1.15`, +`corrects=-0.74`. + +A single-token region with `chain_tokens=[tok]` where +`cga_inner(versor(tok), versor(tok)) < 0` will **always exhaust** in +threshold-mode under any positive threshold, even though the case is +"benign" by naive English semantics. This is a geometric fact about +Cl(4,1), not a regression. + +The 10 cases in this corpus were drawn from the 62-token subset with +**self-`cga_inner > 0.25`**. The case for `correction` was rejected +during authoring (it has `self-cga_inner = -0.036`) and replaced +with `beginning` (`self-cga_inner ≈ 1.36`). + +If you add a case here, verify the expected token's self-score +first: + +```bash +PYTHONPATH=. uv run python -c " +import numpy as np +from algebra.cga import cga_inner +from chat.runtime import ChatRuntime +vocab = ChatRuntime().session.vocab +for tok in ['']: + v = np.asarray(vocab.get_versor(tok), dtype=np.float32) + print(f'{tok}: self cga_inner = {float(cga_inner(v,v)):.4f}') +" +``` + +If the value is ≤ 0.25, the case will exhaust under the operational +threshold `t=0.25` — pick a different token, or use a multi-token +chain whose outer product realigns the blade. + +--- + +## Expected results + +| Condition | exhaustion_rate | pass_rate | +|---|---|---| +| boundary_only | 0.00 | 1.00 | +| null_control | 0.00 | 1.00 | +| inner_loop_t0 (threshold=0.0) | 0.00 | 1.00 | +| inner_loop_tpos (threshold=0.25) | 0.00 | 1.00 | + +If any of these exceeds the 0.05 ceiling, see "When to add cases" below. + +--- + +## When to add cases + +**Add cases when:** +- A new pack ships with new tokens whose semantic role isn't covered. +- A user-reported regression isolates to a benign case the corpus + doesn't cover. + +**Always:** +- Verify self-`cga_inner > 0.25` for the expected token BEFORE adding + the case (see snippet above). +- Pick a `prime` sequence whose final field state lands the + admissible region in a positive blade-score region. Run the case + once via the runner before committing. + +**Never:** +- Lower `EXHAUSTION_CEILING` to accommodate a failing case. The gate + is load-bearing — a real exhaustion here means the inner-loop is + refusing on a benign corpus. + +--- + +## Verifying after edit + +```bash +# Run the full inner_loop_runner against this corpus: +PYTHONPATH=. uv run python -c " +import json +from evals.forward_semantic_control.inner_loop_runner import run_lane, EXHAUSTION_CEILING +cases = [json.loads(l) for l in open('evals/forward_semantic_control/public/inner_loop_benign/cases.jsonl')] +report = run_lane(cases) +for label in ('boundary_only','null_control','inner_loop_t0','inner_loop_tpos'): + pc = report.metrics['per_condition'][label] + flag = 'OK' if pc['exhaustion_rate'] <= EXHAUSTION_CEILING else 'OVER' + print(f'{label:18s}: exhaustion={pc[\"exhaustion_rate\"]:.4f} ({flag})') +" +``` diff --git a/evals/forward_semantic_control/public/v2_phase5/README.md b/evals/forward_semantic_control/public/v2_phase5/README.md new file mode 100644 index 00000000..7b00bdc5 --- /dev/null +++ b/evals/forward_semantic_control/public/v2_phase5/README.md @@ -0,0 +1,136 @@ +# Phase 5 Corpus — Stratified Mechanism-Isolation (`v2_phase5/cases.jsonl`) + +20 hand-curated cases stratified across **five geometric failure-mode +families** so each family reports its own pass rate, refusal rate, and +mechanism-isolation evidence — instead of a single binary verdict over +mixed cases. + +**Runner:** `evals/forward_semantic_control/phase5_runner.py` +**Live:** `core demo phase5` +**Report:** `evals/forward_semantic_control/results/phase5_report.json` +**Contract tests:** `tests/test_phase5_corpus.py` (20 tests) +**Narrative:** `docs/evals/phase5_stratified_findings.md` + +--- + +## The five families + +| Family | Geometric construction | Threshold-mode expectation | Margin-mode expectation | +|---|---|---|---| +| **A. near_forbidden_correct_endpoint** | Expected blade-score > forbidden by a small margin (0.002 to 0.55) | admit expected | admit if gap ≥ δ=0.4, else refuse | +| **B. near_equal_admissible** | Two admissible candidates within ≤ 0.01 blade-score | admit either (tie-break stable) | refuse (diff < δ by construction) | +| **C. no_admissible_path** | Both candidates score ≤ 0 against blade | honest refusal (`INNER_LOOP_EXHAUSTION`) | honest refusal (`INNER_LOOP_EXHAUSTION`) | +| **D. multi_step_admissibility** | Chain of two Family-A configurations | each step admits expected | each step admits expected | +| **E. heterogeneous_relation** | Chained steps with *different blades* at each step | each step admits under its own blade | each step admits under its own blade | + +--- + +## Case schema + +### Single-step case (families A, B, C) + +```json +{ + "id": "FSC-P5-A-001", + "family": "near_forbidden_correct_endpoint", + "kind": "mechanism_isolation", + "semantic_pair": "comparison/reason", + "seed_token": "word", + "admissible_tokens": ["comparison", "reason"], + "relation_blade_token": "comparison", + "expected_endpoint": "comparison", + "forbidden_token": "reason", + "admissibility_threshold": 1.3329, + "rationale": "Sub-margin blade gap (0.0018). Boundary picks ..." +} +``` + +Family C cases additionally set `"expect_refusal": true` and +`"refusal_reason": "inner_loop_exhaustion"`. + +### Chained case (families D, E) + +```json +{ + "id": "FSC-P5-D-001", + "family": "multi_step_admissibility", + "kind": "chain_isolation", + "steps": [ + { "seed_token": "spirit", "admissible_tokens": ["define","explain"], "relation_blade_token": "define", "expected_endpoint": "define", "forbidden_token": "explain", "admissibility_threshold": 1.0249 }, + { "seed_token": "define", "admissible_tokens": ["correct","verify"], "relation_blade_token": "correct", "expected_endpoint": "correct", "forbidden_token": "verify", "admissibility_threshold": 1.0 } + ], + "rationale": "Two-step chain; each step is an independently mined Family-A configuration ..." +} +``` + +Family E cases use the same schema with optional `"relation_label"` per step (e.g. `"compare_with"`, `"causes"`) for documentation. + +--- + +## Required field semantics + +| Field | Meaning | Notes | +|---|---|---| +| `seed_token` | Pack token that seeds the FieldState | Must be present in the active pack | +| `admissible_tokens` | List of pack tokens forming `AdmissibilityRegion.allowed_indices` | All must be pack-grounded | +| `relation_blade_token` | Pack token whose versor is `AdmissibilityRegion.relation_blade` | Single-token blade only | +| `expected_endpoint` | The token the runner asserts as the correct selection | Must be in `admissible_tokens` | +| `forbidden_token` | The token the boundary leg should emit (mechanism-isolation evidence) | Must be in `admissible_tokens` | +| `admissibility_threshold` | Static threshold for threshold-mode leg | Typically set between expected and forbidden blade-scores | +| `expect_refusal` *(Family C)* | If true, both modes must refuse | | +| `refusal_reason` *(Family C)* | Stable enum value the runner asserts on refusal | Use `"inner_loop_exhaustion"` | + +--- + +## How cases were geometrically mined + +The corpus was produced by the offline tool +`evals/forward_semantic_control/phase5_mine.py`, which scans triples +`(seed, admissible_pair, blade)` over a pack subset and reports +candidate geometric configurations for each family. Run it yourself: + +```bash +PYTHONPATH=. uv run python evals/forward_semantic_control/phase5_mine.py --family A --limit 25 +PYTHONPATH=. uv run python evals/forward_semantic_control/phase5_mine.py --family B --limit 25 +PYTHONPATH=. uv run python evals/forward_semantic_control/phase5_mine.py --family C --limit 25 +``` + +The miner is offline only (it imports `chat.runtime.ChatRuntime` for +vocab access, which is too heavy to run inside the contract tests). +Use it to find candidate cases; verify them by hand by running +`uv run python -c "..."` to inspect `cga_inner` scores; commit only +once the geometric construction is confirmed. + +--- + +## When to add cases + +**Always add — never edit existing cases — when:** +- A new failure mode is discovered in production / Phase 6 demo. +- A real corpus case surfaces a δ-margin disagreement that should be + investigated as an ADR-0026 falsification candidate. +- The pack vocabulary expands and a new region geometry becomes + reachable. + +**Do NOT remove cases just because they pass.** They are regression +contracts. + +**Do NOT lower a per-family pass-rate assertion to accommodate a +failing case.** That hides the regression. Either fix the +implementation or document the architectural finding in +`docs/evals/phase5_stratified_findings.md`. + +--- + +## Verifying after edit + +```bash +# 1. The case-schema and pass predicates must hold: +core test --suite phase5 + +# 2. The runner must produce the expected report shape: +core demo phase5 + +# 3. The full chain still passes: +core test --suite adr-0024 +``` diff --git a/evals/forward_semantic_control/public/v2_phase6_demo/README.md b/evals/forward_semantic_control/public/v2_phase6_demo/README.md new file mode 100644 index 00000000..8eee1ecd --- /dev/null +++ b/evals/forward_semantic_control/public/v2_phase6_demo/README.md @@ -0,0 +1,106 @@ +# Phase 6 Corpus — Comparative Demo (`v2_phase6_demo/cases.jsonl`) + +8 focused cases that drive the **three head-to-head conditions** of +the Phase 6 comparative demo. The "head-to-head" is between CORE +(inner-loop + margin + rotor admissibility enabled) and an in-system +baseline (the same codebase with those mechanisms disabled — an +ADR-0023 ablation). + +**Runner:** `evals/forward_semantic_control/phase6_demo.py` +**Live:** `core demo phase6` +**Report:** `evals/forward_semantic_control/results/phase6_demo_report.json` +**Contract tests:** `tests/test_phase6_demo.py` (17 tests) +**Narrative:** `docs/evals/phase6_comparative_demo.md` + +--- + +## The three conditions + +| Condition | Cases | What it proves | +|---|---|---| +| **C1 `replay_determinism`** | 2 | Both baseline AND CORE produce byte-identical trace hashes across 5 reruns. CORE additionally folds refusal_reason into the hash, so refusal events themselves are replayable. | +| **C2 `traced_rejection`** | 3 | When the boundary picks the *forbidden* token, baseline emits it with `admitted=False` (silent emit). CORE overrides, the rejection appears in `rejected_attempts`, and the selection difference is causally attributable to the inner-loop. | +| **C3 `coherent_refusal`** | 3 | When no candidate is admissible, baseline emits an inadmissible candidate. CORE raises `InnerLoopExhaustion` with a typed `RefusalReason` carrying evidence. Typed refusal is *new* in CORE. | + +--- + +## Why the baseline is in-system, not a transformer LLM + +| Concern | In-system baseline | Transformer LLM | +|---|---|---| +| Deterministic | Yes | No (sampling temperature, top-k, etc.) | +| CI-enforceable | Yes (17 contract tests) | No | +| Apples-to-apples | Yes (same field state, vocab, persona) | No (different corpus, training, etc.) | +| Attributable | Yes (only the chain toggled) | No (any difference could be from any layer) | + +A transformer comparison would tell us nothing about whether the +ADR-0024 chain mechanisms are doing real work — only that two +unrelated systems produce different outputs. The honest comparison is +the ablation. + +--- + +## Case schema + +Same single-step shape as Phase 5 Family A / C, with one additional +required field: `condition`. + +```json +{ + "id": "FSC-P6-C2-001", + "condition": "traced_rejection", + "kind": "mechanism_isolation", + "seed_token": "word", + "admissible_tokens": ["question", "meaning"], + "relation_blade_token": "question", + "expected_endpoint": "question", + "forbidden_token": "meaning", + "admissibility_threshold": 1.3706, + "rationale": "Boundary geometrically prefers 'meaning' (forbidden); ..." +} +``` + +C3 cases additionally set `"expect_refusal": true` and +`"refusal_reason": "inner_loop_exhaustion"`. + +### `condition` field values + +| Value | What it controls | +|---|---| +| `"replay_determinism"` | Runs 5 reruns under both baseline and CORE; pass iff both hash sets are singletons. | +| `"traced_rejection"` | Asserts boundary emits forbidden AND CORE corrects-or-refuses AND CORE rejection in trace. | +| `"coherent_refusal"` | Asserts baseline is NOT a typed refusal AND CORE IS a typed `INNER_LOOP_EXHAUSTION`. | + +--- + +## When to add cases + +**Add new cases when:** +- A new boundary-vs-blade divergence pattern is discovered. +- A new geometric construction surfaces a refusal mode not exercised + by C1/C2/C3 cases. +- Phase 5 surfaces a regression that should also be pinned at the + comparative demo layer for narrative impact. + +**Do NOT add cases that always pass.** This corpus is small by design +— each case must surface a *specific* baseline-vs-CORE asymmetry. + +**Do NOT relax C2/C3 predicates when a case ages out.** If a C2 case +stops surfacing "boundary picks forbidden" (because the underlying +geometry shifted), the case has aged out — add a NEW case that +surfaces the failure mode, then archive the old one. + +--- + +## Verifying after edit + +```bash +# 1. Contract tests pin the case-aggregate behaviour: +core test --suite phase6 + +# 2. Live demo produces an investor-readable table: +core demo phase6 + +# 3. The headline metric MUST be all_three_conditions_pass=true: +core demo list-results --json | python3 -c "import json,sys; print(json.loads(sys.stdin.read())['reports'])" | grep all_three_conditions_pass +``` diff --git a/evals/forward_semantic_control/results/README.md b/evals/forward_semantic_control/results/README.md new file mode 100644 index 00000000..e3482d46 --- /dev/null +++ b/evals/forward_semantic_control/results/README.md @@ -0,0 +1,156 @@ +# Forward Semantic Control — Results Directory + +This directory holds the canonical evidence reports for every phase of +the ADR-0024 Forward Semantic Control chain. Every report here is +machine-generated by a runner under `evals/forward_semantic_control/` +and re-runnable via the `core` CLI. + +If you are reading this cold and want to know "what does CORE actually +do that an LLM cannot," **start with `phase6_demo_report.json`** — it +holds the three head-to-head conditions vs the in-system baseline. + +--- + +## How to regenerate everything + +```bash +core demo all # runs phase5 + phase6, refreshes reports + index +core demo list-results # prints the index with headline metrics +``` + +To run a single phase: + +```bash +core demo phase5 # stratified mechanism-isolation (~30 s) +core demo phase6 # 3-condition comparative demo (~10 s) +``` + +Both write JSON reports here and refresh `index.json`. + +--- + +## Reports — what's in this directory + +### `phase2_inner_loop_report.json` + +**Lane:** Corpus observation (4-condition matrix on the existing FSC v1 corpus) +**Runner:** `inner_loop_runner.py` +**ADR:** ADR-0024 Phase 2 +**Status:** Historical evidence — the case schema is teaching-driven (v1/dev) and is no longer the load-bearing benign corpus (see `phase5_benign_inner_loop_report.json` below). + +Reports per condition (boundary_only / null_control / inner_loop_t0 / inner_loop_tpos): +`pass_rate`, `mean_rejection_count_per_turn`, `non_empty_rejected_attempts_rate`, +`exhaustion_rate`, `mean_admissibility_checks_per_turn`, latency stats, +`trace_hash_stability_pass_rate`. + +**Headline interpretation.** The Phase 2 corpus is *adversarial-by-accident*: the v1 case schema probes teaching-driven walks, not inner-loop admissibility. Exhaustion rates above 0.05 on this corpus are an *architectural finding* (documented in `docs/decisions/ADR-0024-inner-loop-admissibility.md` Phase 1 addendum), not a regression. + +### `phase3_v2_report.json` + +**Lane:** Mechanism isolation (5 adversarial v2 cases) +**Runner:** `v2_runner.py` +**ADR:** ADR-0024 Phase 3 +**Headline metric:** `mechanism_isolated == True` iff +`pass_rate == 1.0 AND boundary_decoy_rate == 1.0 AND rejection_traced_rate == 1.0`. + +**Interpretation.** Each case is constructed so the boundary picks a *forbidden* token and the inner-loop must override and trace the rejection. A pass on all three conditions causally attributes the selection difference to the inner-loop, not to any other code path. + +### `phase4_characterization_v1_plus_dev.json` / `phase4_characterization_v2.json` / `phase4_characterization_combined.json` / `phase4_summary.json` + +**Lane:** Threshold characterization (diagnostic sweep) +**Runner:** `threshold_characterization.py` +**ADR:** ADR-0024 Phase 4 (diagnostic, not a tuning artifact) +**Headline metric:** `best_separation_quality` and `geometry_supports_static_threshold` per threshold value swept over `{-1.0, -0.5, 0.0, 0.1, 0.25, 0.5, 1.0}`. + +**Interpretation.** **No static threshold delivers separation_quality ≥ 0.8 on the v1+dev+v2 corpus.** This is the load-bearing finding that motivated ADR-0026's switch from threshold to ranked-with-margin. If you want the geometric justification for the δ-margin gate, this report is it. + +### `phase5_report.json` *(load-bearing — read first if you only read one)* + +**Lane:** Stratified mechanism-isolation across 5 failure-mode families +**Runner:** `phase5_runner.py` +**ADR:** ADR-0024 Phase 5 +**Corpus:** 20 cases under `public/v2_phase5/cases.jsonl` +**Headline metrics:** +- `metrics.pass_rate_threshold` — overall under static threshold +- `metrics.pass_rate_margin` — overall under δ-margin (the operational mode) +- `metrics.mechanism_isolated_threshold` / `mechanism_isolated_margin` — booleans for each mode +- `per_family[].pass_rate_threshold` / `pass_rate_margin` / `refusal_rate_margin` — per-family breakdown + +**Interpretation.** All five families should pass at 100% under both modes. Family B (`near_equal_admissible`) should additionally show `refusal_rate_margin = 100%` — those cases are constructed so δ-margin MUST refuse. Family C (`no_admissible_path`) should show `refusal_rate = 100%` in BOTH modes. + +**When to look here:** any per-family pass_rate drop. The family that drops is the failure mode that broke. + +### `phase5_benign_inner_loop_report.json` + +**Lane:** Benign corpus exhaustion-ceiling lane +**Runner:** `inner_loop_runner.py` (same as Phase 2, different corpus) +**Corpus:** `public/inner_loop_benign/cases.jsonl` (10 single-token cases) +**Headline gate:** `per_condition[*].exhaustion_rate ≤ 0.05` (EXHAUSTION_CEILING) + +**Interpretation.** This is the corpus the EXHAUSTION_CEILING gate was *actually* designed against — a curated benign corpus where the inner-loop should rarely refuse. Expected: 0.00 exhaustion in all four conditions. If exhaustion rises here, either (a) a benign case's expected token now has negative self-`cga_inner` under the active pack (Cl(4,1) signature artifact, see `docs/evals/phase5_stratified_findings.md`), or (b) the inner-loop has regressed. + +### `phase6_demo_report.json` *(headline demo — start here if you're new)* + +**Lane:** Comparative demo, 3 conditions vs in-system baseline (ADR-0023 ablation) +**Runner:** `phase6_demo.py` +**ADR:** ADR-0024 Phase 6 +**Corpus:** 8 cases under `public/v2_phase6_demo/cases.jsonl` +**Headline metrics:** +- `metrics.c1_pass` — Replay determinism (baseline AND CORE byte-identical across 5 reruns) +- `metrics.c2_pass` — Traced rejection (boundary emits forbidden, CORE corrects + traces) +- `metrics.c3_pass` — Coherent refusal (baseline emits inadmissible, CORE raises typed refusal) +- `metrics.all_three_conditions_pass` — boolean AND + +**Interpretation.** This is THE comparative demo. Each `cN_pass` is a *single boolean claim*. If `all_three_conditions_pass` is true, the chain delivers on its three head-to-head claims. If any is false, see the per-case detail and the C-specific breakdown in the same JSON. + +### `index.json` + +**Auto-generated by `core demo`.** Lists every report file with size and a curated subset of headline metrics. Refreshed by every `core demo` run. Use: + +```bash +core demo list-results # human-readable +core demo list-results --json # machine-readable index +``` + +--- + +## How to interpret a per-case detail (general schema) + +Each report's `case_details` list carries per-case evidence. Common fields: + +| Field | Meaning | +|---|---| +| `id` | Stable case ID (e.g. `FSC-P5-A-001`) | +| `family` | Failure-mode family (Phase 5 only) | +| `condition` | One of `replay_determinism`, `traced_rejection`, `coherent_refusal` (Phase 6 only) | +| `boundary` | Boundary-only leg result: `{selected, admitted, rejected_words, ...}` | +| `threshold_leg` / `margin_leg` / `core` | Per-mode leg results | +| `passed_threshold` / `passed_margin` / `passed_threshold` | Pass predicate verdicts | +| `replay_hashes_*` | List of N trace hashes from N reruns (Phase 6 C1) | +| `c2_*` / `c3_*` | Phase 6 condition-specific predicates | + +A `refused: true` leg additionally carries `refusal_reason` (the stable enum value), `refusal_message`, and `rejected_attempts`. + +--- + +## When something looks wrong + +1. **Re-run the responsible phase first.** + `core demo phase5` or `core demo phase6` regenerates the report from scratch. +2. **Compare against the contract tests.** + `core test --suite phase5` (20 tests) and `core test --suite phase6` (17 tests) pin the headline numbers. If those pass and the JSON looks wrong, the runner has drifted from the test predicate — not the implementation. +3. **Check the central architectural finding doc.** + `docs/evals/phase5_stratified_findings.md` documents the known geometric quirks of Cl(4,1) that motivate the δ-margin gate. Many "regressions" turn out to be self-`cga_inner < 0` cases that need a different region shape, not a code fix. +4. **Last resort: full chain.** + `core test --suite adr-0024` (98 tests, ~2 minutes) runs every Phase 2-6 contract test. If THAT passes and a demo report still looks wrong, the corpus has aged out of the failure mode it was designed to surface — see each corpus's own README for guidance on adding new cases. + +--- + +## Related docs + +- `docs/runtime_contracts.md` — Refusal / Margin / Rotor admissibility contracts +- `docs/evals/phase5_stratified_findings.md` — Phase 5 architectural findings +- `docs/evals/phase6_comparative_demo.md` — Phase 6 narrative + "what this does NOT claim" +- `docs/decisions/ADR-0024-inner-loop-admissibility.md` — the foundational ADR +- `docs/decisions/ADR-0025-rotor-frame-admissibility-design-note.md` — rotor admissibility (accepted) +- `docs/decisions/ADR-0026-ranked-admissibility-with-margin.md` — δ-margin gate (accepted) diff --git a/tests/test_cli_demo.py b/tests/test_cli_demo.py index cf810ca5..4dae9693 100644 --- a/tests/test_cli_demo.py +++ b/tests/test_cli_demo.py @@ -132,3 +132,91 @@ class TestDemoSubcommand: data = json.loads(index_path.read_text()) names = [e["file"] for e in data["reports"]] assert "phase6_demo_report.json" in names + + +class TestDemoPreambles: + """Pin the preamble explanations so they don't drift silently.""" + + def test_phase6_preamble_explains_three_conditions(self, capsys) -> None: + cli.main(["demo", "phase6"]) + out = capsys.readouterr().out + assert "WHAT THIS DEMO TESTS" in out + assert "C1 Replay determinism" in out + assert "C2 Traced rejection" in out + assert "C3 Coherent refusal" in out + assert "WHAT TO EXPECT" in out + assert "WHEN TO TWEAK" in out + + def test_phase6_preamble_states_in_system_baseline(self, capsys) -> None: + cli.main(["demo", "phase6"]) + out = capsys.readouterr().out + # The "why not a transformer LLM" explanation must be present. + assert "ADR-0023 ablation" in out + assert "non-deterministic" in out or "Non-deterministic" in out + + def test_phase5_preamble_explains_five_families(self, capsys) -> None: + cli.main(["demo", "phase5"]) + out = capsys.readouterr().out + assert "WHAT THIS DEMO TESTS" in out + for family in ( + "near_forbidden_correct_endpoint", + "near_equal_admissible", + "no_admissible_path", + "multi_step_admissibility", + "heterogeneous_relation", + ): + assert family in out + assert "WHAT TO LOOK FOR" in out + + def test_phase5_preamble_states_delta_falsifiable(self, capsys) -> None: + cli.main(["demo", "phase5"]) + out = capsys.readouterr().out + assert "FALSIFIABLE" in out or "falsifiable" in out + + def test_preamble_suppressed_under_json(self, capsys) -> None: + cli.main(["demo", "phase6", "--json"]) + out = capsys.readouterr().out + # No preamble text should leak into --json mode. + assert "WHAT THIS DEMO TESTS" not in out + # Output must be parseable JSON from the first character. + payload = json.loads(out.split("\n\n")[0]) + assert "metrics" in payload + + def test_all_preamble_explains_combined_run(self, capsys) -> None: + cli.main(["demo", "all"]) + out = capsys.readouterr().out + assert "Combined Demo" in out + # Both phase preambles fire for `demo all`. + assert "Phase 5 Demo" in out + assert "Phase 6 Demo" in out + # Combined summary at the end. + assert "Combined demo summary" in out + assert "load-bearing claim of the ADR-0024 chain" in out + + +class TestResultsReadme: + """The results/ directory ships with an explanatory README so cold readers + can interpret each report without spelunking the runner source.""" + + def test_results_readme_exists(self) -> None: + readme = Path("evals/forward_semantic_control/results/README.md") + assert readme.exists() + text = readme.read_text() + # The README must explicitly call out each phase's report file. + for fname in ( + "phase5_report.json", + "phase6_demo_report.json", + "phase5_benign_inner_loop_report.json", + "phase4_characterization", + "phase3_v2_report.json", + "phase2_inner_loop_report.json", + ): + assert fname in text, f"{fname} missing from results/README.md" + + def test_corpus_readmes_exist(self) -> None: + for path in ( + "evals/forward_semantic_control/public/v2_phase5/README.md", + "evals/forward_semantic_control/public/v2_phase6_demo/README.md", + "evals/forward_semantic_control/public/inner_loop_benign/README.md", + ): + assert Path(path).exists(), f"{path} missing"