diff --git a/CLAIMS.md b/CLAIMS.md index 8d5c20b8..f946c782 100644 --- a/CLAIMS.md +++ b/CLAIMS.md @@ -43,7 +43,7 @@ is a CI failure (`.github/workflows/lane-shas.yml`). | ADR-0104 | `curriculum_loop_closure` | Curriculum-sourced proposals route through single reviewed teaching path | `evals/curriculum_loop_closure/results/v1_dev.json` | `cb94ca0042d78ec2624129ff6493d52e767b69feea32d2997b85d88f1c0883af` | | ADR-0131 | `math_teaching_corpus_v1` | Math teaching corpus replays deterministically; all chains pass exit criterion (correct_rate=1.0, wrong=0) | `evals/math_teaching_corpus/v1/report.json` | `eaf160d145da29f9050ede8d58bf111b0f651dd40aeae9201857d0b97e014dd4` | | ADR-0206 | `deductive_logic_v1` | Propositional entailment scored against an independent truth-table oracle; dev+holdout+external 716/716 correct, wrong=0, refused=0 | `evals/deductive_logic/report.json` | `97a230949016e38d5e3f37a69e4245b320575ee70e5af92ff7607f7b05f74b5f` | -| ADR-0256 | `deduction_serve_v1` | Flag-gated deduction serving decides real English/member/fused/verb/existential arguments end-to-end under earned SERVE licenses; wrong=0 across all splits | `evals/deduction_serve/report.json` | `c855d55cf316471fdfe092aa0d5c954e5ceb6f30c9a8db283e0b9a5d5e8b419a` | +| ADR-0256 | `deduction_serve_v1` | Flag-gated deduction serving decides real English/member/fused/verb/existential arguments end-to-end, wrong=0 across all splits; 4 of 25 shape-bands hold an earned SERVE licence on DISTINCT evidence (R-13 re-count, 2026-07-28) and the other 21 decide with disclosure | `evals/deduction_serve/report.json` | `c855d55cf316471fdfe092aa0d5c954e5ceb6f30c9a8db283e0b9a5d5e8b419a` | | ADR-0262 | `curriculum_serve_v1` | Flag-gated curriculum serving answers exam questions from a subject's RATIFIED chain corpus only; untaught facts return UNKNOWN (anti-recall probes enforced), wrong=0 | `evals/curriculum_serve/report.json` | `d9e7ba500f040b865870413a940ee9a49910ac22e1a89c9feec1a60bdd2513f1` | ## Verification diff --git a/chat/data/deduction_serve_ledger.json b/chat/data/deduction_serve_ledger.json index 43a9e5a1..3f2f83d2 100644 --- a/chat/data/deduction_serve_ledger.json +++ b/chat/data/deduction_serve_ledger.json @@ -1,42 +1,42 @@ { "classes": { "atomic": { - "correct": 720, + "correct": 98, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, "wrong": 0 }, "categorical": { - "correct": 720, + "correct": 294, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, "wrong": 0 }, "conditional_chain": { - "correct": 720, + "correct": 294, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, "wrong": 0 }, "conditional_single": { - "correct": 720, + "correct": 294, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, "wrong": 0 }, "disjunctive": { - "correct": 720, + "correct": 294, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, "wrong": 0 }, "en_atomic": { - "correct": 720, + "correct": 474, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, @@ -50,35 +50,35 @@ "wrong": 0 }, "en_conditional_single": { - "correct": 720, + "correct": 474, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, "wrong": 0 }, "en_condmem_chain": { - "correct": 720, + "correct": 52, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, "wrong": 0 }, "en_condmem_conditional": { - "correct": 720, + "correct": 60, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, "wrong": 0 }, "en_condmem_disjunctive": { - "correct": 720, + "correct": 28, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, "wrong": 0 }, "en_condmem_fused": { - "correct": 720, + "correct": 60, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, @@ -92,92 +92,92 @@ "wrong": 0 }, "en_exist_chain": { - "correct": 720, + "correct": 28, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, "wrong": 0 }, "en_exist_negative": { - "correct": 720, + "correct": 28, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, "wrong": 0 }, "en_exist_universal": { - "correct": 720, + "correct": 46, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, "wrong": 0 }, "en_exist_witness": { - "correct": 720, + "correct": 35, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, "wrong": 0 }, "en_member_atomic": { - "correct": 720, + "correct": 60, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, "wrong": 0 }, "en_member_chain": { - "correct": 720, + "correct": 52, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, "wrong": 0 }, "en_member_negative": { - "correct": 720, + "correct": 52, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, "wrong": 0 }, "en_member_single": { - "correct": 720, + "correct": 60, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, "wrong": 0 }, "en_verb_chain": { - "correct": 720, + "correct": 308, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, "wrong": 0 }, "en_verb_fact": { - "correct": 720, + "correct": 660, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, "wrong": 0 }, "en_verb_negative": { - "correct": 720, + "correct": 308, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, "wrong": 0 }, "en_verb_universal": { - "correct": 720, + "correct": 660, "refused": 0, "t2_agrees_gold": 0, "t2_verified": 0, "wrong": 0 } }, - "content_sha256": "a056f899a47bbd4536e3141f47a9131c9058ab55f2e9567fdcb254145e2dc252", - "note": "Sealed-practice committed ledger for deduction serving (ADR-0256). Engine reads, never writes. Ceilings stay at safe defaults (theta_SERVE=0.99). A band earns SERVE by demonstrated pipeline reliability (reader+projector+engine) at volume >= 657 committed.", + "content_sha256": "8db89aed12aafb68625f4ee0ba994f91964775eef8f8f9e9f92c6ed030ecb1ed", + "note": "Sealed-practice committed ledger for deduction serving (ADR-0256). Engine reads, never writes. Ceilings stay at safe defaults (theta_SERVE=0.99). A band earns SERVE by demonstrated pipeline reliability (reader+projector+engine) at volume >= 657 committed. RE-COUNTED 2026-07-28 under R-13: one committed case per DISTINCT decision (case text), so committed == distinct evidence by construction. The prior seal folded the raw corpus and counted replays as independent trials, which the Wilson floor assumes they are not; 21 of 25 bands held SERVE licences their evidence never supported and are demoted here.", "provenance": "evals.deduction_serve.practice.runner.seal_ledger", "schema": "deduction_serve_ledger_v1" } diff --git a/core/ratified_ledger.py b/core/ratified_ledger.py index cb8f3e82..ad4d598e 100644 --- a/core/ratified_ledger.py +++ b/core/ratified_ledger.py @@ -95,7 +95,12 @@ CAPABILITY_LEDGERS: dict[str, LedgerSpec] = { capability="deduction_serve", path=_PROJECT_ROOT / "chat" / "data" / "deduction_serve_ledger.json", missing_ok=False, - note="ADR-0256 — ships sealed; 25 bands at 720/720 wrong=0.", + note=( + "ADR-0256 — ships sealed; 25 bands, wrong=0. RE-COUNTED 2026-07-28 " + "(R-13): committed == DISTINCT evidence, so 4 bands hold a SERVE " + "licence and 21 decide with disclosure. The prior note said " + "'25 bands at 720/720', which counted replays as independent trials." + ), ), "curriculum_serve": LedgerSpec( capability="curriculum_serve", diff --git a/docs/assessment/50-execution-plan.md b/docs/assessment/50-execution-plan.md index f5d25ebc..d698a768 100644 --- a/docs/assessment/50-execution-plan.md +++ b/docs/assessment/50-execution-plan.md @@ -11,7 +11,7 @@ | **Track A — ADR-0252 §5 run to NO-GO**, criterion pre-registered, artifact + digest committed, **verdict ratified 2026-07-28** | PR-8 — a small ADR (H-3/G-20) · PR-10 — a refactor ADR (H-4) | | PR-4 membership + reachability ratchets · PR-6 three doctrine pins · PR-7 M2 trust table · PR-9 count-the-swallow · **PR-5 the flag register (R-3+R-4)** | PR-12 — an ADR **amendment** (R-13 authorizes it; the amendment is still owed) | | PR-3 + PR-3b (**R-7**, Wave 1) · **PR-11 the soak to committed evidence (R-9+R-2)** · H-13 · H-8e · G-22 · G-23 | Track B — the fabrication ADR (standing instruction, unchanged) | -| **Closed:** G-5, G-6, G-7, G-8, G-9, G-15, G-22, H-6, H-7, H-8 (**all five**), H-11, H-13 · **Answered:** G-1 · **Discharged:** H-10 · **Pinned:** G-23 | Track C — invention, deliberately last | +| **Closed:** G-5, G-6, G-7, G-8, G-9, G-15, G-19, G-22, H-1, H-6, H-7, H-8 (**all five**), H-11, H-13 · **Answered:** G-1 · **Discharged:** H-10 · **Pinned:** G-23 | Track C — invention, deliberately last | **New this arc, not in the original assessment:** **N-8** (the §5 experiment had already returned GO twice, on unmerged branches, both unsound), **N-9** (the gate "drift" was a recorded decision — PR-4's pin 1 **withdrawn** rather than built, R-14 dissolved), **G-21** (the math reader decides **1.0%** of `holdout_dev/v1`), **G-22** (`main` was red before the arc, and `CLAIMS.md` published a superseded evidence digest for two days), **H-13**, **H-14**, **H-8e**. @@ -218,7 +218,7 @@ The packet's original order was by register number. That is the wrong axis: it i | 2 | ~~R-7~~ | ~~Register supersession~~ | — | **EXECUTED** — PR-3 + PR-3b, landed 2026-07-28. Listed so the order stays readable. | | 3 | ~~**R-3 + R-4**~~ | PR-5 — the flag register, all 32 booleans classified, profiles as units, the bidirectional pin, `accrue_realized_knowledge` into the daemon profile | **M** | **EXECUTED 2026-07-28.** Closed **G-6, G-8, H-6, H-8(c), H-8(d)** — and **H-8 in full**. The N-6 docstring correction **dissolved**: the docstrings were right and the code was incomplete. **Wave 4's hard gate is lifted.** | | 4 | ~~**R-9 + R-2**~~ | PR-11 — the soak re-run at 5000 beats under the **corrected** profile, artifact committed, digest pinned, cadence enforced by a source-hash pin, H1–H4 on the gate | **M** | **EXECUTED 2026-07-28.** Closed **G-5**. R-2 landed in the contract *before* the re-run, as designed. The staleness pin **would have caught PR-5** — verified by sabotage. | -| 5 | **R-13** | PR-12 — distinct-evidence counting at the seal boundary + the honest re-count of 25 ratified bands, demotions applied, `test_volume_honesty.py` moved in the same PR | **L** | Needs an ADR **amendment** (0175/0263 lineage). R-13 authorizes the demotions; it does not write the amendment. Expect served capability to shrink — that is the mechanism working. | +| 5 | ~~**R-13**~~ | PR-12 — distinct-evidence counting at the seal boundary, the honest re-count, demotions applied, the producer hardened against re-padding | **L** | **EXECUTED 2026-07-28. 25 licensed bands → 4.** Served capability shrank by 21 bands and `wrong` stayed 0: no answer changed, only the claim attached to it. | | 6 | **R-8** | PR-14 — the outcome-mix rule, then seal `curriculum_serve_ledger.json`. Four bands qualify on sealing (N-5) | **M** | Depends on PR-12's counting basis. This is the entire throughput frontier, one ruling wide. | | 7 | **R-1 · R-5 · R-6 · R-11** | Posture statements: efferent-action deferral with entry criterion (G-12) · the identity-enforcement discrimination bar (G-11) · non-text ingest deferral with the falsification bench as its standard (G-17) · **R-11 B — measure before gating** | **S** each | All four are prose with a criterion. None blocks anything; leaving them silent is the cost. They go last because they are the only items where lateness is cheap. | @@ -340,7 +340,17 @@ Lift `core/cognition/surface_resolution.py`'s declared-precedence pattern to arm The run is known to pass at 5000 beats (N-4). The deliverable is therefore the ceremony the machinery never got: re-run at the contract's long horizon, **commit the report**, **pin the `deterministic_digest`** so a regression flips it (the contract's own unmet closing instruction), schedule per R-9, and promote the H1–H4 holds/bites pins into a curated suite (which PR-4 makes visible). If a hold *bites* at a longer horizon than 5000 beats, that is the highest-value result in this wave and it stops the wave. ### PR-12 · The Wilson re-count · **L** · H-1/G-19 — *ADR amendment (0175/0263 lineage), authorized by R-13* -Distinct-evidence counting at the seal boundary: a replay refreshes, never increments. Declared in the ledger schema. Then re-count the 25 ratified bands and **apply the demotions honestly** — 21 fall short, a number already pinned in `test_volume_honesty.py` "in BOTH directions," so that pin moves in the same PR. Expect served capability to shrink. **Verification:** the re-count is a pinned computation, not a spreadsheet; the demoted bands' licenses are revoked in the same PR. +Distinct-evidence counting at the seal boundary: a replay refreshes, never increments. Declared in the ledger schema. Then re-count the 25 ratified bands and **apply the demotions honestly** — 21 fall short, a number already pinned in `test_volume_honesty.py` "in BOTH directions," so that pin moves in the same PR. Expect served capability to shrink. **Verification:** the re-count is a pinned computation, not a spreadsheet; the demoted bands' licences are revoked in the same PR. + +**LANDED 2026-07-28. 25 licensed bands → 4; 21 demoted.** The four survivors are `en_conditional_chain`, `en_disjunctive`, `en_verb_fact`, `en_verb_universal` — the only bands whose corpus holds enough *independent* cases. The worst offenders inflated **28 distinct cases into 720 committed decisions**, taking `conservative_floor` from `0.808413` to `0.990868` and clearing θ_SERVE on evidence that never existed. + +**What the demotion did and did not do, stated exactly:** no answer changed, no answer became wrong, `wrong` stayed **0** across all 25 bands. The engine is as correct as it was. What changed is the *claim* attached to the answer — 21 bands now serve the same sound conclusion with *"reasoned, but I haven't yet earned a verified track record on arguments of this shape."* **The reasoning did not get worse; the boast did.** + +**The producer was hardened, which is the half that stops it recurring.** `seal_ledger` now refuses outright: `assert_sealed_evidence_distinct` compares the ledger about to be written against the corpus's distinct-case count and raises before anything is written. Curriculum's sealer has carried this guarantee since ADR-0264 R9; deduction's had no equivalent, which is exactly how the exposure arose. + +**A hollow guard of my own, caught by sabotage rather than trusted.** The first version compared `all_gold_problems()` with `distinct_gold_problems()` — two functions that agree by construction, neither of which is what `build_ledger` folds. Reverting `build_ledger` to the raw corpus walked straight past it *and re-sealed the inflated artifact*. The guard now takes the **built ledger** as input, because the invariant is about the artifact. Re-run of the identical attack: refused, and **wrote nothing**. + +**Four records corrected in the same PR**, each of which would otherwise have kept asserting the old capability: `CAPABILITY_LEDGERS`'s manifest note ("25 bands at 720/720"), the published `CLAIMS.md` claim (caught immediately by G-22's own pin), `test_volume_honesty.py`'s module docstring ("NOT silently repaired" — R-13 *is* that repair), and the exposure inventory's framing (it now pins the fix, not the gap). Eight tests encoded the unearned expectation and were updated to the honest one. ### ~~PR-13 · Curriculum query-scoping~~ · **WITHDRAWN** · N-5 Shipped as ADR-0264 R5, discharged 2026-07-26. The 16-premise cap does not exist in the running system. G-10 is re-scoped to R-8 alone. diff --git a/docs/assessment/50-rulings.md b/docs/assessment/50-rulings.md index 6a1f69cf..ed80dc59 100644 --- a/docs/assessment/50-rulings.md +++ b/docs/assessment/50-rulings.md @@ -9,9 +9,9 @@ **Also ratified:** the ADR-0252 §5 **NO-GO** verdict. **Execution order (corrected):** R-12 → R-7 → R-3+R-4 → R-9+R-2 → R-13 → R-8 → R-1/R-5/R-6/R-11. -**Executed so far:** R-7 (PR-3, PR-3b) · **R-12** (both ADR amendments + the §5 verdict banner + the §6 retirement-condition amendment) · **R-3 + R-4** (PR-5 — the flag register, the profile mechanism, the daemon's fourth flag) · **R-9 + R-2** (PR-11 — the soak's committed artifact, pinned digest, and change-triggered cadence) — all landed 2026-07-28. +**Executed so far:** R-7 (PR-3, PR-3b) · **R-12** (both ADR amendments + the §5 verdict banner + the §6 retirement-condition amendment) · **R-3 + R-4** (PR-5 — the flag register, the profile mechanism, the daemon's fourth flag) · **R-9 + R-2** (PR-11 — the soak's committed artifact, pinned digest, change-triggered cadence) · **R-13** (PR-12 — the Wilson re-count; 21 of 25 licences revoked) — all landed 2026-07-28. -**Remaining:** R-13 → R-8 → R-1/R-5/R-6/R-11. +**Remaining:** R-8 → R-1/R-5/R-6/R-11. --- @@ -324,7 +324,7 @@ Two flags' documentation describes a production profile the production profile d ## R-13 · Wilson re-count — authorize, knowing licenses may be demoted -**Status:** **RULED A** — 2026-07-28, Shay (explicit adoption) · *authorize the re-count; **served capability will shrink, and that is the mechanism working*** · **Register:** H-1 / G-19 · **Blocks:** PR-12, and PR-14 through it. +**Status:** **RULED A — EXECUTED 2026-07-28 (PR-12)** · *re-counted on distinct evidence: **25 licensed bands → 4**, 21 demoted to disclosed serving; `wrong` stayed 0 throughout. The producer now refuses to seal a padded ledger.* · **Register:** H-1 / G-19 · **Blocks:** PR-12, and PR-14 through it. **The question.** Wilson lower-bound licensing (θ_SERVE=0.99, ADR-0175 lineage) assumes independent trials. A replay of the same sealed case is one trial observed again, not a new one. Re-counting on a distinct-evidence basis will demote licenses. diff --git a/evals/deduction_serve/practice/runner.py b/evals/deduction_serve/practice/runner.py index 0b526b44..7c523c13 100644 --- a/evals/deduction_serve/practice/runner.py +++ b/evals/deduction_serve/practice/runner.py @@ -38,9 +38,89 @@ _SEALED_LEDGER_PATH = ( ) +def distinct_gold_problems() -> list[Any]: + """The practice corpus with replayed cases collapsed — one per distinct decision. + + **R-13, 2026-07-28.** Wilson lower-bound licensing (`conservative_floor`) is a + statement about *independent trials*. A replay of the same sealed case is one + trial observed again, not a new one, so folding the raw corpus inflated every + band: 720 "committed" decisions drawn from as few as **28** distinct cases, and + `conservative_floor(720, 720) = 0.990868` cleared θ_SERVE=0.99 on evidence that + `conservative_floor(28, 28) = 0.808413` does not come close to supporting. + + The key is the case **text**, which is the definition already agreed in-repo by + ``tests/test_volume_honesty.py`` and the audit behind it: two cases with identical + text are indisputably the same decision. It deliberately *under*-reports — a + tighter key (the normalized atom tuple) would collapse spelling variants and find + more inflation — because under-reporting makes the measured gap a **floor** on the + real gap rather than a guess at it. + + The raw corpus is left alone: it is the practice *material*, and running a case + twice is a legitimate thing for practice to do. What may not happen is a replay + being *counted* as new evidence when a license is computed from the count. + """ + seen: set[tuple[str, str]] = set() + out: list[Any] = [] + for problem in all_gold_problems(): + key = (problem.class_name, problem.payload["text"]) + if key in seen: + continue + seen.add(key) + out.append(problem) + return out + + +def assert_sealed_evidence_distinct(ledger: dict[str, ClassTally]) -> None: + """The R9 invariant on the LEDGER ABOUT TO BE SEALED: a padded producer cannot seal. + + Mirrors ``evals.curriculum_serve.practice.runner.assert_practice_atoms_distinct``, + which has carried this guarantee on the sibling ledger since ADR-0264 R9. The + deduction sealer had no equivalent, which is precisely how 21 of 25 bands came to + hold SERVE licences their evidence never supported. + + Catching it *here* rather than in the audit matters: an audit finds a padded ledger + after it is committed and trusted, and unwinding that exposure then needs a ruling + (it needed one — R-13). This raises before the artifact exists. + + **It takes the built ledger, and that is the whole point.** The first version of + this guard compared ``all_gold_problems()`` to ``distinct_gold_problems()`` — two + functions that agree with each other by construction, and neither of which is what + ``build_ledger`` folds. Reverting ``build_ledger`` to the raw corpus sailed straight + past it and re-sealed the inflated artifact. A guard that checks something + *adjacent* to the thing it protects is the exact failure this ledger already + suffered once; it was caught here only because the guard was sabotage-tested rather + than trusted. The invariant is about the **artifact**, so the artifact is the input. + """ + distinct: dict[str, set[str]] = {} + for problem in all_gold_problems(): + distinct.setdefault(problem.class_name, set()).add(problem.payload["text"]) + + padded = { + band: (tally.committed, len(distinct.get(band, ()))) + for band, tally in ledger.items() + if tally.committed != len(distinct.get(band, ())) + } + if padded: + detail = ", ".join( + f"{band}: ledger committed={got} but only {want} distinct cases exist" + for band, (got, want) in sorted(padded.items()) + ) + raise ValueError( + "refusing to seal — the ledger counts replays as independent trials, which " + "is the precondition conservative_floor assumes and these bands violate: " + f"{detail}" + ) + + def build_ledger() -> dict[str, ClassTally]: - """Run sealed practice over the synthetic corpus → per-band ledger.""" - report = run_practice(all_gold_problems(), DeductionSolver(), ConstructionGoldTether()) + """Run sealed practice over the corpus → per-band ledger, on DISTINCT evidence. + + ``committed == distinct`` by construction, which is the same guarantee the + curriculum ledger states in its own seal note. Before R-13 this folded the raw + corpus and the difference was invisible in the artifact: a band recorded 720/720 + whether that was 720 decisions or 28 decisions replayed. + """ + report = run_practice(distinct_gold_problems(), DeductionSolver(), ConstructionGoldTether()) return dict(report.ledger) @@ -73,7 +153,7 @@ def run(ceilings: Ceilings | None = None) -> dict[str, Any]: } -def build_sealed_artifact() -> dict[str, Any]: +def build_sealed_artifact(ledger: dict[str, ClassTally] | None = None) -> dict[str, Any]: """The committed sealed-ledger dict (self-verifying ``content_sha256``). Formatting and hashing come from the shared bridge (ADR-0263) — byte @@ -81,23 +161,37 @@ def build_sealed_artifact() -> dict[str, Any]: the refactor is proven safe: re-sealing must not move the artifact. """ return seal_artifact( - build_ledger(), + build_ledger() if ledger is None else ledger, schema="deduction_serve_ledger_v1", note=( "Sealed-practice committed ledger for deduction serving (ADR-0256). " "Engine reads, never writes. Ceilings stay at safe defaults " "(theta_SERVE=0.99). A band earns SERVE by demonstrated pipeline " - "reliability (reader+projector+engine) at volume >= 657 committed." + "reliability (reader+projector+engine) at volume >= 657 committed. " + "RE-COUNTED 2026-07-28 under R-13: one committed case per DISTINCT " + "decision (case text), so committed == distinct evidence by construction. " + "The prior seal folded the raw corpus and counted replays as independent " + "trials, which the Wilson floor assumes they are not; 21 of 25 bands held " + "SERVE licences their evidence never supported and are demoted here." ), provenance="evals.deduction_serve.practice.runner.seal_ledger", ) def seal_ledger(path: Path = _SEALED_LEDGER_PATH) -> dict[str, Any]: - """Regenerate + write the committed sealed ledger. Verifies corpus soundness - against the independent oracle first (a mis-stated gold can never seal).""" + """Regenerate + write the committed sealed ledger. + + Two preconditions, both checked before anything is written: + + * ``assert_practice_gold_sound()`` — a mis-stated gold can never seal. + * ``assert_sealed_evidence_distinct(ledger)`` — a producer counting replays as + independent trials can never seal (R-13 / ADR-0264 R9). It is handed the built + ledger rather than the corpus, because the corpus is not what gets sealed. + """ assert_practice_gold_sound() - return write_sealed_ledger(path, build_sealed_artifact()) + ledger = build_ledger() + assert_sealed_evidence_distinct(ledger) + return write_sealed_ledger(path, build_sealed_artifact(ledger)) def main(argv: list[str] | None = None) -> int: diff --git a/scripts/generate_claims.py b/scripts/generate_claims.py index 4c8284a4..6b468fef 100644 --- a/scripts/generate_claims.py +++ b/scripts/generate_claims.py @@ -80,7 +80,7 @@ _LANE_ADR: dict[str, tuple[str, str]] = { ), "deduction_serve_v1": ( "ADR-0256", - "Flag-gated deduction serving decides real English/member/fused/verb/existential arguments end-to-end under earned SERVE licenses; wrong=0 across all splits", + "Flag-gated deduction serving decides real English/member/fused/verb/existential arguments end-to-end, wrong=0 across all splits; 4 of 25 shape-bands hold an earned SERVE licence on DISTINCT evidence (R-13 re-count, 2026-07-28) and the other 21 decide with disclosure", ), } diff --git a/tests/test_construction_inventory.py b/tests/test_construction_inventory.py index 779b324d..9b026842 100644 --- a/tests/test_construction_inventory.py +++ b/tests/test_construction_inventory.py @@ -150,14 +150,36 @@ def test_fabricated_premise_reaches_a_served_deduction_surface() -> None: The verdict is unaffected here, but the *recital* is: CORE tells the user they said ``furthermore``. Revise only by removing the fabrication. + + **Updated 2026-07-28 (R-13), and deliberately not weakened.** The re-count + demoted ``conditional_single``, so both surfaces now carry the unverified-shape + disclosure. The prefix is licensing, not comprehension: it says *"I have not + earned a track record on this argument shape"*, which is a true statement about + evidence volume and says nothing about whether the recited premises are real. + **The fabrication survives the demotion untouched** — that is the finding, and it + is sharper now than before: a disclosure prefix might look like it covers the + hazard, and it does not. The user is still told they said ``furthermore``. + + The assertion therefore stays exact-match on the full string, including the + prefix, so neither the fabrication nor the licence state can drift unnoticed. """ from chat.deduction_surface import deduction_grounded_surface + disclosure = ( + "(reasoned, but I haven't yet earned a verified track record on " + "arguments of this shape) " + ) clean = deduction_grounded_surface("if p then q. p. therefore q.") leaked = deduction_grounded_surface("furthermore, if p then q. p. therefore q.") - assert clean == "Given: p implies q; p. Your premises entail: q." - assert leaked == "Given: furthermore; p implies q; p. Your premises entail: q." + assert clean == disclosure + "Given: p implies q; p. Your premises entail: q." + assert leaked == disclosure + "Given: furthermore; p implies q; p. Your premises entail: q." + + # The defect, isolated from the licence state: the ONLY difference between a + # clean and a leaked recital is the fabricated premise. If this ever passes + # because both surfaces became refusals, the exact-match assertions above fail + # first — which is why they are kept rather than replaced by this. + assert leaked.replace("furthermore; ", "") == clean def test_aspect_arms_drop_negation() -> None: diff --git a/tests/test_deduction_serve_license.py b/tests/test_deduction_serve_license.py index d9faf2ce..c326d1e8 100644 --- a/tests/test_deduction_serve_license.py +++ b/tests/test_deduction_serve_license.py @@ -78,17 +78,60 @@ def test_corpus_is_sound_against_independent_oracle() -> None: assert_practice_gold_sound() # raises AssertionError on any mis-stated gold -def test_every_band_earns_serve_wrong_zero() -> None: +#: The four bands whose DISTINCT evidence clears theta_SERVE=0.99 after the R-13 +#: re-count. Derived from the audit, not chosen: they are the only bands whose +#: corpus contains enough *independent* cases. The other 21 held licences that +#: replay volume manufactured — 720 committed from as few as 28 distinct cases. +SERVE_LICENSED_BANDS = frozenset( + {"en_conditional_chain", "en_disjunctive", "en_verb_fact", "en_verb_universal"} +) + + +def test_practice_is_wrong_zero_on_distinct_evidence() -> None: + """``wrong=0`` survives the re-count — it was never the thing at issue. + + R-13 demoted 21 of 25 bands, and it is worth being exact about what that did + and did not mean: **no answer changed and no answer became wrong.** The engine + is exactly as correct as it was. What changed is the *claim* the ledger makes + about how much independent evidence backs each band, and therefore which bands + may answer authoritatively rather than with disclosure. + """ from evals.deduction_serve.practice.runner import run report = run() assert report["wrong_is_zero"] is True - assert report["all_bands_serve_licensed"] is True assert set(report["classes"]) == set(ALL_SHAPE_BANDS) for band, c in report["classes"].items(): assert c["wrong"] == 0, band - assert c["serve_licensed"] is True, band - assert c["reliability"] >= 0.99, (band, c["reliability"]) + + +def test_only_bands_with_distinct_evidence_earn_serve() -> None: + """The demotion, pinned in BOTH directions (R-13). + + Down: a band outside :data:`SERVE_LICENSED_BANDS` must not be licensed — that is + the exposure this ruling closed. Up: a band inside it must stay licensed — so a + future change cannot quietly revoke earned capability either. + """ + from evals.deduction_serve.practice.runner import run + + report = run() + licensed = {b for b, c in report["classes"].items() if c["serve_licensed"]} + assert licensed == set(SERVE_LICENSED_BANDS), ( + "the SERVE-licensed set moved. Down = capability lost; up = a band earned a " + f"licence. Either is a reviewed decision.\n now: {sorted(licensed)}\n" + f" pinned: {sorted(SERVE_LICENSED_BANDS)}" + ) + assert report["all_bands_serve_licensed"] is False, ( + "all 25 bands are licensed again — the producer is counting replays as " + "independent trials, which is exactly what R-13 removed" + ) + for band, c in report["classes"].items(): + expected = band in SERVE_LICENSED_BANDS + assert c["serve_licensed"] is expected, (band, c["reliability"]) + if expected: + assert c["reliability"] >= 0.99, (band, c["reliability"]) + else: + assert c["reliability"] < 0.99, (band, c["reliability"]) # --------------------------------------------------------------------------- @@ -96,13 +139,23 @@ def test_every_band_earns_serve_wrong_zero() -> None: # --------------------------------------------------------------------------- -def test_committed_ledger_verifies_and_earns_serve() -> None: +def test_committed_ledger_verifies_and_matches_the_recount() -> None: + """The committed artifact must agree with the re-counted licence set. + + All 25 bands stay *in* the ledger — a demoted band is not a deleted band. It + keeps its evidence on the record and serves with disclosure, which is a + stronger honesty position than vanishing: the reader can see exactly how much + evidence the band has and why it is not enough. + """ ledger = load_ratified_ledger() assert set(ledger) == set(ALL_SHAPE_BANDS) ceilings = Ceilings.default() + licensed = set() for band, tally in ledger.items(): assert tally.wrong == 0, band - assert license_for(tally, Action.SERVE, ceilings).licensed is True, band + if license_for(tally, Action.SERVE, ceilings).licensed: + licensed.add(band) + assert licensed == set(SERVE_LICENSED_BANDS), sorted(licensed) def test_serve_license_returns_none_for_unknown_band() -> None: @@ -141,14 +194,36 @@ def test_tampered_ledger_is_rejected(tmp_path, monkeypatch) -> None: def test_earned_band_serves_authoritatively() -> None: - """With the committed (earned) ledger, an in-band argument is served - plainly — no hedge (Phase 1 behavior preserved).""" - surface = deduction_grounded_surface("If p then q. p. Therefore q.") + """An band that DID earn its licence on distinct evidence still serves plainly. + + ``en_conditional_chain`` is one of the four survivors (720 committed, 720 + distinct — no inflation at all). This is the direction that proves R-13 removed + unearned licences rather than simply breaking authoritative serving. + """ + surface = deduction_grounded_surface( + "If the kettle boils then the whistle sounds. " + "If the whistle sounds then the cat wakes. " + "The kettle boils. Therefore the cat wakes." + ) assert surface is not None assert surface.startswith("Given:") assert _UNVERIFIED_SHAPE_DISCLOSURE not in surface +def test_demoted_band_still_answers_but_discloses() -> None: + """The demotion changes the CLAIM, not the answer — the load-bearing half of R-13. + + ``conditional_single`` was licensed on 720 committed decisions drawn from 294 + distinct cases. After the re-count it is not licensed, and the same sound + conclusion is served with the unverified-shape disclosure. The reasoning did not + get worse; the boast did. + """ + surface = deduction_grounded_surface("If p then q. p. Therefore q.") + assert surface is not None + assert surface.startswith(_UNVERIFIED_SHAPE_DISCLOSURE) + assert "Your premises entail: q" in surface + + def test_unearned_band_is_disclosed_not_committed() -> None: """Strip the ledger (inject an empty lookup) and the SAME sound answer is served DISCLOSED — proving the capability is earned, not merely flagged.""" @@ -158,3 +233,44 @@ def test_unearned_band_is_disclosed_not_committed() -> None: assert surface is not None assert surface.startswith(_UNVERIFIED_SHAPE_DISCLOSURE) assert "Your premises entail: q" in surface # the sound answer still served + + +# --------------------------------------------------------------------------- +# R-13 — the seal-time guard that makes the re-count unrepeatable +# --------------------------------------------------------------------------- + + +def test_sealer_refuses_a_ledger_that_counts_replays() -> None: + """A padded ledger can never be sealed again (R-13 / ADR-0264 R9). + + The re-count fixed the artifact. This fixes the *producer*, which is the half + that stops it recurring: ``seal_ledger`` refuses before writing anything. + + Worth recording how this assertion came to exist. The first version of the guard + compared ``all_gold_problems()`` with ``distinct_gold_problems()`` — two functions + that agree by construction and neither of which is what ``build_ledger`` folds. + Reverting ``build_ledger`` to the raw corpus walked straight past it and re-sealed + the inflated artifact. It was caught only because the guard was sabotage-tested + instead of trusted, which is why the guard now takes the **built ledger** as input: + the invariant is about the artifact, so the artifact is what it reads. + """ + from core.reliability_gate import ClassTally + from evals.deduction_serve.practice.runner import assert_sealed_evidence_distinct + + from evals.deduction_serve.practice.runner import build_ledger + + honest = build_ledger() + assert_sealed_evidence_distinct(honest) # the real ledger passes + + padded = dict(honest) + band = "atomic" + padded[band] = ClassTally( + class_name=band, + correct=720, + wrong=0, + refused=honest[band].refused, + t2_verified=honest[band].t2_verified, + t2_agrees_gold=honest[band].t2_agrees_gold, + ) + with pytest.raises(ValueError, match="counts replays as independent trials"): + assert_sealed_evidence_distinct(padded) diff --git a/tests/test_deduction_surface.py b/tests/test_deduction_surface.py index a8ce5105..85633052 100644 --- a/tests/test_deduction_surface.py +++ b/tests/test_deduction_surface.py @@ -123,7 +123,13 @@ def test_multiword_conditional_is_decided_band_v2_en() -> None: ) assert surface is not None assert "Your premises entail: the ground is wet" in surface - assert not surface.startswith(_UNVERIFIED_SHAPE_DISCLOSURE) + # R-13 (2026-07-28): this band was DEMOTED by the distinct-evidence re-count + # — en_conditional_single (474 distinct of 720 committed). The shape is still DECIDED, which is what this + # test is actually about; what changed is the claim attached to the answer. + # The engine now discloses that it has not earned a track record on this + # shape instead of answering authoritatively. Asserting the old behaviour + # here would re-assert a licence the evidence never supported. + assert surface.startswith(_UNVERIFIED_SHAPE_DISCLOSURE) def test_contraposition_is_decided_band_v2_en() -> None: @@ -185,7 +191,13 @@ def test_membership_syllogism_is_decided_band_v3_mem() -> None: ) assert surface is not None assert "Your premises entail: socrates is mortal" in surface - assert not surface.startswith(_UNVERIFIED_SHAPE_DISCLOSURE) + # R-13 (2026-07-28): this band was DEMOTED by the distinct-evidence re-count + # — en_member_single (60 distinct of 720 committed). The shape is still DECIDED, which is what this + # test is actually about; what changed is the claim attached to the answer. + # The engine now discloses that it has not earned a track record on this + # shape instead of answering authoritatively. Asserting the old behaviour + # here would re-assert a licence the evidence never supported. + assert surface.startswith(_UNVERIFIED_SHAPE_DISCLOSURE) def test_member_negative_universal_is_decided() -> None: @@ -243,7 +255,13 @@ def test_conditional_membership_fusion_is_decided_band_v4_cm() -> None: ) assert surface is not None assert "Your premises entail: socrates is mortal" in surface - assert not surface.startswith(_UNVERIFIED_SHAPE_DISCLOSURE) + # R-13 (2026-07-28): this band was DEMOTED by the distinct-evidence re-count + # — en_condmem_conditional (60 distinct of 720 committed). The shape is still DECIDED, which is what this + # test is actually about; what changed is the claim attached to the answer. + # The engine now discloses that it has not earned a track record on this + # shape instead of answering authoritatively. Asserting the old behaviour + # here would re-assert a licence the evidence never supported. + assert surface.startswith(_UNVERIFIED_SHAPE_DISCLOSURE) def test_conditional_membership_fusion_genuinely_fuses() -> None: @@ -450,7 +468,13 @@ def test_existential_is_decided_band_v6_ex() -> None: ) assert surface is not None assert "Your premises entail: some whales are vertebrates" in surface - assert not surface.startswith(_UNVERIFIED_SHAPE_DISCLOSURE) + # R-13 (2026-07-28): this band was DEMOTED by the distinct-evidence re-count + # — en_exist_universal (46 distinct of 720 committed). The shape is still DECIDED, which is what this + # test is actually about; what changed is the claim attached to the answer. + # The engine now discloses that it has not earned a track record on this + # shape instead of answering authoritatively. Asserting the old behaviour + # here would re-assert a licence the evidence never supported. + assert surface.startswith(_UNVERIFIED_SHAPE_DISCLOSURE) def test_existential_conclusion_does_not_close_the_domain() -> None: diff --git a/tests/test_volume_honesty.py b/tests/test_volume_honesty.py index b76e95bc..84e18ff7 100644 --- a/tests/test_volume_honesty.py +++ b/tests/test_volume_honesty.py @@ -21,10 +21,15 @@ Three producers back the three entries in `core.ratified_ledger.CAPABILITY_LEDGE record needs, on distinct cases. This is the standard the other producers are measured against, and it is why the deduction finding is a regression from an established practice rather than a gap nobody had thought about. -- **deduction_serve** (ADR-0256) — 21 of 25 bands do not clear θ_SERVE on - distinct evidence; the worst three inflate 28 distinct cases to 720 committed. - Recorded, pinned, and NOT silently repaired: the ledger is SHA-sealed, ratified, - and gating a live flag, so changing it is Shay's ratification, not a test's. +- **deduction_serve** (ADR-0256) — 21 of 25 bands did not clear θ_SERVE on + distinct evidence; the worst three inflated 28 distinct cases to 720 committed. + For three days this was recorded, pinned, and deliberately NOT repaired, because + the ledger is SHA-sealed, ratified, and gates a live flag — changing it was Shay's + ratification, not a test's. **That ratification is R-13 (2026-07-28), and it has + been applied:** the ledger is re-sealed on distinct evidence, the 21 are demoted + and decide with disclosure, four hold earned licences, and the producer now + REFUSES to seal a padded ledger at all. The audit below no longer measures an + exposure; it pins the repair. - **curriculum_serve** (ADR-0262/0264) — clean, and clean *structurally*: the producer's case identity IS the query atom, so ``committed == distinct`` cannot drift apart the way it did for deduction, where case identity was a quota index. @@ -128,10 +133,15 @@ AUDIT_SOURCES: dict[str, Callable[[], dict[str, list[Hashable]]] | None] = { } -#: Measured 2026-07-25 at main @ `6ada6f7a`. Every band is 720 committed. -#: This is an EXPOSURE INVENTORY, not an approved baseline: 21 of these 25 do not -#: clear θ_SERVE=0.99 on distinct evidence. It is pinned so the exposure cannot -#: grow or drift unnoticed while the ratification question is open. +#: Distinct-evidence counts per band. Measured 2026-07-25 at main @ `6ada6f7a`, +#: when every band was 720 committed and this was an EXPOSURE INVENTORY: 21 of 25 +#: did not clear θ_SERVE=0.99 on the evidence below. +#: +#: **APPLIED 2026-07-28 (R-13).** It is no longer an exposure. The sealed ledger is +#: re-counted on exactly these numbers — committed == distinct — so the 21 are +#: demoted and decide with disclosure, and four hold earned licences. This list is +#: now the *expected* shape of the ledger rather than a measurement of a gap, and +#: `test_deduction_sealed_ledger_matches_the_producer_it_names` pins that equality. DEDUCTION_DISTINCT_EVIDENCE: dict[str, int] = { "atomic": 98, "categorical": 294, @@ -386,11 +396,25 @@ def test_deduction_sealed_ledger_matches_the_producer_it_names( ledger = json.loads(path.read_text(encoding="utf-8")) classes = ledger["classes"] assert ledger["provenance"] == "evals.deduction_serve.practice.runner.seal_ledger" - produced = {a.band: a.committed for a in deduction_audits} + # R-13 (2026-07-28): the sealed ledger now counts DISTINCT evidence, so this + # invariant is strictly stronger than it was. Before the re-count it compared the + # sealed committed counts to the producer's committed counts and both were the + # inflated 720 — the assertion held perfectly while the number it agreed on was + # the wrong number. Two records agreeing is not evidence that either is right. + # It now pins what the re-count actually guarantees: committed == distinct. + produced_distinct = {a.band: a.distinct for a in deduction_audits} + produced_raw = {a.band: a.committed for a in deduction_audits} sealed = {name: tally["correct"] + tally["wrong"] for name, tally in classes.items()} - assert sealed == produced, ( - "sealed ledger committed counts diverge from the producer — the audit " - "below would then be measuring something the engine does not read" + assert sealed == produced_distinct, ( + "sealed ledger committed counts diverge from the producer's DISTINCT counts " + "— either the seal is stale, or build_ledger stopped folding the " + "de-duplicated corpus and replays are being counted as evidence again" + ) + inflated = sorted(b for b in sealed if produced_raw[b] != produced_distinct[b]) + assert inflated, ( + "no band shows any raw-vs-distinct gap, so this assertion proves nothing " + "about de-duplication. Either the corpus lost its replays, or the audit has " + "gone blind — check before assuming the good news." ) assert all(tally["wrong"] == 0 for tally in classes.values()), "wrong=0 must hold"