From 71d6366fa378c0081d6777eb774544edb4b9831c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 28 Jul 2026 07:59:01 +0000 Subject: [PATCH] =?UTF-8?q?feat(ledger,tests,adr):=20PR-14=20=E2=80=94=20t?= =?UTF-8?q?he=20outcome-mix=20rule,=20and=20the=20four=20expected=20licenc?= =?UTF-8?q?es=20dissolve=20under=20it=20(R-8)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rank 6, the last docket item before the posture statements, and the throughput frontier N-5 called "one ruling wide". The ruling landed. It closed the frontier rather than opening it, and that is the correct outcome. R-8 ruled C: committing to an entailment and correctly declining to commit are DIFFERENT capabilities, licensed on DIFFERENT evidence, and may not be pooled. THE FINDING: split, ZERO bands license — not four. band unknown floor entailed floor philosophy_theology_contrast 652 0.989925 8 0.000000 philosophy_theology_modal 652 0.989925 8 0.000000 physics_causal 653 0.989940 7 0.000000 systems_software_causal 653 0.989940 7 0.000000 (pooled, the old basis) 660 0.990046 -- clears by 0.000046 N-5 recorded that "four bands would earn SERVE the moment a ledger is sealed." That is true on the POOLED basis and false on the ruled one. Those four cleared theta_SERVE=0.99 only because 7-8 entailments were counted alongside 652-653 correct refusals to reach 660. The licence was manufactured by the pooling, not earned by the evidence. conservative_floor(9,9) is 0.000000 — the Wilson lower bound at nine trials is not merely below theta, it is zero. This is exactly why the plan blocked PR-14 on R-8 instead of sealing first and ruling after. A ledger sealed on the pooled basis would have granted four licences that the mix rule then had to revoke — and revoking a granted licence is the expensive direction, as PR-12 had just demonstrated across 21 deduction bands in this same session. DELEGATED RULING — R-8 C's entailed floor N: no new constant. The packet recommended "C, with a floor from A applied to the entailed capability only," and N was never named. It does not need naming. theta_SERVE =0.99 through conservative_floor — the bar every other capability already meets, 657 distinct correct decisions — applied to each separated capability on its own evidence licenses nothing, by a factor of 73. Inventing a second, weaker constant for the capability that most needs the strong one would institutionalise two standards, which is precisely the objection that sank option B. Recorded under the standing delegation with its reasoning, so it can be overturned. NO LEDGER IS SEALED, AND THAT IS THE DELIVERABLE. Under the rule there is nothing to license, and the registered missing_ok=True absence already says so once. An artifact whose only content is its own emptiness would be a second statement of the same fact — the defect class registered as G-23 this same week. THE USEFUL RESULT IS THE SHAPE OF THE GAP, WHICH POOLING HAD HIDDEN. Non-commitment serving is FOUR TO FIVE distinct query atoms short per band. Entailed serving is ~648 short. Those are content tasks of completely different size, and a single pooled figure could not distinguish "almost there" from "two orders of magnitude away". Four cases is a morning's work; 648 is a program. Nobody could see that before the split. Delivered: - curriculum_serve_entailed registered in CAPABILITY_LEDGERS, missing_ok=True (absent = nothing licensed, the honest state), with the rule declared in the manifest table rather than at a call site (ADR-0263 rule 5) - an audit source for it — demanded immediately by the manifest's own pin (test_every_licensed_capability_has_an_audit_source went red the moment the capability was declared, which is PR-5's declared-table discipline working) - ADR-0264 §5 amendment recording the ruling and correcting §4.1's "four bands would earn SERVE" expectation. Changes no decision in that ADR; curriculum_serving_enabled stays False - tests/test_curriculum_outcome_mix.py on the gate: both capabilities pinned, the shortfall pinned exactly, AND the counterfactual pinned — pooling licenses 4 — so the argument for the rule cannot drift away from the number it rests on The serving path needs no change, and that is stated rather than left implicit: under the rule nothing is licensed, so there is no licensed-vs-disclosed branch to route. Building that machinery now would be building for a case that cannot occur yet. Closes G-10. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Wcw2pnMBwyvmNyQg4uPEt4 --- core/cli_test.py | 8 + core/ratified_ledger.py | 21 ++- ...4-negative-curriculum-and-premise-scope.md | 29 ++++ docs/assessment/50-execution-plan.md | 18 +- docs/assessment/50-rulings.md | 8 +- tests/test_curriculum_outcome_mix.py | 161 ++++++++++++++++++ tests/test_volume_honesty.py | 24 +++ 7 files changed, 261 insertions(+), 8 deletions(-) create mode 100644 tests/test_curriculum_outcome_mix.py diff --git a/core/cli_test.py b/core/cli_test.py index 1a5c856a..471b4696 100644 --- a/core/cli_test.py +++ b/core/cli_test.py @@ -163,6 +163,14 @@ TEST_SUITES: dict[str, tuple[str, ...]] = { # weeks: the 2026-07-19 soak ran without accrue_realized_knowledge in the # daemon profile, and nothing anywhere could notice. Had this pin existed, # PR-5 would have failed it, which is exactly right. <1s. + # PR-14 / R-8 (ruled C) / G-10 — the curriculum OUTCOME-MIX rule. Committing + # to an entailment and correctly declining to are different capabilities and + # may not be pooled. Pooled, four bands cleared theta_SERVE by 0.000046 on + # 652-653 correct UNKNOWNs plus 7-8 entailments; split, NEITHER half clears. + # Pins both directions plus the counterfactual, so the rule cannot become + # decorative and the moment a band genuinely earns a licence is a reviewed + # decision rather than a drift. ~6s (runs the practice producer). + "tests/test_curriculum_outcome_mix.py", "tests/test_l10_soak_evidence.py", # PR-11 / R-9 — the H1-H4 holds+bites pins, promoted onto the gate. # Measured 20.95s for the four real-soak "holds" and 0.46s for the eight diff --git a/core/ratified_ledger.py b/core/ratified_ledger.py index ad4d598e..fe14af0b 100644 --- a/core/ratified_ledger.py +++ b/core/ratified_ledger.py @@ -109,7 +109,26 @@ CAPABILITY_LEDGERS: dict[str, LedgerSpec] = { note=( "ADR-0262 §5 — no band has earned a license from present curriculum " "volume; every served band is DISCLOSED. Flips to False when the " - "first band earns SERVE and the ledger is committed." + "first band earns SERVE and the ledger is committed. R-8 (ruled C, " + "2026-07-28) splits the earning basis: see 'curriculum_serve_entailed' " + "below. This entry now governs NON-COMMITMENT (correct UNKNOWN) " + "serving only." + ), + ), + "curriculum_serve_entailed": LedgerSpec( + capability="curriculum_serve_entailed", + path=_PROJECT_ROOT / "chat" / "data" / "curriculum_serve_entailed_ledger.json", + missing_ok=True, + note=( + "R-8, ruled C (2026-07-28) — the outcome-mix rule. Committing to an " + "entailment and correctly declining to are DIFFERENT capabilities and " + "are licensed on DIFFERENT evidence. Pooling them let a band clear " + "theta_SERVE=0.99 on correct refusals alone: the four leading bands " + "held 652-653 correct UNKNOWNs and 7-8 entailments, and only the " + "pooled 660 cleared. Split, NEITHER capability licenses anything " + "today — entailed evidence tops out at 9 (conservative_floor(9,9)=0.0 " + "against a required 657), and unknown evidence tops out at 653, four " + "short (five for two of the four bands). Absent = nothing licensed, which is the honest state." ), ), } diff --git a/docs/adr/ADR-0264-negative-curriculum-and-premise-scope.md b/docs/adr/ADR-0264-negative-curriculum-and-premise-scope.md index f898ce27..c71dd886 100644 --- a/docs/adr/ADR-0264-negative-curriculum-and-premise-scope.md +++ b/docs/adr/ADR-0264-negative-curriculum-and-premise-scope.md @@ -400,8 +400,37 @@ band is at its ceiling. ## 5. What this ADR does not decide +> ### Amendment — 2026-07-28: the mix floor is decided, and it decided nothing licensable +> +> *R-8, ruled **C** (`docs/assessment/50-rulings.md`), executed as PR-14. This amendment records a ruling; it reverses no decision in this ADR.* +> +> The first bullet below defers **the mix floor** to Phase B. It is now ruled, and the ruling is a *separation* rather than a number: +> +> **Committing to an entailment and correctly declining to commit are different capabilities, licensed on different evidence, and may not be pooled.** `CAPABILITY_LEDGERS` carries them as two entries — `curriculum_serve` (non-commitment) and `curriculum_serve_entailed` (commitment). +> +> **No new floor constant was introduced, deliberately.** The recommendation was *"C, with a floor from A applied to the entailed capability only,"* and the floor's value was never named. It does not need one: θ_SERVE=0.99 through `conservative_floor` — the bar every other capability already meets, **657** distinct correct decisions — applied to each separated capability on its own evidence is sufficient and is not close. A second, weaker constant for the capability that most needs the strong one would institutionalise two standards, which is the objection that sank option B. +> +> **Measured at the ruling (2026-07-28), and it overturns §4.1's expectation:** +> +> | | evidence | `conservative_floor` | vs 0.99 | +> |---|---|---|---| +> | entailed, best band | **9** | `0.000000` | fails by 648 cases | +> | non-commitment, best band | **653** | `0.989940` | fails by **4 cases** | +> | *pooled* (the old basis) | 660 | `0.990046` | *clears by 0.000046* | +> +> §4.1's banner says *"four bands would earn SERVE the moment a ledger is sealed."* That is true of the **pooled** basis and false of the ruled one. **Split, zero bands license.** The four leading bands cleared only because 7–8 entailments were counted alongside 652–653 correct refusals to reach 660; the licence was manufactured by the pooling, not by the evidence. +> +> **No ledger is sealed, and that is the deliverable.** Under the rule there is nothing to license, and the registered `missing_ok=True` absence already states that once. An artifact whose only content is its own emptiness would be a second statement of the same fact. +> +> **The useful result is the shape of the gap.** Non-commitment serving is **four to five distinct query atoms** short per band; entailed serving is ~648 short. Those are content tasks of completely different size, and the pooled figure could not distinguish *"almost there"* from *"two orders of magnitude away."* Pinned in both directions by `tests/test_curriculum_outcome_mix.py`, including the counterfactual — so the moment a band genuinely earns either licence, it is a reviewed decision rather than a drift. +> +> **`curriculum_serving_enabled` stays False** (last bullet of this section, unchanged). + - **The bound in R9.** The specific de-duplication tuple and mix floor are Phase B's deliverable. R9 fixes only that a bound must exist and be reported. + — **The mix floor is ruled; see the amendment above. The de-duplication tuple was + discharged separately: the curriculum producer's case identity IS the query atom, + so `committed == distinct` by construction.** - **Raising `MAX_PREMISE_SENTENCES`.** Rejected as an approach, not deferred: it is a shared honesty cap that the 25 deduction bands also decide under, raising it would move hash-pinned lanes, and a 219-premise argument is not honest at diff --git a/docs/assessment/50-execution-plan.md b/docs/assessment/50-execution-plan.md index d698a768..4a6c0c05 100644 --- a/docs/assessment/50-execution-plan.md +++ b/docs/assessment/50-execution-plan.md @@ -11,7 +11,7 @@ | **Track A — ADR-0252 §5 run to NO-GO**, criterion pre-registered, artifact + digest committed, **verdict ratified 2026-07-28** | PR-8 — a small ADR (H-3/G-20) · PR-10 — a refactor ADR (H-4) | | PR-4 membership + reachability ratchets · PR-6 three doctrine pins · PR-7 M2 trust table · PR-9 count-the-swallow · **PR-5 the flag register (R-3+R-4)** | PR-12 — an ADR **amendment** (R-13 authorizes it; the amendment is still owed) | | PR-3 + PR-3b (**R-7**, Wave 1) · **PR-11 the soak to committed evidence (R-9+R-2)** · H-13 · H-8e · G-22 · G-23 | Track B — the fabrication ADR (standing instruction, unchanged) | -| **Closed:** G-5, G-6, G-7, G-8, G-9, G-15, G-19, G-22, H-1, H-6, H-7, H-8 (**all five**), H-11, H-13 · **Answered:** G-1 · **Discharged:** H-10 · **Pinned:** G-23 | Track C — invention, deliberately last | +| **Closed:** G-5, G-6, G-7, G-8, G-9, G-10, G-15, G-19, G-22, H-1, H-6, H-7, H-8 (**all five**), H-11, H-13 · **Answered:** G-1 · **Discharged:** H-10 · **Pinned:** G-23 | Track C — invention, deliberately last | **New this arc, not in the original assessment:** **N-8** (the §5 experiment had already returned GO twice, on unmerged branches, both unsound), **N-9** (the gate "drift" was a recorded decision — PR-4's pin 1 **withdrawn** rather than built, R-14 dissolved), **G-21** (the math reader decides **1.0%** of `holdout_dev/v1`), **G-22** (`main` was red before the arc, and `CLAIMS.md` published a superseded evidence digest for two days), **H-13**, **H-14**, **H-8e**. @@ -219,7 +219,7 @@ The packet's original order was by register number. That is the wrong axis: it i | 3 | ~~**R-3 + R-4**~~ | PR-5 — the flag register, all 32 booleans classified, profiles as units, the bidirectional pin, `accrue_realized_knowledge` into the daemon profile | **M** | **EXECUTED 2026-07-28.** Closed **G-6, G-8, H-6, H-8(c), H-8(d)** — and **H-8 in full**. The N-6 docstring correction **dissolved**: the docstrings were right and the code was incomplete. **Wave 4's hard gate is lifted.** | | 4 | ~~**R-9 + R-2**~~ | PR-11 — the soak re-run at 5000 beats under the **corrected** profile, artifact committed, digest pinned, cadence enforced by a source-hash pin, H1–H4 on the gate | **M** | **EXECUTED 2026-07-28.** Closed **G-5**. R-2 landed in the contract *before* the re-run, as designed. The staleness pin **would have caught PR-5** — verified by sabotage. | | 5 | ~~**R-13**~~ | PR-12 — distinct-evidence counting at the seal boundary, the honest re-count, demotions applied, the producer hardened against re-padding | **L** | **EXECUTED 2026-07-28. 25 licensed bands → 4.** Served capability shrank by 21 bands and `wrong` stayed 0: no answer changed, only the claim attached to it. | -| 6 | **R-8** | PR-14 — the outcome-mix rule, then seal `curriculum_serve_ledger.json`. Four bands qualify on sealing (N-5) | **M** | Depends on PR-12's counting basis. This is the entire throughput frontier, one ruling wide. | +| 6 | ~~**R-8**~~ | PR-14 — the outcome-mix rule as two capabilities. **No ledger sealed: under the rule, zero bands license** | **M** | **EXECUTED 2026-07-28.** Overturns N-5 — the four bands cleared only on *pooled* evidence, by 0.000046. Closed **G-10**. | | 7 | **R-1 · R-5 · R-6 · R-11** | Posture statements: efferent-action deferral with entry criterion (G-12) · the identity-enforcement discrimination bar (G-11) · non-text ingest deferral with the falsification bench as its standard (G-17) · **R-11 B — measure before gating** | **S** each | All four are prose with a criterion. None blocks anything; leaving them silent is the cost. They go last because they are the only items where lateness is cheap. | **R-11's measurement is owed now, not later — and R-11 is the one adopted ruling that does not end in a decision.** Option B is *measure first, then re-ask*: an instrumentation PR (counts only, no behavior change) reporting how often the out-of-inventory reading path is taken on the practice corpora, after which **R-11 is put back to Shay with the number in hand.** So the docket owes two things here, not one: the measurement, and the second ruling. Until the number exists, R-11 has been ruled and nothing follows from it — which is a state worth naming, because a ruled-but-inert item is indistinguishable from a settled one at a glance. It lands with the R-1/R-5/R-6 batch at the latest and may land sooner, since the instrumentation is small and independent. @@ -362,7 +362,19 @@ Shipped as ADR-0264 R5, discharged 2026-07-26. The 16-premise cap does not exist **It also carries G-23's decision, because it is the same band.** `DOMAIN_PACKS` places `en_core_cognition_v1` and `en_core_meta_v1` in `philosophy_theology`; neither manifest declares a `domain_id`, so `domain_contract_predicates` P3 has never run on them. The gap is pinned (`tests/test_domain_pack_binding.py`, on the gate, four sabotages red) and deliberately not fixed: whether those two are *philosophy_theology domain packs* or merely *grouped under it for ledger accounting* decides what the license actually covers. Sealing that band's ledger without answering it licenses a domain whose membership is asserted in one place and denied by silence in the other. -**One sub-question the ruling leaves open, stated rather than assumed.** The packet's recommendation was *"C, with a floor from A applied to the entailed capability only"* — and **N was never named.** Adopting C adopts the floor in principle; the number is not derivable from the evidence, because the whole point is that no band is near it (max entailed volume = 9). This PR must therefore either carry a proposed N with its consequence measured, or land the two capabilities with the entailed one explicitly unlicensable-pending-N. It may not quietly ship without a floor — that would license entailment on 9 cases through an omission, which is the exact outcome C exists to prevent. +**LANDED 2026-07-28, and the result is the opposite of what this item predicted.** + +**Split, zero bands license — not four.** The four leading bands hold **652–653 correct UNKNOWNs** and **7–8 entailments**. Pooled they reach 660 and clear θ_SERVE by **0.000046**; separated, neither half clears. `conservative_floor(9,9)` is **0.000000** — the Wilson bound at nine trials is not merely below θ, it is zero. The licence N-5 predicted was manufactured entirely by counting correct refusals and correct commitments as one kind of evidence. + +**This is why the plan blocked PR-14 on R-8 rather than sealing first.** A ledger sealed on the pooled basis would have granted four licences the mix rule then had to revoke — and revoking a granted licence is the expensive direction, as PR-12 had just demonstrated across 21 deduction bands. + +**No ledger is sealed, and that is the deliverable.** Under the rule there is nothing to license; the registered `missing_ok=True` absence states that once already. An artifact whose only content is its own emptiness would be a second statement of the same fact. + +**The useful result is the *shape* of the gap, which pooling had hidden.** Non-commitment serving is **four to five distinct query atoms short per band**; entailed serving is **~648** short. Those are content tasks of completely different size, and a single pooled figure could not distinguish *"almost there"* from *"two orders of magnitude away."* Four cases is a morning's work; 648 is a program. + +**Delivered:** `curriculum_serve_entailed` registered in `CAPABILITY_LEDGERS` (`missing_ok=True` — absent means nothing licensed, which is the honest state); an audit source for it, which the manifest's own pin demanded the moment the capability was declared; the **ADR-0264 §5 amendment** recording the ruling and correcting §4.1's "four bands would earn SERVE" expectation; and `tests/test_curriculum_outcome_mix.py` on the gate, pinning both capabilities, the shortfall, **and the counterfactual** — so the argument for the rule cannot drift away from the number it rests on. + +**~~One sub-question the ruling leaves open~~ — RULED 2026-07-28 under the standing delegation: no new constant.** The packet's recommendation was *"C, with a floor from A applied to the entailed capability only"* — and **N was never named.** Adopting C adopts the floor in principle; the number is not derivable from the evidence, because the whole point is that no band is near it (max entailed volume = 9). This PR must therefore either carry a proposed N with its consequence measured, or land the two capabilities with the entailed one explicitly unlicensable-pending-N. It may not quietly ship without a floor — that would license entailment on 9 cases through an omission, which is the exact outcome C exists to prevent. ### Track B · The fabrications and the widening — *gated, see §6* G-2's fix lands only under its ratified ADR (R-11 may add an interim defensive gate first). G-3's widening program is **shape-dependent on Track A's verdict** and does not start before it. diff --git a/docs/assessment/50-rulings.md b/docs/assessment/50-rulings.md index ed80dc59..b23c5bb8 100644 --- a/docs/assessment/50-rulings.md +++ b/docs/assessment/50-rulings.md @@ -9,9 +9,9 @@ **Also ratified:** the ADR-0252 §5 **NO-GO** verdict. **Execution order (corrected):** R-12 → R-7 → R-3+R-4 → R-9+R-2 → R-13 → R-8 → R-1/R-5/R-6/R-11. -**Executed so far:** R-7 (PR-3, PR-3b) · **R-12** (both ADR amendments + the §5 verdict banner + the §6 retirement-condition amendment) · **R-3 + R-4** (PR-5 — the flag register, the profile mechanism, the daemon's fourth flag) · **R-9 + R-2** (PR-11 — the soak's committed artifact, pinned digest, change-triggered cadence) · **R-13** (PR-12 — the Wilson re-count; 21 of 25 licences revoked) — all landed 2026-07-28. +**Executed so far:** R-7 (PR-3, PR-3b) · **R-12** (both ADR amendments + the §5 verdict banner + the §6 retirement-condition amendment) · **R-3 + R-4** (PR-5 — the flag register, the profile mechanism, the daemon's fourth flag) · **R-9 + R-2** (PR-11 — the soak's committed artifact, pinned digest, change-triggered cadence) · **R-13** (PR-12 — the Wilson re-count; 21 of 25 licences revoked) · **R-8** (PR-14 — the outcome-mix rule; the four expected licences dissolve under it) — all landed 2026-07-28. -**Remaining:** R-8 → R-1/R-5/R-6/R-11. +**Remaining:** R-1/R-5/R-6/R-11 (the four posture statements) + R-11's measurement and its second ruling. --- @@ -19,7 +19,7 @@ > *"If you think you might need me for a ruling, I instruct you to handle it based on the most masterful logic accordingly. No more pauses."* -**Scope.** The residual decisions inside the adopted docket — the sub-questions the twelve rulings left open, and any new question of the same size that arises while executing them. Concretely and by name: **R-8 C's entailed-capability floor `N`** (never specified when C was adopted); **R-11's second ruling** (option B ends in *re-ask*, not a decision); **PR-4b's gate-width choice**; **G-23's domain-membership question**; and the sequencing latitude to reorder the docket where evidence warrants. +**Scope.** The residual decisions inside the adopted docket — the sub-questions the twelve rulings left open, and any new question of the same size that arises while executing them. Concretely and by name: ~~**R-8 C's entailed-capability floor `N`**~~ (**ruled 2026-07-28: no new constant** — the existing θ_SERVE bar, applied to each separated capability on its own evidence, licenses nothing by a factor of 73; a second weaker constant for the capability that most needs the strong one would institutionalise the two standards option B was rejected for); **R-11's second ruling** (option B ends in *re-ask*, not a decision); **PR-4b's gate-width choice**; **G-23's domain-membership question**; and the sequencing latitude to reorder the docket where evidence warrants. **What the delegation does not do, and this matters more than what it does.** @@ -188,7 +188,7 @@ Two flags' documentation describes a production profile the production profile d ## R-8 · The curriculum outcome-mix rule — *now the sole blocker of an entire frontier* -**Status:** **RULED C** — 2026-07-28, Shay (explicit adoption) · *separate `unknown`- and `entailed`-serving licenses* · **Register:** G-10 · **Blocks:** PR-14, Track E in full. +**Status:** **RULED C — EXECUTED 2026-07-28 (PR-14)** · *`unknown`- and `entailed`-serving separated as two capabilities. **Split, ZERO bands license — not four.** The floor `N` was ruled unnecessary (DELEGATED): the existing θ_SERVE bar suffices and is not close.* · **Register:** G-10 · **Blocks:** PR-14, Track E in full. **The question changed shape since the register was written (N-5).** G-10 named the 16-premise compilation cap as the blocking engineering item. ADR-0264 §4.1 carries a banner directly beneath its own heading: diff --git a/tests/test_curriculum_outcome_mix.py b/tests/test_curriculum_outcome_mix.py new file mode 100644 index 00000000..dbbe434b --- /dev/null +++ b/tests/test_curriculum_outcome_mix.py @@ -0,0 +1,161 @@ +"""R-8 (ruled C) / G-10 — the curriculum outcome-mix rule, and what it measures. + +**The rule.** Committing to an entailment and correctly declining to commit are +different capabilities. They are licensed on different evidence and may not be +pooled. ``core.ratified_ledger.CAPABILITY_LEDGERS`` declares them as two entries; +this module measures them. + +**Why the rule was needed, in one number.** Pooled, the four leading bands reach +660 committed decisions and ``conservative_floor(660, 660) = 0.990046``, clearing +θ_SERVE=0.99 — by a margin of **0.000046**. Split, the same bands hold **652–653 +correct UNKNOWNs** and **7–8 entailments**. Neither half clears. The licence was +manufactured entirely by counting correct refusals and correct commitments as one +kind of evidence. + +**This overturns the plan's own prediction.** N-5 recorded that *"four bands would +earn SERVE the moment a ledger is sealed."* That is true on the pooled basis and +false on the ruled one. Sealing under R-8 C licenses **nothing** — which is exactly +why the plan blocked PR-14 on this ruling rather than sealing first and ruling +after. A ledger sealed on the pooled basis would have granted four licences that the +mix rule then had to revoke, and revoking a granted licence is the expensive +direction (see R-13 and the 21 deduction bands). + +**The shortfall is small and exact, and that is the useful part.** Unknown-serving +needs **four to five** more distinct query atoms per band (657 − 653 for two bands, +657 − 652 for the other two). Entailed-serving needs roughly **648** more. Those are +content tasks of wildly different size, and before the split nobody could see that — +the pooled number hid a four-case gap behind a 648-case one. + +This module deliberately does **not** seal a ledger. Under the rule there is nothing +to license, and an artifact asserting zero licences is a file whose only content is +its own emptiness; the registered ``missing_ok=True`` absence already says it, once. +""" + +from __future__ import annotations + +import pytest + +from core.ratified_ledger import CAPABILITY_LEDGERS +from core.reliability_gate import conservative_floor +from core.reliability_gate.evidence import SERVE_VOLUME_AT_THETA_099 + +THETA_SERVE = 0.99 + + +@pytest.fixture(scope="module") +def mix() -> dict[str, dict[str, int]]: + """Per-band ``{'entailed': n, 'unknown': n}`` from the practice producer.""" + from evals.curriculum_serve.practice.runner import run + + report = run() + return {band: dict(c["gold_mix"]) for band, c in report["classes"].items()} + + +def test_both_capabilities_are_registered() -> None: + """The split is declared in the manifest, not asserted at a call site. + + ADR-0263 rule 5. If this fails, the outcome-mix rule exists only in prose — + which is the state R-8 was raised to end. + """ + assert "curriculum_serve" in CAPABILITY_LEDGERS + assert "curriculum_serve_entailed" in CAPABILITY_LEDGERS + assert CAPABILITY_LEDGERS["curriculum_serve_entailed"].missing_ok is True, ( + "absent must mean 'nothing licensed' for the entailed capability — a " + "missing_ok=False entry would turn an honest absence into a broken deployment" + ) + + +def test_the_mix_is_not_vacuous(mix: dict[str, dict[str, int]]) -> None: + """Guard the measurement before asserting anything about it.""" + assert len(mix) >= 10, f"parsed only {len(mix)} bands from the producer" + assert any(m.get("entailed", 0) for m in mix.values()), "no entailed cases parsed" + assert any(m.get("unknown", 0) for m in mix.values()), "no unknown cases parsed" + + +def test_no_band_licenses_entailed_serving(mix: dict[str, dict[str, int]]) -> None: + """The committing capability, on its own evidence: nothing qualifies. + + ``conservative_floor(9, 9) == 0.0`` — the Wilson lower bound at nine trials is + not merely below θ, it is zero. This is the assertion that stops an entailment + licence being granted on a handful of cases, which is what pooling did. + """ + offenders = { + band: n + for band, m in mix.items() + if (n := m.get("entailed", 0)) and conservative_floor(n, n) >= THETA_SERVE + } + assert not offenders, ( + f"these bands now hold enough entailed evidence to license: {offenders}. " + "That is good news and a reviewed decision — seal the entailed ledger and " + "move this pin." + ) + best = max(m.get("entailed", 0) for m in mix.values()) + assert best < SERVE_VOLUME_AT_THETA_099, ( + f"max entailed evidence is {best}, at or above the {SERVE_VOLUME_AT_THETA_099} " + "θ_SERVE requires — the entailed capability is now earnable" + ) + + +def test_no_band_licenses_unknown_serving_either(mix: dict[str, dict[str, int]]) -> None: + """The declining capability, on its OWN evidence: also nothing — the finding. + + This is the assertion that contradicts N-5. The four bands everyone expected to + license hold 652–653 correct UNKNOWNs against a 657 requirement: **four to five + short**. + They only ever cleared because 7–8 entailments were pooled in to reach 660. + """ + licensed = { + band: n + for band, m in mix.items() + if (n := m.get("unknown", 0)) and conservative_floor(n, n) >= THETA_SERVE + } + assert not licensed, ( + f"these bands now license unknown-serving on distinct non-commitments: " + f"{licensed}. Expected under the ruling once curriculum volume grows — seal " + "the ledger and move this pin, in the same change." + ) + + +def test_pooling_would_have_licensed_four_bands(mix: dict[str, dict[str, int]]) -> None: + """Pin the counterfactual, because it is the whole justification for the rule. + + If this ever fails it means pooling no longer manufactures a licence, and the + mix rule has become free rather than load-bearing — worth knowing either way. It + is asserted rather than written in prose so the number cannot quietly drift away + from the argument that rests on it. + """ + pooled = { + band: total + for band, m in mix.items() + if conservative_floor(total := sum(m.values()), total) >= THETA_SERVE + } + assert len(pooled) == 4, ( + f"pooled evidence licenses {len(pooled)} bands, not the 4 recorded when R-8 " + f"was ruled: {sorted(pooled)}" + ) + for band, total in pooled.items(): + entailed = mix[band].get("entailed", 0) + unknown = mix[band].get("unknown", 0) + assert conservative_floor(unknown, unknown) < THETA_SERVE, band + assert entailed < 10, (band, entailed) + assert total >= SERVE_VOLUME_AT_THETA_099, (band, total) + + +def test_the_shortfall_is_recorded_exactly(mix: dict[str, dict[str, int]]) -> None: + """Four or five cases, not "more volume" — the gap is small, and its size is the point. + + Before the split, the unknown shortfall (4-5) was hidden behind the entailed one + (~648). A single pooled figure could not distinguish "almost there" from + "two orders of magnitude away," and those call for completely different work. + """ + best_unknown = max(m.get("unknown", 0) for m in mix.values()) + best_entailed = max(m.get("entailed", 0) for m in mix.values()) + assert SERVE_VOLUME_AT_THETA_099 - best_unknown == 4, ( + f"the unknown-serving shortfall moved: best band has {best_unknown} distinct " + f"non-commitments against {SERVE_VOLUME_AT_THETA_099} required" + ) + assert SERVE_VOLUME_AT_THETA_099 - best_entailed > 600, ( + f"the entailed shortfall is now {SERVE_VOLUME_AT_THETA_099 - best_entailed}; " + "if it has closed to within a few hundred, the entailed capability is worth " + "re-planning rather than deferring" + ) diff --git a/tests/test_volume_honesty.py b/tests/test_volume_honesty.py index 84e18ff7..bca2c312 100644 --- a/tests/test_volume_honesty.py +++ b/tests/test_volume_honesty.py @@ -123,6 +123,29 @@ def _curriculum_keys() -> dict[str, list[Hashable]]: return out +def _curriculum_entailed_keys() -> dict[str, list[Hashable]]: + """Decision keys for the ENTAILED half of the curriculum corpus (R-8, ruled C). + + R-8 split the curriculum earning basis: committing to an entailment and + correctly declining to commit are different capabilities, licensed on different + evidence. This is the audit source for the committing half — the query atoms + whose oracle verdict is ``entailed``. + + It exists so the split cannot be declared in the manifest and left unmeasured. + The numbers it reports are small on purpose: 3-9 per band, against the 657 that + theta_SERVE=0.99 requires. That gap IS the finding. + """ + from evals.curriculum_serve.practice import all_gold_problems + from evals.curriculum_serve.practice.generator import CurriculumOracleTether + + tether = CurriculumOracleTether() + out: dict[str, list[Hashable]] = {} + for problem in all_gold_problems(): + if tether.gold_answer(problem) == "entailed": + out.setdefault(problem.class_name, []).append(problem.payload.key) + return out + + #: capability -> keys provider, or ``None`` when the producer is not built. #: ``None`` is a declaration, not an omission: :func:`test_every_licensed_capability_has_an_audit_source` #: accepts it only while the capability's ledger is genuinely absent. @@ -130,6 +153,7 @@ AUDIT_SOURCES: dict[str, Callable[[], dict[str, list[Hashable]]] | None] = { "estimation": _estimation_keys, "deduction_serve": _deduction_keys, "curriculum_serve": _curriculum_keys, + "curriculum_serve_entailed": _curriculum_entailed_keys, }