diff --git a/core/cli_test.py b/core/cli_test.py index 1a5c856a..471b4696 100644 --- a/core/cli_test.py +++ b/core/cli_test.py @@ -163,6 +163,14 @@ TEST_SUITES: dict[str, tuple[str, ...]] = { # weeks: the 2026-07-19 soak ran without accrue_realized_knowledge in the # daemon profile, and nothing anywhere could notice. Had this pin existed, # PR-5 would have failed it, which is exactly right. <1s. + # PR-14 / R-8 (ruled C) / G-10 — the curriculum OUTCOME-MIX rule. Committing + # to an entailment and correctly declining to are different capabilities and + # may not be pooled. Pooled, four bands cleared theta_SERVE by 0.000046 on + # 652-653 correct UNKNOWNs plus 7-8 entailments; split, NEITHER half clears. + # Pins both directions plus the counterfactual, so the rule cannot become + # decorative and the moment a band genuinely earns a licence is a reviewed + # decision rather than a drift. ~6s (runs the practice producer). + "tests/test_curriculum_outcome_mix.py", "tests/test_l10_soak_evidence.py", # PR-11 / R-9 — the H1-H4 holds+bites pins, promoted onto the gate. # Measured 20.95s for the four real-soak "holds" and 0.46s for the eight diff --git a/core/ratified_ledger.py b/core/ratified_ledger.py index ad4d598e..fe14af0b 100644 --- a/core/ratified_ledger.py +++ b/core/ratified_ledger.py @@ -109,7 +109,26 @@ CAPABILITY_LEDGERS: dict[str, LedgerSpec] = { note=( "ADR-0262 §5 — no band has earned a license from present curriculum " "volume; every served band is DISCLOSED. Flips to False when the " - "first band earns SERVE and the ledger is committed." + "first band earns SERVE and the ledger is committed. R-8 (ruled C, " + "2026-07-28) splits the earning basis: see 'curriculum_serve_entailed' " + "below. This entry now governs NON-COMMITMENT (correct UNKNOWN) " + "serving only." + ), + ), + "curriculum_serve_entailed": LedgerSpec( + capability="curriculum_serve_entailed", + path=_PROJECT_ROOT / "chat" / "data" / "curriculum_serve_entailed_ledger.json", + missing_ok=True, + note=( + "R-8, ruled C (2026-07-28) — the outcome-mix rule. Committing to an " + "entailment and correctly declining to are DIFFERENT capabilities and " + "are licensed on DIFFERENT evidence. Pooling them let a band clear " + "theta_SERVE=0.99 on correct refusals alone: the four leading bands " + "held 652-653 correct UNKNOWNs and 7-8 entailments, and only the " + "pooled 660 cleared. Split, NEITHER capability licenses anything " + "today — entailed evidence tops out at 9 (conservative_floor(9,9)=0.0 " + "against a required 657), and unknown evidence tops out at 653, four " + "short (five for two of the four bands). Absent = nothing licensed, which is the honest state." ), ), } diff --git a/docs/adr/ADR-0264-negative-curriculum-and-premise-scope.md b/docs/adr/ADR-0264-negative-curriculum-and-premise-scope.md index f898ce27..c71dd886 100644 --- a/docs/adr/ADR-0264-negative-curriculum-and-premise-scope.md +++ b/docs/adr/ADR-0264-negative-curriculum-and-premise-scope.md @@ -400,8 +400,37 @@ band is at its ceiling. ## 5. What this ADR does not decide +> ### Amendment — 2026-07-28: the mix floor is decided, and it decided nothing licensable +> +> *R-8, ruled **C** (`docs/assessment/50-rulings.md`), executed as PR-14. This amendment records a ruling; it reverses no decision in this ADR.* +> +> The first bullet below defers **the mix floor** to Phase B. It is now ruled, and the ruling is a *separation* rather than a number: +> +> **Committing to an entailment and correctly declining to commit are different capabilities, licensed on different evidence, and may not be pooled.** `CAPABILITY_LEDGERS` carries them as two entries — `curriculum_serve` (non-commitment) and `curriculum_serve_entailed` (commitment). +> +> **No new floor constant was introduced, deliberately.** The recommendation was *"C, with a floor from A applied to the entailed capability only,"* and the floor's value was never named. It does not need one: θ_SERVE=0.99 through `conservative_floor` — the bar every other capability already meets, **657** distinct correct decisions — applied to each separated capability on its own evidence is sufficient and is not close. A second, weaker constant for the capability that most needs the strong one would institutionalise two standards, which is the objection that sank option B. +> +> **Measured at the ruling (2026-07-28), and it overturns §4.1's expectation:** +> +> | | evidence | `conservative_floor` | vs 0.99 | +> |---|---|---|---| +> | entailed, best band | **9** | `0.000000` | fails by 648 cases | +> | non-commitment, best band | **653** | `0.989940` | fails by **4 cases** | +> | *pooled* (the old basis) | 660 | `0.990046` | *clears by 0.000046* | +> +> §4.1's banner says *"four bands would earn SERVE the moment a ledger is sealed."* That is true of the **pooled** basis and false of the ruled one. **Split, zero bands license.** The four leading bands cleared only because 7–8 entailments were counted alongside 652–653 correct refusals to reach 660; the licence was manufactured by the pooling, not by the evidence. +> +> **No ledger is sealed, and that is the deliverable.** Under the rule there is nothing to license, and the registered `missing_ok=True` absence already states that once. An artifact whose only content is its own emptiness would be a second statement of the same fact. +> +> **The useful result is the shape of the gap.** Non-commitment serving is **four to five distinct query atoms** short per band; entailed serving is ~648 short. Those are content tasks of completely different size, and the pooled figure could not distinguish *"almost there"* from *"two orders of magnitude away."* Pinned in both directions by `tests/test_curriculum_outcome_mix.py`, including the counterfactual — so the moment a band genuinely earns either licence, it is a reviewed decision rather than a drift. +> +> **`curriculum_serving_enabled` stays False** (last bullet of this section, unchanged). + - **The bound in R9.** The specific de-duplication tuple and mix floor are Phase B's deliverable. R9 fixes only that a bound must exist and be reported. + — **The mix floor is ruled; see the amendment above. The de-duplication tuple was + discharged separately: the curriculum producer's case identity IS the query atom, + so `committed == distinct` by construction.** - **Raising `MAX_PREMISE_SENTENCES`.** Rejected as an approach, not deferred: it is a shared honesty cap that the 25 deduction bands also decide under, raising it would move hash-pinned lanes, and a 219-premise argument is not honest at diff --git a/docs/assessment/50-execution-plan.md b/docs/assessment/50-execution-plan.md index d698a768..4a6c0c05 100644 --- a/docs/assessment/50-execution-plan.md +++ b/docs/assessment/50-execution-plan.md @@ -11,7 +11,7 @@ | **Track A — ADR-0252 §5 run to NO-GO**, criterion pre-registered, artifact + digest committed, **verdict ratified 2026-07-28** | PR-8 — a small ADR (H-3/G-20) · PR-10 — a refactor ADR (H-4) | | PR-4 membership + reachability ratchets · PR-6 three doctrine pins · PR-7 M2 trust table · PR-9 count-the-swallow · **PR-5 the flag register (R-3+R-4)** | PR-12 — an ADR **amendment** (R-13 authorizes it; the amendment is still owed) | | PR-3 + PR-3b (**R-7**, Wave 1) · **PR-11 the soak to committed evidence (R-9+R-2)** · H-13 · H-8e · G-22 · G-23 | Track B — the fabrication ADR (standing instruction, unchanged) | -| **Closed:** G-5, G-6, G-7, G-8, G-9, G-15, G-19, G-22, H-1, H-6, H-7, H-8 (**all five**), H-11, H-13 · **Answered:** G-1 · **Discharged:** H-10 · **Pinned:** G-23 | Track C — invention, deliberately last | +| **Closed:** G-5, G-6, G-7, G-8, G-9, G-10, G-15, G-19, G-22, H-1, H-6, H-7, H-8 (**all five**), H-11, H-13 · **Answered:** G-1 · **Discharged:** H-10 · **Pinned:** G-23 | Track C — invention, deliberately last | **New this arc, not in the original assessment:** **N-8** (the §5 experiment had already returned GO twice, on unmerged branches, both unsound), **N-9** (the gate "drift" was a recorded decision — PR-4's pin 1 **withdrawn** rather than built, R-14 dissolved), **G-21** (the math reader decides **1.0%** of `holdout_dev/v1`), **G-22** (`main` was red before the arc, and `CLAIMS.md` published a superseded evidence digest for two days), **H-13**, **H-14**, **H-8e**. @@ -219,7 +219,7 @@ The packet's original order was by register number. That is the wrong axis: it i | 3 | ~~**R-3 + R-4**~~ | PR-5 — the flag register, all 32 booleans classified, profiles as units, the bidirectional pin, `accrue_realized_knowledge` into the daemon profile | **M** | **EXECUTED 2026-07-28.** Closed **G-6, G-8, H-6, H-8(c), H-8(d)** — and **H-8 in full**. The N-6 docstring correction **dissolved**: the docstrings were right and the code was incomplete. **Wave 4's hard gate is lifted.** | | 4 | ~~**R-9 + R-2**~~ | PR-11 — the soak re-run at 5000 beats under the **corrected** profile, artifact committed, digest pinned, cadence enforced by a source-hash pin, H1–H4 on the gate | **M** | **EXECUTED 2026-07-28.** Closed **G-5**. R-2 landed in the contract *before* the re-run, as designed. The staleness pin **would have caught PR-5** — verified by sabotage. | | 5 | ~~**R-13**~~ | PR-12 — distinct-evidence counting at the seal boundary, the honest re-count, demotions applied, the producer hardened against re-padding | **L** | **EXECUTED 2026-07-28. 25 licensed bands → 4.** Served capability shrank by 21 bands and `wrong` stayed 0: no answer changed, only the claim attached to it. | -| 6 | **R-8** | PR-14 — the outcome-mix rule, then seal `curriculum_serve_ledger.json`. Four bands qualify on sealing (N-5) | **M** | Depends on PR-12's counting basis. This is the entire throughput frontier, one ruling wide. | +| 6 | ~~**R-8**~~ | PR-14 — the outcome-mix rule as two capabilities. **No ledger sealed: under the rule, zero bands license** | **M** | **EXECUTED 2026-07-28.** Overturns N-5 — the four bands cleared only on *pooled* evidence, by 0.000046. Closed **G-10**. | | 7 | **R-1 · R-5 · R-6 · R-11** | Posture statements: efferent-action deferral with entry criterion (G-12) · the identity-enforcement discrimination bar (G-11) · non-text ingest deferral with the falsification bench as its standard (G-17) · **R-11 B — measure before gating** | **S** each | All four are prose with a criterion. None blocks anything; leaving them silent is the cost. They go last because they are the only items where lateness is cheap. | **R-11's measurement is owed now, not later — and R-11 is the one adopted ruling that does not end in a decision.** Option B is *measure first, then re-ask*: an instrumentation PR (counts only, no behavior change) reporting how often the out-of-inventory reading path is taken on the practice corpora, after which **R-11 is put back to Shay with the number in hand.** So the docket owes two things here, not one: the measurement, and the second ruling. Until the number exists, R-11 has been ruled and nothing follows from it — which is a state worth naming, because a ruled-but-inert item is indistinguishable from a settled one at a glance. It lands with the R-1/R-5/R-6 batch at the latest and may land sooner, since the instrumentation is small and independent. @@ -362,7 +362,19 @@ Shipped as ADR-0264 R5, discharged 2026-07-26. The 16-premise cap does not exist **It also carries G-23's decision, because it is the same band.** `DOMAIN_PACKS` places `en_core_cognition_v1` and `en_core_meta_v1` in `philosophy_theology`; neither manifest declares a `domain_id`, so `domain_contract_predicates` P3 has never run on them. The gap is pinned (`tests/test_domain_pack_binding.py`, on the gate, four sabotages red) and deliberately not fixed: whether those two are *philosophy_theology domain packs* or merely *grouped under it for ledger accounting* decides what the license actually covers. Sealing that band's ledger without answering it licenses a domain whose membership is asserted in one place and denied by silence in the other. -**One sub-question the ruling leaves open, stated rather than assumed.** The packet's recommendation was *"C, with a floor from A applied to the entailed capability only"* — and **N was never named.** Adopting C adopts the floor in principle; the number is not derivable from the evidence, because the whole point is that no band is near it (max entailed volume = 9). This PR must therefore either carry a proposed N with its consequence measured, or land the two capabilities with the entailed one explicitly unlicensable-pending-N. It may not quietly ship without a floor — that would license entailment on 9 cases through an omission, which is the exact outcome C exists to prevent. +**LANDED 2026-07-28, and the result is the opposite of what this item predicted.** + +**Split, zero bands license — not four.** The four leading bands hold **652–653 correct UNKNOWNs** and **7–8 entailments**. Pooled they reach 660 and clear θ_SERVE by **0.000046**; separated, neither half clears. `conservative_floor(9,9)` is **0.000000** — the Wilson bound at nine trials is not merely below θ, it is zero. The licence N-5 predicted was manufactured entirely by counting correct refusals and correct commitments as one kind of evidence. + +**This is why the plan blocked PR-14 on R-8 rather than sealing first.** A ledger sealed on the pooled basis would have granted four licences the mix rule then had to revoke — and revoking a granted licence is the expensive direction, as PR-12 had just demonstrated across 21 deduction bands. + +**No ledger is sealed, and that is the deliverable.** Under the rule there is nothing to license; the registered `missing_ok=True` absence states that once already. An artifact whose only content is its own emptiness would be a second statement of the same fact. + +**The useful result is the *shape* of the gap, which pooling had hidden.** Non-commitment serving is **four to five distinct query atoms short per band**; entailed serving is **~648** short. Those are content tasks of completely different size, and a single pooled figure could not distinguish *"almost there"* from *"two orders of magnitude away."* Four cases is a morning's work; 648 is a program. + +**Delivered:** `curriculum_serve_entailed` registered in `CAPABILITY_LEDGERS` (`missing_ok=True` — absent means nothing licensed, which is the honest state); an audit source for it, which the manifest's own pin demanded the moment the capability was declared; the **ADR-0264 §5 amendment** recording the ruling and correcting §4.1's "four bands would earn SERVE" expectation; and `tests/test_curriculum_outcome_mix.py` on the gate, pinning both capabilities, the shortfall, **and the counterfactual** — so the argument for the rule cannot drift away from the number it rests on. + +**~~One sub-question the ruling leaves open~~ — RULED 2026-07-28 under the standing delegation: no new constant.** The packet's recommendation was *"C, with a floor from A applied to the entailed capability only"* — and **N was never named.** Adopting C adopts the floor in principle; the number is not derivable from the evidence, because the whole point is that no band is near it (max entailed volume = 9). This PR must therefore either carry a proposed N with its consequence measured, or land the two capabilities with the entailed one explicitly unlicensable-pending-N. It may not quietly ship without a floor — that would license entailment on 9 cases through an omission, which is the exact outcome C exists to prevent. ### Track B · The fabrications and the widening — *gated, see §6* G-2's fix lands only under its ratified ADR (R-11 may add an interim defensive gate first). G-3's widening program is **shape-dependent on Track A's verdict** and does not start before it. diff --git a/docs/assessment/50-rulings.md b/docs/assessment/50-rulings.md index ed80dc59..b23c5bb8 100644 --- a/docs/assessment/50-rulings.md +++ b/docs/assessment/50-rulings.md @@ -9,9 +9,9 @@ **Also ratified:** the ADR-0252 §5 **NO-GO** verdict. **Execution order (corrected):** R-12 → R-7 → R-3+R-4 → R-9+R-2 → R-13 → R-8 → R-1/R-5/R-6/R-11. -**Executed so far:** R-7 (PR-3, PR-3b) · **R-12** (both ADR amendments + the §5 verdict banner + the §6 retirement-condition amendment) · **R-3 + R-4** (PR-5 — the flag register, the profile mechanism, the daemon's fourth flag) · **R-9 + R-2** (PR-11 — the soak's committed artifact, pinned digest, change-triggered cadence) · **R-13** (PR-12 — the Wilson re-count; 21 of 25 licences revoked) — all landed 2026-07-28. +**Executed so far:** R-7 (PR-3, PR-3b) · **R-12** (both ADR amendments + the §5 verdict banner + the §6 retirement-condition amendment) · **R-3 + R-4** (PR-5 — the flag register, the profile mechanism, the daemon's fourth flag) · **R-9 + R-2** (PR-11 — the soak's committed artifact, pinned digest, change-triggered cadence) · **R-13** (PR-12 — the Wilson re-count; 21 of 25 licences revoked) · **R-8** (PR-14 — the outcome-mix rule; the four expected licences dissolve under it) — all landed 2026-07-28. -**Remaining:** R-8 → R-1/R-5/R-6/R-11. +**Remaining:** R-1/R-5/R-6/R-11 (the four posture statements) + R-11's measurement and its second ruling. --- @@ -19,7 +19,7 @@ > *"If you think you might need me for a ruling, I instruct you to handle it based on the most masterful logic accordingly. No more pauses."* -**Scope.** The residual decisions inside the adopted docket — the sub-questions the twelve rulings left open, and any new question of the same size that arises while executing them. Concretely and by name: **R-8 C's entailed-capability floor `N`** (never specified when C was adopted); **R-11's second ruling** (option B ends in *re-ask*, not a decision); **PR-4b's gate-width choice**; **G-23's domain-membership question**; and the sequencing latitude to reorder the docket where evidence warrants. +**Scope.** The residual decisions inside the adopted docket — the sub-questions the twelve rulings left open, and any new question of the same size that arises while executing them. Concretely and by name: ~~**R-8 C's entailed-capability floor `N`**~~ (**ruled 2026-07-28: no new constant** — the existing θ_SERVE bar, applied to each separated capability on its own evidence, licenses nothing by a factor of 73; a second weaker constant for the capability that most needs the strong one would institutionalise the two standards option B was rejected for); **R-11's second ruling** (option B ends in *re-ask*, not a decision); **PR-4b's gate-width choice**; **G-23's domain-membership question**; and the sequencing latitude to reorder the docket where evidence warrants. **What the delegation does not do, and this matters more than what it does.** @@ -188,7 +188,7 @@ Two flags' documentation describes a production profile the production profile d ## R-8 · The curriculum outcome-mix rule — *now the sole blocker of an entire frontier* -**Status:** **RULED C** — 2026-07-28, Shay (explicit adoption) · *separate `unknown`- and `entailed`-serving licenses* · **Register:** G-10 · **Blocks:** PR-14, Track E in full. +**Status:** **RULED C — EXECUTED 2026-07-28 (PR-14)** · *`unknown`- and `entailed`-serving separated as two capabilities. **Split, ZERO bands license — not four.** The floor `N` was ruled unnecessary (DELEGATED): the existing θ_SERVE bar suffices and is not close.* · **Register:** G-10 · **Blocks:** PR-14, Track E in full. **The question changed shape since the register was written (N-5).** G-10 named the 16-premise compilation cap as the blocking engineering item. ADR-0264 §4.1 carries a banner directly beneath its own heading: diff --git a/tests/test_curriculum_outcome_mix.py b/tests/test_curriculum_outcome_mix.py new file mode 100644 index 00000000..dbbe434b --- /dev/null +++ b/tests/test_curriculum_outcome_mix.py @@ -0,0 +1,161 @@ +"""R-8 (ruled C) / G-10 — the curriculum outcome-mix rule, and what it measures. + +**The rule.** Committing to an entailment and correctly declining to commit are +different capabilities. They are licensed on different evidence and may not be +pooled. ``core.ratified_ledger.CAPABILITY_LEDGERS`` declares them as two entries; +this module measures them. + +**Why the rule was needed, in one number.** Pooled, the four leading bands reach +660 committed decisions and ``conservative_floor(660, 660) = 0.990046``, clearing +θ_SERVE=0.99 — by a margin of **0.000046**. Split, the same bands hold **652–653 +correct UNKNOWNs** and **7–8 entailments**. Neither half clears. The licence was +manufactured entirely by counting correct refusals and correct commitments as one +kind of evidence. + +**This overturns the plan's own prediction.** N-5 recorded that *"four bands would +earn SERVE the moment a ledger is sealed."* That is true on the pooled basis and +false on the ruled one. Sealing under R-8 C licenses **nothing** — which is exactly +why the plan blocked PR-14 on this ruling rather than sealing first and ruling +after. A ledger sealed on the pooled basis would have granted four licences that the +mix rule then had to revoke, and revoking a granted licence is the expensive +direction (see R-13 and the 21 deduction bands). + +**The shortfall is small and exact, and that is the useful part.** Unknown-serving +needs **four to five** more distinct query atoms per band (657 − 653 for two bands, +657 − 652 for the other two). Entailed-serving needs roughly **648** more. Those are +content tasks of wildly different size, and before the split nobody could see that — +the pooled number hid a four-case gap behind a 648-case one. + +This module deliberately does **not** seal a ledger. Under the rule there is nothing +to license, and an artifact asserting zero licences is a file whose only content is +its own emptiness; the registered ``missing_ok=True`` absence already says it, once. +""" + +from __future__ import annotations + +import pytest + +from core.ratified_ledger import CAPABILITY_LEDGERS +from core.reliability_gate import conservative_floor +from core.reliability_gate.evidence import SERVE_VOLUME_AT_THETA_099 + +THETA_SERVE = 0.99 + + +@pytest.fixture(scope="module") +def mix() -> dict[str, dict[str, int]]: + """Per-band ``{'entailed': n, 'unknown': n}`` from the practice producer.""" + from evals.curriculum_serve.practice.runner import run + + report = run() + return {band: dict(c["gold_mix"]) for band, c in report["classes"].items()} + + +def test_both_capabilities_are_registered() -> None: + """The split is declared in the manifest, not asserted at a call site. + + ADR-0263 rule 5. If this fails, the outcome-mix rule exists only in prose — + which is the state R-8 was raised to end. + """ + assert "curriculum_serve" in CAPABILITY_LEDGERS + assert "curriculum_serve_entailed" in CAPABILITY_LEDGERS + assert CAPABILITY_LEDGERS["curriculum_serve_entailed"].missing_ok is True, ( + "absent must mean 'nothing licensed' for the entailed capability — a " + "missing_ok=False entry would turn an honest absence into a broken deployment" + ) + + +def test_the_mix_is_not_vacuous(mix: dict[str, dict[str, int]]) -> None: + """Guard the measurement before asserting anything about it.""" + assert len(mix) >= 10, f"parsed only {len(mix)} bands from the producer" + assert any(m.get("entailed", 0) for m in mix.values()), "no entailed cases parsed" + assert any(m.get("unknown", 0) for m in mix.values()), "no unknown cases parsed" + + +def test_no_band_licenses_entailed_serving(mix: dict[str, dict[str, int]]) -> None: + """The committing capability, on its own evidence: nothing qualifies. + + ``conservative_floor(9, 9) == 0.0`` — the Wilson lower bound at nine trials is + not merely below θ, it is zero. This is the assertion that stops an entailment + licence being granted on a handful of cases, which is what pooling did. + """ + offenders = { + band: n + for band, m in mix.items() + if (n := m.get("entailed", 0)) and conservative_floor(n, n) >= THETA_SERVE + } + assert not offenders, ( + f"these bands now hold enough entailed evidence to license: {offenders}. " + "That is good news and a reviewed decision — seal the entailed ledger and " + "move this pin." + ) + best = max(m.get("entailed", 0) for m in mix.values()) + assert best < SERVE_VOLUME_AT_THETA_099, ( + f"max entailed evidence is {best}, at or above the {SERVE_VOLUME_AT_THETA_099} " + "θ_SERVE requires — the entailed capability is now earnable" + ) + + +def test_no_band_licenses_unknown_serving_either(mix: dict[str, dict[str, int]]) -> None: + """The declining capability, on its OWN evidence: also nothing — the finding. + + This is the assertion that contradicts N-5. The four bands everyone expected to + license hold 652–653 correct UNKNOWNs against a 657 requirement: **four to five + short**. + They only ever cleared because 7–8 entailments were pooled in to reach 660. + """ + licensed = { + band: n + for band, m in mix.items() + if (n := m.get("unknown", 0)) and conservative_floor(n, n) >= THETA_SERVE + } + assert not licensed, ( + f"these bands now license unknown-serving on distinct non-commitments: " + f"{licensed}. Expected under the ruling once curriculum volume grows — seal " + "the ledger and move this pin, in the same change." + ) + + +def test_pooling_would_have_licensed_four_bands(mix: dict[str, dict[str, int]]) -> None: + """Pin the counterfactual, because it is the whole justification for the rule. + + If this ever fails it means pooling no longer manufactures a licence, and the + mix rule has become free rather than load-bearing — worth knowing either way. It + is asserted rather than written in prose so the number cannot quietly drift away + from the argument that rests on it. + """ + pooled = { + band: total + for band, m in mix.items() + if conservative_floor(total := sum(m.values()), total) >= THETA_SERVE + } + assert len(pooled) == 4, ( + f"pooled evidence licenses {len(pooled)} bands, not the 4 recorded when R-8 " + f"was ruled: {sorted(pooled)}" + ) + for band, total in pooled.items(): + entailed = mix[band].get("entailed", 0) + unknown = mix[band].get("unknown", 0) + assert conservative_floor(unknown, unknown) < THETA_SERVE, band + assert entailed < 10, (band, entailed) + assert total >= SERVE_VOLUME_AT_THETA_099, (band, total) + + +def test_the_shortfall_is_recorded_exactly(mix: dict[str, dict[str, int]]) -> None: + """Four or five cases, not "more volume" — the gap is small, and its size is the point. + + Before the split, the unknown shortfall (4-5) was hidden behind the entailed one + (~648). A single pooled figure could not distinguish "almost there" from + "two orders of magnitude away," and those call for completely different work. + """ + best_unknown = max(m.get("unknown", 0) for m in mix.values()) + best_entailed = max(m.get("entailed", 0) for m in mix.values()) + assert SERVE_VOLUME_AT_THETA_099 - best_unknown == 4, ( + f"the unknown-serving shortfall moved: best band has {best_unknown} distinct " + f"non-commitments against {SERVE_VOLUME_AT_THETA_099} required" + ) + assert SERVE_VOLUME_AT_THETA_099 - best_entailed > 600, ( + f"the entailed shortfall is now {SERVE_VOLUME_AT_THETA_099 - best_entailed}; " + "if it has closed to within a few hundred, the entailed capability is worth " + "re-planning rather than deferring" + ) diff --git a/tests/test_volume_honesty.py b/tests/test_volume_honesty.py index 84e18ff7..bca2c312 100644 --- a/tests/test_volume_honesty.py +++ b/tests/test_volume_honesty.py @@ -123,6 +123,29 @@ def _curriculum_keys() -> dict[str, list[Hashable]]: return out +def _curriculum_entailed_keys() -> dict[str, list[Hashable]]: + """Decision keys for the ENTAILED half of the curriculum corpus (R-8, ruled C). + + R-8 split the curriculum earning basis: committing to an entailment and + correctly declining to commit are different capabilities, licensed on different + evidence. This is the audit source for the committing half — the query atoms + whose oracle verdict is ``entailed``. + + It exists so the split cannot be declared in the manifest and left unmeasured. + The numbers it reports are small on purpose: 3-9 per band, against the 657 that + theta_SERVE=0.99 requires. That gap IS the finding. + """ + from evals.curriculum_serve.practice import all_gold_problems + from evals.curriculum_serve.practice.generator import CurriculumOracleTether + + tether = CurriculumOracleTether() + out: dict[str, list[Hashable]] = {} + for problem in all_gold_problems(): + if tether.gold_answer(problem) == "entailed": + out.setdefault(problem.class_name, []).append(problem.payload.key) + return out + + #: capability -> keys provider, or ``None`` when the producer is not built. #: ``None`` is a declaration, not an omission: :func:`test_every_licensed_capability_has_an_audit_source` #: accepts it only while the capability's ledger is genuinely absent. @@ -130,6 +153,7 @@ AUDIT_SOURCES: dict[str, Callable[[], dict[str, list[Hashable]]] | None] = { "estimation": _estimation_keys, "deduction_serve": _deduction_keys, "curriculum_serve": _curriculum_keys, + "curriculum_serve_entailed": _curriculum_entailed_keys, }