From 13a4b05d62308118159f79b63563852d18229e9c Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 25 Jul 2026 16:03:10 -0700 Subject: [PATCH] test(reliability): volume-honesty invariant + distinct-evidence audit Phase B of the curriculum-license-loop arc. Stacked on the ADR-0264 branch. The invariant: conservative_floor is a one-sided Wilson lower bound and Wilson assumes INDEPENDENT trials. CORE's pipeline is deterministic, so replaying an identical case is not a second trial -- it is the same trial with a guaranteed outcome. A ledger's `committed` count is an upper bound on its evidence; the defensible figure is its distinct-decision count. Phase B was scoped as a precaution for a curriculum producer that does not exist yet. Running the instrument against the producers that DO exist found it already violated: 21 of the 25 ratified deduction_serve bands do not clear theta_SERVE=0.99 on distinct evidence. Three inflate 28 distinct cases into 720 committed (honest floor 0.8084 vs claimed 0.9909). deduction_serving_enabled was ratified ON 2026-07-24 and is live. The estimation producer (ADR-0175) is clean -- 660 committed, 660 distinct, zero repeats, evidently chosen just above the 657 a perfect record needs. So this is a regression from a standard already established in the repo, not an architectural gap. `claimed` is identical for all 25 deduction bands because every band commits 720 with a perfect record: the gate sees 25 indistinguishable passes while honest floors span 0.8084..0.9909. NOT REPAIRED. The ledger is SHA-sealed, ratified, and gating a live flag, so re-sealing it is Shay's ratification, not a test's. wrong=0 still holds -- no band answered anything incorrectly. What is not established is reliability at the volume claimed. Measured, pinned in both directions, written down. Added: - core/reliability_gate/evidence.py -- pure measurement, imported by no serving path. volume_for_theta() DERIVES 657 from conservative_floor rather than restating it, so a WILSON_Z change cannot leave a stale literal behind. - tests/test_volume_honesty.py -- 13 tests. AUDIT_SOURCES is checked against CAPABILITY_LEDGERS, so a new licensed capability cannot ship unaudited; the curriculum entry is a declared None that fails the moment its ledger exists. That is Phase C's forcing function against repeating the pattern. - docs/research/distinct-evidence-audit-2026-07-25.md -- full audit + the open outcome-mix ruling for Shay. - ADR-0264 R9 amended with the measured exposure. Decision key is case TEXT deliberately: a tighter key (normalized atom) would collapse spelling variants and report MORE inflation, so text-identity under-reports and every number is a floor on the real gap. Outcome mix is deliberately NOT pinned. ClassTally has no verdict axis, the deduction producer already balances 240/240/240 by construction, and imposing a per-verdict-class floor would retroactively fail all 25 bands on a criterion no ADR has ratified (smallest per-band class count is 120). Recorded for a ruling with numbers instead. [Verification]: uv sync --locked on canonical CPython 3.12.13; in-worktree smoke 569 passed (556 baseline + 13), deductive 285 passed. Registered in the `smoke` curated suite -- not left to `full`, which gates nothing. Mutation- checked: doubling CASES_PER_BAND -> 2 failed; reporting committed as distinct -> 4 failed; gating on `claimed` instead of `honest` -> 3 failed; tree restores to 13 passed. --- core/cli_test.py | 8 + core/reliability_gate/__init__.py | 20 + core/reliability_gate/evidence.py | 175 +++++++++ ...4-negative-curriculum-and-premise-scope.md | 44 ++- .../distinct-evidence-audit-2026-07-25.md | 183 +++++++++ tests/test_volume_honesty.py | 367 ++++++++++++++++++ 6 files changed, 787 insertions(+), 10 deletions(-) create mode 100644 core/reliability_gate/evidence.py create mode 100644 docs/research/distinct-evidence-audit-2026-07-25.md create mode 100644 tests/test_volume_honesty.py diff --git a/core/cli_test.py b/core/cli_test.py index 7c4e45cc..ab018585 100644 --- a/core/cli_test.py +++ b/core/cli_test.py @@ -77,6 +77,14 @@ TEST_SUITES: dict[str, tuple[str, ...]] = { # curated suite, which is the same silent-red shape it exists to catch. "tests/test_adr_status_governance.py", "tests/test_adr_index.py", + # Volume honesty (ADR-0264 R9). The Wilson floor assumes independent + # trials; a deterministic pipeline replaying one case N times supplies + # one trial's evidence, not N. This pins the measured exposure in the + # ratified deduction ledger (21 of 25 bands short on distinct evidence) + # in BOTH directions, and forces any new licensed capability to register + # an audit source. It gates a live, flag-ON serving path, so it belongs + # on the pre-push gate. ~1.5s. + "tests/test_volume_honesty.py", ), "runtime": ( "tests/test_chat_runtime.py", diff --git a/core/reliability_gate/__init__.py b/core/reliability_gate/__init__.py index 04458053..ab8c458f 100644 --- a/core/reliability_gate/__init__.py +++ b/core/reliability_gate/__init__.py @@ -9,11 +9,24 @@ Public surface: - :class:`ClassTally` — per-class counted ledger; reliability = commitment precision. - :class:`Action`, :class:`Ceilings` — human-set θ ceilings (engine never mutates). - :func:`license_for`, :class:`LicenseDecision` — the deterministic gate. +- :class:`EvidenceAudit`, :func:`audit_band` — distinct-evidence audit (ADR-0264 + R9). Measurement only, imported by no serving path: the Wilson floor assumes + independent trials, and a deterministic pipeline replaying one input N times + supplies one trial's evidence, not N. """ from __future__ import annotations from core.reliability_gate.ceilings import Action, Ceilings +from core.reliability_gate.evidence import ( + SERVE_VOLUME_AT_THETA_099, + EvidenceAudit, + audit_band, + audit_bands, + below_floor, + format_report, + volume_for_theta, +) from core.reliability_gate.floor import N_MIN, WILSON_Z, conservative_floor from core.reliability_gate.gate import LicenseDecision, license_for from core.reliability_gate.ledger import ClassTally @@ -23,11 +36,18 @@ __all__ = [ "Action", "Ceilings", "ClassTally", + "EvidenceAudit", "LicenseDecision", "N_MIN", "RatifiableProposal", + "SERVE_VOLUME_AT_THETA_099", "WILSON_Z", + "audit_band", + "audit_bands", + "below_floor", "conservative_floor", + "format_report", "license_for", "propose_from_ledger", + "volume_for_theta", ] diff --git a/core/reliability_gate/evidence.py b/core/reliability_gate/evidence.py new file mode 100644 index 00000000..f5e4a911 --- /dev/null +++ b/core/reliability_gate/evidence.py @@ -0,0 +1,175 @@ +"""ADR-0264 R9 — distinct-evidence audit: the precondition :func:`conservative_floor` assumes. + +:func:`core.reliability_gate.floor.conservative_floor` is a one-sided Wilson lower +bound on a success proportion. Wilson assumes **independent trials**. CORE's +pipeline is deterministic by construction — that is the whole thesis — so +replaying an identical input yields the identical outcome with certainty. Two +identical cases in a practice ledger are therefore perfectly correlated, not +independent, and contribute one trial's worth of evidence between them, not two. + +So a ledger's ``committed`` count is an upper bound on its evidence, and the +honest figure is the count of *distinct decisions* it exercised. This module +computes both and reports the gap. It is deliberately a pure measurement with no +authority: it never edits a ledger, never gates anything, and nothing in the +serving path imports it. What to do about a gap is a ratification decision. + +Why this exists as code rather than a note: S6 predicted the exposure in the +abstract (`docs/research/curriculum-volume-quantification-2026-07-24.md` §1 — +``conservative_floor`` cannot distinguish "657 independent facts" from "16 facts +asked 41 times"). It was then measured in the *live, ratified* deduction ledger, +where 21 of 25 bands do not clear θ_SERVE on distinct evidence. A prediction that +has come true needs an instrument, not a second warning. + +**The key is supplied by the caller, and must be conservative.** A decision key +identifies what the pipeline actually decided. Case *text* is the safe default: +two cases with identical text are indisputably the same decision. A tighter key +(e.g. the normalized propositional atom, which collapses spelling variants of one +question) reports *more* inflation. Text-identity therefore under-reports, which +is the correct direction for a claim of this kind — the measured gap is a floor on +the real gap, never an exaggeration of it. +""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Hashable, Iterable +from dataclasses import dataclass + +from core.reliability_gate.floor import conservative_floor + +#: The committed volume a *perfect* record needs to clear θ=0.99 under the pinned +#: ``WILSON_Z``. Derived, not chosen: the bound for ``successes == committed`` +#: reduces to ``n / (n + z²)``, so ``n / (n + 6.635776) ≥ 0.99`` ⟺ ``n ≥ 656.93``. +#: Recomputed by :func:`volume_for_theta` rather than trusted as a constant. +SERVE_VOLUME_AT_THETA_099: int = 657 + + +def volume_for_theta(theta: float) -> int: + """Smallest perfect-record volume whose conservative floor reaches *theta*. + + Derives the figure from :func:`conservative_floor` itself instead of + restating it, so a change to ``WILSON_Z`` cannot leave a stale number behind + in a docstring or a test. Search, not algebra, because the bound is rounded + to 1e-9 and the comparison downstream is the rounded one. + """ + if not 0.0 < theta < 1.0: + raise ValueError(f"theta must lie in (0, 1); got {theta!r}") + n = 1 + while conservative_floor(n, n) < theta: + n *= 2 + if n > 1 << 40: # pragma: no cover - unreachable for theta < 1 + raise ValueError(f"no finite volume reaches theta={theta!r}") + low, high = n // 2, n + while low < high: + mid = (low + high) // 2 + if conservative_floor(mid, mid) >= theta: + high = mid + else: + low = mid + 1 + return low + + +@dataclass(frozen=True, slots=True) +class EvidenceAudit: + """What one band's practice volume is worth as evidence. + + ``claimed`` is the floor the ledger's own ``committed`` count produces — + what the gate sees today. ``honest`` is the floor over distinct decisions — + what a bound assuming independent trials is entitled to claim. + """ + + band: str + committed: int + distinct: int + max_repeat: int + + def __post_init__(self) -> None: + if self.committed < 0 or self.distinct < 0 or self.max_repeat < 0: + raise ValueError("counts must be non-negative") + if self.distinct > self.committed: + raise ValueError( + f"{self.band}: distinct ({self.distinct}) cannot exceed " + f"committed ({self.committed})" + ) + if self.committed and not self.distinct: + raise ValueError(f"{self.band}: committed cases with no distinct keys") + + @property + def claimed(self) -> float: + """Floor from ``committed`` — assumes every case was a fresh trial.""" + return conservative_floor(self.committed, self.committed) + + @property + def honest(self) -> float: + """Floor from distinct decisions — the defensible figure.""" + return conservative_floor(self.distinct, self.distinct) + + @property + def inflation(self) -> float: + """``committed / distinct``. 1.0 means every case was a distinct decision.""" + return round(self.committed / self.distinct, 9) if self.distinct else 0.0 + + def clears(self, theta: float) -> bool: + """Does the *honest* floor reach *theta*? The question the gate should ask.""" + return round(self.honest, 9) >= round(theta, 9) + + +def audit_band(band: str, keys: Iterable[Hashable]) -> EvidenceAudit: + """Audit one band from the decision keys of its committed cases. + + *keys* is one entry per committed case — repeats are the point, so this + takes an iterable of keys rather than a set. + """ + counts = Counter(keys) + committed = sum(counts.values()) + return EvidenceAudit( + band=band, + committed=committed, + distinct=len(counts), + max_repeat=max(counts.values()) if counts else 0, + ) + + +def audit_bands(cases_by_band: dict[str, Iterable[Hashable]]) -> tuple[EvidenceAudit, ...]: + """Audit every band, ordered by band name for replay-stable reporting.""" + return tuple(audit_band(band, cases_by_band[band]) for band in sorted(cases_by_band)) + + +def below_floor( + audits: Iterable[EvidenceAudit], theta: float +) -> tuple[EvidenceAudit, ...]: + """The audits whose honest floor does not reach *theta*, worst gap first.""" + failing = [a for a in audits if not a.clears(theta)] + return tuple(sorted(failing, key=lambda a: (a.honest, a.band))) + + +def format_report(audits: Iterable[EvidenceAudit], theta: float) -> str: + """A fixed-width report — the text a failing pin should print. + + Assertion messages are the whole value of an audit that fails months from + now, so the numbers are formatted here rather than at each call site. + """ + rows = sorted(audits, key=lambda a: a.band) + need = volume_for_theta(theta) + lines = [ + f"theta={theta} perfect-record volume needed={need}", + f"{'band':30s} {'committed':>9s} {'distinct':>8s} {'x':>6s} " + f"{'claimed':>8s} {'honest':>7s} clears", + ] + for a in rows: + lines.append( + f"{a.band:30s} {a.committed:9d} {a.distinct:8d} {a.inflation:6.1f} " + f"{a.claimed:8.4f} {a.honest:7.4f} {'yes' if a.clears(theta) else 'NO'}" + ) + return "\n".join(lines) + + +__all__ = [ + "SERVE_VOLUME_AT_THETA_099", + "EvidenceAudit", + "audit_band", + "audit_bands", + "below_floor", + "format_report", + "volume_for_theta", +] diff --git a/docs/adr/ADR-0264-negative-curriculum-and-premise-scope.md b/docs/adr/ADR-0264-negative-curriculum-and-premise-scope.md index 3c165e55..fc1eda0c 100644 --- a/docs/adr/ADR-0264-negative-curriculum-and-premise-scope.md +++ b/docs/adr/ADR-0264-negative-curriculum-and-premise-scope.md @@ -185,16 +185,40 @@ being right. Additionally: **negative edges are excluded from the BFS adjacency* A negative edge is not a traversable path, and including it would inflate the reachability depths the lane asserts against. -**R9 — Earning requires an outcome-mix floor, and a band that cannot reach the -threshold must say so before anyone authors for it.** ADR-0262 §5.1 already -states the acceptability criterion — "a band whose entailed class is 1% of its -volume is passed by a pipeline that never entails anything" — but leaves it as -prose. It becomes a measured precondition: the practice producer reports, per -band, the maximum honest committed volume and the maximum entailed share -achievable from the ratified corpus, and a band whose ceiling is below the -licensing threshold is reported **structurally unable to earn SERVE** rather than -left for authoring effort to discover. Phase B specifies the bound; this ADR -fixes the requirement that the bound exist. +**R9 — Committed volume must be DISTINCT evidence, and a band that cannot reach +the threshold must say so before anyone authors for it.** + +`conservative_floor` is a one-sided Wilson lower bound, and Wilson assumes +**independent trials**. CORE's pipeline is deterministic, so replaying an +identical case is not a second trial — it is the same trial with a guaranteed +outcome. A ledger's `committed` count is therefore an upper bound on its +evidence, and the defensible figure is its distinct-decision count. A producer +must commit at most one case per distinct decision, where the decision key for +this path is the query atom `(domain, subject, connective, object)` — spelling +variants of one question are one case, because R2/R3 normalize them to one atom. + +Additionally the producer reports, per band, the maximum honest committed volume +(§4.2) and the maximum entailed share the ratified corpus can support, and a band +whose ceiling is below the licensing threshold is reported **structurally unable +to earn SERVE** rather than left for authoring effort to discover. + +> **Amended 2026-07-25, Phase B.** This rule was written as a precaution for a +> producer that does not exist yet. Measuring the producers that *do* exist found +> it already violated: **21 of the 25 ratified `deduction_serve` bands do not +> clear θ_SERVE=0.99 on distinct evidence**, three of them inflating 28 distinct +> cases into 720 committed, under a flag ratified ON. The `estimation` producer +> (ADR-0175) is clean at 660 distinct per class, so this is a regression from an +> established standard rather than an architectural gap. Measured, pinned in both +> directions by `tests/test_volume_honesty.py`, and **not repaired** — the ledger +> is SHA-sealed and ratified, so re-sealing it is Shay's decision. Full audit: +> `docs/research/distinct-evidence-audit-2026-07-25.md`. +> +> Outcome *mix* is deliberately excluded from the pin. `ClassTally` has no verdict +> axis, so mix can only be enforced at the producer; the deduction producer +> already balances 240/240/240 by construction; and whether each verdict class +> needs its own 657 is an open ruling that would retroactively fail all 25 bands +> (smallest per-band verdict-class count is 120). Recorded for Shay with numbers, +> not decided. ## 3. Why this is sound diff --git a/docs/research/distinct-evidence-audit-2026-07-25.md b/docs/research/distinct-evidence-audit-2026-07-25.md new file mode 100644 index 00000000..2fd5ff96 --- /dev/null +++ b/docs/research/distinct-evidence-audit-2026-07-25.md @@ -0,0 +1,183 @@ +# Distinct-evidence audit of the three licensed-ledger producers + +**Date:** 2026-07-25 · **Arc:** curriculum-license-loop Phase B · +**Instrument:** `core/reliability_gate/evidence.py` · +**Pin:** `tests/test_volume_honesty.py` · **Governing rule:** ADR-0264 R9 +**Base:** main @ `6ada6f7a` + +## Summary + +Phase B set out to write an anti-gaming gate for a curriculum practice producer +that does not exist yet. Running the instrument against the producers that *do* +exist found the exposure already live in one of them. + +**21 of the 25 ratified `deduction_serve` bands do not clear θ_SERVE = 0.99 on +distinct evidence.** Three of them inflate **28 distinct cases into 720 +committed**. `deduction_serving_enabled` was ratified ON on 2026-07-24 and is +live. + +The other two producers are clean. `estimation` (ADR-0175) commits 660 distinct +cases per class with zero repeats — just above the 657 a perfect record needs. +The GSM8K practice corpus is 150 cases, 150 distinct. So this is **a regression +from a standard already established in the repository**, not a gap in the +architecture and not something nobody had thought about. + +Nothing was repaired. The deduction ledger is SHA-sealed, ratified, and gating a +live flag; changing it is Shay's ratification. The exposure is measured, pinned +in both directions, and written down here. + +## Why replay is not evidence + +`conservative_floor` (ADR-0175 §4a) is a one-sided Wilson lower bound on a +success proportion. Wilson assumes **independent trials**. + +CORE's pipeline is deterministic — that is the founding property, not an +implementation detail. So submitting an identical case a second time is not a +second trial; it is the same trial with a guaranteed outcome. Two identical cases +in a practice ledger are perfectly correlated, and contribute one trial's worth of +evidence between them. + +It follows that a ledger's `committed` count is an **upper bound** on its +evidence, and the defensible figure is its distinct-decision count. For a perfect +record the bound reduces to `n / (n + z²)`, so with `WILSON_Z = 2.576`: + +| n distinct | conservative_floor | clears θ=0.99 | +|---|---|---| +| 28 | 0.8084 | no | +| 35 | 0.8406 | no | +| 98 | 0.9366 | no | +| 294 | 0.9779 | no | +| 474 | 0.9862 | no | +| 656 | 0.98999 | no | +| **657** | **0.99001** | **yes** | +| 720 | 0.9909 | yes | + +657 is derived, not chosen: `n/(n + 6.635776) ≥ 0.99 ⟺ n ≥ 656.93`. +`volume_for_theta` recomputes it from `conservative_floor` so the figure cannot go +stale in a docstring. + +S6 predicted exactly this in the abstract — `conservative_floor` "cannot +distinguish 657 independent facts from 16 facts asked 41 times" +(`docs/research/curriculum-volume-quantification-2026-07-24.md` §1). What is new +is that it is not hypothetical. + +## Measurement + +Key = the case **text**. Two cases with identical text are indisputably the same +decision. A tighter key — the normalized propositional atom — would additionally +collapse spelling variants of one question and report *more* inflation. So +text-identity **under-reports**, which is the correct direction: every number +below is a floor on the real gap. + +``` +theta=0.99 perfect-record volume needed=657 + +band committed distinct x claimed honest clears +en_condmem_disjunctive 720 28 25.7 0.9909 0.8084 NO +en_exist_chain 720 28 25.7 0.9909 0.8084 NO +en_exist_negative 720 28 25.7 0.9909 0.8084 NO +en_exist_witness 720 35 20.6 0.9909 0.8406 NO +en_exist_universal 720 46 15.7 0.9909 0.8739 NO +en_condmem_chain 720 52 13.8 0.9909 0.8868 NO +en_member_chain 720 52 13.8 0.9909 0.8868 NO +en_member_negative 720 52 13.8 0.9909 0.8868 NO +en_condmem_conditional 720 60 12.0 0.9909 0.9004 NO +en_condmem_fused 720 60 12.0 0.9909 0.9004 NO +en_member_atomic 720 60 12.0 0.9909 0.9004 NO +en_member_single 720 60 12.0 0.9909 0.9004 NO +atomic 720 98 7.3 0.9909 0.9366 NO +categorical 720 294 2.4 0.9909 0.9779 NO +conditional_chain 720 294 2.4 0.9909 0.9779 NO +conditional_single 720 294 2.4 0.9909 0.9779 NO +disjunctive 720 294 2.4 0.9909 0.9779 NO +en_verb_chain 720 308 2.3 0.9909 0.9789 NO +en_verb_negative 720 308 2.3 0.9909 0.9789 NO +en_atomic 720 474 1.5 0.9909 0.9862 NO +en_conditional_single 720 474 1.5 0.9909 0.9862 NO +en_verb_fact 720 660 1.1 0.9909 0.9909 yes +en_verb_universal 720 660 1.1 0.9909 0.9909 yes +en_conditional_chain 720 720 1.0 0.9909 0.9909 yes +en_disjunctive 720 720 1.0 0.9909 0.9909 yes + +estimation +converse:parent_of 660 660 1.0 0.9900 0.9900 yes +converse:sibling_of 660 660 1.0 0.9900 0.9900 yes +``` + +Note the `claimed` column: **identical for every deduction band**, because every +band commits 720 with a perfect record. The gate sees 25 indistinguishable +passes. The `honest` column spans 0.8084 to 0.9909. + +The sealed artifact matches the producer band-for-band +(`test_deduction_sealed_ledger_matches_the_producer_it_names`), so this is a fact +about `chat/data/deduction_serve_ledger.json` — the file the engine actually +reads — and not only about the generator. + +`CASES_PER_BAND = 720` is a constant applied uniformly to bands whose template × +vocabulary space ranges from 28 to 720 distinct instances. The producer fills the +quota by cycling. That is the mechanism, and it is a single constant, which is +part of why it went unnoticed. + +## What this does and does not establish + +**Established.** `wrong = 0` still holds — no band answered anything +incorrectly, and none of the pinned lanes were wrong about the answers they gave. +What is not established is *reliability at the volume claimed*. A band that +decided 28 distinct questions correctly has earned the floor for 28 distinct +questions (0.8084), not the floor for 720 (0.9909). The licences on 21 bands +rest on a bound whose independence precondition does not hold. + +**Not established.** That any served answer was wrong. That the bands cannot +reach 657 distinct — most of them plainly can, by widening the template +vocabulary the way `en_conditional_chain` and `en_disjunctive` already do. The +remedy looks mechanical; it is the *authority to re-seal a ratified ledger* that +is not. + +## Open ruling for Shay — outcome mix + +Separate from distinctness, and deliberately **not** pinned by +`tests/test_volume_honesty.py`. + +`ClassTally` has no verdict axis: `correct` and `wrong` are its only outcome +fields, so a case answered correctly as UNKNOWN is indistinguishable from one +answered correctly as ENTAILED. The gate therefore cannot see mix at all, which +is why any mix rule has to live in the producer. + +The deduction producer already handles this well by construction — every band is +exactly 240 entailed / 240 refuted / 240 unknown, and `gold.py` says so +explicitly ("each band mixes entailed/refuted/unknown gold so reliability +measures decision quality"). ADR-0262 §5.1's "≈⅓ entailed" target is that +practice, restated. + +The open question is whether each verdict class needs **its own** volume. The +claim a SERVE licence makes covers entailed, refuted and unknown answers alike, +yet the smallest per-band verdict-class count across the deduction ledger is +**120** (`en_atomic`), and the largest is 240 — every one of them below 657. + +- Imposing a per-class floor would retroactively fail all 25 deduction bands on a + criterion no ADR has ratified. This audit does not do that. +- Not imposing one means a licence's aggregate floor can rest on a mixture where + one class is thinly sampled. + +Recorded for a ruling, with the numbers, rather than decided here. This is the +same shape as the ADR-0199 per-class-vs-Wilson-floor question and should probably +be answered once for both. + +## Reproduction + +```bash +uv run python -m pytest tests/test_volume_honesty.py -q +``` + +Ad hoc, against any tree: + +```python +from collections import Counter +from core.reliability_gate.evidence import audit_band, format_report +from evals.deduction_serve.practice.gold import all_gold_problems + +by = {} +for p in all_gold_problems(): + by.setdefault(p.class_name, []).append(p.payload["text"]) +print(format_report([audit_band(b, k) for b, k in sorted(by.items())], 0.99)) +``` diff --git a/tests/test_volume_honesty.py b/tests/test_volume_honesty.py new file mode 100644 index 00000000..a85bd61a --- /dev/null +++ b/tests/test_volume_honesty.py @@ -0,0 +1,367 @@ +"""The volume-honesty invariant — practice volume must be distinct evidence. + +ADR-0264 R9. `conservative_floor` is a one-sided Wilson lower bound, and Wilson +assumes independent trials. CORE's pipeline is deterministic, so replaying an +identical case is not a second trial — it is the same trial with a guaranteed +outcome. A ledger's ``committed`` count is therefore an upper bound on its +evidence, and the honest figure is its distinct-decision count. + +This file is the specification of that invariant, written before the curriculum +practice producer exists (Phase C). It is also the audit instrument, because the +exposure S6 predicted in the abstract +(`docs/research/curriculum-volume-quantification-2026-07-24.md` §1) turned out to +be live and measurable in the deduction ledger — see +:data:`DEDUCTION_DISTINCT_EVIDENCE` and +`docs/research/distinct-evidence-audit-2026-07-25.md`. + +Three producers back the three entries in `core.ratified_ledger.CAPABILITY_LEDGERS`: + +- **estimation** (ADR-0175) — clean. 660 committed, 660 distinct, inflation 1.0. + ``CASES_PER_CLASS = 660`` was evidently chosen just above the 657 a perfect + record needs, on distinct cases. This is the standard the other producers are + measured against, and it is why the deduction finding is a regression from an + established practice rather than a gap nobody had thought about. +- **deduction_serve** (ADR-0256) — 21 of 25 bands do not clear θ_SERVE on + distinct evidence; the worst three inflate 28 distinct cases to 720 committed. + Recorded, pinned, and NOT silently repaired: the ledger is SHA-sealed, ratified, + and gating a live flag, so changing it is Shay's ratification, not a test's. +- **curriculum_serve** (ADR-0262/0264) — producer unbuilt. The registry pin below + fails the moment it is built without an audit source, which is the forcing + function that stops Phase C from repeating the deduction pattern. + +Deliberately narrow. This pins evidence *distinctness*. It does not pin outcome +mix — whether each verdict class needs its own volume is an open ruling recorded +in the research note, and imposing it here would retroactively fail all 25 +deduction bands on a criterion no ADR has ratified. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable, Hashable +from pathlib import Path + +import pytest + +from core.ratified_ledger import CAPABILITY_LEDGERS +from core.reliability_gate import conservative_floor +from core.reliability_gate.evidence import ( + SERVE_VOLUME_AT_THETA_099, + EvidenceAudit, + audit_band, + audit_bands, + below_floor, + format_report, + volume_for_theta, +) + +REPO_ROOT = Path(__file__).resolve().parents[1] + +#: θ the sealed-practice ledgers are gated at (both ledger notes state it). +THETA_SERVE = 0.99 + + +# --------------------------------------------------------------------------- +# Audit sources, one per licensed-ledger capability. +# +# Keyed by the SAME capability names as `CAPABILITY_LEDGERS` and checked against +# it below, so a new capability cannot be added to the manifest without either an +# audit source or an explicit declaration that its producer does not exist yet. +# A hand-copied list here would fall behind the manifest exactly the way the +# workbench grounding whitelist fell behind `GROUNDING_SOURCES`. +# --------------------------------------------------------------------------- + +def _deduction_keys() -> dict[str, list[Hashable]]: + """Decision keys per band for the deduction practice corpus. + + The key is the case TEXT. Two cases with identical text are indisputably the + same decision, so text-identity *under*-reports inflation (a tighter key — + the normalized atom tuple — would collapse spelling variants and report + more). Under-reporting is the correct direction: the measured gap is a floor + on the real gap. + """ + from evals.deduction_serve.practice.gold import all_gold_problems + + out: dict[str, list[Hashable]] = {} + for problem in all_gold_problems(): + out.setdefault(problem.class_name, []).append(problem.payload["text"]) + return out + + +def _estimation_keys() -> dict[str, list[Hashable]]: + from evals.determination_estimation import all_gold_problems + + out: dict[str, list[Hashable]] = {} + for problem in all_gold_problems(): + payload = problem.payload + key = repr(sorted(payload.items())) if isinstance(payload, dict) else repr(payload) + out.setdefault(problem.class_name, []).append(key) + return out + + +#: capability -> keys provider, or ``None`` when the producer is not built. +#: ``None`` is a declaration, not an omission: :func:`test_every_licensed_capability_has_an_audit_source` +#: accepts it only while the capability's ledger is genuinely absent. +AUDIT_SOURCES: dict[str, Callable[[], dict[str, list[Hashable]]] | None] = { + "estimation": _estimation_keys, + "deduction_serve": _deduction_keys, + "curriculum_serve": None, # Phase C — see module docstring +} + + +#: Measured 2026-07-25 at main @ `6ada6f7a`. Every band is 720 committed. +#: This is an EXPOSURE INVENTORY, not an approved baseline: 21 of these 25 do not +#: clear θ_SERVE=0.99 on distinct evidence. It is pinned so the exposure cannot +#: grow or drift unnoticed while the ratification question is open. +DEDUCTION_DISTINCT_EVIDENCE: dict[str, int] = { + "atomic": 98, + "categorical": 294, + "conditional_chain": 294, + "conditional_single": 294, + "disjunctive": 294, + "en_atomic": 474, + "en_conditional_chain": 720, + "en_conditional_single": 474, + "en_condmem_chain": 52, + "en_condmem_conditional": 60, + "en_condmem_disjunctive": 28, + "en_condmem_fused": 60, + "en_disjunctive": 720, + "en_exist_chain": 28, + "en_exist_negative": 28, + "en_exist_universal": 46, + "en_exist_witness": 35, + "en_member_atomic": 60, + "en_member_chain": 52, + "en_member_negative": 52, + "en_member_single": 60, + "en_verb_chain": 308, + "en_verb_fact": 660, + "en_verb_negative": 308, + "en_verb_universal": 660, +} + +#: The four that DO clear on distinct evidence. Derived below, not restated. +_EXPECTED_CLEAN = {"en_conditional_chain", "en_disjunctive", "en_verb_fact", "en_verb_universal"} + + +@pytest.fixture(scope="module") +def deduction_audits() -> tuple[EvidenceAudit, ...]: + return audit_bands(_deduction_keys()) + + +@pytest.fixture(scope="module") +def estimation_audits() -> tuple[EvidenceAudit, ...]: + return audit_bands(_estimation_keys()) + + +# --------------------------------------------------------------------------- +# 1. The instrument itself must be correct, and must fail under mutation. +# --------------------------------------------------------------------------- + +def test_serve_volume_is_derived_from_the_pinned_floor() -> None: + """657 must be computed, never trusted as a literal. + + If `WILSON_Z` ever moves, a hard-coded 657 becomes silently wrong in every + doc and test that repeats it. This asserts the derivation and the boundary. + """ + assert volume_for_theta(THETA_SERVE) == SERVE_VOLUME_AT_THETA_099 == 657 + n = SERVE_VOLUME_AT_THETA_099 + assert conservative_floor(n, n) >= THETA_SERVE + assert conservative_floor(n - 1, n - 1) < THETA_SERVE, ( + "657 must be the SMALLEST perfect-record volume clearing theta" + ) + + +def test_audit_detects_padding() -> None: + """The mutation check. Pad a corpus and the instrument must catch it. + + Both corpora below commit exactly 720 cases with a perfect record, so + `conservative_floor` returns the *same passing number* for each and the gate + as it stands cannot tell them apart. One exercised 720 decisions; the other + exercised 28. That gap is the whole point. + """ + honest = audit_band("honest", [f"case-{i}" for i in range(720)]) + padded = audit_band("padded", [f"case-{i % 28}" for i in range(720)]) + + assert honest.committed == padded.committed == 720 + assert honest.claimed == padded.claimed >= THETA_SERVE, ( + "the existing gate must be shown to see these as identical" + ) + + assert honest.distinct == 720 + assert honest.inflation == 1.0 + assert honest.max_repeat == 1 + assert honest.clears(THETA_SERVE) + + assert padded.distinct == 28 + assert padded.inflation > 25 + assert padded.max_repeat > 25 + assert not padded.clears(THETA_SERVE) + assert padded.honest < 0.81 + + +def test_a_single_repeat_is_enough_to_move_the_honest_floor() -> None: + """No dead zone: one duplicate in an otherwise-exact-threshold corpus drops it.""" + exact = audit_band("exact", [f"c-{i}" for i in range(657)]) + assert exact.clears(THETA_SERVE) + # Replace one distinct case with a duplicate: same committed, 656 distinct. + padded = audit_band("padded", [f"c-{i}" for i in range(656)] + ["c-0"]) + assert padded.committed == 657 + assert padded.distinct == 656 + assert not padded.clears(THETA_SERVE) + + +def test_tighter_keys_can_only_report_more_inflation() -> None: + """Key choice is safe in one direction only, and that is the direction used. + + A coarser key (case text) merges fewer cases than a finer one (normalized + atom). So an audit over text is a lower bound on the audit over atoms, and a + finding built on it cannot be an exaggeration. + """ + coarse = ["Does a require b?", "Does a requires b?", "Does c require d?"] + fine = ["(a,requires,b)", "(a,requires,b)", "(c,requires,d)"] + assert audit_band("coarse", coarse).inflation <= audit_band("fine", fine).inflation + + +def test_audit_rejects_impossible_counts() -> None: + with pytest.raises(ValueError, match="cannot exceed"): + EvidenceAudit(band="b", committed=5, distinct=6, max_repeat=1) + with pytest.raises(ValueError, match="no distinct keys"): + EvidenceAudit(band="b", committed=5, distinct=0, max_repeat=0) + + +# --------------------------------------------------------------------------- +# 2. Every licensed capability must be audited — the anti-silence guard. +# --------------------------------------------------------------------------- + +def test_every_licensed_capability_has_an_audit_source() -> None: + """Derived from `CAPABILITY_LEDGERS`, so a new capability cannot slip past. + + This is the forcing function for Phase C: the moment + `chat/data/curriculum_serve_ledger.json` exists, a ``None`` audit source + stops being an honest declaration and this fails. + """ + assert set(AUDIT_SOURCES) == set(CAPABILITY_LEDGERS), ( + "AUDIT_SOURCES and CAPABILITY_LEDGERS disagree — a licensed capability is " + "unaudited, or an audit source names a capability that is not licensed" + ) + for capability, source in sorted(AUDIT_SOURCES.items()): + if source is not None: + continue + ledger_path = CAPABILITY_LEDGERS[capability].path + assert not ledger_path.exists(), ( + f"{capability}: ledger {ledger_path.name} now exists but has no audit " + f"source. Register one in AUDIT_SOURCES — a sealed ledger whose " + f"distinct evidence is unmeasured is exactly the ADR-0264 R9 exposure." + ) + + +@pytest.mark.parametrize("capability", sorted(c for c, s in AUDIT_SOURCES.items() if s)) +def test_audited_capability_bands_are_non_degenerate(capability: str) -> None: + """Whatever a producer emits, it must be auditable and non-empty.""" + audits = audit_bands(AUDIT_SOURCES[capability]()) # type: ignore[misc] + assert audits, f"{capability}: producer yielded no bands" + for audit in audits: + assert audit.committed > 0 + assert audit.distinct > 0 + assert audit.inflation >= 1.0 + + +# --------------------------------------------------------------------------- +# 3. The estimation producer — the clean standard the invariant is calibrated to. +# --------------------------------------------------------------------------- + +def test_estimation_producer_has_no_inflation( + estimation_audits: tuple[EvidenceAudit, ...], +) -> None: + """ADR-0175's producer commits distinct cases only, just above the threshold. + + Recorded as a test because it is the existence proof that the invariant is + satisfiable and was in fact satisfied before ADR-0256's producer: 660 + distinct cases against the 657 a perfect record needs. + """ + assert [a.band for a in estimation_audits] == [ + "converse:parent_of", + "converse:sibling_of", + ] + for audit in estimation_audits: + assert audit.committed == audit.distinct == 660, format_report( + estimation_audits, THETA_SERVE + ) + assert audit.max_repeat == 1 + assert audit.clears(THETA_SERVE) + assert below_floor(estimation_audits, THETA_SERVE) == () + + +# --------------------------------------------------------------------------- +# 4. The deduction producer — the measured exposure, pinned in both directions. +# --------------------------------------------------------------------------- + +def test_deduction_distinct_evidence_matches_the_recorded_exposure( + deduction_audits: tuple[EvidenceAudit, ...], +) -> None: + """Pins the exact inventory, so the exposure can neither grow nor drift. + + Fails if a band's distinct count changes in EITHER direction. A drop is the + exposure worsening. A rise means someone improved a band — welcome, and the + inventory must then be updated deliberately rather than by a silently + loosening test. + """ + measured = {a.band: a.distinct for a in deduction_audits} + assert measured == DEDUCTION_DISTINCT_EVIDENCE, format_report( + deduction_audits, THETA_SERVE + ) + for audit in deduction_audits: + assert audit.committed == 720, "CASES_PER_BAND changed — re-audit the ledger" + + +def test_deduction_exposure_membership_is_exactly_as_recorded( + deduction_audits: tuple[EvidenceAudit, ...], +) -> None: + """Which bands fail is derived from the floor, not restated alongside it.""" + failing = {a.band for a in below_floor(deduction_audits, THETA_SERVE)} + clean = {a.band for a in deduction_audits} - failing + assert clean == _EXPECTED_CLEAN, format_report(deduction_audits, THETA_SERVE) + assert len(failing) == 21, ( + f"expected 21 bands below the distinct-evidence floor, found {len(failing)}" + ) + + +def test_deduction_sealed_ledger_matches_the_producer_it_names( + deduction_audits: tuple[EvidenceAudit, ...], +) -> None: + """The exposure is in the shipped artifact, not only in the generator. + + Without this the finding could be dismissed as being about a generator that + the sealed ledger does not reflect. The ledger's committed counts are the + producer's committed counts, band for band. + """ + path = CAPABILITY_LEDGERS["deduction_serve"].path + ledger = json.loads(path.read_text(encoding="utf-8")) + classes = ledger["classes"] + assert ledger["provenance"] == "evals.deduction_serve.practice.runner.seal_ledger" + produced = {a.band: a.committed for a in deduction_audits} + sealed = {name: tally["correct"] + tally["wrong"] for name, tally in classes.items()} + assert sealed == produced, ( + "sealed ledger committed counts diverge from the producer — the audit " + "below would then be measuring something the engine does not read" + ) + assert all(tally["wrong"] == 0 for tally in classes.values()), "wrong=0 must hold" + + +def test_recorded_exposure_is_declared_in_a_decision_record() -> None: + """A pinned exposure with no decision record behind it is just a stale test. + + Ties this file to the ADR that rules on it, so the inventory above cannot + outlive the reasoning that justified leaving it in place. + """ + adr = REPO_ROOT / "docs" / "adr" / "ADR-0264-negative-curriculum-and-premise-scope.md" + assert adr.exists(), "ADR-0264 governs this invariant and must be present" + text = adr.read_text(encoding="utf-8") + assert "R9" in text, "ADR-0264 must state the R9 volume-honesty rule" + research = REPO_ROOT / "docs" / "research" / "distinct-evidence-audit-2026-07-25.md" + assert research.exists(), ( + "the measured audit must stay written down — a 21-band exposure recorded " + "only in a test's data structure is not a finding anyone will read" + )