core-labs/core
core-labs/core
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request 'Curriculum sealed-practice producer — and the ledger it deliberately does not commit (Phase C)' (#121) from feat/curriculum-practice-producer into main

Browse source
This commit is contained in:
Joshua Matthew-Catudio Shay 2026-07-26 20:19:45 +00:00
commit e336ddfd77
14 changed files with 2346 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

122
core/cli_proposal_queue.py
View file

@ -119,6 +119,7 @@ def cmd_proposal_queue_ratify(args: argparse.Namespace) -> int:
reviewer=args.reviewer,
rationale=args.rationale,
intent=args.intent,
polarity=args.polarity,
)
receipt = ratify_chain(record, dry_run=args.dry_run)
except RatificationError as exc:
@ -138,10 +139,90 @@ def cmd_proposal_queue_ratify(args: argparse.Namespace) -> int:
f"{receipt.family_chains_before} -> {receipt.family_chains_after}")
if not args.dry_run:
print(f" still pending : {', '.join(receipt.pending_stages)}")
# Wiring the receipt's `ledger_reseal` stage to the verb that performs
# it. Naming a pending stage without naming its command is how this one
# stayed unperformed since the ceremony was written.
if "ledger_reseal" in receipt.pending_stages:
print(
" next : core proposal-queue reseal curriculum_serve "
"--dry-run (the ledger is stale until you do)"
)
return 0
def cmd_proposal_queue_reseal(args: argparse.Namespace) -> int:
"""The ``ledger_reseal`` stage of the ceremony — explicit, never automatic.
``ratify_chain`` grows the corpus and names this stage in its receipt's
``pending_stages``; nothing performed it until now, so a sealed ledger went
stale the moment curriculum changed. This is the verb a human runs.
It refuses by default to grant a license. A band can clear θ_SERVE on correct
NON-COMMITMENTS alone, so "regenerate the artifact" and "make four bands
authoritative" would otherwise be the same keystroke.
"""
from teaching.ledger_reseal import ReadyToReseal, apply_reseal, plan_reseal
try:
plan = plan_reseal(args.capability)
except ReadyToReseal as exc:
print(f"REFUSED: {exc}")
return 2
payload: dict[str, object] = {
"capability": plan.capability,
"path": str(plan.path),
"licensed_before": list(plan.licensed_before),
"licensed_after": list(plan.licensed_after),
"newly_licensed": list(plan.newly_licensed),
"revoked": list(plan.revoked),
"committed": plan.committed,
"mix": plan.mix,
"is_housekeeping": plan.is_housekeeping,
"written": False,
}
if not args.json:
print(f"reseal plan for {plan.capability}:")
print(f" artifact : {plan.path}")
for band in sorted(plan.committed):
mix = ", ".join(f"{k}={v}" for k, v in sorted(plan.mix.get(band, {}).items()))
lic = "SERVE" if band in plan.licensed_after else " "
print(f" {lic} {band:44s} committed={plan.committed[band]:6d} {mix}")
print(f" licensed now : {len(plan.licensed_before)}")
print(f" licensed after: {len(plan.licensed_after)}")
if plan.newly_licensed:
print(f" NEWLY LICENSED: {', '.join(plan.newly_licensed)}")
if plan.revoked:
print(f" revoked : {', '.join(plan.revoked)}")
if args.dry_run:
if args.json:
print(json.dumps(payload, indent=2, sort_keys=True))
else:
print(" (dry run — nothing written)")
return 0
try:
path = apply_reseal(plan, allow_new_licenses=args.allow_new_licenses)
except ReadyToReseal as exc:
if args.json:
payload["refused"] = str(exc)
print(json.dumps(payload, indent=2, sort_keys=True))
else:
print(f"REFUSED: {exc}")
return 1
if args.json:
payload["written"] = True
print(json.dumps(payload, indent=2, sort_keys=True))
else:
print(f" SEALED -> {path}")
return 0
def register(subparsers: argparse._SubParsersAction) -> None:
from teaching.curriculum_premises import AFFIRMATIVE, POLARITIES
"""Attach the ``core proposal-queue`` subcommand tree to a top-level parser."""
queue = subparsers.add_parser(
"proposal-queue",
@ -195,7 +276,7 @@ def register(subparsers: argparse._SubParsersAction) -> None:
"curriculum loader admits the chain -> receipt. Refuses, and rolls "
"back, if the appended row would be silently dropped. Does not "
"write a ledger (bridge rule 1) or queue an arena entry; the "
"receipt names those stages."
"receipt names those stages, and `reseal` performs the first."
),
)
ratify.add_argument("domain")
@ -205,10 +286,49 @@ def register(subparsers: argparse._SubParsersAction) -> None:
ratify.add_argument("--reviewer", required=True, help="who ratified this")
ratify.add_argument("--rationale", required=True, help="why it is ratified")
ratify.add_argument("--intent", default="cause")
ratify.add_argument(
"--polarity", choices=list(POLARITIES), default=AFFIRMATIVE,
help=(
"ADR-0264 R1. `negative` ratifies an explicitly TAUGHT refutation: "
"the atom compiles under the sentential-negation prefix and the "
"question answers `refuted`, which is different from an absent edge "
"(UNKNOWN, the open-world reading). Must reuse the AFFIRMATIVE "
"connective (R3) — `causes`, never `does not cause` — or it mints a "
"different atom and silently fails to refute. Refused if the atom is "
"already taught with the other polarity (R4)."
),
)
ratify.add_argument("--dry-run", action="store_true",
help="report the band delta without writing")
ratify.add_argument("--json", action="store_true")
ratify.set_defaults(func=cmd_proposal_queue_ratify)
from teaching.ledger_reseal import resealable_capabilities
reseal = sub.add_parser(
"reseal",
help="re-run sealed practice and rewrite a capability's committed ledger",
description=(
"The `ledger_reseal` stage `ratify_chain` names but never performs. "
"Ratifying a chain grows the corpus without re-running practice, so "
"the ledger the serving gate reads goes stale. This rebuilds it from "
"sealed practice. REFUSES by default if the reseal would newly "
"license a band: reliability is commitment precision, so a band can "
"clear theta_SERVE on correct NON-COMMITMENTS alone, and granting a "
"serving license is a ratification rather than housekeeping."
),
)
reseal.add_argument("capability", choices=resealable_capabilities())
reseal.add_argument(
"--dry-run", action="store_true",
help="report the license delta without writing",
)
reseal.add_argument(
"--allow-new-licenses", action="store_true",
help="authorize bands that would become newly licensed (a RATIFICATION)",
)
reseal.add_argument("--json", action="store_true")
reseal.set_defaults(func=cmd_proposal_queue_reseal)
__all__ = ["register"]

9
core/cli_test.py
View file

@ -85,6 +85,12 @@ TEST_SUITES: dict[str, tuple[str, ...]] = {
# an audit source. It gates a live, flag-ON serving path, so it belongs
# on the pre-push gate. ~1.5s.
"tests/test_volume_honesty.py",
# ADR-0264 R1-R4/R8 — taught curriculum NEGATIVES. Before this, a row
# authored to refute compiled to a premise ASSERTING the atom, and the
# independent oracle ignored polarity too, so gold agreed and wrong=0
# stayed green. Corpus-level epistemology is unfalsifiable by every
# other gate here, so this one belongs on the pre-push gate. ~0.4s.
"tests/test_curriculum_polarity.py",
),
"runtime": (
"tests/test_chat_runtime.py",
@ -229,6 +235,9 @@ TEST_SUITES: dict[str, tuple[str, ...]] = {
"tests/test_exist_argument_reader.py",
"tests/test_deduction_serve_e2e.py",
"tests/test_curriculum_serve.py",
"tests/test_curriculum_practice.py",
"tests/test_curriculum_polarity.py",
"tests/test_ledger_reseal.py",
"tests/test_ratified_ledger_bridge.py",
"tests/test_vocab_trigger_instrument.py",
),

166
docs/research/curriculum-practice-producer-2026-07-26.md Normal file
View file

@ -0,0 +1,166 @@
# Curriculum practice producer — what building it measured (Phase C)
**Date:** 2026-07-26 · **Arc:** curriculum-license-loop-2026-07 · **Unit:** Phase C
**Code:** `evals/curriculum_serve/practice/{generator,runner}.py`,
`tests/test_curriculum_practice.py`
**Governs:** ADR-0262 §5 (what a curriculum band can earn), ADR-0264 R9 (volume honesty)
The producer named by `chat/curriculum_serve_license.py`'s own docstring —
`evals.curriculum_serve.practice.runner.seal_ledger`, declared "the only writer"
of the ledger that module reads — did not exist. It does now. Building it
measured three things, two of which correct records I wrote earlier in this arc.
Reproduce everything below with:
```bash
uv run python -m evals.curriculum_serve.practice.runner # capped corpus
uv run python -m evals.curriculum_serve.practice.runner --full # every atom (~75s)
```
---
## 0. What the producer is
A band's case space is **enumerated, not authored**: every exam question that
routes to that subject (`chat.curriculum_surface.resolve_domain`) whose relation
falls in that family. One committed case per distinct query atom, so
`committed == distinct` is not a property the generator maintains — it is the same
fact twice. This is the structural version of what ADR-0264 R9 asks for, and it is
why the curriculum producer cannot develop the deduction producer's exposure
(`CASES_PER_BAND = 720` filled by cycling spaces as small as 28).
`CASES_PER_BAND = 660` follows `evals/determination_estimation`, per
`DIVISION-OF-WORK.md` §4. It is a **ceiling on committed volume, not a quota**: a
band with a smaller space emits its whole space and reports a number below the
floor. Selection above the cap is every taught edge plus a fixed stride across the
rest — a lexicographic prefix of 660 out of 45,300 would cover about five subject
terms of 151 and report one corner of a band as the band.
Result over the capped corpus: **11 bands, wrong = 0, inflation 1.0 everywhere.**
---
## 1. Finding — a committed ledger is necessarily an *earning* ledger
The plan of record's Phase-1 exit criterion asks that
`curriculum_serve_license()` read "a real (still-unearned) ledger instead of an
absent one."
**That state is unreachable.** Reliability is commitment precision, and a correct
UNKNOWN is a commitment. `conservative_floor(660, 660) = 0.990046 ≥ 0.99`, so any
band with ≥657 routable atoms clears θ_SERVE on non-commitments alone, whatever its
curriculum says. Sealing today licenses **four** bands:
| band | committed | refused | reliability | SERVE | entailed | entailed share |
|---|---:|---:|---:|:---:|---:|---:|
| `curriculum_philosophy_theology_modal` | 45,188 | 112 | 0.99985 | **yes** | 8 | 0.0177% |
| `curriculum_philosophy_theology_contrast` | 22,598 | 52 | 0.99971 | **yes** | 8 | 0.0354% |
| `curriculum_physics_causal` | 720 | 0 | 0.99087 | **yes** | 7 | 0.9722% |
| `curriculum_systems_software_causal` | 720 | 0 | 0.99087 | **yes** | 7 | 0.9722% |
| `curriculum_physics_modal` | 480 | 0 | 0.98636 | no | 9 | 1.8750% |
| `curriculum_mathematics_logic_modal` | 480 | 0 | 0.98636 | no | 4 | 0.8333% |
| `curriculum_systems_software_modal` | 480 | 0 | 0.98636 | no | 6 | 1.2500% |
| `curriculum_mathematics_logic_contrast` | 240 | 0 | 0.97309 | no | 8 | 3.3333% |
| `curriculum_mathematics_logic_evidential` | 240 | 0 | 0.97309 | no | 8 | 3.3333% |
| `curriculum_mathematics_logic_sequence` | 240 | 0 | 0.97309 | no | 4 | 1.6667% |
| `curriculum_systems_software_sequence` | 240 | 0 | 0.97309 | no | 3 | 1.2500% |
(Exhaustive sweep: 71,790 atoms, `wrong = 0`, 164 refusals.)
A licensed band would therefore be certified on evidence that is **99.0%–99.98%
correct non-commitment**. ADR-0262 §5.1 explicitly rules that unacceptable. The
gate cannot see it: `ClassTally` carries `correct`/`wrong`/`refused` and **no
verdict axis**, so a correct UNKNOWN is indistinguishable from a correct ENTAILED
once tallied. Mix can only be enforced at the producer, which is exactly the
open outcome-mix ruling recorded in
`docs/research/distinct-evidence-audit-2026-07-25.md`.
**Consequence for this arc:** that ruling is no longer academic — it is the last
thing between a built loop and four unearned licenses. Phase C therefore ships the
writer and **does not commit the artifact**; `chat/data/curriculum_serve_ledger.json`
stays absent, `missing_ok=True` stays true, and every band stays DISCLOSED. Writing
it is a ratification, so it belongs to the explicit `core proposal-queue reseal`
verb a human runs (Phase D), and it should not be run before the mix ruling.
`tests/test_curriculum_practice.py::test_curriculum_ledger_is_not_committed` pins
the absence, and `test_sealing_would_license_exactly_the_bands_that_reach_the_floor`
pins the consequence, so neither can be lost and re-derived wrongly.
### Recommendation (still Shay's call)
Enforce mix **at the producer**, as a per-verdict-class floor rather than a ratio:
a band earns SERVE only when each verdict class it can express independently
reaches the volume the floor requires. Under that rule no band earns anything today
(max entailed volume is 9), which is the honest reading, and it makes ADR-0262 §5.1
mechanical instead of advisory. It also keeps the deduction bands' aggregate
licenses intact, since imposing a per-class 657 retroactively would fail all 25.
---
## 2. Correction — ADR-0264 §4.2's ceiling table understates every band
I sized bands from **per-term exclusivity** (a term taught by only one subject).
The router's actual predicate is **per-pair**: `resolve_domain` routes a question
iff exactly one served subject holds *both* terms. A term taught in two subjects
can still appear in a pair only one subject holds both halves of, so per-term
exclusivity is a strictly tighter bound and under-counts.
| band | ADR-0264 §4.2 | true | verdict change |
|---|---:|---:|---|
| `curriculum_systems_software_causal` | 630 | **720** | **cannot reach 657 → CAN** |
| `curriculum_philosophy_theology_modal` | 44,104 | 45,300 | — |
| `curriculum_philosophy_theology_contrast` | 22,052 | 22,650 | — |
| `curriculum_mathematics_logic_modal` | 420 | 480 | — |
| `curriculum_mathematics_logic_{sequence,contrast,evidential}` | 210 | 240 | — |
| `curriculum_systems_software_modal` | 420 | 480 | — |
| `curriculum_systems_software_sequence` | 210 | 240 | — |
| `curriculum_physics_{causal,modal}` | 720 / 480 | 720 / 480 | — |
So **4 of 11 bands can reach 657, not 3**, and **7 cannot, not 8**. The retarget
conclusion is unaffected — `philosophy_theology · modal` is still the largest space
by two orders of magnitude, and `physics · modal` (480) is still impossible at any
authoring volume. The numbers are now pinned in
`tests/test_curriculum_practice.py::BAND_ATOM_SPACE`, measured rather than derived,
so this class of error cannot recur silently.
---
## 3. Finding — two taught lemmas collide with the argument reader's grammar
`then` and `therefore` are lemmas `philosophy_theology`'s packs teach **and**
control words of the argument reader: `therefore` is literally the conclusion
marker in `". Therefore <conclusion>."`, and `then` is the conditional consequent
marker. An atom using either as a term compiles an argument the reader cannot parse
back, so it refuses `compiled_premises_unreadable`.
- **164 routable atoms** affected (112 modal + 52 contrast) out of 71,790.
- All 164 are **coverage misses, not wrongs** — excluded from reliability's
denominator (ADR-0175 §4), never a confabulation. `wrong = 0` holds.
- Exactly two lemmas are responsible; every other implicated term is only ever the
*partner* of one of them.
- It is in the band Phase F retargets to.
The narrow cost is small. The mechanism is not: **the vocabulary boundary and the
reader's grammar are never screened against each other.** A future pack teaching
`and`, `or`, `not`, or `if` would open a much larger hole the same way, and it would
present as a coverage dip rather than an error. A reserved-word check at pack mount
or at chain ratification would close it.
Not fixed here — outside Phase C's unit (`DIVISION-OF-WORK.md` §6.5). Pinned as a
bound by `test_the_collision_is_bounded_to_two_lemmas`, so a third reserved word
entering the curriculum fails loudly instead of quietly enlarging the gap.
---
## 4. What did not move
- `evals/curriculum_serve` lane: `[physics] n=32 correct=32 wrong=0 declined_mismatch=0
anti_recall=5`, unchanged; `report.json` bytes and the `curriculum_serve_v1` SHA
pin unchanged. The producer reads the same solver the lane does and writes nothing
the lane reads.
- No serving behaviour change of any kind: no ledger committed, so
`curriculum_serve_license()` still returns `None` for every band and every
curriculum answer is still served DISCLOSED.
- `refuted` remains unreachable — no corpus row can express a negative until
ADR-0264 R1–R4 is implemented. `test_gold_mix_has_no_refuted_class` pins that as a
red test for the polarity unit to turn green.

63
evals/curriculum_serve/oracle.py
View file

@ -79,13 +79,34 @@ def _lemmas(pack_id: str) -> set[str]:
return out
def _edges(domain: str) -> list[tuple[str, str, str, str]]:
"""``(subject, connective, object, chain_id)`` for every ratified row."""
#: ADR-0264 R1/R8, restated independently — the serving path has its own copy of
#: this vocabulary and the two must not share a polarity helper. Code-level
#: independence is the whole evidentiary value of this oracle: if it imported the
#: compiler's notion of polarity, agreement would only prove the compiler agrees
#: with itself, and a polarity bug would be invisible on both sides at once.
_AFFIRMATIVE = "affirmative"
_NEGATIVE = "negative"
_POLARITIES = (_AFFIRMATIVE, _NEGATIVE)
def _row_polarity(row: dict) -> str | None:
"""This oracle's own reading of a row's polarity.
Absent ⇒ affirmative. An unrecognized token ⇒ ``None``, and the caller drops
the row: reading an unknown value as affirmative would let a row someone
wrote to refute be scored as an assertion.
"""
value = str(row.get("polarity") or _AFFIRMATIVE).strip().lower()
return value if value in _POLARITIES else None
def _edges(domain: str) -> list[tuple[str, str, str, str, str]]:
"""``(subject, connective, object, chain_id, polarity)`` per ratified row."""
corpora, packs = _DOMAIN_SOURCES[domain]
vocabulary = set()
for pack in packs:
vocabulary |= _lemmas(pack)
out: list[tuple[str, str, str, str]] = []
out: list[tuple[str, str, str, str, str]] = []
for corpus in corpora:
path = _CHAIN_DIR / f"{corpus}.jsonl"
if not path.exists():
@ -105,7 +126,12 @@ def _edges(domain: str) -> list[tuple[str, str, str, str]]:
continue
if connective not in _FAMILY_OF_CONNECTIVE:
continue
out.append((subject, connective, obj, row.get("chain_id", "")))
polarity = _row_polarity(row)
if polarity is None:
continue
out.append(
(subject, connective, obj, row.get("chain_id", ""), polarity)
)
return out
@ -159,12 +185,28 @@ def oracle_answer(domain: str, subject: str, relation: str, obj: str) -> OracleV
# curriculum teaching "entropy reveals energy" has not thereby taught
# "entropy causes energy". Family scoping decides which premises are in
# play; the connective decides what was actually said.
if any(s == subject and c == connective and o == obj for s, c, o, _id in edges):
return OracleVerdict("entailed", "", family, 1)
#
# ADR-0264 R1/R8 — and the POLARITY decides which way. A taught negative row
# is a taught refutation: the curriculum has said something about this atom,
# and what it said is "no". That is categorically different from an absent
# edge, which stays UNKNOWN under the open-world reading. Both are checked at
# depth 1, against the same atom, so the two cannot be confused.
for s, c, o, _id, polarity in edges:
if s == subject and c == connective and o == obj:
if polarity == _NEGATIVE:
return OracleVerdict("refuted", "", family, 1)
return OracleVerdict("entailed", "", family, 1)
# No taught edge. Report the shortest path so the lane can prove the
# serving path does NOT compose a chain into a claim.
#
# NEGATIVE rows are EXCLUDED from the adjacency (ADR-0264 R8). Reachability
# here exists to measure whether an untaught pair is *composable* from taught
# edges, and "a does not X b" supplies no step from a to b — treating it as
# one would report a path built out of a denial.
adjacency: dict[str, list[str]] = {}
for s, _c, o, _id in edges:
for s, _c, o, _id, polarity in edges:
if polarity == _NEGATIVE:
continue
adjacency.setdefault(s, []).append(o)
seen = {subject}
queue: deque[tuple[str, int]] = deque([(subject, 0)])
@ -181,8 +223,11 @@ def oracle_answer(domain: str, subject: str, relation: str, obj: str) -> OracleV
return OracleVerdict("unknown", "", family, 0)
def taught_edges(domain: str) -> list[tuple[str, str, str, str]]:
"""Public view for the lane's provenance assertions."""
def taught_edges(domain: str) -> list[tuple[str, str, str, str, str]]:
"""Public view for the lane's provenance assertions.
Each row carries its polarity as the fifth element (ADR-0264 R1).
"""
return _edges(domain)

32
evals/curriculum_serve/practice/__init__.py Normal file
View file

@ -0,0 +1,32 @@
"""Sealed practice for curriculum-grounded serving (ADR-0262/0264, Phase C).
The missing producer named by ``chat/curriculum_serve_license.py``'s docstring:
that module reads ``chat/data/curriculum_serve_ledger.json`` and declares
``evals.curriculum_serve.practice.runner.seal_ledger`` "the only writer" of it.
Until this package existed the writer did not, so authoring curriculum could not
earn a license at any volume.
"""
from evals.curriculum_serve.practice.generator import (
CASES_PER_BAND,
CurriculumOracleTether,
CurriculumSolver,
QueryAtom,
all_gold_problems,
assert_practice_atoms_distinct,
band_cases,
routable_atoms,
taught_atoms,
)
__all__ = [
"CASES_PER_BAND",
"CurriculumOracleTether",
"CurriculumSolver",
"QueryAtom",
"all_gold_problems",
"assert_practice_atoms_distinct",
"band_cases",
"routable_atoms",
"taught_atoms",
]

280
evals/curriculum_serve/practice/generator.py Normal file
View file

@ -0,0 +1,280 @@
"""The curriculum practice corpus — one committed case per DISTINCT query atom.
ADR-0262 (curriculum-grounded serving) supplies the solver; ADR-0264 R9 supplies
the sizing rule. The corpus is enumerated, never authored: a band's case space is
exactly the set of exam questions that ROUTE to its subject and whose relation
falls in its family, so there is no hand-written gold to drift and nothing to
paraphrase-pad.
**Why enumeration is the honest shape here.** ``conservative_floor`` is a Wilson
bound and Wilson assumes independent trials, so a deterministic pipeline replaying
one input N times supplies one trial's evidence, not N (ADR-0264 R9). The
``deduction_serve`` producer sets a flat ``CASES_PER_BAND = 720`` and reaches it by
cycling a template x vocabulary space as small as 28 distinct instances; 21 of its
25 ratified bands therefore do not clear θ_SERVE on distinct evidence
(``tests/test_volume_honesty.py``). This producer cannot repeat that: its case
identity IS the query atom, so ``committed == distinct`` holds by construction
rather than by discipline. The model is ``evals/determination_estimation``
(660 distinct cases per class, zero repeats), which is what
``DIVISION-OF-WORK.md`` §4 directs Phase C to follow.
**Routability is asked, not re-derived.** An atom is in a band's space iff
``chat.curriculum_surface.resolve_domain`` routes its two terms to that subject.
Restating the predicate here would let the corpus and the serving path disagree
about which questions exist — so the corpus asks the serving path. This is not a
gold dependency: gold comes from ``evals.curriculum_serve.oracle``, which shares
no code with the serving path (ADR-0199 L-2).
Note that per-*term* exclusivity is a strictly tighter bound than the per-*pair*
predicate the router actually applies: a term taught in two subjects can still
appear in a pair that only one subject holds both halves of. Sizing a band from
exclusive terms therefore UNDER-counts its space — ``systems_software · causal``
reads as 630 under that bound and is really 720, which is the difference between
"cannot reach 657" and "can".
Determinism: no clock, no RNG. The atom space is sorted, selection is a fixed
stride, so ``all_gold_problems()`` is byte-stable across runs and the sealed
ledger is safe to commit and SHA-verify on load.
"""
from __future__ import annotations
from dataclasses import dataclass
from functools import lru_cache
from chat.curriculum_surface import (
SERVED_DOMAINS,
CurriculumQuery,
band_for,
decide_curriculum_question,
resolve_domain,
)
from core.learning_arena.protocols import BaseAttempt, DomainProblem, Problem
from evals.curriculum_serve.oracle import oracle_answer
from teaching.curriculum_premises import CONNECTIVE_FAMILY, FAMILIES, load_curriculum
#: Committed cases per band, when the band's atom space is larger than this.
#:
#: 660 is ``evals/determination_estimation``'s constant, chosen just above the 657
#: a perfect record needs to clear θ_SERVE=0.99 (``volume_for_theta(0.99)``). It is
#: a ceiling on *committed volume*, not a quota to be filled: a band whose space is
#: smaller emits its whole space and honestly reports a number below the floor.
CASES_PER_BAND = 660
@dataclass(frozen=True, slots=True)
class QueryAtom:
"""One distinct curriculum decision — the unit of committed evidence.
The atom IS the decision key (ADR-0264 R9). Two cases with the same atom are
the same decision replayed, so the corpus admits each atom at most once and
:meth:`key` is what ``tests/test_volume_honesty.py`` audits.
"""
domain: str
family: str
subject: str
connective: str
obj: str
@property
def text(self) -> str:
"""The exam question, in the closed ``Does <s> <v> <o>?`` grammar."""
return f"Does {self.subject} {self.connective} {self.obj}?"
@property
def key(self) -> tuple[str, str, str, str]:
"""The distinct-decision key. Polarity is deliberately absent: a
negative curriculum row (ADR-0264 R1-R4) changes a question's ANSWER,
never its identity, and R4 rejects a corpus that states both polarities
of one atom at ratification. So polarity cannot distinguish two cases."""
return (self.domain, self.subject, self.connective, self.obj)
@property
def case_id(self) -> str:
return f"{self.domain}:{self.connective}:{self.subject}->{self.obj}"
@lru_cache(maxsize=None)
def _routes_to(subject: str, obj: str) -> str | None:
"""The subject a term pair routes to, or ``None`` — the router's own answer."""
domain = resolve_domain(CurriculumQuery(subject, "", obj))
return domain if isinstance(domain, str) else None
@lru_cache(maxsize=None)
def routable_atoms(domain: str, family: str) -> tuple[QueryAtom, ...]:
"""Every exam question that reaches *(domain, family)*, in sorted order.
The band's complete case space: ordered term pairs the router sends to
*domain*, crossed with the family's connectives. Self-pairs are excluded —
``Does x cause x?`` is not a curriculum question any corpus states.
"""
connectives = sorted(c for c, f in CONNECTIVE_FAMILY.items() if f == family)
terms = sorted(load_curriculum(domain).vocabulary)
atoms = [
QueryAtom(domain, family, subject, connective, obj)
for subject in terms
for obj in terms
if subject != obj and _routes_to(subject, obj) == domain
for connective in connectives
]
return tuple(sorted(atoms, key=lambda a: (a.subject, a.connective, a.obj)))
@lru_cache(maxsize=None)
def taught_atoms(domain: str, family: str) -> frozenset[tuple[str, str, str, str]]:
"""The atom keys a ratified chain states directly — the COMMITTED classes.
Scarce by nature: a band has as many taught edges as the curriculum states,
which is why ADR-0262 §5.1 makes authored volume the binding constraint on
what a band can demonstrate beyond non-commitment.
Polarity-independent on purpose (ADR-0264 R1). A negative row states its atom
just as directly as an affirmative one — it says "no" about it — so both are
scarce committed evidence and both are force-included in a capped sample. The
verdict they earn (``entailed`` vs ``refuted``) is the oracle's call, not this
function's; classifying here would duplicate the gold rule inside the corpus.
"""
return frozenset(
QueryAtom(domain, family, c.subject, c.connective, c.obj).key
for c in load_curriculum(domain).family(family)
)
def band_cases(domain: str, family: str, cap: int = CASES_PER_BAND) -> tuple[QueryAtom, ...]:
"""The committed cases for one band — distinct atoms, at most *cap* of them.
Whole space when it fits. Otherwise: every taught edge, then a fixed STRIDE
across the remaining atoms in sorted order.
The stride is deliberate rather than a prefix. A lexicographic prefix of 660
atoms out of 45,300 would cover about five subject terms of 151, measuring one
corner of the band and reporting it as the band; a stride spreads the sample
across every subject the curriculum teaches. It is not a random sample — there
is no RNG here — but it is an unbiased-by-construction one, and it cannot be
tuned per band, which is the property that matters for an evidence artifact.
Taught edges are force-included because they are the only committed-POSITIVE
evidence a band has and a sample that missed them would report a band's
reliability while exercising none of its content.
"""
atoms = routable_atoms(domain, family)
if len(atoms) <= cap:
return atoms
taught_keys = taught_atoms(domain, family)
taught = [a for a in atoms if a.key in taught_keys]
rest = [a for a in atoms if a.key not in taught_keys]
need = max(cap - len(taught), 0)
stride = max(len(rest) // need, 1) if need else 1
picked = rest[::stride][:need]
return tuple(sorted(taught + picked, key=lambda a: (a.subject, a.connective, a.obj)))
def practice_bands() -> tuple[tuple[str, str], ...]:
"""Every *(domain, family)* pair the ratified corpora actually populate.
A family with no chains is not a band with zero volume — it is not a band.
``decide_curriculum_question`` refuses it ``empty_curriculum`` before any
band key is assigned, so committing cases under one would invent a capability
axis the serving path never uses.
"""
return tuple(
(domain, family)
for domain in SERVED_DOMAINS
for family in FAMILIES
if load_curriculum(domain).family(family)
)
def all_gold_problems(cap: int = CASES_PER_BAND) -> tuple[Problem, ...]:
"""The full practice corpus, every band, in a deterministic order."""
problems: list[Problem] = []
for domain, family in practice_bands():
band = band_for(domain, family)
for atom in band_cases(domain, family, cap):
problems.append(
Problem(problem_id=atom.case_id, class_name=band, payload=atom)
)
return tuple(problems)
class DuplicateAtom(AssertionError):
"""A band committed the same query atom twice — the R9 invariant, violated."""
def assert_practice_atoms_distinct(cap: int = CASES_PER_BAND) -> None:
"""``committed == distinct``, enforced at the producer (ADR-0264 R9).
The Phase B audit MEASURES inflation; this refuses to emit it. Both matter:
a producer that cannot pad is better than one that is caught padding, and the
audit still guards the sealed artifact against a future producer change.
"""
seen: dict[str, set[tuple[str, str, str, str]]] = {}
for problem in all_gold_problems(cap):
atom: QueryAtom = problem.payload
keys = seen.setdefault(problem.class_name, set())
if atom.key in keys:
raise DuplicateAtom(
f"{problem.class_name}: query atom {atom.key} committed twice — "
"practice volume must be distinct evidence (ADR-0264 R9)"
)
keys.add(atom.key)
class CurriculumSolver:
"""The production decision path, under test (``chat/curriculum_surface.py``).
``committed`` is ``verdict != "declined"``. A typed refusal is an honest
non-commitment, excluded from reliability's denominator (ADR-0175 §4) and
counted as coverage — never conflated with a confabulation. That mapping is
the same one ``evals/curriculum_serve/runner.py`` applies when it separates a
``declined`` mismatch from a ``wrong``.
"""
domain_id = "curriculum_serve"
def attempt(self, problem: DomainProblem) -> BaseAttempt:
atom: QueryAtom = problem.payload
decision = decide_curriculum_question(atom.text)
committed = decision.verdict != "declined"
return BaseAttempt(
committed=committed,
answer=decision.verdict,
reason=decision.reason,
case_id=problem.problem_id,
)
class CurriculumOracleTether:
"""Tier-1 gold: the INDEPENDENT curriculum oracle (ADR-0199 L-2).
``evals/curriculum_serve/oracle.py`` shares no code with the serving path —
its own loader, ratification predicate, family table, agreement normalization
and verdict rule. Two independently written procedures agreeing on every atom
is real evidence the serving path reads the curriculum correctly; a tether
built from the compiler would only prove the compiler agrees with itself.
"""
domain_id = "curriculum_serve"
def is_correct(self, attempt: BaseAttempt, problem: DomainProblem) -> bool:
return attempt.answer == self.gold_answer(problem)
def gold_answer(self, problem: DomainProblem) -> str:
atom: QueryAtom = problem.payload
return oracle_answer(atom.domain, atom.subject, atom.connective, atom.obj).verdict
__all__ = [
"CASES_PER_BAND",
"CurriculumOracleTether",
"CurriculumSolver",
"DuplicateAtom",
"QueryAtom",
"all_gold_problems",
"assert_practice_atoms_distinct",
"band_cases",
"practice_bands",
"routable_atoms",
"taught_atoms",
]

187
evals/curriculum_serve/practice/runner.py Normal file
View file

@ -0,0 +1,187 @@
"""Sealed-practice runner for curriculum serving (ADR-0262/0264, Phase C).
Folds the ADR-0199 ``run_practice`` engine over the enumerated atom corpus
(``generator.py``) and reads each band's ``ClassTally`` through the reliability
gate. Two outputs, exactly mirroring ``evals/deduction_serve/practice/runner.py``:
- ``run()`` — the falsifiable report: which *(subject × relation family)* bands
earned SERVE, at what committed volume, and with what verdict MIX.
- ``seal_ledger()`` — writes the SHA-sealed artifact the serving reader
(``chat/curriculum_serve_license.py``) trusts. That module's docstring already
names this function as the only writer; this is the function it named.
**The artifact is NOT committed by this arc, and that is a finding, not an
omission.** Every band with a routable space ≥657 reaches θ_SERVE on the UNKNOWN
class alone: ``physics · causal`` commits 7 taught edges and 713 correct
non-commitments, and ``conservative_floor(660, 660) = 0.990046 ≥ 0.99``. So
sealing this ledger today would flip four bands from disclosed to authoritative on
a mix that is ~99% non-entailed — which ADR-0262 §5.1 explicitly rules
unacceptable, and which the still-open outcome-mix ruling (ADR-0264 R9's recorded
non-decision; ``docs/research/distinct-evidence-audit-2026-07-25.md``) exists to
settle. The plan of record's Phase-1 exit criterion asked for a ledger that is
"real (still-unearned)"; building the producer shows that state is unreachable.
Committing the artifact is therefore a ratification, and it is left to the
explicit ``core proposal-queue reseal`` verb a human runs (Phase D).
Determinism: the corpus is enumerated + sorted (no clock, no RNG) and
``run_practice`` is a pure fold, so the sealed ledger is byte-identical across
runs — safe to commit and SHA-verify on load, whenever it is committed.
"""
from __future__ import annotations
import argparse
from pathlib import Path
from typing import Any
from core.learning_arena.engine import run_practice
from core.ratified_ledger import seal_artifact, write_sealed_ledger
from core.reliability_gate import Action, Ceilings, ClassTally, license_for
from evals.curriculum_serve.practice.generator import (
CASES_PER_BAND,
CurriculumOracleTether,
CurriculumSolver,
all_gold_problems,
assert_practice_atoms_distinct,
)
#: The committed sealed ledger lives next to its serving READER (chat/), the
#: topology the estimation and deduction ledgers already use.
_SEALED_LEDGER_PATH = (
Path(__file__).resolve().parents[3] / "chat" / "data" / "curriculum_serve_ledger.json"
)
def build_ledger(cap: int = CASES_PER_BAND) -> dict[str, ClassTally]:
"""Run sealed practice over the enumerated corpus → per-band ledger."""
report = run_practice(
all_gold_problems(cap), CurriculumSolver(), CurriculumOracleTether()
)
return dict(report.ledger)
def build_mix(cap: int = CASES_PER_BAND) -> dict[str, dict[str, int]]:
"""Per-band GOLD verdict mix — the figure the reliability gate cannot see.
``ClassTally`` carries ``correct``/``wrong``/``refused`` and no verdict axis, so
a correct UNKNOWN is indistinguishable from a correct ENTAILED once tallied.
That is precisely why a band can clear θ_SERVE on non-commitments alone, and
why the mix has to be reported alongside the license rather than inferred from
it.
"""
tether = CurriculumOracleTether()
mix: dict[str, dict[str, int]] = {}
for problem in all_gold_problems(cap):
band = mix.setdefault(problem.class_name, {})
gold = tether.gold_answer(problem)
band[gold] = band.get(gold, 0) + 1
return mix
def run(ceilings: Ceilings | None = None, cap: int = CASES_PER_BAND) -> dict[str, Any]:
"""Build the ledger and report the SERVE verdict + mix per band."""
ceilings = ceilings if ceilings is not None else Ceilings.default()
ledger = build_ledger(cap)
mix = build_mix(cap)
classes: dict[str, Any] = {}
for band, tally in sorted(ledger.items()):
serve = license_for(tally, Action.SERVE, ceilings)
band_mix = mix.get(band, {})
entailed = band_mix.get("entailed", 0)
classes[band] = {
"correct": tally.correct,
"wrong": tally.wrong,
"refused": tally.refused,
"committed": tally.committed,
"reliability": tally.reliability,
"coverage": tally.coverage,
"serve_licensed": serve.licensed,
"serve_ratio": serve.ratio,
"gold_mix": dict(sorted(band_mix.items())),
"entailed_share": (
round(entailed / tally.committed, 6) if tally.committed else 0.0
),
}
return {
"lane": "curriculum-serve-practice",
"classes": classes,
"any_band_serve_licensed": any(c["serve_licensed"] for c in classes.values()),
"wrong_is_zero": all(c["wrong"] == 0 for c in classes.values()),
}
def build_sealed_artifact(cap: int = CASES_PER_BAND) -> dict[str, Any]:
"""The sealed-ledger dict (self-verifying ``content_sha256``)."""
return seal_artifact(
build_ledger(cap),
schema="curriculum_serve_ledger_v1",
note=(
"Sealed-practice committed ledger for curriculum-grounded serving "
"(ADR-0262, sized per ADR-0264 R9). Engine reads, never writes. "
"Ceilings stay at safe defaults (theta_SERVE=0.99). One committed case "
"per DISTINCT routable query atom, so committed == distinct evidence by "
"construction. NOTE: reliability here is dominated by correct "
"non-commitments; read `gold_mix` from the runner report before "
"treating a licensed band as demonstrated curriculum competence."
),
provenance="evals.curriculum_serve.practice.runner.seal_ledger",
)
def seal_ledger(
path: Path = _SEALED_LEDGER_PATH, cap: int = CASES_PER_BAND
) -> dict[str, Any]:
"""Regenerate + write the committed sealed ledger.
Verifies the R9 distinct-evidence invariant first: a producer that padded
could never seal, rather than sealing and being caught by the audit later.
"""
assert_practice_atoms_distinct(cap)
return write_sealed_ledger(path, build_sealed_artifact(cap))
def _full_sweep() -> dict[str, Any]:
"""Every routable atom in every band — the exhaustive agreement measurement.
Not the sealed corpus (that is capped at :data:`CASES_PER_BAND`); this is the
evidence that the cap is a sampling choice and not a hiding place. Runs the
whole ~70k-atom space, so it is a deliberate CLI invocation and not a test.
"""
return run(cap=10**9)
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--seal", action="store_true",
help="regenerate + write chat/data/curriculum_serve_ledger.json "
"(a RATIFICATION — see the module docstring before running it)",
)
parser.add_argument(
"--full", action="store_true",
help="report over EVERY routable atom instead of the capped corpus (slow)",
)
args = parser.parse_args(argv)
if args.seal:
artifact = seal_ledger()
print(f"sealed {len(artifact['classes'])} bands -> {_SEALED_LEDGER_PATH}")
return 0
report = _full_sweep() if args.full else run()
for band, c in report["classes"].items():
print(
f" {band:44s} committed={c['committed']:6d} wrong={c['wrong']} "
f"refused={c['refused']:4d} reliability={c['reliability']:.5f} "
f"SERVE={str(c['serve_licensed']):5s} entailed={c['gold_mix'].get('entailed', 0):3d} "
f"({c['entailed_share']:.4%})"
)
print(
f"any_band_serve_licensed={report['any_band_serve_licensed']} "
f"wrong_is_zero={report['wrong_is_zero']}"
)
return 0 if report["wrong_is_zero"] else 1
if __name__ == "__main__":
raise SystemExit(main())

65
teaching/curriculum_premises.py
View file

@ -73,6 +73,21 @@ CONNECTIVE_FAMILY: dict[str, str] = {
#: Relation families, in a stable order (the band-key axis — ADR-0262 §4).
FAMILIES: tuple[str, ...] = ("causal", "modal", "sequence", "contrast", "evidential")
#: The two row polarities (ADR-0264 R1). ``polarity`` is a ROW-LEVEL field, not
#: an ``intent`` value and not a new ``operator_family``: bands key on
#: *(domain, connective-derived family)*, so a ``modal_negative`` family would
#: create a fresh band at n=0 instead of adding refuted volume to the band it
#: belongs to. An ABSENT field reads as affirmative, which is what makes the
#: existing corpora valid unchanged.
AFFIRMATIVE = "affirmative"
NEGATIVE = "negative"
POLARITIES: tuple[str, str] = (AFFIRMATIVE, NEGATIVE)
#: ADR-0264 R2 — the sentential-negation prefix the argument reader already
#: parses (``generate/proof_chain/english.py``). Stated once here so the
#: compiled form cannot drift from the form the reader accepts.
NEGATION_PREFIX = "it is not the case that "
@dataclass(frozen=True, slots=True)
class CurriculumChain:
@ -83,13 +98,42 @@ class CurriculumChain:
connective: str
obj: str
family: str
#: ADR-0264 R1. Defaulted, so every existing construction site stays valid
#: and an un-migrated corpus row reads as the affirmative it always was.
polarity: str = AFFIRMATIVE
@property
def negated(self) -> bool:
return self.polarity == NEGATIVE
@property
def atom_sentence(self) -> str:
"""The affirmative core, independent of polarity — the ATOM this chain
is about.
ADR-0264 R3 in one property: a negative row must mint the *same*
propositional atom as its affirmative counterpart, or it cannot refute
anything. A negative row that reworded the relation would produce a
second, unrelated atom and the query would come back UNKNOWN instead of
REFUTED — a taught refutation silently failing to refute, which is the
worst available outcome here because nothing goes red.
"""
return f"{self.subject} {self.connective} {self.obj}"
@property
def sentence(self) -> str:
"""The English premise this chain compiles to. The connective is
already a third-person-singular verb form, so the sentence lands in
the verb-predicate grammar (ADR-0260) exactly as written."""
return f"{self.subject} {self.connective} {self.obj}"
"""The English premise this chain compiles to.
The connective is already a third-person-singular verb form, so the
affirmative sentence lands in the verb-predicate grammar (ADR-0260)
exactly as written. A negative row is the same sentence under the
sentential-negation prefix (ADR-0264 R2), which the verb band reads as
a negation OF THAT ATOM — so `(¬p, therefore p)` is a tautological
refutation and the query decides REFUTED.
"""
if self.negated:
return f"{NEGATION_PREFIX}{self.atom_sentence}"
return self.atom_sentence
@dataclass(frozen=True, slots=True)
@ -189,6 +233,14 @@ def load_curriculum(domain: str) -> Curriculum:
family = CONNECTIVE_FAMILY.get(connective)
if family is None:
continue
# ADR-0264 R1 — absent polarity IS affirmative. An UNRECOGNIZED
# polarity is dropped here rather than guessed: reading an
# unknown token as affirmative would turn a row someone wrote to
# refute into a row that asserts, which is the one direction of
# error this whole rule exists to prevent.
polarity = str(row.get("polarity") or AFFIRMATIVE).strip().lower()
if polarity not in POLARITIES:
continue
chains.append(
CurriculumChain(
chain_id=str(row.get("chain_id") or ""),
@ -196,6 +248,7 @@ def load_curriculum(domain: str) -> Curriculum:
connective=connective,
obj=str(row.get("object") or "").strip(),
family=family,
polarity=polarity,
)
)
return Curriculum(
@ -275,9 +328,13 @@ def resolve_pinned(curriculum: Curriculum, chain_ids: tuple[str, ...]) -> tuple[
__all__ = [
"AFFIRMATIVE",
"CONNECTIVE_FAMILY",
"FAMILIES",
"MAX_PREMISE_SENTENCES",
"NEGATION_PREFIX",
"NEGATIVE",
"POLARITIES",
"Curriculum",
"CurriculumChain",
"UnratifiedChain",

195
teaching/ledger_reseal.py Normal file
View file

@ -0,0 +1,195 @@
"""teaching/ledger_reseal.py — the ``ledger_reseal`` stage of the ceremony.
``RatificationReceipt.pending_stages`` has named ``ledger_reseal`` since the
ratification ceremony was written, and nothing performed it. Ratifying a chain
grows the corpus; it does not re-run practice, so the sealed ledger the serving
gate reads goes stale the moment curriculum changes. This module is that stage,
and it is deliberately **not** reachable from ``ratify_chain``.
Why it is a separate, human-run act
-----------------------------------
``AGENTS.md`` forbids ratification automation, and an auto-reseal on every
``ratify`` would churn the ledger's ``content_sha256`` once per row. But the
stronger reason is what Phase C measured: **resealing can grant licenses.**
Reliability is commitment precision and a correct UNKNOWN is a commitment, so
any band with ≥657 routable atoms clears θ_SERVE on non-commitments alone. A
plain "regenerate the artifact" verb would therefore flip four curriculum bands
from disclosed to authoritative as a side effect of housekeeping, on evidence
that is ~99% non-entailed — which ADR-0262 §5.1 rules unacceptable
(``docs/research/curriculum-practice-producer-2026-07-26.md`` §1).
So :func:`plan_reseal` computes the license DELTA before writing anything, and
the caller must opt in explicitly to any band that would become newly licensed.
The default is refusal. Regenerating an artifact is housekeeping; changing what
the engine will assert authoritatively is a ratification, and the two must not
share a keystroke.
Scope: this module writes a ledger and nothing else. It has no opinion on
whether curriculum content is true, cannot ratify a chain, and cannot flip a
serving flag.
"""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable
from core.ratified_ledger import (
CAPABILITY_LEDGERS,
load_capability_ledger,
write_sealed_ledger,
)
from core.reliability_gate import Action, Ceilings, ClassTally, license_for
class ReadyToReseal(ValueError):
"""A reseal was requested that cannot proceed as asked."""
@dataclass(frozen=True, slots=True)
class ResealPlan:
"""What a reseal would do, computed before it does it."""
capability: str
path: Path
#: Bands licensed by the ledger currently on disk (empty when absent).
licensed_before: tuple[str, ...]
#: Bands the freshly-built artifact would license.
licensed_after: tuple[str, ...]
#: Per-band committed volume in the new artifact.
committed: dict[str, int]
#: Per-band gold verdict mix — the axis ``ClassTally`` cannot carry.
mix: dict[str, dict[str, int]]
artifact: dict[str, Any]
@property
def newly_licensed(self) -> tuple[str, ...]:
"""Bands this reseal would grant SERVE that do not hold it today.
The figure that decides whether the operation is housekeeping or a
ratification.
"""
before = set(self.licensed_before)
return tuple(b for b in self.licensed_after if b not in before)
@property
def revoked(self) -> tuple[str, ...]:
"""Bands that would LOSE a license — always allowed, never gated.
Losing a license is the gate becoming more conservative, which needs no
authorization. Only granting one does.
"""
after = set(self.licensed_after)
return tuple(b for b in self.licensed_before if b not in after)
@property
def is_housekeeping(self) -> bool:
return not self.newly_licensed
#: capability -> (build artifact, build ledger, build mix).
#:
#: ``deduction_serve`` is deliberately ABSENT. Its ledger is SHA-sealed,
#: ratified, and gating a live flag, and 21 of its 25 bands do not clear
#: θ_SERVE on distinct evidence (``tests/test_volume_honesty.py``). Re-sealing it
#: is Shay's ratification, not a CLI verb's, and a reseal that silently
#: regenerated it would erase the exposure this arc measured. ``estimation``
#: ships sealed with its gate and has no reason to move.
_RESEALABLE: dict[str, tuple[Callable[[], dict[str, Any]], Callable[[], dict[str, ClassTally]], Callable[[], dict[str, dict[str, int]]]]] = {}
def _register_curriculum() -> None:
from evals.curriculum_serve.practice.runner import (
build_ledger,
build_mix,
build_sealed_artifact,
)
_RESEALABLE["curriculum_serve"] = (build_sealed_artifact, build_ledger, build_mix)
def resealable_capabilities() -> tuple[str, ...]:
"""Capabilities this stage will rebuild, in a stable order."""
if not _RESEALABLE:
_register_curriculum()
return tuple(sorted(_RESEALABLE))
def _licensed_bands(ledger: dict[str, ClassTally], ceilings: Ceilings) -> tuple[str, ...]:
return tuple(
sorted(
band
for band, tally in ledger.items()
if license_for(tally, Action.SERVE, ceilings).licensed
)
)
def plan_reseal(capability: str, *, ceilings: Ceilings | None = None) -> ResealPlan:
"""Compute the reseal and its license delta — writing nothing.
The read of the CURRENT ledger goes through :func:`load_capability_ledger`,
the same production entry point the serving adapters use, so two properties
come for free. A hand-edited ledger refuses here rather than being silently
overwritten (losing the evidence that it was tampered with), and this stage
inherits the capability's DECLARED absence policy instead of choosing its own
— bridge rule 5: ``missing_ok`` is a fact about the capability, not about the
call, and no production path may pass it.
"""
if capability not in resealable_capabilities():
raise ReadyToReseal(
f"{capability!r} is not resealable — known: {list(resealable_capabilities())}. "
"Ledgers absent from that list are sealed by deliberate ratification only."
)
spec = CAPABILITY_LEDGERS[capability]
ceilings = ceilings if ceilings is not None else Ceilings.default()
build_artifact, build_ledger, build_mix = _RESEALABLE[capability]
current = load_capability_ledger(capability)
fresh_ledger = build_ledger()
artifact = build_artifact()
return ResealPlan(
capability=capability,
path=spec.path,
licensed_before=_licensed_bands(current, ceilings),
licensed_after=_licensed_bands(fresh_ledger, ceilings),
committed={band: t.committed for band, t in sorted(fresh_ledger.items())},
mix=build_mix(),
artifact=artifact,
)
def apply_reseal(plan: ResealPlan, *, allow_new_licenses: bool = False) -> Path:
"""Write the planned artifact, refusing to grant a license by accident.
``allow_new_licenses`` is the ratification. Without it, a plan that would
newly license any band refuses and names them — the operator then either
authorizes the grant or fixes the producer, but never discovers afterwards
that a serving surface became authoritative.
"""
if plan.newly_licensed and not allow_new_licenses:
detail = ", ".join(
f"{band} (committed={plan.committed.get(band, 0)}, "
f"entailed={plan.mix.get(band, {}).get('entailed', 0)})"
for band in plan.newly_licensed
)
raise ReadyToReseal(
f"{plan.capability}: this reseal would newly license {len(plan.newly_licensed)} "
f"band(s) — {detail}. That is a ratification, not a reseal: those bands "
"would begin serving authoritatively. Re-run with --allow-new-licenses "
"only if the grant is intended, and read the `entailed` counts first — a "
"band can clear theta_SERVE on correct NON-COMMITMENTS alone (ADR-0262 §5.1)."
)
write_sealed_ledger(plan.path, plan.artifact)
return plan.path
__all__ = [
"ReadyToReseal",
"ResealPlan",
"apply_reseal",
"plan_reseal",
"resealable_capabilities",
]

75
teaching/ratification.py
View file

@ -57,7 +57,13 @@ from core.capability.domains import (
DOMAIN_CORPORA,
DOMAIN_PACKS,
)
from teaching.curriculum_premises import CONNECTIVE_FAMILY, load_curriculum
from teaching.curriculum_premises import (
AFFIRMATIVE,
CONNECTIVE_FAMILY,
NEGATIVE,
POLARITIES,
load_curriculum,
)
_REPO_ROOT = Path(__file__).resolve().parents[1]
@ -74,6 +80,10 @@ _ROW_FIELDS: tuple[str, ...] = (
"intent",
"connective",
"object",
# ADR-0264 R1 — row-level polarity, emitted only for NEGATIVE rows (see
# ``ChainRecord.as_row``). An affirmative row omits it, so every committed
# corpus stays byte-identical and no existing lane hash moves.
"polarity",
"subject_pack_id",
"object_pack_id",
"review_status",
@ -100,9 +110,34 @@ class ChainRecord:
object_pack_id: str
review_status: str
provenance: str
#: ADR-0264 R1. Defaulted so existing construction sites are unchanged.
polarity: str = AFFIRMATIVE
def as_row(self) -> dict[str, Any]:
return {name: getattr(self, name) for name in _ROW_FIELDS}
"""The committed row.
``polarity`` is OMITTED when affirmative. An absent field already reads
as affirmative (ADR-0264 R1), so emitting it would add a redundant key
to every future row while the committed corpora lack it — two spellings
of the same fact, and a diff that looks like a schema change on rows
whose meaning did not move.
"""
row = {
name: getattr(self, name)
for name in _ROW_FIELDS
if name != "polarity"
}
if self.polarity != AFFIRMATIVE:
# Re-insert in _ROW_FIELDS order — chain corpora are reviewed as
# diffs, so key order is part of the artifact.
ordered = {}
for name in _ROW_FIELDS:
if name == "polarity":
ordered["polarity"] = self.polarity
else:
ordered[name] = row[name]
return ordered
return row
def as_jsonl_line(self) -> str:
"""Byte-compatible with the committed corpora: compact separators,
@ -221,6 +256,7 @@ def build_chain_record(
reviewer: str,
rationale: str,
intent: str = "cause",
polarity: str = AFFIRMATIVE,
chain_id: str | None = None,
) -> ChainRecord:
"""Construct the record a reviewed decision implies.
@ -236,11 +272,20 @@ def build_chain_record(
if not rationale:
raise RatificationError("ratification requires a stated rationale")
polarity = polarity.strip().lower()
if polarity not in POLARITIES:
raise RatificationError(
f"polarity {polarity!r} is not one of {list(POLARITIES)} — the "
"curriculum loader would drop this row (ADR-0264 R1)"
)
family = CONNECTIVE_FAMILY.get(connective.strip())
if family is None:
raise RatificationError(
f"connective {connective!r} is outside CONNECTIVE_FAMILY "
f"{sorted(CONNECTIVE_FAMILY)} — the loader would drop this row"
f"{sorted(CONNECTIVE_FAMILY)} — the loader would drop this row. "
"A NEGATIVE row must reuse the affirmative connective (ADR-0264 R3), "
"never a negated paraphrase of it."
)
packs = DOMAIN_PACKS.get(domain, ())
@ -273,6 +318,7 @@ def build_chain_record(
object_pack_id=object_pack,
review_status="reviewed",
provenance=f"ratification-ceremony:{reviewer}:{rationale}",
polarity=polarity,
)
@ -298,6 +344,10 @@ def validate_admissible(record: ChainRecord) -> None:
raise RatificationError(
f"object {record.object!r} absent from pack {record.object_pack_id}"
)
if record.polarity not in POLARITIES:
raise RatificationError(
f"polarity {record.polarity!r} has no meaning (ADR-0264 R1)"
)
for row in existing_rows(record.domain):
if str(row.get("chain_id")) == record.chain_id:
raise RatificationError(f"chain_id {record.chain_id} already committed")
@ -306,6 +356,25 @@ def validate_admissible(record: ChainRecord) -> None:
and str(row.get("connective")) == record.connective
and str(row.get("object")) == record.object
):
# ADR-0264 R4 — a corpus stating both polarities of ONE atom is
# contradictory, and the contradiction is rejected HERE rather than
# discovered at serve time. `evaluate_entailment_with_trace` would
# meet `p ∧ ¬p` and refuse INCONSISTENT_PREMISES, which is honest
# but arrives as a band that has stopped answering anything in that
# family — the curriculum silently going dark instead of the
# authoring mistake being named.
existing_polarity = (
str(row.get("polarity") or AFFIRMATIVE).strip().lower()
)
if existing_polarity != record.polarity:
raise RatificationError(
f"contradiction: {row.get('chain_id')} already teaches "
f"{existing_polarity} '{record.subject} {record.connective} "
f"{record.object}'; this row would teach {record.polarity}. "
"One atom cannot hold both polarities (ADR-0264 R4) — the "
"compiled premises would be inconsistent and the band would "
"refuse every question in the family."
)
raise RatificationError(
f"duplicate edge: {row.get('chain_id')} already teaches "
f"{record.subject} {record.connective} {record.object}"

427
tests/test_curriculum_polarity.py Normal file
View file

@ -0,0 +1,427 @@
"""ADR-0264 R1-R4 + R8 — explicitly-taught curriculum negatives.
Before this, `polarity` was read by NOTHING. `CurriculumChain.sentence` was
unconditionally affirmative, so a row authored to REFUTE an atom compiled to a
premise asserting it, and the question came back `entailed`. The independent
oracle ignored the field too — so **gold agreed and `wrong=0` stayed green.**
That is the shape of defect no gate in this repository can catch, which is why
the epistemology was settled in an ADR before any code moved.
The five rules, and what each one is defending against:
- **R1** polarity is a ROW-LEVEL field; absent ⇒ affirmative. Not an `intent`
value, not a new `operator_family`: bands key on the connective-derived family,
so `modal_negative` would open a fresh band at n=0 instead of adding refuted
volume to the band it belongs to.
- **R2** a negative row compiles under the sentential-negation prefix the argument
reader already parses.
- **R3** a negative row MUST reuse the affirmative connective, so it mints the
SAME propositional atom. A negated paraphrase mints a different atom and the
query returns UNKNOWN — a taught refutation that silently fails to refute.
- **R4** one atom cannot hold both polarities; the contradiction is rejected at
ratification, not discovered at serve time as a band that stopped answering.
- **R8** the oracle learns polarity INDEPENDENTLY (its own constants, its own
reader) and excludes negatives from the reachability adjacency.
The end-to-end tests write a temporary row into the committed corpus and restore
it, asserting **byte-identity** on the way out. No negative row is committed:
authoring curriculum is Phase F and Shay's.
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Iterator
import pytest
from chat.curriculum_surface import decide_curriculum_question
from evals.curriculum_serve.oracle import oracle_answer, taught_edges
from generate.proof_chain.entail import Entailment, evaluate_entailment_with_trace
from generate.proof_chain.verb import VerbArgument, read_verb_argument
from teaching.curriculum_premises import (
AFFIRMATIVE,
NEGATION_PREFIX,
NEGATIVE,
POLARITIES,
CurriculumChain,
compile_premises,
load_curriculum,
)
from teaching.ratification import (
ChainRecord,
RatificationError,
build_chain_record,
validate_admissible,
)
REPO_ROOT = Path(__file__).resolve().parents[1]
_PHYSICS_CORPUS = REPO_ROOT / "teaching" / "domain_chains" / "physics_chains_v1.jsonl"
#: An atom physics does NOT teach, in taught vocabulary, so it routes and is
#: UNKNOWN today — the clean slot to plant a negative in.
_UNTAUGHT_ATOM = ("mass", "causes", "charge")
def _chain(polarity: str = AFFIRMATIVE) -> CurriculumChain:
return CurriculumChain(
chain_id="x-causal-001",
subject="force",
connective="causes",
obj="acceleration",
family="causal",
polarity=polarity,
)
# ---------------------------------------------------------------------------
# R1 — the field, and its default.
# ---------------------------------------------------------------------------
def test_r1_polarity_defaults_to_affirmative() -> None:
"""An absent field must read as affirmative, or every committed row changes
meaning the moment the field exists."""
bare = CurriculumChain("i", "force", "causes", "acceleration", "causal")
assert bare.polarity == AFFIRMATIVE
assert bare.negated is False
assert bare.sentence == "force causes acceleration"
def test_r1_committed_corpora_carry_no_polarity_field_and_still_load() -> None:
"""The migration is a no-op on disk: nothing was rewritten."""
rows = [
json.loads(line)
for line in _PHYSICS_CORPUS.read_text(encoding="utf-8").splitlines()
if line.strip()
]
assert rows
assert not any("polarity" in row for row in rows), (
"no committed row should carry polarity yet — Phase F authors those"
)
for chain in load_curriculum("physics").chains:
assert chain.polarity == AFFIRMATIVE
def test_r1_unrecognized_polarity_is_dropped_not_guessed() -> None:
"""Reading an unknown token as affirmative would turn a row someone wrote to
refute into a row that asserts — the one direction of error that matters."""
assert set(POLARITIES) == {AFFIRMATIVE, NEGATIVE}
with pytest.raises(RatificationError, match="polarity"):
build_chain_record(
domain="physics", subject="mass", connective="causes", obj="charge",
reviewer="t", rationale="t", polarity="maybe",
)
def test_r1_polarity_is_not_a_new_operator_family() -> None:
"""Bands key on the connective-derived family; a negative row keeps its band."""
from chat.curriculum_surface import band_for
neg = _chain(NEGATIVE)
assert neg.family == "causal"
assert band_for("physics", neg.family) == "curriculum_physics_causal"
# ---------------------------------------------------------------------------
# R2 — the compiled form, and that the reader accepts it.
# ---------------------------------------------------------------------------
def test_r2_negative_row_compiles_under_the_negation_prefix() -> None:
neg = _chain(NEGATIVE)
assert neg.sentence == f"{NEGATION_PREFIX}force causes acceleration"
assert neg.negated is True
def test_r2_the_reader_actually_parses_that_form() -> None:
"""The prefix is not decorative — the verb band must read it as a negation
of the atom, or R2 compiles premises nothing can decide."""
argument = f"{_chain(NEGATIVE).sentence}. Therefore force causes acceleration."
arg = read_verb_argument(argument)
assert isinstance(arg, VerbArgument), f"reader refused: {arg}"
trace = evaluate_entailment_with_trace(arg.premise_formulas, arg.query_formula)
assert trace.outcome is Entailment.REFUTED
assert trace.reason == "tautological_refutation"
def test_r2_affirmative_control_still_entails() -> None:
argument = f"{_chain().sentence}. Therefore force causes acceleration."
arg = read_verb_argument(argument)
assert isinstance(arg, VerbArgument)
trace = evaluate_entailment_with_trace(arg.premise_formulas, arg.query_formula)
assert trace.outcome is Entailment.ENTAILED
# ---------------------------------------------------------------------------
# R3 — the same atom. This is the silent-failure rule.
# ---------------------------------------------------------------------------
def test_r3_negative_reuses_the_affirmative_atom() -> None:
"""`atom_sentence` is polarity-independent — that IS R3."""
assert _chain().atom_sentence == _chain(NEGATIVE).atom_sentence
assert _chain(NEGATIVE).sentence.endswith(_chain(NEGATIVE).atom_sentence)
def test_r3_a_different_connective_would_silently_fail_to_refute() -> None:
"""Why R3 is a rule and not a style note.
A negative row spelled with a DIFFERENT relation word mints a different
propositional atom, so it cannot contradict the query. The result is UNKNOWN
— the curriculum was taught a refutation and answers "I don't know". Nothing
goes red. This is the mechanism, demonstrated.
"""
same_atom = f"{NEGATION_PREFIX}force causes acceleration. Therefore force causes acceleration."
other_atom = f"{NEGATION_PREFIX}force enables acceleration. Therefore force causes acceleration."
a = read_verb_argument(same_atom)
b = read_verb_argument(other_atom)
assert isinstance(a, VerbArgument) and isinstance(b, VerbArgument)
assert evaluate_entailment_with_trace(a.premise_formulas, a.query_formula).outcome is Entailment.REFUTED
assert evaluate_entailment_with_trace(b.premise_formulas, b.query_formula).outcome is Entailment.UNKNOWN, (
"a negated paraphrase must be shown NOT to refute — that is the hazard R3 forbids"
)
def test_r3_ratification_refuses_a_connective_outside_the_closed_set() -> None:
with pytest.raises(RatificationError, match="ADR-0264 R3"):
build_chain_record(
domain="physics", subject="mass", connective="does not cause",
obj="charge", reviewer="t", rationale="t", polarity=NEGATIVE,
)
# ---------------------------------------------------------------------------
# R4 — contradiction rejected at ratification.
# ---------------------------------------------------------------------------
def _record(polarity: str, subject: str = "force", obj: str = "acceleration") -> ChainRecord:
return ChainRecord(
chain_id="physics-causal-900", domain="physics", operator_family="causal",
subject=subject, intent="cause", connective="causes", object=obj,
subject_pack_id="en_physics_v1", object_pack_id="en_physics_v1",
review_status="reviewed", provenance="test", polarity=polarity,
)
def test_r4_negating_an_already_taught_atom_is_rejected() -> None:
"""physics-causal-001 teaches `force causes acceleration` affirmatively."""
with pytest.raises(RatificationError, match="contradiction"):
validate_admissible(_record(NEGATIVE))
def test_r4_the_refusal_names_both_polarities_and_the_consequence() -> None:
with pytest.raises(RatificationError) as exc:
validate_admissible(_record(NEGATIVE))
message = str(exc.value)
assert AFFIRMATIVE in message and NEGATIVE in message
assert "ADR-0264 R4" in message
assert "inconsistent" in message
def test_r4_same_polarity_duplicate_is_still_a_duplicate_not_a_contradiction() -> None:
"""The two failures are different and must stay distinguishable."""
with pytest.raises(RatificationError, match="duplicate edge"):
validate_admissible(_record(AFFIRMATIVE))
def test_r4_an_untaught_atom_admits_either_polarity() -> None:
"""R4 forbids contradiction, not negatives."""
subject, _connective, obj = _UNTAUGHT_ATOM
validate_admissible(_record(NEGATIVE, subject=subject, obj=obj))
validate_admissible(_record(AFFIRMATIVE, subject=subject, obj=obj))
# ---------------------------------------------------------------------------
# R8 — the oracle, independently.
# ---------------------------------------------------------------------------
def test_r8_oracle_does_not_import_the_compiler_s_polarity_vocabulary() -> None:
"""Code-level independence is the whole evidentiary value.
If the oracle imported the compiler's polarity constants or helper, agreement
would only prove the compiler agrees with itself and a polarity bug would be
invisible on both sides at once.
"""
import inspect
import evals.curriculum_serve.oracle as oracle
source = inspect.getsource(oracle)
assert "from teaching.curriculum_premises import" not in source
assert "teaching.curriculum_premises" not in source
# It has its own constants.
assert oracle._AFFIRMATIVE == "affirmative"
assert oracle._NEGATIVE == "negative"
def test_r8_taught_edges_exposes_polarity() -> None:
edges = taught_edges("physics")
assert edges
for _s, _c, _o, _id, polarity in edges:
assert polarity == AFFIRMATIVE
# ---------------------------------------------------------------------------
# End to end, against a temporary corpus row. Restored byte-identically.
# ---------------------------------------------------------------------------
def _plant(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, *, polarity: str, chain_id: str
) -> tuple[str, str, str]:
"""Repoint BOTH loaders at a temp corpus carrying one extra row.
The committed corpus is never written. An earlier version of this fixture
appended to `teaching/domain_chains/physics_chains_v1.jsonl` and restored it;
that is safe under `smoke`/`deductive` (serial) but NOT under `full`, which
injects `-n auto` — a parallel worker reading the corpus mid-mutation would
see a row that is not committed, and the failure would look like a corpus
problem rather than a test-isolation one.
Each loader is still redirected SEPARATELY, at its own module constant, and
each still parses with its own code. Only the corpus LOCATION is shared; the
ADR-0264 R8 independence being tested is untouched.
"""
import evals.curriculum_serve.oracle as oracle
import teaching.curriculum_premises as premises
subject, connective, obj = _UNTAUGHT_ATOM
row = {
"chain_id": chain_id,
"domain": "physics",
"operator_family": "causal",
"subject": subject,
"intent": "cause",
"connective": connective,
"object": obj,
"subject_pack_id": "en_physics_v1",
"object_pack_id": "en_physics_v1",
"review_status": "reviewed",
"provenance": "test:polarity",
}
if polarity != AFFIRMATIVE:
row["polarity"] = polarity
chain_dir = tmp_path / "teaching" / "domain_chains"
chain_dir.mkdir(parents=True)
for src in (REPO_ROOT / "teaching" / "domain_chains").glob("*.jsonl"):
text = src.read_text(encoding="utf-8")
if not text.endswith("\n"):
text += "\n"
if src.name == _PHYSICS_CORPUS.name:
text += json.dumps(row, separators=(",", ":")) + "\n"
(chain_dir / src.name).write_text(text, encoding="utf-8")
# The compiler resolves corpora AND packs from its repo root, so the temp
# tree needs the real packs visible under the same name.
(tmp_path / "packs").symlink_to(REPO_ROOT / "packs")
monkeypatch.setattr(premises, "_REPO_ROOT", tmp_path)
monkeypatch.setattr(oracle, "_CHAIN_DIR", chain_dir)
premises.load_curriculum.cache_clear()
premises._pack_lemmas.cache_clear()
monkeypatch.setattr(
premises.load_curriculum, "cache_clear", premises.load_curriculum.cache_clear
)
return subject, connective, obj
@pytest.fixture
def negative_row(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> Iterator[tuple[str, str, str]]:
"""A TAUGHT NEGATIVE for an atom physics does not otherwise teach."""
atom = _plant(monkeypatch, tmp_path, polarity=NEGATIVE, chain_id="physics-causal-999")
yield atom
load_curriculum.cache_clear()
def test_e2e_a_taught_negative_serves_refuted(negative_row: tuple[str, str, str]) -> None:
"""The whole point: `polarity` now changes the answer.
Before this unit the same row served a confident `entailed` — a taught
refutation answering "yes".
"""
subject, connective, obj = negative_row
decision = decide_curriculum_question(f"Does {subject} {connective} {obj}?")
assert decision.verdict == "refuted", decision
assert decision.band == "curriculum_physics_causal"
def test_e2e_the_independent_oracle_agrees(negative_row: tuple[str, str, str]) -> None:
"""Two independently-written procedures must reach the same verdict, or the
lane's `wrong=0` is measuring agreement with itself."""
subject, connective, obj = negative_row
gold = oracle_answer("physics", subject, connective, obj)
served = decide_curriculum_question(f"Does {subject} {connective} {obj}?")
assert gold.verdict == "refuted"
assert gold.depth == 1, "a taught negative is a depth-1 statement, not a composition"
assert served.verdict == gold.verdict
def test_e2e_the_negative_premise_is_actually_compiled(
negative_row: tuple[str, str, str],
) -> None:
subject, connective, obj = negative_row
curriculum = load_curriculum("physics")
premises, chain_ids = compile_premises(
curriculum, "causal", query=(subject, connective, obj)
)
assert "physics-causal-999" in chain_ids
assert any(p.startswith(NEGATION_PREFIX) for p in premises), premises
def test_e2e_the_atom_was_unknown_before_the_negative_row() -> None:
"""Without the fixture the same question is UNKNOWN — so the test above is
measuring the row, not a pre-existing verdict."""
subject, connective, obj = _UNTAUGHT_ATOM
load_curriculum.cache_clear()
decision = decide_curriculum_question(f"Does {subject} {connective} {obj}?")
assert decision.verdict == "unknown", decision
assert oracle_answer("physics", subject, connective, obj).verdict == "unknown"
def test_e2e_negatives_are_excluded_from_reachability(
negative_row: tuple[str, str, str],
) -> None:
"""R8 — a denial supplies no step, demonstrated on a path it would complete.
The fixture plants `mass does not cause charge`, and physics already teaches
`charge causes field` (physics-causal-004). So if negatives entered the
adjacency, `(mass, causes, field)` would report a 2-hop path built out of a
DENIAL. `mass` has no other outgoing causal edge, so depth must stay 0.
Constructed this way on purpose: the assertion distinguishes the correct
behaviour from the specific wrong one, rather than merely observing UNKNOWN
(which both would produce).
"""
subject, _connective, obj = negative_row
assert (obj, "causes", "field") in {
(c.subject, c.connective, c.obj) for c in load_curriculum("physics").family("causal")
}, "fixture premise moved: physics must still teach `charge causes field`"
verdict = oracle_answer("physics", subject, "causes", "field")
assert verdict.verdict == "unknown"
assert verdict.depth == 0, (
f"reachability composed a {verdict.depth}-hop path through a denial "
f"({subject} -/-> {obj} -> field) — ADR-0264 R8 excludes negatives from "
"the adjacency precisely so this cannot happen"
)
def test_e2e_an_affirmative_row_in_the_same_slot_WOULD_create_that_path(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
"""The control for the test above: the path is real when the row is positive.
Without this, `depth == 0` could just mean the oracle never finds 2-hop paths
at all, and the R8 assertion would be vacuous. Same slot, same terms, only the
polarity differs — so the two tests isolate polarity and nothing else.
"""
subject, _connective, _obj = _plant(
monkeypatch, tmp_path, polarity=AFFIRMATIVE, chain_id="physics-causal-998"
)
verdict = oracle_answer("physics", subject, "causes", "field")
assert verdict.verdict == "unknown", "a 2-hop path must not be ENTAILED"
assert verdict.depth == 2, (
f"expected the affirmative row to open a 2-hop path, got {verdict}"
)
load_curriculum.cache_clear()

361
tests/test_curriculum_practice.py Normal file
View file

@ -0,0 +1,361 @@
"""Phase C — the curriculum practice producer (ADR-0262 solver, ADR-0264 R9 sizing).
Four things are pinned here, in decreasing order of how quietly they would fail:
1. **The ledger is not committed, and a band would earn SERVE if it were.** The
producer works; that is exactly why the artifact is a ratification. Pinned so
the reasoning cannot be lost and re-derived wrongly later.
2. **`then` / `therefore` collide with the argument reader's grammar.** Two taught
`philosophy_theology` lemmas are the reader's own control words, so 164 routable
atoms refuse `compiled_premises_unreadable`. Coverage misses, never wrongs — but
in the band Phase F retargets to.
3. **`committed == distinct` by construction**, not by discipline.
4. **wrong=0** over the capped corpus, per band.
"""
from __future__ import annotations
import json
from pathlib import Path
import pytest
from chat.curriculum_surface import band_for, decide_curriculum_question
from core.ratified_ledger import CAPABILITY_LEDGERS, load_sealed_ledger
from core.reliability_gate import Action, Ceilings, license_for
from core.reliability_gate.evidence import audit_bands
from evals.curriculum_serve.practice.generator import (
CASES_PER_BAND,
DuplicateAtom,
QueryAtom,
all_gold_problems,
assert_practice_atoms_distinct,
band_cases,
practice_bands,
routable_atoms,
taught_atoms,
)
from evals.curriculum_serve.practice.runner import (
build_ledger,
build_mix,
build_sealed_artifact,
run,
seal_ledger,
)
REPO_ROOT = Path(__file__).resolve().parents[1]
THETA_SERVE = 0.99
#: The 11 bands the ratified corpora populate, and each one's FULL routable atom
#: space. Pinned in both directions: a rise means new curriculum (welcome, update
#: deliberately), a fall means a band lost reachable questions.
#:
#: These supersede ADR-0264 §4.2's table, which sized bands from per-TERM
#: exclusivity. That bound is strictly tighter than the router's per-PAIR
#: predicate, so it under-counted — `systems_software_causal` reads as 630 there
#: and is really 720, which flips it from "cannot reach 657" to "can".
BAND_ATOM_SPACE: dict[str, int] = {
"curriculum_mathematics_logic_contrast": 240,
"curriculum_mathematics_logic_evidential": 240,
"curriculum_mathematics_logic_modal": 480,
"curriculum_mathematics_logic_sequence": 240,
"curriculum_philosophy_theology_contrast": 22650,
"curriculum_philosophy_theology_modal": 45300,
"curriculum_physics_causal": 720,
"curriculum_physics_modal": 480,
"curriculum_systems_software_causal": 720,
"curriculum_systems_software_modal": 480,
"curriculum_systems_software_sequence": 240,
}
#: Bands whose space reaches the 657 a perfect record needs. Derived below, never
#: restated: these are the four that WOULD earn SERVE the moment a ledger is sealed.
_CAN_REACH_FLOOR = {
band for band, space in BAND_ATOM_SPACE.items() if space >= 657
}
#: The reader-grammar collision, exact. Both lemmas are taught by
#: `philosophy_theology`'s packs AND are control words of the argument reader:
#: `therefore` is literally the conclusion marker `". Therefore <x>."`, and `then`
#: is the conditional consequent marker. An atom using either as a term compiles an
#: argument the reader cannot parse back.
RESERVED_WORD_LEMMAS = ("then", "therefore")
@pytest.fixture(scope="module")
def report() -> dict:
return run()
@pytest.fixture(scope="module")
def ledger() -> dict:
return build_ledger()
@pytest.fixture(scope="module")
def mix() -> dict:
return build_mix()
@pytest.fixture(scope="module")
def problems() -> tuple:
return all_gold_problems()
# ---------------------------------------------------------------------------
# 1. The artifact is uncommitted, and that is a decision with a reason.
# ---------------------------------------------------------------------------
def test_curriculum_ledger_is_not_committed() -> None:
"""Phase C builds the writer; committing the artifact is a ratification.
The plan of record's exit criterion asked for a ledger that is "real
(still-unearned)". That state is unreachable: every band with >=657 routable
atoms clears theta_SERVE on correct NON-COMMITMENTS alone. So a committed
ledger is necessarily an earning one, and this asserts nobody committed it as
housekeeping.
"""
path = CAPABILITY_LEDGERS["curriculum_serve"].path
assert not path.exists(), (
f"{path.name} exists. If that was deliberate ratification, update this "
"test and ADR-0262 §5 together — a curriculum ledger licenses bands whose "
"committed evidence is ~99% non-entailed."
)
assert CAPABILITY_LEDGERS["curriculum_serve"].missing_ok, (
"an absent curriculum ledger must stay legitimately absent (serves disclosed)"
)
def test_sealing_would_license_exactly_the_bands_that_reach_the_floor(
ledger: dict,
) -> None:
"""The consequence of sealing, measured rather than asserted in prose.
This is the finding Phase C exists to produce: the license is reachable today
and it is reachable on the UNKNOWN class. If this ever reports a different set,
the corpus moved and the ADR-0262 §5 discussion needs revisiting.
"""
ceilings = Ceilings.default()
licensed = {
band
for band, tally in ledger.items()
if license_for(tally, Action.SERVE, ceilings).licensed
}
assert licensed == _CAN_REACH_FLOOR
assert len(licensed) == 4
def test_a_licensed_band_is_licensed_on_non_commitments(report: dict) -> None:
"""The mix behind the license — the number `ClassTally` structurally cannot see.
`ClassTally` carries correct/wrong/refused and no verdict axis, so a correct
UNKNOWN counts exactly like a correct ENTAILED. Every band that would earn
SERVE does so with an entailed share under 5%, which is what ADR-0262 §5.1
rules unacceptable and what the open outcome-mix ruling has to settle.
"""
for band in sorted(_CAN_REACH_FLOOR):
entry = report["classes"][band]
assert entry["serve_licensed"] is True
assert entry["gold_mix"].get("entailed", 0) <= 9
assert entry["entailed_share"] < 0.05, (
f"{band}: entailed share {entry['entailed_share']:.4%} — if a band ever "
"earns SERVE on a genuine mix, that is the ADR-0262 §5.1 bar being met "
"and this pin should be revisited, not relaxed"
)
def test_seal_ledger_writes_a_verifying_artifact(tmp_path: Path) -> None:
"""The writer `chat/curriculum_serve_license.py` names must actually work.
Written to a tmp path, never to `chat/data/` — the point of the test is that
the mechanism is real, not that the repository gains a ledger.
"""
path = tmp_path / "curriculum_serve_ledger.json"
artifact = seal_ledger(path)
assert path.exists()
assert artifact["provenance"] == "evals.curriculum_serve.practice.runner.seal_ledger"
assert artifact["schema"] == "curriculum_serve_ledger_v1"
# Round-trips through the same verifier the serving reader uses.
tallies = load_sealed_ledger(path)
assert set(tallies) == set(BAND_ATOM_SPACE)
assert all(t.wrong == 0 for t in tallies.values())
def test_sealed_artifact_is_byte_identical_across_runs() -> None:
"""Determinism (ADR-0199 L-4): no clock, no RNG, so re-sealing must not move it.
Two independent computations are the minimum that proves this and the maximum
worth paying for — each is a full fold over the corpus.
"""
first = build_sealed_artifact()
second = build_sealed_artifact()
assert json.dumps(first, indent=2, sort_keys=True) == json.dumps(
second, indent=2, sort_keys=True
)
assert first["content_sha256"] == second["content_sha256"]
# ---------------------------------------------------------------------------
# 2. The reader-grammar collision — in the band Phase F retargets to.
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("lemma", RESERVED_WORD_LEMMAS)
def test_reserved_reader_words_are_taught_curriculum_lemmas(lemma: str) -> None:
"""The precondition of the collision: these really are taught terms."""
from teaching.curriculum_premises import load_curriculum
assert lemma in load_curriculum("philosophy_theology").vocabulary
@pytest.mark.parametrize("lemma", RESERVED_WORD_LEMMAS)
def test_reserved_word_atoms_refuse_rather_than_answer(lemma: str) -> None:
"""A control word as a TERM refuses typed — it never answers wrongly.
This is the honest failure: `compiled_premises_unreadable` is a coverage miss
excluded from reliability's denominator (ADR-0175 §4), not a confabulation. The
hazard is that the vocabulary boundary and the reader's grammar are not screened
against each other, so a future pack teaching "and"/"or"/"if" would open a much
larger hole silently. Recorded, not fixed — out of Phase C's scope
(DIVISION-OF-WORK §6.5).
"""
decision = decide_curriculum_question(f"Does knowledge requires {lemma}?")
assert decision.verdict == "declined"
assert decision.reason == "compiled_premises_unreadable"
def test_the_collision_is_bounded_to_two_lemmas() -> None:
"""Every refusal in the whole 69,666-atom space traces to `then`/`therefore`.
Pinned as a bound, so the day a third reserved word enters the curriculum this
fails instead of quietly enlarging the gap.
"""
offenders: set[str] = set()
checked = 0
for domain, family in practice_bands():
for atom in routable_atoms(domain, family):
reserved = {atom.subject, atom.obj} & set(RESERVED_WORD_LEMMAS)
if not reserved:
continue
checked += 1
if decide_curriculum_question(atom.text).verdict == "declined":
offenders |= reserved
assert checked > 0, "no reserved-word atoms found — the sweep is not exercising anything"
assert offenders == set(RESERVED_WORD_LEMMAS)
# ---------------------------------------------------------------------------
# 3. Distinct evidence, structurally.
# ---------------------------------------------------------------------------
def test_committed_equals_distinct_for_every_band(problems: tuple) -> None:
"""ADR-0264 R9 holds by construction: case identity IS the decision key."""
by_band: dict[str, list] = {}
for problem in problems:
by_band.setdefault(problem.class_name, []).append(problem.payload.key)
audits = audit_bands(by_band)
assert audits
for audit in audits:
assert audit.committed == audit.distinct
assert audit.inflation == 1.0
assert audit.max_repeat == 1
def test_distinctness_guard_fails_under_mutation() -> None:
"""The R9 producer guard must be able to fail, or it is worthless.
Same discipline as `tests/test_volume_honesty.py::test_audit_detects_padding`:
a pin that cannot go red proves nothing. Feeds the guard a corpus with one
atom repeated and asserts it refuses.
"""
atom = QueryAtom("physics", "causal", "force", "causes", "acceleration")
assert atom.key == ("physics", "force", "causes", "acceleration")
import evals.curriculum_serve.practice.generator as gen
original = gen.all_gold_problems
from core.learning_arena.protocols import Problem
def padded(cap: int = CASES_PER_BAND) -> tuple[Problem, ...]:
return (
Problem("a", "curriculum_physics_causal", atom),
Problem("b", "curriculum_physics_causal", atom),
)
gen.all_gold_problems = padded # type: ignore[assignment]
try:
with pytest.raises(DuplicateAtom, match="committed twice"):
gen.assert_practice_atoms_distinct()
finally:
gen.all_gold_problems = original # type: ignore[assignment]
assert_practice_atoms_distinct() # the real corpus still passes
# ---------------------------------------------------------------------------
# 4. The corpus itself: shape, determinism, wrong=0.
# ---------------------------------------------------------------------------
def test_band_atom_spaces_are_exactly_as_recorded() -> None:
measured = {
band_for(d, f): len(routable_atoms(d, f)) for d, f in practice_bands()
}
assert measured == BAND_ATOM_SPACE
def test_every_band_is_a_populated_family() -> None:
"""A family with no chains is not a band with zero volume — it is not a band."""
from teaching.curriculum_premises import load_curriculum
for domain, family in practice_bands():
assert load_curriculum(domain).family(family)
assert len(practice_bands()) == len(BAND_ATOM_SPACE)
def test_cap_selects_taught_edges_first() -> None:
"""Taught edges are the only committed-POSITIVE evidence; a sample that
dropped them would report reliability while exercising no curriculum."""
for domain, family in practice_bands():
space = len(routable_atoms(domain, family))
cases = band_cases(domain, family)
assert len(cases) == min(space, CASES_PER_BAND)
taught = taught_atoms(domain, family)
assert taught <= {c.key for c in cases}, f"{domain}/{family} dropped a taught edge"
def test_corpus_is_deterministic(problems: tuple) -> None:
assert [p.problem_id for p in all_gold_problems()] == [p.problem_id for p in problems]
def test_wrong_is_zero_for_every_band(report: dict) -> None:
"""The serving path and the independent oracle agree on every committed atom."""
assert report["wrong_is_zero"] is True
for band, entry in report["classes"].items():
assert entry["wrong"] == 0, f"{band}: {entry}"
assert set(report["classes"]) == set(BAND_ATOM_SPACE)
def test_gold_mix_has_no_refuted_class(report: dict) -> None:
"""`refuted` is reachable in MECHANISM but absent from CONTENT.
ADR-0264 R1–R4/R8 are implemented, so a negative row would be compiled,
decided and scored as `refuted` — `tests/test_curriculum_polarity.py` proves
that end to end against a temporary corpus row. No negative row is COMMITTED,
because authoring curriculum is Phase F and Shay's. So the shipped mix is
still entailed/unknown only, and this stays a true statement about the corpus
rather than about the machinery.
"""
for band, entry in report["classes"].items():
assert "refuted" not in entry["gold_mix"], (
f"{band} produced refuted — a negative row was committed. That is "
"Phase F content; update this pin deliberately alongside it."
)
assert set(entry["gold_mix"]) <= {"entailed", "unknown", "declined"}
def test_mix_totals_reconcile_with_the_ledger(
ledger: dict, report: dict, mix: dict
) -> None:
"""The mix is counted over the same corpus the ledger tallies, not a second one."""
for band, tally in ledger.items():
assert sum(mix[band].values()) == tally.attempted
assert report["classes"][band]["committed"] == tally.committed

329
tests/test_ledger_reseal.py Normal file
View file

@ -0,0 +1,329 @@
"""Phase D — the ``ledger_reseal`` ceremony stage and its CLI verb.
`RatificationReceipt.pending_stages` has named `ledger_reseal` since the
ratification ceremony was written and nothing performed it. This pins the verb
that performs it, and — more importantly — the reason it refuses by default.
The load-bearing property is **not** "the ledger can be rewritten". It is that
rewriting it cannot grant a serving license by accident. Reliability is
commitment precision, so a curriculum band clears θ_SERVE on correct
NON-COMMITMENTS alone; without the gate below, `reseal` would be a housekeeping
verb that silently made four bands authoritative.
"""
from __future__ import annotations
import dataclasses
import json
from pathlib import Path
import pytest
from core.ratified_ledger import CAPABILITY_LEDGERS, LedgerSpec, load_sealed_ledger
from core.reliability_gate import ClassTally
from teaching.ledger_reseal import (
ReadyToReseal,
ResealPlan,
apply_reseal,
plan_reseal,
resealable_capabilities,
)
@pytest.fixture(scope="module")
def plan() -> ResealPlan:
return plan_reseal("curriculum_serve")
# ---------------------------------------------------------------------------
# 1. What is resealable, and what must never be.
# ---------------------------------------------------------------------------
def test_only_curriculum_is_resealable() -> None:
"""`deduction_serve` must not be reachable from a CLI verb.
Its ledger is SHA-sealed, ratified, and gating a live flag, and 21 of its 25
bands do not clear θ_SERVE on distinct evidence
(`tests/test_volume_honesty.py`). A reseal verb that regenerated it would
erase a measured exposure as a side effect of housekeeping. `estimation`
ships sealed with its own gate and has no reason to move.
"""
assert resealable_capabilities() == ("curriculum_serve",)
assert "deduction_serve" not in resealable_capabilities()
assert "estimation" not in resealable_capabilities()
# ...and both are still registered capabilities, so this is an exclusion by
# decision rather than by them being unknown.
assert {"deduction_serve", "estimation"} <= set(CAPABILITY_LEDGERS)
def test_unregistered_capability_refuses() -> None:
with pytest.raises(ReadyToReseal, match="not resealable"):
plan_reseal("deduction_serve")
# ---------------------------------------------------------------------------
# 2. Planning writes nothing, and reports the license delta.
# ---------------------------------------------------------------------------
def test_planning_writes_nothing(plan: ResealPlan) -> None:
"""A plan is a read. The artifact must still be absent afterwards."""
assert not plan.path.exists(), (
"planning a reseal must not create the ledger — see "
"tests/test_curriculum_practice.py::test_curriculum_ledger_is_not_committed"
)
assert plan.capability == "curriculum_serve"
assert plan.artifact["schema"] == "curriculum_serve_ledger_v1"
def test_plan_reports_the_bands_a_reseal_would_newly_license(plan: ResealPlan) -> None:
"""The whole point of the stage being a separate act.
Nothing is licensed today (the ledger is absent), and a reseal would license
four bands — every band whose routable atom space reaches 657. Those are the
numbers `docs/research/curriculum-practice-producer-2026-07-26.md` §1 records.
"""
assert plan.licensed_before == ()
assert len(plan.licensed_after) == 4
assert set(plan.newly_licensed) == set(plan.licensed_after)
assert plan.revoked == ()
assert plan.is_housekeeping is False
def test_newly_licensed_bands_are_licensed_on_non_commitments(plan: ResealPlan) -> None:
"""The mix the operator must see before authorizing — `ClassTally` has no
verdict axis, so the gate itself cannot distinguish these from real
competence."""
for band in plan.newly_licensed:
mix = plan.mix[band]
assert mix.get("entailed", 0) <= 9
assert mix.get("unknown", 0) > 600
assert plan.committed[band] >= 657
# ---------------------------------------------------------------------------
# 3. Applying refuses to grant a license by accident. This is the unit.
# ---------------------------------------------------------------------------
def test_apply_refuses_to_grant_a_license_and_writes_nothing(
plan: ResealPlan, tmp_path: Path
) -> None:
target = tmp_path / "curriculum_serve_ledger.json"
redirected = dataclasses.replace(plan, path=target)
with pytest.raises(ReadyToReseal, match="would newly license 4 band"):
apply_reseal(redirected)
assert not target.exists(), "a refused reseal must not have written"
def test_refusal_names_the_bands_and_their_entailed_counts(
plan: ResealPlan, tmp_path: Path
) -> None:
"""A refusal an operator cannot act on is just an obstacle."""
with pytest.raises(ReadyToReseal) as exc:
apply_reseal(dataclasses.replace(plan, path=tmp_path / "l.json"))
message = str(exc.value)
for band in plan.newly_licensed:
assert band in message
assert "entailed=" in message
assert "--allow-new-licenses" in message
assert "NON-COMMITMENTS" in message
def test_apply_with_explicit_authorization_writes_a_verifying_artifact(
plan: ResealPlan, tmp_path: Path
) -> None:
"""The grant path works — into tmp, never into `chat/data/`."""
target = tmp_path / "curriculum_serve_ledger.json"
written = apply_reseal(
dataclasses.replace(plan, path=target), allow_new_licenses=True
)
assert written == target
tallies = load_sealed_ledger(target)
assert len(tallies) == 11
assert all(t.wrong == 0 for t in tallies.values())
artifact = json.loads(target.read_text(encoding="utf-8"))
assert artifact["provenance"] == "evals.curriculum_serve.practice.runner.seal_ledger"
def test_a_reseal_that_only_revokes_needs_no_authorization(tmp_path: Path) -> None:
"""Losing a license is the gate becoming MORE conservative.
Only granting is gated. A producer change that drops a band below the floor
must be applicable without an authorization flag, or the safe direction
becomes the hard one.
"""
plan = ResealPlan(
capability="curriculum_serve",
path=tmp_path / "l.json",
licensed_before=("band_a", "band_b"),
licensed_after=("band_a",),
committed={"band_a": 700, "band_b": 3},
mix={"band_a": {"entailed": 700}, "band_b": {"entailed": 3}},
artifact={"schema": "x", "classes": {}, "content_sha256": "y"},
)
assert plan.newly_licensed == ()
assert plan.revoked == ("band_b",)
assert plan.is_housekeeping is True
assert apply_reseal(plan) == plan.path # no flag needed
assert plan.path.exists()
def test_a_pure_housekeeping_reseal_is_allowed(tmp_path: Path) -> None:
"""Identical before/after — the ordinary case once volume is settled."""
plan = ResealPlan(
capability="curriculum_serve",
path=tmp_path / "l.json",
licensed_before=("band_a",),
licensed_after=("band_a",),
committed={"band_a": 700},
mix={"band_a": {"entailed": 700}},
artifact={"schema": "x", "classes": {}, "content_sha256": "y"},
)
assert plan.is_housekeeping is True
apply_reseal(plan)
assert plan.path.exists()
# ---------------------------------------------------------------------------
# 4. A tampered ledger on disk refuses rather than being silently overwritten.
# ---------------------------------------------------------------------------
def test_tampered_current_ledger_refuses_instead_of_being_overwritten(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Overwriting a hand-edited ledger would destroy the evidence of tampering.
`plan_reseal` reads the current ledger through the SAME verifier the serving
path uses, so a broken `content_sha256` surfaces here.
"""
target = tmp_path / "curriculum_serve_ledger.json"
target.write_text(
json.dumps(
{
"schema": "curriculum_serve_ledger_v1",
"classes": {"curriculum_physics_causal": {"correct": 999, "wrong": 0}},
"content_sha256": "0" * 64,
"note": "hand-edited",
"provenance": "not-sealed-practice",
},
indent=2,
),
encoding="utf-8",
)
import teaching.ledger_reseal as mod
monkeypatch.setitem(
mod.CAPABILITY_LEDGERS,
"curriculum_serve",
LedgerSpec(
capability="curriculum_serve", path=target, missing_ok=True, note="test"
),
)
with pytest.raises(Exception, match="content_sha256 mismatch"):
plan_reseal("curriculum_serve")
# The tampered bytes are still there to be investigated.
assert "hand-edited" in target.read_text(encoding="utf-8")
# ---------------------------------------------------------------------------
# 5. The CLI verb.
# ---------------------------------------------------------------------------
def _run_cli(*argv: str) -> tuple[int, str]:
import argparse
import io
import contextlib
from core.cli_proposal_queue import register
parser = argparse.ArgumentParser()
register(parser.add_subparsers(dest="command", required=True))
args = parser.parse_args(["proposal-queue", *argv])
buf = io.StringIO()
with contextlib.redirect_stdout(buf):
code = args.func(args)
return code, buf.getvalue()
def test_cli_dry_run_reports_without_writing() -> None:
code, out = _run_cli("reseal", "curriculum_serve", "--dry-run")
assert code == 0
assert "nothing written" in out
assert "NEWLY LICENSED" in out
assert not CAPABILITY_LEDGERS["curriculum_serve"].path.exists()
def test_cli_refuses_without_authorization() -> None:
code, out = _run_cli("reseal", "curriculum_serve")
assert code == 1
assert "REFUSED" in out
assert not CAPABILITY_LEDGERS["curriculum_serve"].path.exists()
def test_cli_json_is_machine_readable() -> None:
code, out = _run_cli("reseal", "curriculum_serve", "--dry-run", "--json")
assert code == 0
payload = json.loads(out)
assert payload["written"] is False
assert payload["is_housekeeping"] is False
assert len(payload["newly_licensed"]) == 4
assert payload["licensed_before"] == []
def test_reseal_verb_is_registered_under_proposal_queue() -> None:
"""The verb has to be reachable from `core proposal-queue`, not only importable."""
import argparse
from core.cli_proposal_queue import register
parser = argparse.ArgumentParser()
register(parser.add_subparsers(dest="command", required=True))
args = parser.parse_args(
["proposal-queue", "reseal", "curriculum_serve", "--dry-run"]
)
assert args.proposal_queue_command == "reseal"
assert args.capability == "curriculum_serve"
assert args.dry_run is True
assert args.allow_new_licenses is False
# ---------------------------------------------------------------------------
# 6. The receipt now names the command that performs its pending stage.
# ---------------------------------------------------------------------------
def test_ratify_output_names_the_reseal_command() -> None:
"""Naming a pending stage without naming its command is how `ledger_reseal`
stayed unperformed since the ceremony was written.
Asserted at SOURCE level deliberately. Exercising the branch needs a
successful non-dry-run ratification, which appends to the real committed
corpus — the test would either mutate `teaching/domain_chains/` or stub out
the very ceremony it claims to check. A source pin is the honest instrument
here, and it is stated as one rather than dressed up as a behavioural test.
"""
import inspect
from teaching.ratification import RatificationReceipt
import core.cli_proposal_queue as cli
assert "ledger_reseal" in RatificationReceipt.__dataclass_fields__[
"pending_stages"
].default
source = inspect.getsource(cli.cmd_proposal_queue_ratify)
assert 'if "ledger_reseal" in receipt.pending_stages' in source
assert "core proposal-queue reseal curriculum_serve" in source
def test_reseal_stage_is_not_reachable_from_ratify_chain() -> None:
"""Doctrine: no ratification automation. `ratify_chain` must not seal."""
import inspect
import teaching.ratification as ratification
source = inspect.getsource(ratification)
for forbidden in ("ledger_reseal", "seal_ledger", "write_sealed_ledger"):
assert f"import {forbidden}" not in source
assert f"{forbidden}(" not in source
tally = ClassTally("x", correct=1)
assert tally.committed == 1 # sanity: the gate module is importable here

59
tests/test_volume_honesty.py
View file

@ -25,9 +25,11 @@ Three producers back the three entries in `core.ratified_ledger.CAPABILITY_LEDGE
distinct evidence; the worst three inflate 28 distinct cases to 720 committed.
Recorded, pinned, and NOT silently repaired: the ledger is SHA-sealed, ratified,
and gating a live flag, so changing it is Shay's ratification, not a test's.
- **curriculum_serve** (ADR-0262/0264) — producer unbuilt. The registry pin below
fails the moment it is built without an audit source, which is the forcing
function that stops Phase C from repeating the deduction pattern.
- **curriculum_serve** (ADR-0262/0264) — clean, and clean *structurally*: the
producer's case identity IS the query atom, so ``committed == distinct`` cannot
drift apart the way it did for deduction, where case identity was a quota index.
Its ledger is deliberately uncommitted — see
``evals/curriculum_serve/practice/runner.py``.
Deliberately narrow. This pins evidence *distinctness*. It does not pin outcome
mix — whether each verdict class needs its own volume is an open ruling recorded
@ -99,13 +101,30 @@ def _estimation_keys() -> dict[str, list[Hashable]]:
return out
def _curriculum_keys() -> dict[str, list[Hashable]]:
"""Decision keys per band for the curriculum practice corpus (Phase C).
The key is the query ATOM, which is also the producer's case identity — so
this audit can only ever report inflation 1.0 while the producer is correct,
and reports the real figure the moment it is not. Contrast the deduction
source above, where the key had to be recovered from case text because the
producer's identity was a quota index.
"""
from evals.curriculum_serve.practice import all_gold_problems
out: dict[str, list[Hashable]] = {}
for problem in all_gold_problems():
out.setdefault(problem.class_name, []).append(problem.payload.key)
return out
#: capability -> keys provider, or ``None`` when the producer is not built.
#: ``None`` is a declaration, not an omission: :func:`test_every_licensed_capability_has_an_audit_source`
#: accepts it only while the capability's ledger is genuinely absent.
AUDIT_SOURCES: dict[str, Callable[[], dict[str, list[Hashable]]] | None] = {
"estimation": _estimation_keys,
"deduction_serve": _deduction_keys,
"curriculum_serve": None, # Phase C — see module docstring
"curriculum_serve": _curriculum_keys,
}
@ -238,9 +257,10 @@ def test_audit_rejects_impossible_counts() -> None:
def test_every_licensed_capability_has_an_audit_source() -> None:
"""Derived from `CAPABILITY_LEDGERS`, so a new capability cannot slip past.
This is the forcing function for Phase C: the moment
`chat/data/curriculum_serve_ledger.json` exists, a ``None`` audit source
stops being an honest declaration and this fails.
This was the forcing function for Phase C, and it worked: a ``None`` audit
source stops being an honest declaration the moment the capability's ledger
exists. All three producers now carry one, so the guard is holding the line
for the NEXT capability rather than for curriculum.
"""
assert set(AUDIT_SOURCES) == set(CAPABILITY_LEDGERS), (
"AUDIT_SOURCES and CAPABILITY_LEDGERS disagree — a licensed capability is "
@ -294,6 +314,31 @@ def test_estimation_producer_has_no_inflation(
assert below_floor(estimation_audits, THETA_SERVE) == ()
# ---------------------------------------------------------------------------
# 3b. The curriculum producer — clean, and clean by construction (Phase C).
# ---------------------------------------------------------------------------
def test_curriculum_producer_has_no_inflation() -> None:
"""The second producer that satisfies the invariant, and the first that
satisfies it STRUCTURALLY.
Its case identity is the query atom, so ``committed == distinct`` is not a
property the generator has to maintain — it is the same fact twice. Pinned here
beside the estimation standard because this file is where a reader looks to ask
"which producers are honest about volume?".
"""
audits = audit_bands(_curriculum_keys())
assert audits, "curriculum producer yielded no bands"
assert len(audits) == 11, f"expected 11 populated bands, found {len(audits)}"
for audit in audits:
assert audit.committed == audit.distinct, format_report(audits, THETA_SERVE)
assert audit.inflation == 1.0
assert audit.max_repeat == 1
assert {a.band for a in below_floor(audits, THETA_SERVE)} == {
a.band for a in audits if a.distinct < SERVE_VOLUME_AT_THETA_099
}, "a band below the floor must be below it for lack of VOLUME, never inflation"
# ---------------------------------------------------------------------------
# 4. The deduction producer — the measured exposure, pinned in both directions.
# ---------------------------------------------------------------------------