From c1d08ff9d55944a3372783cf55e30d02baf0cdad Mon Sep 17 00:00:00 2001 From: Shay Date: Sun, 26 Jul 2026 09:52:54 -0700 Subject: [PATCH 1/3] feat(curriculum): sealed-practice producer for curriculum serving (Phase C) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds the producer `chat/curriculum_serve_license.py` already named as the only writer of the ledger it reads — `evals.curriculum_serve.practice.runner .seal_ledger` — which did not exist. Until now, authoring curriculum could not earn a license at any volume. A band's case space is ENUMERATED, not authored: every exam question that routes to that subject (resolve_domain) whose relation is in that family, one committed case per distinct query atom. So `committed == distinct` is not a property the generator maintains, it is the same fact twice — the structural form of ADR-0264 R9, and why this producer cannot develop the deduction producer's exposure (CASES_PER_BAND=720 filled from spaces as small as 28). CASES_PER_BAND=660 follows `evals/determination_estimation` per DIVISION-OF-WORK §4: a ceiling on committed volume, not a quota. 11 bands, wrong=0, inflation 1.0 everywhere. THE LEDGER IS DELIBERATELY NOT COMMITTED. The plan's Phase-1 exit criterion asked for "a real (still-unearned) ledger"; that state is unreachable. Reliability is commitment precision and a correct UNKNOWN is a commitment, so conservative_floor(660,660)=0.990046 clears theta_SERVE on non-commitments alone. Sealing today licenses FOUR bands on evidence that is 99.0%-99.98% correct non-commitment — which ADR-0262 §5.1 rules unacceptable and which ClassTally structurally cannot see (no verdict axis). Writing the artifact is therefore a ratification and belongs to Phase D's explicit human-run verb. The open outcome-mix ruling is now blocking, not academic. Two corrections to my own earlier records, both measured: - ADR-0264 §4.2 sized bands from per-TERM exclusivity; the router's predicate is per-PAIR, which is looser. Every ceiling was understated, and systems_software_causal is 720 not 630 — flipping it from "cannot reach 657" to "can". 4 of 11 bands can reach the floor, not 3. Now pinned as measured values in BAND_ATOM_SPACE. - `then` and `therefore` are taught philosophy_theology lemmas AND control words of the argument reader (`therefore` is the conclusion marker). 164 routable atoms refuse compiled_premises_unreadable — coverage misses, never wrongs — in the band Phase F retargets to. The vocabulary boundary is not screened against the reader's grammar; a pack teaching and/or/not/if would open a much larger hole the same way. Recorded, not fixed (§6.5), pinned as a two-lemma bound. Registers curriculum_serve in AUDIT_SOURCES, retiring the declared `None` that was Phase C's forcing function, and adds the producer to `deductive`. [Verification]: in-worktree on canonical CPython 3.12.13 with `uv sync --locked`: smoke 571 (569 + 1 test + 1 parametrization), deductive 310 (291 + 19). Mutation-checked: injecting the deduction padding pattern into band_cases turns 5 tests red including the R9 audit and the seal guard. curriculum_serve lane unchanged at n=32 correct=32 wrong=0 anti_recall=5; all 11 lane SHA pins match; no ledger written, so no serving change. --- core/cli_test.py | 1 + ...curriculum-practice-producer-2026-07-26.md | 166 +++++++++ evals/curriculum_serve/practice/__init__.py | 32 ++ evals/curriculum_serve/practice/generator.py | 274 ++++++++++++++ evals/curriculum_serve/practice/runner.py | 187 ++++++++++ tests/test_curriculum_practice.py | 352 ++++++++++++++++++ tests/test_volume_honesty.py | 59 ++- 7 files changed, 1064 insertions(+), 7 deletions(-) create mode 100644 docs/research/curriculum-practice-producer-2026-07-26.md create mode 100644 evals/curriculum_serve/practice/__init__.py create mode 100644 evals/curriculum_serve/practice/generator.py create mode 100644 evals/curriculum_serve/practice/runner.py create mode 100644 tests/test_curriculum_practice.py diff --git a/core/cli_test.py b/core/cli_test.py index ab018585..f43e5be2 100644 --- a/core/cli_test.py +++ b/core/cli_test.py @@ -229,6 +229,7 @@ TEST_SUITES: dict[str, tuple[str, ...]] = { "tests/test_exist_argument_reader.py", "tests/test_deduction_serve_e2e.py", "tests/test_curriculum_serve.py", + "tests/test_curriculum_practice.py", "tests/test_ratified_ledger_bridge.py", "tests/test_vocab_trigger_instrument.py", ), diff --git a/docs/research/curriculum-practice-producer-2026-07-26.md b/docs/research/curriculum-practice-producer-2026-07-26.md new file mode 100644 index 00000000..c7f9d03e --- /dev/null +++ b/docs/research/curriculum-practice-producer-2026-07-26.md @@ -0,0 +1,166 @@ +# Curriculum practice producer — what building it measured (Phase C) + +**Date:** 2026-07-26 · **Arc:** curriculum-license-loop-2026-07 · **Unit:** Phase C +**Code:** `evals/curriculum_serve/practice/{generator,runner}.py`, +`tests/test_curriculum_practice.py` +**Governs:** ADR-0262 §5 (what a curriculum band can earn), ADR-0264 R9 (volume honesty) + +The producer named by `chat/curriculum_serve_license.py`'s own docstring — +`evals.curriculum_serve.practice.runner.seal_ledger`, declared "the only writer" +of the ledger that module reads — did not exist. It does now. Building it +measured three things, two of which correct records I wrote earlier in this arc. + +Reproduce everything below with: + +```bash +uv run python -m evals.curriculum_serve.practice.runner # capped corpus +uv run python -m evals.curriculum_serve.practice.runner --full # every atom (~75s) +``` + +--- + +## 0. What the producer is + +A band's case space is **enumerated, not authored**: every exam question that +routes to that subject (`chat.curriculum_surface.resolve_domain`) whose relation +falls in that family. One committed case per distinct query atom, so +`committed == distinct` is not a property the generator maintains — it is the same +fact twice. This is the structural version of what ADR-0264 R9 asks for, and it is +why the curriculum producer cannot develop the deduction producer's exposure +(`CASES_PER_BAND = 720` filled by cycling spaces as small as 28). + +`CASES_PER_BAND = 660` follows `evals/determination_estimation`, per +`DIVISION-OF-WORK.md` §4. It is a **ceiling on committed volume, not a quota**: a +band with a smaller space emits its whole space and reports a number below the +floor. Selection above the cap is every taught edge plus a fixed stride across the +rest — a lexicographic prefix of 660 out of 45,300 would cover about five subject +terms of 151 and report one corner of a band as the band. + +Result over the capped corpus: **11 bands, wrong = 0, inflation 1.0 everywhere.** + +--- + +## 1. Finding — a committed ledger is necessarily an *earning* ledger + +The plan of record's Phase-1 exit criterion asks that +`curriculum_serve_license()` read "a real (still-unearned) ledger instead of an +absent one." + +**That state is unreachable.** Reliability is commitment precision, and a correct +UNKNOWN is a commitment. `conservative_floor(660, 660) = 0.990046 ≥ 0.99`, so any +band with ≥657 routable atoms clears θ_SERVE on non-commitments alone, whatever its +curriculum says. Sealing today licenses **four** bands: + +| band | committed | refused | reliability | SERVE | entailed | entailed share | +|---|---:|---:|---:|:---:|---:|---:| +| `curriculum_philosophy_theology_modal` | 45,188 | 112 | 0.99985 | **yes** | 8 | 0.0177% | +| `curriculum_philosophy_theology_contrast` | 22,598 | 52 | 0.99971 | **yes** | 8 | 0.0354% | +| `curriculum_physics_causal` | 720 | 0 | 0.99087 | **yes** | 7 | 0.9722% | +| `curriculum_systems_software_causal` | 720 | 0 | 0.99087 | **yes** | 7 | 0.9722% | +| `curriculum_physics_modal` | 480 | 0 | 0.98636 | no | 9 | 1.8750% | +| `curriculum_mathematics_logic_modal` | 480 | 0 | 0.98636 | no | 4 | 0.8333% | +| `curriculum_systems_software_modal` | 480 | 0 | 0.98636 | no | 6 | 1.2500% | +| `curriculum_mathematics_logic_contrast` | 240 | 0 | 0.97309 | no | 8 | 3.3333% | +| `curriculum_mathematics_logic_evidential` | 240 | 0 | 0.97309 | no | 8 | 3.3333% | +| `curriculum_mathematics_logic_sequence` | 240 | 0 | 0.97309 | no | 4 | 1.6667% | +| `curriculum_systems_software_sequence` | 240 | 0 | 0.97309 | no | 3 | 1.2500% | + +(Exhaustive sweep: 71,790 atoms, `wrong = 0`, 164 refusals.) + +A licensed band would therefore be certified on evidence that is **99.0%–99.98% +correct non-commitment**. ADR-0262 §5.1 explicitly rules that unacceptable. The +gate cannot see it: `ClassTally` carries `correct`/`wrong`/`refused` and **no +verdict axis**, so a correct UNKNOWN is indistinguishable from a correct ENTAILED +once tallied. Mix can only be enforced at the producer, which is exactly the +open outcome-mix ruling recorded in +`docs/research/distinct-evidence-audit-2026-07-25.md`. + +**Consequence for this arc:** that ruling is no longer academic — it is the last +thing between a built loop and four unearned licenses. Phase C therefore ships the +writer and **does not commit the artifact**; `chat/data/curriculum_serve_ledger.json` +stays absent, `missing_ok=True` stays true, and every band stays DISCLOSED. Writing +it is a ratification, so it belongs to the explicit `core proposal-queue reseal` +verb a human runs (Phase D), and it should not be run before the mix ruling. + +`tests/test_curriculum_practice.py::test_curriculum_ledger_is_not_committed` pins +the absence, and `test_sealing_would_license_exactly_the_bands_that_reach_the_floor` +pins the consequence, so neither can be lost and re-derived wrongly. + +### Recommendation (still Shay's call) + +Enforce mix **at the producer**, as a per-verdict-class floor rather than a ratio: +a band earns SERVE only when each verdict class it can express independently +reaches the volume the floor requires. Under that rule no band earns anything today +(max entailed volume is 9), which is the honest reading, and it makes ADR-0262 §5.1 +mechanical instead of advisory. It also keeps the deduction bands' aggregate +licenses intact, since imposing a per-class 657 retroactively would fail all 25. + +--- + +## 2. Correction — ADR-0264 §4.2's ceiling table understates every band + +I sized bands from **per-term exclusivity** (a term taught by only one subject). +The router's actual predicate is **per-pair**: `resolve_domain` routes a question +iff exactly one served subject holds *both* terms. A term taught in two subjects +can still appear in a pair only one subject holds both halves of, so per-term +exclusivity is a strictly tighter bound and under-counts. + +| band | ADR-0264 §4.2 | true | verdict change | +|---|---:|---:|---| +| `curriculum_systems_software_causal` | 630 | **720** | **cannot reach 657 → CAN** | +| `curriculum_philosophy_theology_modal` | 44,104 | 45,300 | — | +| `curriculum_philosophy_theology_contrast` | 22,052 | 22,650 | — | +| `curriculum_mathematics_logic_modal` | 420 | 480 | — | +| `curriculum_mathematics_logic_{sequence,contrast,evidential}` | 210 | 240 | — | +| `curriculum_systems_software_modal` | 420 | 480 | — | +| `curriculum_systems_software_sequence` | 210 | 240 | — | +| `curriculum_physics_{causal,modal}` | 720 / 480 | 720 / 480 | — | + +So **4 of 11 bands can reach 657, not 3**, and **7 cannot, not 8**. The retarget +conclusion is unaffected — `philosophy_theology · modal` is still the largest space +by two orders of magnitude, and `physics · modal` (480) is still impossible at any +authoring volume. The numbers are now pinned in +`tests/test_curriculum_practice.py::BAND_ATOM_SPACE`, measured rather than derived, +so this class of error cannot recur silently. + +--- + +## 3. Finding — two taught lemmas collide with the argument reader's grammar + +`then` and `therefore` are lemmas `philosophy_theology`'s packs teach **and** +control words of the argument reader: `therefore` is literally the conclusion +marker in `". Therefore ."`, and `then` is the conditional consequent +marker. An atom using either as a term compiles an argument the reader cannot parse +back, so it refuses `compiled_premises_unreadable`. + +- **164 routable atoms** affected (112 modal + 52 contrast) out of 71,790. +- All 164 are **coverage misses, not wrongs** — excluded from reliability's + denominator (ADR-0175 §4), never a confabulation. `wrong = 0` holds. +- Exactly two lemmas are responsible; every other implicated term is only ever the + *partner* of one of them. +- It is in the band Phase F retargets to. + +The narrow cost is small. The mechanism is not: **the vocabulary boundary and the +reader's grammar are never screened against each other.** A future pack teaching +`and`, `or`, `not`, or `if` would open a much larger hole the same way, and it would +present as a coverage dip rather than an error. A reserved-word check at pack mount +or at chain ratification would close it. + +Not fixed here — outside Phase C's unit (`DIVISION-OF-WORK.md` §6.5). Pinned as a +bound by `test_the_collision_is_bounded_to_two_lemmas`, so a third reserved word +entering the curriculum fails loudly instead of quietly enlarging the gap. + +--- + +## 4. What did not move + +- `evals/curriculum_serve` lane: `[physics] n=32 correct=32 wrong=0 declined_mismatch=0 + anti_recall=5`, unchanged; `report.json` bytes and the `curriculum_serve_v1` SHA + pin unchanged. The producer reads the same solver the lane does and writes nothing + the lane reads. +- No serving behaviour change of any kind: no ledger committed, so + `curriculum_serve_license()` still returns `None` for every band and every + curriculum answer is still served DISCLOSED. +- `refuted` remains unreachable — no corpus row can express a negative until + ADR-0264 R1–R4 is implemented. `test_gold_mix_has_no_refuted_class` pins that as a + red test for the polarity unit to turn green. diff --git a/evals/curriculum_serve/practice/__init__.py b/evals/curriculum_serve/practice/__init__.py new file mode 100644 index 00000000..793d77f5 --- /dev/null +++ b/evals/curriculum_serve/practice/__init__.py @@ -0,0 +1,32 @@ +"""Sealed practice for curriculum-grounded serving (ADR-0262/0264, Phase C). + +The missing producer named by ``chat/curriculum_serve_license.py``'s docstring: +that module reads ``chat/data/curriculum_serve_ledger.json`` and declares +``evals.curriculum_serve.practice.runner.seal_ledger`` "the only writer" of it. +Until this package existed the writer did not, so authoring curriculum could not +earn a license at any volume. +""" + +from evals.curriculum_serve.practice.generator import ( + CASES_PER_BAND, + CurriculumOracleTether, + CurriculumSolver, + QueryAtom, + all_gold_problems, + assert_practice_atoms_distinct, + band_cases, + routable_atoms, + taught_atoms, +) + +__all__ = [ + "CASES_PER_BAND", + "CurriculumOracleTether", + "CurriculumSolver", + "QueryAtom", + "all_gold_problems", + "assert_practice_atoms_distinct", + "band_cases", + "routable_atoms", + "taught_atoms", +] diff --git a/evals/curriculum_serve/practice/generator.py b/evals/curriculum_serve/practice/generator.py new file mode 100644 index 00000000..df7c32ca --- /dev/null +++ b/evals/curriculum_serve/practice/generator.py @@ -0,0 +1,274 @@ +"""The curriculum practice corpus — one committed case per DISTINCT query atom. + +ADR-0262 (curriculum-grounded serving) supplies the solver; ADR-0264 R9 supplies +the sizing rule. The corpus is enumerated, never authored: a band's case space is +exactly the set of exam questions that ROUTE to its subject and whose relation +falls in its family, so there is no hand-written gold to drift and nothing to +paraphrase-pad. + +**Why enumeration is the honest shape here.** ``conservative_floor`` is a Wilson +bound and Wilson assumes independent trials, so a deterministic pipeline replaying +one input N times supplies one trial's evidence, not N (ADR-0264 R9). The +``deduction_serve`` producer sets a flat ``CASES_PER_BAND = 720`` and reaches it by +cycling a template x vocabulary space as small as 28 distinct instances; 21 of its +25 ratified bands therefore do not clear θ_SERVE on distinct evidence +(``tests/test_volume_honesty.py``). This producer cannot repeat that: its case +identity IS the query atom, so ``committed == distinct`` holds by construction +rather than by discipline. The model is ``evals/determination_estimation`` +(660 distinct cases per class, zero repeats), which is what +``DIVISION-OF-WORK.md`` §4 directs Phase C to follow. + +**Routability is asked, not re-derived.** An atom is in a band's space iff +``chat.curriculum_surface.resolve_domain`` routes its two terms to that subject. +Restating the predicate here would let the corpus and the serving path disagree +about which questions exist — so the corpus asks the serving path. This is not a +gold dependency: gold comes from ``evals.curriculum_serve.oracle``, which shares +no code with the serving path (ADR-0199 L-2). + +Note that per-*term* exclusivity is a strictly tighter bound than the per-*pair* +predicate the router actually applies: a term taught in two subjects can still +appear in a pair that only one subject holds both halves of. Sizing a band from +exclusive terms therefore UNDER-counts its space — ``systems_software · causal`` +reads as 630 under that bound and is really 720, which is the difference between +"cannot reach 657" and "can". + +Determinism: no clock, no RNG. The atom space is sorted, selection is a fixed +stride, so ``all_gold_problems()`` is byte-stable across runs and the sealed +ledger is safe to commit and SHA-verify on load. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from functools import lru_cache + +from chat.curriculum_surface import ( + SERVED_DOMAINS, + CurriculumQuery, + band_for, + decide_curriculum_question, + resolve_domain, +) +from core.learning_arena.protocols import BaseAttempt, DomainProblem, Problem +from evals.curriculum_serve.oracle import oracle_answer +from teaching.curriculum_premises import CONNECTIVE_FAMILY, FAMILIES, load_curriculum + +#: Committed cases per band, when the band's atom space is larger than this. +#: +#: 660 is ``evals/determination_estimation``'s constant, chosen just above the 657 +#: a perfect record needs to clear θ_SERVE=0.99 (``volume_for_theta(0.99)``). It is +#: a ceiling on *committed volume*, not a quota to be filled: a band whose space is +#: smaller emits its whole space and honestly reports a number below the floor. +CASES_PER_BAND = 660 + + +@dataclass(frozen=True, slots=True) +class QueryAtom: + """One distinct curriculum decision — the unit of committed evidence. + + The atom IS the decision key (ADR-0264 R9). Two cases with the same atom are + the same decision replayed, so the corpus admits each atom at most once and + :meth:`key` is what ``tests/test_volume_honesty.py`` audits. + """ + + domain: str + family: str + subject: str + connective: str + obj: str + + @property + def text(self) -> str: + """The exam question, in the closed ``Does ?`` grammar.""" + return f"Does {self.subject} {self.connective} {self.obj}?" + + @property + def key(self) -> tuple[str, str, str, str]: + """The distinct-decision key. Polarity is deliberately absent: a + negative curriculum row (ADR-0264 R1-R4) changes a question's ANSWER, + never its identity, and R4 rejects a corpus that states both polarities + of one atom at ratification. So polarity cannot distinguish two cases.""" + return (self.domain, self.subject, self.connective, self.obj) + + @property + def case_id(self) -> str: + return f"{self.domain}:{self.connective}:{self.subject}->{self.obj}" + + +@lru_cache(maxsize=None) +def _routes_to(subject: str, obj: str) -> str | None: + """The subject a term pair routes to, or ``None`` — the router's own answer.""" + domain = resolve_domain(CurriculumQuery(subject, "", obj)) + return domain if isinstance(domain, str) else None + + +@lru_cache(maxsize=None) +def routable_atoms(domain: str, family: str) -> tuple[QueryAtom, ...]: + """Every exam question that reaches *(domain, family)*, in sorted order. + + The band's complete case space: ordered term pairs the router sends to + *domain*, crossed with the family's connectives. Self-pairs are excluded — + ``Does x cause x?`` is not a curriculum question any corpus states. + """ + connectives = sorted(c for c, f in CONNECTIVE_FAMILY.items() if f == family) + terms = sorted(load_curriculum(domain).vocabulary) + atoms = [ + QueryAtom(domain, family, subject, connective, obj) + for subject in terms + for obj in terms + if subject != obj and _routes_to(subject, obj) == domain + for connective in connectives + ] + return tuple(sorted(atoms, key=lambda a: (a.subject, a.connective, a.obj))) + + +@lru_cache(maxsize=None) +def taught_atoms(domain: str, family: str) -> frozenset[tuple[str, str, str, str]]: + """The atom keys a ratified chain states directly — the ``entailed`` class. + + Scarce by nature: a band has as many taught edges as the curriculum states, + which is why ADR-0262 §5.1 makes authored volume the binding constraint on + what a band can demonstrate beyond non-commitment. + """ + return frozenset( + QueryAtom(domain, family, c.subject, c.connective, c.obj).key + for c in load_curriculum(domain).family(family) + ) + + +def band_cases(domain: str, family: str, cap: int = CASES_PER_BAND) -> tuple[QueryAtom, ...]: + """The committed cases for one band — distinct atoms, at most *cap* of them. + + Whole space when it fits. Otherwise: every taught edge, then a fixed STRIDE + across the remaining atoms in sorted order. + + The stride is deliberate rather than a prefix. A lexicographic prefix of 660 + atoms out of 45,300 would cover about five subject terms of 151, measuring one + corner of the band and reporting it as the band; a stride spreads the sample + across every subject the curriculum teaches. It is not a random sample — there + is no RNG here — but it is an unbiased-by-construction one, and it cannot be + tuned per band, which is the property that matters for an evidence artifact. + Taught edges are force-included because they are the only committed-POSITIVE + evidence a band has and a sample that missed them would report a band's + reliability while exercising none of its content. + """ + atoms = routable_atoms(domain, family) + if len(atoms) <= cap: + return atoms + taught_keys = taught_atoms(domain, family) + taught = [a for a in atoms if a.key in taught_keys] + rest = [a for a in atoms if a.key not in taught_keys] + need = max(cap - len(taught), 0) + stride = max(len(rest) // need, 1) if need else 1 + picked = rest[::stride][:need] + return tuple(sorted(taught + picked, key=lambda a: (a.subject, a.connective, a.obj))) + + +def practice_bands() -> tuple[tuple[str, str], ...]: + """Every *(domain, family)* pair the ratified corpora actually populate. + + A family with no chains is not a band with zero volume — it is not a band. + ``decide_curriculum_question`` refuses it ``empty_curriculum`` before any + band key is assigned, so committing cases under one would invent a capability + axis the serving path never uses. + """ + return tuple( + (domain, family) + for domain in SERVED_DOMAINS + for family in FAMILIES + if load_curriculum(domain).family(family) + ) + + +def all_gold_problems(cap: int = CASES_PER_BAND) -> tuple[Problem, ...]: + """The full practice corpus, every band, in a deterministic order.""" + problems: list[Problem] = [] + for domain, family in practice_bands(): + band = band_for(domain, family) + for atom in band_cases(domain, family, cap): + problems.append( + Problem(problem_id=atom.case_id, class_name=band, payload=atom) + ) + return tuple(problems) + + +class DuplicateAtom(AssertionError): + """A band committed the same query atom twice — the R9 invariant, violated.""" + + +def assert_practice_atoms_distinct(cap: int = CASES_PER_BAND) -> None: + """``committed == distinct``, enforced at the producer (ADR-0264 R9). + + The Phase B audit MEASURES inflation; this refuses to emit it. Both matter: + a producer that cannot pad is better than one that is caught padding, and the + audit still guards the sealed artifact against a future producer change. + """ + seen: dict[str, set[tuple[str, str, str, str]]] = {} + for problem in all_gold_problems(cap): + atom: QueryAtom = problem.payload + keys = seen.setdefault(problem.class_name, set()) + if atom.key in keys: + raise DuplicateAtom( + f"{problem.class_name}: query atom {atom.key} committed twice — " + "practice volume must be distinct evidence (ADR-0264 R9)" + ) + keys.add(atom.key) + + +class CurriculumSolver: + """The production decision path, under test (``chat/curriculum_surface.py``). + + ``committed`` is ``verdict != "declined"``. A typed refusal is an honest + non-commitment, excluded from reliability's denominator (ADR-0175 §4) and + counted as coverage — never conflated with a confabulation. That mapping is + the same one ``evals/curriculum_serve/runner.py`` applies when it separates a + ``declined`` mismatch from a ``wrong``. + """ + + domain_id = "curriculum_serve" + + def attempt(self, problem: DomainProblem) -> BaseAttempt: + atom: QueryAtom = problem.payload + decision = decide_curriculum_question(atom.text) + committed = decision.verdict != "declined" + return BaseAttempt( + committed=committed, + answer=decision.verdict, + reason=decision.reason, + case_id=problem.problem_id, + ) + + +class CurriculumOracleTether: + """Tier-1 gold: the INDEPENDENT curriculum oracle (ADR-0199 L-2). + + ``evals/curriculum_serve/oracle.py`` shares no code with the serving path — + its own loader, ratification predicate, family table, agreement normalization + and verdict rule. Two independently written procedures agreeing on every atom + is real evidence the serving path reads the curriculum correctly; a tether + built from the compiler would only prove the compiler agrees with itself. + """ + + domain_id = "curriculum_serve" + + def is_correct(self, attempt: BaseAttempt, problem: DomainProblem) -> bool: + return attempt.answer == self.gold_answer(problem) + + def gold_answer(self, problem: DomainProblem) -> str: + atom: QueryAtom = problem.payload + return oracle_answer(atom.domain, atom.subject, atom.connective, atom.obj).verdict + + +__all__ = [ + "CASES_PER_BAND", + "CurriculumOracleTether", + "CurriculumSolver", + "DuplicateAtom", + "QueryAtom", + "all_gold_problems", + "assert_practice_atoms_distinct", + "band_cases", + "practice_bands", + "routable_atoms", + "taught_atoms", +] diff --git a/evals/curriculum_serve/practice/runner.py b/evals/curriculum_serve/practice/runner.py new file mode 100644 index 00000000..6e26ba07 --- /dev/null +++ b/evals/curriculum_serve/practice/runner.py @@ -0,0 +1,187 @@ +"""Sealed-practice runner for curriculum serving (ADR-0262/0264, Phase C). + +Folds the ADR-0199 ``run_practice`` engine over the enumerated atom corpus +(``generator.py``) and reads each band's ``ClassTally`` through the reliability +gate. Two outputs, exactly mirroring ``evals/deduction_serve/practice/runner.py``: + +- ``run()`` — the falsifiable report: which *(subject × relation family)* bands + earned SERVE, at what committed volume, and with what verdict MIX. +- ``seal_ledger()`` — writes the SHA-sealed artifact the serving reader + (``chat/curriculum_serve_license.py``) trusts. That module's docstring already + names this function as the only writer; this is the function it named. + +**The artifact is NOT committed by this arc, and that is a finding, not an +omission.** Every band with a routable space ≥657 reaches θ_SERVE on the UNKNOWN +class alone: ``physics · causal`` commits 7 taught edges and 713 correct +non-commitments, and ``conservative_floor(660, 660) = 0.990046 ≥ 0.99``. So +sealing this ledger today would flip four bands from disclosed to authoritative on +a mix that is ~99% non-entailed — which ADR-0262 §5.1 explicitly rules +unacceptable, and which the still-open outcome-mix ruling (ADR-0264 R9's recorded +non-decision; ``docs/research/distinct-evidence-audit-2026-07-25.md``) exists to +settle. The plan of record's Phase-1 exit criterion asked for a ledger that is +"real (still-unearned)"; building the producer shows that state is unreachable. +Committing the artifact is therefore a ratification, and it is left to the +explicit ``core proposal-queue reseal`` verb a human runs (Phase D). + +Determinism: the corpus is enumerated + sorted (no clock, no RNG) and +``run_practice`` is a pure fold, so the sealed ledger is byte-identical across +runs — safe to commit and SHA-verify on load, whenever it is committed. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path +from typing import Any + +from core.learning_arena.engine import run_practice +from core.ratified_ledger import seal_artifact, write_sealed_ledger +from core.reliability_gate import Action, Ceilings, ClassTally, license_for +from evals.curriculum_serve.practice.generator import ( + CASES_PER_BAND, + CurriculumOracleTether, + CurriculumSolver, + all_gold_problems, + assert_practice_atoms_distinct, +) + +#: The committed sealed ledger lives next to its serving READER (chat/), the +#: topology the estimation and deduction ledgers already use. +_SEALED_LEDGER_PATH = ( + Path(__file__).resolve().parents[3] / "chat" / "data" / "curriculum_serve_ledger.json" +) + + +def build_ledger(cap: int = CASES_PER_BAND) -> dict[str, ClassTally]: + """Run sealed practice over the enumerated corpus → per-band ledger.""" + report = run_practice( + all_gold_problems(cap), CurriculumSolver(), CurriculumOracleTether() + ) + return dict(report.ledger) + + +def build_mix(cap: int = CASES_PER_BAND) -> dict[str, dict[str, int]]: + """Per-band GOLD verdict mix — the figure the reliability gate cannot see. + + ``ClassTally`` carries ``correct``/``wrong``/``refused`` and no verdict axis, so + a correct UNKNOWN is indistinguishable from a correct ENTAILED once tallied. + That is precisely why a band can clear θ_SERVE on non-commitments alone, and + why the mix has to be reported alongside the license rather than inferred from + it. + """ + tether = CurriculumOracleTether() + mix: dict[str, dict[str, int]] = {} + for problem in all_gold_problems(cap): + band = mix.setdefault(problem.class_name, {}) + gold = tether.gold_answer(problem) + band[gold] = band.get(gold, 0) + 1 + return mix + + +def run(ceilings: Ceilings | None = None, cap: int = CASES_PER_BAND) -> dict[str, Any]: + """Build the ledger and report the SERVE verdict + mix per band.""" + ceilings = ceilings if ceilings is not None else Ceilings.default() + ledger = build_ledger(cap) + mix = build_mix(cap) + classes: dict[str, Any] = {} + for band, tally in sorted(ledger.items()): + serve = license_for(tally, Action.SERVE, ceilings) + band_mix = mix.get(band, {}) + entailed = band_mix.get("entailed", 0) + classes[band] = { + "correct": tally.correct, + "wrong": tally.wrong, + "refused": tally.refused, + "committed": tally.committed, + "reliability": tally.reliability, + "coverage": tally.coverage, + "serve_licensed": serve.licensed, + "serve_ratio": serve.ratio, + "gold_mix": dict(sorted(band_mix.items())), + "entailed_share": ( + round(entailed / tally.committed, 6) if tally.committed else 0.0 + ), + } + return { + "lane": "curriculum-serve-practice", + "classes": classes, + "any_band_serve_licensed": any(c["serve_licensed"] for c in classes.values()), + "wrong_is_zero": all(c["wrong"] == 0 for c in classes.values()), + } + + +def build_sealed_artifact(cap: int = CASES_PER_BAND) -> dict[str, Any]: + """The sealed-ledger dict (self-verifying ``content_sha256``).""" + return seal_artifact( + build_ledger(cap), + schema="curriculum_serve_ledger_v1", + note=( + "Sealed-practice committed ledger for curriculum-grounded serving " + "(ADR-0262, sized per ADR-0264 R9). Engine reads, never writes. " + "Ceilings stay at safe defaults (theta_SERVE=0.99). One committed case " + "per DISTINCT routable query atom, so committed == distinct evidence by " + "construction. NOTE: reliability here is dominated by correct " + "non-commitments; read `gold_mix` from the runner report before " + "treating a licensed band as demonstrated curriculum competence." + ), + provenance="evals.curriculum_serve.practice.runner.seal_ledger", + ) + + +def seal_ledger( + path: Path = _SEALED_LEDGER_PATH, cap: int = CASES_PER_BAND +) -> dict[str, Any]: + """Regenerate + write the committed sealed ledger. + + Verifies the R9 distinct-evidence invariant first: a producer that padded + could never seal, rather than sealing and being caught by the audit later. + """ + assert_practice_atoms_distinct(cap) + return write_sealed_ledger(path, build_sealed_artifact(cap)) + + +def _full_sweep() -> dict[str, Any]: + """Every routable atom in every band — the exhaustive agreement measurement. + + Not the sealed corpus (that is capped at :data:`CASES_PER_BAND`); this is the + evidence that the cap is a sampling choice and not a hiding place. Runs the + whole ~70k-atom space, so it is a deliberate CLI invocation and not a test. + """ + return run(cap=10**9) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--seal", action="store_true", + help="regenerate + write chat/data/curriculum_serve_ledger.json " + "(a RATIFICATION — see the module docstring before running it)", + ) + parser.add_argument( + "--full", action="store_true", + help="report over EVERY routable atom instead of the capped corpus (slow)", + ) + args = parser.parse_args(argv) + + if args.seal: + artifact = seal_ledger() + print(f"sealed {len(artifact['classes'])} bands -> {_SEALED_LEDGER_PATH}") + return 0 + + report = _full_sweep() if args.full else run() + for band, c in report["classes"].items(): + print( + f" {band:44s} committed={c['committed']:6d} wrong={c['wrong']} " + f"refused={c['refused']:4d} reliability={c['reliability']:.5f} " + f"SERVE={str(c['serve_licensed']):5s} entailed={c['gold_mix'].get('entailed', 0):3d} " + f"({c['entailed_share']:.4%})" + ) + print( + f"any_band_serve_licensed={report['any_band_serve_licensed']} " + f"wrong_is_zero={report['wrong_is_zero']}" + ) + return 0 if report["wrong_is_zero"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_curriculum_practice.py b/tests/test_curriculum_practice.py new file mode 100644 index 00000000..3359fc9e --- /dev/null +++ b/tests/test_curriculum_practice.py @@ -0,0 +1,352 @@ +"""Phase C — the curriculum practice producer (ADR-0262 solver, ADR-0264 R9 sizing). + +Four things are pinned here, in decreasing order of how quietly they would fail: + +1. **The ledger is not committed, and a band would earn SERVE if it were.** The + producer works; that is exactly why the artifact is a ratification. Pinned so + the reasoning cannot be lost and re-derived wrongly later. +2. **`then` / `therefore` collide with the argument reader's grammar.** Two taught + `philosophy_theology` lemmas are the reader's own control words, so 164 routable + atoms refuse `compiled_premises_unreadable`. Coverage misses, never wrongs — but + in the band Phase F retargets to. +3. **`committed == distinct` by construction**, not by discipline. +4. **wrong=0** over the capped corpus, per band. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from chat.curriculum_surface import band_for, decide_curriculum_question +from core.ratified_ledger import CAPABILITY_LEDGERS, load_sealed_ledger +from core.reliability_gate import Action, Ceilings, license_for +from core.reliability_gate.evidence import audit_bands +from evals.curriculum_serve.practice.generator import ( + CASES_PER_BAND, + DuplicateAtom, + QueryAtom, + all_gold_problems, + assert_practice_atoms_distinct, + band_cases, + practice_bands, + routable_atoms, + taught_atoms, +) +from evals.curriculum_serve.practice.runner import ( + build_ledger, + build_mix, + build_sealed_artifact, + run, + seal_ledger, +) + +REPO_ROOT = Path(__file__).resolve().parents[1] +THETA_SERVE = 0.99 + +#: The 11 bands the ratified corpora populate, and each one's FULL routable atom +#: space. Pinned in both directions: a rise means new curriculum (welcome, update +#: deliberately), a fall means a band lost reachable questions. +#: +#: These supersede ADR-0264 §4.2's table, which sized bands from per-TERM +#: exclusivity. That bound is strictly tighter than the router's per-PAIR +#: predicate, so it under-counted — `systems_software_causal` reads as 630 there +#: and is really 720, which flips it from "cannot reach 657" to "can". +BAND_ATOM_SPACE: dict[str, int] = { + "curriculum_mathematics_logic_contrast": 240, + "curriculum_mathematics_logic_evidential": 240, + "curriculum_mathematics_logic_modal": 480, + "curriculum_mathematics_logic_sequence": 240, + "curriculum_philosophy_theology_contrast": 22650, + "curriculum_philosophy_theology_modal": 45300, + "curriculum_physics_causal": 720, + "curriculum_physics_modal": 480, + "curriculum_systems_software_causal": 720, + "curriculum_systems_software_modal": 480, + "curriculum_systems_software_sequence": 240, +} + +#: Bands whose space reaches the 657 a perfect record needs. Derived below, never +#: restated: these are the four that WOULD earn SERVE the moment a ledger is sealed. +_CAN_REACH_FLOOR = { + band for band, space in BAND_ATOM_SPACE.items() if space >= 657 +} + +#: The reader-grammar collision, exact. Both lemmas are taught by +#: `philosophy_theology`'s packs AND are control words of the argument reader: +#: `therefore` is literally the conclusion marker `". Therefore ."`, and `then` +#: is the conditional consequent marker. An atom using either as a term compiles an +#: argument the reader cannot parse back. +RESERVED_WORD_LEMMAS = ("then", "therefore") + + +@pytest.fixture(scope="module") +def report() -> dict: + return run() + + +@pytest.fixture(scope="module") +def ledger() -> dict: + return build_ledger() + + +@pytest.fixture(scope="module") +def mix() -> dict: + return build_mix() + + +@pytest.fixture(scope="module") +def problems() -> tuple: + return all_gold_problems() + + +# --------------------------------------------------------------------------- +# 1. The artifact is uncommitted, and that is a decision with a reason. +# --------------------------------------------------------------------------- + +def test_curriculum_ledger_is_not_committed() -> None: + """Phase C builds the writer; committing the artifact is a ratification. + + The plan of record's exit criterion asked for a ledger that is "real + (still-unearned)". That state is unreachable: every band with >=657 routable + atoms clears theta_SERVE on correct NON-COMMITMENTS alone. So a committed + ledger is necessarily an earning one, and this asserts nobody committed it as + housekeeping. + """ + path = CAPABILITY_LEDGERS["curriculum_serve"].path + assert not path.exists(), ( + f"{path.name} exists. If that was deliberate ratification, update this " + "test and ADR-0262 §5 together — a curriculum ledger licenses bands whose " + "committed evidence is ~99% non-entailed." + ) + assert CAPABILITY_LEDGERS["curriculum_serve"].missing_ok, ( + "an absent curriculum ledger must stay legitimately absent (serves disclosed)" + ) + + +def test_sealing_would_license_exactly_the_bands_that_reach_the_floor( + ledger: dict, +) -> None: + """The consequence of sealing, measured rather than asserted in prose. + + This is the finding Phase C exists to produce: the license is reachable today + and it is reachable on the UNKNOWN class. If this ever reports a different set, + the corpus moved and the ADR-0262 §5 discussion needs revisiting. + """ + ceilings = Ceilings.default() + licensed = { + band + for band, tally in ledger.items() + if license_for(tally, Action.SERVE, ceilings).licensed + } + assert licensed == _CAN_REACH_FLOOR + assert len(licensed) == 4 + + +def test_a_licensed_band_is_licensed_on_non_commitments(report: dict) -> None: + """The mix behind the license — the number `ClassTally` structurally cannot see. + + `ClassTally` carries correct/wrong/refused and no verdict axis, so a correct + UNKNOWN counts exactly like a correct ENTAILED. Every band that would earn + SERVE does so with an entailed share under 5%, which is what ADR-0262 §5.1 + rules unacceptable and what the open outcome-mix ruling has to settle. + """ + for band in sorted(_CAN_REACH_FLOOR): + entry = report["classes"][band] + assert entry["serve_licensed"] is True + assert entry["gold_mix"].get("entailed", 0) <= 9 + assert entry["entailed_share"] < 0.05, ( + f"{band}: entailed share {entry['entailed_share']:.4%} — if a band ever " + "earns SERVE on a genuine mix, that is the ADR-0262 §5.1 bar being met " + "and this pin should be revisited, not relaxed" + ) + + +def test_seal_ledger_writes_a_verifying_artifact(tmp_path: Path) -> None: + """The writer `chat/curriculum_serve_license.py` names must actually work. + + Written to a tmp path, never to `chat/data/` — the point of the test is that + the mechanism is real, not that the repository gains a ledger. + """ + path = tmp_path / "curriculum_serve_ledger.json" + artifact = seal_ledger(path) + assert path.exists() + assert artifact["provenance"] == "evals.curriculum_serve.practice.runner.seal_ledger" + assert artifact["schema"] == "curriculum_serve_ledger_v1" + # Round-trips through the same verifier the serving reader uses. + tallies = load_sealed_ledger(path) + assert set(tallies) == set(BAND_ATOM_SPACE) + assert all(t.wrong == 0 for t in tallies.values()) + + +def test_sealed_artifact_is_byte_identical_across_runs() -> None: + """Determinism (ADR-0199 L-4): no clock, no RNG, so re-sealing must not move it. + + Two independent computations are the minimum that proves this and the maximum + worth paying for — each is a full fold over the corpus. + """ + first = build_sealed_artifact() + second = build_sealed_artifact() + assert json.dumps(first, indent=2, sort_keys=True) == json.dumps( + second, indent=2, sort_keys=True + ) + assert first["content_sha256"] == second["content_sha256"] + + +# --------------------------------------------------------------------------- +# 2. The reader-grammar collision — in the band Phase F retargets to. +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("lemma", RESERVED_WORD_LEMMAS) +def test_reserved_reader_words_are_taught_curriculum_lemmas(lemma: str) -> None: + """The precondition of the collision: these really are taught terms.""" + from teaching.curriculum_premises import load_curriculum + + assert lemma in load_curriculum("philosophy_theology").vocabulary + + +@pytest.mark.parametrize("lemma", RESERVED_WORD_LEMMAS) +def test_reserved_word_atoms_refuse_rather_than_answer(lemma: str) -> None: + """A control word as a TERM refuses typed — it never answers wrongly. + + This is the honest failure: `compiled_premises_unreadable` is a coverage miss + excluded from reliability's denominator (ADR-0175 §4), not a confabulation. The + hazard is that the vocabulary boundary and the reader's grammar are not screened + against each other, so a future pack teaching "and"/"or"/"if" would open a much + larger hole silently. Recorded, not fixed — out of Phase C's scope + (DIVISION-OF-WORK §6.5). + """ + decision = decide_curriculum_question(f"Does knowledge requires {lemma}?") + assert decision.verdict == "declined" + assert decision.reason == "compiled_premises_unreadable" + + +def test_the_collision_is_bounded_to_two_lemmas() -> None: + """Every refusal in the whole 69,666-atom space traces to `then`/`therefore`. + + Pinned as a bound, so the day a third reserved word enters the curriculum this + fails instead of quietly enlarging the gap. + """ + offenders: set[str] = set() + checked = 0 + for domain, family in practice_bands(): + for atom in routable_atoms(domain, family): + reserved = {atom.subject, atom.obj} & set(RESERVED_WORD_LEMMAS) + if not reserved: + continue + checked += 1 + if decide_curriculum_question(atom.text).verdict == "declined": + offenders |= reserved + assert checked > 0, "no reserved-word atoms found — the sweep is not exercising anything" + assert offenders == set(RESERVED_WORD_LEMMAS) + + +# --------------------------------------------------------------------------- +# 3. Distinct evidence, structurally. +# --------------------------------------------------------------------------- + +def test_committed_equals_distinct_for_every_band(problems: tuple) -> None: + """ADR-0264 R9 holds by construction: case identity IS the decision key.""" + by_band: dict[str, list] = {} + for problem in problems: + by_band.setdefault(problem.class_name, []).append(problem.payload.key) + audits = audit_bands(by_band) + assert audits + for audit in audits: + assert audit.committed == audit.distinct + assert audit.inflation == 1.0 + assert audit.max_repeat == 1 + + +def test_distinctness_guard_fails_under_mutation() -> None: + """The R9 producer guard must be able to fail, or it is worthless. + + Same discipline as `tests/test_volume_honesty.py::test_audit_detects_padding`: + a pin that cannot go red proves nothing. Feeds the guard a corpus with one + atom repeated and asserts it refuses. + """ + atom = QueryAtom("physics", "causal", "force", "causes", "acceleration") + assert atom.key == ("physics", "force", "causes", "acceleration") + + import evals.curriculum_serve.practice.generator as gen + + original = gen.all_gold_problems + from core.learning_arena.protocols import Problem + + def padded(cap: int = CASES_PER_BAND) -> tuple[Problem, ...]: + return ( + Problem("a", "curriculum_physics_causal", atom), + Problem("b", "curriculum_physics_causal", atom), + ) + + gen.all_gold_problems = padded # type: ignore[assignment] + try: + with pytest.raises(DuplicateAtom, match="committed twice"): + gen.assert_practice_atoms_distinct() + finally: + gen.all_gold_problems = original # type: ignore[assignment] + + assert_practice_atoms_distinct() # the real corpus still passes + + +# --------------------------------------------------------------------------- +# 4. The corpus itself: shape, determinism, wrong=0. +# --------------------------------------------------------------------------- + +def test_band_atom_spaces_are_exactly_as_recorded() -> None: + measured = { + band_for(d, f): len(routable_atoms(d, f)) for d, f in practice_bands() + } + assert measured == BAND_ATOM_SPACE + + +def test_every_band_is_a_populated_family() -> None: + """A family with no chains is not a band with zero volume — it is not a band.""" + from teaching.curriculum_premises import load_curriculum + + for domain, family in practice_bands(): + assert load_curriculum(domain).family(family) + assert len(practice_bands()) == len(BAND_ATOM_SPACE) + + +def test_cap_selects_taught_edges_first() -> None: + """Taught edges are the only committed-POSITIVE evidence; a sample that + dropped them would report reliability while exercising no curriculum.""" + for domain, family in practice_bands(): + space = len(routable_atoms(domain, family)) + cases = band_cases(domain, family) + assert len(cases) == min(space, CASES_PER_BAND) + taught = taught_atoms(domain, family) + assert taught <= {c.key for c in cases}, f"{domain}/{family} dropped a taught edge" + + +def test_corpus_is_deterministic(problems: tuple) -> None: + assert [p.problem_id for p in all_gold_problems()] == [p.problem_id for p in problems] + + +def test_wrong_is_zero_for_every_band(report: dict) -> None: + """The serving path and the independent oracle agree on every committed atom.""" + assert report["wrong_is_zero"] is True + for band, entry in report["classes"].items(): + assert entry["wrong"] == 0, f"{band}: {entry}" + assert set(report["classes"]) == set(BAND_ATOM_SPACE) + + +def test_gold_mix_has_no_refuted_class(report: dict) -> None: + """No corpus row can express a negative yet (ADR-0264 R1-R4 unimplemented), so + `refuted` is unreachable and the mix is entailed/unknown only. Pinned so the + polarity unit has a red test to turn green.""" + for band, entry in report["classes"].items(): + assert "refuted" not in entry["gold_mix"], f"{band} produced refuted — polarity landed?" + assert set(entry["gold_mix"]) <= {"entailed", "unknown", "declined"} + + +def test_mix_totals_reconcile_with_the_ledger( + ledger: dict, report: dict, mix: dict +) -> None: + """The mix is counted over the same corpus the ledger tallies, not a second one.""" + for band, tally in ledger.items(): + assert sum(mix[band].values()) == tally.attempted + assert report["classes"][band]["committed"] == tally.committed diff --git a/tests/test_volume_honesty.py b/tests/test_volume_honesty.py index a85bd61a..b76e95bc 100644 --- a/tests/test_volume_honesty.py +++ b/tests/test_volume_honesty.py @@ -25,9 +25,11 @@ Three producers back the three entries in `core.ratified_ledger.CAPABILITY_LEDGE distinct evidence; the worst three inflate 28 distinct cases to 720 committed. Recorded, pinned, and NOT silently repaired: the ledger is SHA-sealed, ratified, and gating a live flag, so changing it is Shay's ratification, not a test's. -- **curriculum_serve** (ADR-0262/0264) — producer unbuilt. The registry pin below - fails the moment it is built without an audit source, which is the forcing - function that stops Phase C from repeating the deduction pattern. +- **curriculum_serve** (ADR-0262/0264) — clean, and clean *structurally*: the + producer's case identity IS the query atom, so ``committed == distinct`` cannot + drift apart the way it did for deduction, where case identity was a quota index. + Its ledger is deliberately uncommitted — see + ``evals/curriculum_serve/practice/runner.py``. Deliberately narrow. This pins evidence *distinctness*. It does not pin outcome mix — whether each verdict class needs its own volume is an open ruling recorded @@ -99,13 +101,30 @@ def _estimation_keys() -> dict[str, list[Hashable]]: return out +def _curriculum_keys() -> dict[str, list[Hashable]]: + """Decision keys per band for the curriculum practice corpus (Phase C). + + The key is the query ATOM, which is also the producer's case identity — so + this audit can only ever report inflation 1.0 while the producer is correct, + and reports the real figure the moment it is not. Contrast the deduction + source above, where the key had to be recovered from case text because the + producer's identity was a quota index. + """ + from evals.curriculum_serve.practice import all_gold_problems + + out: dict[str, list[Hashable]] = {} + for problem in all_gold_problems(): + out.setdefault(problem.class_name, []).append(problem.payload.key) + return out + + #: capability -> keys provider, or ``None`` when the producer is not built. #: ``None`` is a declaration, not an omission: :func:`test_every_licensed_capability_has_an_audit_source` #: accepts it only while the capability's ledger is genuinely absent. AUDIT_SOURCES: dict[str, Callable[[], dict[str, list[Hashable]]] | None] = { "estimation": _estimation_keys, "deduction_serve": _deduction_keys, - "curriculum_serve": None, # Phase C — see module docstring + "curriculum_serve": _curriculum_keys, } @@ -238,9 +257,10 @@ def test_audit_rejects_impossible_counts() -> None: def test_every_licensed_capability_has_an_audit_source() -> None: """Derived from `CAPABILITY_LEDGERS`, so a new capability cannot slip past. - This is the forcing function for Phase C: the moment - `chat/data/curriculum_serve_ledger.json` exists, a ``None`` audit source - stops being an honest declaration and this fails. + This was the forcing function for Phase C, and it worked: a ``None`` audit + source stops being an honest declaration the moment the capability's ledger + exists. All three producers now carry one, so the guard is holding the line + for the NEXT capability rather than for curriculum. """ assert set(AUDIT_SOURCES) == set(CAPABILITY_LEDGERS), ( "AUDIT_SOURCES and CAPABILITY_LEDGERS disagree — a licensed capability is " @@ -294,6 +314,31 @@ def test_estimation_producer_has_no_inflation( assert below_floor(estimation_audits, THETA_SERVE) == () +# --------------------------------------------------------------------------- +# 3b. The curriculum producer — clean, and clean by construction (Phase C). +# --------------------------------------------------------------------------- + +def test_curriculum_producer_has_no_inflation() -> None: + """The second producer that satisfies the invariant, and the first that + satisfies it STRUCTURALLY. + + Its case identity is the query atom, so ``committed == distinct`` is not a + property the generator has to maintain — it is the same fact twice. Pinned here + beside the estimation standard because this file is where a reader looks to ask + "which producers are honest about volume?". + """ + audits = audit_bands(_curriculum_keys()) + assert audits, "curriculum producer yielded no bands" + assert len(audits) == 11, f"expected 11 populated bands, found {len(audits)}" + for audit in audits: + assert audit.committed == audit.distinct, format_report(audits, THETA_SERVE) + assert audit.inflation == 1.0 + assert audit.max_repeat == 1 + assert {a.band for a in below_floor(audits, THETA_SERVE)} == { + a.band for a in audits if a.distinct < SERVE_VOLUME_AT_THETA_099 + }, "a band below the floor must be below it for lack of VOLUME, never inflation" + + # --------------------------------------------------------------------------- # 4. The deduction producer — the measured exposure, pinned in both directions. # --------------------------------------------------------------------------- From 0a78545e3d3ba2d496690ea5eff2eb92d70f1a24 Mon Sep 17 00:00:00 2001 From: Shay Date: Sun, 26 Jul 2026 10:20:56 -0700 Subject: [PATCH 2/3] =?UTF-8?q?feat(teaching):=20core=20proposal-queue=20r?= =?UTF-8?q?eseal=20=E2=80=94=20the=20ledger=5Freseal=20stage=20(Phase=20D)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `RatificationReceipt.pending_stages` has named `ledger_reseal` since the ratification ceremony was written and nothing performed it, so a sealed ledger went stale the moment curriculum changed. This is that stage, plus the CLI verb a human runs, and it is deliberately NOT reachable from `ratify_chain` (doctrine forbids ratification automation, and a per-row reseal would churn the ledger's content_sha256). The load-bearing property is not "the ledger can be rewritten" — it is that rewriting it cannot grant a serving license by accident. Reliability is commitment precision, so a curriculum band clears theta_SERVE on correct NON-COMMITMENTS alone (Phase C, PR #121). A plain regenerate verb would flip four bands from disclosed to authoritative as a side effect of housekeeping, on evidence that is ~99% non-entailed, which ADR-0262 §5.1 rules unacceptable. So: - `plan_reseal` computes the license DELTA before writing anything, reading the current ledger through `load_capability_ledger` — the production entry point — so a tampered ledger refuses here instead of being overwritten and losing the evidence, and the stage inherits the capability's DECLARED absence policy rather than choosing its own (bridge rule 5). - `apply_reseal` REFUSES by default when any band would become newly licensed, naming the bands and their `entailed` counts. `--allow-new- licenses` is the ratification. Revocations are never gated: losing a license is the gate becoming more conservative and needs no authorization. - `deduction_serve` is deliberately absent from `_RESEALABLE`. Its ledger is SHA-sealed, ratified, and gating a live flag, and 21 of its 25 bands do not clear theta_SERVE on distinct evidence. A verb that regenerated it would erase a measured exposure as housekeeping. - `ratify`'s output now names the command that performs its pending stage. Naming a stage without naming its command is how this one stayed unperformed. Exit codes: 0 dry-run/applied, 1 refused, 2 unknown capability. [Verification]: in-worktree on canonical CPython 3.12.13 with `uv sync --locked`: smoke 571 (unchanged), deductive 327 (310 + 17). `chat/data/curriculum_serve_ledger.json` still absent after the full suite — no test writes it. Caught and fixed one real gate failure: `test_no_production_adapter_passes_missing_ok` correctly rejected an explicit `missing_ok=True`; the fix routes through the manifest instead. --- core/cli_proposal_queue.py | 107 +++++++++++- core/cli_test.py | 1 + teaching/ledger_reseal.py | 195 +++++++++++++++++++++ tests/test_ledger_reseal.py | 329 ++++++++++++++++++++++++++++++++++++ 4 files changed, 631 insertions(+), 1 deletion(-) create mode 100644 teaching/ledger_reseal.py create mode 100644 tests/test_ledger_reseal.py diff --git a/core/cli_proposal_queue.py b/core/cli_proposal_queue.py index 509d3cba..daf118c2 100644 --- a/core/cli_proposal_queue.py +++ b/core/cli_proposal_queue.py @@ -138,6 +138,84 @@ def cmd_proposal_queue_ratify(args: argparse.Namespace) -> int: f"{receipt.family_chains_before} -> {receipt.family_chains_after}") if not args.dry_run: print(f" still pending : {', '.join(receipt.pending_stages)}") + # Wiring the receipt's `ledger_reseal` stage to the verb that performs + # it. Naming a pending stage without naming its command is how this one + # stayed unperformed since the ceremony was written. + if "ledger_reseal" in receipt.pending_stages: + print( + " next : core proposal-queue reseal curriculum_serve " + "--dry-run (the ledger is stale until you do)" + ) + return 0 + + +def cmd_proposal_queue_reseal(args: argparse.Namespace) -> int: + """The ``ledger_reseal`` stage of the ceremony — explicit, never automatic. + + ``ratify_chain`` grows the corpus and names this stage in its receipt's + ``pending_stages``; nothing performed it until now, so a sealed ledger went + stale the moment curriculum changed. This is the verb a human runs. + + It refuses by default to grant a license. A band can clear θ_SERVE on correct + NON-COMMITMENTS alone, so "regenerate the artifact" and "make four bands + authoritative" would otherwise be the same keystroke. + """ + from teaching.ledger_reseal import ReadyToReseal, apply_reseal, plan_reseal + + try: + plan = plan_reseal(args.capability) + except ReadyToReseal as exc: + print(f"REFUSED: {exc}") + return 2 + + payload: dict[str, object] = { + "capability": plan.capability, + "path": str(plan.path), + "licensed_before": list(plan.licensed_before), + "licensed_after": list(plan.licensed_after), + "newly_licensed": list(plan.newly_licensed), + "revoked": list(plan.revoked), + "committed": plan.committed, + "mix": plan.mix, + "is_housekeeping": plan.is_housekeeping, + "written": False, + } + if not args.json: + print(f"reseal plan for {plan.capability}:") + print(f" artifact : {plan.path}") + for band in sorted(plan.committed): + mix = ", ".join(f"{k}={v}" for k, v in sorted(plan.mix.get(band, {}).items())) + lic = "SERVE" if band in plan.licensed_after else " " + print(f" {lic} {band:44s} committed={plan.committed[band]:6d} {mix}") + print(f" licensed now : {len(plan.licensed_before)}") + print(f" licensed after: {len(plan.licensed_after)}") + if plan.newly_licensed: + print(f" NEWLY LICENSED: {', '.join(plan.newly_licensed)}") + if plan.revoked: + print(f" revoked : {', '.join(plan.revoked)}") + + if args.dry_run: + if args.json: + print(json.dumps(payload, indent=2, sort_keys=True)) + else: + print(" (dry run — nothing written)") + return 0 + + try: + path = apply_reseal(plan, allow_new_licenses=args.allow_new_licenses) + except ReadyToReseal as exc: + if args.json: + payload["refused"] = str(exc) + print(json.dumps(payload, indent=2, sort_keys=True)) + else: + print(f"REFUSED: {exc}") + return 1 + + if args.json: + payload["written"] = True + print(json.dumps(payload, indent=2, sort_keys=True)) + else: + print(f" SEALED -> {path}") return 0 @@ -195,7 +273,7 @@ def register(subparsers: argparse._SubParsersAction) -> None: "curriculum loader admits the chain -> receipt. Refuses, and rolls " "back, if the appended row would be silently dropped. Does not " "write a ledger (bridge rule 1) or queue an arena entry; the " - "receipt names those stages." + "receipt names those stages, and `reseal` performs the first." ), ) ratify.add_argument("domain") @@ -210,5 +288,32 @@ def register(subparsers: argparse._SubParsersAction) -> None: ratify.add_argument("--json", action="store_true") ratify.set_defaults(func=cmd_proposal_queue_ratify) + from teaching.ledger_reseal import resealable_capabilities + + reseal = sub.add_parser( + "reseal", + help="re-run sealed practice and rewrite a capability's committed ledger", + description=( + "The `ledger_reseal` stage `ratify_chain` names but never performs. " + "Ratifying a chain grows the corpus without re-running practice, so " + "the ledger the serving gate reads goes stale. This rebuilds it from " + "sealed practice. REFUSES by default if the reseal would newly " + "license a band: reliability is commitment precision, so a band can " + "clear theta_SERVE on correct NON-COMMITMENTS alone, and granting a " + "serving license is a ratification rather than housekeeping." + ), + ) + reseal.add_argument("capability", choices=resealable_capabilities()) + reseal.add_argument( + "--dry-run", action="store_true", + help="report the license delta without writing", + ) + reseal.add_argument( + "--allow-new-licenses", action="store_true", + help="authorize bands that would become newly licensed (a RATIFICATION)", + ) + reseal.add_argument("--json", action="store_true") + reseal.set_defaults(func=cmd_proposal_queue_reseal) + __all__ = ["register"] diff --git a/core/cli_test.py b/core/cli_test.py index f43e5be2..c88d1b17 100644 --- a/core/cli_test.py +++ b/core/cli_test.py @@ -230,6 +230,7 @@ TEST_SUITES: dict[str, tuple[str, ...]] = { "tests/test_deduction_serve_e2e.py", "tests/test_curriculum_serve.py", "tests/test_curriculum_practice.py", + "tests/test_ledger_reseal.py", "tests/test_ratified_ledger_bridge.py", "tests/test_vocab_trigger_instrument.py", ), diff --git a/teaching/ledger_reseal.py b/teaching/ledger_reseal.py new file mode 100644 index 00000000..eaea14bb --- /dev/null +++ b/teaching/ledger_reseal.py @@ -0,0 +1,195 @@ +"""teaching/ledger_reseal.py — the ``ledger_reseal`` stage of the ceremony. + +``RatificationReceipt.pending_stages`` has named ``ledger_reseal`` since the +ratification ceremony was written, and nothing performed it. Ratifying a chain +grows the corpus; it does not re-run practice, so the sealed ledger the serving +gate reads goes stale the moment curriculum changes. This module is that stage, +and it is deliberately **not** reachable from ``ratify_chain``. + +Why it is a separate, human-run act +----------------------------------- +``AGENTS.md`` forbids ratification automation, and an auto-reseal on every +``ratify`` would churn the ledger's ``content_sha256`` once per row. But the +stronger reason is what Phase C measured: **resealing can grant licenses.** +Reliability is commitment precision and a correct UNKNOWN is a commitment, so +any band with ≥657 routable atoms clears θ_SERVE on non-commitments alone. A +plain "regenerate the artifact" verb would therefore flip four curriculum bands +from disclosed to authoritative as a side effect of housekeeping, on evidence +that is ~99% non-entailed — which ADR-0262 §5.1 rules unacceptable +(``docs/research/curriculum-practice-producer-2026-07-26.md`` §1). + +So :func:`plan_reseal` computes the license DELTA before writing anything, and +the caller must opt in explicitly to any band that would become newly licensed. +The default is refusal. Regenerating an artifact is housekeeping; changing what +the engine will assert authoritatively is a ratification, and the two must not +share a keystroke. + +Scope: this module writes a ledger and nothing else. It has no opinion on +whether curriculum content is true, cannot ratify a chain, and cannot flip a +serving flag. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable + +from core.ratified_ledger import ( + CAPABILITY_LEDGERS, + load_capability_ledger, + write_sealed_ledger, +) +from core.reliability_gate import Action, Ceilings, ClassTally, license_for + + +class ReadyToReseal(ValueError): + """A reseal was requested that cannot proceed as asked.""" + + +@dataclass(frozen=True, slots=True) +class ResealPlan: + """What a reseal would do, computed before it does it.""" + + capability: str + path: Path + #: Bands licensed by the ledger currently on disk (empty when absent). + licensed_before: tuple[str, ...] + #: Bands the freshly-built artifact would license. + licensed_after: tuple[str, ...] + #: Per-band committed volume in the new artifact. + committed: dict[str, int] + #: Per-band gold verdict mix — the axis ``ClassTally`` cannot carry. + mix: dict[str, dict[str, int]] + artifact: dict[str, Any] + + @property + def newly_licensed(self) -> tuple[str, ...]: + """Bands this reseal would grant SERVE that do not hold it today. + + The figure that decides whether the operation is housekeeping or a + ratification. + """ + before = set(self.licensed_before) + return tuple(b for b in self.licensed_after if b not in before) + + @property + def revoked(self) -> tuple[str, ...]: + """Bands that would LOSE a license — always allowed, never gated. + + Losing a license is the gate becoming more conservative, which needs no + authorization. Only granting one does. + """ + after = set(self.licensed_after) + return tuple(b for b in self.licensed_before if b not in after) + + @property + def is_housekeeping(self) -> bool: + return not self.newly_licensed + + +#: capability -> (build artifact, build ledger, build mix). +#: +#: ``deduction_serve`` is deliberately ABSENT. Its ledger is SHA-sealed, +#: ratified, and gating a live flag, and 21 of its 25 bands do not clear +#: θ_SERVE on distinct evidence (``tests/test_volume_honesty.py``). Re-sealing it +#: is Shay's ratification, not a CLI verb's, and a reseal that silently +#: regenerated it would erase the exposure this arc measured. ``estimation`` +#: ships sealed with its gate and has no reason to move. +_RESEALABLE: dict[str, tuple[Callable[[], dict[str, Any]], Callable[[], dict[str, ClassTally]], Callable[[], dict[str, dict[str, int]]]]] = {} + + +def _register_curriculum() -> None: + from evals.curriculum_serve.practice.runner import ( + build_ledger, + build_mix, + build_sealed_artifact, + ) + + _RESEALABLE["curriculum_serve"] = (build_sealed_artifact, build_ledger, build_mix) + + +def resealable_capabilities() -> tuple[str, ...]: + """Capabilities this stage will rebuild, in a stable order.""" + if not _RESEALABLE: + _register_curriculum() + return tuple(sorted(_RESEALABLE)) + + +def _licensed_bands(ledger: dict[str, ClassTally], ceilings: Ceilings) -> tuple[str, ...]: + return tuple( + sorted( + band + for band, tally in ledger.items() + if license_for(tally, Action.SERVE, ceilings).licensed + ) + ) + + +def plan_reseal(capability: str, *, ceilings: Ceilings | None = None) -> ResealPlan: + """Compute the reseal and its license delta — writing nothing. + + The read of the CURRENT ledger goes through :func:`load_capability_ledger`, + the same production entry point the serving adapters use, so two properties + come for free. A hand-edited ledger refuses here rather than being silently + overwritten (losing the evidence that it was tampered with), and this stage + inherits the capability's DECLARED absence policy instead of choosing its own + — bridge rule 5: ``missing_ok`` is a fact about the capability, not about the + call, and no production path may pass it. + """ + if capability not in resealable_capabilities(): + raise ReadyToReseal( + f"{capability!r} is not resealable — known: {list(resealable_capabilities())}. " + "Ledgers absent from that list are sealed by deliberate ratification only." + ) + spec = CAPABILITY_LEDGERS[capability] + ceilings = ceilings if ceilings is not None else Ceilings.default() + build_artifact, build_ledger, build_mix = _RESEALABLE[capability] + + current = load_capability_ledger(capability) + fresh_ledger = build_ledger() + artifact = build_artifact() + + return ResealPlan( + capability=capability, + path=spec.path, + licensed_before=_licensed_bands(current, ceilings), + licensed_after=_licensed_bands(fresh_ledger, ceilings), + committed={band: t.committed for band, t in sorted(fresh_ledger.items())}, + mix=build_mix(), + artifact=artifact, + ) + + +def apply_reseal(plan: ResealPlan, *, allow_new_licenses: bool = False) -> Path: + """Write the planned artifact, refusing to grant a license by accident. + + ``allow_new_licenses`` is the ratification. Without it, a plan that would + newly license any band refuses and names them — the operator then either + authorizes the grant or fixes the producer, but never discovers afterwards + that a serving surface became authoritative. + """ + if plan.newly_licensed and not allow_new_licenses: + detail = ", ".join( + f"{band} (committed={plan.committed.get(band, 0)}, " + f"entailed={plan.mix.get(band, {}).get('entailed', 0)})" + for band in plan.newly_licensed + ) + raise ReadyToReseal( + f"{plan.capability}: this reseal would newly license {len(plan.newly_licensed)} " + f"band(s) — {detail}. That is a ratification, not a reseal: those bands " + "would begin serving authoritatively. Re-run with --allow-new-licenses " + "only if the grant is intended, and read the `entailed` counts first — a " + "band can clear theta_SERVE on correct NON-COMMITMENTS alone (ADR-0262 §5.1)." + ) + write_sealed_ledger(plan.path, plan.artifact) + return plan.path + + +__all__ = [ + "ReadyToReseal", + "ResealPlan", + "apply_reseal", + "plan_reseal", + "resealable_capabilities", +] diff --git a/tests/test_ledger_reseal.py b/tests/test_ledger_reseal.py new file mode 100644 index 00000000..0d1cd662 --- /dev/null +++ b/tests/test_ledger_reseal.py @@ -0,0 +1,329 @@ +"""Phase D — the ``ledger_reseal`` ceremony stage and its CLI verb. + +`RatificationReceipt.pending_stages` has named `ledger_reseal` since the +ratification ceremony was written and nothing performed it. This pins the verb +that performs it, and — more importantly — the reason it refuses by default. + +The load-bearing property is **not** "the ledger can be rewritten". It is that +rewriting it cannot grant a serving license by accident. Reliability is +commitment precision, so a curriculum band clears θ_SERVE on correct +NON-COMMITMENTS alone; without the gate below, `reseal` would be a housekeeping +verb that silently made four bands authoritative. +""" + +from __future__ import annotations + +import dataclasses +import json +from pathlib import Path + +import pytest + +from core.ratified_ledger import CAPABILITY_LEDGERS, LedgerSpec, load_sealed_ledger +from core.reliability_gate import ClassTally +from teaching.ledger_reseal import ( + ReadyToReseal, + ResealPlan, + apply_reseal, + plan_reseal, + resealable_capabilities, +) + + +@pytest.fixture(scope="module") +def plan() -> ResealPlan: + return plan_reseal("curriculum_serve") + + +# --------------------------------------------------------------------------- +# 1. What is resealable, and what must never be. +# --------------------------------------------------------------------------- + +def test_only_curriculum_is_resealable() -> None: + """`deduction_serve` must not be reachable from a CLI verb. + + Its ledger is SHA-sealed, ratified, and gating a live flag, and 21 of its 25 + bands do not clear θ_SERVE on distinct evidence + (`tests/test_volume_honesty.py`). A reseal verb that regenerated it would + erase a measured exposure as a side effect of housekeeping. `estimation` + ships sealed with its own gate and has no reason to move. + """ + assert resealable_capabilities() == ("curriculum_serve",) + assert "deduction_serve" not in resealable_capabilities() + assert "estimation" not in resealable_capabilities() + # ...and both are still registered capabilities, so this is an exclusion by + # decision rather than by them being unknown. + assert {"deduction_serve", "estimation"} <= set(CAPABILITY_LEDGERS) + + +def test_unregistered_capability_refuses() -> None: + with pytest.raises(ReadyToReseal, match="not resealable"): + plan_reseal("deduction_serve") + + +# --------------------------------------------------------------------------- +# 2. Planning writes nothing, and reports the license delta. +# --------------------------------------------------------------------------- + +def test_planning_writes_nothing(plan: ResealPlan) -> None: + """A plan is a read. The artifact must still be absent afterwards.""" + assert not plan.path.exists(), ( + "planning a reseal must not create the ledger — see " + "tests/test_curriculum_practice.py::test_curriculum_ledger_is_not_committed" + ) + assert plan.capability == "curriculum_serve" + assert plan.artifact["schema"] == "curriculum_serve_ledger_v1" + + +def test_plan_reports_the_bands_a_reseal_would_newly_license(plan: ResealPlan) -> None: + """The whole point of the stage being a separate act. + + Nothing is licensed today (the ledger is absent), and a reseal would license + four bands — every band whose routable atom space reaches 657. Those are the + numbers `docs/research/curriculum-practice-producer-2026-07-26.md` §1 records. + """ + assert plan.licensed_before == () + assert len(plan.licensed_after) == 4 + assert set(plan.newly_licensed) == set(plan.licensed_after) + assert plan.revoked == () + assert plan.is_housekeeping is False + + +def test_newly_licensed_bands_are_licensed_on_non_commitments(plan: ResealPlan) -> None: + """The mix the operator must see before authorizing — `ClassTally` has no + verdict axis, so the gate itself cannot distinguish these from real + competence.""" + for band in plan.newly_licensed: + mix = plan.mix[band] + assert mix.get("entailed", 0) <= 9 + assert mix.get("unknown", 0) > 600 + assert plan.committed[band] >= 657 + + +# --------------------------------------------------------------------------- +# 3. Applying refuses to grant a license by accident. This is the unit. +# --------------------------------------------------------------------------- + +def test_apply_refuses_to_grant_a_license_and_writes_nothing( + plan: ResealPlan, tmp_path: Path +) -> None: + target = tmp_path / "curriculum_serve_ledger.json" + redirected = dataclasses.replace(plan, path=target) + with pytest.raises(ReadyToReseal, match="would newly license 4 band"): + apply_reseal(redirected) + assert not target.exists(), "a refused reseal must not have written" + + +def test_refusal_names_the_bands_and_their_entailed_counts( + plan: ResealPlan, tmp_path: Path +) -> None: + """A refusal an operator cannot act on is just an obstacle.""" + with pytest.raises(ReadyToReseal) as exc: + apply_reseal(dataclasses.replace(plan, path=tmp_path / "l.json")) + message = str(exc.value) + for band in plan.newly_licensed: + assert band in message + assert "entailed=" in message + assert "--allow-new-licenses" in message + assert "NON-COMMITMENTS" in message + + +def test_apply_with_explicit_authorization_writes_a_verifying_artifact( + plan: ResealPlan, tmp_path: Path +) -> None: + """The grant path works — into tmp, never into `chat/data/`.""" + target = tmp_path / "curriculum_serve_ledger.json" + written = apply_reseal( + dataclasses.replace(plan, path=target), allow_new_licenses=True + ) + assert written == target + tallies = load_sealed_ledger(target) + assert len(tallies) == 11 + assert all(t.wrong == 0 for t in tallies.values()) + artifact = json.loads(target.read_text(encoding="utf-8")) + assert artifact["provenance"] == "evals.curriculum_serve.practice.runner.seal_ledger" + + +def test_a_reseal_that_only_revokes_needs_no_authorization(tmp_path: Path) -> None: + """Losing a license is the gate becoming MORE conservative. + + Only granting is gated. A producer change that drops a band below the floor + must be applicable without an authorization flag, or the safe direction + becomes the hard one. + """ + plan = ResealPlan( + capability="curriculum_serve", + path=tmp_path / "l.json", + licensed_before=("band_a", "band_b"), + licensed_after=("band_a",), + committed={"band_a": 700, "band_b": 3}, + mix={"band_a": {"entailed": 700}, "band_b": {"entailed": 3}}, + artifact={"schema": "x", "classes": {}, "content_sha256": "y"}, + ) + assert plan.newly_licensed == () + assert plan.revoked == ("band_b",) + assert plan.is_housekeeping is True + assert apply_reseal(plan) == plan.path # no flag needed + assert plan.path.exists() + + +def test_a_pure_housekeeping_reseal_is_allowed(tmp_path: Path) -> None: + """Identical before/after — the ordinary case once volume is settled.""" + plan = ResealPlan( + capability="curriculum_serve", + path=tmp_path / "l.json", + licensed_before=("band_a",), + licensed_after=("band_a",), + committed={"band_a": 700}, + mix={"band_a": {"entailed": 700}}, + artifact={"schema": "x", "classes": {}, "content_sha256": "y"}, + ) + assert plan.is_housekeeping is True + apply_reseal(plan) + assert plan.path.exists() + + +# --------------------------------------------------------------------------- +# 4. A tampered ledger on disk refuses rather than being silently overwritten. +# --------------------------------------------------------------------------- + +def test_tampered_current_ledger_refuses_instead_of_being_overwritten( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Overwriting a hand-edited ledger would destroy the evidence of tampering. + + `plan_reseal` reads the current ledger through the SAME verifier the serving + path uses, so a broken `content_sha256` surfaces here. + """ + target = tmp_path / "curriculum_serve_ledger.json" + target.write_text( + json.dumps( + { + "schema": "curriculum_serve_ledger_v1", + "classes": {"curriculum_physics_causal": {"correct": 999, "wrong": 0}}, + "content_sha256": "0" * 64, + "note": "hand-edited", + "provenance": "not-sealed-practice", + }, + indent=2, + ), + encoding="utf-8", + ) + import teaching.ledger_reseal as mod + + monkeypatch.setitem( + mod.CAPABILITY_LEDGERS, + "curriculum_serve", + LedgerSpec( + capability="curriculum_serve", path=target, missing_ok=True, note="test" + ), + ) + with pytest.raises(Exception, match="content_sha256 mismatch"): + plan_reseal("curriculum_serve") + # The tampered bytes are still there to be investigated. + assert "hand-edited" in target.read_text(encoding="utf-8") + + +# --------------------------------------------------------------------------- +# 5. The CLI verb. +# --------------------------------------------------------------------------- + +def _run_cli(*argv: str) -> tuple[int, str]: + import argparse + import io + import contextlib + + from core.cli_proposal_queue import register + + parser = argparse.ArgumentParser() + register(parser.add_subparsers(dest="command", required=True)) + args = parser.parse_args(["proposal-queue", *argv]) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + code = args.func(args) + return code, buf.getvalue() + + +def test_cli_dry_run_reports_without_writing() -> None: + code, out = _run_cli("reseal", "curriculum_serve", "--dry-run") + assert code == 0 + assert "nothing written" in out + assert "NEWLY LICENSED" in out + assert not CAPABILITY_LEDGERS["curriculum_serve"].path.exists() + + +def test_cli_refuses_without_authorization() -> None: + code, out = _run_cli("reseal", "curriculum_serve") + assert code == 1 + assert "REFUSED" in out + assert not CAPABILITY_LEDGERS["curriculum_serve"].path.exists() + + +def test_cli_json_is_machine_readable() -> None: + code, out = _run_cli("reseal", "curriculum_serve", "--dry-run", "--json") + assert code == 0 + payload = json.loads(out) + assert payload["written"] is False + assert payload["is_housekeeping"] is False + assert len(payload["newly_licensed"]) == 4 + assert payload["licensed_before"] == [] + + +def test_reseal_verb_is_registered_under_proposal_queue() -> None: + """The verb has to be reachable from `core proposal-queue`, not only importable.""" + import argparse + + from core.cli_proposal_queue import register + + parser = argparse.ArgumentParser() + register(parser.add_subparsers(dest="command", required=True)) + args = parser.parse_args( + ["proposal-queue", "reseal", "curriculum_serve", "--dry-run"] + ) + assert args.proposal_queue_command == "reseal" + assert args.capability == "curriculum_serve" + assert args.dry_run is True + assert args.allow_new_licenses is False + + +# --------------------------------------------------------------------------- +# 6. The receipt now names the command that performs its pending stage. +# --------------------------------------------------------------------------- + +def test_ratify_output_names_the_reseal_command() -> None: + """Naming a pending stage without naming its command is how `ledger_reseal` + stayed unperformed since the ceremony was written. + + Asserted at SOURCE level deliberately. Exercising the branch needs a + successful non-dry-run ratification, which appends to the real committed + corpus — the test would either mutate `teaching/domain_chains/` or stub out + the very ceremony it claims to check. A source pin is the honest instrument + here, and it is stated as one rather than dressed up as a behavioural test. + """ + import inspect + + from teaching.ratification import RatificationReceipt + + import core.cli_proposal_queue as cli + + assert "ledger_reseal" in RatificationReceipt.__dataclass_fields__[ + "pending_stages" + ].default + + source = inspect.getsource(cli.cmd_proposal_queue_ratify) + assert 'if "ledger_reseal" in receipt.pending_stages' in source + assert "core proposal-queue reseal curriculum_serve" in source + + +def test_reseal_stage_is_not_reachable_from_ratify_chain() -> None: + """Doctrine: no ratification automation. `ratify_chain` must not seal.""" + import inspect + + import teaching.ratification as ratification + + source = inspect.getsource(ratification) + for forbidden in ("ledger_reseal", "seal_ledger", "write_sealed_ledger"): + assert f"import {forbidden}" not in source + assert f"{forbidden}(" not in source + tally = ClassTally("x", correct=1) + assert tally.committed == 1 # sanity: the gate module is importable here From 08f6d6802c7686b57250044097bc11c1efeef8a0 Mon Sep 17 00:00:00 2001 From: Shay Date: Sun, 26 Jul 2026 12:19:19 -0700 Subject: [PATCH 3/3] =?UTF-8?q?feat(curriculum):=20taught=20negatives=20?= =?UTF-8?q?=E2=80=94=20polarity=20is=20read=20(ADR-0264=20R1-R4,=20R8)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `polarity` was read by NOTHING. `CurriculumChain.sentence` was unconditionally affirmative, so a row authored to REFUTE an atom compiled to a premise ASSERTING it and the question came back `entailed`. The independent oracle ignored the field too — so gold agreed and wrong=0 stayed green. That is the shape of defect no gate in this repo can catch, which is why ADR-0264 settled the epistemology before any code moved. R1 — row-level `polarity`, absent => affirmative. Not an `intent` value and not a new `operator_family`: bands key on the connective-derived family, so `modal_negative` would open a fresh band at n=0 instead of adding refuted volume to the band it belongs to. An UNRECOGNIZED token is dropped at the admission boundary rather than read as affirmative — that is the one direction of error the rule exists to prevent. `as_row` OMITS the field when affirmative, so committed corpora stay byte-identical and no lane hash moves. R2 — a negative row compiles under the sentential-negation prefix the argument reader already parses, so `(¬p, therefore p)` is a tautological refutation. R3 — `atom_sentence` is polarity-independent, which IS the rule: a negative row must mint the SAME propositional atom. Demonstrated, not just asserted — `test_r3_a_different_connective_would_silently_fail_to_refute` shows a negated paraphrase returning UNKNOWN, i.e. a taught refutation that quietly fails to refute with nothing going red. R4 — one atom cannot hold both polarities; rejected at ratification, naming both polarities and the consequence. Left to serve time it would surface as INCONSISTENT_PREMISES: the band silently going dark across the whole family instead of the authoring mistake being named. Duplicate-edge and contradiction stay distinguishable failures. R8 — the oracle learns polarity INDEPENDENTLY (its own constants, its own row reader; asserted at source level that it imports nothing from teaching.curriculum_premises) and EXCLUDES negatives from the reachability adjacency, because a denial supplies no step from subject to object. NO negative row is committed — authoring curriculum is Phase F and Shay's. So the shipped gold mix is still entailed 72 / unknown 4968, zero refuted, and Phase C's `test_gold_mix_has_no_refuted_class` stays green with its message retargeted from "unimplemented" to "no content yet". Also adds `--polarity` to `core proposal-queue ratify`, so the mechanism is reachable from the operator surface rather than only from Python. The end-to-end tests repoint BOTH loaders at a temp corpus instead of writing the committed one. An earlier draft appended to physics_chains_v1.jsonl and restored it, which is safe under smoke/deductive (serial) but NOT under `full`, which injects `-n auto`: a parallel worker reading mid-mutation would see an uncommitted row and the failure would look like a corpus problem rather than test isolation. Each loader is still redirected separately at its own module constant and still parses with its own code, so R8's independence is untouched. [Verification]: in-worktree on canonical CPython 3.12.13 with `uv sync --locked`: smoke 593 (571 + 22), deductive 332 (310 + 22). curriculum_serve lane unchanged at n=32 correct=32 wrong=0 anti_recall=5; all 11 lane SHA pins match; committed corpora untouched (git status clean under teaching/domain_chains/). Mutation-checked all three load-bearing rules: R2 sentence ignoring polarity -> 5 red; R8 negatives back in the adjacency -> the exclusion test red; R4 check removed -> both R4 tests red. The R8 test has an affirmative control proving the 2-hop path is real when the row is positive, so `depth == 0` cannot pass vacuously. --- core/cli_proposal_queue.py | 15 + core/cli_test.py | 7 + evals/curriculum_serve/oracle.py | 63 ++- evals/curriculum_serve/practice/generator.py | 8 +- teaching/curriculum_premises.py | 65 ++- teaching/ratification.py | 75 +++- tests/test_curriculum_polarity.py | 427 +++++++++++++++++++ tests/test_curriculum_practice.py | 17 +- 8 files changed, 656 insertions(+), 21 deletions(-) create mode 100644 tests/test_curriculum_polarity.py diff --git a/core/cli_proposal_queue.py b/core/cli_proposal_queue.py index 509d3cba..a132efb4 100644 --- a/core/cli_proposal_queue.py +++ b/core/cli_proposal_queue.py @@ -119,6 +119,7 @@ def cmd_proposal_queue_ratify(args: argparse.Namespace) -> int: reviewer=args.reviewer, rationale=args.rationale, intent=args.intent, + polarity=args.polarity, ) receipt = ratify_chain(record, dry_run=args.dry_run) except RatificationError as exc: @@ -142,6 +143,8 @@ def cmd_proposal_queue_ratify(args: argparse.Namespace) -> int: def register(subparsers: argparse._SubParsersAction) -> None: + from teaching.curriculum_premises import AFFIRMATIVE, POLARITIES + """Attach the ``core proposal-queue`` subcommand tree to a top-level parser.""" queue = subparsers.add_parser( "proposal-queue", @@ -205,6 +208,18 @@ def register(subparsers: argparse._SubParsersAction) -> None: ratify.add_argument("--reviewer", required=True, help="who ratified this") ratify.add_argument("--rationale", required=True, help="why it is ratified") ratify.add_argument("--intent", default="cause") + ratify.add_argument( + "--polarity", choices=list(POLARITIES), default=AFFIRMATIVE, + help=( + "ADR-0264 R1. `negative` ratifies an explicitly TAUGHT refutation: " + "the atom compiles under the sentential-negation prefix and the " + "question answers `refuted`, which is different from an absent edge " + "(UNKNOWN, the open-world reading). Must reuse the AFFIRMATIVE " + "connective (R3) — `causes`, never `does not cause` — or it mints a " + "different atom and silently fails to refute. Refused if the atom is " + "already taught with the other polarity (R4)." + ), + ) ratify.add_argument("--dry-run", action="store_true", help="report the band delta without writing") ratify.add_argument("--json", action="store_true") diff --git a/core/cli_test.py b/core/cli_test.py index f43e5be2..3a5bf31d 100644 --- a/core/cli_test.py +++ b/core/cli_test.py @@ -85,6 +85,12 @@ TEST_SUITES: dict[str, tuple[str, ...]] = { # an audit source. It gates a live, flag-ON serving path, so it belongs # on the pre-push gate. ~1.5s. "tests/test_volume_honesty.py", + # ADR-0264 R1-R4/R8 — taught curriculum NEGATIVES. Before this, a row + # authored to refute compiled to a premise ASSERTING the atom, and the + # independent oracle ignored polarity too, so gold agreed and wrong=0 + # stayed green. Corpus-level epistemology is unfalsifiable by every + # other gate here, so this one belongs on the pre-push gate. ~0.4s. + "tests/test_curriculum_polarity.py", ), "runtime": ( "tests/test_chat_runtime.py", @@ -230,6 +236,7 @@ TEST_SUITES: dict[str, tuple[str, ...]] = { "tests/test_deduction_serve_e2e.py", "tests/test_curriculum_serve.py", "tests/test_curriculum_practice.py", + "tests/test_curriculum_polarity.py", "tests/test_ratified_ledger_bridge.py", "tests/test_vocab_trigger_instrument.py", ), diff --git a/evals/curriculum_serve/oracle.py b/evals/curriculum_serve/oracle.py index 7feb3978..c50984eb 100644 --- a/evals/curriculum_serve/oracle.py +++ b/evals/curriculum_serve/oracle.py @@ -79,13 +79,34 @@ def _lemmas(pack_id: str) -> set[str]: return out -def _edges(domain: str) -> list[tuple[str, str, str, str]]: - """``(subject, connective, object, chain_id)`` for every ratified row.""" +#: ADR-0264 R1/R8, restated independently — the serving path has its own copy of +#: this vocabulary and the two must not share a polarity helper. Code-level +#: independence is the whole evidentiary value of this oracle: if it imported the +#: compiler's notion of polarity, agreement would only prove the compiler agrees +#: with itself, and a polarity bug would be invisible on both sides at once. +_AFFIRMATIVE = "affirmative" +_NEGATIVE = "negative" +_POLARITIES = (_AFFIRMATIVE, _NEGATIVE) + + +def _row_polarity(row: dict) -> str | None: + """This oracle's own reading of a row's polarity. + + Absent ⇒ affirmative. An unrecognized token ⇒ ``None``, and the caller drops + the row: reading an unknown value as affirmative would let a row someone + wrote to refute be scored as an assertion. + """ + value = str(row.get("polarity") or _AFFIRMATIVE).strip().lower() + return value if value in _POLARITIES else None + + +def _edges(domain: str) -> list[tuple[str, str, str, str, str]]: + """``(subject, connective, object, chain_id, polarity)`` per ratified row.""" corpora, packs = _DOMAIN_SOURCES[domain] vocabulary = set() for pack in packs: vocabulary |= _lemmas(pack) - out: list[tuple[str, str, str, str]] = [] + out: list[tuple[str, str, str, str, str]] = [] for corpus in corpora: path = _CHAIN_DIR / f"{corpus}.jsonl" if not path.exists(): @@ -105,7 +126,12 @@ def _edges(domain: str) -> list[tuple[str, str, str, str]]: continue if connective not in _FAMILY_OF_CONNECTIVE: continue - out.append((subject, connective, obj, row.get("chain_id", ""))) + polarity = _row_polarity(row) + if polarity is None: + continue + out.append( + (subject, connective, obj, row.get("chain_id", ""), polarity) + ) return out @@ -159,12 +185,28 @@ def oracle_answer(domain: str, subject: str, relation: str, obj: str) -> OracleV # curriculum teaching "entropy reveals energy" has not thereby taught # "entropy causes energy". Family scoping decides which premises are in # play; the connective decides what was actually said. - if any(s == subject and c == connective and o == obj for s, c, o, _id in edges): - return OracleVerdict("entailed", "", family, 1) + # + # ADR-0264 R1/R8 — and the POLARITY decides which way. A taught negative row + # is a taught refutation: the curriculum has said something about this atom, + # and what it said is "no". That is categorically different from an absent + # edge, which stays UNKNOWN under the open-world reading. Both are checked at + # depth 1, against the same atom, so the two cannot be confused. + for s, c, o, _id, polarity in edges: + if s == subject and c == connective and o == obj: + if polarity == _NEGATIVE: + return OracleVerdict("refuted", "", family, 1) + return OracleVerdict("entailed", "", family, 1) # No taught edge. Report the shortest path so the lane can prove the # serving path does NOT compose a chain into a claim. + # + # NEGATIVE rows are EXCLUDED from the adjacency (ADR-0264 R8). Reachability + # here exists to measure whether an untaught pair is *composable* from taught + # edges, and "a does not X b" supplies no step from a to b — treating it as + # one would report a path built out of a denial. adjacency: dict[str, list[str]] = {} - for s, _c, o, _id in edges: + for s, _c, o, _id, polarity in edges: + if polarity == _NEGATIVE: + continue adjacency.setdefault(s, []).append(o) seen = {subject} queue: deque[tuple[str, int]] = deque([(subject, 0)]) @@ -181,8 +223,11 @@ def oracle_answer(domain: str, subject: str, relation: str, obj: str) -> OracleV return OracleVerdict("unknown", "", family, 0) -def taught_edges(domain: str) -> list[tuple[str, str, str, str]]: - """Public view for the lane's provenance assertions.""" +def taught_edges(domain: str) -> list[tuple[str, str, str, str, str]]: + """Public view for the lane's provenance assertions. + + Each row carries its polarity as the fifth element (ADR-0264 R1). + """ return _edges(domain) diff --git a/evals/curriculum_serve/practice/generator.py b/evals/curriculum_serve/practice/generator.py index df7c32ca..ee124b64 100644 --- a/evals/curriculum_serve/practice/generator.py +++ b/evals/curriculum_serve/practice/generator.py @@ -124,11 +124,17 @@ def routable_atoms(domain: str, family: str) -> tuple[QueryAtom, ...]: @lru_cache(maxsize=None) def taught_atoms(domain: str, family: str) -> frozenset[tuple[str, str, str, str]]: - """The atom keys a ratified chain states directly — the ``entailed`` class. + """The atom keys a ratified chain states directly — the COMMITTED classes. Scarce by nature: a band has as many taught edges as the curriculum states, which is why ADR-0262 §5.1 makes authored volume the binding constraint on what a band can demonstrate beyond non-commitment. + + Polarity-independent on purpose (ADR-0264 R1). A negative row states its atom + just as directly as an affirmative one — it says "no" about it — so both are + scarce committed evidence and both are force-included in a capped sample. The + verdict they earn (``entailed`` vs ``refuted``) is the oracle's call, not this + function's; classifying here would duplicate the gold rule inside the corpus. """ return frozenset( QueryAtom(domain, family, c.subject, c.connective, c.obj).key diff --git a/teaching/curriculum_premises.py b/teaching/curriculum_premises.py index fbcae7af..1f351770 100644 --- a/teaching/curriculum_premises.py +++ b/teaching/curriculum_premises.py @@ -73,6 +73,21 @@ CONNECTIVE_FAMILY: dict[str, str] = { #: Relation families, in a stable order (the band-key axis — ADR-0262 §4). FAMILIES: tuple[str, ...] = ("causal", "modal", "sequence", "contrast", "evidential") +#: The two row polarities (ADR-0264 R1). ``polarity`` is a ROW-LEVEL field, not +#: an ``intent`` value and not a new ``operator_family``: bands key on +#: *(domain, connective-derived family)*, so a ``modal_negative`` family would +#: create a fresh band at n=0 instead of adding refuted volume to the band it +#: belongs to. An ABSENT field reads as affirmative, which is what makes the +#: existing corpora valid unchanged. +AFFIRMATIVE = "affirmative" +NEGATIVE = "negative" +POLARITIES: tuple[str, str] = (AFFIRMATIVE, NEGATIVE) + +#: ADR-0264 R2 — the sentential-negation prefix the argument reader already +#: parses (``generate/proof_chain/english.py``). Stated once here so the +#: compiled form cannot drift from the form the reader accepts. +NEGATION_PREFIX = "it is not the case that " + @dataclass(frozen=True, slots=True) class CurriculumChain: @@ -83,13 +98,42 @@ class CurriculumChain: connective: str obj: str family: str + #: ADR-0264 R1. Defaulted, so every existing construction site stays valid + #: and an un-migrated corpus row reads as the affirmative it always was. + polarity: str = AFFIRMATIVE + + @property + def negated(self) -> bool: + return self.polarity == NEGATIVE + + @property + def atom_sentence(self) -> str: + """The affirmative core, independent of polarity — the ATOM this chain + is about. + + ADR-0264 R3 in one property: a negative row must mint the *same* + propositional atom as its affirmative counterpart, or it cannot refute + anything. A negative row that reworded the relation would produce a + second, unrelated atom and the query would come back UNKNOWN instead of + REFUTED — a taught refutation silently failing to refute, which is the + worst available outcome here because nothing goes red. + """ + return f"{self.subject} {self.connective} {self.obj}" @property def sentence(self) -> str: - """The English premise this chain compiles to. The connective is - already a third-person-singular verb form, so the sentence lands in - the verb-predicate grammar (ADR-0260) exactly as written.""" - return f"{self.subject} {self.connective} {self.obj}" + """The English premise this chain compiles to. + + The connective is already a third-person-singular verb form, so the + affirmative sentence lands in the verb-predicate grammar (ADR-0260) + exactly as written. A negative row is the same sentence under the + sentential-negation prefix (ADR-0264 R2), which the verb band reads as + a negation OF THAT ATOM — so `(¬p, therefore p)` is a tautological + refutation and the query decides REFUTED. + """ + if self.negated: + return f"{NEGATION_PREFIX}{self.atom_sentence}" + return self.atom_sentence @dataclass(frozen=True, slots=True) @@ -189,6 +233,14 @@ def load_curriculum(domain: str) -> Curriculum: family = CONNECTIVE_FAMILY.get(connective) if family is None: continue + # ADR-0264 R1 — absent polarity IS affirmative. An UNRECOGNIZED + # polarity is dropped here rather than guessed: reading an + # unknown token as affirmative would turn a row someone wrote to + # refute into a row that asserts, which is the one direction of + # error this whole rule exists to prevent. + polarity = str(row.get("polarity") or AFFIRMATIVE).strip().lower() + if polarity not in POLARITIES: + continue chains.append( CurriculumChain( chain_id=str(row.get("chain_id") or ""), @@ -196,6 +248,7 @@ def load_curriculum(domain: str) -> Curriculum: connective=connective, obj=str(row.get("object") or "").strip(), family=family, + polarity=polarity, ) ) return Curriculum( @@ -275,9 +328,13 @@ def resolve_pinned(curriculum: Curriculum, chain_ids: tuple[str, ...]) -> tuple[ __all__ = [ + "AFFIRMATIVE", "CONNECTIVE_FAMILY", "FAMILIES", "MAX_PREMISE_SENTENCES", + "NEGATION_PREFIX", + "NEGATIVE", + "POLARITIES", "Curriculum", "CurriculumChain", "UnratifiedChain", diff --git a/teaching/ratification.py b/teaching/ratification.py index cebeadfa..8bc36ab1 100644 --- a/teaching/ratification.py +++ b/teaching/ratification.py @@ -57,7 +57,13 @@ from core.capability.domains import ( DOMAIN_CORPORA, DOMAIN_PACKS, ) -from teaching.curriculum_premises import CONNECTIVE_FAMILY, load_curriculum +from teaching.curriculum_premises import ( + AFFIRMATIVE, + CONNECTIVE_FAMILY, + NEGATIVE, + POLARITIES, + load_curriculum, +) _REPO_ROOT = Path(__file__).resolve().parents[1] @@ -74,6 +80,10 @@ _ROW_FIELDS: tuple[str, ...] = ( "intent", "connective", "object", + # ADR-0264 R1 — row-level polarity, emitted only for NEGATIVE rows (see + # ``ChainRecord.as_row``). An affirmative row omits it, so every committed + # corpus stays byte-identical and no existing lane hash moves. + "polarity", "subject_pack_id", "object_pack_id", "review_status", @@ -100,9 +110,34 @@ class ChainRecord: object_pack_id: str review_status: str provenance: str + #: ADR-0264 R1. Defaulted so existing construction sites are unchanged. + polarity: str = AFFIRMATIVE def as_row(self) -> dict[str, Any]: - return {name: getattr(self, name) for name in _ROW_FIELDS} + """The committed row. + + ``polarity`` is OMITTED when affirmative. An absent field already reads + as affirmative (ADR-0264 R1), so emitting it would add a redundant key + to every future row while the committed corpora lack it — two spellings + of the same fact, and a diff that looks like a schema change on rows + whose meaning did not move. + """ + row = { + name: getattr(self, name) + for name in _ROW_FIELDS + if name != "polarity" + } + if self.polarity != AFFIRMATIVE: + # Re-insert in _ROW_FIELDS order — chain corpora are reviewed as + # diffs, so key order is part of the artifact. + ordered = {} + for name in _ROW_FIELDS: + if name == "polarity": + ordered["polarity"] = self.polarity + else: + ordered[name] = row[name] + return ordered + return row def as_jsonl_line(self) -> str: """Byte-compatible with the committed corpora: compact separators, @@ -221,6 +256,7 @@ def build_chain_record( reviewer: str, rationale: str, intent: str = "cause", + polarity: str = AFFIRMATIVE, chain_id: str | None = None, ) -> ChainRecord: """Construct the record a reviewed decision implies. @@ -236,11 +272,20 @@ def build_chain_record( if not rationale: raise RatificationError("ratification requires a stated rationale") + polarity = polarity.strip().lower() + if polarity not in POLARITIES: + raise RatificationError( + f"polarity {polarity!r} is not one of {list(POLARITIES)} — the " + "curriculum loader would drop this row (ADR-0264 R1)" + ) + family = CONNECTIVE_FAMILY.get(connective.strip()) if family is None: raise RatificationError( f"connective {connective!r} is outside CONNECTIVE_FAMILY " - f"{sorted(CONNECTIVE_FAMILY)} — the loader would drop this row" + f"{sorted(CONNECTIVE_FAMILY)} — the loader would drop this row. " + "A NEGATIVE row must reuse the affirmative connective (ADR-0264 R3), " + "never a negated paraphrase of it." ) packs = DOMAIN_PACKS.get(domain, ()) @@ -273,6 +318,7 @@ def build_chain_record( object_pack_id=object_pack, review_status="reviewed", provenance=f"ratification-ceremony:{reviewer}:{rationale}", + polarity=polarity, ) @@ -298,6 +344,10 @@ def validate_admissible(record: ChainRecord) -> None: raise RatificationError( f"object {record.object!r} absent from pack {record.object_pack_id}" ) + if record.polarity not in POLARITIES: + raise RatificationError( + f"polarity {record.polarity!r} has no meaning (ADR-0264 R1)" + ) for row in existing_rows(record.domain): if str(row.get("chain_id")) == record.chain_id: raise RatificationError(f"chain_id {record.chain_id} already committed") @@ -306,6 +356,25 @@ def validate_admissible(record: ChainRecord) -> None: and str(row.get("connective")) == record.connective and str(row.get("object")) == record.object ): + # ADR-0264 R4 — a corpus stating both polarities of ONE atom is + # contradictory, and the contradiction is rejected HERE rather than + # discovered at serve time. `evaluate_entailment_with_trace` would + # meet `p ∧ ¬p` and refuse INCONSISTENT_PREMISES, which is honest + # but arrives as a band that has stopped answering anything in that + # family — the curriculum silently going dark instead of the + # authoring mistake being named. + existing_polarity = ( + str(row.get("polarity") or AFFIRMATIVE).strip().lower() + ) + if existing_polarity != record.polarity: + raise RatificationError( + f"contradiction: {row.get('chain_id')} already teaches " + f"{existing_polarity} '{record.subject} {record.connective} " + f"{record.object}'; this row would teach {record.polarity}. " + "One atom cannot hold both polarities (ADR-0264 R4) — the " + "compiled premises would be inconsistent and the band would " + "refuse every question in the family." + ) raise RatificationError( f"duplicate edge: {row.get('chain_id')} already teaches " f"{record.subject} {record.connective} {record.object}" diff --git a/tests/test_curriculum_polarity.py b/tests/test_curriculum_polarity.py new file mode 100644 index 00000000..b86523b5 --- /dev/null +++ b/tests/test_curriculum_polarity.py @@ -0,0 +1,427 @@ +"""ADR-0264 R1-R4 + R8 — explicitly-taught curriculum negatives. + +Before this, `polarity` was read by NOTHING. `CurriculumChain.sentence` was +unconditionally affirmative, so a row authored to REFUTE an atom compiled to a +premise asserting it, and the question came back `entailed`. The independent +oracle ignored the field too — so **gold agreed and `wrong=0` stayed green.** +That is the shape of defect no gate in this repository can catch, which is why +the epistemology was settled in an ADR before any code moved. + +The five rules, and what each one is defending against: + +- **R1** polarity is a ROW-LEVEL field; absent ⇒ affirmative. Not an `intent` + value, not a new `operator_family`: bands key on the connective-derived family, + so `modal_negative` would open a fresh band at n=0 instead of adding refuted + volume to the band it belongs to. +- **R2** a negative row compiles under the sentential-negation prefix the argument + reader already parses. +- **R3** a negative row MUST reuse the affirmative connective, so it mints the + SAME propositional atom. A negated paraphrase mints a different atom and the + query returns UNKNOWN — a taught refutation that silently fails to refute. +- **R4** one atom cannot hold both polarities; the contradiction is rejected at + ratification, not discovered at serve time as a band that stopped answering. +- **R8** the oracle learns polarity INDEPENDENTLY (its own constants, its own + reader) and excludes negatives from the reachability adjacency. + +The end-to-end tests write a temporary row into the committed corpus and restore +it, asserting **byte-identity** on the way out. No negative row is committed: +authoring curriculum is Phase F and Shay's. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Iterator + +import pytest + +from chat.curriculum_surface import decide_curriculum_question +from evals.curriculum_serve.oracle import oracle_answer, taught_edges +from generate.proof_chain.entail import Entailment, evaluate_entailment_with_trace +from generate.proof_chain.verb import VerbArgument, read_verb_argument +from teaching.curriculum_premises import ( + AFFIRMATIVE, + NEGATION_PREFIX, + NEGATIVE, + POLARITIES, + CurriculumChain, + compile_premises, + load_curriculum, +) +from teaching.ratification import ( + ChainRecord, + RatificationError, + build_chain_record, + validate_admissible, +) + +REPO_ROOT = Path(__file__).resolve().parents[1] +_PHYSICS_CORPUS = REPO_ROOT / "teaching" / "domain_chains" / "physics_chains_v1.jsonl" + +#: An atom physics does NOT teach, in taught vocabulary, so it routes and is +#: UNKNOWN today — the clean slot to plant a negative in. +_UNTAUGHT_ATOM = ("mass", "causes", "charge") + + +def _chain(polarity: str = AFFIRMATIVE) -> CurriculumChain: + return CurriculumChain( + chain_id="x-causal-001", + subject="force", + connective="causes", + obj="acceleration", + family="causal", + polarity=polarity, + ) + + +# --------------------------------------------------------------------------- +# R1 — the field, and its default. +# --------------------------------------------------------------------------- + +def test_r1_polarity_defaults_to_affirmative() -> None: + """An absent field must read as affirmative, or every committed row changes + meaning the moment the field exists.""" + bare = CurriculumChain("i", "force", "causes", "acceleration", "causal") + assert bare.polarity == AFFIRMATIVE + assert bare.negated is False + assert bare.sentence == "force causes acceleration" + + +def test_r1_committed_corpora_carry_no_polarity_field_and_still_load() -> None: + """The migration is a no-op on disk: nothing was rewritten.""" + rows = [ + json.loads(line) + for line in _PHYSICS_CORPUS.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + assert rows + assert not any("polarity" in row for row in rows), ( + "no committed row should carry polarity yet — Phase F authors those" + ) + for chain in load_curriculum("physics").chains: + assert chain.polarity == AFFIRMATIVE + + +def test_r1_unrecognized_polarity_is_dropped_not_guessed() -> None: + """Reading an unknown token as affirmative would turn a row someone wrote to + refute into a row that asserts — the one direction of error that matters.""" + assert set(POLARITIES) == {AFFIRMATIVE, NEGATIVE} + with pytest.raises(RatificationError, match="polarity"): + build_chain_record( + domain="physics", subject="mass", connective="causes", obj="charge", + reviewer="t", rationale="t", polarity="maybe", + ) + + +def test_r1_polarity_is_not_a_new_operator_family() -> None: + """Bands key on the connective-derived family; a negative row keeps its band.""" + from chat.curriculum_surface import band_for + + neg = _chain(NEGATIVE) + assert neg.family == "causal" + assert band_for("physics", neg.family) == "curriculum_physics_causal" + + +# --------------------------------------------------------------------------- +# R2 — the compiled form, and that the reader accepts it. +# --------------------------------------------------------------------------- + +def test_r2_negative_row_compiles_under_the_negation_prefix() -> None: + neg = _chain(NEGATIVE) + assert neg.sentence == f"{NEGATION_PREFIX}force causes acceleration" + assert neg.negated is True + + +def test_r2_the_reader_actually_parses_that_form() -> None: + """The prefix is not decorative — the verb band must read it as a negation + of the atom, or R2 compiles premises nothing can decide.""" + argument = f"{_chain(NEGATIVE).sentence}. Therefore force causes acceleration." + arg = read_verb_argument(argument) + assert isinstance(arg, VerbArgument), f"reader refused: {arg}" + trace = evaluate_entailment_with_trace(arg.premise_formulas, arg.query_formula) + assert trace.outcome is Entailment.REFUTED + assert trace.reason == "tautological_refutation" + + +def test_r2_affirmative_control_still_entails() -> None: + argument = f"{_chain().sentence}. Therefore force causes acceleration." + arg = read_verb_argument(argument) + assert isinstance(arg, VerbArgument) + trace = evaluate_entailment_with_trace(arg.premise_formulas, arg.query_formula) + assert trace.outcome is Entailment.ENTAILED + + +# --------------------------------------------------------------------------- +# R3 — the same atom. This is the silent-failure rule. +# --------------------------------------------------------------------------- + +def test_r3_negative_reuses_the_affirmative_atom() -> None: + """`atom_sentence` is polarity-independent — that IS R3.""" + assert _chain().atom_sentence == _chain(NEGATIVE).atom_sentence + assert _chain(NEGATIVE).sentence.endswith(_chain(NEGATIVE).atom_sentence) + + +def test_r3_a_different_connective_would_silently_fail_to_refute() -> None: + """Why R3 is a rule and not a style note. + + A negative row spelled with a DIFFERENT relation word mints a different + propositional atom, so it cannot contradict the query. The result is UNKNOWN + — the curriculum was taught a refutation and answers "I don't know". Nothing + goes red. This is the mechanism, demonstrated. + """ + same_atom = f"{NEGATION_PREFIX}force causes acceleration. Therefore force causes acceleration." + other_atom = f"{NEGATION_PREFIX}force enables acceleration. Therefore force causes acceleration." + + a = read_verb_argument(same_atom) + b = read_verb_argument(other_atom) + assert isinstance(a, VerbArgument) and isinstance(b, VerbArgument) + assert evaluate_entailment_with_trace(a.premise_formulas, a.query_formula).outcome is Entailment.REFUTED + assert evaluate_entailment_with_trace(b.premise_formulas, b.query_formula).outcome is Entailment.UNKNOWN, ( + "a negated paraphrase must be shown NOT to refute — that is the hazard R3 forbids" + ) + + +def test_r3_ratification_refuses_a_connective_outside_the_closed_set() -> None: + with pytest.raises(RatificationError, match="ADR-0264 R3"): + build_chain_record( + domain="physics", subject="mass", connective="does not cause", + obj="charge", reviewer="t", rationale="t", polarity=NEGATIVE, + ) + + +# --------------------------------------------------------------------------- +# R4 — contradiction rejected at ratification. +# --------------------------------------------------------------------------- + +def _record(polarity: str, subject: str = "force", obj: str = "acceleration") -> ChainRecord: + return ChainRecord( + chain_id="physics-causal-900", domain="physics", operator_family="causal", + subject=subject, intent="cause", connective="causes", object=obj, + subject_pack_id="en_physics_v1", object_pack_id="en_physics_v1", + review_status="reviewed", provenance="test", polarity=polarity, + ) + + +def test_r4_negating_an_already_taught_atom_is_rejected() -> None: + """physics-causal-001 teaches `force causes acceleration` affirmatively.""" + with pytest.raises(RatificationError, match="contradiction"): + validate_admissible(_record(NEGATIVE)) + + +def test_r4_the_refusal_names_both_polarities_and_the_consequence() -> None: + with pytest.raises(RatificationError) as exc: + validate_admissible(_record(NEGATIVE)) + message = str(exc.value) + assert AFFIRMATIVE in message and NEGATIVE in message + assert "ADR-0264 R4" in message + assert "inconsistent" in message + + +def test_r4_same_polarity_duplicate_is_still_a_duplicate_not_a_contradiction() -> None: + """The two failures are different and must stay distinguishable.""" + with pytest.raises(RatificationError, match="duplicate edge"): + validate_admissible(_record(AFFIRMATIVE)) + + +def test_r4_an_untaught_atom_admits_either_polarity() -> None: + """R4 forbids contradiction, not negatives.""" + subject, _connective, obj = _UNTAUGHT_ATOM + validate_admissible(_record(NEGATIVE, subject=subject, obj=obj)) + validate_admissible(_record(AFFIRMATIVE, subject=subject, obj=obj)) + + +# --------------------------------------------------------------------------- +# R8 — the oracle, independently. +# --------------------------------------------------------------------------- + +def test_r8_oracle_does_not_import_the_compiler_s_polarity_vocabulary() -> None: + """Code-level independence is the whole evidentiary value. + + If the oracle imported the compiler's polarity constants or helper, agreement + would only prove the compiler agrees with itself and a polarity bug would be + invisible on both sides at once. + """ + import inspect + + import evals.curriculum_serve.oracle as oracle + + source = inspect.getsource(oracle) + assert "from teaching.curriculum_premises import" not in source + assert "teaching.curriculum_premises" not in source + # It has its own constants. + assert oracle._AFFIRMATIVE == "affirmative" + assert oracle._NEGATIVE == "negative" + + +def test_r8_taught_edges_exposes_polarity() -> None: + edges = taught_edges("physics") + assert edges + for _s, _c, _o, _id, polarity in edges: + assert polarity == AFFIRMATIVE + + +# --------------------------------------------------------------------------- +# End to end, against a temporary corpus row. Restored byte-identically. +# --------------------------------------------------------------------------- + +def _plant( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, *, polarity: str, chain_id: str +) -> tuple[str, str, str]: + """Repoint BOTH loaders at a temp corpus carrying one extra row. + + The committed corpus is never written. An earlier version of this fixture + appended to `teaching/domain_chains/physics_chains_v1.jsonl` and restored it; + that is safe under `smoke`/`deductive` (serial) but NOT under `full`, which + injects `-n auto` — a parallel worker reading the corpus mid-mutation would + see a row that is not committed, and the failure would look like a corpus + problem rather than a test-isolation one. + + Each loader is still redirected SEPARATELY, at its own module constant, and + each still parses with its own code. Only the corpus LOCATION is shared; the + ADR-0264 R8 independence being tested is untouched. + """ + import evals.curriculum_serve.oracle as oracle + import teaching.curriculum_premises as premises + + subject, connective, obj = _UNTAUGHT_ATOM + row = { + "chain_id": chain_id, + "domain": "physics", + "operator_family": "causal", + "subject": subject, + "intent": "cause", + "connective": connective, + "object": obj, + "subject_pack_id": "en_physics_v1", + "object_pack_id": "en_physics_v1", + "review_status": "reviewed", + "provenance": "test:polarity", + } + if polarity != AFFIRMATIVE: + row["polarity"] = polarity + + chain_dir = tmp_path / "teaching" / "domain_chains" + chain_dir.mkdir(parents=True) + for src in (REPO_ROOT / "teaching" / "domain_chains").glob("*.jsonl"): + text = src.read_text(encoding="utf-8") + if not text.endswith("\n"): + text += "\n" + if src.name == _PHYSICS_CORPUS.name: + text += json.dumps(row, separators=(",", ":")) + "\n" + (chain_dir / src.name).write_text(text, encoding="utf-8") + # The compiler resolves corpora AND packs from its repo root, so the temp + # tree needs the real packs visible under the same name. + (tmp_path / "packs").symlink_to(REPO_ROOT / "packs") + + monkeypatch.setattr(premises, "_REPO_ROOT", tmp_path) + monkeypatch.setattr(oracle, "_CHAIN_DIR", chain_dir) + premises.load_curriculum.cache_clear() + premises._pack_lemmas.cache_clear() + monkeypatch.setattr( + premises.load_curriculum, "cache_clear", premises.load_curriculum.cache_clear + ) + return subject, connective, obj + + +@pytest.fixture +def negative_row( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> Iterator[tuple[str, str, str]]: + """A TAUGHT NEGATIVE for an atom physics does not otherwise teach.""" + atom = _plant(monkeypatch, tmp_path, polarity=NEGATIVE, chain_id="physics-causal-999") + yield atom + load_curriculum.cache_clear() + + +def test_e2e_a_taught_negative_serves_refuted(negative_row: tuple[str, str, str]) -> None: + """The whole point: `polarity` now changes the answer. + + Before this unit the same row served a confident `entailed` — a taught + refutation answering "yes". + """ + subject, connective, obj = negative_row + decision = decide_curriculum_question(f"Does {subject} {connective} {obj}?") + assert decision.verdict == "refuted", decision + assert decision.band == "curriculum_physics_causal" + + +def test_e2e_the_independent_oracle_agrees(negative_row: tuple[str, str, str]) -> None: + """Two independently-written procedures must reach the same verdict, or the + lane's `wrong=0` is measuring agreement with itself.""" + subject, connective, obj = negative_row + gold = oracle_answer("physics", subject, connective, obj) + served = decide_curriculum_question(f"Does {subject} {connective} {obj}?") + assert gold.verdict == "refuted" + assert gold.depth == 1, "a taught negative is a depth-1 statement, not a composition" + assert served.verdict == gold.verdict + + +def test_e2e_the_negative_premise_is_actually_compiled( + negative_row: tuple[str, str, str], +) -> None: + subject, connective, obj = negative_row + curriculum = load_curriculum("physics") + premises, chain_ids = compile_premises( + curriculum, "causal", query=(subject, connective, obj) + ) + assert "physics-causal-999" in chain_ids + assert any(p.startswith(NEGATION_PREFIX) for p in premises), premises + + +def test_e2e_the_atom_was_unknown_before_the_negative_row() -> None: + """Without the fixture the same question is UNKNOWN — so the test above is + measuring the row, not a pre-existing verdict.""" + subject, connective, obj = _UNTAUGHT_ATOM + load_curriculum.cache_clear() + decision = decide_curriculum_question(f"Does {subject} {connective} {obj}?") + assert decision.verdict == "unknown", decision + assert oracle_answer("physics", subject, connective, obj).verdict == "unknown" + + +def test_e2e_negatives_are_excluded_from_reachability( + negative_row: tuple[str, str, str], +) -> None: + """R8 — a denial supplies no step, demonstrated on a path it would complete. + + The fixture plants `mass does not cause charge`, and physics already teaches + `charge causes field` (physics-causal-004). So if negatives entered the + adjacency, `(mass, causes, field)` would report a 2-hop path built out of a + DENIAL. `mass` has no other outgoing causal edge, so depth must stay 0. + + Constructed this way on purpose: the assertion distinguishes the correct + behaviour from the specific wrong one, rather than merely observing UNKNOWN + (which both would produce). + """ + subject, _connective, obj = negative_row + assert (obj, "causes", "field") in { + (c.subject, c.connective, c.obj) for c in load_curriculum("physics").family("causal") + }, "fixture premise moved: physics must still teach `charge causes field`" + + verdict = oracle_answer("physics", subject, "causes", "field") + assert verdict.verdict == "unknown" + assert verdict.depth == 0, ( + f"reachability composed a {verdict.depth}-hop path through a denial " + f"({subject} -/-> {obj} -> field) — ADR-0264 R8 excludes negatives from " + "the adjacency precisely so this cannot happen" + ) + + +def test_e2e_an_affirmative_row_in_the_same_slot_WOULD_create_that_path( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The control for the test above: the path is real when the row is positive. + + Without this, `depth == 0` could just mean the oracle never finds 2-hop paths + at all, and the R8 assertion would be vacuous. Same slot, same terms, only the + polarity differs — so the two tests isolate polarity and nothing else. + """ + subject, _connective, _obj = _plant( + monkeypatch, tmp_path, polarity=AFFIRMATIVE, chain_id="physics-causal-998" + ) + verdict = oracle_answer("physics", subject, "causes", "field") + assert verdict.verdict == "unknown", "a 2-hop path must not be ENTAILED" + assert verdict.depth == 2, ( + f"expected the affirmative row to open a 2-hop path, got {verdict}" + ) + load_curriculum.cache_clear() diff --git a/tests/test_curriculum_practice.py b/tests/test_curriculum_practice.py index 3359fc9e..508b1967 100644 --- a/tests/test_curriculum_practice.py +++ b/tests/test_curriculum_practice.py @@ -335,11 +335,20 @@ def test_wrong_is_zero_for_every_band(report: dict) -> None: def test_gold_mix_has_no_refuted_class(report: dict) -> None: - """No corpus row can express a negative yet (ADR-0264 R1-R4 unimplemented), so - `refuted` is unreachable and the mix is entailed/unknown only. Pinned so the - polarity unit has a red test to turn green.""" + """`refuted` is reachable in MECHANISM but absent from CONTENT. + + ADR-0264 R1–R4/R8 are implemented, so a negative row would be compiled, + decided and scored as `refuted` — `tests/test_curriculum_polarity.py` proves + that end to end against a temporary corpus row. No negative row is COMMITTED, + because authoring curriculum is Phase F and Shay's. So the shipped mix is + still entailed/unknown only, and this stays a true statement about the corpus + rather than about the machinery. + """ for band, entry in report["classes"].items(): - assert "refuted" not in entry["gold_mix"], f"{band} produced refuted — polarity landed?" + assert "refuted" not in entry["gold_mix"], ( + f"{band} produced refuted — a negative row was committed. That is " + "Phase F content; update this pin deliberately alongside it." + ) assert set(entry["gold_mix"]) <= {"entailed", "unknown", "declined"}