diff --git a/core/cli_test.py b/core/cli_test.py index ab018585..f43e5be2 100644 --- a/core/cli_test.py +++ b/core/cli_test.py @@ -229,6 +229,7 @@ TEST_SUITES: dict[str, tuple[str, ...]] = { "tests/test_exist_argument_reader.py", "tests/test_deduction_serve_e2e.py", "tests/test_curriculum_serve.py", + "tests/test_curriculum_practice.py", "tests/test_ratified_ledger_bridge.py", "tests/test_vocab_trigger_instrument.py", ), diff --git a/docs/research/curriculum-practice-producer-2026-07-26.md b/docs/research/curriculum-practice-producer-2026-07-26.md new file mode 100644 index 00000000..c7f9d03e --- /dev/null +++ b/docs/research/curriculum-practice-producer-2026-07-26.md @@ -0,0 +1,166 @@ +# Curriculum practice producer — what building it measured (Phase C) + +**Date:** 2026-07-26 · **Arc:** curriculum-license-loop-2026-07 · **Unit:** Phase C +**Code:** `evals/curriculum_serve/practice/{generator,runner}.py`, +`tests/test_curriculum_practice.py` +**Governs:** ADR-0262 §5 (what a curriculum band can earn), ADR-0264 R9 (volume honesty) + +The producer named by `chat/curriculum_serve_license.py`'s own docstring — +`evals.curriculum_serve.practice.runner.seal_ledger`, declared "the only writer" +of the ledger that module reads — did not exist. It does now. Building it +measured three things, two of which correct records I wrote earlier in this arc. + +Reproduce everything below with: + +```bash +uv run python -m evals.curriculum_serve.practice.runner # capped corpus +uv run python -m evals.curriculum_serve.practice.runner --full # every atom (~75s) +``` + +--- + +## 0. What the producer is + +A band's case space is **enumerated, not authored**: every exam question that +routes to that subject (`chat.curriculum_surface.resolve_domain`) whose relation +falls in that family. One committed case per distinct query atom, so +`committed == distinct` is not a property the generator maintains — it is the same +fact twice. This is the structural version of what ADR-0264 R9 asks for, and it is +why the curriculum producer cannot develop the deduction producer's exposure +(`CASES_PER_BAND = 720` filled by cycling spaces as small as 28). + +`CASES_PER_BAND = 660` follows `evals/determination_estimation`, per +`DIVISION-OF-WORK.md` §4. It is a **ceiling on committed volume, not a quota**: a +band with a smaller space emits its whole space and reports a number below the +floor. Selection above the cap is every taught edge plus a fixed stride across the +rest — a lexicographic prefix of 660 out of 45,300 would cover about five subject +terms of 151 and report one corner of a band as the band. + +Result over the capped corpus: **11 bands, wrong = 0, inflation 1.0 everywhere.** + +--- + +## 1. Finding — a committed ledger is necessarily an *earning* ledger + +The plan of record's Phase-1 exit criterion asks that +`curriculum_serve_license()` read "a real (still-unearned) ledger instead of an +absent one." + +**That state is unreachable.** Reliability is commitment precision, and a correct +UNKNOWN is a commitment. `conservative_floor(660, 660) = 0.990046 ≥ 0.99`, so any +band with ≥657 routable atoms clears θ_SERVE on non-commitments alone, whatever its +curriculum says. Sealing today licenses **four** bands: + +| band | committed | refused | reliability | SERVE | entailed | entailed share | +|---|---:|---:|---:|:---:|---:|---:| +| `curriculum_philosophy_theology_modal` | 45,188 | 112 | 0.99985 | **yes** | 8 | 0.0177% | +| `curriculum_philosophy_theology_contrast` | 22,598 | 52 | 0.99971 | **yes** | 8 | 0.0354% | +| `curriculum_physics_causal` | 720 | 0 | 0.99087 | **yes** | 7 | 0.9722% | +| `curriculum_systems_software_causal` | 720 | 0 | 0.99087 | **yes** | 7 | 0.9722% | +| `curriculum_physics_modal` | 480 | 0 | 0.98636 | no | 9 | 1.8750% | +| `curriculum_mathematics_logic_modal` | 480 | 0 | 0.98636 | no | 4 | 0.8333% | +| `curriculum_systems_software_modal` | 480 | 0 | 0.98636 | no | 6 | 1.2500% | +| `curriculum_mathematics_logic_contrast` | 240 | 0 | 0.97309 | no | 8 | 3.3333% | +| `curriculum_mathematics_logic_evidential` | 240 | 0 | 0.97309 | no | 8 | 3.3333% | +| `curriculum_mathematics_logic_sequence` | 240 | 0 | 0.97309 | no | 4 | 1.6667% | +| `curriculum_systems_software_sequence` | 240 | 0 | 0.97309 | no | 3 | 1.2500% | + +(Exhaustive sweep: 71,790 atoms, `wrong = 0`, 164 refusals.) + +A licensed band would therefore be certified on evidence that is **99.0%–99.98% +correct non-commitment**. ADR-0262 §5.1 explicitly rules that unacceptable. The +gate cannot see it: `ClassTally` carries `correct`/`wrong`/`refused` and **no +verdict axis**, so a correct UNKNOWN is indistinguishable from a correct ENTAILED +once tallied. Mix can only be enforced at the producer, which is exactly the +open outcome-mix ruling recorded in +`docs/research/distinct-evidence-audit-2026-07-25.md`. + +**Consequence for this arc:** that ruling is no longer academic — it is the last +thing between a built loop and four unearned licenses. Phase C therefore ships the +writer and **does not commit the artifact**; `chat/data/curriculum_serve_ledger.json` +stays absent, `missing_ok=True` stays true, and every band stays DISCLOSED. Writing +it is a ratification, so it belongs to the explicit `core proposal-queue reseal` +verb a human runs (Phase D), and it should not be run before the mix ruling. + +`tests/test_curriculum_practice.py::test_curriculum_ledger_is_not_committed` pins +the absence, and `test_sealing_would_license_exactly_the_bands_that_reach_the_floor` +pins the consequence, so neither can be lost and re-derived wrongly. + +### Recommendation (still Shay's call) + +Enforce mix **at the producer**, as a per-verdict-class floor rather than a ratio: +a band earns SERVE only when each verdict class it can express independently +reaches the volume the floor requires. Under that rule no band earns anything today +(max entailed volume is 9), which is the honest reading, and it makes ADR-0262 §5.1 +mechanical instead of advisory. It also keeps the deduction bands' aggregate +licenses intact, since imposing a per-class 657 retroactively would fail all 25. + +--- + +## 2. Correction — ADR-0264 §4.2's ceiling table understates every band + +I sized bands from **per-term exclusivity** (a term taught by only one subject). +The router's actual predicate is **per-pair**: `resolve_domain` routes a question +iff exactly one served subject holds *both* terms. A term taught in two subjects +can still appear in a pair only one subject holds both halves of, so per-term +exclusivity is a strictly tighter bound and under-counts. + +| band | ADR-0264 §4.2 | true | verdict change | +|---|---:|---:|---| +| `curriculum_systems_software_causal` | 630 | **720** | **cannot reach 657 → CAN** | +| `curriculum_philosophy_theology_modal` | 44,104 | 45,300 | — | +| `curriculum_philosophy_theology_contrast` | 22,052 | 22,650 | — | +| `curriculum_mathematics_logic_modal` | 420 | 480 | — | +| `curriculum_mathematics_logic_{sequence,contrast,evidential}` | 210 | 240 | — | +| `curriculum_systems_software_modal` | 420 | 480 | — | +| `curriculum_systems_software_sequence` | 210 | 240 | — | +| `curriculum_physics_{causal,modal}` | 720 / 480 | 720 / 480 | — | + +So **4 of 11 bands can reach 657, not 3**, and **7 cannot, not 8**. The retarget +conclusion is unaffected — `philosophy_theology · modal` is still the largest space +by two orders of magnitude, and `physics · modal` (480) is still impossible at any +authoring volume. The numbers are now pinned in +`tests/test_curriculum_practice.py::BAND_ATOM_SPACE`, measured rather than derived, +so this class of error cannot recur silently. + +--- + +## 3. Finding — two taught lemmas collide with the argument reader's grammar + +`then` and `therefore` are lemmas `philosophy_theology`'s packs teach **and** +control words of the argument reader: `therefore` is literally the conclusion +marker in `". Therefore ."`, and `then` is the conditional consequent +marker. An atom using either as a term compiles an argument the reader cannot parse +back, so it refuses `compiled_premises_unreadable`. + +- **164 routable atoms** affected (112 modal + 52 contrast) out of 71,790. +- All 164 are **coverage misses, not wrongs** — excluded from reliability's + denominator (ADR-0175 §4), never a confabulation. `wrong = 0` holds. +- Exactly two lemmas are responsible; every other implicated term is only ever the + *partner* of one of them. +- It is in the band Phase F retargets to. + +The narrow cost is small. The mechanism is not: **the vocabulary boundary and the +reader's grammar are never screened against each other.** A future pack teaching +`and`, `or`, `not`, or `if` would open a much larger hole the same way, and it would +present as a coverage dip rather than an error. A reserved-word check at pack mount +or at chain ratification would close it. + +Not fixed here — outside Phase C's unit (`DIVISION-OF-WORK.md` §6.5). Pinned as a +bound by `test_the_collision_is_bounded_to_two_lemmas`, so a third reserved word +entering the curriculum fails loudly instead of quietly enlarging the gap. + +--- + +## 4. What did not move + +- `evals/curriculum_serve` lane: `[physics] n=32 correct=32 wrong=0 declined_mismatch=0 + anti_recall=5`, unchanged; `report.json` bytes and the `curriculum_serve_v1` SHA + pin unchanged. The producer reads the same solver the lane does and writes nothing + the lane reads. +- No serving behaviour change of any kind: no ledger committed, so + `curriculum_serve_license()` still returns `None` for every band and every + curriculum answer is still served DISCLOSED. +- `refuted` remains unreachable — no corpus row can express a negative until + ADR-0264 R1–R4 is implemented. `test_gold_mix_has_no_refuted_class` pins that as a + red test for the polarity unit to turn green. diff --git a/evals/curriculum_serve/practice/__init__.py b/evals/curriculum_serve/practice/__init__.py new file mode 100644 index 00000000..793d77f5 --- /dev/null +++ b/evals/curriculum_serve/practice/__init__.py @@ -0,0 +1,32 @@ +"""Sealed practice for curriculum-grounded serving (ADR-0262/0264, Phase C). + +The missing producer named by ``chat/curriculum_serve_license.py``'s docstring: +that module reads ``chat/data/curriculum_serve_ledger.json`` and declares +``evals.curriculum_serve.practice.runner.seal_ledger`` "the only writer" of it. +Until this package existed the writer did not, so authoring curriculum could not +earn a license at any volume. +""" + +from evals.curriculum_serve.practice.generator import ( + CASES_PER_BAND, + CurriculumOracleTether, + CurriculumSolver, + QueryAtom, + all_gold_problems, + assert_practice_atoms_distinct, + band_cases, + routable_atoms, + taught_atoms, +) + +__all__ = [ + "CASES_PER_BAND", + "CurriculumOracleTether", + "CurriculumSolver", + "QueryAtom", + "all_gold_problems", + "assert_practice_atoms_distinct", + "band_cases", + "routable_atoms", + "taught_atoms", +] diff --git a/evals/curriculum_serve/practice/generator.py b/evals/curriculum_serve/practice/generator.py new file mode 100644 index 00000000..df7c32ca --- /dev/null +++ b/evals/curriculum_serve/practice/generator.py @@ -0,0 +1,274 @@ +"""The curriculum practice corpus — one committed case per DISTINCT query atom. + +ADR-0262 (curriculum-grounded serving) supplies the solver; ADR-0264 R9 supplies +the sizing rule. The corpus is enumerated, never authored: a band's case space is +exactly the set of exam questions that ROUTE to its subject and whose relation +falls in its family, so there is no hand-written gold to drift and nothing to +paraphrase-pad. + +**Why enumeration is the honest shape here.** ``conservative_floor`` is a Wilson +bound and Wilson assumes independent trials, so a deterministic pipeline replaying +one input N times supplies one trial's evidence, not N (ADR-0264 R9). The +``deduction_serve`` producer sets a flat ``CASES_PER_BAND = 720`` and reaches it by +cycling a template x vocabulary space as small as 28 distinct instances; 21 of its +25 ratified bands therefore do not clear θ_SERVE on distinct evidence +(``tests/test_volume_honesty.py``). This producer cannot repeat that: its case +identity IS the query atom, so ``committed == distinct`` holds by construction +rather than by discipline. The model is ``evals/determination_estimation`` +(660 distinct cases per class, zero repeats), which is what +``DIVISION-OF-WORK.md`` §4 directs Phase C to follow. + +**Routability is asked, not re-derived.** An atom is in a band's space iff +``chat.curriculum_surface.resolve_domain`` routes its two terms to that subject. +Restating the predicate here would let the corpus and the serving path disagree +about which questions exist — so the corpus asks the serving path. This is not a +gold dependency: gold comes from ``evals.curriculum_serve.oracle``, which shares +no code with the serving path (ADR-0199 L-2). + +Note that per-*term* exclusivity is a strictly tighter bound than the per-*pair* +predicate the router actually applies: a term taught in two subjects can still +appear in a pair that only one subject holds both halves of. Sizing a band from +exclusive terms therefore UNDER-counts its space — ``systems_software · causal`` +reads as 630 under that bound and is really 720, which is the difference between +"cannot reach 657" and "can". + +Determinism: no clock, no RNG. The atom space is sorted, selection is a fixed +stride, so ``all_gold_problems()`` is byte-stable across runs and the sealed +ledger is safe to commit and SHA-verify on load. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from functools import lru_cache + +from chat.curriculum_surface import ( + SERVED_DOMAINS, + CurriculumQuery, + band_for, + decide_curriculum_question, + resolve_domain, +) +from core.learning_arena.protocols import BaseAttempt, DomainProblem, Problem +from evals.curriculum_serve.oracle import oracle_answer +from teaching.curriculum_premises import CONNECTIVE_FAMILY, FAMILIES, load_curriculum + +#: Committed cases per band, when the band's atom space is larger than this. +#: +#: 660 is ``evals/determination_estimation``'s constant, chosen just above the 657 +#: a perfect record needs to clear θ_SERVE=0.99 (``volume_for_theta(0.99)``). It is +#: a ceiling on *committed volume*, not a quota to be filled: a band whose space is +#: smaller emits its whole space and honestly reports a number below the floor. +CASES_PER_BAND = 660 + + +@dataclass(frozen=True, slots=True) +class QueryAtom: + """One distinct curriculum decision — the unit of committed evidence. + + The atom IS the decision key (ADR-0264 R9). Two cases with the same atom are + the same decision replayed, so the corpus admits each atom at most once and + :meth:`key` is what ``tests/test_volume_honesty.py`` audits. + """ + + domain: str + family: str + subject: str + connective: str + obj: str + + @property + def text(self) -> str: + """The exam question, in the closed ``Does ?`` grammar.""" + return f"Does {self.subject} {self.connective} {self.obj}?" + + @property + def key(self) -> tuple[str, str, str, str]: + """The distinct-decision key. Polarity is deliberately absent: a + negative curriculum row (ADR-0264 R1-R4) changes a question's ANSWER, + never its identity, and R4 rejects a corpus that states both polarities + of one atom at ratification. So polarity cannot distinguish two cases.""" + return (self.domain, self.subject, self.connective, self.obj) + + @property + def case_id(self) -> str: + return f"{self.domain}:{self.connective}:{self.subject}->{self.obj}" + + +@lru_cache(maxsize=None) +def _routes_to(subject: str, obj: str) -> str | None: + """The subject a term pair routes to, or ``None`` — the router's own answer.""" + domain = resolve_domain(CurriculumQuery(subject, "", obj)) + return domain if isinstance(domain, str) else None + + +@lru_cache(maxsize=None) +def routable_atoms(domain: str, family: str) -> tuple[QueryAtom, ...]: + """Every exam question that reaches *(domain, family)*, in sorted order. + + The band's complete case space: ordered term pairs the router sends to + *domain*, crossed with the family's connectives. Self-pairs are excluded — + ``Does x cause x?`` is not a curriculum question any corpus states. + """ + connectives = sorted(c for c, f in CONNECTIVE_FAMILY.items() if f == family) + terms = sorted(load_curriculum(domain).vocabulary) + atoms = [ + QueryAtom(domain, family, subject, connective, obj) + for subject in terms + for obj in terms + if subject != obj and _routes_to(subject, obj) == domain + for connective in connectives + ] + return tuple(sorted(atoms, key=lambda a: (a.subject, a.connective, a.obj))) + + +@lru_cache(maxsize=None) +def taught_atoms(domain: str, family: str) -> frozenset[tuple[str, str, str, str]]: + """The atom keys a ratified chain states directly — the ``entailed`` class. + + Scarce by nature: a band has as many taught edges as the curriculum states, + which is why ADR-0262 §5.1 makes authored volume the binding constraint on + what a band can demonstrate beyond non-commitment. + """ + return frozenset( + QueryAtom(domain, family, c.subject, c.connective, c.obj).key + for c in load_curriculum(domain).family(family) + ) + + +def band_cases(domain: str, family: str, cap: int = CASES_PER_BAND) -> tuple[QueryAtom, ...]: + """The committed cases for one band — distinct atoms, at most *cap* of them. + + Whole space when it fits. Otherwise: every taught edge, then a fixed STRIDE + across the remaining atoms in sorted order. + + The stride is deliberate rather than a prefix. A lexicographic prefix of 660 + atoms out of 45,300 would cover about five subject terms of 151, measuring one + corner of the band and reporting it as the band; a stride spreads the sample + across every subject the curriculum teaches. It is not a random sample — there + is no RNG here — but it is an unbiased-by-construction one, and it cannot be + tuned per band, which is the property that matters for an evidence artifact. + Taught edges are force-included because they are the only committed-POSITIVE + evidence a band has and a sample that missed them would report a band's + reliability while exercising none of its content. + """ + atoms = routable_atoms(domain, family) + if len(atoms) <= cap: + return atoms + taught_keys = taught_atoms(domain, family) + taught = [a for a in atoms if a.key in taught_keys] + rest = [a for a in atoms if a.key not in taught_keys] + need = max(cap - len(taught), 0) + stride = max(len(rest) // need, 1) if need else 1 + picked = rest[::stride][:need] + return tuple(sorted(taught + picked, key=lambda a: (a.subject, a.connective, a.obj))) + + +def practice_bands() -> tuple[tuple[str, str], ...]: + """Every *(domain, family)* pair the ratified corpora actually populate. + + A family with no chains is not a band with zero volume — it is not a band. + ``decide_curriculum_question`` refuses it ``empty_curriculum`` before any + band key is assigned, so committing cases under one would invent a capability + axis the serving path never uses. + """ + return tuple( + (domain, family) + for domain in SERVED_DOMAINS + for family in FAMILIES + if load_curriculum(domain).family(family) + ) + + +def all_gold_problems(cap: int = CASES_PER_BAND) -> tuple[Problem, ...]: + """The full practice corpus, every band, in a deterministic order.""" + problems: list[Problem] = [] + for domain, family in practice_bands(): + band = band_for(domain, family) + for atom in band_cases(domain, family, cap): + problems.append( + Problem(problem_id=atom.case_id, class_name=band, payload=atom) + ) + return tuple(problems) + + +class DuplicateAtom(AssertionError): + """A band committed the same query atom twice — the R9 invariant, violated.""" + + +def assert_practice_atoms_distinct(cap: int = CASES_PER_BAND) -> None: + """``committed == distinct``, enforced at the producer (ADR-0264 R9). + + The Phase B audit MEASURES inflation; this refuses to emit it. Both matter: + a producer that cannot pad is better than one that is caught padding, and the + audit still guards the sealed artifact against a future producer change. + """ + seen: dict[str, set[tuple[str, str, str, str]]] = {} + for problem in all_gold_problems(cap): + atom: QueryAtom = problem.payload + keys = seen.setdefault(problem.class_name, set()) + if atom.key in keys: + raise DuplicateAtom( + f"{problem.class_name}: query atom {atom.key} committed twice — " + "practice volume must be distinct evidence (ADR-0264 R9)" + ) + keys.add(atom.key) + + +class CurriculumSolver: + """The production decision path, under test (``chat/curriculum_surface.py``). + + ``committed`` is ``verdict != "declined"``. A typed refusal is an honest + non-commitment, excluded from reliability's denominator (ADR-0175 §4) and + counted as coverage — never conflated with a confabulation. That mapping is + the same one ``evals/curriculum_serve/runner.py`` applies when it separates a + ``declined`` mismatch from a ``wrong``. + """ + + domain_id = "curriculum_serve" + + def attempt(self, problem: DomainProblem) -> BaseAttempt: + atom: QueryAtom = problem.payload + decision = decide_curriculum_question(atom.text) + committed = decision.verdict != "declined" + return BaseAttempt( + committed=committed, + answer=decision.verdict, + reason=decision.reason, + case_id=problem.problem_id, + ) + + +class CurriculumOracleTether: + """Tier-1 gold: the INDEPENDENT curriculum oracle (ADR-0199 L-2). + + ``evals/curriculum_serve/oracle.py`` shares no code with the serving path — + its own loader, ratification predicate, family table, agreement normalization + and verdict rule. Two independently written procedures agreeing on every atom + is real evidence the serving path reads the curriculum correctly; a tether + built from the compiler would only prove the compiler agrees with itself. + """ + + domain_id = "curriculum_serve" + + def is_correct(self, attempt: BaseAttempt, problem: DomainProblem) -> bool: + return attempt.answer == self.gold_answer(problem) + + def gold_answer(self, problem: DomainProblem) -> str: + atom: QueryAtom = problem.payload + return oracle_answer(atom.domain, atom.subject, atom.connective, atom.obj).verdict + + +__all__ = [ + "CASES_PER_BAND", + "CurriculumOracleTether", + "CurriculumSolver", + "DuplicateAtom", + "QueryAtom", + "all_gold_problems", + "assert_practice_atoms_distinct", + "band_cases", + "practice_bands", + "routable_atoms", + "taught_atoms", +] diff --git a/evals/curriculum_serve/practice/runner.py b/evals/curriculum_serve/practice/runner.py new file mode 100644 index 00000000..6e26ba07 --- /dev/null +++ b/evals/curriculum_serve/practice/runner.py @@ -0,0 +1,187 @@ +"""Sealed-practice runner for curriculum serving (ADR-0262/0264, Phase C). + +Folds the ADR-0199 ``run_practice`` engine over the enumerated atom corpus +(``generator.py``) and reads each band's ``ClassTally`` through the reliability +gate. Two outputs, exactly mirroring ``evals/deduction_serve/practice/runner.py``: + +- ``run()`` — the falsifiable report: which *(subject × relation family)* bands + earned SERVE, at what committed volume, and with what verdict MIX. +- ``seal_ledger()`` — writes the SHA-sealed artifact the serving reader + (``chat/curriculum_serve_license.py``) trusts. That module's docstring already + names this function as the only writer; this is the function it named. + +**The artifact is NOT committed by this arc, and that is a finding, not an +omission.** Every band with a routable space ≥657 reaches θ_SERVE on the UNKNOWN +class alone: ``physics · causal`` commits 7 taught edges and 713 correct +non-commitments, and ``conservative_floor(660, 660) = 0.990046 ≥ 0.99``. So +sealing this ledger today would flip four bands from disclosed to authoritative on +a mix that is ~99% non-entailed — which ADR-0262 §5.1 explicitly rules +unacceptable, and which the still-open outcome-mix ruling (ADR-0264 R9's recorded +non-decision; ``docs/research/distinct-evidence-audit-2026-07-25.md``) exists to +settle. The plan of record's Phase-1 exit criterion asked for a ledger that is +"real (still-unearned)"; building the producer shows that state is unreachable. +Committing the artifact is therefore a ratification, and it is left to the +explicit ``core proposal-queue reseal`` verb a human runs (Phase D). + +Determinism: the corpus is enumerated + sorted (no clock, no RNG) and +``run_practice`` is a pure fold, so the sealed ledger is byte-identical across +runs — safe to commit and SHA-verify on load, whenever it is committed. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path +from typing import Any + +from core.learning_arena.engine import run_practice +from core.ratified_ledger import seal_artifact, write_sealed_ledger +from core.reliability_gate import Action, Ceilings, ClassTally, license_for +from evals.curriculum_serve.practice.generator import ( + CASES_PER_BAND, + CurriculumOracleTether, + CurriculumSolver, + all_gold_problems, + assert_practice_atoms_distinct, +) + +#: The committed sealed ledger lives next to its serving READER (chat/), the +#: topology the estimation and deduction ledgers already use. +_SEALED_LEDGER_PATH = ( + Path(__file__).resolve().parents[3] / "chat" / "data" / "curriculum_serve_ledger.json" +) + + +def build_ledger(cap: int = CASES_PER_BAND) -> dict[str, ClassTally]: + """Run sealed practice over the enumerated corpus → per-band ledger.""" + report = run_practice( + all_gold_problems(cap), CurriculumSolver(), CurriculumOracleTether() + ) + return dict(report.ledger) + + +def build_mix(cap: int = CASES_PER_BAND) -> dict[str, dict[str, int]]: + """Per-band GOLD verdict mix — the figure the reliability gate cannot see. + + ``ClassTally`` carries ``correct``/``wrong``/``refused`` and no verdict axis, so + a correct UNKNOWN is indistinguishable from a correct ENTAILED once tallied. + That is precisely why a band can clear θ_SERVE on non-commitments alone, and + why the mix has to be reported alongside the license rather than inferred from + it. + """ + tether = CurriculumOracleTether() + mix: dict[str, dict[str, int]] = {} + for problem in all_gold_problems(cap): + band = mix.setdefault(problem.class_name, {}) + gold = tether.gold_answer(problem) + band[gold] = band.get(gold, 0) + 1 + return mix + + +def run(ceilings: Ceilings | None = None, cap: int = CASES_PER_BAND) -> dict[str, Any]: + """Build the ledger and report the SERVE verdict + mix per band.""" + ceilings = ceilings if ceilings is not None else Ceilings.default() + ledger = build_ledger(cap) + mix = build_mix(cap) + classes: dict[str, Any] = {} + for band, tally in sorted(ledger.items()): + serve = license_for(tally, Action.SERVE, ceilings) + band_mix = mix.get(band, {}) + entailed = band_mix.get("entailed", 0) + classes[band] = { + "correct": tally.correct, + "wrong": tally.wrong, + "refused": tally.refused, + "committed": tally.committed, + "reliability": tally.reliability, + "coverage": tally.coverage, + "serve_licensed": serve.licensed, + "serve_ratio": serve.ratio, + "gold_mix": dict(sorted(band_mix.items())), + "entailed_share": ( + round(entailed / tally.committed, 6) if tally.committed else 0.0 + ), + } + return { + "lane": "curriculum-serve-practice", + "classes": classes, + "any_band_serve_licensed": any(c["serve_licensed"] for c in classes.values()), + "wrong_is_zero": all(c["wrong"] == 0 for c in classes.values()), + } + + +def build_sealed_artifact(cap: int = CASES_PER_BAND) -> dict[str, Any]: + """The sealed-ledger dict (self-verifying ``content_sha256``).""" + return seal_artifact( + build_ledger(cap), + schema="curriculum_serve_ledger_v1", + note=( + "Sealed-practice committed ledger for curriculum-grounded serving " + "(ADR-0262, sized per ADR-0264 R9). Engine reads, never writes. " + "Ceilings stay at safe defaults (theta_SERVE=0.99). One committed case " + "per DISTINCT routable query atom, so committed == distinct evidence by " + "construction. NOTE: reliability here is dominated by correct " + "non-commitments; read `gold_mix` from the runner report before " + "treating a licensed band as demonstrated curriculum competence." + ), + provenance="evals.curriculum_serve.practice.runner.seal_ledger", + ) + + +def seal_ledger( + path: Path = _SEALED_LEDGER_PATH, cap: int = CASES_PER_BAND +) -> dict[str, Any]: + """Regenerate + write the committed sealed ledger. + + Verifies the R9 distinct-evidence invariant first: a producer that padded + could never seal, rather than sealing and being caught by the audit later. + """ + assert_practice_atoms_distinct(cap) + return write_sealed_ledger(path, build_sealed_artifact(cap)) + + +def _full_sweep() -> dict[str, Any]: + """Every routable atom in every band — the exhaustive agreement measurement. + + Not the sealed corpus (that is capped at :data:`CASES_PER_BAND`); this is the + evidence that the cap is a sampling choice and not a hiding place. Runs the + whole ~70k-atom space, so it is a deliberate CLI invocation and not a test. + """ + return run(cap=10**9) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--seal", action="store_true", + help="regenerate + write chat/data/curriculum_serve_ledger.json " + "(a RATIFICATION — see the module docstring before running it)", + ) + parser.add_argument( + "--full", action="store_true", + help="report over EVERY routable atom instead of the capped corpus (slow)", + ) + args = parser.parse_args(argv) + + if args.seal: + artifact = seal_ledger() + print(f"sealed {len(artifact['classes'])} bands -> {_SEALED_LEDGER_PATH}") + return 0 + + report = _full_sweep() if args.full else run() + for band, c in report["classes"].items(): + print( + f" {band:44s} committed={c['committed']:6d} wrong={c['wrong']} " + f"refused={c['refused']:4d} reliability={c['reliability']:.5f} " + f"SERVE={str(c['serve_licensed']):5s} entailed={c['gold_mix'].get('entailed', 0):3d} " + f"({c['entailed_share']:.4%})" + ) + print( + f"any_band_serve_licensed={report['any_band_serve_licensed']} " + f"wrong_is_zero={report['wrong_is_zero']}" + ) + return 0 if report["wrong_is_zero"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_curriculum_practice.py b/tests/test_curriculum_practice.py new file mode 100644 index 00000000..3359fc9e --- /dev/null +++ b/tests/test_curriculum_practice.py @@ -0,0 +1,352 @@ +"""Phase C — the curriculum practice producer (ADR-0262 solver, ADR-0264 R9 sizing). + +Four things are pinned here, in decreasing order of how quietly they would fail: + +1. **The ledger is not committed, and a band would earn SERVE if it were.** The + producer works; that is exactly why the artifact is a ratification. Pinned so + the reasoning cannot be lost and re-derived wrongly later. +2. **`then` / `therefore` collide with the argument reader's grammar.** Two taught + `philosophy_theology` lemmas are the reader's own control words, so 164 routable + atoms refuse `compiled_premises_unreadable`. Coverage misses, never wrongs — but + in the band Phase F retargets to. +3. **`committed == distinct` by construction**, not by discipline. +4. **wrong=0** over the capped corpus, per band. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from chat.curriculum_surface import band_for, decide_curriculum_question +from core.ratified_ledger import CAPABILITY_LEDGERS, load_sealed_ledger +from core.reliability_gate import Action, Ceilings, license_for +from core.reliability_gate.evidence import audit_bands +from evals.curriculum_serve.practice.generator import ( + CASES_PER_BAND, + DuplicateAtom, + QueryAtom, + all_gold_problems, + assert_practice_atoms_distinct, + band_cases, + practice_bands, + routable_atoms, + taught_atoms, +) +from evals.curriculum_serve.practice.runner import ( + build_ledger, + build_mix, + build_sealed_artifact, + run, + seal_ledger, +) + +REPO_ROOT = Path(__file__).resolve().parents[1] +THETA_SERVE = 0.99 + +#: The 11 bands the ratified corpora populate, and each one's FULL routable atom +#: space. Pinned in both directions: a rise means new curriculum (welcome, update +#: deliberately), a fall means a band lost reachable questions. +#: +#: These supersede ADR-0264 §4.2's table, which sized bands from per-TERM +#: exclusivity. That bound is strictly tighter than the router's per-PAIR +#: predicate, so it under-counted — `systems_software_causal` reads as 630 there +#: and is really 720, which flips it from "cannot reach 657" to "can". +BAND_ATOM_SPACE: dict[str, int] = { + "curriculum_mathematics_logic_contrast": 240, + "curriculum_mathematics_logic_evidential": 240, + "curriculum_mathematics_logic_modal": 480, + "curriculum_mathematics_logic_sequence": 240, + "curriculum_philosophy_theology_contrast": 22650, + "curriculum_philosophy_theology_modal": 45300, + "curriculum_physics_causal": 720, + "curriculum_physics_modal": 480, + "curriculum_systems_software_causal": 720, + "curriculum_systems_software_modal": 480, + "curriculum_systems_software_sequence": 240, +} + +#: Bands whose space reaches the 657 a perfect record needs. Derived below, never +#: restated: these are the four that WOULD earn SERVE the moment a ledger is sealed. +_CAN_REACH_FLOOR = { + band for band, space in BAND_ATOM_SPACE.items() if space >= 657 +} + +#: The reader-grammar collision, exact. Both lemmas are taught by +#: `philosophy_theology`'s packs AND are control words of the argument reader: +#: `therefore` is literally the conclusion marker `". Therefore ."`, and `then` +#: is the conditional consequent marker. An atom using either as a term compiles an +#: argument the reader cannot parse back. +RESERVED_WORD_LEMMAS = ("then", "therefore") + + +@pytest.fixture(scope="module") +def report() -> dict: + return run() + + +@pytest.fixture(scope="module") +def ledger() -> dict: + return build_ledger() + + +@pytest.fixture(scope="module") +def mix() -> dict: + return build_mix() + + +@pytest.fixture(scope="module") +def problems() -> tuple: + return all_gold_problems() + + +# --------------------------------------------------------------------------- +# 1. The artifact is uncommitted, and that is a decision with a reason. +# --------------------------------------------------------------------------- + +def test_curriculum_ledger_is_not_committed() -> None: + """Phase C builds the writer; committing the artifact is a ratification. + + The plan of record's exit criterion asked for a ledger that is "real + (still-unearned)". That state is unreachable: every band with >=657 routable + atoms clears theta_SERVE on correct NON-COMMITMENTS alone. So a committed + ledger is necessarily an earning one, and this asserts nobody committed it as + housekeeping. + """ + path = CAPABILITY_LEDGERS["curriculum_serve"].path + assert not path.exists(), ( + f"{path.name} exists. If that was deliberate ratification, update this " + "test and ADR-0262 §5 together — a curriculum ledger licenses bands whose " + "committed evidence is ~99% non-entailed." + ) + assert CAPABILITY_LEDGERS["curriculum_serve"].missing_ok, ( + "an absent curriculum ledger must stay legitimately absent (serves disclosed)" + ) + + +def test_sealing_would_license_exactly_the_bands_that_reach_the_floor( + ledger: dict, +) -> None: + """The consequence of sealing, measured rather than asserted in prose. + + This is the finding Phase C exists to produce: the license is reachable today + and it is reachable on the UNKNOWN class. If this ever reports a different set, + the corpus moved and the ADR-0262 §5 discussion needs revisiting. + """ + ceilings = Ceilings.default() + licensed = { + band + for band, tally in ledger.items() + if license_for(tally, Action.SERVE, ceilings).licensed + } + assert licensed == _CAN_REACH_FLOOR + assert len(licensed) == 4 + + +def test_a_licensed_band_is_licensed_on_non_commitments(report: dict) -> None: + """The mix behind the license — the number `ClassTally` structurally cannot see. + + `ClassTally` carries correct/wrong/refused and no verdict axis, so a correct + UNKNOWN counts exactly like a correct ENTAILED. Every band that would earn + SERVE does so with an entailed share under 5%, which is what ADR-0262 §5.1 + rules unacceptable and what the open outcome-mix ruling has to settle. + """ + for band in sorted(_CAN_REACH_FLOOR): + entry = report["classes"][band] + assert entry["serve_licensed"] is True + assert entry["gold_mix"].get("entailed", 0) <= 9 + assert entry["entailed_share"] < 0.05, ( + f"{band}: entailed share {entry['entailed_share']:.4%} — if a band ever " + "earns SERVE on a genuine mix, that is the ADR-0262 §5.1 bar being met " + "and this pin should be revisited, not relaxed" + ) + + +def test_seal_ledger_writes_a_verifying_artifact(tmp_path: Path) -> None: + """The writer `chat/curriculum_serve_license.py` names must actually work. + + Written to a tmp path, never to `chat/data/` — the point of the test is that + the mechanism is real, not that the repository gains a ledger. + """ + path = tmp_path / "curriculum_serve_ledger.json" + artifact = seal_ledger(path) + assert path.exists() + assert artifact["provenance"] == "evals.curriculum_serve.practice.runner.seal_ledger" + assert artifact["schema"] == "curriculum_serve_ledger_v1" + # Round-trips through the same verifier the serving reader uses. + tallies = load_sealed_ledger(path) + assert set(tallies) == set(BAND_ATOM_SPACE) + assert all(t.wrong == 0 for t in tallies.values()) + + +def test_sealed_artifact_is_byte_identical_across_runs() -> None: + """Determinism (ADR-0199 L-4): no clock, no RNG, so re-sealing must not move it. + + Two independent computations are the minimum that proves this and the maximum + worth paying for — each is a full fold over the corpus. + """ + first = build_sealed_artifact() + second = build_sealed_artifact() + assert json.dumps(first, indent=2, sort_keys=True) == json.dumps( + second, indent=2, sort_keys=True + ) + assert first["content_sha256"] == second["content_sha256"] + + +# --------------------------------------------------------------------------- +# 2. The reader-grammar collision — in the band Phase F retargets to. +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("lemma", RESERVED_WORD_LEMMAS) +def test_reserved_reader_words_are_taught_curriculum_lemmas(lemma: str) -> None: + """The precondition of the collision: these really are taught terms.""" + from teaching.curriculum_premises import load_curriculum + + assert lemma in load_curriculum("philosophy_theology").vocabulary + + +@pytest.mark.parametrize("lemma", RESERVED_WORD_LEMMAS) +def test_reserved_word_atoms_refuse_rather_than_answer(lemma: str) -> None: + """A control word as a TERM refuses typed — it never answers wrongly. + + This is the honest failure: `compiled_premises_unreadable` is a coverage miss + excluded from reliability's denominator (ADR-0175 §4), not a confabulation. The + hazard is that the vocabulary boundary and the reader's grammar are not screened + against each other, so a future pack teaching "and"/"or"/"if" would open a much + larger hole silently. Recorded, not fixed — out of Phase C's scope + (DIVISION-OF-WORK §6.5). + """ + decision = decide_curriculum_question(f"Does knowledge requires {lemma}?") + assert decision.verdict == "declined" + assert decision.reason == "compiled_premises_unreadable" + + +def test_the_collision_is_bounded_to_two_lemmas() -> None: + """Every refusal in the whole 69,666-atom space traces to `then`/`therefore`. + + Pinned as a bound, so the day a third reserved word enters the curriculum this + fails instead of quietly enlarging the gap. + """ + offenders: set[str] = set() + checked = 0 + for domain, family in practice_bands(): + for atom in routable_atoms(domain, family): + reserved = {atom.subject, atom.obj} & set(RESERVED_WORD_LEMMAS) + if not reserved: + continue + checked += 1 + if decide_curriculum_question(atom.text).verdict == "declined": + offenders |= reserved + assert checked > 0, "no reserved-word atoms found — the sweep is not exercising anything" + assert offenders == set(RESERVED_WORD_LEMMAS) + + +# --------------------------------------------------------------------------- +# 3. Distinct evidence, structurally. +# --------------------------------------------------------------------------- + +def test_committed_equals_distinct_for_every_band(problems: tuple) -> None: + """ADR-0264 R9 holds by construction: case identity IS the decision key.""" + by_band: dict[str, list] = {} + for problem in problems: + by_band.setdefault(problem.class_name, []).append(problem.payload.key) + audits = audit_bands(by_band) + assert audits + for audit in audits: + assert audit.committed == audit.distinct + assert audit.inflation == 1.0 + assert audit.max_repeat == 1 + + +def test_distinctness_guard_fails_under_mutation() -> None: + """The R9 producer guard must be able to fail, or it is worthless. + + Same discipline as `tests/test_volume_honesty.py::test_audit_detects_padding`: + a pin that cannot go red proves nothing. Feeds the guard a corpus with one + atom repeated and asserts it refuses. + """ + atom = QueryAtom("physics", "causal", "force", "causes", "acceleration") + assert atom.key == ("physics", "force", "causes", "acceleration") + + import evals.curriculum_serve.practice.generator as gen + + original = gen.all_gold_problems + from core.learning_arena.protocols import Problem + + def padded(cap: int = CASES_PER_BAND) -> tuple[Problem, ...]: + return ( + Problem("a", "curriculum_physics_causal", atom), + Problem("b", "curriculum_physics_causal", atom), + ) + + gen.all_gold_problems = padded # type: ignore[assignment] + try: + with pytest.raises(DuplicateAtom, match="committed twice"): + gen.assert_practice_atoms_distinct() + finally: + gen.all_gold_problems = original # type: ignore[assignment] + + assert_practice_atoms_distinct() # the real corpus still passes + + +# --------------------------------------------------------------------------- +# 4. The corpus itself: shape, determinism, wrong=0. +# --------------------------------------------------------------------------- + +def test_band_atom_spaces_are_exactly_as_recorded() -> None: + measured = { + band_for(d, f): len(routable_atoms(d, f)) for d, f in practice_bands() + } + assert measured == BAND_ATOM_SPACE + + +def test_every_band_is_a_populated_family() -> None: + """A family with no chains is not a band with zero volume — it is not a band.""" + from teaching.curriculum_premises import load_curriculum + + for domain, family in practice_bands(): + assert load_curriculum(domain).family(family) + assert len(practice_bands()) == len(BAND_ATOM_SPACE) + + +def test_cap_selects_taught_edges_first() -> None: + """Taught edges are the only committed-POSITIVE evidence; a sample that + dropped them would report reliability while exercising no curriculum.""" + for domain, family in practice_bands(): + space = len(routable_atoms(domain, family)) + cases = band_cases(domain, family) + assert len(cases) == min(space, CASES_PER_BAND) + taught = taught_atoms(domain, family) + assert taught <= {c.key for c in cases}, f"{domain}/{family} dropped a taught edge" + + +def test_corpus_is_deterministic(problems: tuple) -> None: + assert [p.problem_id for p in all_gold_problems()] == [p.problem_id for p in problems] + + +def test_wrong_is_zero_for_every_band(report: dict) -> None: + """The serving path and the independent oracle agree on every committed atom.""" + assert report["wrong_is_zero"] is True + for band, entry in report["classes"].items(): + assert entry["wrong"] == 0, f"{band}: {entry}" + assert set(report["classes"]) == set(BAND_ATOM_SPACE) + + +def test_gold_mix_has_no_refuted_class(report: dict) -> None: + """No corpus row can express a negative yet (ADR-0264 R1-R4 unimplemented), so + `refuted` is unreachable and the mix is entailed/unknown only. Pinned so the + polarity unit has a red test to turn green.""" + for band, entry in report["classes"].items(): + assert "refuted" not in entry["gold_mix"], f"{band} produced refuted — polarity landed?" + assert set(entry["gold_mix"]) <= {"entailed", "unknown", "declined"} + + +def test_mix_totals_reconcile_with_the_ledger( + ledger: dict, report: dict, mix: dict +) -> None: + """The mix is counted over the same corpus the ledger tallies, not a second one.""" + for band, tally in ledger.items(): + assert sum(mix[band].values()) == tally.attempted + assert report["classes"][band]["committed"] == tally.committed diff --git a/tests/test_volume_honesty.py b/tests/test_volume_honesty.py index a85bd61a..b76e95bc 100644 --- a/tests/test_volume_honesty.py +++ b/tests/test_volume_honesty.py @@ -25,9 +25,11 @@ Three producers back the three entries in `core.ratified_ledger.CAPABILITY_LEDGE distinct evidence; the worst three inflate 28 distinct cases to 720 committed. Recorded, pinned, and NOT silently repaired: the ledger is SHA-sealed, ratified, and gating a live flag, so changing it is Shay's ratification, not a test's. -- **curriculum_serve** (ADR-0262/0264) — producer unbuilt. The registry pin below - fails the moment it is built without an audit source, which is the forcing - function that stops Phase C from repeating the deduction pattern. +- **curriculum_serve** (ADR-0262/0264) — clean, and clean *structurally*: the + producer's case identity IS the query atom, so ``committed == distinct`` cannot + drift apart the way it did for deduction, where case identity was a quota index. + Its ledger is deliberately uncommitted — see + ``evals/curriculum_serve/practice/runner.py``. Deliberately narrow. This pins evidence *distinctness*. It does not pin outcome mix — whether each verdict class needs its own volume is an open ruling recorded @@ -99,13 +101,30 @@ def _estimation_keys() -> dict[str, list[Hashable]]: return out +def _curriculum_keys() -> dict[str, list[Hashable]]: + """Decision keys per band for the curriculum practice corpus (Phase C). + + The key is the query ATOM, which is also the producer's case identity — so + this audit can only ever report inflation 1.0 while the producer is correct, + and reports the real figure the moment it is not. Contrast the deduction + source above, where the key had to be recovered from case text because the + producer's identity was a quota index. + """ + from evals.curriculum_serve.practice import all_gold_problems + + out: dict[str, list[Hashable]] = {} + for problem in all_gold_problems(): + out.setdefault(problem.class_name, []).append(problem.payload.key) + return out + + #: capability -> keys provider, or ``None`` when the producer is not built. #: ``None`` is a declaration, not an omission: :func:`test_every_licensed_capability_has_an_audit_source` #: accepts it only while the capability's ledger is genuinely absent. AUDIT_SOURCES: dict[str, Callable[[], dict[str, list[Hashable]]] | None] = { "estimation": _estimation_keys, "deduction_serve": _deduction_keys, - "curriculum_serve": None, # Phase C — see module docstring + "curriculum_serve": _curriculum_keys, } @@ -238,9 +257,10 @@ def test_audit_rejects_impossible_counts() -> None: def test_every_licensed_capability_has_an_audit_source() -> None: """Derived from `CAPABILITY_LEDGERS`, so a new capability cannot slip past. - This is the forcing function for Phase C: the moment - `chat/data/curriculum_serve_ledger.json` exists, a ``None`` audit source - stops being an honest declaration and this fails. + This was the forcing function for Phase C, and it worked: a ``None`` audit + source stops being an honest declaration the moment the capability's ledger + exists. All three producers now carry one, so the guard is holding the line + for the NEXT capability rather than for curriculum. """ assert set(AUDIT_SOURCES) == set(CAPABILITY_LEDGERS), ( "AUDIT_SOURCES and CAPABILITY_LEDGERS disagree — a licensed capability is " @@ -294,6 +314,31 @@ def test_estimation_producer_has_no_inflation( assert below_floor(estimation_audits, THETA_SERVE) == () +# --------------------------------------------------------------------------- +# 3b. The curriculum producer — clean, and clean by construction (Phase C). +# --------------------------------------------------------------------------- + +def test_curriculum_producer_has_no_inflation() -> None: + """The second producer that satisfies the invariant, and the first that + satisfies it STRUCTURALLY. + + Its case identity is the query atom, so ``committed == distinct`` is not a + property the generator has to maintain — it is the same fact twice. Pinned here + beside the estimation standard because this file is where a reader looks to ask + "which producers are honest about volume?". + """ + audits = audit_bands(_curriculum_keys()) + assert audits, "curriculum producer yielded no bands" + assert len(audits) == 11, f"expected 11 populated bands, found {len(audits)}" + for audit in audits: + assert audit.committed == audit.distinct, format_report(audits, THETA_SERVE) + assert audit.inflation == 1.0 + assert audit.max_repeat == 1 + assert {a.band for a in below_floor(audits, THETA_SERVE)} == { + a.band for a in audits if a.distinct < SERVE_VOLUME_AT_THETA_099 + }, "a band below the floor must be below it for lack of VOLUME, never inflation" + + # --------------------------------------------------------------------------- # 4. The deduction producer — the measured exposure, pinned in both directions. # ---------------------------------------------------------------------------