diff --git a/CLAIMS.md b/CLAIMS.md index d180c1d3..51904dcb 100644 --- a/CLAIMS.md +++ b/CLAIMS.md @@ -44,6 +44,7 @@ is a CI failure (`.github/workflows/lane-shas.yml`). | ADR-0131 | `math_teaching_corpus_v1` | Math teaching corpus replays deterministically; all chains pass exit criterion (correct_rate=1.0, wrong=0) | `evals/math_teaching_corpus/v1/report.json` | `eaf160d145da29f9050ede8d58bf111b0f651dd40aeae9201857d0b97e014dd4` | | ADR-0206 | `deductive_logic_v1` | Propositional entailment scored against an independent truth-table oracle; dev+holdout+external 716/716 correct, wrong=0, refused=0 | `evals/deductive_logic/report.json` | `97a230949016e38d5e3f37a69e4245b320575ee70e5af92ff7607f7b05f74b5f` | | ADR-0256 | `deduction_serve_v1` | Flag-gated deduction serving decides real English/member/fused/verb/existential arguments end-to-end under earned SERVE licenses; wrong=0 across all splits | `evals/deduction_serve/report.json` | `0b461a5a49c8f8260ca87d0c9c9f9a17232bd1fdedd982e34649eedf9cca30b5` | +| ADR-0262 | `curriculum_serve_v1` | Flag-gated curriculum serving answers exam questions from a subject's RATIFIED chain corpus only; untaught facts return UNKNOWN (anti-recall probes enforced), wrong=0 | `evals/curriculum_serve/report.json` | `d9e7ba500f040b865870413a940ee9a49910ac22e1a89c9feec1a60bdd2513f1` | ## Verification diff --git a/chat/curriculum_serve_license.py b/chat/curriculum_serve_license.py new file mode 100644 index 00000000..49084bee --- /dev/null +++ b/chat/curriculum_serve_license.py @@ -0,0 +1,68 @@ +"""Serving-side SERVE license for curriculum-grounded answers (ADR-0262). + +Reads the **ratified, committed** curriculum-serve ledger +(``chat/data/curriculum_serve_ledger.json``) and exposes, per *(subject × +relation family)* band, whether the FULL serving pipeline (curriculum +compiler → argument reader → ROBDD engine) has earned ``Action.SERVE`` under +the safe default ceilings (θ_SERVE=0.99). The engine READS this artifact; it +never writes it — the sealed-practice output of +``evals.curriculum_serve.practice.runner.seal_ledger`` is the only writer, and +its ``content_sha256`` is verified on load so a hand-edited ledger is rejected +rather than silently trusted. + +The third instance of the seal → ratify → SHA-verify → serve-gate pattern, and +the one that made the shared bridge worth extracting: this module is now an +ADAPTER over ``core.ratified_ledger`` (ADR-0263). It names the artifact, keeps +the memoization, and declares the one thing that genuinely differs — this +ledger is legitimately ABSENT, because no curriculum band has earned anything +yet (ADR-0262 §5.1: the binding constraint is ratified curriculum volume). An +absent ledger reads as an empty table, so every answer is served DISCLOSED. +""" + +from __future__ import annotations + +from functools import lru_cache +from pathlib import Path + +from core.ratified_ledger import ( + RatifiedLedgerError as RatifiedCurriculumLedgerError, + load_sealed_ledger, + serve_license, +) +from core.reliability_gate import Ceilings, ClassTally, LicenseDecision + +_LEDGER_PATH = Path(__file__).resolve().parent / "data" / "curriculum_serve_ledger.json" + + +@lru_cache(maxsize=1) +def load_ratified_ledger() -> dict[str, ClassTally]: + """Load + verify the ratified curriculum-serve ledger → per-band tallies. + + An ABSENT ledger is not an error here: no curriculum band has earned + anything yet, and the honest reading of "no file" is "no committed + evidence", which the gate turns into a disclosed answer rather than a + withheld one. + """ + return load_sealed_ledger(_LEDGER_PATH, missing_ok=True) + + +def curriculum_serve_license( + band: str, + *, + ledger: dict[str, ClassTally] | None = None, + ceilings: Ceilings | None = None, +) -> LicenseDecision | None: + """The ``Action.SERVE`` license for a curriculum band, or ``None``. + + ``None`` means the band has no committed evidence → never licensed; the + caller serves a disclosed (hedged) surface, the safe default. + """ + ledger = ledger if ledger is not None else load_ratified_ledger() + return serve_license(band, ledger, ceilings=ceilings) + + +__all__ = [ + "RatifiedCurriculumLedgerError", + "curriculum_serve_license", + "load_ratified_ledger", +] diff --git a/chat/curriculum_surface.py b/chat/curriculum_surface.py new file mode 100644 index 00000000..ea2ec0cc --- /dev/null +++ b/chat/curriculum_surface.py @@ -0,0 +1,361 @@ +"""chat/curriculum_surface.py — CURRICULUM composer (generalization arc Phase 2). + +The second question shape of the curriculum-entailment gold contract +(docs/plans/generalization-arc-2026-07-24.md §4.2b): an exam question that +supplies only the QUERY — "Does force cause acceleration?" — answered from the +ratified curriculum of the subject it is about, and from nothing else. + +The decision is made by the SAME argument bands the deduction composer uses +(ADR-0256…0261). This module compiles ratified chains into premise sentences, +appends the question as a "Therefore …" conclusion, and hands the assembled +argument to `deduction_grounded_surface`'s decider. There is no subject- +specific reasoning code anywhere in this path — physics differs from +philosophy only in which curriculum rows load, which is what makes "port the +lifecycle to a new subject" a data operation rather than an engineering one. + +Why an UNTAUGHT fact is UNKNOWN and never "no": the curriculum is read +OPEN-world. "The curriculum does not teach it" is not "it is false", and a +system that decoded the former into the latter would be inventing negative +knowledge it was never given. The only verdicts reachable from a purely +positive curriculum are therefore ENTAILED and UNKNOWN — a real limit of the +present corpora, recorded in ADR-0262 §5 rather than papered over. + +Fail-closed (INV-34): once `looks_like_curriculum_question` fires, every path +below returns a committed, honest surface — typed refusal or decided verdict — +never a silent fall-through to a different composer. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Callable + +from chat.curriculum_serve_license import curriculum_serve_license +from core.reliability_gate import LicenseDecision +from generate.proof_chain.entail import ( + INCONSISTENT_PREMISES, + Entailment, + evaluate_entailment_with_trace, +) +from generate.proof_chain.exist import ExistArgument, read_exist_argument +from generate.proof_chain.verb import VerbArgument, read_verb_argument, verb_forms_link +from teaching.curriculum_premises import ( + CONNECTIVE_FAMILY, + compile_premises, + load_curriculum, +) + +#: Subjects this composer serves, in a stable resolution order. A question is +#: routed to the subject whose ratified vocabulary contains BOTH of its terms; +#: more than one match is an `ambiguous_reading` refusal, never a guess. +SERVED_DOMAINS: tuple[str, ...] = ( + "physics", + "mathematics_logic", + "systems_software", + "philosophy_theology", +) + +#: The closed exam-question grammar: ``Does ?``. +#: Deliberately one shape. A question the grammar cannot read refuses typed — +#: the reader is never asked to guess which token is the relation. +_QUESTION_RE = re.compile(r"^\s*does\s+(.+?)\s*\?\s*$", re.IGNORECASE) + + +@dataclass(frozen=True, slots=True) +class CurriculumQuery: + """A read exam question: the edge the curriculum is being asked about.""" + + subject: str + verb: str + obj: str + + +@dataclass(frozen=True, slots=True) +class CurriculumRefusal: + """A typed refusal — the closed reason vocabulary of ADR-0262 §4.6.""" + + reason: str + detail: str = "" + + +def looks_like_curriculum_question(text: str) -> bool: + """True iff *text* is a ``Does …?`` polar question. + + A cheap, deterministic COMMIT gate — not a decision. The reader below + remains the sole authority on whether the question is actually readable. + """ + return bool(_QUESTION_RE.match(text or "")) + + +def read_curriculum_question(text: str) -> CurriculumQuery | CurriculumRefusal: + """Read *text* as ``Does ?``, or refuse (typed).""" + match = _QUESTION_RE.match(text or "") + if match is None: + return CurriculumRefusal("question_shape_out_of_band", text or "") + tokens = match.group(1).replace(",", " ").lower().split() + if len(tokens) != 3: + # Two-token ("does force act") and four-token ("does a force cause + # acceleration") shapes are genuinely different readings; the closed + # grammar refuses rather than dropping or inventing a slot. + return CurriculumRefusal("question_shape_out_of_band", " ".join(tokens)) + return CurriculumQuery(tokens[0], tokens[1], tokens[2]) + + +def resolve_domain(query: CurriculumQuery) -> str | CurriculumRefusal: + """The subject whose ratified vocabulary contains BOTH of the query's + terms — or a typed refusal (§4.6). + + ``untaught_vocabulary`` is the anti-recall boundary made mechanical: a + term no mounted pack teaches cannot be reasoned about here at all, however + familiar it is. ``ambiguous_reading`` fires when two subjects would both + claim the question; the composer declines rather than picking one. + """ + matches = [ + domain + for domain in SERVED_DOMAINS + if {query.subject, query.obj} <= load_curriculum(domain).vocabulary + ] + if not matches: + return CurriculumRefusal( + "untaught_vocabulary", f"{query.subject} / {query.obj}" + ) + if len(matches) > 1: + return CurriculumRefusal("ambiguous_reading", ", ".join(matches)) + return matches[0] + + +def resolve_family(query: CurriculumQuery) -> str | CurriculumRefusal: + """The relation family of the query's verb, under the SAME closed + agreement relation the verb-predicate band uses (ADR-0260) — so a question + may spell the relation in its base form ("cause") while the curriculum + states it in the third person ("causes").""" + for connective, family in CONNECTIVE_FAMILY.items(): + if verb_forms_link(query.verb, connective): + return family + return CurriculumRefusal("out_of_curriculum", query.verb) + + +def _query_sentence(query: CurriculumQuery, family: str) -> str: + """The conclusion sentence, spelled with the CURRICULUM's own connective + form so the reader's agreement linking has nothing to do — the question's + base form is normalized here, once, visibly.""" + for connective in CONNECTIVE_FAMILY: + if CONNECTIVE_FAMILY[connective] == family and verb_forms_link( + query.verb, connective + ): + return f"{query.subject} {connective} {query.obj}" + return f"{query.subject} {query.verb} {query.obj}" # pragma: no cover - guarded above + + +def band_for(domain: str, family: str) -> str: + """The capability band an answer is licensed under: *(subject × relation + family)* — ADR-0262 §4.""" + return f"curriculum_{domain}_{family}" + + +@dataclass(frozen=True, slots=True) +class CurriculumDecision: + """What the composer decided, before rendering — the lane scores this.""" + + verdict: str # entailed | refuted | unknown | declined + band: str # "" when the question never reached a band + reason: str # typed refusal reason, or "" when decided + domain: str = "" + family: str = "" + premise_count: int = 0 + + +def decide_curriculum_question(text: str) -> CurriculumDecision: + """Read, route, compile, and decide — the production decision path. + + Kept separate from the rendering below so the lane can pin the DECISION + without pinning prose (the same split ``evals/deduction_serve/runner.py`` + uses). + """ + query = read_curriculum_question(text) + if isinstance(query, CurriculumRefusal): + return CurriculumDecision("declined", "", query.reason) + domain = resolve_domain(query) + if isinstance(domain, CurriculumRefusal): + return CurriculumDecision("declined", "", domain.reason) + family = resolve_family(query) + if isinstance(family, CurriculumRefusal): + return CurriculumDecision("declined", "", family.reason, domain=domain) + + curriculum = load_curriculum(domain) + premises, _chain_ids = compile_premises(curriculum, family) + if not premises: + return CurriculumDecision( + "declined", "", "empty_curriculum", domain=domain, family=family + ) + conclusion = _query_sentence(query, family) + argument = ". ".join(premises) + f". Therefore {conclusion}." + + # The SAME argument bands decide it. The verb band reads these sentences + # (each is ` `); the existential band is tried + # after it exactly as in serving, so the cascade order here matches the + # deduction composer's tail and no subject-specific decider exists. + arg = read_verb_argument(argument) + if not isinstance(arg, VerbArgument): + arg = read_exist_argument(argument) + if not isinstance(arg, ExistArgument): + return CurriculumDecision( + "declined", "", "compiled_premises_unreadable", + domain=domain, family=family, premise_count=len(premises), + ) + trace = evaluate_entailment_with_trace(arg.premise_formulas, arg.query_formula) + verdict = { + Entailment.ENTAILED: "entailed", + Entailment.REFUTED: "refuted", + Entailment.UNKNOWN: "unknown", + Entailment.REFUSED: "declined", + }[trace.outcome] + reason = trace.reason if trace.outcome is Entailment.REFUSED else "" + return CurriculumDecision( + verdict, + band_for(domain, family), + reason, + domain=domain, + family=family, + premise_count=len(premises), + ) + + +_LicenseLookup = Callable[[str], LicenseDecision | None] + +_REFUSAL_SURFACE = { + "question_shape_out_of_band": ( + "That reads as a question about a subject I study, but not in a form I " + "can decide yet (I read \"Does ?\")." + ), + "untaught_vocabulary": ( + "I haven't been taught {detail} — so I can't decide anything about it " + "from what I know. That's a gap in my curriculum, not a claim about " + "the world." + ), + "out_of_curriculum": ( + "I haven't been taught the relation \"{detail}\", so I can't decide " + "that question from my curriculum." + ), + "ambiguous_reading": ( + "Those terms are taught in more than one subject I study ({detail}), " + "so I can't tell which curriculum you're asking about." + ), + "empty_curriculum": ( + "I have no ratified material for that kind of relation yet, so there " + "is nothing for me to decide it from." + ), + "compiled_premises_unreadable": ( + "I have the curriculum for that, but I can't read it back precisely " + "enough to decide the question yet." + ), + INCONSISTENT_PREMISES: ( + "The curriculum I have for that is inconsistent — I won't assert " + "anything from it until it is corrected." + ), +} +_DEFAULT_REFUSAL_SURFACE = ( + "I can't decide that from my curriculum as stated." +) + +#: Disclosed hedge for a band that has not earned SERVE — same posture as +#: the deduction composer (ADR-0256): the answer is sound, the track record +#: is not yet demonstrated, so it is served DISCLOSED rather than withheld. +_UNVERIFIED_SHAPE_DISCLOSURE = ( + "(answered from my curriculum, but I haven't yet earned a verified track " + "record on questions of this shape) " +) + + +def curriculum_grounded_surface( + text: str, *, license_lookup: _LicenseLookup = curriculum_serve_license, +) -> str | None: + """Return a deterministic CURRICULUM-tier surface, or ``None``. + + ``None`` only when *text* is not a ``Does …?`` question at all — the + caller then falls through to its pre-existing dispatch, byte-identical to + before this composer existed. + """ + if not looks_like_curriculum_question(text): + return None + decision = decide_curriculum_question(text) + query = read_curriculum_question(text) + # Restate the question in the CURRICULUM's own spelling of the relation — + # the normalization the reader performed is shown, not hidden. + shown = ( + _query_sentence(query, decision.family) + if isinstance(query, CurriculumQuery) and decision.family + else text.strip() + ) + if decision.verdict == "declined": + template = _REFUSAL_SURFACE.get(decision.reason, _DEFAULT_REFUSAL_SURFACE) + return template.format(detail=_detail_of(text, decision)) + if decision.verdict == "entailed": + surface = ( + f"Yes — my {decision.domain} curriculum teaches that {shown}, " + f"directly, among the {decision.premise_count} {decision.family} " + f"relations it states." + ) + elif decision.verdict == "refuted": + surface = ( + f"No — my {decision.domain} curriculum rules out that {shown}." + ) + else: + surface = ( + f"My {decision.domain} curriculum doesn't settle whether {shown}. " + f"It states {decision.premise_count} {decision.family} relations, " + f"and none of them decides this one — which is not the same as " + f"its being false." + ) + return _license_gate(surface, decision.band, license_lookup) + + +def _detail_of(text: str, decision: CurriculumDecision) -> str: + """The user-visible fragment a typed refusal names (empty when the reason + carries no detail slot).""" + query = read_curriculum_question(text) + if not isinstance(query, CurriculumQuery): + return "" + if decision.reason == "untaught_vocabulary": + untaught = [ + term + for term in (query.subject, query.obj) + if not any( + term in load_curriculum(d).vocabulary for d in SERVED_DOMAINS + ) + ] + return " or ".join(untaught) if untaught else f"{query.subject} / {query.obj}" + if decision.reason == "out_of_curriculum": + return query.verb + if decision.reason == "ambiguous_reading": + return ", ".join( + d + for d in SERVED_DOMAINS + if {query.subject, query.obj} <= load_curriculum(d).vocabulary + ) + return "" + + +def _license_gate(surface: str, band: str, license_lookup: _LicenseLookup) -> str: + """Serve *surface* authoritatively iff *band* holds a SERVE license; else + serve the same sound answer DISCLOSED (hedged).""" + decision = license_lookup(band) + if decision is not None and decision.licensed: + return surface + return _UNVERIFIED_SHAPE_DISCLOSURE + surface + + +__all__ = [ + "CurriculumDecision", + "CurriculumQuery", + "CurriculumRefusal", + "SERVED_DOMAINS", + "band_for", + "curriculum_grounded_surface", + "decide_curriculum_question", + "looks_like_curriculum_question", + "read_curriculum_question", + "resolve_domain", + "resolve_family", +] diff --git a/chat/deduction_serve_license.py b/chat/deduction_serve_license.py index 55397aac..2fcc2579 100644 --- a/chat/deduction_serve_license.py +++ b/chat/deduction_serve_license.py @@ -10,27 +10,27 @@ artifact is the sealed-practice output of is verified on load, so a hand-edited (un-ratified) ledger is rejected rather than silently trusted. -Mirrors ``generate.determine.estimation_license`` exactly: immutable ratified -data parsed once and cached; the gate (``license_for``) is pure; ceilings stay -at the safe defaults (invariant #4 — the engine cannot raise its own bar). +The load/verify/gate mechanics live in ``core.ratified_ledger`` (ADR-0263) — +this module is the deduction-serve ADAPTER over that bridge: it names the +artifact, keeps the memoization, and preserves its own public API. Ceilings +stay at the safe defaults (invariant #4 — the engine cannot raise its own bar). """ from __future__ import annotations -import json from functools import lru_cache from pathlib import Path -from core.reliability_gate import Action, Ceilings, ClassTally, LicenseDecision, license_for -from formation.hashing import sha256_of +from core.ratified_ledger import ( + RatifiedLedgerError, + load_sealed_ledger, + serve_license, +) +from core.reliability_gate import Ceilings, ClassTally, LicenseDecision _LEDGER_PATH = Path(__file__).resolve().parent / "data" / "deduction_serve_ledger.json" -class RatifiedLedgerError(ValueError): - """The committed deduction-serve ledger is missing, malformed, or tampered with.""" - - @lru_cache(maxsize=1) def load_ratified_ledger() -> dict[str, ClassTally]: """Load + verify the ratified deduction-serve ledger → per-band ``ClassTally``. @@ -39,30 +39,7 @@ def load_ratified_ledger() -> dict[str, ClassTally]: recomputed ``content_sha256`` does not match the committed one (tamper-evidence: only the sealed-practice output is trusted, never a hand-edited ledger). """ - try: - artifact = json.loads(_LEDGER_PATH.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: # pragma: no cover - defensive - raise RatifiedLedgerError(f"cannot read ratified ledger: {exc}") from exc - - classes = artifact.get("classes") - if not isinstance(classes, dict): - raise RatifiedLedgerError("ratified ledger has no 'classes' table") - if sha256_of(classes) != artifact.get("content_sha256"): - raise RatifiedLedgerError( - "ratified ledger content_sha256 mismatch — not the sealed-practice output" - ) - - ledger: dict[str, ClassTally] = {} - for cls, counts in classes.items(): - ledger[cls] = ClassTally( - class_name=cls, - correct=int(counts.get("correct", 0)), - wrong=int(counts.get("wrong", 0)), - refused=int(counts.get("refused", 0)), - t2_verified=int(counts.get("t2_verified", 0)), - t2_agrees_gold=int(counts.get("t2_agrees_gold", 0)), - ) - return ledger + return load_sealed_ledger(_LEDGER_PATH) def deduction_serve_license( @@ -79,11 +56,7 @@ def deduction_serve_license( safe default ceilings. """ ledger = ledger if ledger is not None else load_ratified_ledger() - tally = ledger.get(shape_band) - if tally is None: - return None - ceilings = ceilings if ceilings is not None else Ceilings.default() - return license_for(tally, Action.SERVE, ceilings) + return serve_license(shape_band, ledger, ceilings=ceilings) __all__ = ["RatifiedLedgerError", "deduction_serve_license", "load_ratified_ledger"] diff --git a/chat/runtime.py b/chat/runtime.py index 0122d8dd..d49df225 100644 --- a/chat/runtime.py +++ b/chat/runtime.py @@ -1757,6 +1757,22 @@ class ChatRuntime: return (deduction_surface, "deduction", ()) if attempts is not None: attempts.append(DispatchAttempt(source="deduction", outcome="skipped", reason="not_argument_shaped")) + # Generalization arc, Phase 2 (ADR-0262) — curriculum-grounded exam + # questions, checked beside deduction serving for the same reason: a + # pure function of the input text and the ratified curriculum, with no + # vault/field dependency, so warm/cold state cannot change the answer. + if self.config.curriculum_serving_enabled: + from chat.curriculum_surface import curriculum_grounded_surface + from generate.intent import DialogueIntent, IntentTag as _IntentTag + + curriculum_surface = curriculum_grounded_surface(text) + if curriculum_surface is not None: + self._last_intent = DialogueIntent(tag=_IntentTag.DEDUCTION, subject=text) + if attempts is not None: + attempts.append(DispatchAttempt(source="curriculum", outcome="admitted", reason="curriculum_composer_committed")) + return (curriculum_surface, "curriculum", ()) + if attempts is not None: + attempts.append(DispatchAttempt(source="curriculum", outcome="skipped", reason="not_curriculum_question")) if not allow_warm and gate_source != "empty_vault": if attempts is not None: for src in ("pack", "teaching", "partial", "oov"): diff --git a/core/cli_test.py b/core/cli_test.py index e66a95ab..a804ae4f 100644 --- a/core/cli_test.py +++ b/core/cli_test.py @@ -183,6 +183,8 @@ TEST_SUITES: dict[str, tuple[str, ...]] = { "tests/test_verb_argument_reader.py", "tests/test_exist_argument_reader.py", "tests/test_deduction_serve_e2e.py", + "tests/test_curriculum_serve.py", + "tests/test_ratified_ledger_bridge.py", ), "full": ("tests/",), } diff --git a/core/config.py b/core/config.py index 8922dd1c..30ce8c18 100644 --- a/core/config.py +++ b/core/config.py @@ -386,6 +386,21 @@ class RuntimeConfig: # (hedged). OFF by default: flag-off is byte-identical to pre-arc dispatch. deduction_serving_enabled: bool = False + # Generalization arc, Phase 2 (ADR-0262) — when on, ``chat/curriculum_surface.py`` + # answers exam-shaped polar questions ("Does force cause acceleration?") from the + # RATIFIED domain-chain curriculum of the subject the question's vocabulary + # belongs to, decided by the same argument bands the deduction composer uses. + # The curriculum is read OPEN-world: a relation it does not teach is UNKNOWN, + # never "no", so an untaught fact can never be decoded into a negative claim. + # Like ``deduction_serving_enabled`` this is an enable for an EARNED path, not a + # direct-serve switch — an answer is served AUTHORITATIVELY only when its + # (subject x relation family) band holds a SERVE license on the committed, + # SHA-sealed curriculum ledger; an unearned band is served DISCLOSED (hedged), + # which is the state of every band today (ADR-0262 SS5: the binding constraint is + # ratified curriculum volume, not machinery). OFF by default: flag-off is + # byte-identical to pre-arc dispatch. + curriculum_serving_enabled: bool = False + # ASK serving gate enable flag. When True, ASK serving is allowed. # Default False (dark). ask_serving_enabled: bool = False diff --git a/core/ratified_ledger.py b/core/ratified_ledger.py new file mode 100644 index 00000000..0bc55061 --- /dev/null +++ b/core/ratified_ledger.py @@ -0,0 +1,157 @@ +"""The ratified-ledger bridge — seal → ratify → SHA-verify → serve-gate. + +ADR-0175 Phase 5's consumption bridge (generalization plan Phase 3.3), +extracted from three working instances rather than designed ahead of them: + +- ``generate/determine/estimation_license.py`` (ADR-0175, the first) +- ``chat/deduction_serve_license.py`` (ADR-0256, the second) +- ``chat/curriculum_serve_license.py`` (ADR-0262, the third) + +All three had converged on the same artifact and the same four rules, which is +what makes this an extraction and not a speculation. The rules, stated once: + +1. **The engine reads; only sealed practice writes.** A ledger is the output + of a practice run over a gold corpus, never of a serving turn. Nothing in a + serving path may call :func:`write_sealed_ledger`. +2. **Tamper-evidence is structural.** The artifact carries + ``content_sha256`` over its ``classes`` table; a load that cannot reproduce + it REFUSES. A hand-edited ledger is not a slightly-wrong ledger, it is an + unratified one. +3. **Ceilings are not negotiable at the call site.** The gate always runs at + the safe defaults unless a caller passes ceilings explicitly, and no + production path does — ADR-0175 invariant #4: an engine cannot raise its own + bar. +4. **Absent evidence is never a license.** A class missing from the ledger + yields ``None``, and every caller's ``None`` branch serves the disclosed + (hedged) surface. A capability with no track record is served honestly, not + withheld and not asserted. + +Byte-compatibility is deliberate: :func:`seal_artifact` and +:func:`write_sealed_ledger` reproduce the exact bytes the three existing +sealers wrote, so adopting the bridge re-seals every committed ledger +identically and no lane pin moves. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from core.reliability_gate import ( + Action, + Ceilings, + ClassTally, + LicenseDecision, + license_for, +) +from formation.hashing import sha256_of + + +class RatifiedLedgerError(ValueError): + """A committed ledger is malformed or does not verify against its own hash.""" + + +def tally_dict(tally: ClassTally) -> dict[str, Any]: + """The committed per-class row. The field set is the contract — a reader + of an older ledger must be able to name every field it finds.""" + return { + "correct": tally.correct, + "wrong": tally.wrong, + "refused": tally.refused, + "t2_verified": tally.t2_verified, + "t2_agrees_gold": tally.t2_agrees_gold, + } + + +def seal_artifact( + ledger: dict[str, ClassTally], *, schema: str, note: str, provenance: str +) -> dict[str, Any]: + """The self-verifying sealed-ledger dict for *ledger*. + + Classes are sorted, so the artifact is a pure function of the practice + result: the same corpus and solver seal byte-identically, which is what + makes a committed ledger reviewable as a diff. + """ + classes = {name: tally_dict(tally) for name, tally in sorted(ledger.items())} + return { + "schema": schema, + "classes": classes, + "content_sha256": sha256_of(classes), + "note": note, + "provenance": provenance, + } + + +def write_sealed_ledger(path: Path, artifact: dict[str, Any]) -> dict[str, Any]: + """Write *artifact* to *path* in the committed formatting.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(artifact, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return artifact + + +def load_sealed_ledger(path: Path, *, missing_ok: bool = False) -> dict[str, ClassTally]: + """Load + verify a sealed ledger → per-class ``ClassTally``. + + ``missing_ok`` distinguishes two genuinely different situations. A ledger + that a capability *ships with* is required: its absence means the + deployment is broken, and refusing is right. A ledger for a capability + whose practice volume is still being built is legitimately absent, and the + honest reading of "no file" is "no class has earned anything yet" — an + empty table, every answer disclosed. Neither case may be answered by + guessing a license. + """ + if not path.exists(): + if missing_ok: + return {} + raise RatifiedLedgerError(f"ratified ledger not found: {path}") + try: + artifact = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise RatifiedLedgerError(f"cannot read ratified ledger: {exc}") from exc + + classes = artifact.get("classes") if isinstance(artifact, dict) else None + if not isinstance(classes, dict): + raise RatifiedLedgerError("ratified ledger has no 'classes' table") + if sha256_of(classes) != artifact.get("content_sha256"): + raise RatifiedLedgerError( + "ratified ledger content_sha256 mismatch — not the sealed-practice output" + ) + + return { + name: ClassTally( + class_name=name, + correct=int(counts.get("correct", 0)), + wrong=int(counts.get("wrong", 0)), + refused=int(counts.get("refused", 0)), + t2_verified=int(counts.get("t2_verified", 0)), + t2_agrees_gold=int(counts.get("t2_agrees_gold", 0)), + ) + for name, counts in classes.items() + } + + +def serve_license( + class_name: str, + ledger: dict[str, ClassTally], + *, + ceilings: Ceilings | None = None, +) -> LicenseDecision | None: + """The ``Action.SERVE`` verdict for *class_name*, or ``None`` when the + class has no committed evidence (never a license — rule 4).""" + tally = ledger.get(class_name) + if tally is None: + return None + return license_for(tally, Action.SERVE, ceilings or Ceilings.default()) + + +__all__ = [ + "RatifiedLedgerError", + "load_sealed_ledger", + "seal_artifact", + "serve_license", + "tally_dict", + "write_sealed_ledger", +] diff --git a/docs/adr/ADR-0262-curriculum-grounded-serving.md b/docs/adr/ADR-0262-curriculum-grounded-serving.md new file mode 100644 index 00000000..959bad43 --- /dev/null +++ b/docs/adr/ADR-0262-curriculum-grounded-serving.md @@ -0,0 +1,164 @@ +# ADR-0262 — Curriculum-grounded serving: exams answered from what was taught + +- **Status:** Proposed +- **Date:** 2026-07-24 +- **Arc:** generalization Phase 2 (docs/plans/generalization-arc-2026-07-24.md §4) +- **Governs:** `teaching/curriculum_premises.py`, `chat/curriculum_surface.py`, + `chat/curriculum_serve_license.py`, the `curriculum__` bands, + the `evals/curriculum_serve` lane and its independent oracle, and the + `curriculum_serving_enabled` flag. + +## 1. Context + +The band cascade (ADR-0256 → 0261) decides arguments whose premises are IN the +text. Exams are not shaped like that: the question supplies only the query, and +the premises are supposed to be what the student was taught. Until now CORE had +no path from "a ratified curriculum exists" to "a question is answered from +it" — the 16 domain seed packs and 5 domain-chain corpora were inert. + +The generalization plan §4 fixes the epistemology for that path in advance, so +that implementing it cannot quietly become "answer from what the model knows". +This ADR implements §4. + +## 2. Decision + +Add a second serving composer, `chat/curriculum_surface.py`, behind the +default-off `curriculum_serving_enabled` flag: + +1. **Closed question grammar** — `Does ?`. One shape. + Anything else refuses `question_shape_out_of_band` rather than guessing + which token is the relation. +2. **Subject routing by vocabulary** — the question goes to the subject whose + ratified pack vocabulary contains BOTH terms. No match is + `untaught_vocabulary`; more than one is `ambiguous_reading`. The subject is + never guessed. +3. **Premise compilation from the ratified curriculum only** + (`teaching/curriculum_premises.py`). A chain is admitted iff + `review_status == "reviewed"` AND both terms are resident in the packs the + chain declares. Compilation is scoped to the question's relation family. +4. **The same decider** — compiled premises plus the question as a "Therefore" + conclusion are handed to the verb-predicate band (ADR-0260), falling through + to the existential band (ADR-0261), then the ROBDD engine. **No + subject-specific decision code exists anywhere in this path.** +5. **Bands = (subject × relation family)**, e.g. `curriculum_physics_causal`, + gated by the same earned-license machinery as deduction serving through a + second ledger reader (`chat/curriculum_serve_license.py`). + +The relation family comes from the CONNECTIVE, not from a chain row's declared +`operator_family`. A question carries a relation word and nothing else; a +family derived from anything the question cannot carry would let premise +compilation and question routing disagree, and a taught edge could then be +missing from the premises compiled to decide it — a wrong answer built out of a +correct curriculum. Two corpus rows currently declare a family their connective +does not imply; under this rule they are read as their connective says. + +## 3. Why this is sound + +1. **Gold is a function of (curriculum, question).** No case-local hidden text, + no world knowledge. A lane case pins the chain ids it draws on and the + runner FAILS the case if a pinned chain is absent or unratified — so a + curriculum that changes under a case breaks loudly instead of quietly + answering from what remains. +2. **Open-world reading.** A relation the curriculum does not state is + UNKNOWN, never "no". Deciding otherwise would manufacture negative + knowledge from silence, which is the same failure as manufacturing positive + knowledge from silence. +3. **No composition.** The curriculum teaches `force causes acceleration` and + `acceleration causes motion`; it does not teach that causation composes, so + `force causes motion` is UNKNOWN. Transitivity of a causal or modal relation + is a substantive claim about the world, not a logical truth, and nothing + ratified asserts it. The independent oracle reports the shortest path length + with its verdict precisely so the lane can assert that a reachable pair is + still answered UNKNOWN — composition is provably not happening rather than + merely absent. +4. **Face-value relations.** `entropy reveals energy` does not license + `entropy causes energy`. Same family, different relation: the reader mints + distinct atoms per verb group (ADR-0260), so a family-level near-miss cannot + become an entailment. +5. **Independent gold.** `evals/curriculum_serve/oracle.py` shares no code with + the serving path — its own JSONL loader, ratification predicate, family + table, agreement normalization, and verdict rule. It disagreed with the + serving path once during authoring, on exactly the point in (4), and the + ORACLE was the side that was wrong; that disagreement is why the check + exists. +6. **Anti-recall is a lane guard, not a hope.** A split must carry ≥3 probes + whose answer is true in the world and absent from the curriculum, and no + probe may carry a committed gold, or the lane refuses to run. + +## 4. License posture + +Every `curriculum_*` band is UNEARNED today, so every answer is served +DISCLOSED through the same hedge deduction serving uses. This is not a +temporary shortcut around the license — it is the license working. The reason +is §5.1. + +## 5. Findings + +### 5.1 The binding constraint on Phase 2 is ratified curriculum VOLUME + +A band earns SERVE at θ_SERVE=0.99 only with n ≥ 657 committed cases and a +genuine outcome mix. The question space of a subject is +`|vocabulary|² − |vocabulary|`, so volume is reachable (physics: 240, +philosophy_theology: 22,650). **Entailed** cases are not: physics teaches 7 +causal and 9 modal relations, so at most 16 distinct questions in the whole +subject can come back ENTAILED. Sampling cannot fix this — repeating the same +16 questions inflates n without adding information, and a band whose entailed +class is 1% of its volume is passed by a pipeline that never entails anything. + +Concretely: a balanced band (≈⅓ entailed) needs **≈219 distinct taught +relations per subject × family**. Physics has 7 and 9. That is a ~25× gap, and +it is a curriculum-authoring and ratification task — Shay's call, not an +engineering one. + +The honest consequence, recorded rather than worked around: the machinery, +the lane, the oracle, and the license gate all ship and pass; the licenses do +not exist yet; every answer is hedged until they do. The plan's O→S checkpoint +("≥1 subject lane serving wrong=0 with earned licenses") is met on every clause +except the last, and the last one cannot be met from present data. + +### 5.2 REFUTED is unreachable from present corpora + +Every ratified chain is a positive assertion. Refuting a question needs the +curriculum to teach a negative ("no X causes Y") or an exclusion. Until a +corpus does, `refuted` is a verdict this path can render but never reach. The +serving code handles it; the lane documents it. + +### 5.3 There is no biology domain-chain corpus + +The plan names "physics and biology" as Phase 2's two subjects because "OOD +lanes, seed packs, and domain chains already exist". For biology only the first +two are true: `evals/foundational_biology_ood` is a **fluency** lane (its own +contract says it measures grammatical English, not truth) and there is no +`biology_chains_*.jsonl`. The four subjects with ratified chains AND mounted +packs are physics, mathematics_logic, systems_software, and +philosophy_theology; the composer serves all four. The second subject should be +chosen from those, or biology's chain corpus authored first. + +## 6. Scope-outs + +- **Question shapes** beyond `Does ?` — multi-word + terms, "why"/"how" questions, negated questions, and quantified questions all + refuse typed. Each is a separate readable unit with its own lane. +- **Composition**, per §3.3 — a future ADR may license it for a family the + curriculum itself declares transitive, but that requires the declaration to + be an executable claim rather than a classification label, which today it is + not. +- **Premise-set scaling** — a family with more than `MAX_PREMISE_SENTENCES` + (16) ratified chains compiles to an argument the reader refuses, surfacing as + `compiled_premises_unreadable`. No current family exceeds 9. The fix when one + does is to compile the query's k-hop neighborhood rather than the whole + family (sound: fewer premises can lose an entailment, never create one), and + to disclose the narrowing. + +## 7. Verification + +- `tests/test_curriculum_serve.py` — 20 tests: premise provenance and the + loud-failure guard, family scoping never losing a taught edge, taught-edge + entailment, untaught composition staying UNKNOWN, untaught fact never + REFUTED, relation read at face value, base-form question normalization, + anti-recall declines, 3 typed refusals, non-question pass-through, routing, + the disclosed-until-earned license posture and the authoritative path when a + band IS licensed, and oracle independence including the depth-2 reachability + assertion. +- `evals/curriculum_serve` lane: 32/32, wrong=0, 5 anti-recall probes, both + physics bands exercised. Pinned as `curriculum_serve_v1`. diff --git a/docs/adr/ADR-0263-ratified-ledger-bridge.md b/docs/adr/ADR-0263-ratified-ledger-bridge.md new file mode 100644 index 00000000..7d4536d8 --- /dev/null +++ b/docs/adr/ADR-0263-ratified-ledger-bridge.md @@ -0,0 +1,80 @@ +# ADR-0263 — The ratified-ledger bridge + +- **Status:** Proposed +- **Date:** 2026-07-24 +- **Arc:** generalization Phase 3.3 (docs/plans/generalization-arc-2026-07-24.md §2) +- **Governs:** `core/ratified_ledger.py` and the three adapters over it — + `generate/determine/estimation_license.py`, `chat/deduction_serve_license.py`, + `chat/curriculum_serve_license.py` — plus the sealed-artifact writer in + `evals/deduction_serve/practice/runner.py`. + +## 1. Context + +Three capabilities had independently converged on the same four-step pattern: +sealed practice writes a per-class tally table → the table is committed as a +ratified artifact → serving verifies its `content_sha256` on load → a per-class +gate decides SERVE against fixed ceilings. + +Three near-identical implementations is the point at which a shared component +is an extraction rather than a guess. It matters now because the generalization +plan puts a per-subject arena behind every new subject: without a bridge, each +subject copies fifty lines of verification, and the copies drift. + +## 2. Decision + +`core/ratified_ledger.py` owns the mechanics — `seal_artifact`, +`write_sealed_ledger`, `load_sealed_ledger`, `serve_license`, `tally_dict` — +and states the four rules once: + +1. **The engine reads; only sealed practice writes.** +2. **Tamper-evidence is structural** — a load that cannot reproduce + `content_sha256` REFUSES. A hand-edited ledger is not a slightly-wrong + ledger; it is an unratified one. +3. **Ceilings are not negotiable at the call site** (ADR-0175 invariant #4). +4. **Absent evidence is never a license** — a missing class yields `None`, and + every caller's `None` branch serves the disclosed surface. + +Each capability keeps a thin adapter that names its artifact, keeps its +memoization, and preserves its public API. One genuine difference is now +declared rather than implied: `load_sealed_ledger(..., missing_ok=True)` +distinguishes a ledger a capability *ships with* (absence = broken deployment, +refuse) from one whose practice volume is still being built (absence = no class +has earned anything, serve disclosed). Curriculum serving is the second kind +today (ADR-0262 §5.1). + +## 3. Why this is safe + +The extraction's safety property is byte-identity: `seal_artifact` and +`write_sealed_ledger` reproduce exactly what the bespoke sealers wrote, so +re-sealing through the bridge leaves every committed artifact and every lane +pin unmoved. That is asserted, not assumed — +`test_committed_deduction_ledger_reseals_byte_identically` re-derives the +committed 25-band ledger through the bridge and compares the bytes. + +## 4. What this unblocks + +A new subject arena now needs a gold corpus and a band key. It does not need to +re-implement ratification — which is what the plan meant by "so each subject +arena plugs in without bespoke wiring", and why §3 sequenced this ahead of the +second subject. + +## 5. Scope-outs + +- **The estimation adapter keeps its predicate → converse-class naming.** That + mapping is genuinely local to estimation; the bridge gates a class name and + does not know how one is derived. +- **No ledger migration.** All three artifacts already share the schema shape; + nothing is rewritten, versioned, or moved. +- **`core/learning_arena`'s practice engine is untouched.** The bridge covers + the consumption half (seal → serve); the production half (run practice, tally) + is already shared. + +## 6. Verification + +`tests/test_ratified_ledger_bridge.py` — 8 tests: round-trip, hand-edit +rejection, required-vs-optional absence, absent class never licensed, the +Wilson volume floor, a single `wrong` costing the license, committed-ledger +byte-identity, and a guard that all three adapters read through the bridge +(they no longer import a hashing module of their own). Plus the existing +consumers unchanged: `core test --suite deductive` 252 passed; the estimation +and license test set 355 passed. diff --git a/docs/handoff/generalization-2026-07/BRIEF-S-sonnet-low-tier.md b/docs/handoff/generalization-2026-07/BRIEF-S-sonnet-low-tier.md index c28170bc..aa4d6ced 100644 --- a/docs/handoff/generalization-2026-07/BRIEF-S-sonnet-low-tier.md +++ b/docs/handoff/generalization-2026-07/BRIEF-S-sonnet-low-tier.md @@ -1,47 +1,136 @@ # Handoff Brief — Tier S (Sonnet 5, LOW risk) -**Read first:** `docs/plans/generalization-arc-2026-07-24.md` §6 -(constraint set is binding) and the Tier-O brief's constraints block — -they apply verbatim. Opus will have updated this brief with actual state -at its exit checkpoint; trust that update over anything stale below. +**Read first:** `docs/plans/generalization-arc-2026-07-24.md` §6 (constraint +set is binding) and the Tier-O brief's constraints block — they apply verbatim. -**You own (independent items, any order unless noted):** +--- + +## STATE AT THE OPUS → SONNET CHECKPOINT (updated 2026-07-24) + +**Tier O is complete.** Four units, all committed, all with local gates green: + +| unit | branch @ commit | what | +|---|---|---| +| O1 | `feat/existential-band` @ `e84c0e84` | Band v6-EX existentials (ADR-0261) + a wrong-answer fix in Band v1b | +| O2 | `feat/curriculum-serve-physics` @ `44e78aa4` | curriculum-grounded serving (ADR-0262), physics lane 32/32 wrong=0 | +| O3 | same branch @ `0a17c496` | ratified-ledger bridge (ADR-0263) | +| O4 | same branch @ `353a52b8` | math Phase 4.1 measured status — null result, documented | + +Branch stack (each on the previous): `main` → `feat/verb-predicate-band` +(PR #111, Shay's, still open) → `feat/existential-band` → +`feat/curriculum-serve-physics`. + +### ⛔ BLOCKER — pushes to core-labs/core are failing server-side + +The Forgejo repo has a **corrupt object**: `refs/heads/feat/verb-predicate-band` +(= PR #111 head, `e41a3e2a`) points at a **zero-length object file** on the +server. Symptoms: + +``` +remote: error: object file .../objects/e4/1a3e2a... is empty +remote: fatal: bad object refs/heads/feat/verb-predicate-band +! [remote rejected] (missing necessary objects) +``` + +This blocks **every** push to the repo, not just the Opus branches — the +server fails its own connectivity check before accepting any ref. Reads +(`git fetch`, `ls-remote`, MCP) still work. Nothing local is damaged: the +identical content exists locally as `d181ae30`. + +Repair needs filesystem access on the Forgejo VM (remove the empty object, +then reset or delete `refs/heads/feat/verb-predicate-band` and +`refs/pull/111/head`, `git fsck`). **This is Shay's call — do not delete or +force-push someone else's PR branch.** Until it is fixed, work locally and +commit; push when the server is repaired. + +### Corrections to the plan's ground truth (verified — act on these) + +1. **There is no biology domain-chain corpus.** `evals/foundational_biology_ood` + is a *fluency* lane (its own contract says so) and no `biology_chains_*.jsonl` + exists. The subjects with ratified chains AND mounted packs are physics, + mathematics_logic, systems_software, philosophy_theology. Pick the second + subject from those, or author biology's chains first. +2. **Phase 4.1 (seeding-sentence injection) was already built** before this + arc and converts 0 cases; the frozen-500 baseline re-measured today is + `correct=5 wrong=0 refused=495`. See + `docs/research/math-reader-phase-4-1-status-2026-07-24.md`. Phase 4 should + be re-pointed at the reader arc's own live recommendation (increment-2 + case-first on cases 0000/0001/0148/0082). +3. **No curriculum band can earn a SERVE license from present data.** A band + needs n≥657 with a real outcome mix; physics teaches 7 causal + 9 modal + relations, so at most 16 questions in the whole subject can ever come back + ENTAILED. A balanced band needs ≈219 taught relations per subject × family. + Phase 2's blocker is **ratified curriculum volume**, not machinery + (ADR-0262 §5.1). Every curriculum answer is served DISCLOSED until that + changes — which is the license working, not a workaround. + +--- + +## You own (independent items, any order unless noted) 1. **S1 — ratification packet for `deduction_serving_enabled`.** Evidence collation only — the flip is Shay's decision, never yours. - Contents: 17+-band sealed ledger stats - (`chat/data/deduction_serve_ledger.json`), serve-lane results - (`evals/deduction_serve/report.json`), byte-identical-when-off proof - (cite ADR-0256 + `tests/test_deduction_surface.py` flag tests), blast - radius (exact dispatch point `chat/runtime.py` deduction branch), and - rollback (flip back, zero residue). One markdown doc in - `docs/research/`, PR'd. -2. **S2 — lane re-pins + flake documentation.** Only AFTER the Phase-0.1 - drift findings doc exists and says re-pin is safe. Surgical - single-line edits in `scripts/verify_lane_shas.py` — NEVER `--update` - (it rewrites every lane and silently drops erroring lanes' pins). - Document the `public_demo` env-timeout flake where the findings doc - says. -3. **S3 — vocab-trigger instrument.** Implement the measurable test - specified in `docs/handoffs/COMPREHENSION-READER-AUDIT.md` - (§measurable-test, lines ~163–166 and ~231–238): refusal histogram - split mechanism-vs-coverage + Phase-2-admissions-per-lexicon-batch - counter. CLI or lane runner per the spec; do not invent policy — the - spec decides, you implement. -4. **S4 — HITL proposal-queue CLI.** A `core` CLI surface listing + - reviewing pending proposals from the existing sinks - (`teaching/proposals/`, contemplation/idle sinks). Read + review-state - transitions only; NO ratification automation, NO corpus mutation, NO - flag flips. -5. **S5 — housekeeping.** Promotion sweeps Opus's checkpoint notes call - for; capability-index entries for new subject lanes; docs/memory - updates; dead-code removal only when unambiguous. + Contents: the **25-band** sealed ledger stats + (`chat/data/deduction_serve_ledger.json`, all 25 at 720/720 wrong=0), + serve-lane results (`evals/deduction_serve/report.json`, **166/166 across + six splits**), byte-identical-when-off proof (cite ADR-0256 + + `tests/test_deduction_surface.py` flag tests), blast radius (the deduction + branch in `chat/runtime.py`), and rollback (flip back, zero residue). + **Add a second section for `curriculum_serving_enabled`** (ADR-0262): same + shape, noting explicitly that every curriculum band is unearned, so the + flip enables a *disclosed* capability. One markdown doc in `docs/research/`. -**Constraints:** identical to Tier O (worktrees, fresh venv, pre-push -gate, compare-URL PRs, Shay merges, wrong=0, flags stay off, no -timelines). When any item's scope turns out larger than described here — -stop and flag it in the PR/handoff notes rather than improvising. +2. **S2 — lane re-pins + the `public_demo` drift.** The Phase-0.1 findings doc + exists (`docs/research/lane-drift-investigation-2026-07-24.md`). + **Correct the standing note: `public_demo` is NOT a timing flake here.** It + reproduces deterministically. Measured this session: + - pinned: `7d8ba0db…` (last set at commit `7f6c497a`, long before this arc) + - content with the register-axis fix reverted: `c6596e11…` + - content today: `da7fad65…` (3 identical runs, incl. on the base commit) -**Arc close:** all Tier-S PRs pushed; memory updated; note whether the -Phase-5 articulation trigger (multi-step decided content exists) has -fired, for the next arc's scoping. + So there are **two** drifts: one predating PR #110 (unexplained — find it + before re-pinning) and one caused by PR #110's register-axis fix (expected, + intended). `all_passed` stays `true` throughout; only the demo payload + digest moved. Do the archaeology, then re-pin surgically — **never + `--update`** (it rewrites every lane and silently drops erroring lanes' + pins). Three arcs now depend on that rule. + +3. **S3 — vocab-trigger instrument.** Implement the measurable test specified + in `docs/handoffs/COMPREHENSION-READER-AUDIT.md` (§measurable-test): + refusal histogram split mechanism-vs-coverage + admissions-per-lexicon-batch + counter. Two ready-made feeds now exist: the curriculum path's + `untaught_vocabulary` / `out_of_curriculum` refusals + (`chat/curriculum_surface.py`) are literally coverage-class refusals, and + the deduction bands' typed refusals are mostly mechanism-class. Do not + invent policy — the spec decides, you implement. + +4. **S4 — HITL proposal-queue CLI.** A `core` CLI surface listing + reviewing + pending proposals from the existing sinks (`teaching/proposals/`, + contemplation/idle sinks). Read + review-state transitions only; NO + ratification automation, NO corpus mutation, NO flag flips. + +5. **S5 — housekeeping.** + - Add `tests/test_register_substantive_consumption.py`'s e2e tests to the + curated smoke suite (a red test outside every gate is a silent red — the + lesson from the #96 regression). + - Capability-index entries for the new lanes (`curriculum_serve_v1`, the + `v2_exist` split). + - Promotion sweep: `ds-mem-0020` was promoted declined→unknown by v6-EX + (already done); check nothing else in the older splits is now decidable. + - Memory/doc updates; dead-code removal only when unambiguous. + +6. **S6 (new) — quantify the curriculum-volume ask.** From ADR-0262 §5.1: + produce a short doc stating, per served subject × relation family, how many + ratified chains exist and how many a balanced 657-case band would need. + Evidence only — authoring or ratifying curriculum content is Shay's. + +**Constraints:** identical to Tier O (worktrees, fresh venv, pre-push gate, +compare-URL PRs, Shay merges, wrong=0, flags stay off, no timelines). When any +item's scope turns out larger than described here — stop and flag it in the +PR/handoff notes rather than improvising. + +**Arc close:** all Tier-S PRs pushed; memory updated; note whether the Phase-5 +articulation trigger (multi-step decided content exists) has fired. Evidence +for it now exists on both sides: the deduction bands produce multi-premise +decided chains, and the curriculum path produces answers that state how much +curriculum they were decided from. diff --git a/docs/research/curriculum-grounded-serving-2026-07-24.md b/docs/research/curriculum-grounded-serving-2026-07-24.md new file mode 100644 index 00000000..0b751a63 --- /dev/null +++ b/docs/research/curriculum-grounded-serving-2026-07-24.md @@ -0,0 +1,109 @@ +# Curriculum-grounded serving — the first exam answered from what was taught + +**Date:** 2026-07-24 · **ADR:** ADR-0262 · **Arc:** generalization Phase 2 (Tier O) + +## What shipped + +"Does force cause acceleration?" is now answered — from the ratified physics +chain corpus and nothing else. "Does gravity cause acceleration?" is declined, +because *gravity* is not in any pack CORE has been taught. "Does force cause +motion?" comes back unsettled, even though the curriculum contains both links +in that chain, because nothing ratified says causation composes. + +That third answer is the one worth staring at. It is the difference between a +system that decodes a curriculum and one that pattern-matches over it. + +The lane: 32 hand-authored physics cases, wrong=0, five anti-recall probes, an +independent oracle re-deriving every gold, and three contract guards that fail +the LANE (not the case) if provenance, soundness, or anti-recall coverage +lapses. 20 unit tests. Zero subject-specific decision code: the composer routes +by vocabulary, compiles premises from ratified chains, and hands the result to +the same argument bands that decide "All philosophers teach. Socrates is a +philosopher." + +## The design decision that carried the weight + +The plan fixed the epistemology in §4 before any code existed, and that turned +out to matter more than any implementation choice. Three of its clauses did +real work: + +- **Gold is a function of (curriculum, question).** No hidden premises in the + case file. A case pins chain ids; the runner reconstructs premises from the + corpus and fails loudly if a pinned chain is absent or unratified. A + curriculum that moves under a case breaks the case instead of quietly + changing its answer. +- **Untaught ⇒ UNKNOWN, never "no".** Open-world reading. Manufacturing a + negative from silence is the same error as manufacturing a positive from it, + and it is the more tempting one because it feels like rigor. +- **Anti-recall probes are mandatory.** The runner refuses to score a split + that does not contain questions whose answers are true in the world and + absent from the curriculum. Without them the lane cannot distinguish + decoding from recall, so it must not ship. + +## Where the independent oracle earned its keep + +The oracle disagreed with the serving path exactly once during authoring: +"Does entropy cause energy?" The curriculum teaches *entropy reveals energy*. +The serving path said UNKNOWN — different relation, different atom. The oracle +said ENTAILED, because I had written its edge lookup to match on relation +*family* rather than the relation itself. + +The oracle was wrong, and the serving path was right. That is the useful shape +of the result: an independent check is worth building even when — especially +when — you expect it to agree, because the thing it catches is the assumption +you did not notice you had made. Family scoping decides which premises are in +play; the connective decides what was actually said. + +A second authoring error surfaced the same way: two lane cases claimed both +directions of `field requires charge` were taught. Only one is. The provenance +and soundness guards caught it before the case file was committed. + +## The finding: Phase 2's blocker is curriculum volume, not machinery + +Every `curriculum_*` band is unearned, so every answer is served hedged. That +is not a shortcut around the license — it is the license working, and the +reason is arithmetic. + +A band earns SERVE at θ_SERVE=0.99 with n ≥ 657 committed cases and a genuine +outcome mix. The question space is large enough (physics: 240 distinct +questions per family; philosophy_theology: 22,650). The *entailed* space is +not: physics teaches 7 causal and 9 modal relations, so at most 16 questions in +the entire subject can come back ENTAILED. Repeating them to reach 657 inflates +n without adding information, and a band whose entailed class is 1% of its +volume is passed by a pipeline that never entails anything at all. + +A balanced band needs roughly **219 distinct taught relations per subject × +family**. Physics has 7 and 9 — a ~25× gap. No sampling scheme closes it. What +closes it is ratified curriculum content, which is an authoring and +ratification decision, not an engineering one. + +So the honest state of Phase 2: the lifecycle is ported and provably correct on +real curriculum data; the earned-license half is blocked on how much has been +taught. That is a much better problem to have than the reverse, and it is the +one the plan's own doctrine (capability → practice → calibration → serve) +predicts you would hit here. + +## Two corrections to the plan's ground truth + +- **There is no biology domain-chain corpus.** The plan names physics and + biology as Phase 2's subjects on the basis that "OOD lanes, seed packs, and + domain chains already exist". For biology the first two are true; the third + is not, and `evals/foundational_biology_ood`'s own contract says it measures + fluency, not truth. The subjects with ratified chains AND mounted packs are + physics, mathematics_logic, systems_software, and philosophy_theology. The + composer serves all four; the second subject should come from that list, or + biology's chains should be authored first. +- **`operator_family` and `connective` disagree in the corpora.** Two rows + declare a family their connective does not imply. The serving path reads the + connective, because that is the only thing an exam question carries — and if + routing and compilation disagreed about a row's family, a taught edge could + go missing from the premises compiled to decide it. That would be a wrong + answer assembled from a correct curriculum, which is the exact failure mode + ADR-0261 §5.1 found in the categorical band a day earlier. + +## What is deliberately not built + +Composition, negation, question shapes beyond `Does ?`, +and premise-set scaling past the reader's 16-sentence honesty cap. Each is +scoped in ADR-0262 §6 with the condition that would unblock it. None of them +is a prerequisite for the next subject: adding one is a data operation now. diff --git a/docs/research/math-reader-phase-4-1-status-2026-07-24.md b/docs/research/math-reader-phase-4-1-status-2026-07-24.md new file mode 100644 index 00000000..17355166 --- /dev/null +++ b/docs/research/math-reader-phase-4-1-status-2026-07-24.md @@ -0,0 +1,85 @@ +# Math reader Phase 4.1 — measured status: the increment is already built, and it converts nothing + +**Date:** 2026-07-24 · **Arc:** generalization Phase 4.1 (Tier O, O4) · **Result:** null, by measurement + +## What was asked + +The generalization plan §2/Phase 4.1 and the Opus brief item O4 both name +"seeding-sentence injection (standing ruling #77 — the 75% 'no injection' +wall)" as the next math move, with wrong=0 on the full 500 as the gate and +parse-rate gain as the goal. + +## What is actually in the tree + +Seeding-sentence injection **is built and merged**. The reader arc's increment +1 shipped it as the #78 positive-polarity substrate: + +- `generate/math_roundtrip.py::NEUTRAL_COUNT_VERBS` — the curated + production/possession/acquisition allowlist, +- consumed by the seed matcher's `inject_discrete_count_statement` path and by + `generate/math_candidate_parser.py::_COMPARISON_ANCHOR_VERBS`, +- depletion/transfer verbs refuse by positive determination (an allowlist, not + a blocklist — the fail-open blocklist was reverted), +- documented in `docs/research/increment-1-band-plan-2026-07-19.md` §BUILD + OUTCOME. + +Its measured yield when it shipped: **PARSED unchanged — tune 3, measure 2, no +conversion.** The scoped band `{seed + forward-comparison + q:simple/summation}` +converted **0** tune cases, measured four independent ways (yield harness, band +map, band oracle, capability histogram). + +## Re-measured today, on the frozen split + +``` +uv run python evals/gsm8k_math/holdout_dev/v1/runner.py +holdout_dev: correct=5 wrong=0 refused=495 (n=500) +``` + +Five correct is exactly the tune-3 + measure-2 the increment-1 report recorded. +**No drift in five days of unrelated arc work, and wrong=0 still holds.** The +tune/measure split (`evals/gsm8k_math/holdout_dev/v1/split.py`) is frozen and +was not touched. + +## Why the honest move is to stop rather than extend + +The arc's own evidence says the wall is not where 4.1 aims: + +1. **The ~30-band bet was falsified.** In-band cases strand on capabilities + *outside* the band — 25 need multi-compound, 17 need compare-additive. + Adding more seed verbs cannot convert a case that also needs rate, + currency, and unit conversion in the same sentence. +2. **The taxonomy is not at bedrock.** The "smallest tractable set" + `{seed, q:simple}` dissolved on inspection: every one of its four cases + really needs rate / compare / currency / copula. Real GSM8K statements are + individually multi-capability — the conjunction recurs *within* a sentence, + so a capability list keeps fragmenting rather than converging. +3. **Growing the allowlist is the forbidden move.** ADR-0251 and + `docs/research/reader-arc-overfit-inventory-2026-07-19.md` exist because + per-case pattern growth previously produced overfit "lift" that committed + *wrong* answers on the real exam. An extension whose only evidence is "it + parses more tune cases" is the exact shape of that failure. + +## The live next move (unchanged, and it is not 4.1) + +The reader arc's own recommendation, carried to Shay with the increment-1 +foundations, is **case-first, not capability-first**: pick 2–3 specific closest +tune cases, hand-enumerate their *exact* end-to-end needs however many +capabilities that is, build precisely that, convert them, measure. The closest +cases are already identified — `0000`, `0001`, `0148`, `0082` — all needing +complex compare forms (chained / mass-noun entities / "than" + aggregate +reference) plus summation or difference. + +`multi-compound` is the single highest-leverage capability (on the critical +path for 70/106 = 66% of tractable needed-sets) but is insufficient alone. +`q:complex` remains the largest intractable wall (~35+ cases). + +## Recommendation for the plan + +Phase 4.1 should be marked **complete-with-null-yield** rather than pending, +and Phase 4 re-pointed at the increment-2 case-first work. The measurement +above is the evidence; nothing in this session's arc work moved it either way, +which is itself the useful signal — the math lane is genuinely orthogonal to +the reading/curriculum work, exactly as the plan's §3 dependency table assumed. + +Phase 4.2 (compare unblock) and 4.3 (q:complex decomposition study) are +untouched by this and remain queued behind the case-first increment. diff --git a/evals/curriculum_serve/__init__.py b/evals/curriculum_serve/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/curriculum_serve/contract.md b/evals/curriculum_serve/contract.md new file mode 100644 index 00000000..829b7686 --- /dev/null +++ b/evals/curriculum_serve/contract.md @@ -0,0 +1,86 @@ +# Curriculum-serve lane contract + +## What this lane scores + +The **production curriculum-grounded answering path** — the exact pipeline +`chat/curriculum_surface.py::decide_curriculum_question` runs on a `core chat` +turn when `curriculum_serving_enabled` is on: + +``` +Does ? (closed exam-question grammar) + → subject routing (the domain whose ratified vocabulary holds BOTH terms) + → relation family (the connective, normalized by ADR-0260 agreement) + → premise compilation (that family's RATIFIED chains, and nothing else) + → the argument bands (ADR-0260 verb reading → ADR-0261 existential) + → the ROBDD engine (ADR-0201/0218) +``` + +There is no subject-specific decision code anywhere in the path. Physics +differs from philosophy only in which curriculum rows load — which is the +property that makes "add a subject" a data operation. + +## Gold vocabulary + +Three classes are reachable: `entailed`, `unknown`, `declined`. + +`refuted` is **unreachable from a purely positive curriculum** and that is not +an omission. The curriculum is read OPEN-world: a relation it does not state +is UNKNOWN, never "no". Refuting would require the curriculum to teach a +negative ("no X causes Y"), which no ratified corpus does yet. See ADR-0262 §5. + +## The three lane guards (all run before any case is scored) + +Failing a guard is a **lane** failure, not a case miss — the lane is unsound, +not merely under-covered. + +1. **Provenance** (plan §4.3) — every chain id a case pins must resolve in the + subject's ratified curriculum. A case whose curriculum moved under it + breaks loudly rather than quietly answering from what remains. +2. **Corpus soundness** (§4.4) — the INDEPENDENT oracle + (`evals/curriculum_serve/oracle.py`, sharing no code with the serving path: + own loader, own ratification predicate, own family table, own agreement + normalization, own verdict rule) must re-derive every committed gold. +3. **Anti-recall coverage** (§4.7) — the split must carry ≥3 probes whose + answer is true in the world and absent from the curriculum, and no probe + may carry a committed gold. Without them the lane cannot show the system + decodes rather than recalls, and it must not ship. + +## Splits + +- `physics/` — 32 hand-authored cases over `physics_chains_v1`: 13 taught + edges (both connectives of each family, incl. the row whose declared + `operator_family` disagrees with its connective), 5 untaught compositions at + depth 2–3, 4 reverse-direction and cross-relation near-misses, 5 anti-recall + probes (3 with untaught vocabulary — *gravity*, *current*, *pressure* — and + 2 with taught vocabulary but untaught relations), and 5 typed refusals. + +## wrong=0 discipline + +Identical to the deduction-serve lane: `wrong` (a committed verdict that +disagrees with gold) MUST stay 0; a decline where gold expected a verdict is a +coverage miss, tracked in `counts.declined`, never conflated with a +confabulation. The runner requires `correct == n`. + +## What the lane deliberately does NOT do + +- **It does not compose chains.** `force causes acceleration` and + `acceleration causes motion` are both taught; `force causes motion` is + UNKNOWN. Causal transitivity is a substantive claim about the world, and no + ratified corpus teaches it. The oracle reports the shortest path length + alongside its verdict precisely so the lane can assert that a reachable pair + is still answered UNKNOWN — composition is provably not happening. +- **It does not read the corpus's `operator_family` field.** The family comes + from the CONNECTIVE, because a question carries a relation word and nothing + else; deriving the family from a field the question cannot carry would let + premise compilation and question routing disagree, and a taught edge could + go missing from the premises compiled to decide it. + +## Reproduce + +```bash +uv run python -m evals.curriculum_serve.runner # human-facing +uv run python -m evals.curriculum_serve.runner --report evals/curriculum_serve/report.json +``` + +Pinned in `scripts/verify_lane_shas.py` as lane id `curriculum_serve_v1`. +`core test --suite deductive` runs `tests/test_curriculum_serve.py`. diff --git a/evals/curriculum_serve/oracle.py b/evals/curriculum_serve/oracle.py new file mode 100644 index 00000000..7feb3978 --- /dev/null +++ b/evals/curriculum_serve/oracle.py @@ -0,0 +1,189 @@ +"""Independent curriculum oracle — the gold for the curriculum-serve lane. + +This is **deliberately a second, independent decision procedure** (plan §4.4). +It shares no code with the serving path: its own JSONL loader, its own +ratification predicate, its own connective→family table, its own agreement +normalization, and its own verdict rule. Two independently-written procedures +agreeing on every case is real evidence the serving path reads the curriculum +correctly; a shared-code "oracle" would only prove the compiler agrees with +itself. + +It is intentionally simple: a closed-world reachability oracle over the chain +graph. "Closed-world" describes only what the ORACLE can see — the ratified +corpus and nothing else. It does NOT mean untaught facts are false: an edge +absent from the corpus is UNKNOWN, never refuted, because a curriculum that +does not mention something has said nothing about it. The oracle also reports +the shortest path length between the two terms, which is what lets the lane +assert the property that matters most here — that a reachable-but-untaught +pair is answered UNKNOWN rather than composed into a claim. +""" + +from __future__ import annotations + +import json +from collections import deque +from dataclasses import dataclass +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_CHAIN_DIR = _REPO_ROOT / "teaching" / "domain_chains" +_PACK_DIR = _REPO_ROOT / "packs" / "data" + +#: Independently restated (the serving path has its own copy — that is the point). +_FAMILY_OF_CONNECTIVE = { + "causes": "causal", + "reveals": "causal", + "grounds": "causal", + "requires": "modal", + "enables": "modal", + "precedes": "sequence", + "opposes": "contrast", + "supports": "evidential", +} + +#: Which corpora and packs each subject is taught from — restated, not imported. +_DOMAIN_SOURCES: dict[str, tuple[tuple[str, ...], tuple[str, ...]]] = { + "physics": (("physics_chains_v1",), ("en_physics_v1",)), + "mathematics_logic": (("mathematics_logic_chains_v1",), ("en_mathematics_logic_v1",)), + "systems_software": (("systems_software_chains_v1",), ("en_systems_software_v1",)), + "philosophy_theology": ( + ("philosophy_theology_chains_v1",), + ("en_core_cognition_v1", "en_core_meta_v1"), + ), +} + + +@dataclass(frozen=True, slots=True) +class OracleVerdict: + """The gold verdict, plus the evidence the lane asserts against.""" + + verdict: str # entailed | unknown | declined + reason: str # typed reason when declined, else "" + family: str # "" when the relation is not taught anywhere + depth: int # 1 = taught edge, >1 = reachable in n hops, 0 = no path + + +def _lemmas(pack_id: str) -> set[str]: + path = _PACK_DIR / pack_id / "lexicon.jsonl" + out: set[str] = set() + if not path.exists(): + return out + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + row = json.loads(line) + lemma = row.get("lemma") + if isinstance(lemma, str): + out.add(lemma) + return out + + +def _edges(domain: str) -> list[tuple[str, str, str, str]]: + """``(subject, connective, object, chain_id)`` for every ratified row.""" + corpora, packs = _DOMAIN_SOURCES[domain] + vocabulary = set() + for pack in packs: + vocabulary |= _lemmas(pack) + out: list[tuple[str, str, str, str]] = [] + for corpus in corpora: + path = _CHAIN_DIR / f"{corpus}.jsonl" + if not path.exists(): + continue + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + row = json.loads(line) + if row.get("review_status") != "reviewed": + continue + if row.get("domain") != domain: + continue + subject, obj = row.get("subject"), row.get("object") + connective = row.get("connective") + if subject not in vocabulary or obj not in vocabulary: + continue + if connective not in _FAMILY_OF_CONNECTIVE: + continue + out.append((subject, connective, obj, row.get("chain_id", ""))) + return out + + +def vocabulary(domain: str) -> set[str]: + """Every lemma the subject's packs teach.""" + out: set[str] = set() + for pack in _DOMAIN_SOURCES[domain][1]: + out |= _lemmas(pack) + return out + + +def _base_forms(word: str) -> set[str]: + """The spellings a question's relation word may take — independently + written, covering the same +s/+es/y↔ies agreement the serving path + normalizes with its own table.""" + forms = {word} + forms.add(word + "s") + forms.add(word + "es") + if word.endswith("y"): + forms.add(word[:-1] + "ies") + if word.endswith("s"): + forms.add(word[:-1]) + if word.endswith("es"): + forms.add(word[:-2]) + if word.endswith("ies"): + forms.add(word[:-3] + "y") + return forms + + +def _connective_for(relation: str) -> str | None: + candidates = _base_forms(relation) + for connective in _FAMILY_OF_CONNECTIVE: + if connective in candidates or relation == connective: + return connective + return None + + +def oracle_answer(domain: str, subject: str, relation: str, obj: str) -> OracleVerdict: + """The gold verdict for one exam question against *domain*'s curriculum.""" + vocab = vocabulary(domain) + if subject not in vocab or obj not in vocab: + return OracleVerdict("declined", "untaught_vocabulary", "", 0) + connective = _connective_for(relation) + if connective is None: + return OracleVerdict("declined", "out_of_curriculum", "", 0) + family = _FAMILY_OF_CONNECTIVE[connective] + edges = [e for e in _edges(domain) if _FAMILY_OF_CONNECTIVE[e[1]] == family] + if not edges: + return OracleVerdict("declined", "empty_curriculum", family, 0) + # Entailment needs the SAME relation, not merely the same family: the + # curriculum teaching "entropy reveals energy" has not thereby taught + # "entropy causes energy". Family scoping decides which premises are in + # play; the connective decides what was actually said. + if any(s == subject and c == connective and o == obj for s, c, o, _id in edges): + return OracleVerdict("entailed", "", family, 1) + # No taught edge. Report the shortest path so the lane can prove the + # serving path does NOT compose a chain into a claim. + adjacency: dict[str, list[str]] = {} + for s, _c, o, _id in edges: + adjacency.setdefault(s, []).append(o) + seen = {subject} + queue: deque[tuple[str, int]] = deque([(subject, 0)]) + while queue: + node, dist = queue.popleft() + for nxt in adjacency.get(node, ()): + if nxt == obj: + # ``dist`` counts edges already traversed to reach ``node``; + # this one closes the path, so the shortest path is dist + 1. + return OracleVerdict("unknown", "", family, dist + 1) + if nxt not in seen: + seen.add(nxt) + queue.append((nxt, dist + 1)) + return OracleVerdict("unknown", "", family, 0) + + +def taught_edges(domain: str) -> list[tuple[str, str, str, str]]: + """Public view for the lane's provenance assertions.""" + return _edges(domain) + + +__all__ = ["OracleVerdict", "oracle_answer", "taught_edges", "vocabulary"] diff --git a/evals/curriculum_serve/physics/__init__.py b/evals/curriculum_serve/physics/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/curriculum_serve/physics/cases.jsonl b/evals/curriculum_serve/physics/cases.jsonl new file mode 100644 index 00000000..772cb2c4 --- /dev/null +++ b/evals/curriculum_serve/physics/cases.jsonl @@ -0,0 +1,32 @@ +{"id": "cs-phy-0001", "text": "Does force cause acceleration?", "gold": "entailed", "chains": ["physics-causal-001"], "class": "taught_edge_causal"} +{"id": "cs-phy-0002", "text": "Does acceleration cause motion?", "gold": "entailed", "chains": ["physics-causal-002"], "class": "taught_edge_causal"} +{"id": "cs-phy-0003", "text": "Does work cause energy?", "gold": "entailed", "chains": ["physics-causal-003"], "class": "taught_edge_causal"} +{"id": "cs-phy-0004", "text": "Does charge cause field?", "gold": "entailed", "chains": ["physics-causal-004"], "class": "taught_edge_causal"} +{"id": "cs-phy-0005", "text": "Does field cause force?", "gold": "entailed", "chains": ["physics-causal-005"], "class": "taught_edge_causal"} +{"id": "cs-phy-0006", "text": "Does temperature reveal motion?", "gold": "entailed", "chains": ["physics-causal-006"], "class": "taught_edge_reveals"} +{"id": "cs-phy-0007", "text": "Does entropy reveal energy?", "gold": "entailed", "chains": ["physics-causal-007"], "class": "taught_edge_reveals"} +{"id": "cs-phy-0008", "text": "Does conservation require symmetry?", "gold": "entailed", "chains": ["physics-modal-001"], "class": "taught_edge_modal"} +{"id": "cs-phy-0009", "text": "Does mass require force?", "gold": "entailed", "chains": ["physics-modal-002"], "class": "taught_edge_modal"} +{"id": "cs-phy-0010", "text": "Does momentum require mass?", "gold": "entailed", "chains": ["physics-modal-003"], "class": "taught_edge_modal"} +{"id": "cs-phy-0011", "text": "Does energy require conservation?", "gold": "entailed", "chains": ["physics-modal-004"], "class": "taught_edge_modal"} +{"id": "cs-phy-0012", "text": "Does frequency require wave?", "gold": "entailed", "chains": ["physics-modal-005"], "class": "taught_edge_modal"} +{"id": "cs-phy-0013", "text": "Does wave require field?", "gold": "entailed", "chains": ["physics-causal-008"], "class": "taught_edge_family_by_connective"} +{"id": "cs-phy-0014", "text": "Does force cause motion?", "gold": "unknown", "chains": ["physics-causal-001", "physics-causal-002"], "class": "two_hop_not_composed"} +{"id": "cs-phy-0015", "text": "Does charge cause acceleration?", "gold": "unknown", "chains": ["physics-causal-004", "physics-causal-005", "physics-causal-001"], "class": "three_hop_not_composed"} +{"id": "cs-phy-0016", "text": "Does energy require symmetry?", "gold": "unknown", "chains": ["physics-modal-004", "physics-modal-001"], "class": "two_hop_modal_not_composed"} +{"id": "cs-phy-0017", "text": "Does momentum require force?", "gold": "unknown", "chains": ["physics-modal-003", "physics-modal-002"], "class": "two_hop_modal_not_composed"} +{"id": "cs-phy-0018", "text": "Does acceleration cause force?", "gold": "unknown", "chains": ["physics-causal-001"], "class": "reverse_direction"} +{"id": "cs-phy-0019", "text": "Does motion cause acceleration?", "gold": "unknown", "chains": ["physics-causal-002"], "class": "reverse_direction"} +{"id": "cs-phy-0020", "text": "Does entropy cause energy?", "gold": "unknown", "chains": ["physics-causal-007"], "class": "relation_not_interchangeable"} +{"id": "cs-phy-0021", "text": "Does work cause motion?", "gold": "unknown", "chains": [], "class": "unlinked_pair"} +{"id": "cs-phy-0022", "text": "Does temperature require symmetry?", "gold": "unknown", "chains": [], "class": "unlinked_pair"} +{"id": "cs-phy-0023", "text": "Does mass require energy?", "gold": "unknown", "chains": [], "class": "anti_recall_true_but_untaught"} +{"id": "cs-phy-0024", "text": "Does force cause work?", "gold": "unknown", "chains": [], "class": "anti_recall_true_but_untaught"} +{"id": "cs-phy-0025", "text": "Does charge require field?", "gold": "unknown", "chains": ["physics-modal-007"], "class": "reverse_direction_modal"} +{"id": "cs-phy-0026", "text": "Does field require charge?", "gold": "entailed", "chains": ["physics-modal-007"], "class": "taught_edge_modal"} +{"id": "cs-phy-0027", "text": "Does gravity cause acceleration?", "gold": "declined", "chains": [], "class": "anti_recall_untaught_vocabulary"} +{"id": "cs-phy-0028", "text": "Does current cause field?", "gold": "declined", "chains": [], "class": "anti_recall_untaught_vocabulary"} +{"id": "cs-phy-0029", "text": "Does pressure require volume?", "gold": "declined", "chains": [], "class": "anti_recall_untaught_vocabulary"} +{"id": "cs-phy-0030", "text": "Does force explain acceleration?", "gold": "declined", "chains": [], "class": "out_of_curriculum_relation"} +{"id": "cs-phy-0031", "text": "Does force accelerate mass quickly?", "gold": "declined", "chains": [], "class": "question_shape_out_of_band"} +{"id": "cs-phy-0032", "text": "Does the force cause acceleration?", "gold": "declined", "chains": [], "class": "question_shape_out_of_band"} diff --git a/evals/curriculum_serve/report.json b/evals/curriculum_serve/report.json new file mode 100644 index 00000000..e4828347 --- /dev/null +++ b/evals/curriculum_serve/report.json @@ -0,0 +1,40 @@ +{ + "aggregate": { + "correct": 32, + "declined": 0, + "n": 32, + "wrong": 0 + }, + "all_correct": true, + "arc": "generalization", + "lane": "curriculum_serve", + "schema_version": 1, + "splits": { + "physics": { + "all_cases_correct": true, + "anti_recall_probes": 5, + "by_band": { + "curriculum_physics_causal": 14, + "curriculum_physics_modal": 12 + }, + "by_gold": { + "declined": 6, + "entailed": 14, + "unknown": 12 + }, + "correct_by_gold": { + "declined": 6, + "entailed": 14, + "unknown": 12 + }, + "counts": { + "correct": 32, + "declined": 0, + "wrong": 0 + }, + "domain": "physics", + "n": 32 + } + }, + "wrong_is_zero": true +} diff --git a/evals/curriculum_serve/runner.py b/evals/curriculum_serve/runner.py new file mode 100644 index 00000000..83498321 --- /dev/null +++ b/evals/curriculum_serve/runner.py @@ -0,0 +1,216 @@ +"""Curriculum-serve lane — scores curriculum-grounded exam answering. + +The Phase-2 lane of the generalization arc. For each committed case, the raw +question text runs through the exact production path +``chat/curriculum_surface.py::decide_curriculum_question`` — question reader → +subject routing → ratified-curriculum premise compilation → the argument bands +→ the ROBDD engine — and the resulting verdict is compared to gold authored +INDEPENDENTLY by ``evals.curriculum_serve.oracle`` (plan §4.4). + +Three guards run before any case is scored, and any of them failing is a lane +failure, not a case miss: + +1. **Provenance** (§4.3) — every chain id a case pins must resolve in the + subject's ratified curriculum. A case whose curriculum moved under it + breaks loudly instead of quietly answering from what is left. +2. **Corpus soundness** (§4.4) — the independent oracle must agree with every + committed gold. A gold nobody can re-derive from the curriculum is not gold. +3. **Anti-recall coverage** (§4.7) — the split MUST contain cases whose answer + is true in the world but absent from the curriculum. Without them the lane + cannot show the system decodes rather than recalls, and it must not ship. + +Counts mirror the deduction-serve lane exactly: ``wrong`` (a committed verdict +that disagrees with gold) MUST stay 0; a decline where gold expected a verdict +is a coverage miss, tracked but never conflated with a confabulation. +""" + +from __future__ import annotations + +import argparse +import json +from collections import Counter +from pathlib import Path + +from chat.curriculum_surface import decide_curriculum_question, read_curriculum_question +from evals.curriculum_serve.oracle import oracle_answer +from teaching.curriculum_premises import load_curriculum, resolve_pinned + +_ROOT = Path(__file__).resolve().parent + +_SPLITS: tuple[tuple[str, str, Path], ...] = ( + # (split name, subject domain, cases) + ("physics", "physics", _ROOT / "physics" / "cases.jsonl"), +) + +#: A split must carry at least this many anti-recall probes (§4.7). The class +#: prefix is the contract: a probe is a case whose answer is true in the world +#: and absent from the curriculum, so the honest verdict is a non-commitment. +_ANTI_RECALL_PREFIX = "anti_recall" +_MIN_ANTI_RECALL = 3 + + +class LaneContractError(AssertionError): + """A guard failed — the lane itself is unsound, not merely under-covered.""" + + +def _load(path: Path) -> list[dict]: + with path.open(encoding="utf-8") as fh: + return [json.loads(line) for line in fh if line.strip()] + + +def assert_provenance(domain: str, cases: list[dict]) -> None: + """Every pinned chain id must resolve in the ratified curriculum (§4.3).""" + curriculum = load_curriculum(domain) + for case in cases: + pinned = tuple(case.get("chains", ())) + if not pinned: + continue + resolve_pinned(curriculum, pinned) # raises UnratifiedChain + + +def assert_corpus_sound(domain: str, cases: list[dict]) -> None: + """The independent oracle must re-derive every committed gold (§4.4).""" + for case in cases: + query = read_curriculum_question(case["text"]) + gold = case["gold"] + if not hasattr(query, "subject"): + # Unreadable question shapes are gold-``declined`` by construction; + # the oracle has no opinion on a text it cannot parse either. + if gold != "declined": + raise LaneContractError( + f"{case['id']}: unreadable question with gold={gold}" + ) + continue + verdict = oracle_answer(domain, query.subject, query.verb, query.obj) + if verdict.verdict != gold: + raise LaneContractError( + f"{case['id']}: oracle={verdict.verdict} gold={gold} " + f"(family={verdict.family} depth={verdict.depth}) text={case['text']!r}" + ) + + +def assert_anti_recall_coverage(cases: list[dict]) -> None: + """The split must probe recall, or it must not ship (§4.7).""" + probes = [c for c in cases if str(c.get("class", "")).startswith(_ANTI_RECALL_PREFIX)] + if len(probes) < _MIN_ANTI_RECALL: + raise LaneContractError( + f"split has {len(probes)} anti-recall probes, needs >= {_MIN_ANTI_RECALL}" + ) + for probe in probes: + if probe["gold"] in ("entailed", "refuted"): + raise LaneContractError( + f"{probe['id']}: an anti-recall probe cannot have a committed gold" + ) + + +def build_report(domain: str, cases: list[dict]) -> dict: + assert_provenance(domain, cases) + assert_corpus_sound(domain, cases) + assert_anti_recall_coverage(cases) + + counts = Counter({"correct": 0, "wrong": 0, "declined": 0}) + by_gold: Counter[str] = Counter() + correct_by_gold: Counter[str] = Counter() + by_band: Counter[str] = Counter() + mismatches: list[dict] = [] + + for case in cases: + gold = case["gold"] + by_gold[gold] += 1 + decision = decide_curriculum_question(case["text"]) + if decision.band: + by_band[decision.band] += 1 + if decision.verdict == gold: + counts["correct"] += 1 + correct_by_gold[gold] += 1 + elif decision.verdict == "declined": + counts["declined"] += 1 + mismatches.append( + {"id": case["id"], "gold": gold, "got": decision.verdict, + "reason": decision.reason, "text": case["text"]} + ) + else: + counts["wrong"] += 1 + mismatches.append( + {"id": case["id"], "gold": gold, "got": decision.verdict, + "reason": decision.reason, "text": case["text"]} + ) + + return { + "n": len(cases), + "counts": dict(counts), + "by_gold": dict(by_gold), + "correct_by_gold": dict(correct_by_gold), + "by_band": dict(by_band), + "anti_recall_probes": sum( + 1 for c in cases if str(c.get("class", "")).startswith(_ANTI_RECALL_PREFIX) + ), + "all_cases_correct": counts["correct"] == len(cases), + "mismatch_examples": mismatches[:10], + } + + +def build_combined_report() -> dict: + """Deterministic per-split + aggregate report — safe to SHA-pin.""" + splits: dict[str, dict] = {} + aggregate = {"n": 0, "correct": 0, "wrong": 0, "declined": 0} + for name, domain, path in _SPLITS: + report = build_report(domain, _load(path)) + splits[name] = { + "n": report["n"], + "domain": domain, + "counts": report["counts"], + "by_gold": report["by_gold"], + "correct_by_gold": report["correct_by_gold"], + "by_band": report["by_band"], + "anti_recall_probes": report["anti_recall_probes"], + "all_cases_correct": report["all_cases_correct"], + } + aggregate["n"] += report["n"] + for key in ("correct", "wrong", "declined"): + aggregate[key] += report["counts"][key] + return { + "schema_version": 1, + "lane": "curriculum_serve", + "arc": "generalization", + "splits": splits, + "aggregate": aggregate, + "wrong_is_zero": aggregate["wrong"] == 0, + "all_correct": all(s["all_cases_correct"] for s in splits.values()), + } + + +def write_combined_report(path: Path) -> dict: + report = build_combined_report() + path.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return report + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--report", type=Path, default=None) + args = parser.parse_args(argv) + + if args.report is not None: + report = write_combined_report(args.report) + return 0 if (report["wrong_is_zero"] and report["all_correct"]) else 1 + + all_ok = True + for name, domain, path in _SPLITS: + report = build_report(domain, _load(path)) + c = report["counts"] + print( + f"[{name}] n={report['n']} correct={c['correct']} wrong={c['wrong']} " + f"declined_mismatch={c['declined']} anti_recall={report['anti_recall_probes']}" + ) + for m in report["mismatch_examples"]: + print(f" {m['id']}: gold={m['gold']} got={m['got']} " + f"reason={m['reason']} text={m['text']!r}") + all_ok = all_ok and report["all_cases_correct"] + return 0 if all_ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/deduction_serve/practice/runner.py b/evals/deduction_serve/practice/runner.py index 5c9498b9..236644e2 100644 --- a/evals/deduction_serve/practice/runner.py +++ b/evals/deduction_serve/practice/runner.py @@ -18,11 +18,11 @@ commit and SHA-verify on load. from __future__ import annotations import argparse -import json from pathlib import Path from typing import Any from core.learning_arena.engine import run_practice +from core.ratified_ledger import seal_artifact, tally_dict, write_sealed_ledger from core.reliability_gate import Action, Ceilings, ClassTally, license_for from evals.deduction_serve.practice.gold import ( ConstructionGoldTether, @@ -30,7 +30,6 @@ from evals.deduction_serve.practice.gold import ( all_gold_problems, assert_corpus_sound, ) -from formation.hashing import sha256_of #: The committed sealed ledger lives next to its serving READER (chat/), mirroring #: the estimation ledger's topology (producer in evals/, artifact by the reader). @@ -46,13 +45,7 @@ def build_ledger() -> dict[str, ClassTally]: def _tally_dict(tally: ClassTally) -> dict[str, Any]: - return { - "correct": tally.correct, - "wrong": tally.wrong, - "refused": tally.refused, - "t2_verified": tally.t2_verified, - "t2_agrees_gold": tally.t2_agrees_gold, - } + return tally_dict(tally) def run(ceilings: Ceilings | None = None) -> dict[str, Any]: @@ -81,31 +74,30 @@ def run(ceilings: Ceilings | None = None) -> dict[str, Any]: def build_sealed_artifact() -> dict[str, Any]: - """The committed sealed-ledger dict (self-verifying ``content_sha256``).""" - ledger = build_ledger() - classes = {cls: _tally_dict(tally) for cls, tally in sorted(ledger.items())} - return { - "schema": "deduction_serve_ledger_v1", - "classes": classes, - "content_sha256": sha256_of(classes), - "note": ( + """The committed sealed-ledger dict (self-verifying ``content_sha256``). + + Formatting and hashing come from the shared bridge (ADR-0263) — byte + -identical to what this module wrote before the extraction, which is how + the refactor is proven safe: re-sealing must not move the artifact. + """ + return seal_artifact( + build_ledger(), + schema="deduction_serve_ledger_v1", + note=( "Sealed-practice committed ledger for deduction serving (ADR-0256). " "Engine reads, never writes. Ceilings stay at safe defaults " "(theta_SERVE=0.99). A band earns SERVE by demonstrated pipeline " "reliability (reader+projector+engine) at volume >= 657 committed." ), - "provenance": "evals.deduction_serve.practice.runner.seal_ledger", - } + provenance="evals.deduction_serve.practice.runner.seal_ledger", + ) def seal_ledger(path: Path = _SEALED_LEDGER_PATH) -> dict[str, Any]: """Regenerate + write the committed sealed ledger. Verifies corpus soundness against the independent oracle first (a mis-stated gold can never seal).""" assert_corpus_sound() - artifact = build_sealed_artifact() - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(artifact, indent=2, sort_keys=True) + "\n", encoding="utf-8") - return artifact + return write_sealed_ledger(path, build_sealed_artifact()) def main(argv: list[str] | None = None) -> int: diff --git a/generate/determine/estimation_license.py b/generate/determine/estimation_license.py index a10a3e38..ce8110b4 100644 --- a/generate/determine/estimation_license.py +++ b/generate/determine/estimation_license.py @@ -8,27 +8,28 @@ never writes it. The artifact is the sealed-practice output of load, so a hand-edited (un-ratified) ledger is rejected rather than silently trusted. Determinism: the ledger is immutable ratified data, parsed once and cached; the gate -(``license_for``) is pure. No engine self-authorization — ceilings stay at the safe -defaults (raising one's own bar is structurally impossible, ADR-0175 invariant #4). +is pure. No engine self-authorization — ceilings stay at the safe defaults (raising +one's own bar is structurally impossible, ADR-0175 invariant #4). + +The load/verify/gate mechanics live in ``core.ratified_ledger`` (ADR-0263), the +bridge extracted from this module and its two successors; this is the estimation +ADAPTER over it, preserving its own public API (including the predicate → +converse-class naming, which is the one thing genuinely local to estimation). """ from __future__ import annotations -import json from functools import lru_cache from pathlib import Path -from core.reliability_gate import Action, Ceilings, ClassTally, LicenseDecision, license_for -from formation.hashing import sha256_of +from core.ratified_ledger import RatifiedLedgerError, load_sealed_ledger +from core.ratified_ledger import serve_license as _serve_license +from core.reliability_gate import Ceilings, ClassTally, LicenseDecision from generate.determine.estimate import converse_class_name _LEDGER_PATH = Path(__file__).resolve().parent / "data" / "estimation_ledger.json" -class RatifiedLedgerError(ValueError): - """The committed estimation ledger is missing, malformed, or tampered with.""" - - @lru_cache(maxsize=1) def load_ratified_ledger() -> dict[str, ClassTally]: """Load + verify the ratified estimation ledger → per-class ``ClassTally``. @@ -37,30 +38,7 @@ def load_ratified_ledger() -> dict[str, ClassTally]: recomputed ``content_sha256`` does not match the committed one (tamper-evidence: only the sealed-practice output is trusted, never a hand-edited ledger). """ - try: - artifact = json.loads(_LEDGER_PATH.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: # pragma: no cover - defensive - raise RatifiedLedgerError(f"cannot read ratified ledger: {exc}") from exc - - classes = artifact.get("classes") - if not isinstance(classes, dict): - raise RatifiedLedgerError("ratified ledger has no 'classes' table") - if sha256_of(classes) != artifact.get("content_sha256"): - raise RatifiedLedgerError( - "ratified ledger content_sha256 mismatch — not the sealed-practice output" - ) - - ledger: dict[str, ClassTally] = {} - for cls, counts in classes.items(): - ledger[cls] = ClassTally( - class_name=cls, - correct=int(counts.get("correct", 0)), - wrong=int(counts.get("wrong", 0)), - refused=int(counts.get("refused", 0)), - t2_verified=int(counts.get("t2_verified", 0)), - t2_agrees_gold=int(counts.get("t2_agrees_gold", 0)), - ) - return ledger + return load_sealed_ledger(_LEDGER_PATH) def serve_license( @@ -76,11 +54,7 @@ def serve_license( deterministic ``license_for`` verdict under the safe default ceilings. """ ledger = ledger if ledger is not None else load_ratified_ledger() - tally = ledger.get(converse_class_name(predicate)) - if tally is None: - return None - ceilings = ceilings if ceilings is not None else Ceilings.default() - return license_for(tally, Action.SERVE, ceilings) + return _serve_license(converse_class_name(predicate), ledger, ceilings=ceilings) __all__ = ["RatifiedLedgerError", "load_ratified_ledger", "serve_license"] diff --git a/generate/proof_chain/verb.py b/generate/proof_chain/verb.py index 914ce5c4..157294f1 100644 --- a/generate/proof_chain/verb.py +++ b/generate/proof_chain/verb.py @@ -509,10 +509,22 @@ def read_verb_argument(text: str) -> VerbArgument | VerbRefusal: ) +def verb_forms_link(a: str, b: str) -> bool: + """Public view of this band's closed verb-agreement relation. + + Exported so paths OUTSIDE the proof_chain package (the curriculum + composer, ADR-0262) can normalize a base form against a stated + third-person form using the SAME closed table and suffix rules the reader + uses — one agreement authority, not two. + """ + return _verb_tokens_link(a, b) + + __all__ = [ "MAX_ATOMS", "MAX_PREMISE_SENTENCES", "VerbArgument", "VerbRefusal", "read_verb_argument", + "verb_forms_link", ] diff --git a/scripts/generate_claims.py b/scripts/generate_claims.py index 69920837..4c8284a4 100644 --- a/scripts/generate_claims.py +++ b/scripts/generate_claims.py @@ -74,6 +74,10 @@ _LANE_ADR: dict[str, tuple[str, str]] = { "ADR-0206", "Propositional entailment scored against an independent truth-table oracle; dev+holdout+external 716/716 correct, wrong=0, refused=0", ), + "curriculum_serve_v1": ( + "ADR-0262", + "Flag-gated curriculum serving answers exam questions from a subject's RATIFIED chain corpus only; untaught facts return UNKNOWN (anti-recall probes enforced), wrong=0", + ), "deduction_serve_v1": ( "ADR-0256", "Flag-gated deduction serving decides real English/member/fused/verb/existential arguments end-to-end under earned SERVE licenses; wrong=0 across all splits", diff --git a/scripts/verify_lane_shas.py b/scripts/verify_lane_shas.py index d26394b2..29b7931a 100644 --- a/scripts/verify_lane_shas.py +++ b/scripts/verify_lane_shas.py @@ -52,6 +52,7 @@ PINNED_SHAS: dict[str, str] = { "math_teaching_corpus_v1": "eaf160d145da29f9050ede8d58bf111b0f651dd40aeae9201857d0b97e014dd4", "deductive_logic_v1": "97a230949016e38d5e3f37a69e4245b320575ee70e5af92ff7607f7b05f74b5f", "deduction_serve_v1": "0b461a5a49c8f8260ca87d0c9c9f9a17232bd1fdedd982e34649eedf9cca30b5", + "curriculum_serve_v1": "d9e7ba500f040b865870413a940ee9a49910ac22e1a89c9feec1a60bdd2513f1", } @@ -137,6 +138,12 @@ LANE_SPECS: tuple[LaneSpec, ...] = ( report_relative="evals/deduction_serve/report.json", run_as_module=True, ), + LaneSpec( + lane_id="curriculum_serve_v1", + runner_module="evals/curriculum_serve/runner.py", + report_relative="evals/curriculum_serve/report.json", + run_as_module=True, + ), ) diff --git a/teaching/curriculum_premises.py b/teaching/curriculum_premises.py new file mode 100644 index 00000000..d7efadb4 --- /dev/null +++ b/teaching/curriculum_premises.py @@ -0,0 +1,251 @@ +"""Curriculum → premise compiler (generalization arc Phase 2, ADR-0262). + +The load-bearing half of the curriculum-entailment gold contract +(docs/plans/generalization-arc-2026-07-24.md §4): an exam question supplies +only the QUERY; its premises are compiled here, from the RATIFIED domain-chain +corpus of the subject being examined, and from nowhere else. Gold is therefore +a function of *(curriculum, question)* — never of case-local hidden text, and +never of what a model happens to know about the world. That is the +decoding-not-generating line in mechanical form: a fact absent from the +curriculum is UNKNOWN, however true it is. + +**Ratification** (§4.3) is the same predicate the capability reporter applies +to these corpora, restated here rather than imported: a chain counts iff its +``review_status`` is ``reviewed`` AND both its subject and object lemmas are +resident in the packs the chain itself declares. The duplication is +deliberate and narrow — a serving path must not take a dependency on the +reporting subsystem, and this module must be able to state its own admission +rule in the terms the serve-time refusals name. + +**Compilation is family-scoped.** A question about a `causes` relation is +answered from the `causes` chains. Restricting cannot lose an entailment (a +chain's family is fixed by its connective, so an edge that would answer the +question is always in the question's own family), and it keeps the compiled +premise set inside the reader's honesty caps as corpora grow. + +The compiler emits plain English sentences — ``"force causes acceleration"`` — +because the argument bands are the decider (ADR-0256…0261). There is no +subject-specific decision code anywhere in this path: physics differs from +philosophy only in which rows load. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path + +from core.capability.domains import ( + DOMAIN_CAPABILITY_CORPORA, + DOMAIN_CORPORA, + DOMAIN_PACKS, +) + +_REPO_ROOT = Path(__file__).resolve().parents[1] + +#: Connective → relation family. The closed set this path serves; a chain +#: whose connective is absent is not compiled (and cannot be questioned about +#: either, so the two ends stay consistent). +#: +#: The CONNECTIVE is the sole family authority here, deliberately — not the +#: row's own ``operator_family`` field. An exam question carries a relation +#: word and nothing else, so a family derived from anything the question does +#: not carry would let premise compilation and question routing disagree, and +#: a taught edge could then be missing from the premises compiled to decide +#: it — a wrong answer built from a correct curriculum. Two corpus rows +#: currently declare a family their connective does not imply +#: (``physics-causal-008`` and one systems_software row, both +#: ``requires``/``causal``); under this rule they are read as modal, which is +#: what their connective says. +CONNECTIVE_FAMILY: dict[str, str] = { + "causes": "causal", + "reveals": "causal", + "grounds": "causal", + "requires": "modal", + "enables": "modal", + "precedes": "sequence", + "opposes": "contrast", + "supports": "evidential", +} + +#: Relation families, in a stable order (the band-key axis — ADR-0262 §4). +FAMILIES: tuple[str, ...] = ("causal", "modal", "sequence", "contrast", "evidential") + + +@dataclass(frozen=True, slots=True) +class CurriculumChain: + """One ratified curriculum edge, as the exam path reads it.""" + + chain_id: str + subject: str + connective: str + obj: str + family: str + + @property + def sentence(self) -> str: + """The English premise this chain compiles to. The connective is + already a third-person-singular verb form, so the sentence lands in + the verb-predicate grammar (ADR-0260) exactly as written.""" + return f"{self.subject} {self.connective} {self.obj}" + + +@dataclass(frozen=True, slots=True) +class Curriculum: + """The ratified curriculum of one subject — the ONLY premise source.""" + + domain: str + chains: tuple[CurriculumChain, ...] + vocabulary: frozenset[str] + + def family(self, family: str) -> tuple[CurriculumChain, ...]: + return tuple(c for c in self.chains if c.family == family) + + def chain_by_id(self, chain_id: str) -> CurriculumChain | None: + return next((c for c in self.chains if c.chain_id == chain_id), None) + + +@lru_cache(maxsize=None) +def _pack_lemmas(pack_id: str) -> frozenset[str]: + path = _REPO_ROOT / "packs" / "data" / pack_id / "lexicon.jsonl" + if not path.exists(): + return frozenset() + lemmas: set[str] = set() + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + lemma = row.get("lemma") if isinstance(row, dict) else None + if isinstance(lemma, str): + lemmas.add(lemma) + return frozenset(lemmas) + + +def _domain_vocabulary(domain: str) -> frozenset[str]: + """Every lemma the subject's mounted packs teach — the vocabulary boundary + an exam question may not cross (§4.6 ``untaught_vocabulary``).""" + lemmas: set[str] = set() + for pack_id in DOMAIN_PACKS.get(domain, ()): + lemmas |= _pack_lemmas(pack_id) + return frozenset(lemmas) + + +def _ratified_rows(corpus_id: str, domain: str) -> list[dict]: + rel = DOMAIN_CAPABILITY_CORPORA.get(corpus_id) + if rel is None: + return [] + path = _REPO_ROOT / rel + if not path.exists(): + return [] + rows: list[dict] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(row, dict): + continue + if row.get("review_status") != "reviewed": + continue + if str(row.get("domain") or "").strip() != domain: + continue + subject = str(row.get("subject") or "").strip() + obj = str(row.get("object") or "").strip() + subject_pack = str(row.get("subject_pack_id") or "").strip() + object_pack = str(row.get("object_pack_id") or "").strip() + if not subject or not obj: + continue + if subject_pack and subject not in _pack_lemmas(subject_pack): + continue + if object_pack and obj not in _pack_lemmas(object_pack): + continue + rows.append(row) + return rows + + +@lru_cache(maxsize=None) +def load_curriculum(domain: str) -> Curriculum: + """The ratified curriculum for *domain* — deterministic, corpus order. + + Unratified rows, rows whose terms have left their packs, and rows whose + connective is outside :data:`CONNECTIVE_FAMILY` are DROPPED here, at the + admission boundary, so that everything downstream can treat every chain it + sees as licensed premise material. A dropped chain is not a silent + weakening of an argument the way a dropped *premise* would be (ADR-0261 + §5.1): the exam path pins chain ids and fails loudly when a pinned chain + did not survive admission — see ``resolve_pinned``. + """ + chains: list[CurriculumChain] = [] + for corpus_id in DOMAIN_CORPORA.get(domain, ()): + for row in _ratified_rows(corpus_id, domain): + connective = str(row.get("connective") or "").strip() + family = CONNECTIVE_FAMILY.get(connective) + if family is None: + continue + chains.append( + CurriculumChain( + chain_id=str(row.get("chain_id") or ""), + subject=str(row.get("subject") or "").strip(), + connective=connective, + obj=str(row.get("object") or "").strip(), + family=family, + ) + ) + return Curriculum( + domain=domain, + chains=tuple(chains), + vocabulary=_domain_vocabulary(domain), + ) + + +def compile_premises( + curriculum: Curriculum, family: str +) -> tuple[tuple[str, ...], tuple[str, ...]]: + """``(premise_sentences, chain_ids)`` for *family*, in corpus order. + + Deterministic: the same curriculum and family always compile to the same + sentences in the same order, so a lane report over them is SHA-pinnable. + """ + chains = curriculum.family(family) + return tuple(c.sentence for c in chains), tuple(c.chain_id for c in chains) + + +class UnratifiedChain(LookupError): + """A lane case pinned a chain that is absent or did not survive admission.""" + + +def resolve_pinned(curriculum: Curriculum, chain_ids: tuple[str, ...]) -> tuple[CurriculumChain, ...]: + """The pinned chains, or raise (§4.3). + + A case that pins a chain the curriculum no longer ratifies MUST fail — not + quietly answer from whatever else is present. This is the provenance guard + that keeps "gold is a function of the curriculum" true over time: if the + curriculum changes under a case, the case breaks loudly. + """ + resolved: list[CurriculumChain] = [] + for chain_id in chain_ids: + chain = curriculum.chain_by_id(chain_id) + if chain is None: + raise UnratifiedChain( + f"{curriculum.domain}: pinned chain {chain_id!r} is absent or unratified" + ) + resolved.append(chain) + return tuple(resolved) + + +__all__ = [ + "CONNECTIVE_FAMILY", + "FAMILIES", + "Curriculum", + "CurriculumChain", + "UnratifiedChain", + "compile_premises", + "load_curriculum", + "resolve_pinned", +] diff --git a/tests/test_curriculum_serve.py b/tests/test_curriculum_serve.py new file mode 100644 index 00000000..a027e6f2 --- /dev/null +++ b/tests/test_curriculum_serve.py @@ -0,0 +1,211 @@ +"""Curriculum-grounded serving (ADR-0262) — the Phase-2 contract. + +These tests pin the epistemology, not just the plumbing: that answers come +from the ratified curriculum and nowhere else, that an untaught fact is +UNKNOWN rather than "no", that a chain the curriculum does not teach is never +composed into a claim, and that a question whose vocabulary the subject has +never been taught is declined rather than answered from world knowledge. +""" + +from __future__ import annotations + +import pytest + +from chat.curriculum_serve_license import curriculum_serve_license +from chat.curriculum_surface import ( + CurriculumQuery, + CurriculumRefusal, + band_for, + curriculum_grounded_surface, + decide_curriculum_question, + looks_like_curriculum_question, + read_curriculum_question, + resolve_domain, + resolve_family, +) +from evals.curriculum_serve.oracle import oracle_answer +from teaching.curriculum_premises import ( + UnratifiedChain, + compile_premises, + load_curriculum, + resolve_pinned, +) + + +# --- the curriculum is the only premise source -------------------------------- + + +def test_premises_come_only_from_ratified_chains() -> None: + curriculum = load_curriculum("physics") + premises, chain_ids = compile_premises(curriculum, "causal") + assert premises == ( + "force causes acceleration", + "acceleration causes motion", + "work causes energy", + "charge causes field", + "field causes force", + "temperature reveals motion", + "entropy reveals energy", + ) + assert all(cid.startswith("physics-") for cid in chain_ids) + + +def test_pinned_chain_that_is_not_ratified_fails_loudly() -> None: + """§4.3 provenance: a case whose curriculum moved under it must break, not + quietly answer from what remains.""" + curriculum = load_curriculum("physics") + with pytest.raises(UnratifiedChain): + resolve_pinned(curriculum, ("physics-causal-999",)) + + +def test_family_scoping_never_loses_a_taught_edge() -> None: + """Compilation is family-scoped; every taught edge is reachable from the + family its own connective implies, so scoping cannot hide an answer.""" + curriculum = load_curriculum("physics") + for chain in curriculum.chains: + premises, _ = compile_premises(curriculum, chain.family) + assert chain.sentence in premises + + +# --- the verdicts that matter ------------------------------------------------- + + +def test_taught_edge_is_entailed() -> None: + decision = decide_curriculum_question("Does force cause acceleration?") + assert decision.verdict == "entailed" + assert decision.band == band_for("physics", "causal") + + +def test_untaught_composition_is_unknown_not_entailed() -> None: + """The curriculum teaches force→acceleration and acceleration→motion. It + does NOT teach that causation composes, so "force causes motion" is + unsettled. Composing it would be inventing a rule nobody taught.""" + assert decide_curriculum_question("Does force cause motion?").verdict == "unknown" + + +def test_untaught_fact_is_unknown_never_refuted() -> None: + """Open-world: silence is not denial. No question answerable from a purely + positive curriculum may come back "no".""" + for text in ( + "Does mass require energy?", # true in the world, untaught + "Does motion cause force?", # the taught edge runs the other way + "Does work cause motion?", # both terms taught, no relation + ): + assert decide_curriculum_question(text).verdict == "unknown", text + + +def test_relation_is_read_at_face_value() -> None: + """"entropy reveals energy" is taught; "entropy causes energy" is not. Same + family, different relation — the curriculum said one and not the other.""" + assert decide_curriculum_question("Does entropy reveal energy?").verdict == "entailed" + assert decide_curriculum_question("Does entropy cause energy?").verdict == "unknown" + + +def test_question_may_spell_the_relation_in_its_base_form() -> None: + """The question says "cause", the curriculum says "causes" — normalized by + the SAME closed agreement relation the verb band uses (ADR-0260).""" + assert decide_curriculum_question("Does charge cause field?").verdict == "entailed" + + +# --- anti-recall --------------------------------------------------------------- + + +@pytest.mark.parametrize( + "text", ["Does gravity cause acceleration?", "Does current cause field?"] +) +def test_untaught_vocabulary_declines_rather_than_recalling(text: str) -> None: + """Both are true physics and both are outside every mounted pack. The + system has no curriculum to decide them from, and says so.""" + decision = decide_curriculum_question(text) + assert decision.verdict == "declined" + assert decision.reason == "untaught_vocabulary" + surface = curriculum_grounded_surface(text) + assert surface is not None + assert "haven't been taught" in surface + assert "not a claim about the world" in surface + + +# --- typed refusals ------------------------------------------------------------ + + +@pytest.mark.parametrize( + "text, reason", + [ + ("Does force explain acceleration?", "out_of_curriculum"), + ("Does the force cause acceleration?", "question_shape_out_of_band"), + ("Does force accelerate?", "question_shape_out_of_band"), + ], +) +def test_typed_refusals(text: str, reason: str) -> None: + assert decide_curriculum_question(text).reason == reason + + +def test_non_question_text_is_not_claimed_at_all() -> None: + """The composer returns None for anything that is not a ``Does …?`` + question, so pre-existing dispatch is byte-identical.""" + assert curriculum_grounded_surface("Force causes acceleration.") is None + assert not looks_like_curriculum_question("Is force a cause of acceleration?") + + +def test_question_reader_shape() -> None: + query = read_curriculum_question("Does force cause acceleration?") + assert isinstance(query, CurriculumQuery) + assert (query.subject, query.verb, query.obj) == ("force", "cause", "acceleration") + assert isinstance(read_curriculum_question("Why does it move?"), CurriculumRefusal) + + +def test_domain_and_family_routing() -> None: + query = read_curriculum_question("Does force cause acceleration?") + assert isinstance(query, CurriculumQuery) + assert resolve_domain(query) == "physics" + assert resolve_family(query) == "causal" + + +# --- the license posture ------------------------------------------------------- + + +def test_answers_are_disclosed_until_a_band_earns_serve() -> None: + """No curriculum band holds a SERVE license yet (ADR-0262 §5: ratified + curriculum volume, not machinery, is the binding constraint), so every + answer is served DISCLOSED — sound, and honest about its track record.""" + assert curriculum_serve_license(band_for("physics", "causal")) is None + surface = curriculum_grounded_surface("Does force cause acceleration?") + assert surface is not None + assert surface.startswith("(answered from my curriculum") + + +def test_an_earned_band_is_served_authoritatively() -> None: + """The gate is wired, not stubbed: hand it a licensed band and the hedge + disappears.""" + + class _Licensed: + licensed = True + + surface = curriculum_grounded_surface( + "Does force cause acceleration?", license_lookup=lambda band: _Licensed() + ) + assert surface is not None + assert not surface.startswith("(answered from my curriculum") + assert "physics curriculum teaches that force causes acceleration" in surface + + +# --- the independent oracle ---------------------------------------------------- + + +def test_oracle_is_independent_and_agrees() -> None: + """The oracle shares no code with the serving path; agreement across the + verdict classes is the evidence the compiler reads the curriculum right.""" + for subject, relation, obj, expected in ( + ("force", "cause", "acceleration", "entailed"), + ("force", "cause", "motion", "unknown"), + ("gravity", "cause", "acceleration", "declined"), + ): + assert oracle_answer("physics", subject, relation, obj).verdict == expected + + +def test_oracle_reports_reachability_without_claiming_it() -> None: + """force→acceleration→motion is reachable at depth 2 and still UNKNOWN — + the property that proves composition is not being silently performed.""" + verdict = oracle_answer("physics", "force", "cause", "motion") + assert verdict.verdict == "unknown" + assert verdict.depth == 2 diff --git a/tests/test_ratified_ledger_bridge.py b/tests/test_ratified_ledger_bridge.py new file mode 100644 index 00000000..107be40a --- /dev/null +++ b/tests/test_ratified_ledger_bridge.py @@ -0,0 +1,113 @@ +"""The ratified-ledger bridge (ADR-0263) — seal → ratify → SHA-verify → gate. + +These tests pin the four rules the three adapters depend on, and the property +that made the extraction safe: re-sealing through the bridge reproduces the +committed artifacts byte-for-byte. +""" + +from __future__ import annotations + +import json + +import pytest + +from core.ratified_ledger import ( + RatifiedLedgerError, + load_sealed_ledger, + seal_artifact, + serve_license, + write_sealed_ledger, +) +from core.reliability_gate import ClassTally + + +def _tally(name: str, correct: int, wrong: int = 0) -> ClassTally: + return ClassTally(class_name=name, correct=correct, wrong=wrong, refused=0) + + +def test_seal_then_load_round_trips(tmp_path) -> None: + ledger = {"alpha": _tally("alpha", 700), "beta": _tally("beta", 12)} + path = tmp_path / "ledger.json" + write_sealed_ledger( + path, seal_artifact(ledger, schema="t_v1", note="n", provenance="p") + ) + loaded = load_sealed_ledger(path) + assert set(loaded) == {"alpha", "beta"} + assert loaded["alpha"].correct == 700 + + +def test_a_hand_edited_ledger_is_rejected(tmp_path) -> None: + """Rule 2 — tamper-evidence is structural. Editing a tally without + re-sealing does not produce a slightly-wrong ledger; it produces an + unratified one, and loading REFUSES.""" + path = tmp_path / "ledger.json" + write_sealed_ledger( + path, + seal_artifact({"alpha": _tally("alpha", 10)}, schema="t_v1", note="n", provenance="p"), + ) + artifact = json.loads(path.read_text()) + artifact["classes"]["alpha"]["correct"] = 9_999 + path.write_text(json.dumps(artifact, indent=2, sort_keys=True) + "\n") + with pytest.raises(RatifiedLedgerError): + load_sealed_ledger(path) + + +def test_missing_ledger_refuses_unless_declared_optional(tmp_path) -> None: + """A capability that SHIPS with a ledger is broken without it; one whose + practice volume is still being built legitimately has none. Neither is + answered by guessing a license.""" + path = tmp_path / "absent.json" + with pytest.raises(RatifiedLedgerError): + load_sealed_ledger(path) + assert load_sealed_ledger(path, missing_ok=True) == {} + + +def test_absent_class_is_never_licensed() -> None: + """Rule 4 — no evidence is not a license; the caller's ``None`` branch is + what serves the disclosed surface.""" + assert serve_license("nobody", {"alpha": _tally("alpha", 700)}) is None + + +def test_volume_floor_is_enforced_by_the_gate() -> None: + """A perfect but small record does not clear θ_SERVE=0.99 — the Wilson + floor is what makes a license *earned* rather than merely clean.""" + small = serve_license("alpha", {"alpha": _tally("alpha", 12)}) + large = serve_license("alpha", {"alpha": _tally("alpha", 720)}) + assert small is not None and not small.licensed + assert large is not None and large.licensed + + +def test_a_single_wrong_costs_the_license() -> None: + dirty = serve_license("alpha", {"alpha": _tally("alpha", 720, wrong=1)}) + assert dirty is not None and not dirty.licensed + + +def test_committed_deduction_ledger_reseals_byte_identically() -> None: + """The extraction's safety property: the bridge writes what the bespoke + sealers wrote, so adopting it moves no committed artifact and no lane pin.""" + from pathlib import Path + + from evals.deduction_serve.practice.runner import build_sealed_artifact + + committed = Path("chat/data/deduction_serve_ledger.json") + expected = json.dumps(build_sealed_artifact(), indent=2, sort_keys=True) + "\n" + assert committed.read_text(encoding="utf-8") == expected + + +def test_every_adapter_reads_through_the_bridge() -> None: + """All three instances now share one loader — the property that makes a + future change to the ratification rule land everywhere at once.""" + import chat.curriculum_serve_license as curriculum + import chat.deduction_serve_license as deduction + import generate.determine.estimation_license as estimation + + for module in (deduction, estimation, curriculum): + source = module.__file__ or "" + assert source + text = open(source, encoding="utf-8").read() + assert "core.ratified_ledger" in text + # The signal that an adapter still verifies for itself is that it + # hashes for itself; docstrings may (and do) still explain the rule. + assert "formation.hashing" not in text, ( + f"{module.__name__} still re-implements verification" + )