Merge 'feat(generalization): curriculum-grounded serving, ratified-ledger bridge, math Phase 4.1 status' from feat/curriculum-serve-physics into main
ADR-0262 curriculum-grounded serving (physics lane 32/32 wrong=0), ADR-0263 the ratified-ledger bridge, and the measured Phase 4.1 null result.
This commit is contained in:
commit
0ae54ebb7b
28 changed files with 2386 additions and 139 deletions
|
|
@ -44,6 +44,7 @@ is a CI failure (`.github/workflows/lane-shas.yml`).
|
||||||
| ADR-0131 | `math_teaching_corpus_v1` | Math teaching corpus replays deterministically; all chains pass exit criterion (correct_rate=1.0, wrong=0) | `evals/math_teaching_corpus/v1/report.json` | `eaf160d145da29f9050ede8d58bf111b0f651dd40aeae9201857d0b97e014dd4` |
|
| ADR-0131 | `math_teaching_corpus_v1` | Math teaching corpus replays deterministically; all chains pass exit criterion (correct_rate=1.0, wrong=0) | `evals/math_teaching_corpus/v1/report.json` | `eaf160d145da29f9050ede8d58bf111b0f651dd40aeae9201857d0b97e014dd4` |
|
||||||
| ADR-0206 | `deductive_logic_v1` | Propositional entailment scored against an independent truth-table oracle; dev+holdout+external 716/716 correct, wrong=0, refused=0 | `evals/deductive_logic/report.json` | `97a230949016e38d5e3f37a69e4245b320575ee70e5af92ff7607f7b05f74b5f` |
|
| ADR-0206 | `deductive_logic_v1` | Propositional entailment scored against an independent truth-table oracle; dev+holdout+external 716/716 correct, wrong=0, refused=0 | `evals/deductive_logic/report.json` | `97a230949016e38d5e3f37a69e4245b320575ee70e5af92ff7607f7b05f74b5f` |
|
||||||
| ADR-0256 | `deduction_serve_v1` | Flag-gated deduction serving decides real English/member/fused/verb/existential arguments end-to-end under earned SERVE licenses; wrong=0 across all splits | `evals/deduction_serve/report.json` | `0b461a5a49c8f8260ca87d0c9c9f9a17232bd1fdedd982e34649eedf9cca30b5` |
|
| ADR-0256 | `deduction_serve_v1` | Flag-gated deduction serving decides real English/member/fused/verb/existential arguments end-to-end under earned SERVE licenses; wrong=0 across all splits | `evals/deduction_serve/report.json` | `0b461a5a49c8f8260ca87d0c9c9f9a17232bd1fdedd982e34649eedf9cca30b5` |
|
||||||
|
| ADR-0262 | `curriculum_serve_v1` | Flag-gated curriculum serving answers exam questions from a subject's RATIFIED chain corpus only; untaught facts return UNKNOWN (anti-recall probes enforced), wrong=0 | `evals/curriculum_serve/report.json` | `d9e7ba500f040b865870413a940ee9a49910ac22e1a89c9feec1a60bdd2513f1` |
|
||||||
|
|
||||||
## Verification
|
## Verification
|
||||||
|
|
||||||
|
|
|
||||||
68
chat/curriculum_serve_license.py
Normal file
68
chat/curriculum_serve_license.py
Normal file
|
|
@ -0,0 +1,68 @@
|
||||||
|
"""Serving-side SERVE license for curriculum-grounded answers (ADR-0262).
|
||||||
|
|
||||||
|
Reads the **ratified, committed** curriculum-serve ledger
|
||||||
|
(``chat/data/curriculum_serve_ledger.json``) and exposes, per *(subject ×
|
||||||
|
relation family)* band, whether the FULL serving pipeline (curriculum
|
||||||
|
compiler → argument reader → ROBDD engine) has earned ``Action.SERVE`` under
|
||||||
|
the safe default ceilings (θ_SERVE=0.99). The engine READS this artifact; it
|
||||||
|
never writes it — the sealed-practice output of
|
||||||
|
``evals.curriculum_serve.practice.runner.seal_ledger`` is the only writer, and
|
||||||
|
its ``content_sha256`` is verified on load so a hand-edited ledger is rejected
|
||||||
|
rather than silently trusted.
|
||||||
|
|
||||||
|
The third instance of the seal → ratify → SHA-verify → serve-gate pattern, and
|
||||||
|
the one that made the shared bridge worth extracting: this module is now an
|
||||||
|
ADAPTER over ``core.ratified_ledger`` (ADR-0263). It names the artifact, keeps
|
||||||
|
the memoization, and declares the one thing that genuinely differs — this
|
||||||
|
ledger is legitimately ABSENT, because no curriculum band has earned anything
|
||||||
|
yet (ADR-0262 §5.1: the binding constraint is ratified curriculum volume). An
|
||||||
|
absent ledger reads as an empty table, so every answer is served DISCLOSED.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from functools import lru_cache
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from core.ratified_ledger import (
|
||||||
|
RatifiedLedgerError as RatifiedCurriculumLedgerError,
|
||||||
|
load_sealed_ledger,
|
||||||
|
serve_license,
|
||||||
|
)
|
||||||
|
from core.reliability_gate import Ceilings, ClassTally, LicenseDecision
|
||||||
|
|
||||||
|
_LEDGER_PATH = Path(__file__).resolve().parent / "data" / "curriculum_serve_ledger.json"
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=1)
|
||||||
|
def load_ratified_ledger() -> dict[str, ClassTally]:
|
||||||
|
"""Load + verify the ratified curriculum-serve ledger → per-band tallies.
|
||||||
|
|
||||||
|
An ABSENT ledger is not an error here: no curriculum band has earned
|
||||||
|
anything yet, and the honest reading of "no file" is "no committed
|
||||||
|
evidence", which the gate turns into a disclosed answer rather than a
|
||||||
|
withheld one.
|
||||||
|
"""
|
||||||
|
return load_sealed_ledger(_LEDGER_PATH, missing_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
def curriculum_serve_license(
|
||||||
|
band: str,
|
||||||
|
*,
|
||||||
|
ledger: dict[str, ClassTally] | None = None,
|
||||||
|
ceilings: Ceilings | None = None,
|
||||||
|
) -> LicenseDecision | None:
|
||||||
|
"""The ``Action.SERVE`` license for a curriculum band, or ``None``.
|
||||||
|
|
||||||
|
``None`` means the band has no committed evidence → never licensed; the
|
||||||
|
caller serves a disclosed (hedged) surface, the safe default.
|
||||||
|
"""
|
||||||
|
ledger = ledger if ledger is not None else load_ratified_ledger()
|
||||||
|
return serve_license(band, ledger, ceilings=ceilings)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"RatifiedCurriculumLedgerError",
|
||||||
|
"curriculum_serve_license",
|
||||||
|
"load_ratified_ledger",
|
||||||
|
]
|
||||||
361
chat/curriculum_surface.py
Normal file
361
chat/curriculum_surface.py
Normal file
|
|
@ -0,0 +1,361 @@
|
||||||
|
"""chat/curriculum_surface.py — CURRICULUM composer (generalization arc Phase 2).
|
||||||
|
|
||||||
|
The second question shape of the curriculum-entailment gold contract
|
||||||
|
(docs/plans/generalization-arc-2026-07-24.md §4.2b): an exam question that
|
||||||
|
supplies only the QUERY — "Does force cause acceleration?" — answered from the
|
||||||
|
ratified curriculum of the subject it is about, and from nothing else.
|
||||||
|
|
||||||
|
The decision is made by the SAME argument bands the deduction composer uses
|
||||||
|
(ADR-0256…0261). This module compiles ratified chains into premise sentences,
|
||||||
|
appends the question as a "Therefore …" conclusion, and hands the assembled
|
||||||
|
argument to `deduction_grounded_surface`'s decider. There is no subject-
|
||||||
|
specific reasoning code anywhere in this path — physics differs from
|
||||||
|
philosophy only in which curriculum rows load, which is what makes "port the
|
||||||
|
lifecycle to a new subject" a data operation rather than an engineering one.
|
||||||
|
|
||||||
|
Why an UNTAUGHT fact is UNKNOWN and never "no": the curriculum is read
|
||||||
|
OPEN-world. "The curriculum does not teach it" is not "it is false", and a
|
||||||
|
system that decoded the former into the latter would be inventing negative
|
||||||
|
knowledge it was never given. The only verdicts reachable from a purely
|
||||||
|
positive curriculum are therefore ENTAILED and UNKNOWN — a real limit of the
|
||||||
|
present corpora, recorded in ADR-0262 §5 rather than papered over.
|
||||||
|
|
||||||
|
Fail-closed (INV-34): once `looks_like_curriculum_question` fires, every path
|
||||||
|
below returns a committed, honest surface — typed refusal or decided verdict —
|
||||||
|
never a silent fall-through to a different composer.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Callable
|
||||||
|
|
||||||
|
from chat.curriculum_serve_license import curriculum_serve_license
|
||||||
|
from core.reliability_gate import LicenseDecision
|
||||||
|
from generate.proof_chain.entail import (
|
||||||
|
INCONSISTENT_PREMISES,
|
||||||
|
Entailment,
|
||||||
|
evaluate_entailment_with_trace,
|
||||||
|
)
|
||||||
|
from generate.proof_chain.exist import ExistArgument, read_exist_argument
|
||||||
|
from generate.proof_chain.verb import VerbArgument, read_verb_argument, verb_forms_link
|
||||||
|
from teaching.curriculum_premises import (
|
||||||
|
CONNECTIVE_FAMILY,
|
||||||
|
compile_premises,
|
||||||
|
load_curriculum,
|
||||||
|
)
|
||||||
|
|
||||||
|
#: Subjects this composer serves, in a stable resolution order. A question is
|
||||||
|
#: routed to the subject whose ratified vocabulary contains BOTH of its terms;
|
||||||
|
#: more than one match is an `ambiguous_reading` refusal, never a guess.
|
||||||
|
SERVED_DOMAINS: tuple[str, ...] = (
|
||||||
|
"physics",
|
||||||
|
"mathematics_logic",
|
||||||
|
"systems_software",
|
||||||
|
"philosophy_theology",
|
||||||
|
)
|
||||||
|
|
||||||
|
#: The closed exam-question grammar: ``Does <subject> <verb> <object>?``.
|
||||||
|
#: Deliberately one shape. A question the grammar cannot read refuses typed —
|
||||||
|
#: the reader is never asked to guess which token is the relation.
|
||||||
|
_QUESTION_RE = re.compile(r"^\s*does\s+(.+?)\s*\?\s*$", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class CurriculumQuery:
|
||||||
|
"""A read exam question: the edge the curriculum is being asked about."""
|
||||||
|
|
||||||
|
subject: str
|
||||||
|
verb: str
|
||||||
|
obj: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class CurriculumRefusal:
|
||||||
|
"""A typed refusal — the closed reason vocabulary of ADR-0262 §4.6."""
|
||||||
|
|
||||||
|
reason: str
|
||||||
|
detail: str = ""
|
||||||
|
|
||||||
|
|
||||||
|
def looks_like_curriculum_question(text: str) -> bool:
|
||||||
|
"""True iff *text* is a ``Does …?`` polar question.
|
||||||
|
|
||||||
|
A cheap, deterministic COMMIT gate — not a decision. The reader below
|
||||||
|
remains the sole authority on whether the question is actually readable.
|
||||||
|
"""
|
||||||
|
return bool(_QUESTION_RE.match(text or ""))
|
||||||
|
|
||||||
|
|
||||||
|
def read_curriculum_question(text: str) -> CurriculumQuery | CurriculumRefusal:
|
||||||
|
"""Read *text* as ``Does <subject> <verb> <object>?``, or refuse (typed)."""
|
||||||
|
match = _QUESTION_RE.match(text or "")
|
||||||
|
if match is None:
|
||||||
|
return CurriculumRefusal("question_shape_out_of_band", text or "")
|
||||||
|
tokens = match.group(1).replace(",", " ").lower().split()
|
||||||
|
if len(tokens) != 3:
|
||||||
|
# Two-token ("does force act") and four-token ("does a force cause
|
||||||
|
# acceleration") shapes are genuinely different readings; the closed
|
||||||
|
# grammar refuses rather than dropping or inventing a slot.
|
||||||
|
return CurriculumRefusal("question_shape_out_of_band", " ".join(tokens))
|
||||||
|
return CurriculumQuery(tokens[0], tokens[1], tokens[2])
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_domain(query: CurriculumQuery) -> str | CurriculumRefusal:
|
||||||
|
"""The subject whose ratified vocabulary contains BOTH of the query's
|
||||||
|
terms — or a typed refusal (§4.6).
|
||||||
|
|
||||||
|
``untaught_vocabulary`` is the anti-recall boundary made mechanical: a
|
||||||
|
term no mounted pack teaches cannot be reasoned about here at all, however
|
||||||
|
familiar it is. ``ambiguous_reading`` fires when two subjects would both
|
||||||
|
claim the question; the composer declines rather than picking one.
|
||||||
|
"""
|
||||||
|
matches = [
|
||||||
|
domain
|
||||||
|
for domain in SERVED_DOMAINS
|
||||||
|
if {query.subject, query.obj} <= load_curriculum(domain).vocabulary
|
||||||
|
]
|
||||||
|
if not matches:
|
||||||
|
return CurriculumRefusal(
|
||||||
|
"untaught_vocabulary", f"{query.subject} / {query.obj}"
|
||||||
|
)
|
||||||
|
if len(matches) > 1:
|
||||||
|
return CurriculumRefusal("ambiguous_reading", ", ".join(matches))
|
||||||
|
return matches[0]
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_family(query: CurriculumQuery) -> str | CurriculumRefusal:
|
||||||
|
"""The relation family of the query's verb, under the SAME closed
|
||||||
|
agreement relation the verb-predicate band uses (ADR-0260) — so a question
|
||||||
|
may spell the relation in its base form ("cause") while the curriculum
|
||||||
|
states it in the third person ("causes")."""
|
||||||
|
for connective, family in CONNECTIVE_FAMILY.items():
|
||||||
|
if verb_forms_link(query.verb, connective):
|
||||||
|
return family
|
||||||
|
return CurriculumRefusal("out_of_curriculum", query.verb)
|
||||||
|
|
||||||
|
|
||||||
|
def _query_sentence(query: CurriculumQuery, family: str) -> str:
|
||||||
|
"""The conclusion sentence, spelled with the CURRICULUM's own connective
|
||||||
|
form so the reader's agreement linking has nothing to do — the question's
|
||||||
|
base form is normalized here, once, visibly."""
|
||||||
|
for connective in CONNECTIVE_FAMILY:
|
||||||
|
if CONNECTIVE_FAMILY[connective] == family and verb_forms_link(
|
||||||
|
query.verb, connective
|
||||||
|
):
|
||||||
|
return f"{query.subject} {connective} {query.obj}"
|
||||||
|
return f"{query.subject} {query.verb} {query.obj}" # pragma: no cover - guarded above
|
||||||
|
|
||||||
|
|
||||||
|
def band_for(domain: str, family: str) -> str:
|
||||||
|
"""The capability band an answer is licensed under: *(subject × relation
|
||||||
|
family)* — ADR-0262 §4."""
|
||||||
|
return f"curriculum_{domain}_{family}"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class CurriculumDecision:
|
||||||
|
"""What the composer decided, before rendering — the lane scores this."""
|
||||||
|
|
||||||
|
verdict: str # entailed | refuted | unknown | declined
|
||||||
|
band: str # "" when the question never reached a band
|
||||||
|
reason: str # typed refusal reason, or "" when decided
|
||||||
|
domain: str = ""
|
||||||
|
family: str = ""
|
||||||
|
premise_count: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
def decide_curriculum_question(text: str) -> CurriculumDecision:
|
||||||
|
"""Read, route, compile, and decide — the production decision path.
|
||||||
|
|
||||||
|
Kept separate from the rendering below so the lane can pin the DECISION
|
||||||
|
without pinning prose (the same split ``evals/deduction_serve/runner.py``
|
||||||
|
uses).
|
||||||
|
"""
|
||||||
|
query = read_curriculum_question(text)
|
||||||
|
if isinstance(query, CurriculumRefusal):
|
||||||
|
return CurriculumDecision("declined", "", query.reason)
|
||||||
|
domain = resolve_domain(query)
|
||||||
|
if isinstance(domain, CurriculumRefusal):
|
||||||
|
return CurriculumDecision("declined", "", domain.reason)
|
||||||
|
family = resolve_family(query)
|
||||||
|
if isinstance(family, CurriculumRefusal):
|
||||||
|
return CurriculumDecision("declined", "", family.reason, domain=domain)
|
||||||
|
|
||||||
|
curriculum = load_curriculum(domain)
|
||||||
|
premises, _chain_ids = compile_premises(curriculum, family)
|
||||||
|
if not premises:
|
||||||
|
return CurriculumDecision(
|
||||||
|
"declined", "", "empty_curriculum", domain=domain, family=family
|
||||||
|
)
|
||||||
|
conclusion = _query_sentence(query, family)
|
||||||
|
argument = ". ".join(premises) + f". Therefore {conclusion}."
|
||||||
|
|
||||||
|
# The SAME argument bands decide it. The verb band reads these sentences
|
||||||
|
# (each is `<term> <connective> <term>`); the existential band is tried
|
||||||
|
# after it exactly as in serving, so the cascade order here matches the
|
||||||
|
# deduction composer's tail and no subject-specific decider exists.
|
||||||
|
arg = read_verb_argument(argument)
|
||||||
|
if not isinstance(arg, VerbArgument):
|
||||||
|
arg = read_exist_argument(argument)
|
||||||
|
if not isinstance(arg, ExistArgument):
|
||||||
|
return CurriculumDecision(
|
||||||
|
"declined", "", "compiled_premises_unreadable",
|
||||||
|
domain=domain, family=family, premise_count=len(premises),
|
||||||
|
)
|
||||||
|
trace = evaluate_entailment_with_trace(arg.premise_formulas, arg.query_formula)
|
||||||
|
verdict = {
|
||||||
|
Entailment.ENTAILED: "entailed",
|
||||||
|
Entailment.REFUTED: "refuted",
|
||||||
|
Entailment.UNKNOWN: "unknown",
|
||||||
|
Entailment.REFUSED: "declined",
|
||||||
|
}[trace.outcome]
|
||||||
|
reason = trace.reason if trace.outcome is Entailment.REFUSED else ""
|
||||||
|
return CurriculumDecision(
|
||||||
|
verdict,
|
||||||
|
band_for(domain, family),
|
||||||
|
reason,
|
||||||
|
domain=domain,
|
||||||
|
family=family,
|
||||||
|
premise_count=len(premises),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
_LicenseLookup = Callable[[str], LicenseDecision | None]
|
||||||
|
|
||||||
|
_REFUSAL_SURFACE = {
|
||||||
|
"question_shape_out_of_band": (
|
||||||
|
"That reads as a question about a subject I study, but not in a form I "
|
||||||
|
"can decide yet (I read \"Does <term> <relation> <term>?\")."
|
||||||
|
),
|
||||||
|
"untaught_vocabulary": (
|
||||||
|
"I haven't been taught {detail} — so I can't decide anything about it "
|
||||||
|
"from what I know. That's a gap in my curriculum, not a claim about "
|
||||||
|
"the world."
|
||||||
|
),
|
||||||
|
"out_of_curriculum": (
|
||||||
|
"I haven't been taught the relation \"{detail}\", so I can't decide "
|
||||||
|
"that question from my curriculum."
|
||||||
|
),
|
||||||
|
"ambiguous_reading": (
|
||||||
|
"Those terms are taught in more than one subject I study ({detail}), "
|
||||||
|
"so I can't tell which curriculum you're asking about."
|
||||||
|
),
|
||||||
|
"empty_curriculum": (
|
||||||
|
"I have no ratified material for that kind of relation yet, so there "
|
||||||
|
"is nothing for me to decide it from."
|
||||||
|
),
|
||||||
|
"compiled_premises_unreadable": (
|
||||||
|
"I have the curriculum for that, but I can't read it back precisely "
|
||||||
|
"enough to decide the question yet."
|
||||||
|
),
|
||||||
|
INCONSISTENT_PREMISES: (
|
||||||
|
"The curriculum I have for that is inconsistent — I won't assert "
|
||||||
|
"anything from it until it is corrected."
|
||||||
|
),
|
||||||
|
}
|
||||||
|
_DEFAULT_REFUSAL_SURFACE = (
|
||||||
|
"I can't decide that from my curriculum as stated."
|
||||||
|
)
|
||||||
|
|
||||||
|
#: Disclosed hedge for a band that has not earned SERVE — same posture as
|
||||||
|
#: the deduction composer (ADR-0256): the answer is sound, the track record
|
||||||
|
#: is not yet demonstrated, so it is served DISCLOSED rather than withheld.
|
||||||
|
_UNVERIFIED_SHAPE_DISCLOSURE = (
|
||||||
|
"(answered from my curriculum, but I haven't yet earned a verified track "
|
||||||
|
"record on questions of this shape) "
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def curriculum_grounded_surface(
|
||||||
|
text: str, *, license_lookup: _LicenseLookup = curriculum_serve_license,
|
||||||
|
) -> str | None:
|
||||||
|
"""Return a deterministic CURRICULUM-tier surface, or ``None``.
|
||||||
|
|
||||||
|
``None`` only when *text* is not a ``Does …?`` question at all — the
|
||||||
|
caller then falls through to its pre-existing dispatch, byte-identical to
|
||||||
|
before this composer existed.
|
||||||
|
"""
|
||||||
|
if not looks_like_curriculum_question(text):
|
||||||
|
return None
|
||||||
|
decision = decide_curriculum_question(text)
|
||||||
|
query = read_curriculum_question(text)
|
||||||
|
# Restate the question in the CURRICULUM's own spelling of the relation —
|
||||||
|
# the normalization the reader performed is shown, not hidden.
|
||||||
|
shown = (
|
||||||
|
_query_sentence(query, decision.family)
|
||||||
|
if isinstance(query, CurriculumQuery) and decision.family
|
||||||
|
else text.strip()
|
||||||
|
)
|
||||||
|
if decision.verdict == "declined":
|
||||||
|
template = _REFUSAL_SURFACE.get(decision.reason, _DEFAULT_REFUSAL_SURFACE)
|
||||||
|
return template.format(detail=_detail_of(text, decision))
|
||||||
|
if decision.verdict == "entailed":
|
||||||
|
surface = (
|
||||||
|
f"Yes — my {decision.domain} curriculum teaches that {shown}, "
|
||||||
|
f"directly, among the {decision.premise_count} {decision.family} "
|
||||||
|
f"relations it states."
|
||||||
|
)
|
||||||
|
elif decision.verdict == "refuted":
|
||||||
|
surface = (
|
||||||
|
f"No — my {decision.domain} curriculum rules out that {shown}."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
surface = (
|
||||||
|
f"My {decision.domain} curriculum doesn't settle whether {shown}. "
|
||||||
|
f"It states {decision.premise_count} {decision.family} relations, "
|
||||||
|
f"and none of them decides this one — which is not the same as "
|
||||||
|
f"its being false."
|
||||||
|
)
|
||||||
|
return _license_gate(surface, decision.band, license_lookup)
|
||||||
|
|
||||||
|
|
||||||
|
def _detail_of(text: str, decision: CurriculumDecision) -> str:
|
||||||
|
"""The user-visible fragment a typed refusal names (empty when the reason
|
||||||
|
carries no detail slot)."""
|
||||||
|
query = read_curriculum_question(text)
|
||||||
|
if not isinstance(query, CurriculumQuery):
|
||||||
|
return ""
|
||||||
|
if decision.reason == "untaught_vocabulary":
|
||||||
|
untaught = [
|
||||||
|
term
|
||||||
|
for term in (query.subject, query.obj)
|
||||||
|
if not any(
|
||||||
|
term in load_curriculum(d).vocabulary for d in SERVED_DOMAINS
|
||||||
|
)
|
||||||
|
]
|
||||||
|
return " or ".join(untaught) if untaught else f"{query.subject} / {query.obj}"
|
||||||
|
if decision.reason == "out_of_curriculum":
|
||||||
|
return query.verb
|
||||||
|
if decision.reason == "ambiguous_reading":
|
||||||
|
return ", ".join(
|
||||||
|
d
|
||||||
|
for d in SERVED_DOMAINS
|
||||||
|
if {query.subject, query.obj} <= load_curriculum(d).vocabulary
|
||||||
|
)
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def _license_gate(surface: str, band: str, license_lookup: _LicenseLookup) -> str:
|
||||||
|
"""Serve *surface* authoritatively iff *band* holds a SERVE license; else
|
||||||
|
serve the same sound answer DISCLOSED (hedged)."""
|
||||||
|
decision = license_lookup(band)
|
||||||
|
if decision is not None and decision.licensed:
|
||||||
|
return surface
|
||||||
|
return _UNVERIFIED_SHAPE_DISCLOSURE + surface
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"CurriculumDecision",
|
||||||
|
"CurriculumQuery",
|
||||||
|
"CurriculumRefusal",
|
||||||
|
"SERVED_DOMAINS",
|
||||||
|
"band_for",
|
||||||
|
"curriculum_grounded_surface",
|
||||||
|
"decide_curriculum_question",
|
||||||
|
"looks_like_curriculum_question",
|
||||||
|
"read_curriculum_question",
|
||||||
|
"resolve_domain",
|
||||||
|
"resolve_family",
|
||||||
|
]
|
||||||
|
|
@ -10,27 +10,27 @@ artifact is the sealed-practice output of
|
||||||
is verified on load, so a hand-edited (un-ratified) ledger is rejected rather
|
is verified on load, so a hand-edited (un-ratified) ledger is rejected rather
|
||||||
than silently trusted.
|
than silently trusted.
|
||||||
|
|
||||||
Mirrors ``generate.determine.estimation_license`` exactly: immutable ratified
|
The load/verify/gate mechanics live in ``core.ratified_ledger`` (ADR-0263) —
|
||||||
data parsed once and cached; the gate (``license_for``) is pure; ceilings stay
|
this module is the deduction-serve ADAPTER over that bridge: it names the
|
||||||
at the safe defaults (invariant #4 — the engine cannot raise its own bar).
|
artifact, keeps the memoization, and preserves its own public API. Ceilings
|
||||||
|
stay at the safe defaults (invariant #4 — the engine cannot raise its own bar).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from core.reliability_gate import Action, Ceilings, ClassTally, LicenseDecision, license_for
|
from core.ratified_ledger import (
|
||||||
from formation.hashing import sha256_of
|
RatifiedLedgerError,
|
||||||
|
load_sealed_ledger,
|
||||||
|
serve_license,
|
||||||
|
)
|
||||||
|
from core.reliability_gate import Ceilings, ClassTally, LicenseDecision
|
||||||
|
|
||||||
_LEDGER_PATH = Path(__file__).resolve().parent / "data" / "deduction_serve_ledger.json"
|
_LEDGER_PATH = Path(__file__).resolve().parent / "data" / "deduction_serve_ledger.json"
|
||||||
|
|
||||||
|
|
||||||
class RatifiedLedgerError(ValueError):
|
|
||||||
"""The committed deduction-serve ledger is missing, malformed, or tampered with."""
|
|
||||||
|
|
||||||
|
|
||||||
@lru_cache(maxsize=1)
|
@lru_cache(maxsize=1)
|
||||||
def load_ratified_ledger() -> dict[str, ClassTally]:
|
def load_ratified_ledger() -> dict[str, ClassTally]:
|
||||||
"""Load + verify the ratified deduction-serve ledger → per-band ``ClassTally``.
|
"""Load + verify the ratified deduction-serve ledger → per-band ``ClassTally``.
|
||||||
|
|
@ -39,30 +39,7 @@ def load_ratified_ledger() -> dict[str, ClassTally]:
|
||||||
recomputed ``content_sha256`` does not match the committed one (tamper-evidence:
|
recomputed ``content_sha256`` does not match the committed one (tamper-evidence:
|
||||||
only the sealed-practice output is trusted, never a hand-edited ledger).
|
only the sealed-practice output is trusted, never a hand-edited ledger).
|
||||||
"""
|
"""
|
||||||
try:
|
return load_sealed_ledger(_LEDGER_PATH)
|
||||||
artifact = json.loads(_LEDGER_PATH.read_text(encoding="utf-8"))
|
|
||||||
except (OSError, json.JSONDecodeError) as exc: # pragma: no cover - defensive
|
|
||||||
raise RatifiedLedgerError(f"cannot read ratified ledger: {exc}") from exc
|
|
||||||
|
|
||||||
classes = artifact.get("classes")
|
|
||||||
if not isinstance(classes, dict):
|
|
||||||
raise RatifiedLedgerError("ratified ledger has no 'classes' table")
|
|
||||||
if sha256_of(classes) != artifact.get("content_sha256"):
|
|
||||||
raise RatifiedLedgerError(
|
|
||||||
"ratified ledger content_sha256 mismatch — not the sealed-practice output"
|
|
||||||
)
|
|
||||||
|
|
||||||
ledger: dict[str, ClassTally] = {}
|
|
||||||
for cls, counts in classes.items():
|
|
||||||
ledger[cls] = ClassTally(
|
|
||||||
class_name=cls,
|
|
||||||
correct=int(counts.get("correct", 0)),
|
|
||||||
wrong=int(counts.get("wrong", 0)),
|
|
||||||
refused=int(counts.get("refused", 0)),
|
|
||||||
t2_verified=int(counts.get("t2_verified", 0)),
|
|
||||||
t2_agrees_gold=int(counts.get("t2_agrees_gold", 0)),
|
|
||||||
)
|
|
||||||
return ledger
|
|
||||||
|
|
||||||
|
|
||||||
def deduction_serve_license(
|
def deduction_serve_license(
|
||||||
|
|
@ -79,11 +56,7 @@ def deduction_serve_license(
|
||||||
safe default ceilings.
|
safe default ceilings.
|
||||||
"""
|
"""
|
||||||
ledger = ledger if ledger is not None else load_ratified_ledger()
|
ledger = ledger if ledger is not None else load_ratified_ledger()
|
||||||
tally = ledger.get(shape_band)
|
return serve_license(shape_band, ledger, ceilings=ceilings)
|
||||||
if tally is None:
|
|
||||||
return None
|
|
||||||
ceilings = ceilings if ceilings is not None else Ceilings.default()
|
|
||||||
return license_for(tally, Action.SERVE, ceilings)
|
|
||||||
|
|
||||||
|
|
||||||
__all__ = ["RatifiedLedgerError", "deduction_serve_license", "load_ratified_ledger"]
|
__all__ = ["RatifiedLedgerError", "deduction_serve_license", "load_ratified_ledger"]
|
||||||
|
|
|
||||||
|
|
@ -1757,6 +1757,22 @@ class ChatRuntime:
|
||||||
return (deduction_surface, "deduction", ())
|
return (deduction_surface, "deduction", ())
|
||||||
if attempts is not None:
|
if attempts is not None:
|
||||||
attempts.append(DispatchAttempt(source="deduction", outcome="skipped", reason="not_argument_shaped"))
|
attempts.append(DispatchAttempt(source="deduction", outcome="skipped", reason="not_argument_shaped"))
|
||||||
|
# Generalization arc, Phase 2 (ADR-0262) — curriculum-grounded exam
|
||||||
|
# questions, checked beside deduction serving for the same reason: a
|
||||||
|
# pure function of the input text and the ratified curriculum, with no
|
||||||
|
# vault/field dependency, so warm/cold state cannot change the answer.
|
||||||
|
if self.config.curriculum_serving_enabled:
|
||||||
|
from chat.curriculum_surface import curriculum_grounded_surface
|
||||||
|
from generate.intent import DialogueIntent, IntentTag as _IntentTag
|
||||||
|
|
||||||
|
curriculum_surface = curriculum_grounded_surface(text)
|
||||||
|
if curriculum_surface is not None:
|
||||||
|
self._last_intent = DialogueIntent(tag=_IntentTag.DEDUCTION, subject=text)
|
||||||
|
if attempts is not None:
|
||||||
|
attempts.append(DispatchAttempt(source="curriculum", outcome="admitted", reason="curriculum_composer_committed"))
|
||||||
|
return (curriculum_surface, "curriculum", ())
|
||||||
|
if attempts is not None:
|
||||||
|
attempts.append(DispatchAttempt(source="curriculum", outcome="skipped", reason="not_curriculum_question"))
|
||||||
if not allow_warm and gate_source != "empty_vault":
|
if not allow_warm and gate_source != "empty_vault":
|
||||||
if attempts is not None:
|
if attempts is not None:
|
||||||
for src in ("pack", "teaching", "partial", "oov"):
|
for src in ("pack", "teaching", "partial", "oov"):
|
||||||
|
|
|
||||||
|
|
@ -183,6 +183,8 @@ TEST_SUITES: dict[str, tuple[str, ...]] = {
|
||||||
"tests/test_verb_argument_reader.py",
|
"tests/test_verb_argument_reader.py",
|
||||||
"tests/test_exist_argument_reader.py",
|
"tests/test_exist_argument_reader.py",
|
||||||
"tests/test_deduction_serve_e2e.py",
|
"tests/test_deduction_serve_e2e.py",
|
||||||
|
"tests/test_curriculum_serve.py",
|
||||||
|
"tests/test_ratified_ledger_bridge.py",
|
||||||
),
|
),
|
||||||
"full": ("tests/",),
|
"full": ("tests/",),
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -386,6 +386,21 @@ class RuntimeConfig:
|
||||||
# (hedged). OFF by default: flag-off is byte-identical to pre-arc dispatch.
|
# (hedged). OFF by default: flag-off is byte-identical to pre-arc dispatch.
|
||||||
deduction_serving_enabled: bool = False
|
deduction_serving_enabled: bool = False
|
||||||
|
|
||||||
|
# Generalization arc, Phase 2 (ADR-0262) — when on, ``chat/curriculum_surface.py``
|
||||||
|
# answers exam-shaped polar questions ("Does force cause acceleration?") from the
|
||||||
|
# RATIFIED domain-chain curriculum of the subject the question's vocabulary
|
||||||
|
# belongs to, decided by the same argument bands the deduction composer uses.
|
||||||
|
# The curriculum is read OPEN-world: a relation it does not teach is UNKNOWN,
|
||||||
|
# never "no", so an untaught fact can never be decoded into a negative claim.
|
||||||
|
# Like ``deduction_serving_enabled`` this is an enable for an EARNED path, not a
|
||||||
|
# direct-serve switch — an answer is served AUTHORITATIVELY only when its
|
||||||
|
# (subject x relation family) band holds a SERVE license on the committed,
|
||||||
|
# SHA-sealed curriculum ledger; an unearned band is served DISCLOSED (hedged),
|
||||||
|
# which is the state of every band today (ADR-0262 SS5: the binding constraint is
|
||||||
|
# ratified curriculum volume, not machinery). OFF by default: flag-off is
|
||||||
|
# byte-identical to pre-arc dispatch.
|
||||||
|
curriculum_serving_enabled: bool = False
|
||||||
|
|
||||||
# ASK serving gate enable flag. When True, ASK serving is allowed.
|
# ASK serving gate enable flag. When True, ASK serving is allowed.
|
||||||
# Default False (dark).
|
# Default False (dark).
|
||||||
ask_serving_enabled: bool = False
|
ask_serving_enabled: bool = False
|
||||||
|
|
|
||||||
157
core/ratified_ledger.py
Normal file
157
core/ratified_ledger.py
Normal file
|
|
@ -0,0 +1,157 @@
|
||||||
|
"""The ratified-ledger bridge — seal → ratify → SHA-verify → serve-gate.
|
||||||
|
|
||||||
|
ADR-0175 Phase 5's consumption bridge (generalization plan Phase 3.3),
|
||||||
|
extracted from three working instances rather than designed ahead of them:
|
||||||
|
|
||||||
|
- ``generate/determine/estimation_license.py`` (ADR-0175, the first)
|
||||||
|
- ``chat/deduction_serve_license.py`` (ADR-0256, the second)
|
||||||
|
- ``chat/curriculum_serve_license.py`` (ADR-0262, the third)
|
||||||
|
|
||||||
|
All three had converged on the same artifact and the same four rules, which is
|
||||||
|
what makes this an extraction and not a speculation. The rules, stated once:
|
||||||
|
|
||||||
|
1. **The engine reads; only sealed practice writes.** A ledger is the output
|
||||||
|
of a practice run over a gold corpus, never of a serving turn. Nothing in a
|
||||||
|
serving path may call :func:`write_sealed_ledger`.
|
||||||
|
2. **Tamper-evidence is structural.** The artifact carries
|
||||||
|
``content_sha256`` over its ``classes`` table; a load that cannot reproduce
|
||||||
|
it REFUSES. A hand-edited ledger is not a slightly-wrong ledger, it is an
|
||||||
|
unratified one.
|
||||||
|
3. **Ceilings are not negotiable at the call site.** The gate always runs at
|
||||||
|
the safe defaults unless a caller passes ceilings explicitly, and no
|
||||||
|
production path does — ADR-0175 invariant #4: an engine cannot raise its own
|
||||||
|
bar.
|
||||||
|
4. **Absent evidence is never a license.** A class missing from the ledger
|
||||||
|
yields ``None``, and every caller's ``None`` branch serves the disclosed
|
||||||
|
(hedged) surface. A capability with no track record is served honestly, not
|
||||||
|
withheld and not asserted.
|
||||||
|
|
||||||
|
Byte-compatibility is deliberate: :func:`seal_artifact` and
|
||||||
|
:func:`write_sealed_ledger` reproduce the exact bytes the three existing
|
||||||
|
sealers wrote, so adopting the bridge re-seals every committed ledger
|
||||||
|
identically and no lane pin moves.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from core.reliability_gate import (
|
||||||
|
Action,
|
||||||
|
Ceilings,
|
||||||
|
ClassTally,
|
||||||
|
LicenseDecision,
|
||||||
|
license_for,
|
||||||
|
)
|
||||||
|
from formation.hashing import sha256_of
|
||||||
|
|
||||||
|
|
||||||
|
class RatifiedLedgerError(ValueError):
|
||||||
|
"""A committed ledger is malformed or does not verify against its own hash."""
|
||||||
|
|
||||||
|
|
||||||
|
def tally_dict(tally: ClassTally) -> dict[str, Any]:
|
||||||
|
"""The committed per-class row. The field set is the contract — a reader
|
||||||
|
of an older ledger must be able to name every field it finds."""
|
||||||
|
return {
|
||||||
|
"correct": tally.correct,
|
||||||
|
"wrong": tally.wrong,
|
||||||
|
"refused": tally.refused,
|
||||||
|
"t2_verified": tally.t2_verified,
|
||||||
|
"t2_agrees_gold": tally.t2_agrees_gold,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def seal_artifact(
|
||||||
|
ledger: dict[str, ClassTally], *, schema: str, note: str, provenance: str
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""The self-verifying sealed-ledger dict for *ledger*.
|
||||||
|
|
||||||
|
Classes are sorted, so the artifact is a pure function of the practice
|
||||||
|
result: the same corpus and solver seal byte-identically, which is what
|
||||||
|
makes a committed ledger reviewable as a diff.
|
||||||
|
"""
|
||||||
|
classes = {name: tally_dict(tally) for name, tally in sorted(ledger.items())}
|
||||||
|
return {
|
||||||
|
"schema": schema,
|
||||||
|
"classes": classes,
|
||||||
|
"content_sha256": sha256_of(classes),
|
||||||
|
"note": note,
|
||||||
|
"provenance": provenance,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def write_sealed_ledger(path: Path, artifact: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
"""Write *artifact* to *path* in the committed formatting."""
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(
|
||||||
|
json.dumps(artifact, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||||
|
)
|
||||||
|
return artifact
|
||||||
|
|
||||||
|
|
||||||
|
def load_sealed_ledger(path: Path, *, missing_ok: bool = False) -> dict[str, ClassTally]:
|
||||||
|
"""Load + verify a sealed ledger → per-class ``ClassTally``.
|
||||||
|
|
||||||
|
``missing_ok`` distinguishes two genuinely different situations. A ledger
|
||||||
|
that a capability *ships with* is required: its absence means the
|
||||||
|
deployment is broken, and refusing is right. A ledger for a capability
|
||||||
|
whose practice volume is still being built is legitimately absent, and the
|
||||||
|
honest reading of "no file" is "no class has earned anything yet" — an
|
||||||
|
empty table, every answer disclosed. Neither case may be answered by
|
||||||
|
guessing a license.
|
||||||
|
"""
|
||||||
|
if not path.exists():
|
||||||
|
if missing_ok:
|
||||||
|
return {}
|
||||||
|
raise RatifiedLedgerError(f"ratified ledger not found: {path}")
|
||||||
|
try:
|
||||||
|
artifact = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, json.JSONDecodeError) as exc:
|
||||||
|
raise RatifiedLedgerError(f"cannot read ratified ledger: {exc}") from exc
|
||||||
|
|
||||||
|
classes = artifact.get("classes") if isinstance(artifact, dict) else None
|
||||||
|
if not isinstance(classes, dict):
|
||||||
|
raise RatifiedLedgerError("ratified ledger has no 'classes' table")
|
||||||
|
if sha256_of(classes) != artifact.get("content_sha256"):
|
||||||
|
raise RatifiedLedgerError(
|
||||||
|
"ratified ledger content_sha256 mismatch — not the sealed-practice output"
|
||||||
|
)
|
||||||
|
|
||||||
|
return {
|
||||||
|
name: ClassTally(
|
||||||
|
class_name=name,
|
||||||
|
correct=int(counts.get("correct", 0)),
|
||||||
|
wrong=int(counts.get("wrong", 0)),
|
||||||
|
refused=int(counts.get("refused", 0)),
|
||||||
|
t2_verified=int(counts.get("t2_verified", 0)),
|
||||||
|
t2_agrees_gold=int(counts.get("t2_agrees_gold", 0)),
|
||||||
|
)
|
||||||
|
for name, counts in classes.items()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def serve_license(
|
||||||
|
class_name: str,
|
||||||
|
ledger: dict[str, ClassTally],
|
||||||
|
*,
|
||||||
|
ceilings: Ceilings | None = None,
|
||||||
|
) -> LicenseDecision | None:
|
||||||
|
"""The ``Action.SERVE`` verdict for *class_name*, or ``None`` when the
|
||||||
|
class has no committed evidence (never a license — rule 4)."""
|
||||||
|
tally = ledger.get(class_name)
|
||||||
|
if tally is None:
|
||||||
|
return None
|
||||||
|
return license_for(tally, Action.SERVE, ceilings or Ceilings.default())
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"RatifiedLedgerError",
|
||||||
|
"load_sealed_ledger",
|
||||||
|
"seal_artifact",
|
||||||
|
"serve_license",
|
||||||
|
"tally_dict",
|
||||||
|
"write_sealed_ledger",
|
||||||
|
]
|
||||||
164
docs/adr/ADR-0262-curriculum-grounded-serving.md
Normal file
164
docs/adr/ADR-0262-curriculum-grounded-serving.md
Normal file
|
|
@ -0,0 +1,164 @@
|
||||||
|
# ADR-0262 — Curriculum-grounded serving: exams answered from what was taught
|
||||||
|
|
||||||
|
- **Status:** Proposed
|
||||||
|
- **Date:** 2026-07-24
|
||||||
|
- **Arc:** generalization Phase 2 (docs/plans/generalization-arc-2026-07-24.md §4)
|
||||||
|
- **Governs:** `teaching/curriculum_premises.py`, `chat/curriculum_surface.py`,
|
||||||
|
`chat/curriculum_serve_license.py`, the `curriculum_<subject>_<family>` bands,
|
||||||
|
the `evals/curriculum_serve` lane and its independent oracle, and the
|
||||||
|
`curriculum_serving_enabled` flag.
|
||||||
|
|
||||||
|
## 1. Context
|
||||||
|
|
||||||
|
The band cascade (ADR-0256 → 0261) decides arguments whose premises are IN the
|
||||||
|
text. Exams are not shaped like that: the question supplies only the query, and
|
||||||
|
the premises are supposed to be what the student was taught. Until now CORE had
|
||||||
|
no path from "a ratified curriculum exists" to "a question is answered from
|
||||||
|
it" — the 16 domain seed packs and 5 domain-chain corpora were inert.
|
||||||
|
|
||||||
|
The generalization plan §4 fixes the epistemology for that path in advance, so
|
||||||
|
that implementing it cannot quietly become "answer from what the model knows".
|
||||||
|
This ADR implements §4.
|
||||||
|
|
||||||
|
## 2. Decision
|
||||||
|
|
||||||
|
Add a second serving composer, `chat/curriculum_surface.py`, behind the
|
||||||
|
default-off `curriculum_serving_enabled` flag:
|
||||||
|
|
||||||
|
1. **Closed question grammar** — `Does <term> <relation> <term>?`. One shape.
|
||||||
|
Anything else refuses `question_shape_out_of_band` rather than guessing
|
||||||
|
which token is the relation.
|
||||||
|
2. **Subject routing by vocabulary** — the question goes to the subject whose
|
||||||
|
ratified pack vocabulary contains BOTH terms. No match is
|
||||||
|
`untaught_vocabulary`; more than one is `ambiguous_reading`. The subject is
|
||||||
|
never guessed.
|
||||||
|
3. **Premise compilation from the ratified curriculum only**
|
||||||
|
(`teaching/curriculum_premises.py`). A chain is admitted iff
|
||||||
|
`review_status == "reviewed"` AND both terms are resident in the packs the
|
||||||
|
chain declares. Compilation is scoped to the question's relation family.
|
||||||
|
4. **The same decider** — compiled premises plus the question as a "Therefore"
|
||||||
|
conclusion are handed to the verb-predicate band (ADR-0260), falling through
|
||||||
|
to the existential band (ADR-0261), then the ROBDD engine. **No
|
||||||
|
subject-specific decision code exists anywhere in this path.**
|
||||||
|
5. **Bands = (subject × relation family)**, e.g. `curriculum_physics_causal`,
|
||||||
|
gated by the same earned-license machinery as deduction serving through a
|
||||||
|
second ledger reader (`chat/curriculum_serve_license.py`).
|
||||||
|
|
||||||
|
The relation family comes from the CONNECTIVE, not from a chain row's declared
|
||||||
|
`operator_family`. A question carries a relation word and nothing else; a
|
||||||
|
family derived from anything the question cannot carry would let premise
|
||||||
|
compilation and question routing disagree, and a taught edge could then be
|
||||||
|
missing from the premises compiled to decide it — a wrong answer built out of a
|
||||||
|
correct curriculum. Two corpus rows currently declare a family their connective
|
||||||
|
does not imply; under this rule they are read as their connective says.
|
||||||
|
|
||||||
|
## 3. Why this is sound
|
||||||
|
|
||||||
|
1. **Gold is a function of (curriculum, question).** No case-local hidden text,
|
||||||
|
no world knowledge. A lane case pins the chain ids it draws on and the
|
||||||
|
runner FAILS the case if a pinned chain is absent or unratified — so a
|
||||||
|
curriculum that changes under a case breaks loudly instead of quietly
|
||||||
|
answering from what remains.
|
||||||
|
2. **Open-world reading.** A relation the curriculum does not state is
|
||||||
|
UNKNOWN, never "no". Deciding otherwise would manufacture negative
|
||||||
|
knowledge from silence, which is the same failure as manufacturing positive
|
||||||
|
knowledge from silence.
|
||||||
|
3. **No composition.** The curriculum teaches `force causes acceleration` and
|
||||||
|
`acceleration causes motion`; it does not teach that causation composes, so
|
||||||
|
`force causes motion` is UNKNOWN. Transitivity of a causal or modal relation
|
||||||
|
is a substantive claim about the world, not a logical truth, and nothing
|
||||||
|
ratified asserts it. The independent oracle reports the shortest path length
|
||||||
|
with its verdict precisely so the lane can assert that a reachable pair is
|
||||||
|
still answered UNKNOWN — composition is provably not happening rather than
|
||||||
|
merely absent.
|
||||||
|
4. **Face-value relations.** `entropy reveals energy` does not license
|
||||||
|
`entropy causes energy`. Same family, different relation: the reader mints
|
||||||
|
distinct atoms per verb group (ADR-0260), so a family-level near-miss cannot
|
||||||
|
become an entailment.
|
||||||
|
5. **Independent gold.** `evals/curriculum_serve/oracle.py` shares no code with
|
||||||
|
the serving path — its own JSONL loader, ratification predicate, family
|
||||||
|
table, agreement normalization, and verdict rule. It disagreed with the
|
||||||
|
serving path once during authoring, on exactly the point in (4), and the
|
||||||
|
ORACLE was the side that was wrong; that disagreement is why the check
|
||||||
|
exists.
|
||||||
|
6. **Anti-recall is a lane guard, not a hope.** A split must carry ≥3 probes
|
||||||
|
whose answer is true in the world and absent from the curriculum, and no
|
||||||
|
probe may carry a committed gold, or the lane refuses to run.
|
||||||
|
|
||||||
|
## 4. License posture
|
||||||
|
|
||||||
|
Every `curriculum_*` band is UNEARNED today, so every answer is served
|
||||||
|
DISCLOSED through the same hedge deduction serving uses. This is not a
|
||||||
|
temporary shortcut around the license — it is the license working. The reason
|
||||||
|
is §5.1.
|
||||||
|
|
||||||
|
## 5. Findings
|
||||||
|
|
||||||
|
### 5.1 The binding constraint on Phase 2 is ratified curriculum VOLUME
|
||||||
|
|
||||||
|
A band earns SERVE at θ_SERVE=0.99 only with n ≥ 657 committed cases and a
|
||||||
|
genuine outcome mix. The question space of a subject is
|
||||||
|
`|vocabulary|² − |vocabulary|`, so volume is reachable (physics: 240,
|
||||||
|
philosophy_theology: 22,650). **Entailed** cases are not: physics teaches 7
|
||||||
|
causal and 9 modal relations, so at most 16 distinct questions in the whole
|
||||||
|
subject can come back ENTAILED. Sampling cannot fix this — repeating the same
|
||||||
|
16 questions inflates n without adding information, and a band whose entailed
|
||||||
|
class is 1% of its volume is passed by a pipeline that never entails anything.
|
||||||
|
|
||||||
|
Concretely: a balanced band (≈⅓ entailed) needs **≈219 distinct taught
|
||||||
|
relations per subject × family**. Physics has 7 and 9. That is a ~25× gap, and
|
||||||
|
it is a curriculum-authoring and ratification task — Shay's call, not an
|
||||||
|
engineering one.
|
||||||
|
|
||||||
|
The honest consequence, recorded rather than worked around: the machinery,
|
||||||
|
the lane, the oracle, and the license gate all ship and pass; the licenses do
|
||||||
|
not exist yet; every answer is hedged until they do. The plan's O→S checkpoint
|
||||||
|
("≥1 subject lane serving wrong=0 with earned licenses") is met on every clause
|
||||||
|
except the last, and the last one cannot be met from present data.
|
||||||
|
|
||||||
|
### 5.2 REFUTED is unreachable from present corpora
|
||||||
|
|
||||||
|
Every ratified chain is a positive assertion. Refuting a question needs the
|
||||||
|
curriculum to teach a negative ("no X causes Y") or an exclusion. Until a
|
||||||
|
corpus does, `refuted` is a verdict this path can render but never reach. The
|
||||||
|
serving code handles it; the lane documents it.
|
||||||
|
|
||||||
|
### 5.3 There is no biology domain-chain corpus
|
||||||
|
|
||||||
|
The plan names "physics and biology" as Phase 2's two subjects because "OOD
|
||||||
|
lanes, seed packs, and domain chains already exist". For biology only the first
|
||||||
|
two are true: `evals/foundational_biology_ood` is a **fluency** lane (its own
|
||||||
|
contract says it measures grammatical English, not truth) and there is no
|
||||||
|
`biology_chains_*.jsonl`. The four subjects with ratified chains AND mounted
|
||||||
|
packs are physics, mathematics_logic, systems_software, and
|
||||||
|
philosophy_theology; the composer serves all four. The second subject should be
|
||||||
|
chosen from those, or biology's chain corpus authored first.
|
||||||
|
|
||||||
|
## 6. Scope-outs
|
||||||
|
|
||||||
|
- **Question shapes** beyond `Does <term> <relation> <term>?` — multi-word
|
||||||
|
terms, "why"/"how" questions, negated questions, and quantified questions all
|
||||||
|
refuse typed. Each is a separate readable unit with its own lane.
|
||||||
|
- **Composition**, per §3.3 — a future ADR may license it for a family the
|
||||||
|
curriculum itself declares transitive, but that requires the declaration to
|
||||||
|
be an executable claim rather than a classification label, which today it is
|
||||||
|
not.
|
||||||
|
- **Premise-set scaling** — a family with more than `MAX_PREMISE_SENTENCES`
|
||||||
|
(16) ratified chains compiles to an argument the reader refuses, surfacing as
|
||||||
|
`compiled_premises_unreadable`. No current family exceeds 9. The fix when one
|
||||||
|
does is to compile the query's k-hop neighborhood rather than the whole
|
||||||
|
family (sound: fewer premises can lose an entailment, never create one), and
|
||||||
|
to disclose the narrowing.
|
||||||
|
|
||||||
|
## 7. Verification
|
||||||
|
|
||||||
|
- `tests/test_curriculum_serve.py` — 20 tests: premise provenance and the
|
||||||
|
loud-failure guard, family scoping never losing a taught edge, taught-edge
|
||||||
|
entailment, untaught composition staying UNKNOWN, untaught fact never
|
||||||
|
REFUTED, relation read at face value, base-form question normalization,
|
||||||
|
anti-recall declines, 3 typed refusals, non-question pass-through, routing,
|
||||||
|
the disclosed-until-earned license posture and the authoritative path when a
|
||||||
|
band IS licensed, and oracle independence including the depth-2 reachability
|
||||||
|
assertion.
|
||||||
|
- `evals/curriculum_serve` lane: 32/32, wrong=0, 5 anti-recall probes, both
|
||||||
|
physics bands exercised. Pinned as `curriculum_serve_v1`.
|
||||||
80
docs/adr/ADR-0263-ratified-ledger-bridge.md
Normal file
80
docs/adr/ADR-0263-ratified-ledger-bridge.md
Normal file
|
|
@ -0,0 +1,80 @@
|
||||||
|
# ADR-0263 — The ratified-ledger bridge
|
||||||
|
|
||||||
|
- **Status:** Proposed
|
||||||
|
- **Date:** 2026-07-24
|
||||||
|
- **Arc:** generalization Phase 3.3 (docs/plans/generalization-arc-2026-07-24.md §2)
|
||||||
|
- **Governs:** `core/ratified_ledger.py` and the three adapters over it —
|
||||||
|
`generate/determine/estimation_license.py`, `chat/deduction_serve_license.py`,
|
||||||
|
`chat/curriculum_serve_license.py` — plus the sealed-artifact writer in
|
||||||
|
`evals/deduction_serve/practice/runner.py`.
|
||||||
|
|
||||||
|
## 1. Context
|
||||||
|
|
||||||
|
Three capabilities had independently converged on the same four-step pattern:
|
||||||
|
sealed practice writes a per-class tally table → the table is committed as a
|
||||||
|
ratified artifact → serving verifies its `content_sha256` on load → a per-class
|
||||||
|
gate decides SERVE against fixed ceilings.
|
||||||
|
|
||||||
|
Three near-identical implementations is the point at which a shared component
|
||||||
|
is an extraction rather than a guess. It matters now because the generalization
|
||||||
|
plan puts a per-subject arena behind every new subject: without a bridge, each
|
||||||
|
subject copies fifty lines of verification, and the copies drift.
|
||||||
|
|
||||||
|
## 2. Decision
|
||||||
|
|
||||||
|
`core/ratified_ledger.py` owns the mechanics — `seal_artifact`,
|
||||||
|
`write_sealed_ledger`, `load_sealed_ledger`, `serve_license`, `tally_dict` —
|
||||||
|
and states the four rules once:
|
||||||
|
|
||||||
|
1. **The engine reads; only sealed practice writes.**
|
||||||
|
2. **Tamper-evidence is structural** — a load that cannot reproduce
|
||||||
|
`content_sha256` REFUSES. A hand-edited ledger is not a slightly-wrong
|
||||||
|
ledger; it is an unratified one.
|
||||||
|
3. **Ceilings are not negotiable at the call site** (ADR-0175 invariant #4).
|
||||||
|
4. **Absent evidence is never a license** — a missing class yields `None`, and
|
||||||
|
every caller's `None` branch serves the disclosed surface.
|
||||||
|
|
||||||
|
Each capability keeps a thin adapter that names its artifact, keeps its
|
||||||
|
memoization, and preserves its public API. One genuine difference is now
|
||||||
|
declared rather than implied: `load_sealed_ledger(..., missing_ok=True)`
|
||||||
|
distinguishes a ledger a capability *ships with* (absence = broken deployment,
|
||||||
|
refuse) from one whose practice volume is still being built (absence = no class
|
||||||
|
has earned anything, serve disclosed). Curriculum serving is the second kind
|
||||||
|
today (ADR-0262 §5.1).
|
||||||
|
|
||||||
|
## 3. Why this is safe
|
||||||
|
|
||||||
|
The extraction's safety property is byte-identity: `seal_artifact` and
|
||||||
|
`write_sealed_ledger` reproduce exactly what the bespoke sealers wrote, so
|
||||||
|
re-sealing through the bridge leaves every committed artifact and every lane
|
||||||
|
pin unmoved. That is asserted, not assumed —
|
||||||
|
`test_committed_deduction_ledger_reseals_byte_identically` re-derives the
|
||||||
|
committed 25-band ledger through the bridge and compares the bytes.
|
||||||
|
|
||||||
|
## 4. What this unblocks
|
||||||
|
|
||||||
|
A new subject arena now needs a gold corpus and a band key. It does not need to
|
||||||
|
re-implement ratification — which is what the plan meant by "so each subject
|
||||||
|
arena plugs in without bespoke wiring", and why §3 sequenced this ahead of the
|
||||||
|
second subject.
|
||||||
|
|
||||||
|
## 5. Scope-outs
|
||||||
|
|
||||||
|
- **The estimation adapter keeps its predicate → converse-class naming.** That
|
||||||
|
mapping is genuinely local to estimation; the bridge gates a class name and
|
||||||
|
does not know how one is derived.
|
||||||
|
- **No ledger migration.** All three artifacts already share the schema shape;
|
||||||
|
nothing is rewritten, versioned, or moved.
|
||||||
|
- **`core/learning_arena`'s practice engine is untouched.** The bridge covers
|
||||||
|
the consumption half (seal → serve); the production half (run practice, tally)
|
||||||
|
is already shared.
|
||||||
|
|
||||||
|
## 6. Verification
|
||||||
|
|
||||||
|
`tests/test_ratified_ledger_bridge.py` — 8 tests: round-trip, hand-edit
|
||||||
|
rejection, required-vs-optional absence, absent class never licensed, the
|
||||||
|
Wilson volume floor, a single `wrong` costing the license, committed-ledger
|
||||||
|
byte-identity, and a guard that all three adapters read through the bridge
|
||||||
|
(they no longer import a hashing module of their own). Plus the existing
|
||||||
|
consumers unchanged: `core test --suite deductive` 252 passed; the estimation
|
||||||
|
and license test set 355 passed.
|
||||||
|
|
@ -1,47 +1,136 @@
|
||||||
# Handoff Brief — Tier S (Sonnet 5, LOW risk)
|
# Handoff Brief — Tier S (Sonnet 5, LOW risk)
|
||||||
|
|
||||||
**Read first:** `docs/plans/generalization-arc-2026-07-24.md` §6
|
**Read first:** `docs/plans/generalization-arc-2026-07-24.md` §6 (constraint
|
||||||
(constraint set is binding) and the Tier-O brief's constraints block —
|
set is binding) and the Tier-O brief's constraints block — they apply verbatim.
|
||||||
they apply verbatim. Opus will have updated this brief with actual state
|
|
||||||
at its exit checkpoint; trust that update over anything stale below.
|
|
||||||
|
|
||||||
**You own (independent items, any order unless noted):**
|
---
|
||||||
|
|
||||||
|
## STATE AT THE OPUS → SONNET CHECKPOINT (updated 2026-07-24)
|
||||||
|
|
||||||
|
**Tier O is complete.** Four units, all committed, all with local gates green:
|
||||||
|
|
||||||
|
| unit | branch @ commit | what |
|
||||||
|
|---|---|---|
|
||||||
|
| O1 | `feat/existential-band` @ `e84c0e84` | Band v6-EX existentials (ADR-0261) + a wrong-answer fix in Band v1b |
|
||||||
|
| O2 | `feat/curriculum-serve-physics` @ `44e78aa4` | curriculum-grounded serving (ADR-0262), physics lane 32/32 wrong=0 |
|
||||||
|
| O3 | same branch @ `0a17c496` | ratified-ledger bridge (ADR-0263) |
|
||||||
|
| O4 | same branch @ `353a52b8` | math Phase 4.1 measured status — null result, documented |
|
||||||
|
|
||||||
|
Branch stack (each on the previous): `main` → `feat/verb-predicate-band`
|
||||||
|
(PR #111, Shay's, still open) → `feat/existential-band` →
|
||||||
|
`feat/curriculum-serve-physics`.
|
||||||
|
|
||||||
|
### ⛔ BLOCKER — pushes to core-labs/core are failing server-side
|
||||||
|
|
||||||
|
The Forgejo repo has a **corrupt object**: `refs/heads/feat/verb-predicate-band`
|
||||||
|
(= PR #111 head, `e41a3e2a`) points at a **zero-length object file** on the
|
||||||
|
server. Symptoms:
|
||||||
|
|
||||||
|
```
|
||||||
|
remote: error: object file .../objects/e4/1a3e2a... is empty
|
||||||
|
remote: fatal: bad object refs/heads/feat/verb-predicate-band
|
||||||
|
! [remote rejected] <your-branch> (missing necessary objects)
|
||||||
|
```
|
||||||
|
|
||||||
|
This blocks **every** push to the repo, not just the Opus branches — the
|
||||||
|
server fails its own connectivity check before accepting any ref. Reads
|
||||||
|
(`git fetch`, `ls-remote`, MCP) still work. Nothing local is damaged: the
|
||||||
|
identical content exists locally as `d181ae30`.
|
||||||
|
|
||||||
|
Repair needs filesystem access on the Forgejo VM (remove the empty object,
|
||||||
|
then reset or delete `refs/heads/feat/verb-predicate-band` and
|
||||||
|
`refs/pull/111/head`, `git fsck`). **This is Shay's call — do not delete or
|
||||||
|
force-push someone else's PR branch.** Until it is fixed, work locally and
|
||||||
|
commit; push when the server is repaired.
|
||||||
|
|
||||||
|
### Corrections to the plan's ground truth (verified — act on these)
|
||||||
|
|
||||||
|
1. **There is no biology domain-chain corpus.** `evals/foundational_biology_ood`
|
||||||
|
is a *fluency* lane (its own contract says so) and no `biology_chains_*.jsonl`
|
||||||
|
exists. The subjects with ratified chains AND mounted packs are physics,
|
||||||
|
mathematics_logic, systems_software, philosophy_theology. Pick the second
|
||||||
|
subject from those, or author biology's chains first.
|
||||||
|
2. **Phase 4.1 (seeding-sentence injection) was already built** before this
|
||||||
|
arc and converts 0 cases; the frozen-500 baseline re-measured today is
|
||||||
|
`correct=5 wrong=0 refused=495`. See
|
||||||
|
`docs/research/math-reader-phase-4-1-status-2026-07-24.md`. Phase 4 should
|
||||||
|
be re-pointed at the reader arc's own live recommendation (increment-2
|
||||||
|
case-first on cases 0000/0001/0148/0082).
|
||||||
|
3. **No curriculum band can earn a SERVE license from present data.** A band
|
||||||
|
needs n≥657 with a real outcome mix; physics teaches 7 causal + 9 modal
|
||||||
|
relations, so at most 16 questions in the whole subject can ever come back
|
||||||
|
ENTAILED. A balanced band needs ≈219 taught relations per subject × family.
|
||||||
|
Phase 2's blocker is **ratified curriculum volume**, not machinery
|
||||||
|
(ADR-0262 §5.1). Every curriculum answer is served DISCLOSED until that
|
||||||
|
changes — which is the license working, not a workaround.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## You own (independent items, any order unless noted)
|
||||||
|
|
||||||
1. **S1 — ratification packet for `deduction_serving_enabled`.**
|
1. **S1 — ratification packet for `deduction_serving_enabled`.**
|
||||||
Evidence collation only — the flip is Shay's decision, never yours.
|
Evidence collation only — the flip is Shay's decision, never yours.
|
||||||
Contents: 17+-band sealed ledger stats
|
Contents: the **25-band** sealed ledger stats
|
||||||
(`chat/data/deduction_serve_ledger.json`), serve-lane results
|
(`chat/data/deduction_serve_ledger.json`, all 25 at 720/720 wrong=0),
|
||||||
(`evals/deduction_serve/report.json`), byte-identical-when-off proof
|
serve-lane results (`evals/deduction_serve/report.json`, **166/166 across
|
||||||
(cite ADR-0256 + `tests/test_deduction_surface.py` flag tests), blast
|
six splits**), byte-identical-when-off proof (cite ADR-0256 +
|
||||||
radius (exact dispatch point `chat/runtime.py` deduction branch), and
|
`tests/test_deduction_surface.py` flag tests), blast radius (the deduction
|
||||||
rollback (flip back, zero residue). One markdown doc in
|
branch in `chat/runtime.py`), and rollback (flip back, zero residue).
|
||||||
`docs/research/`, PR'd.
|
**Add a second section for `curriculum_serving_enabled`** (ADR-0262): same
|
||||||
2. **S2 — lane re-pins + flake documentation.** Only AFTER the Phase-0.1
|
shape, noting explicitly that every curriculum band is unearned, so the
|
||||||
drift findings doc exists and says re-pin is safe. Surgical
|
flip enables a *disclosed* capability. One markdown doc in `docs/research/`.
|
||||||
single-line edits in `scripts/verify_lane_shas.py` — NEVER `--update`
|
|
||||||
(it rewrites every lane and silently drops erroring lanes' pins).
|
|
||||||
Document the `public_demo` env-timeout flake where the findings doc
|
|
||||||
says.
|
|
||||||
3. **S3 — vocab-trigger instrument.** Implement the measurable test
|
|
||||||
specified in `docs/handoffs/COMPREHENSION-READER-AUDIT.md`
|
|
||||||
(§measurable-test, lines ~163–166 and ~231–238): refusal histogram
|
|
||||||
split mechanism-vs-coverage + Phase-2-admissions-per-lexicon-batch
|
|
||||||
counter. CLI or lane runner per the spec; do not invent policy — the
|
|
||||||
spec decides, you implement.
|
|
||||||
4. **S4 — HITL proposal-queue CLI.** A `core` CLI surface listing +
|
|
||||||
reviewing pending proposals from the existing sinks
|
|
||||||
(`teaching/proposals/`, contemplation/idle sinks). Read + review-state
|
|
||||||
transitions only; NO ratification automation, NO corpus mutation, NO
|
|
||||||
flag flips.
|
|
||||||
5. **S5 — housekeeping.** Promotion sweeps Opus's checkpoint notes call
|
|
||||||
for; capability-index entries for new subject lanes; docs/memory
|
|
||||||
updates; dead-code removal only when unambiguous.
|
|
||||||
|
|
||||||
**Constraints:** identical to Tier O (worktrees, fresh venv, pre-push
|
2. **S2 — lane re-pins + the `public_demo` drift.** The Phase-0.1 findings doc
|
||||||
gate, compare-URL PRs, Shay merges, wrong=0, flags stay off, no
|
exists (`docs/research/lane-drift-investigation-2026-07-24.md`).
|
||||||
timelines). When any item's scope turns out larger than described here —
|
**Correct the standing note: `public_demo` is NOT a timing flake here.** It
|
||||||
stop and flag it in the PR/handoff notes rather than improvising.
|
reproduces deterministically. Measured this session:
|
||||||
|
- pinned: `7d8ba0db…` (last set at commit `7f6c497a`, long before this arc)
|
||||||
|
- content with the register-axis fix reverted: `c6596e11…`
|
||||||
|
- content today: `da7fad65…` (3 identical runs, incl. on the base commit)
|
||||||
|
|
||||||
**Arc close:** all Tier-S PRs pushed; memory updated; note whether the
|
So there are **two** drifts: one predating PR #110 (unexplained — find it
|
||||||
Phase-5 articulation trigger (multi-step decided content exists) has
|
before re-pinning) and one caused by PR #110's register-axis fix (expected,
|
||||||
fired, for the next arc's scoping.
|
intended). `all_passed` stays `true` throughout; only the demo payload
|
||||||
|
digest moved. Do the archaeology, then re-pin surgically — **never
|
||||||
|
`--update`** (it rewrites every lane and silently drops erroring lanes'
|
||||||
|
pins). Three arcs now depend on that rule.
|
||||||
|
|
||||||
|
3. **S3 — vocab-trigger instrument.** Implement the measurable test specified
|
||||||
|
in `docs/handoffs/COMPREHENSION-READER-AUDIT.md` (§measurable-test):
|
||||||
|
refusal histogram split mechanism-vs-coverage + admissions-per-lexicon-batch
|
||||||
|
counter. Two ready-made feeds now exist: the curriculum path's
|
||||||
|
`untaught_vocabulary` / `out_of_curriculum` refusals
|
||||||
|
(`chat/curriculum_surface.py`) are literally coverage-class refusals, and
|
||||||
|
the deduction bands' typed refusals are mostly mechanism-class. Do not
|
||||||
|
invent policy — the spec decides, you implement.
|
||||||
|
|
||||||
|
4. **S4 — HITL proposal-queue CLI.** A `core` CLI surface listing + reviewing
|
||||||
|
pending proposals from the existing sinks (`teaching/proposals/`,
|
||||||
|
contemplation/idle sinks). Read + review-state transitions only; NO
|
||||||
|
ratification automation, NO corpus mutation, NO flag flips.
|
||||||
|
|
||||||
|
5. **S5 — housekeeping.**
|
||||||
|
- Add `tests/test_register_substantive_consumption.py`'s e2e tests to the
|
||||||
|
curated smoke suite (a red test outside every gate is a silent red — the
|
||||||
|
lesson from the #96 regression).
|
||||||
|
- Capability-index entries for the new lanes (`curriculum_serve_v1`, the
|
||||||
|
`v2_exist` split).
|
||||||
|
- Promotion sweep: `ds-mem-0020` was promoted declined→unknown by v6-EX
|
||||||
|
(already done); check nothing else in the older splits is now decidable.
|
||||||
|
- Memory/doc updates; dead-code removal only when unambiguous.
|
||||||
|
|
||||||
|
6. **S6 (new) — quantify the curriculum-volume ask.** From ADR-0262 §5.1:
|
||||||
|
produce a short doc stating, per served subject × relation family, how many
|
||||||
|
ratified chains exist and how many a balanced 657-case band would need.
|
||||||
|
Evidence only — authoring or ratifying curriculum content is Shay's.
|
||||||
|
|
||||||
|
**Constraints:** identical to Tier O (worktrees, fresh venv, pre-push gate,
|
||||||
|
compare-URL PRs, Shay merges, wrong=0, flags stay off, no timelines). When any
|
||||||
|
item's scope turns out larger than described here — stop and flag it in the
|
||||||
|
PR/handoff notes rather than improvising.
|
||||||
|
|
||||||
|
**Arc close:** all Tier-S PRs pushed; memory updated; note whether the Phase-5
|
||||||
|
articulation trigger (multi-step decided content exists) has fired. Evidence
|
||||||
|
for it now exists on both sides: the deduction bands produce multi-premise
|
||||||
|
decided chains, and the curriculum path produces answers that state how much
|
||||||
|
curriculum they were decided from.
|
||||||
|
|
|
||||||
109
docs/research/curriculum-grounded-serving-2026-07-24.md
Normal file
109
docs/research/curriculum-grounded-serving-2026-07-24.md
Normal file
|
|
@ -0,0 +1,109 @@
|
||||||
|
# Curriculum-grounded serving — the first exam answered from what was taught
|
||||||
|
|
||||||
|
**Date:** 2026-07-24 · **ADR:** ADR-0262 · **Arc:** generalization Phase 2 (Tier O)
|
||||||
|
|
||||||
|
## What shipped
|
||||||
|
|
||||||
|
"Does force cause acceleration?" is now answered — from the ratified physics
|
||||||
|
chain corpus and nothing else. "Does gravity cause acceleration?" is declined,
|
||||||
|
because *gravity* is not in any pack CORE has been taught. "Does force cause
|
||||||
|
motion?" comes back unsettled, even though the curriculum contains both links
|
||||||
|
in that chain, because nothing ratified says causation composes.
|
||||||
|
|
||||||
|
That third answer is the one worth staring at. It is the difference between a
|
||||||
|
system that decodes a curriculum and one that pattern-matches over it.
|
||||||
|
|
||||||
|
The lane: 32 hand-authored physics cases, wrong=0, five anti-recall probes, an
|
||||||
|
independent oracle re-deriving every gold, and three contract guards that fail
|
||||||
|
the LANE (not the case) if provenance, soundness, or anti-recall coverage
|
||||||
|
lapses. 20 unit tests. Zero subject-specific decision code: the composer routes
|
||||||
|
by vocabulary, compiles premises from ratified chains, and hands the result to
|
||||||
|
the same argument bands that decide "All philosophers teach. Socrates is a
|
||||||
|
philosopher."
|
||||||
|
|
||||||
|
## The design decision that carried the weight
|
||||||
|
|
||||||
|
The plan fixed the epistemology in §4 before any code existed, and that turned
|
||||||
|
out to matter more than any implementation choice. Three of its clauses did
|
||||||
|
real work:
|
||||||
|
|
||||||
|
- **Gold is a function of (curriculum, question).** No hidden premises in the
|
||||||
|
case file. A case pins chain ids; the runner reconstructs premises from the
|
||||||
|
corpus and fails loudly if a pinned chain is absent or unratified. A
|
||||||
|
curriculum that moves under a case breaks the case instead of quietly
|
||||||
|
changing its answer.
|
||||||
|
- **Untaught ⇒ UNKNOWN, never "no".** Open-world reading. Manufacturing a
|
||||||
|
negative from silence is the same error as manufacturing a positive from it,
|
||||||
|
and it is the more tempting one because it feels like rigor.
|
||||||
|
- **Anti-recall probes are mandatory.** The runner refuses to score a split
|
||||||
|
that does not contain questions whose answers are true in the world and
|
||||||
|
absent from the curriculum. Without them the lane cannot distinguish
|
||||||
|
decoding from recall, so it must not ship.
|
||||||
|
|
||||||
|
## Where the independent oracle earned its keep
|
||||||
|
|
||||||
|
The oracle disagreed with the serving path exactly once during authoring:
|
||||||
|
"Does entropy cause energy?" The curriculum teaches *entropy reveals energy*.
|
||||||
|
The serving path said UNKNOWN — different relation, different atom. The oracle
|
||||||
|
said ENTAILED, because I had written its edge lookup to match on relation
|
||||||
|
*family* rather than the relation itself.
|
||||||
|
|
||||||
|
The oracle was wrong, and the serving path was right. That is the useful shape
|
||||||
|
of the result: an independent check is worth building even when — especially
|
||||||
|
when — you expect it to agree, because the thing it catches is the assumption
|
||||||
|
you did not notice you had made. Family scoping decides which premises are in
|
||||||
|
play; the connective decides what was actually said.
|
||||||
|
|
||||||
|
A second authoring error surfaced the same way: two lane cases claimed both
|
||||||
|
directions of `field requires charge` were taught. Only one is. The provenance
|
||||||
|
and soundness guards caught it before the case file was committed.
|
||||||
|
|
||||||
|
## The finding: Phase 2's blocker is curriculum volume, not machinery
|
||||||
|
|
||||||
|
Every `curriculum_*` band is unearned, so every answer is served hedged. That
|
||||||
|
is not a shortcut around the license — it is the license working, and the
|
||||||
|
reason is arithmetic.
|
||||||
|
|
||||||
|
A band earns SERVE at θ_SERVE=0.99 with n ≥ 657 committed cases and a genuine
|
||||||
|
outcome mix. The question space is large enough (physics: 240 distinct
|
||||||
|
questions per family; philosophy_theology: 22,650). The *entailed* space is
|
||||||
|
not: physics teaches 7 causal and 9 modal relations, so at most 16 questions in
|
||||||
|
the entire subject can come back ENTAILED. Repeating them to reach 657 inflates
|
||||||
|
n without adding information, and a band whose entailed class is 1% of its
|
||||||
|
volume is passed by a pipeline that never entails anything at all.
|
||||||
|
|
||||||
|
A balanced band needs roughly **219 distinct taught relations per subject ×
|
||||||
|
family**. Physics has 7 and 9 — a ~25× gap. No sampling scheme closes it. What
|
||||||
|
closes it is ratified curriculum content, which is an authoring and
|
||||||
|
ratification decision, not an engineering one.
|
||||||
|
|
||||||
|
So the honest state of Phase 2: the lifecycle is ported and provably correct on
|
||||||
|
real curriculum data; the earned-license half is blocked on how much has been
|
||||||
|
taught. That is a much better problem to have than the reverse, and it is the
|
||||||
|
one the plan's own doctrine (capability → practice → calibration → serve)
|
||||||
|
predicts you would hit here.
|
||||||
|
|
||||||
|
## Two corrections to the plan's ground truth
|
||||||
|
|
||||||
|
- **There is no biology domain-chain corpus.** The plan names physics and
|
||||||
|
biology as Phase 2's subjects on the basis that "OOD lanes, seed packs, and
|
||||||
|
domain chains already exist". For biology the first two are true; the third
|
||||||
|
is not, and `evals/foundational_biology_ood`'s own contract says it measures
|
||||||
|
fluency, not truth. The subjects with ratified chains AND mounted packs are
|
||||||
|
physics, mathematics_logic, systems_software, and philosophy_theology. The
|
||||||
|
composer serves all four; the second subject should come from that list, or
|
||||||
|
biology's chains should be authored first.
|
||||||
|
- **`operator_family` and `connective` disagree in the corpora.** Two rows
|
||||||
|
declare a family their connective does not imply. The serving path reads the
|
||||||
|
connective, because that is the only thing an exam question carries — and if
|
||||||
|
routing and compilation disagreed about a row's family, a taught edge could
|
||||||
|
go missing from the premises compiled to decide it. That would be a wrong
|
||||||
|
answer assembled from a correct curriculum, which is the exact failure mode
|
||||||
|
ADR-0261 §5.1 found in the categorical band a day earlier.
|
||||||
|
|
||||||
|
## What is deliberately not built
|
||||||
|
|
||||||
|
Composition, negation, question shapes beyond `Does <term> <relation> <term>?`,
|
||||||
|
and premise-set scaling past the reader's 16-sentence honesty cap. Each is
|
||||||
|
scoped in ADR-0262 §6 with the condition that would unblock it. None of them
|
||||||
|
is a prerequisite for the next subject: adding one is a data operation now.
|
||||||
85
docs/research/math-reader-phase-4-1-status-2026-07-24.md
Normal file
85
docs/research/math-reader-phase-4-1-status-2026-07-24.md
Normal file
|
|
@ -0,0 +1,85 @@
|
||||||
|
# Math reader Phase 4.1 — measured status: the increment is already built, and it converts nothing
|
||||||
|
|
||||||
|
**Date:** 2026-07-24 · **Arc:** generalization Phase 4.1 (Tier O, O4) · **Result:** null, by measurement
|
||||||
|
|
||||||
|
## What was asked
|
||||||
|
|
||||||
|
The generalization plan §2/Phase 4.1 and the Opus brief item O4 both name
|
||||||
|
"seeding-sentence injection (standing ruling #77 — the 75% 'no injection'
|
||||||
|
wall)" as the next math move, with wrong=0 on the full 500 as the gate and
|
||||||
|
parse-rate gain as the goal.
|
||||||
|
|
||||||
|
## What is actually in the tree
|
||||||
|
|
||||||
|
Seeding-sentence injection **is built and merged**. The reader arc's increment
|
||||||
|
1 shipped it as the #78 positive-polarity substrate:
|
||||||
|
|
||||||
|
- `generate/math_roundtrip.py::NEUTRAL_COUNT_VERBS` — the curated
|
||||||
|
production/possession/acquisition allowlist,
|
||||||
|
- consumed by the seed matcher's `inject_discrete_count_statement` path and by
|
||||||
|
`generate/math_candidate_parser.py::_COMPARISON_ANCHOR_VERBS`,
|
||||||
|
- depletion/transfer verbs refuse by positive determination (an allowlist, not
|
||||||
|
a blocklist — the fail-open blocklist was reverted),
|
||||||
|
- documented in `docs/research/increment-1-band-plan-2026-07-19.md` §BUILD
|
||||||
|
OUTCOME.
|
||||||
|
|
||||||
|
Its measured yield when it shipped: **PARSED unchanged — tune 3, measure 2, no
|
||||||
|
conversion.** The scoped band `{seed + forward-comparison + q:simple/summation}`
|
||||||
|
converted **0** tune cases, measured four independent ways (yield harness, band
|
||||||
|
map, band oracle, capability histogram).
|
||||||
|
|
||||||
|
## Re-measured today, on the frozen split
|
||||||
|
|
||||||
|
```
|
||||||
|
uv run python evals/gsm8k_math/holdout_dev/v1/runner.py
|
||||||
|
holdout_dev: correct=5 wrong=0 refused=495 (n=500)
|
||||||
|
```
|
||||||
|
|
||||||
|
Five correct is exactly the tune-3 + measure-2 the increment-1 report recorded.
|
||||||
|
**No drift in five days of unrelated arc work, and wrong=0 still holds.** The
|
||||||
|
tune/measure split (`evals/gsm8k_math/holdout_dev/v1/split.py`) is frozen and
|
||||||
|
was not touched.
|
||||||
|
|
||||||
|
## Why the honest move is to stop rather than extend
|
||||||
|
|
||||||
|
The arc's own evidence says the wall is not where 4.1 aims:
|
||||||
|
|
||||||
|
1. **The ~30-band bet was falsified.** In-band cases strand on capabilities
|
||||||
|
*outside* the band — 25 need multi-compound, 17 need compare-additive.
|
||||||
|
Adding more seed verbs cannot convert a case that also needs rate,
|
||||||
|
currency, and unit conversion in the same sentence.
|
||||||
|
2. **The taxonomy is not at bedrock.** The "smallest tractable set"
|
||||||
|
`{seed, q:simple}` dissolved on inspection: every one of its four cases
|
||||||
|
really needs rate / compare / currency / copula. Real GSM8K statements are
|
||||||
|
individually multi-capability — the conjunction recurs *within* a sentence,
|
||||||
|
so a capability list keeps fragmenting rather than converging.
|
||||||
|
3. **Growing the allowlist is the forbidden move.** ADR-0251 and
|
||||||
|
`docs/research/reader-arc-overfit-inventory-2026-07-19.md` exist because
|
||||||
|
per-case pattern growth previously produced overfit "lift" that committed
|
||||||
|
*wrong* answers on the real exam. An extension whose only evidence is "it
|
||||||
|
parses more tune cases" is the exact shape of that failure.
|
||||||
|
|
||||||
|
## The live next move (unchanged, and it is not 4.1)
|
||||||
|
|
||||||
|
The reader arc's own recommendation, carried to Shay with the increment-1
|
||||||
|
foundations, is **case-first, not capability-first**: pick 2–3 specific closest
|
||||||
|
tune cases, hand-enumerate their *exact* end-to-end needs however many
|
||||||
|
capabilities that is, build precisely that, convert them, measure. The closest
|
||||||
|
cases are already identified — `0000`, `0001`, `0148`, `0082` — all needing
|
||||||
|
complex compare forms (chained / mass-noun entities / "than" + aggregate
|
||||||
|
reference) plus summation or difference.
|
||||||
|
|
||||||
|
`multi-compound` is the single highest-leverage capability (on the critical
|
||||||
|
path for 70/106 = 66% of tractable needed-sets) but is insufficient alone.
|
||||||
|
`q:complex` remains the largest intractable wall (~35+ cases).
|
||||||
|
|
||||||
|
## Recommendation for the plan
|
||||||
|
|
||||||
|
Phase 4.1 should be marked **complete-with-null-yield** rather than pending,
|
||||||
|
and Phase 4 re-pointed at the increment-2 case-first work. The measurement
|
||||||
|
above is the evidence; nothing in this session's arc work moved it either way,
|
||||||
|
which is itself the useful signal — the math lane is genuinely orthogonal to
|
||||||
|
the reading/curriculum work, exactly as the plan's §3 dependency table assumed.
|
||||||
|
|
||||||
|
Phase 4.2 (compare unblock) and 4.3 (q:complex decomposition study) are
|
||||||
|
untouched by this and remain queued behind the case-first increment.
|
||||||
0
evals/curriculum_serve/__init__.py
Normal file
0
evals/curriculum_serve/__init__.py
Normal file
86
evals/curriculum_serve/contract.md
Normal file
86
evals/curriculum_serve/contract.md
Normal file
|
|
@ -0,0 +1,86 @@
|
||||||
|
# Curriculum-serve lane contract
|
||||||
|
|
||||||
|
## What this lane scores
|
||||||
|
|
||||||
|
The **production curriculum-grounded answering path** — the exact pipeline
|
||||||
|
`chat/curriculum_surface.py::decide_curriculum_question` runs on a `core chat`
|
||||||
|
turn when `curriculum_serving_enabled` is on:
|
||||||
|
|
||||||
|
```
|
||||||
|
Does <term> <relation> <term>? (closed exam-question grammar)
|
||||||
|
→ subject routing (the domain whose ratified vocabulary holds BOTH terms)
|
||||||
|
→ relation family (the connective, normalized by ADR-0260 agreement)
|
||||||
|
→ premise compilation (that family's RATIFIED chains, and nothing else)
|
||||||
|
→ the argument bands (ADR-0260 verb reading → ADR-0261 existential)
|
||||||
|
→ the ROBDD engine (ADR-0201/0218)
|
||||||
|
```
|
||||||
|
|
||||||
|
There is no subject-specific decision code anywhere in the path. Physics
|
||||||
|
differs from philosophy only in which curriculum rows load — which is the
|
||||||
|
property that makes "add a subject" a data operation.
|
||||||
|
|
||||||
|
## Gold vocabulary
|
||||||
|
|
||||||
|
Three classes are reachable: `entailed`, `unknown`, `declined`.
|
||||||
|
|
||||||
|
`refuted` is **unreachable from a purely positive curriculum** and that is not
|
||||||
|
an omission. The curriculum is read OPEN-world: a relation it does not state
|
||||||
|
is UNKNOWN, never "no". Refuting would require the curriculum to teach a
|
||||||
|
negative ("no X causes Y"), which no ratified corpus does yet. See ADR-0262 §5.
|
||||||
|
|
||||||
|
## The three lane guards (all run before any case is scored)
|
||||||
|
|
||||||
|
Failing a guard is a **lane** failure, not a case miss — the lane is unsound,
|
||||||
|
not merely under-covered.
|
||||||
|
|
||||||
|
1. **Provenance** (plan §4.3) — every chain id a case pins must resolve in the
|
||||||
|
subject's ratified curriculum. A case whose curriculum moved under it
|
||||||
|
breaks loudly rather than quietly answering from what remains.
|
||||||
|
2. **Corpus soundness** (§4.4) — the INDEPENDENT oracle
|
||||||
|
(`evals/curriculum_serve/oracle.py`, sharing no code with the serving path:
|
||||||
|
own loader, own ratification predicate, own family table, own agreement
|
||||||
|
normalization, own verdict rule) must re-derive every committed gold.
|
||||||
|
3. **Anti-recall coverage** (§4.7) — the split must carry ≥3 probes whose
|
||||||
|
answer is true in the world and absent from the curriculum, and no probe
|
||||||
|
may carry a committed gold. Without them the lane cannot show the system
|
||||||
|
decodes rather than recalls, and it must not ship.
|
||||||
|
|
||||||
|
## Splits
|
||||||
|
|
||||||
|
- `physics/` — 32 hand-authored cases over `physics_chains_v1`: 13 taught
|
||||||
|
edges (both connectives of each family, incl. the row whose declared
|
||||||
|
`operator_family` disagrees with its connective), 5 untaught compositions at
|
||||||
|
depth 2–3, 4 reverse-direction and cross-relation near-misses, 5 anti-recall
|
||||||
|
probes (3 with untaught vocabulary — *gravity*, *current*, *pressure* — and
|
||||||
|
2 with taught vocabulary but untaught relations), and 5 typed refusals.
|
||||||
|
|
||||||
|
## wrong=0 discipline
|
||||||
|
|
||||||
|
Identical to the deduction-serve lane: `wrong` (a committed verdict that
|
||||||
|
disagrees with gold) MUST stay 0; a decline where gold expected a verdict is a
|
||||||
|
coverage miss, tracked in `counts.declined`, never conflated with a
|
||||||
|
confabulation. The runner requires `correct == n`.
|
||||||
|
|
||||||
|
## What the lane deliberately does NOT do
|
||||||
|
|
||||||
|
- **It does not compose chains.** `force causes acceleration` and
|
||||||
|
`acceleration causes motion` are both taught; `force causes motion` is
|
||||||
|
UNKNOWN. Causal transitivity is a substantive claim about the world, and no
|
||||||
|
ratified corpus teaches it. The oracle reports the shortest path length
|
||||||
|
alongside its verdict precisely so the lane can assert that a reachable pair
|
||||||
|
is still answered UNKNOWN — composition is provably not happening.
|
||||||
|
- **It does not read the corpus's `operator_family` field.** The family comes
|
||||||
|
from the CONNECTIVE, because a question carries a relation word and nothing
|
||||||
|
else; deriving the family from a field the question cannot carry would let
|
||||||
|
premise compilation and question routing disagree, and a taught edge could
|
||||||
|
go missing from the premises compiled to decide it.
|
||||||
|
|
||||||
|
## Reproduce
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv run python -m evals.curriculum_serve.runner # human-facing
|
||||||
|
uv run python -m evals.curriculum_serve.runner --report evals/curriculum_serve/report.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Pinned in `scripts/verify_lane_shas.py` as lane id `curriculum_serve_v1`.
|
||||||
|
`core test --suite deductive` runs `tests/test_curriculum_serve.py`.
|
||||||
189
evals/curriculum_serve/oracle.py
Normal file
189
evals/curriculum_serve/oracle.py
Normal file
|
|
@ -0,0 +1,189 @@
|
||||||
|
"""Independent curriculum oracle — the gold for the curriculum-serve lane.
|
||||||
|
|
||||||
|
This is **deliberately a second, independent decision procedure** (plan §4.4).
|
||||||
|
It shares no code with the serving path: its own JSONL loader, its own
|
||||||
|
ratification predicate, its own connective→family table, its own agreement
|
||||||
|
normalization, and its own verdict rule. Two independently-written procedures
|
||||||
|
agreeing on every case is real evidence the serving path reads the curriculum
|
||||||
|
correctly; a shared-code "oracle" would only prove the compiler agrees with
|
||||||
|
itself.
|
||||||
|
|
||||||
|
It is intentionally simple: a closed-world reachability oracle over the chain
|
||||||
|
graph. "Closed-world" describes only what the ORACLE can see — the ratified
|
||||||
|
corpus and nothing else. It does NOT mean untaught facts are false: an edge
|
||||||
|
absent from the corpus is UNKNOWN, never refuted, because a curriculum that
|
||||||
|
does not mention something has said nothing about it. The oracle also reports
|
||||||
|
the shortest path length between the two terms, which is what lets the lane
|
||||||
|
assert the property that matters most here — that a reachable-but-untaught
|
||||||
|
pair is answered UNKNOWN rather than composed into a claim.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from collections import deque
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
_CHAIN_DIR = _REPO_ROOT / "teaching" / "domain_chains"
|
||||||
|
_PACK_DIR = _REPO_ROOT / "packs" / "data"
|
||||||
|
|
||||||
|
#: Independently restated (the serving path has its own copy — that is the point).
|
||||||
|
_FAMILY_OF_CONNECTIVE = {
|
||||||
|
"causes": "causal",
|
||||||
|
"reveals": "causal",
|
||||||
|
"grounds": "causal",
|
||||||
|
"requires": "modal",
|
||||||
|
"enables": "modal",
|
||||||
|
"precedes": "sequence",
|
||||||
|
"opposes": "contrast",
|
||||||
|
"supports": "evidential",
|
||||||
|
}
|
||||||
|
|
||||||
|
#: Which corpora and packs each subject is taught from — restated, not imported.
|
||||||
|
_DOMAIN_SOURCES: dict[str, tuple[tuple[str, ...], tuple[str, ...]]] = {
|
||||||
|
"physics": (("physics_chains_v1",), ("en_physics_v1",)),
|
||||||
|
"mathematics_logic": (("mathematics_logic_chains_v1",), ("en_mathematics_logic_v1",)),
|
||||||
|
"systems_software": (("systems_software_chains_v1",), ("en_systems_software_v1",)),
|
||||||
|
"philosophy_theology": (
|
||||||
|
("philosophy_theology_chains_v1",),
|
||||||
|
("en_core_cognition_v1", "en_core_meta_v1"),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class OracleVerdict:
|
||||||
|
"""The gold verdict, plus the evidence the lane asserts against."""
|
||||||
|
|
||||||
|
verdict: str # entailed | unknown | declined
|
||||||
|
reason: str # typed reason when declined, else ""
|
||||||
|
family: str # "" when the relation is not taught anywhere
|
||||||
|
depth: int # 1 = taught edge, >1 = reachable in n hops, 0 = no path
|
||||||
|
|
||||||
|
|
||||||
|
def _lemmas(pack_id: str) -> set[str]:
|
||||||
|
path = _PACK_DIR / pack_id / "lexicon.jsonl"
|
||||||
|
out: set[str] = set()
|
||||||
|
if not path.exists():
|
||||||
|
return out
|
||||||
|
for line in path.read_text(encoding="utf-8").splitlines():
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
row = json.loads(line)
|
||||||
|
lemma = row.get("lemma")
|
||||||
|
if isinstance(lemma, str):
|
||||||
|
out.add(lemma)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _edges(domain: str) -> list[tuple[str, str, str, str]]:
|
||||||
|
"""``(subject, connective, object, chain_id)`` for every ratified row."""
|
||||||
|
corpora, packs = _DOMAIN_SOURCES[domain]
|
||||||
|
vocabulary = set()
|
||||||
|
for pack in packs:
|
||||||
|
vocabulary |= _lemmas(pack)
|
||||||
|
out: list[tuple[str, str, str, str]] = []
|
||||||
|
for corpus in corpora:
|
||||||
|
path = _CHAIN_DIR / f"{corpus}.jsonl"
|
||||||
|
if not path.exists():
|
||||||
|
continue
|
||||||
|
for line in path.read_text(encoding="utf-8").splitlines():
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
row = json.loads(line)
|
||||||
|
if row.get("review_status") != "reviewed":
|
||||||
|
continue
|
||||||
|
if row.get("domain") != domain:
|
||||||
|
continue
|
||||||
|
subject, obj = row.get("subject"), row.get("object")
|
||||||
|
connective = row.get("connective")
|
||||||
|
if subject not in vocabulary or obj not in vocabulary:
|
||||||
|
continue
|
||||||
|
if connective not in _FAMILY_OF_CONNECTIVE:
|
||||||
|
continue
|
||||||
|
out.append((subject, connective, obj, row.get("chain_id", "")))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def vocabulary(domain: str) -> set[str]:
|
||||||
|
"""Every lemma the subject's packs teach."""
|
||||||
|
out: set[str] = set()
|
||||||
|
for pack in _DOMAIN_SOURCES[domain][1]:
|
||||||
|
out |= _lemmas(pack)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _base_forms(word: str) -> set[str]:
|
||||||
|
"""The spellings a question's relation word may take — independently
|
||||||
|
written, covering the same +s/+es/y↔ies agreement the serving path
|
||||||
|
normalizes with its own table."""
|
||||||
|
forms = {word}
|
||||||
|
forms.add(word + "s")
|
||||||
|
forms.add(word + "es")
|
||||||
|
if word.endswith("y"):
|
||||||
|
forms.add(word[:-1] + "ies")
|
||||||
|
if word.endswith("s"):
|
||||||
|
forms.add(word[:-1])
|
||||||
|
if word.endswith("es"):
|
||||||
|
forms.add(word[:-2])
|
||||||
|
if word.endswith("ies"):
|
||||||
|
forms.add(word[:-3] + "y")
|
||||||
|
return forms
|
||||||
|
|
||||||
|
|
||||||
|
def _connective_for(relation: str) -> str | None:
|
||||||
|
candidates = _base_forms(relation)
|
||||||
|
for connective in _FAMILY_OF_CONNECTIVE:
|
||||||
|
if connective in candidates or relation == connective:
|
||||||
|
return connective
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def oracle_answer(domain: str, subject: str, relation: str, obj: str) -> OracleVerdict:
|
||||||
|
"""The gold verdict for one exam question against *domain*'s curriculum."""
|
||||||
|
vocab = vocabulary(domain)
|
||||||
|
if subject not in vocab or obj not in vocab:
|
||||||
|
return OracleVerdict("declined", "untaught_vocabulary", "", 0)
|
||||||
|
connective = _connective_for(relation)
|
||||||
|
if connective is None:
|
||||||
|
return OracleVerdict("declined", "out_of_curriculum", "", 0)
|
||||||
|
family = _FAMILY_OF_CONNECTIVE[connective]
|
||||||
|
edges = [e for e in _edges(domain) if _FAMILY_OF_CONNECTIVE[e[1]] == family]
|
||||||
|
if not edges:
|
||||||
|
return OracleVerdict("declined", "empty_curriculum", family, 0)
|
||||||
|
# Entailment needs the SAME relation, not merely the same family: the
|
||||||
|
# curriculum teaching "entropy reveals energy" has not thereby taught
|
||||||
|
# "entropy causes energy". Family scoping decides which premises are in
|
||||||
|
# play; the connective decides what was actually said.
|
||||||
|
if any(s == subject and c == connective and o == obj for s, c, o, _id in edges):
|
||||||
|
return OracleVerdict("entailed", "", family, 1)
|
||||||
|
# No taught edge. Report the shortest path so the lane can prove the
|
||||||
|
# serving path does NOT compose a chain into a claim.
|
||||||
|
adjacency: dict[str, list[str]] = {}
|
||||||
|
for s, _c, o, _id in edges:
|
||||||
|
adjacency.setdefault(s, []).append(o)
|
||||||
|
seen = {subject}
|
||||||
|
queue: deque[tuple[str, int]] = deque([(subject, 0)])
|
||||||
|
while queue:
|
||||||
|
node, dist = queue.popleft()
|
||||||
|
for nxt in adjacency.get(node, ()):
|
||||||
|
if nxt == obj:
|
||||||
|
# ``dist`` counts edges already traversed to reach ``node``;
|
||||||
|
# this one closes the path, so the shortest path is dist + 1.
|
||||||
|
return OracleVerdict("unknown", "", family, dist + 1)
|
||||||
|
if nxt not in seen:
|
||||||
|
seen.add(nxt)
|
||||||
|
queue.append((nxt, dist + 1))
|
||||||
|
return OracleVerdict("unknown", "", family, 0)
|
||||||
|
|
||||||
|
|
||||||
|
def taught_edges(domain: str) -> list[tuple[str, str, str, str]]:
|
||||||
|
"""Public view for the lane's provenance assertions."""
|
||||||
|
return _edges(domain)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["OracleVerdict", "oracle_answer", "taught_edges", "vocabulary"]
|
||||||
0
evals/curriculum_serve/physics/__init__.py
Normal file
0
evals/curriculum_serve/physics/__init__.py
Normal file
32
evals/curriculum_serve/physics/cases.jsonl
Normal file
32
evals/curriculum_serve/physics/cases.jsonl
Normal file
|
|
@ -0,0 +1,32 @@
|
||||||
|
{"id": "cs-phy-0001", "text": "Does force cause acceleration?", "gold": "entailed", "chains": ["physics-causal-001"], "class": "taught_edge_causal"}
|
||||||
|
{"id": "cs-phy-0002", "text": "Does acceleration cause motion?", "gold": "entailed", "chains": ["physics-causal-002"], "class": "taught_edge_causal"}
|
||||||
|
{"id": "cs-phy-0003", "text": "Does work cause energy?", "gold": "entailed", "chains": ["physics-causal-003"], "class": "taught_edge_causal"}
|
||||||
|
{"id": "cs-phy-0004", "text": "Does charge cause field?", "gold": "entailed", "chains": ["physics-causal-004"], "class": "taught_edge_causal"}
|
||||||
|
{"id": "cs-phy-0005", "text": "Does field cause force?", "gold": "entailed", "chains": ["physics-causal-005"], "class": "taught_edge_causal"}
|
||||||
|
{"id": "cs-phy-0006", "text": "Does temperature reveal motion?", "gold": "entailed", "chains": ["physics-causal-006"], "class": "taught_edge_reveals"}
|
||||||
|
{"id": "cs-phy-0007", "text": "Does entropy reveal energy?", "gold": "entailed", "chains": ["physics-causal-007"], "class": "taught_edge_reveals"}
|
||||||
|
{"id": "cs-phy-0008", "text": "Does conservation require symmetry?", "gold": "entailed", "chains": ["physics-modal-001"], "class": "taught_edge_modal"}
|
||||||
|
{"id": "cs-phy-0009", "text": "Does mass require force?", "gold": "entailed", "chains": ["physics-modal-002"], "class": "taught_edge_modal"}
|
||||||
|
{"id": "cs-phy-0010", "text": "Does momentum require mass?", "gold": "entailed", "chains": ["physics-modal-003"], "class": "taught_edge_modal"}
|
||||||
|
{"id": "cs-phy-0011", "text": "Does energy require conservation?", "gold": "entailed", "chains": ["physics-modal-004"], "class": "taught_edge_modal"}
|
||||||
|
{"id": "cs-phy-0012", "text": "Does frequency require wave?", "gold": "entailed", "chains": ["physics-modal-005"], "class": "taught_edge_modal"}
|
||||||
|
{"id": "cs-phy-0013", "text": "Does wave require field?", "gold": "entailed", "chains": ["physics-causal-008"], "class": "taught_edge_family_by_connective"}
|
||||||
|
{"id": "cs-phy-0014", "text": "Does force cause motion?", "gold": "unknown", "chains": ["physics-causal-001", "physics-causal-002"], "class": "two_hop_not_composed"}
|
||||||
|
{"id": "cs-phy-0015", "text": "Does charge cause acceleration?", "gold": "unknown", "chains": ["physics-causal-004", "physics-causal-005", "physics-causal-001"], "class": "three_hop_not_composed"}
|
||||||
|
{"id": "cs-phy-0016", "text": "Does energy require symmetry?", "gold": "unknown", "chains": ["physics-modal-004", "physics-modal-001"], "class": "two_hop_modal_not_composed"}
|
||||||
|
{"id": "cs-phy-0017", "text": "Does momentum require force?", "gold": "unknown", "chains": ["physics-modal-003", "physics-modal-002"], "class": "two_hop_modal_not_composed"}
|
||||||
|
{"id": "cs-phy-0018", "text": "Does acceleration cause force?", "gold": "unknown", "chains": ["physics-causal-001"], "class": "reverse_direction"}
|
||||||
|
{"id": "cs-phy-0019", "text": "Does motion cause acceleration?", "gold": "unknown", "chains": ["physics-causal-002"], "class": "reverse_direction"}
|
||||||
|
{"id": "cs-phy-0020", "text": "Does entropy cause energy?", "gold": "unknown", "chains": ["physics-causal-007"], "class": "relation_not_interchangeable"}
|
||||||
|
{"id": "cs-phy-0021", "text": "Does work cause motion?", "gold": "unknown", "chains": [], "class": "unlinked_pair"}
|
||||||
|
{"id": "cs-phy-0022", "text": "Does temperature require symmetry?", "gold": "unknown", "chains": [], "class": "unlinked_pair"}
|
||||||
|
{"id": "cs-phy-0023", "text": "Does mass require energy?", "gold": "unknown", "chains": [], "class": "anti_recall_true_but_untaught"}
|
||||||
|
{"id": "cs-phy-0024", "text": "Does force cause work?", "gold": "unknown", "chains": [], "class": "anti_recall_true_but_untaught"}
|
||||||
|
{"id": "cs-phy-0025", "text": "Does charge require field?", "gold": "unknown", "chains": ["physics-modal-007"], "class": "reverse_direction_modal"}
|
||||||
|
{"id": "cs-phy-0026", "text": "Does field require charge?", "gold": "entailed", "chains": ["physics-modal-007"], "class": "taught_edge_modal"}
|
||||||
|
{"id": "cs-phy-0027", "text": "Does gravity cause acceleration?", "gold": "declined", "chains": [], "class": "anti_recall_untaught_vocabulary"}
|
||||||
|
{"id": "cs-phy-0028", "text": "Does current cause field?", "gold": "declined", "chains": [], "class": "anti_recall_untaught_vocabulary"}
|
||||||
|
{"id": "cs-phy-0029", "text": "Does pressure require volume?", "gold": "declined", "chains": [], "class": "anti_recall_untaught_vocabulary"}
|
||||||
|
{"id": "cs-phy-0030", "text": "Does force explain acceleration?", "gold": "declined", "chains": [], "class": "out_of_curriculum_relation"}
|
||||||
|
{"id": "cs-phy-0031", "text": "Does force accelerate mass quickly?", "gold": "declined", "chains": [], "class": "question_shape_out_of_band"}
|
||||||
|
{"id": "cs-phy-0032", "text": "Does the force cause acceleration?", "gold": "declined", "chains": [], "class": "question_shape_out_of_band"}
|
||||||
40
evals/curriculum_serve/report.json
Normal file
40
evals/curriculum_serve/report.json
Normal file
|
|
@ -0,0 +1,40 @@
|
||||||
|
{
|
||||||
|
"aggregate": {
|
||||||
|
"correct": 32,
|
||||||
|
"declined": 0,
|
||||||
|
"n": 32,
|
||||||
|
"wrong": 0
|
||||||
|
},
|
||||||
|
"all_correct": true,
|
||||||
|
"arc": "generalization",
|
||||||
|
"lane": "curriculum_serve",
|
||||||
|
"schema_version": 1,
|
||||||
|
"splits": {
|
||||||
|
"physics": {
|
||||||
|
"all_cases_correct": true,
|
||||||
|
"anti_recall_probes": 5,
|
||||||
|
"by_band": {
|
||||||
|
"curriculum_physics_causal": 14,
|
||||||
|
"curriculum_physics_modal": 12
|
||||||
|
},
|
||||||
|
"by_gold": {
|
||||||
|
"declined": 6,
|
||||||
|
"entailed": 14,
|
||||||
|
"unknown": 12
|
||||||
|
},
|
||||||
|
"correct_by_gold": {
|
||||||
|
"declined": 6,
|
||||||
|
"entailed": 14,
|
||||||
|
"unknown": 12
|
||||||
|
},
|
||||||
|
"counts": {
|
||||||
|
"correct": 32,
|
||||||
|
"declined": 0,
|
||||||
|
"wrong": 0
|
||||||
|
},
|
||||||
|
"domain": "physics",
|
||||||
|
"n": 32
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"wrong_is_zero": true
|
||||||
|
}
|
||||||
216
evals/curriculum_serve/runner.py
Normal file
216
evals/curriculum_serve/runner.py
Normal file
|
|
@ -0,0 +1,216 @@
|
||||||
|
"""Curriculum-serve lane — scores curriculum-grounded exam answering.
|
||||||
|
|
||||||
|
The Phase-2 lane of the generalization arc. For each committed case, the raw
|
||||||
|
question text runs through the exact production path
|
||||||
|
``chat/curriculum_surface.py::decide_curriculum_question`` — question reader →
|
||||||
|
subject routing → ratified-curriculum premise compilation → the argument bands
|
||||||
|
→ the ROBDD engine — and the resulting verdict is compared to gold authored
|
||||||
|
INDEPENDENTLY by ``evals.curriculum_serve.oracle`` (plan §4.4).
|
||||||
|
|
||||||
|
Three guards run before any case is scored, and any of them failing is a lane
|
||||||
|
failure, not a case miss:
|
||||||
|
|
||||||
|
1. **Provenance** (§4.3) — every chain id a case pins must resolve in the
|
||||||
|
subject's ratified curriculum. A case whose curriculum moved under it
|
||||||
|
breaks loudly instead of quietly answering from what is left.
|
||||||
|
2. **Corpus soundness** (§4.4) — the independent oracle must agree with every
|
||||||
|
committed gold. A gold nobody can re-derive from the curriculum is not gold.
|
||||||
|
3. **Anti-recall coverage** (§4.7) — the split MUST contain cases whose answer
|
||||||
|
is true in the world but absent from the curriculum. Without them the lane
|
||||||
|
cannot show the system decodes rather than recalls, and it must not ship.
|
||||||
|
|
||||||
|
Counts mirror the deduction-serve lane exactly: ``wrong`` (a committed verdict
|
||||||
|
that disagrees with gold) MUST stay 0; a decline where gold expected a verdict
|
||||||
|
is a coverage miss, tracked but never conflated with a confabulation.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
from collections import Counter
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from chat.curriculum_surface import decide_curriculum_question, read_curriculum_question
|
||||||
|
from evals.curriculum_serve.oracle import oracle_answer
|
||||||
|
from teaching.curriculum_premises import load_curriculum, resolve_pinned
|
||||||
|
|
||||||
|
_ROOT = Path(__file__).resolve().parent
|
||||||
|
|
||||||
|
_SPLITS: tuple[tuple[str, str, Path], ...] = (
|
||||||
|
# (split name, subject domain, cases)
|
||||||
|
("physics", "physics", _ROOT / "physics" / "cases.jsonl"),
|
||||||
|
)
|
||||||
|
|
||||||
|
#: A split must carry at least this many anti-recall probes (§4.7). The class
|
||||||
|
#: prefix is the contract: a probe is a case whose answer is true in the world
|
||||||
|
#: and absent from the curriculum, so the honest verdict is a non-commitment.
|
||||||
|
_ANTI_RECALL_PREFIX = "anti_recall"
|
||||||
|
_MIN_ANTI_RECALL = 3
|
||||||
|
|
||||||
|
|
||||||
|
class LaneContractError(AssertionError):
|
||||||
|
"""A guard failed — the lane itself is unsound, not merely under-covered."""
|
||||||
|
|
||||||
|
|
||||||
|
def _load(path: Path) -> list[dict]:
|
||||||
|
with path.open(encoding="utf-8") as fh:
|
||||||
|
return [json.loads(line) for line in fh if line.strip()]
|
||||||
|
|
||||||
|
|
||||||
|
def assert_provenance(domain: str, cases: list[dict]) -> None:
|
||||||
|
"""Every pinned chain id must resolve in the ratified curriculum (§4.3)."""
|
||||||
|
curriculum = load_curriculum(domain)
|
||||||
|
for case in cases:
|
||||||
|
pinned = tuple(case.get("chains", ()))
|
||||||
|
if not pinned:
|
||||||
|
continue
|
||||||
|
resolve_pinned(curriculum, pinned) # raises UnratifiedChain
|
||||||
|
|
||||||
|
|
||||||
|
def assert_corpus_sound(domain: str, cases: list[dict]) -> None:
|
||||||
|
"""The independent oracle must re-derive every committed gold (§4.4)."""
|
||||||
|
for case in cases:
|
||||||
|
query = read_curriculum_question(case["text"])
|
||||||
|
gold = case["gold"]
|
||||||
|
if not hasattr(query, "subject"):
|
||||||
|
# Unreadable question shapes are gold-``declined`` by construction;
|
||||||
|
# the oracle has no opinion on a text it cannot parse either.
|
||||||
|
if gold != "declined":
|
||||||
|
raise LaneContractError(
|
||||||
|
f"{case['id']}: unreadable question with gold={gold}"
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
verdict = oracle_answer(domain, query.subject, query.verb, query.obj)
|
||||||
|
if verdict.verdict != gold:
|
||||||
|
raise LaneContractError(
|
||||||
|
f"{case['id']}: oracle={verdict.verdict} gold={gold} "
|
||||||
|
f"(family={verdict.family} depth={verdict.depth}) text={case['text']!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def assert_anti_recall_coverage(cases: list[dict]) -> None:
|
||||||
|
"""The split must probe recall, or it must not ship (§4.7)."""
|
||||||
|
probes = [c for c in cases if str(c.get("class", "")).startswith(_ANTI_RECALL_PREFIX)]
|
||||||
|
if len(probes) < _MIN_ANTI_RECALL:
|
||||||
|
raise LaneContractError(
|
||||||
|
f"split has {len(probes)} anti-recall probes, needs >= {_MIN_ANTI_RECALL}"
|
||||||
|
)
|
||||||
|
for probe in probes:
|
||||||
|
if probe["gold"] in ("entailed", "refuted"):
|
||||||
|
raise LaneContractError(
|
||||||
|
f"{probe['id']}: an anti-recall probe cannot have a committed gold"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def build_report(domain: str, cases: list[dict]) -> dict:
|
||||||
|
assert_provenance(domain, cases)
|
||||||
|
assert_corpus_sound(domain, cases)
|
||||||
|
assert_anti_recall_coverage(cases)
|
||||||
|
|
||||||
|
counts = Counter({"correct": 0, "wrong": 0, "declined": 0})
|
||||||
|
by_gold: Counter[str] = Counter()
|
||||||
|
correct_by_gold: Counter[str] = Counter()
|
||||||
|
by_band: Counter[str] = Counter()
|
||||||
|
mismatches: list[dict] = []
|
||||||
|
|
||||||
|
for case in cases:
|
||||||
|
gold = case["gold"]
|
||||||
|
by_gold[gold] += 1
|
||||||
|
decision = decide_curriculum_question(case["text"])
|
||||||
|
if decision.band:
|
||||||
|
by_band[decision.band] += 1
|
||||||
|
if decision.verdict == gold:
|
||||||
|
counts["correct"] += 1
|
||||||
|
correct_by_gold[gold] += 1
|
||||||
|
elif decision.verdict == "declined":
|
||||||
|
counts["declined"] += 1
|
||||||
|
mismatches.append(
|
||||||
|
{"id": case["id"], "gold": gold, "got": decision.verdict,
|
||||||
|
"reason": decision.reason, "text": case["text"]}
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
counts["wrong"] += 1
|
||||||
|
mismatches.append(
|
||||||
|
{"id": case["id"], "gold": gold, "got": decision.verdict,
|
||||||
|
"reason": decision.reason, "text": case["text"]}
|
||||||
|
)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"n": len(cases),
|
||||||
|
"counts": dict(counts),
|
||||||
|
"by_gold": dict(by_gold),
|
||||||
|
"correct_by_gold": dict(correct_by_gold),
|
||||||
|
"by_band": dict(by_band),
|
||||||
|
"anti_recall_probes": sum(
|
||||||
|
1 for c in cases if str(c.get("class", "")).startswith(_ANTI_RECALL_PREFIX)
|
||||||
|
),
|
||||||
|
"all_cases_correct": counts["correct"] == len(cases),
|
||||||
|
"mismatch_examples": mismatches[:10],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def build_combined_report() -> dict:
|
||||||
|
"""Deterministic per-split + aggregate report — safe to SHA-pin."""
|
||||||
|
splits: dict[str, dict] = {}
|
||||||
|
aggregate = {"n": 0, "correct": 0, "wrong": 0, "declined": 0}
|
||||||
|
for name, domain, path in _SPLITS:
|
||||||
|
report = build_report(domain, _load(path))
|
||||||
|
splits[name] = {
|
||||||
|
"n": report["n"],
|
||||||
|
"domain": domain,
|
||||||
|
"counts": report["counts"],
|
||||||
|
"by_gold": report["by_gold"],
|
||||||
|
"correct_by_gold": report["correct_by_gold"],
|
||||||
|
"by_band": report["by_band"],
|
||||||
|
"anti_recall_probes": report["anti_recall_probes"],
|
||||||
|
"all_cases_correct": report["all_cases_correct"],
|
||||||
|
}
|
||||||
|
aggregate["n"] += report["n"]
|
||||||
|
for key in ("correct", "wrong", "declined"):
|
||||||
|
aggregate[key] += report["counts"][key]
|
||||||
|
return {
|
||||||
|
"schema_version": 1,
|
||||||
|
"lane": "curriculum_serve",
|
||||||
|
"arc": "generalization",
|
||||||
|
"splits": splits,
|
||||||
|
"aggregate": aggregate,
|
||||||
|
"wrong_is_zero": aggregate["wrong"] == 0,
|
||||||
|
"all_correct": all(s["all_cases_correct"] for s in splits.values()),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def write_combined_report(path: Path) -> dict:
|
||||||
|
report = build_combined_report()
|
||||||
|
path.write_text(
|
||||||
|
json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||||
|
)
|
||||||
|
return report
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--report", type=Path, default=None)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
|
||||||
|
if args.report is not None:
|
||||||
|
report = write_combined_report(args.report)
|
||||||
|
return 0 if (report["wrong_is_zero"] and report["all_correct"]) else 1
|
||||||
|
|
||||||
|
all_ok = True
|
||||||
|
for name, domain, path in _SPLITS:
|
||||||
|
report = build_report(domain, _load(path))
|
||||||
|
c = report["counts"]
|
||||||
|
print(
|
||||||
|
f"[{name}] n={report['n']} correct={c['correct']} wrong={c['wrong']} "
|
||||||
|
f"declined_mismatch={c['declined']} anti_recall={report['anti_recall_probes']}"
|
||||||
|
)
|
||||||
|
for m in report["mismatch_examples"]:
|
||||||
|
print(f" {m['id']}: gold={m['gold']} got={m['got']} "
|
||||||
|
f"reason={m['reason']} text={m['text']!r}")
|
||||||
|
all_ok = all_ok and report["all_cases_correct"]
|
||||||
|
return 0 if all_ok else 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
|
|
@ -18,11 +18,11 @@ commit and SHA-verify on load.
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import json
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from core.learning_arena.engine import run_practice
|
from core.learning_arena.engine import run_practice
|
||||||
|
from core.ratified_ledger import seal_artifact, tally_dict, write_sealed_ledger
|
||||||
from core.reliability_gate import Action, Ceilings, ClassTally, license_for
|
from core.reliability_gate import Action, Ceilings, ClassTally, license_for
|
||||||
from evals.deduction_serve.practice.gold import (
|
from evals.deduction_serve.practice.gold import (
|
||||||
ConstructionGoldTether,
|
ConstructionGoldTether,
|
||||||
|
|
@ -30,7 +30,6 @@ from evals.deduction_serve.practice.gold import (
|
||||||
all_gold_problems,
|
all_gold_problems,
|
||||||
assert_corpus_sound,
|
assert_corpus_sound,
|
||||||
)
|
)
|
||||||
from formation.hashing import sha256_of
|
|
||||||
|
|
||||||
#: The committed sealed ledger lives next to its serving READER (chat/), mirroring
|
#: The committed sealed ledger lives next to its serving READER (chat/), mirroring
|
||||||
#: the estimation ledger's topology (producer in evals/, artifact by the reader).
|
#: the estimation ledger's topology (producer in evals/, artifact by the reader).
|
||||||
|
|
@ -46,13 +45,7 @@ def build_ledger() -> dict[str, ClassTally]:
|
||||||
|
|
||||||
|
|
||||||
def _tally_dict(tally: ClassTally) -> dict[str, Any]:
|
def _tally_dict(tally: ClassTally) -> dict[str, Any]:
|
||||||
return {
|
return tally_dict(tally)
|
||||||
"correct": tally.correct,
|
|
||||||
"wrong": tally.wrong,
|
|
||||||
"refused": tally.refused,
|
|
||||||
"t2_verified": tally.t2_verified,
|
|
||||||
"t2_agrees_gold": tally.t2_agrees_gold,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def run(ceilings: Ceilings | None = None) -> dict[str, Any]:
|
def run(ceilings: Ceilings | None = None) -> dict[str, Any]:
|
||||||
|
|
@ -81,31 +74,30 @@ def run(ceilings: Ceilings | None = None) -> dict[str, Any]:
|
||||||
|
|
||||||
|
|
||||||
def build_sealed_artifact() -> dict[str, Any]:
|
def build_sealed_artifact() -> dict[str, Any]:
|
||||||
"""The committed sealed-ledger dict (self-verifying ``content_sha256``)."""
|
"""The committed sealed-ledger dict (self-verifying ``content_sha256``).
|
||||||
ledger = build_ledger()
|
|
||||||
classes = {cls: _tally_dict(tally) for cls, tally in sorted(ledger.items())}
|
Formatting and hashing come from the shared bridge (ADR-0263) — byte
|
||||||
return {
|
-identical to what this module wrote before the extraction, which is how
|
||||||
"schema": "deduction_serve_ledger_v1",
|
the refactor is proven safe: re-sealing must not move the artifact.
|
||||||
"classes": classes,
|
"""
|
||||||
"content_sha256": sha256_of(classes),
|
return seal_artifact(
|
||||||
"note": (
|
build_ledger(),
|
||||||
|
schema="deduction_serve_ledger_v1",
|
||||||
|
note=(
|
||||||
"Sealed-practice committed ledger for deduction serving (ADR-0256). "
|
"Sealed-practice committed ledger for deduction serving (ADR-0256). "
|
||||||
"Engine reads, never writes. Ceilings stay at safe defaults "
|
"Engine reads, never writes. Ceilings stay at safe defaults "
|
||||||
"(theta_SERVE=0.99). A band earns SERVE by demonstrated pipeline "
|
"(theta_SERVE=0.99). A band earns SERVE by demonstrated pipeline "
|
||||||
"reliability (reader+projector+engine) at volume >= 657 committed."
|
"reliability (reader+projector+engine) at volume >= 657 committed."
|
||||||
),
|
),
|
||||||
"provenance": "evals.deduction_serve.practice.runner.seal_ledger",
|
provenance="evals.deduction_serve.practice.runner.seal_ledger",
|
||||||
}
|
)
|
||||||
|
|
||||||
|
|
||||||
def seal_ledger(path: Path = _SEALED_LEDGER_PATH) -> dict[str, Any]:
|
def seal_ledger(path: Path = _SEALED_LEDGER_PATH) -> dict[str, Any]:
|
||||||
"""Regenerate + write the committed sealed ledger. Verifies corpus soundness
|
"""Regenerate + write the committed sealed ledger. Verifies corpus soundness
|
||||||
against the independent oracle first (a mis-stated gold can never seal)."""
|
against the independent oracle first (a mis-stated gold can never seal)."""
|
||||||
assert_corpus_sound()
|
assert_corpus_sound()
|
||||||
artifact = build_sealed_artifact()
|
return write_sealed_ledger(path, build_sealed_artifact())
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
path.write_text(json.dumps(artifact, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
||||||
return artifact
|
|
||||||
|
|
||||||
|
|
||||||
def main(argv: list[str] | None = None) -> int:
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
|
|
||||||
|
|
@ -8,27 +8,28 @@ never writes it. The artifact is the sealed-practice output of
|
||||||
load, so a hand-edited (un-ratified) ledger is rejected rather than silently trusted.
|
load, so a hand-edited (un-ratified) ledger is rejected rather than silently trusted.
|
||||||
|
|
||||||
Determinism: the ledger is immutable ratified data, parsed once and cached; the gate
|
Determinism: the ledger is immutable ratified data, parsed once and cached; the gate
|
||||||
(``license_for``) is pure. No engine self-authorization — ceilings stay at the safe
|
is pure. No engine self-authorization — ceilings stay at the safe defaults (raising
|
||||||
defaults (raising one's own bar is structurally impossible, ADR-0175 invariant #4).
|
one's own bar is structurally impossible, ADR-0175 invariant #4).
|
||||||
|
|
||||||
|
The load/verify/gate mechanics live in ``core.ratified_ledger`` (ADR-0263), the
|
||||||
|
bridge extracted from this module and its two successors; this is the estimation
|
||||||
|
ADAPTER over it, preserving its own public API (including the predicate →
|
||||||
|
converse-class naming, which is the one thing genuinely local to estimation).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from core.reliability_gate import Action, Ceilings, ClassTally, LicenseDecision, license_for
|
from core.ratified_ledger import RatifiedLedgerError, load_sealed_ledger
|
||||||
from formation.hashing import sha256_of
|
from core.ratified_ledger import serve_license as _serve_license
|
||||||
|
from core.reliability_gate import Ceilings, ClassTally, LicenseDecision
|
||||||
from generate.determine.estimate import converse_class_name
|
from generate.determine.estimate import converse_class_name
|
||||||
|
|
||||||
_LEDGER_PATH = Path(__file__).resolve().parent / "data" / "estimation_ledger.json"
|
_LEDGER_PATH = Path(__file__).resolve().parent / "data" / "estimation_ledger.json"
|
||||||
|
|
||||||
|
|
||||||
class RatifiedLedgerError(ValueError):
|
|
||||||
"""The committed estimation ledger is missing, malformed, or tampered with."""
|
|
||||||
|
|
||||||
|
|
||||||
@lru_cache(maxsize=1)
|
@lru_cache(maxsize=1)
|
||||||
def load_ratified_ledger() -> dict[str, ClassTally]:
|
def load_ratified_ledger() -> dict[str, ClassTally]:
|
||||||
"""Load + verify the ratified estimation ledger → per-class ``ClassTally``.
|
"""Load + verify the ratified estimation ledger → per-class ``ClassTally``.
|
||||||
|
|
@ -37,30 +38,7 @@ def load_ratified_ledger() -> dict[str, ClassTally]:
|
||||||
recomputed ``content_sha256`` does not match the committed one (tamper-evidence:
|
recomputed ``content_sha256`` does not match the committed one (tamper-evidence:
|
||||||
only the sealed-practice output is trusted, never a hand-edited ledger).
|
only the sealed-practice output is trusted, never a hand-edited ledger).
|
||||||
"""
|
"""
|
||||||
try:
|
return load_sealed_ledger(_LEDGER_PATH)
|
||||||
artifact = json.loads(_LEDGER_PATH.read_text(encoding="utf-8"))
|
|
||||||
except (OSError, json.JSONDecodeError) as exc: # pragma: no cover - defensive
|
|
||||||
raise RatifiedLedgerError(f"cannot read ratified ledger: {exc}") from exc
|
|
||||||
|
|
||||||
classes = artifact.get("classes")
|
|
||||||
if not isinstance(classes, dict):
|
|
||||||
raise RatifiedLedgerError("ratified ledger has no 'classes' table")
|
|
||||||
if sha256_of(classes) != artifact.get("content_sha256"):
|
|
||||||
raise RatifiedLedgerError(
|
|
||||||
"ratified ledger content_sha256 mismatch — not the sealed-practice output"
|
|
||||||
)
|
|
||||||
|
|
||||||
ledger: dict[str, ClassTally] = {}
|
|
||||||
for cls, counts in classes.items():
|
|
||||||
ledger[cls] = ClassTally(
|
|
||||||
class_name=cls,
|
|
||||||
correct=int(counts.get("correct", 0)),
|
|
||||||
wrong=int(counts.get("wrong", 0)),
|
|
||||||
refused=int(counts.get("refused", 0)),
|
|
||||||
t2_verified=int(counts.get("t2_verified", 0)),
|
|
||||||
t2_agrees_gold=int(counts.get("t2_agrees_gold", 0)),
|
|
||||||
)
|
|
||||||
return ledger
|
|
||||||
|
|
||||||
|
|
||||||
def serve_license(
|
def serve_license(
|
||||||
|
|
@ -76,11 +54,7 @@ def serve_license(
|
||||||
deterministic ``license_for`` verdict under the safe default ceilings.
|
deterministic ``license_for`` verdict under the safe default ceilings.
|
||||||
"""
|
"""
|
||||||
ledger = ledger if ledger is not None else load_ratified_ledger()
|
ledger = ledger if ledger is not None else load_ratified_ledger()
|
||||||
tally = ledger.get(converse_class_name(predicate))
|
return _serve_license(converse_class_name(predicate), ledger, ceilings=ceilings)
|
||||||
if tally is None:
|
|
||||||
return None
|
|
||||||
ceilings = ceilings if ceilings is not None else Ceilings.default()
|
|
||||||
return license_for(tally, Action.SERVE, ceilings)
|
|
||||||
|
|
||||||
|
|
||||||
__all__ = ["RatifiedLedgerError", "load_ratified_ledger", "serve_license"]
|
__all__ = ["RatifiedLedgerError", "load_ratified_ledger", "serve_license"]
|
||||||
|
|
|
||||||
|
|
@ -509,10 +509,22 @@ def read_verb_argument(text: str) -> VerbArgument | VerbRefusal:
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def verb_forms_link(a: str, b: str) -> bool:
|
||||||
|
"""Public view of this band's closed verb-agreement relation.
|
||||||
|
|
||||||
|
Exported so paths OUTSIDE the proof_chain package (the curriculum
|
||||||
|
composer, ADR-0262) can normalize a base form against a stated
|
||||||
|
third-person form using the SAME closed table and suffix rules the reader
|
||||||
|
uses — one agreement authority, not two.
|
||||||
|
"""
|
||||||
|
return _verb_tokens_link(a, b)
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"MAX_ATOMS",
|
"MAX_ATOMS",
|
||||||
"MAX_PREMISE_SENTENCES",
|
"MAX_PREMISE_SENTENCES",
|
||||||
"VerbArgument",
|
"VerbArgument",
|
||||||
"VerbRefusal",
|
"VerbRefusal",
|
||||||
"read_verb_argument",
|
"read_verb_argument",
|
||||||
|
"verb_forms_link",
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -74,6 +74,10 @@ _LANE_ADR: dict[str, tuple[str, str]] = {
|
||||||
"ADR-0206",
|
"ADR-0206",
|
||||||
"Propositional entailment scored against an independent truth-table oracle; dev+holdout+external 716/716 correct, wrong=0, refused=0",
|
"Propositional entailment scored against an independent truth-table oracle; dev+holdout+external 716/716 correct, wrong=0, refused=0",
|
||||||
),
|
),
|
||||||
|
"curriculum_serve_v1": (
|
||||||
|
"ADR-0262",
|
||||||
|
"Flag-gated curriculum serving answers exam questions from a subject's RATIFIED chain corpus only; untaught facts return UNKNOWN (anti-recall probes enforced), wrong=0",
|
||||||
|
),
|
||||||
"deduction_serve_v1": (
|
"deduction_serve_v1": (
|
||||||
"ADR-0256",
|
"ADR-0256",
|
||||||
"Flag-gated deduction serving decides real English/member/fused/verb/existential arguments end-to-end under earned SERVE licenses; wrong=0 across all splits",
|
"Flag-gated deduction serving decides real English/member/fused/verb/existential arguments end-to-end under earned SERVE licenses; wrong=0 across all splits",
|
||||||
|
|
|
||||||
|
|
@ -52,6 +52,7 @@ PINNED_SHAS: dict[str, str] = {
|
||||||
"math_teaching_corpus_v1": "eaf160d145da29f9050ede8d58bf111b0f651dd40aeae9201857d0b97e014dd4",
|
"math_teaching_corpus_v1": "eaf160d145da29f9050ede8d58bf111b0f651dd40aeae9201857d0b97e014dd4",
|
||||||
"deductive_logic_v1": "97a230949016e38d5e3f37a69e4245b320575ee70e5af92ff7607f7b05f74b5f",
|
"deductive_logic_v1": "97a230949016e38d5e3f37a69e4245b320575ee70e5af92ff7607f7b05f74b5f",
|
||||||
"deduction_serve_v1": "0b461a5a49c8f8260ca87d0c9c9f9a17232bd1fdedd982e34649eedf9cca30b5",
|
"deduction_serve_v1": "0b461a5a49c8f8260ca87d0c9c9f9a17232bd1fdedd982e34649eedf9cca30b5",
|
||||||
|
"curriculum_serve_v1": "d9e7ba500f040b865870413a940ee9a49910ac22e1a89c9feec1a60bdd2513f1",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -137,6 +138,12 @@ LANE_SPECS: tuple[LaneSpec, ...] = (
|
||||||
report_relative="evals/deduction_serve/report.json",
|
report_relative="evals/deduction_serve/report.json",
|
||||||
run_as_module=True,
|
run_as_module=True,
|
||||||
),
|
),
|
||||||
|
LaneSpec(
|
||||||
|
lane_id="curriculum_serve_v1",
|
||||||
|
runner_module="evals/curriculum_serve/runner.py",
|
||||||
|
report_relative="evals/curriculum_serve/report.json",
|
||||||
|
run_as_module=True,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
251
teaching/curriculum_premises.py
Normal file
251
teaching/curriculum_premises.py
Normal file
|
|
@ -0,0 +1,251 @@
|
||||||
|
"""Curriculum → premise compiler (generalization arc Phase 2, ADR-0262).
|
||||||
|
|
||||||
|
The load-bearing half of the curriculum-entailment gold contract
|
||||||
|
(docs/plans/generalization-arc-2026-07-24.md §4): an exam question supplies
|
||||||
|
only the QUERY; its premises are compiled here, from the RATIFIED domain-chain
|
||||||
|
corpus of the subject being examined, and from nowhere else. Gold is therefore
|
||||||
|
a function of *(curriculum, question)* — never of case-local hidden text, and
|
||||||
|
never of what a model happens to know about the world. That is the
|
||||||
|
decoding-not-generating line in mechanical form: a fact absent from the
|
||||||
|
curriculum is UNKNOWN, however true it is.
|
||||||
|
|
||||||
|
**Ratification** (§4.3) is the same predicate the capability reporter applies
|
||||||
|
to these corpora, restated here rather than imported: a chain counts iff its
|
||||||
|
``review_status`` is ``reviewed`` AND both its subject and object lemmas are
|
||||||
|
resident in the packs the chain itself declares. The duplication is
|
||||||
|
deliberate and narrow — a serving path must not take a dependency on the
|
||||||
|
reporting subsystem, and this module must be able to state its own admission
|
||||||
|
rule in the terms the serve-time refusals name.
|
||||||
|
|
||||||
|
**Compilation is family-scoped.** A question about a `causes` relation is
|
||||||
|
answered from the `causes` chains. Restricting cannot lose an entailment (a
|
||||||
|
chain's family is fixed by its connective, so an edge that would answer the
|
||||||
|
question is always in the question's own family), and it keeps the compiled
|
||||||
|
premise set inside the reader's honesty caps as corpora grow.
|
||||||
|
|
||||||
|
The compiler emits plain English sentences — ``"force causes acceleration"`` —
|
||||||
|
because the argument bands are the decider (ADR-0256…0261). There is no
|
||||||
|
subject-specific decision code anywhere in this path: physics differs from
|
||||||
|
philosophy only in which rows load.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from functools import lru_cache
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from core.capability.domains import (
|
||||||
|
DOMAIN_CAPABILITY_CORPORA,
|
||||||
|
DOMAIN_CORPORA,
|
||||||
|
DOMAIN_PACKS,
|
||||||
|
)
|
||||||
|
|
||||||
|
_REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
#: Connective → relation family. The closed set this path serves; a chain
|
||||||
|
#: whose connective is absent is not compiled (and cannot be questioned about
|
||||||
|
#: either, so the two ends stay consistent).
|
||||||
|
#:
|
||||||
|
#: The CONNECTIVE is the sole family authority here, deliberately — not the
|
||||||
|
#: row's own ``operator_family`` field. An exam question carries a relation
|
||||||
|
#: word and nothing else, so a family derived from anything the question does
|
||||||
|
#: not carry would let premise compilation and question routing disagree, and
|
||||||
|
#: a taught edge could then be missing from the premises compiled to decide
|
||||||
|
#: it — a wrong answer built from a correct curriculum. Two corpus rows
|
||||||
|
#: currently declare a family their connective does not imply
|
||||||
|
#: (``physics-causal-008`` and one systems_software row, both
|
||||||
|
#: ``requires``/``causal``); under this rule they are read as modal, which is
|
||||||
|
#: what their connective says.
|
||||||
|
CONNECTIVE_FAMILY: dict[str, str] = {
|
||||||
|
"causes": "causal",
|
||||||
|
"reveals": "causal",
|
||||||
|
"grounds": "causal",
|
||||||
|
"requires": "modal",
|
||||||
|
"enables": "modal",
|
||||||
|
"precedes": "sequence",
|
||||||
|
"opposes": "contrast",
|
||||||
|
"supports": "evidential",
|
||||||
|
}
|
||||||
|
|
||||||
|
#: Relation families, in a stable order (the band-key axis — ADR-0262 §4).
|
||||||
|
FAMILIES: tuple[str, ...] = ("causal", "modal", "sequence", "contrast", "evidential")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class CurriculumChain:
|
||||||
|
"""One ratified curriculum edge, as the exam path reads it."""
|
||||||
|
|
||||||
|
chain_id: str
|
||||||
|
subject: str
|
||||||
|
connective: str
|
||||||
|
obj: str
|
||||||
|
family: str
|
||||||
|
|
||||||
|
@property
|
||||||
|
def sentence(self) -> str:
|
||||||
|
"""The English premise this chain compiles to. The connective is
|
||||||
|
already a third-person-singular verb form, so the sentence lands in
|
||||||
|
the verb-predicate grammar (ADR-0260) exactly as written."""
|
||||||
|
return f"{self.subject} {self.connective} {self.obj}"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class Curriculum:
|
||||||
|
"""The ratified curriculum of one subject — the ONLY premise source."""
|
||||||
|
|
||||||
|
domain: str
|
||||||
|
chains: tuple[CurriculumChain, ...]
|
||||||
|
vocabulary: frozenset[str]
|
||||||
|
|
||||||
|
def family(self, family: str) -> tuple[CurriculumChain, ...]:
|
||||||
|
return tuple(c for c in self.chains if c.family == family)
|
||||||
|
|
||||||
|
def chain_by_id(self, chain_id: str) -> CurriculumChain | None:
|
||||||
|
return next((c for c in self.chains if c.chain_id == chain_id), None)
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=None)
|
||||||
|
def _pack_lemmas(pack_id: str) -> frozenset[str]:
|
||||||
|
path = _REPO_ROOT / "packs" / "data" / pack_id / "lexicon.jsonl"
|
||||||
|
if not path.exists():
|
||||||
|
return frozenset()
|
||||||
|
lemmas: set[str] = set()
|
||||||
|
for line in path.read_text(encoding="utf-8").splitlines():
|
||||||
|
if not line.strip():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
row = json.loads(line)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
continue
|
||||||
|
lemma = row.get("lemma") if isinstance(row, dict) else None
|
||||||
|
if isinstance(lemma, str):
|
||||||
|
lemmas.add(lemma)
|
||||||
|
return frozenset(lemmas)
|
||||||
|
|
||||||
|
|
||||||
|
def _domain_vocabulary(domain: str) -> frozenset[str]:
|
||||||
|
"""Every lemma the subject's mounted packs teach — the vocabulary boundary
|
||||||
|
an exam question may not cross (§4.6 ``untaught_vocabulary``)."""
|
||||||
|
lemmas: set[str] = set()
|
||||||
|
for pack_id in DOMAIN_PACKS.get(domain, ()):
|
||||||
|
lemmas |= _pack_lemmas(pack_id)
|
||||||
|
return frozenset(lemmas)
|
||||||
|
|
||||||
|
|
||||||
|
def _ratified_rows(corpus_id: str, domain: str) -> list[dict]:
|
||||||
|
rel = DOMAIN_CAPABILITY_CORPORA.get(corpus_id)
|
||||||
|
if rel is None:
|
||||||
|
return []
|
||||||
|
path = _REPO_ROOT / rel
|
||||||
|
if not path.exists():
|
||||||
|
return []
|
||||||
|
rows: list[dict] = []
|
||||||
|
for line in path.read_text(encoding="utf-8").splitlines():
|
||||||
|
if not line.strip():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
row = json.loads(line)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
continue
|
||||||
|
if not isinstance(row, dict):
|
||||||
|
continue
|
||||||
|
if row.get("review_status") != "reviewed":
|
||||||
|
continue
|
||||||
|
if str(row.get("domain") or "").strip() != domain:
|
||||||
|
continue
|
||||||
|
subject = str(row.get("subject") or "").strip()
|
||||||
|
obj = str(row.get("object") or "").strip()
|
||||||
|
subject_pack = str(row.get("subject_pack_id") or "").strip()
|
||||||
|
object_pack = str(row.get("object_pack_id") or "").strip()
|
||||||
|
if not subject or not obj:
|
||||||
|
continue
|
||||||
|
if subject_pack and subject not in _pack_lemmas(subject_pack):
|
||||||
|
continue
|
||||||
|
if object_pack and obj not in _pack_lemmas(object_pack):
|
||||||
|
continue
|
||||||
|
rows.append(row)
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=None)
|
||||||
|
def load_curriculum(domain: str) -> Curriculum:
|
||||||
|
"""The ratified curriculum for *domain* — deterministic, corpus order.
|
||||||
|
|
||||||
|
Unratified rows, rows whose terms have left their packs, and rows whose
|
||||||
|
connective is outside :data:`CONNECTIVE_FAMILY` are DROPPED here, at the
|
||||||
|
admission boundary, so that everything downstream can treat every chain it
|
||||||
|
sees as licensed premise material. A dropped chain is not a silent
|
||||||
|
weakening of an argument the way a dropped *premise* would be (ADR-0261
|
||||||
|
§5.1): the exam path pins chain ids and fails loudly when a pinned chain
|
||||||
|
did not survive admission — see ``resolve_pinned``.
|
||||||
|
"""
|
||||||
|
chains: list[CurriculumChain] = []
|
||||||
|
for corpus_id in DOMAIN_CORPORA.get(domain, ()):
|
||||||
|
for row in _ratified_rows(corpus_id, domain):
|
||||||
|
connective = str(row.get("connective") or "").strip()
|
||||||
|
family = CONNECTIVE_FAMILY.get(connective)
|
||||||
|
if family is None:
|
||||||
|
continue
|
||||||
|
chains.append(
|
||||||
|
CurriculumChain(
|
||||||
|
chain_id=str(row.get("chain_id") or ""),
|
||||||
|
subject=str(row.get("subject") or "").strip(),
|
||||||
|
connective=connective,
|
||||||
|
obj=str(row.get("object") or "").strip(),
|
||||||
|
family=family,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return Curriculum(
|
||||||
|
domain=domain,
|
||||||
|
chains=tuple(chains),
|
||||||
|
vocabulary=_domain_vocabulary(domain),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def compile_premises(
|
||||||
|
curriculum: Curriculum, family: str
|
||||||
|
) -> tuple[tuple[str, ...], tuple[str, ...]]:
|
||||||
|
"""``(premise_sentences, chain_ids)`` for *family*, in corpus order.
|
||||||
|
|
||||||
|
Deterministic: the same curriculum and family always compile to the same
|
||||||
|
sentences in the same order, so a lane report over them is SHA-pinnable.
|
||||||
|
"""
|
||||||
|
chains = curriculum.family(family)
|
||||||
|
return tuple(c.sentence for c in chains), tuple(c.chain_id for c in chains)
|
||||||
|
|
||||||
|
|
||||||
|
class UnratifiedChain(LookupError):
|
||||||
|
"""A lane case pinned a chain that is absent or did not survive admission."""
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_pinned(curriculum: Curriculum, chain_ids: tuple[str, ...]) -> tuple[CurriculumChain, ...]:
|
||||||
|
"""The pinned chains, or raise (§4.3).
|
||||||
|
|
||||||
|
A case that pins a chain the curriculum no longer ratifies MUST fail — not
|
||||||
|
quietly answer from whatever else is present. This is the provenance guard
|
||||||
|
that keeps "gold is a function of the curriculum" true over time: if the
|
||||||
|
curriculum changes under a case, the case breaks loudly.
|
||||||
|
"""
|
||||||
|
resolved: list[CurriculumChain] = []
|
||||||
|
for chain_id in chain_ids:
|
||||||
|
chain = curriculum.chain_by_id(chain_id)
|
||||||
|
if chain is None:
|
||||||
|
raise UnratifiedChain(
|
||||||
|
f"{curriculum.domain}: pinned chain {chain_id!r} is absent or unratified"
|
||||||
|
)
|
||||||
|
resolved.append(chain)
|
||||||
|
return tuple(resolved)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"CONNECTIVE_FAMILY",
|
||||||
|
"FAMILIES",
|
||||||
|
"Curriculum",
|
||||||
|
"CurriculumChain",
|
||||||
|
"UnratifiedChain",
|
||||||
|
"compile_premises",
|
||||||
|
"load_curriculum",
|
||||||
|
"resolve_pinned",
|
||||||
|
]
|
||||||
211
tests/test_curriculum_serve.py
Normal file
211
tests/test_curriculum_serve.py
Normal file
|
|
@ -0,0 +1,211 @@
|
||||||
|
"""Curriculum-grounded serving (ADR-0262) — the Phase-2 contract.
|
||||||
|
|
||||||
|
These tests pin the epistemology, not just the plumbing: that answers come
|
||||||
|
from the ratified curriculum and nowhere else, that an untaught fact is
|
||||||
|
UNKNOWN rather than "no", that a chain the curriculum does not teach is never
|
||||||
|
composed into a claim, and that a question whose vocabulary the subject has
|
||||||
|
never been taught is declined rather than answered from world knowledge.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from chat.curriculum_serve_license import curriculum_serve_license
|
||||||
|
from chat.curriculum_surface import (
|
||||||
|
CurriculumQuery,
|
||||||
|
CurriculumRefusal,
|
||||||
|
band_for,
|
||||||
|
curriculum_grounded_surface,
|
||||||
|
decide_curriculum_question,
|
||||||
|
looks_like_curriculum_question,
|
||||||
|
read_curriculum_question,
|
||||||
|
resolve_domain,
|
||||||
|
resolve_family,
|
||||||
|
)
|
||||||
|
from evals.curriculum_serve.oracle import oracle_answer
|
||||||
|
from teaching.curriculum_premises import (
|
||||||
|
UnratifiedChain,
|
||||||
|
compile_premises,
|
||||||
|
load_curriculum,
|
||||||
|
resolve_pinned,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# --- the curriculum is the only premise source --------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_premises_come_only_from_ratified_chains() -> None:
|
||||||
|
curriculum = load_curriculum("physics")
|
||||||
|
premises, chain_ids = compile_premises(curriculum, "causal")
|
||||||
|
assert premises == (
|
||||||
|
"force causes acceleration",
|
||||||
|
"acceleration causes motion",
|
||||||
|
"work causes energy",
|
||||||
|
"charge causes field",
|
||||||
|
"field causes force",
|
||||||
|
"temperature reveals motion",
|
||||||
|
"entropy reveals energy",
|
||||||
|
)
|
||||||
|
assert all(cid.startswith("physics-") for cid in chain_ids)
|
||||||
|
|
||||||
|
|
||||||
|
def test_pinned_chain_that_is_not_ratified_fails_loudly() -> None:
|
||||||
|
"""§4.3 provenance: a case whose curriculum moved under it must break, not
|
||||||
|
quietly answer from what remains."""
|
||||||
|
curriculum = load_curriculum("physics")
|
||||||
|
with pytest.raises(UnratifiedChain):
|
||||||
|
resolve_pinned(curriculum, ("physics-causal-999",))
|
||||||
|
|
||||||
|
|
||||||
|
def test_family_scoping_never_loses_a_taught_edge() -> None:
|
||||||
|
"""Compilation is family-scoped; every taught edge is reachable from the
|
||||||
|
family its own connective implies, so scoping cannot hide an answer."""
|
||||||
|
curriculum = load_curriculum("physics")
|
||||||
|
for chain in curriculum.chains:
|
||||||
|
premises, _ = compile_premises(curriculum, chain.family)
|
||||||
|
assert chain.sentence in premises
|
||||||
|
|
||||||
|
|
||||||
|
# --- the verdicts that matter -------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_taught_edge_is_entailed() -> None:
|
||||||
|
decision = decide_curriculum_question("Does force cause acceleration?")
|
||||||
|
assert decision.verdict == "entailed"
|
||||||
|
assert decision.band == band_for("physics", "causal")
|
||||||
|
|
||||||
|
|
||||||
|
def test_untaught_composition_is_unknown_not_entailed() -> None:
|
||||||
|
"""The curriculum teaches force→acceleration and acceleration→motion. It
|
||||||
|
does NOT teach that causation composes, so "force causes motion" is
|
||||||
|
unsettled. Composing it would be inventing a rule nobody taught."""
|
||||||
|
assert decide_curriculum_question("Does force cause motion?").verdict == "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def test_untaught_fact_is_unknown_never_refuted() -> None:
|
||||||
|
"""Open-world: silence is not denial. No question answerable from a purely
|
||||||
|
positive curriculum may come back "no"."""
|
||||||
|
for text in (
|
||||||
|
"Does mass require energy?", # true in the world, untaught
|
||||||
|
"Does motion cause force?", # the taught edge runs the other way
|
||||||
|
"Does work cause motion?", # both terms taught, no relation
|
||||||
|
):
|
||||||
|
assert decide_curriculum_question(text).verdict == "unknown", text
|
||||||
|
|
||||||
|
|
||||||
|
def test_relation_is_read_at_face_value() -> None:
|
||||||
|
""""entropy reveals energy" is taught; "entropy causes energy" is not. Same
|
||||||
|
family, different relation — the curriculum said one and not the other."""
|
||||||
|
assert decide_curriculum_question("Does entropy reveal energy?").verdict == "entailed"
|
||||||
|
assert decide_curriculum_question("Does entropy cause energy?").verdict == "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def test_question_may_spell_the_relation_in_its_base_form() -> None:
|
||||||
|
"""The question says "cause", the curriculum says "causes" — normalized by
|
||||||
|
the SAME closed agreement relation the verb band uses (ADR-0260)."""
|
||||||
|
assert decide_curriculum_question("Does charge cause field?").verdict == "entailed"
|
||||||
|
|
||||||
|
|
||||||
|
# --- anti-recall ---------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"text", ["Does gravity cause acceleration?", "Does current cause field?"]
|
||||||
|
)
|
||||||
|
def test_untaught_vocabulary_declines_rather_than_recalling(text: str) -> None:
|
||||||
|
"""Both are true physics and both are outside every mounted pack. The
|
||||||
|
system has no curriculum to decide them from, and says so."""
|
||||||
|
decision = decide_curriculum_question(text)
|
||||||
|
assert decision.verdict == "declined"
|
||||||
|
assert decision.reason == "untaught_vocabulary"
|
||||||
|
surface = curriculum_grounded_surface(text)
|
||||||
|
assert surface is not None
|
||||||
|
assert "haven't been taught" in surface
|
||||||
|
assert "not a claim about the world" in surface
|
||||||
|
|
||||||
|
|
||||||
|
# --- typed refusals ------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"text, reason",
|
||||||
|
[
|
||||||
|
("Does force explain acceleration?", "out_of_curriculum"),
|
||||||
|
("Does the force cause acceleration?", "question_shape_out_of_band"),
|
||||||
|
("Does force accelerate?", "question_shape_out_of_band"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_typed_refusals(text: str, reason: str) -> None:
|
||||||
|
assert decide_curriculum_question(text).reason == reason
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_question_text_is_not_claimed_at_all() -> None:
|
||||||
|
"""The composer returns None for anything that is not a ``Does …?``
|
||||||
|
question, so pre-existing dispatch is byte-identical."""
|
||||||
|
assert curriculum_grounded_surface("Force causes acceleration.") is None
|
||||||
|
assert not looks_like_curriculum_question("Is force a cause of acceleration?")
|
||||||
|
|
||||||
|
|
||||||
|
def test_question_reader_shape() -> None:
|
||||||
|
query = read_curriculum_question("Does force cause acceleration?")
|
||||||
|
assert isinstance(query, CurriculumQuery)
|
||||||
|
assert (query.subject, query.verb, query.obj) == ("force", "cause", "acceleration")
|
||||||
|
assert isinstance(read_curriculum_question("Why does it move?"), CurriculumRefusal)
|
||||||
|
|
||||||
|
|
||||||
|
def test_domain_and_family_routing() -> None:
|
||||||
|
query = read_curriculum_question("Does force cause acceleration?")
|
||||||
|
assert isinstance(query, CurriculumQuery)
|
||||||
|
assert resolve_domain(query) == "physics"
|
||||||
|
assert resolve_family(query) == "causal"
|
||||||
|
|
||||||
|
|
||||||
|
# --- the license posture -------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_answers_are_disclosed_until_a_band_earns_serve() -> None:
|
||||||
|
"""No curriculum band holds a SERVE license yet (ADR-0262 §5: ratified
|
||||||
|
curriculum volume, not machinery, is the binding constraint), so every
|
||||||
|
answer is served DISCLOSED — sound, and honest about its track record."""
|
||||||
|
assert curriculum_serve_license(band_for("physics", "causal")) is None
|
||||||
|
surface = curriculum_grounded_surface("Does force cause acceleration?")
|
||||||
|
assert surface is not None
|
||||||
|
assert surface.startswith("(answered from my curriculum")
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_earned_band_is_served_authoritatively() -> None:
|
||||||
|
"""The gate is wired, not stubbed: hand it a licensed band and the hedge
|
||||||
|
disappears."""
|
||||||
|
|
||||||
|
class _Licensed:
|
||||||
|
licensed = True
|
||||||
|
|
||||||
|
surface = curriculum_grounded_surface(
|
||||||
|
"Does force cause acceleration?", license_lookup=lambda band: _Licensed()
|
||||||
|
)
|
||||||
|
assert surface is not None
|
||||||
|
assert not surface.startswith("(answered from my curriculum")
|
||||||
|
assert "physics curriculum teaches that force causes acceleration" in surface
|
||||||
|
|
||||||
|
|
||||||
|
# --- the independent oracle ----------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_oracle_is_independent_and_agrees() -> None:
|
||||||
|
"""The oracle shares no code with the serving path; agreement across the
|
||||||
|
verdict classes is the evidence the compiler reads the curriculum right."""
|
||||||
|
for subject, relation, obj, expected in (
|
||||||
|
("force", "cause", "acceleration", "entailed"),
|
||||||
|
("force", "cause", "motion", "unknown"),
|
||||||
|
("gravity", "cause", "acceleration", "declined"),
|
||||||
|
):
|
||||||
|
assert oracle_answer("physics", subject, relation, obj).verdict == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_oracle_reports_reachability_without_claiming_it() -> None:
|
||||||
|
"""force→acceleration→motion is reachable at depth 2 and still UNKNOWN —
|
||||||
|
the property that proves composition is not being silently performed."""
|
||||||
|
verdict = oracle_answer("physics", "force", "cause", "motion")
|
||||||
|
assert verdict.verdict == "unknown"
|
||||||
|
assert verdict.depth == 2
|
||||||
113
tests/test_ratified_ledger_bridge.py
Normal file
113
tests/test_ratified_ledger_bridge.py
Normal file
|
|
@ -0,0 +1,113 @@
|
||||||
|
"""The ratified-ledger bridge (ADR-0263) — seal → ratify → SHA-verify → gate.
|
||||||
|
|
||||||
|
These tests pin the four rules the three adapters depend on, and the property
|
||||||
|
that made the extraction safe: re-sealing through the bridge reproduces the
|
||||||
|
committed artifacts byte-for-byte.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from core.ratified_ledger import (
|
||||||
|
RatifiedLedgerError,
|
||||||
|
load_sealed_ledger,
|
||||||
|
seal_artifact,
|
||||||
|
serve_license,
|
||||||
|
write_sealed_ledger,
|
||||||
|
)
|
||||||
|
from core.reliability_gate import ClassTally
|
||||||
|
|
||||||
|
|
||||||
|
def _tally(name: str, correct: int, wrong: int = 0) -> ClassTally:
|
||||||
|
return ClassTally(class_name=name, correct=correct, wrong=wrong, refused=0)
|
||||||
|
|
||||||
|
|
||||||
|
def test_seal_then_load_round_trips(tmp_path) -> None:
|
||||||
|
ledger = {"alpha": _tally("alpha", 700), "beta": _tally("beta", 12)}
|
||||||
|
path = tmp_path / "ledger.json"
|
||||||
|
write_sealed_ledger(
|
||||||
|
path, seal_artifact(ledger, schema="t_v1", note="n", provenance="p")
|
||||||
|
)
|
||||||
|
loaded = load_sealed_ledger(path)
|
||||||
|
assert set(loaded) == {"alpha", "beta"}
|
||||||
|
assert loaded["alpha"].correct == 700
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_hand_edited_ledger_is_rejected(tmp_path) -> None:
|
||||||
|
"""Rule 2 — tamper-evidence is structural. Editing a tally without
|
||||||
|
re-sealing does not produce a slightly-wrong ledger; it produces an
|
||||||
|
unratified one, and loading REFUSES."""
|
||||||
|
path = tmp_path / "ledger.json"
|
||||||
|
write_sealed_ledger(
|
||||||
|
path,
|
||||||
|
seal_artifact({"alpha": _tally("alpha", 10)}, schema="t_v1", note="n", provenance="p"),
|
||||||
|
)
|
||||||
|
artifact = json.loads(path.read_text())
|
||||||
|
artifact["classes"]["alpha"]["correct"] = 9_999
|
||||||
|
path.write_text(json.dumps(artifact, indent=2, sort_keys=True) + "\n")
|
||||||
|
with pytest.raises(RatifiedLedgerError):
|
||||||
|
load_sealed_ledger(path)
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_ledger_refuses_unless_declared_optional(tmp_path) -> None:
|
||||||
|
"""A capability that SHIPS with a ledger is broken without it; one whose
|
||||||
|
practice volume is still being built legitimately has none. Neither is
|
||||||
|
answered by guessing a license."""
|
||||||
|
path = tmp_path / "absent.json"
|
||||||
|
with pytest.raises(RatifiedLedgerError):
|
||||||
|
load_sealed_ledger(path)
|
||||||
|
assert load_sealed_ledger(path, missing_ok=True) == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_absent_class_is_never_licensed() -> None:
|
||||||
|
"""Rule 4 — no evidence is not a license; the caller's ``None`` branch is
|
||||||
|
what serves the disclosed surface."""
|
||||||
|
assert serve_license("nobody", {"alpha": _tally("alpha", 700)}) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_volume_floor_is_enforced_by_the_gate() -> None:
|
||||||
|
"""A perfect but small record does not clear θ_SERVE=0.99 — the Wilson
|
||||||
|
floor is what makes a license *earned* rather than merely clean."""
|
||||||
|
small = serve_license("alpha", {"alpha": _tally("alpha", 12)})
|
||||||
|
large = serve_license("alpha", {"alpha": _tally("alpha", 720)})
|
||||||
|
assert small is not None and not small.licensed
|
||||||
|
assert large is not None and large.licensed
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_single_wrong_costs_the_license() -> None:
|
||||||
|
dirty = serve_license("alpha", {"alpha": _tally("alpha", 720, wrong=1)})
|
||||||
|
assert dirty is not None and not dirty.licensed
|
||||||
|
|
||||||
|
|
||||||
|
def test_committed_deduction_ledger_reseals_byte_identically() -> None:
|
||||||
|
"""The extraction's safety property: the bridge writes what the bespoke
|
||||||
|
sealers wrote, so adopting it moves no committed artifact and no lane pin."""
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from evals.deduction_serve.practice.runner import build_sealed_artifact
|
||||||
|
|
||||||
|
committed = Path("chat/data/deduction_serve_ledger.json")
|
||||||
|
expected = json.dumps(build_sealed_artifact(), indent=2, sort_keys=True) + "\n"
|
||||||
|
assert committed.read_text(encoding="utf-8") == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_adapter_reads_through_the_bridge() -> None:
|
||||||
|
"""All three instances now share one loader — the property that makes a
|
||||||
|
future change to the ratification rule land everywhere at once."""
|
||||||
|
import chat.curriculum_serve_license as curriculum
|
||||||
|
import chat.deduction_serve_license as deduction
|
||||||
|
import generate.determine.estimation_license as estimation
|
||||||
|
|
||||||
|
for module in (deduction, estimation, curriculum):
|
||||||
|
source = module.__file__ or ""
|
||||||
|
assert source
|
||||||
|
text = open(source, encoding="utf-8").read()
|
||||||
|
assert "core.ratified_ledger" in text
|
||||||
|
# The signal that an adapter still verifies for itself is that it
|
||||||
|
# hashes for itself; docstrings may (and do) still explain the rule.
|
||||||
|
assert "formation.hashing" not in text, (
|
||||||
|
f"{module.__name__} still re-implements verification"
|
||||||
|
)
|
||||||
Loading…
Reference in a new issue