Audit of the one-mutation-path invariant (ADR-0021 §3) found three leaks
where pack authority or session-state writes could substitute for coherence
judgment. All three landed fixes or partial closures in this push.
Leaks closed:
- Leak A: pack vocab defaulted to COHERENT — flipped to SPECULATIVE in
language_packs/{compiler,schema}.py; docstring corrected to align with
ADR-0021 (it was rationalizing the leak).
- Leak B: vault.recall was epistemic-blind — VaultStore.store() now stamps
every entry with EpistemicStatus (default SPECULATIVE); recall(min_status=)
filters to admissible-as-evidence tier. All 4 vault-write sites updated.
- Leak C (write-side): generate/proposition.py:198 stored articulated
propositions unmarked — now stamps SPECULATIVE, breaking the
fabrication-feedback loop in principle. Read-side audit of 5 call sites
is the residual.
New architectural invariants (tests/test_architectural_invariants.py):
- INV-21: one-mutation-path allowlist (caught Leak C on first run)
- INV-22: pack lexicon default is SPECULATIVE (Leak A guard)
- INV-23: vault recall epistemic-aware (Leak B guard)
New eval lanes:
- teaching_injection_resistance — ships GREEN at 1.00/1.00/0 (the
structural anti-injection claim is real and measurable)
- refusal_calibration — honest gap: 0% refusal, 0% fabrication
- contradiction_detection — honest gap: 50% flag via versor-delta heuristic,
100% false-positive; motivates the proper coherence-checker
- articulation_of_status — honest gap: 0% speculative articulation, 60%
false certainty; output-side leak surface
New benchmarks:
- benchmarks/footprint.py — total deployed runtime is 7.06 MiB
(109,358x smaller than Llama 3.1 405B, runs offline, no GPU)
- benchmarks/learning_curve.py — monotonic + replay-deterministic curve
per lane
Documentation:
- docs/truth_seeking_schema.md — foundational architectural commitment,
five rules, mapped to human failure modes, leaks published openly
- evals/CLAIMS.md — five-tier public claims doc; Tier 4.5 publishes
known gaps with named fixes; verification contract at top
- README.md — new pillar between algebraic substrate and language pillar
Includes in-flight formation pipeline scaffolding (formation/, tests/formation/,
docs/formation_pipeline_plan.md) and minor CLI/contracts/gitignore edits
that were already in the working tree at session start.
Verification: 798 passed, 2 skipped, 1 deselected (pre-existing pack-count
test drift unrelated to schema changes).
213 lines
7.1 KiB
Python
213 lines
7.1 KiB
Python
"""Tests for ``formation.index`` and ``formation.promote`` — Stage 8."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from formation.candidate import CandidateState, RelationCandidate, SourceRef
|
|
from formation.course import (
|
|
GateMeasurement,
|
|
MasteryReport,
|
|
SubjectSpec,
|
|
ValidatedTripleSet,
|
|
)
|
|
from formation.index import MasteredCourseEntry, MasteredCoursesIndex
|
|
from formation.mastery import emit_report
|
|
from formation.promote import PromoteRefused, promote
|
|
|
|
|
|
def _src() -> SourceRef:
|
|
return SourceRef(
|
|
source_sha="a" * 64,
|
|
span="span",
|
|
adapter="wikipedia",
|
|
retrieved_at="2026-05-16T00:00:00Z",
|
|
)
|
|
|
|
|
|
def _vts(*triples: tuple[str, str, str]) -> ValidatedTripleSet:
|
|
rels = tuple(
|
|
RelationCandidate(
|
|
head=h, relation=r, tail=t, sources=(_src(),),
|
|
state=CandidateState.VALIDATED,
|
|
)
|
|
for h, r, t in triples
|
|
)
|
|
return ValidatedTripleSet(
|
|
subject_id="subject.x",
|
|
concepts=(),
|
|
relations=rels,
|
|
counters=(),
|
|
ordering_hints=(),
|
|
)
|
|
|
|
|
|
def _spec(*requires: str) -> SubjectSpec:
|
|
return SubjectSpec(
|
|
subject_id="subject.x",
|
|
title="Subject X",
|
|
target_depth="introductory",
|
|
requires_courses=requires,
|
|
)
|
|
|
|
|
|
def _report(course_id: str = "x.v1", ratified: bool = True) -> MasteryReport:
|
|
return emit_report(
|
|
course_id=course_id,
|
|
source_bundle_sha="0" * 64,
|
|
validated_set_sha="1" * 64,
|
|
course_sha256="2" * 64,
|
|
plan_sha256="3" * 64,
|
|
gates=(
|
|
GateMeasurement(
|
|
name="g", passed=ratified, measurement="1.0", threshold="1.0",
|
|
),
|
|
),
|
|
trace_hashes=("t1",),
|
|
failure_reasons=() if ratified else ("nope",),
|
|
issued_at="2026-05-16T00:00:00Z",
|
|
)
|
|
|
|
|
|
class TestIndex:
|
|
def test_empty_index_round_trips(self, tmp_path) -> None:
|
|
idx = MasteredCoursesIndex(tmp_path / "idx.json")
|
|
assert idx.all_courses() == ()
|
|
assert idx.contains_course("x.v1") is False
|
|
|
|
def test_add_and_persist(self, tmp_path) -> None:
|
|
path = tmp_path / "idx.json"
|
|
idx = MasteredCoursesIndex(path)
|
|
entry = MasteredCourseEntry(
|
|
course_id="x.v1",
|
|
report_sha256="r" * 64,
|
|
issued_at="2026-05-16T00:00:00Z",
|
|
course_sha256="c" * 64,
|
|
validated_set_sha="v" * 64,
|
|
)
|
|
idx.add(entry)
|
|
# Reload.
|
|
idx2 = MasteredCoursesIndex(path)
|
|
assert idx2.contains_course("x.v1") is True
|
|
assert idx2.contains_report("r" * 64) is True
|
|
|
|
def test_add_same_entry_idempotent(self, tmp_path) -> None:
|
|
idx = MasteredCoursesIndex(tmp_path / "idx.json")
|
|
entry = MasteredCourseEntry(
|
|
course_id="x.v1", report_sha256="r" * 64,
|
|
issued_at="2026-05-16T00:00:00Z",
|
|
course_sha256="c" * 64, validated_set_sha="v" * 64,
|
|
)
|
|
idx.add(entry)
|
|
idx.add(entry) # no-op
|
|
assert len(idx.all_courses()) == 1
|
|
|
|
def test_conflicting_report_for_same_course_raises(self, tmp_path) -> None:
|
|
idx = MasteredCoursesIndex(tmp_path / "idx.json")
|
|
idx.add(MasteredCourseEntry(
|
|
course_id="x.v1", report_sha256="r" * 64,
|
|
issued_at="2026-05-16T00:00:00Z",
|
|
course_sha256="c" * 64, validated_set_sha="v" * 64,
|
|
))
|
|
with pytest.raises(ValueError, match="different report"):
|
|
idx.add(MasteredCourseEntry(
|
|
course_id="x.v1", report_sha256="r2" * 32,
|
|
issued_at="2026-05-16T00:00:00Z",
|
|
course_sha256="c" * 64, validated_set_sha="v" * 64,
|
|
))
|
|
|
|
|
|
class TestPromoteGates:
|
|
def test_unratified_report_refused(self, tmp_path) -> None:
|
|
idx = MasteredCoursesIndex(tmp_path / "idx.json")
|
|
with pytest.raises(PromoteRefused, match="not ratified"):
|
|
promote(
|
|
report=_report(ratified=False),
|
|
spec=_spec(),
|
|
validated_set=_vts(("wisdom", "is", "judgment")),
|
|
index=idx,
|
|
)
|
|
|
|
def test_tampered_seal_refused(self, tmp_path) -> None:
|
|
rpt = _report()
|
|
tampered = MasteryReport(
|
|
course_id="WRONG", # mutated post-seal
|
|
source_bundle_sha=rpt.source_bundle_sha,
|
|
validated_set_sha=rpt.validated_set_sha,
|
|
course_sha256=rpt.course_sha256,
|
|
plan_sha256=rpt.plan_sha256,
|
|
gates=rpt.gates,
|
|
trace_hashes=rpt.trace_hashes,
|
|
ratified=rpt.ratified,
|
|
report_sha256=rpt.report_sha256,
|
|
issued_at=rpt.issued_at,
|
|
failure_reasons=rpt.failure_reasons,
|
|
)
|
|
idx = MasteredCoursesIndex(tmp_path / "idx.json")
|
|
with pytest.raises(PromoteRefused, match="self-seal"):
|
|
promote(
|
|
report=tampered,
|
|
spec=_spec(),
|
|
validated_set=_vts(("wisdom", "is", "judgment")),
|
|
index=idx,
|
|
)
|
|
|
|
def test_missing_prerequisite_refused(self, tmp_path) -> None:
|
|
idx = MasteredCoursesIndex(tmp_path / "idx.json")
|
|
with pytest.raises(PromoteRefused, match="missing prerequisites"):
|
|
promote(
|
|
report=_report(),
|
|
spec=_spec("not.present.v1"),
|
|
validated_set=_vts(("wisdom", "is", "judgment")),
|
|
index=idx,
|
|
)
|
|
|
|
|
|
class TestPromoteHappyPath:
|
|
def test_promotion_routes_triples_through_review(self, tmp_path) -> None:
|
|
idx = MasteredCoursesIndex(tmp_path / "idx.json")
|
|
rpt = _report()
|
|
result = promote(
|
|
report=rpt,
|
|
spec=_spec(),
|
|
validated_set=_vts(("wisdom", "is", "judgment")),
|
|
index=idx,
|
|
)
|
|
assert result.idempotent_skipped is False
|
|
assert len(result.promoted) == 1
|
|
assert result.promoted[0].triple == ("wisdom", "is", "judgment")
|
|
assert result.promoted[0].example.accepted is True
|
|
# Index now has it.
|
|
assert idx.contains_course("x.v1")
|
|
assert idx.contains_report(rpt.report_sha256)
|
|
|
|
def test_promotion_idempotent_on_second_call(self, tmp_path) -> None:
|
|
idx = MasteredCoursesIndex(tmp_path / "idx.json")
|
|
rpt = _report()
|
|
promote(
|
|
report=rpt, spec=_spec(),
|
|
validated_set=_vts(("wisdom", "is", "judgment")), index=idx,
|
|
)
|
|
result2 = promote(
|
|
report=rpt, spec=_spec(),
|
|
validated_set=_vts(("wisdom", "is", "judgment")), index=idx,
|
|
)
|
|
assert result2.idempotent_skipped is True
|
|
assert result2.promoted == ()
|
|
|
|
|
|
class TestPromoteWithPrerequisites:
|
|
def test_prereq_present_proceeds(self, tmp_path) -> None:
|
|
idx = MasteredCoursesIndex(tmp_path / "idx.json")
|
|
idx.add(MasteredCourseEntry(
|
|
course_id="prereq.v1", report_sha256="p" * 64,
|
|
issued_at="2026-05-16T00:00:00Z",
|
|
course_sha256="c" * 64, validated_set_sha="v" * 64,
|
|
))
|
|
result = promote(
|
|
report=_report(),
|
|
spec=_spec("prereq.v1"),
|
|
validated_set=_vts(("wisdom", "is", "judgment")),
|
|
index=idx,
|
|
)
|
|
assert result.idempotent_skipped is False
|