core/tests/formation/test_compose.py
Shay 64c5bc4619 feat(epistemic): truth-seeking schema audit — 3 leaks closed, 4 new lanes, 3 new invariants
Audit of the one-mutation-path invariant (ADR-0021 §3) found three leaks
where pack authority or session-state writes could substitute for coherence
judgment. All three landed fixes or partial closures in this push.

Leaks closed:
- Leak A: pack vocab defaulted to COHERENT — flipped to SPECULATIVE in
  language_packs/{compiler,schema}.py; docstring corrected to align with
  ADR-0021 (it was rationalizing the leak).
- Leak B: vault.recall was epistemic-blind — VaultStore.store() now stamps
  every entry with EpistemicStatus (default SPECULATIVE); recall(min_status=)
  filters to admissible-as-evidence tier. All 4 vault-write sites updated.
- Leak C (write-side): generate/proposition.py:198 stored articulated
  propositions unmarked — now stamps SPECULATIVE, breaking the
  fabrication-feedback loop in principle. Read-side audit of 5 call sites
  is the residual.

New architectural invariants (tests/test_architectural_invariants.py):
- INV-21: one-mutation-path allowlist (caught Leak C on first run)
- INV-22: pack lexicon default is SPECULATIVE (Leak A guard)
- INV-23: vault recall epistemic-aware (Leak B guard)

New eval lanes:
- teaching_injection_resistance — ships GREEN at 1.00/1.00/0 (the
  structural anti-injection claim is real and measurable)
- refusal_calibration — honest gap: 0% refusal, 0% fabrication
- contradiction_detection — honest gap: 50% flag via versor-delta heuristic,
  100% false-positive; motivates the proper coherence-checker
- articulation_of_status — honest gap: 0% speculative articulation, 60%
  false certainty; output-side leak surface

New benchmarks:
- benchmarks/footprint.py — total deployed runtime is 7.06 MiB
  (109,358x smaller than Llama 3.1 405B, runs offline, no GPU)
- benchmarks/learning_curve.py — monotonic + replay-deterministic curve
  per lane

Documentation:
- docs/truth_seeking_schema.md — foundational architectural commitment,
  five rules, mapped to human failure modes, leaks published openly
- evals/CLAIMS.md — five-tier public claims doc; Tier 4.5 publishes
  known gaps with named fixes; verification contract at top
- README.md — new pillar between algebraic substrate and language pillar

Includes in-flight formation pipeline scaffolding (formation/, tests/formation/,
docs/formation_pipeline_plan.md) and minor CLI/contracts/gitignore edits
that were already in the working tree at session start.

Verification: 798 passed, 2 skipped, 1 deselected (pre-existing pack-count
test drift unrelated to schema changes).
2026-05-17 07:27:41 -07:00

254 lines
8.5 KiB
Python

"""Tests for ``formation.compose`` — deterministic Course YAML composition.
Per Phase 3 of ``docs/formation_pipeline_plan.md``:
* same input -> identical YAML bytes
* reorder input tuples -> identical YAML bytes
* YAML round-trips through ``yaml.safe_load`` if PyYAML is available
* ``course_sha256`` is stable across calls (proxy for "across Python sessions")
* different input -> different SHA
* template version participates in the SHA
"""
from __future__ import annotations
import subprocess
import sys
from pathlib import Path
import pytest
from formation.candidate import (
ConceptCandidate,
CounterCandidate,
OrderingHint,
RelationCandidate,
SourceRef,
)
from formation.compose import compose
from formation.course import SubjectSpec, ValidatedTripleSet
# ---------- fixtures ----------
_SHA_A = "a" * 64
_SHA_B = "b" * 64
_SHA_C = "c" * 64
_BUNDLE_SHA = "f" * 64
def _src(sha: str = _SHA_A, adapter: str = "wikipedia") -> SourceRef:
return SourceRef(
source_sha=sha,
span="...quoted span...",
adapter=adapter,
retrieved_at="2026-05-16T00:00:00Z",
)
def _spec() -> SubjectSpec:
return SubjectSpec(
subject_id="subject.geometry",
title="Geometric Algebra Primer",
target_depth="introductory",
requires_courses=("course.calculus",),
identity_axis_constraints=("preserve_identity",),
)
def _validated_set(
*,
relations: tuple[RelationCandidate, ...] | None = None,
concepts: tuple[ConceptCandidate, ...] | None = None,
) -> ValidatedTripleSet:
default_concepts = (
ConceptCandidate(
canonical_term="vector",
definition="A directed quantity in a linear space.",
sources=(_src(_SHA_A),),
),
ConceptCandidate(
canonical_term="bivector",
definition="An oriented planar element.",
sources=(_src(_SHA_B),),
),
ConceptCandidate(
canonical_term="rotor",
definition="A unit even-grade element generating rotation.",
sources=(_src(_SHA_A), _src(_SHA_B, "stackexchange")),
),
)
default_relations = (
RelationCandidate(
head="vector",
relation="combines_to",
tail="bivector",
sources=(_src(_SHA_A),),
),
RelationCandidate(
head="bivector",
relation="exponentiates_to",
tail="rotor",
sources=(_src(_SHA_B),),
),
)
counters = (
CounterCandidate(
head="vector",
relation="equals",
tail="rotor",
sources=(_src(_SHA_C),),
),
)
hints = (OrderingHint(before="vector", after="bivector", sources=(_src(_SHA_A),)),)
return ValidatedTripleSet(
subject_id="subject.geometry",
concepts=concepts if concepts is not None else default_concepts,
relations=relations if relations is not None else default_relations,
counters=counters,
ordering_hints=hints,
)
# ---------- determinism ----------
class TestDeterminism:
def test_same_input_identical_bytes(self) -> None:
vs = _validated_set()
a = compose(vs, _spec(), _BUNDLE_SHA)
b = compose(vs, _spec(), _BUNDLE_SHA)
assert a.yaml_bytes == b.yaml_bytes
assert a.course_sha256 == b.course_sha256
def test_reordered_relations_same_bytes(self) -> None:
base = _validated_set()
reordered = _validated_set(relations=tuple(reversed(base.relations)))
a = compose(base, _spec(), _BUNDLE_SHA)
b = compose(reordered, _spec(), _BUNDLE_SHA)
assert a.yaml_bytes == b.yaml_bytes
assert a.course_sha256 == b.course_sha256
def test_reordered_concepts_same_bytes(self) -> None:
base = _validated_set()
reordered_concepts = tuple(reversed(base.concepts))
b_set = _validated_set(concepts=reordered_concepts)
a = compose(base, _spec(), _BUNDLE_SHA)
b = compose(b_set, _spec(), _BUNDLE_SHA)
assert a.yaml_bytes == b.yaml_bytes
# ---------- round-trip ----------
class TestRoundTrip:
def test_yaml_safe_load_roundtrip(self) -> None:
yaml = pytest.importorskip("yaml")
out = compose(_validated_set(), _spec(), _BUNDLE_SHA)
loaded = yaml.safe_load(out.yaml_bytes.decode("utf-8"))
# Re-emit through compose by handing the parsed structure back is not
# meaningful; instead, assert structure invariants survive parsing.
assert loaded["template_id"] == "definition"
assert loaded["template_version"] == "1.0.0"
assert loaded["source_bundle_sha"] == _BUNDLE_SHA
assert (
loaded["substrate_invariants"]["max_versor_condition"] == "1.0e-6"
)
# Concepts present and ordered.
terms = [c["canonical_term"] for c in loaded["phase_1_ontological_seeding"]["concepts"]]
assert terms == sorted(terms)
# Relations topo-sorted: vector -> bivector before bivector -> rotor.
rels = loaded["phase_2_axiomatic_rotor_scaffolding"]["relations"]
assert rels[0]["head"] == "vector"
assert rels[1]["head"] == "bivector"
# Walks present.
walks = loaded["phase_3_holonomic_syllabus_walk"]["walks"]
assert len(walks) >= 1
# Adversarial includes identity-override probes.
probes = loaded["phase_4_epistemic_boundary_hardening"]["adversarial_corrections"]
probe_ids = {p.get("probe_id") for p in probes}
assert "identity_override_axis_rewrite" in probe_ids
# ---------- SHA sensitivity ----------
class TestShaSensitivity:
def test_different_input_different_sha(self) -> None:
a = compose(_validated_set(), _spec(), _BUNDLE_SHA)
# Different validated set: add an extra relation.
extra = RelationCandidate(
head="rotor",
relation="acts_on",
tail="vector",
sources=(_src(_SHA_A),),
)
vs2 = _validated_set(relations=_validated_set().relations + (extra,))
b = compose(vs2, _spec(), _BUNDLE_SHA)
assert a.course_sha256 != b.course_sha256
def test_different_source_bundle_different_sha(self) -> None:
a = compose(_validated_set(), _spec(), _BUNDLE_SHA)
b = compose(_validated_set(), _spec(), "0" * 64)
assert a.course_sha256 != b.course_sha256
def test_template_version_mismatch_raises(self) -> None:
with pytest.raises(ValueError, match="template_version"):
compose(
_validated_set(),
_spec(),
_BUNDLE_SHA,
template_version="9.9.9",
)
def test_unknown_template_raises(self) -> None:
with pytest.raises(KeyError):
compose(
_validated_set(),
_spec(),
_BUNDLE_SHA,
template_id="does_not_exist",
)
# ---------- cross-session SHA stability ----------
class TestCrossSession:
def test_sha_stable_across_subprocess(self) -> None:
"""Compute the course SHA in a fresh Python process; must match.
This is the strongest proxy for "stable across Python sessions" we can
achieve without persisting fixtures. Hash randomization is reset per
process; if anything in compose depends on it, this fails.
"""
repo_root = Path(__file__).resolve().parents[2]
script = (
"import sys; sys.path.insert(0, %r);"
"from tests.formation.test_compose import _validated_set, _spec, _BUNDLE_SHA;"
"from formation.compose import compose;"
"print(compose(_validated_set(), _spec(), _BUNDLE_SHA).course_sha256)"
) % str(repo_root)
in_proc = compose(_validated_set(), _spec(), _BUNDLE_SHA).course_sha256
result = subprocess.run(
[sys.executable, "-c", script],
check=True,
capture_output=True,
text=True,
cwd=str(repo_root),
env={"PYTHONHASHSEED": "random", "PATH": ""},
)
assert result.stdout.strip() == in_proc
# ---------- floats forbidden ----------
class TestFloatsForbidden:
def test_payload_has_no_floats(self) -> None:
# Compose should succeed even when we exercise every path; the
# underlying _reject_floats guarantee is asserted by passing.
out = compose(_validated_set(), _spec(), _BUNDLE_SHA)
assert out.yaml_bytes # non-empty
# The literal threshold appears as a string in the YAML.
assert b'max_versor_condition: "1.0e-6"' in out.yaml_bytes