""" Lab Eval: Full Teaching Pipeline Trace — All Three Identity Packs For each identity pack configuration, runs a complete teaching session and traces every layer with structured output. Nothing is written to packs or manifolds. The vault is in-process and session-scoped. Outputs structured JSON to stdout. Exit 0 always (this is an exploration trace, not a pass/fail assertion). To run: python -m evals.lab.teaching_trace python -m evals.lab.teaching_trace | python -m json.tool """ from __future__ import annotations import hashlib import json import sys _IDENTITY_PACKS = [ "default_general_v1", "precision_first_v1", "generosity_first_v1", ] _TEACHING_INPUTS = [ ("light is truth", "light is the ground of all knowledge"), ("word carries meaning", "meaning depends on relational context, not isolated form"), ("truth is fixed", "truth is not fixed — it is a coherence judgment relative to a field"), ] def _versor_digest(v) -> str: import numpy as np return hashlib.sha256(np.asarray(v, dtype=np.float32).tobytes()).hexdigest()[:16] def _trace_pack(pack_id: str) -> dict: from chat.runtime import ChatRuntime from core.config import RuntimeConfig from teaching.correction import CorrectionCandidate from teaching.review import review_correction from teaching.store import TeachingStore from teaching.epistemic import EpistemicStatus import uuid config = RuntimeConfig(identity_pack=pack_id) rt = ChatRuntime(config=config) store = TeachingStore(capacity=64) pack_trace = { "pack_id": pack_id, "alignment_threshold": float(rt.identity_manifold.alignment_threshold), "value_axes": [ { "axis_id": ax.axis_id, "name": ax.name, "weight": float(ax.weight), } for ax in rt.identity_manifold.value_axes ], "turns": [], } for input_text, correction_text in _TEACHING_INPUTS: resp = rt.chat(input_text) # --- Layer trace --- identity_score = resp.identity_score safety_v = resp.safety_verdict ethics_v = resp.ethics_verdict turn_trace = { "input": input_text, "input_versor_digest": _versor_digest(rt.session.state.F), "gate_decision": { "vault_size": len(rt.session.vault), }, "proposition": { "subject": resp.proposition.subject, "predicate": resp.proposition.predicate, "frame_id": resp.proposition.frame_id, "relation_norm": float(resp.proposition.relation_norm), }, "identity": { "alignment": float(identity_score.alignment) if identity_score else None, "flagged": identity_score.flagged if identity_score else None, "deviation_axes": list(identity_score.deviation_axes) if identity_score else [], }, "safety": { "upheld": safety_v.upheld if safety_v else None, "violated": [ p.predicate_id for p in (safety_v.predicate_results if safety_v else []) if not p.result ], }, "ethics": { "upheld": ethics_v.upheld if ethics_v else None, "violated": [ c.commitment_id for c in (ethics_v.commitment_results if ethics_v else []) if not c.result ], }, "surface": resp.surface, "versor_condition": float(resp.versor_condition), "dialogue_role": resp.dialogue_role, "walk_surface": resp.walk_surface, } # --- Teaching path --- candidate = CorrectionCandidate( candidate_id=str(uuid.uuid4()), correction_text=correction_text, prior_surface=resp.surface, prior_turn=rt.session.turn - 1, intent=__import__("generate.intent", fromlist=["classify_intent"]).classify_intent(correction_text), ) reviewed = review_correction( candidate, identity_score=identity_score, identity_manifold=rt.identity_manifold, epistemic_status=EpistemicStatus.SPECULATIVE, ) proposal = store.add(reviewed) turn_trace["teaching"] = { "correction_text": correction_text, "review_outcome": reviewed.outcome.value, "review_hash": reviewed.review_hash[:16], "proposal_id": proposal.proposal_id if proposal else None, "triple": list(proposal.triple) if proposal and proposal.triple else None, "epistemic_status_after_store": proposal.epistemic_status.value if proposal else None, } pack_trace["turns"].append(turn_trace) pack_trace["store_summary"] = { "total_examples": len(store), "pending_proposals": len(store.pending_proposals()), "triples": [list(t) for t in store.triples()], } return pack_trace def _cross_pack_diff(traces: list[dict]) -> dict: """Compare alignment scores, flags, hedge rates across packs on same inputs.""" if len(traces) < 2: return {} diffs = [] n_turns = min(len(t["turns"]) for t in traces) for i in range(n_turns): input_text = traces[0]["turns"][i]["input"] row = {"input": input_text} for trace in traces: turn = trace["turns"][i] row[trace["pack_id"]] = { "alignment": turn["identity"]["alignment"], "flagged": turn["identity"]["flagged"], "surface": turn["surface"], "dialogue_role": turn["dialogue_role"], "review_outcome": turn["teaching"]["review_outcome"], } diffs.append(row) return {"cross_pack_diff": diffs} def run() -> dict: traces = [_trace_pack(pack_id) for pack_id in _IDENTITY_PACKS] diff = _cross_pack_diff(traces) return { "eval": "teaching_trace", "packs_traced": _IDENTITY_PACKS, "per_pack": traces, **diff, } if __name__ == "__main__": result = run() print(json.dumps(result, indent=2)) sys.exit(0)