""" IngestCompiler — deterministic three-gate validation pipeline. Gate order (sequential; a packet failing any gate does not proceed): 1. ProvenanceGate — SourceSpan integrity: non-empty text, valid SHA-256, byte offsets non-negative and ordered 2. SemanticGate — span completeness: non-empty payload, balanced delimiters (for code/math), minimum length 3. GovernanceGate — ReviewLevel vs DeterminismClass consistency; ReviewDecision override check The compiler: - performs structural deduplication by pressure_id (exact structural identity) - tracks semantic convergence by semantic_key and annotates packets that assert the same claim as prior packets with a warning - applies ReviewDecision overrides without mutating the original packet - produces a ValidationReport alongside the original immutable packets - exports accepted packets as LearningArtifact objects Ingest/gate.py is NOT imported or called here. """ from __future__ import annotations from collections import defaultdict from typing import Sequence from core_ingest.types import ( CandidateGeometricPressure, DeterminismClass, GateDisposition, LearningArtifact, ReviewDecision, ReviewLevel, ValidationReport, ValidationResult, ) # --------------------------------------------------------------------------- # Individual gates # --------------------------------------------------------------------------- class ProvenanceGate: """ Gate 1: SourceSpan integrity. Checks: - At least one SourceSpan (enforced at packet construction; double-checked here) - source_sha256 is 64 hex characters - byte_start >= 0 - byte_end > byte_start """ def check(self, packet: CandidateGeometricPressure) -> str | None: """Return a failure reason string, or None if the packet passes.""" if not packet.provenance: return "provenance is empty" for span in packet.provenance: if span.byte_start < 0: return f"negative byte_start ({span.byte_start}) in SourceSpan" if span.byte_end <= span.byte_start: return ( f"byte_end ({span.byte_end}) <= byte_start ({span.byte_start})" ) if len(span.source_sha256) != 64: return ( f"source_sha256 is {len(span.source_sha256)} chars, expected 64" ) return None class SemanticGate: """ Gate 2: Span completeness. Checks: - payload_json is non-empty (not just '{}') - lemma is non-empty for TEXT and SCRIPTURE modalities - For CODE modality: no unbalanced braces/brackets/parens in lemma - Minimum combined span length of 1 byte """ _PAIRS = {"{": "}", "[": "]", "(": ")", "<": ">"} def check(self, packet: CandidateGeometricPressure) -> str | None: import json from core_ingest.types import Modality if packet.payload_json in ("{}", ""): return "payload_json is empty — no content to ingest" # Require non-empty lemma for text / scripture if packet.modality in (Modality.TEXT, Modality.SCRIPTURE) and not packet.lemma: return ( f"modality {packet.modality.value} requires a non-empty lemma" ) # CODE: balanced delimiter check on lemma if packet.modality == Modality.CODE and packet.lemma: stack: list[str] = [] for ch in packet.lemma: if ch in self._PAIRS: stack.append(self._PAIRS[ch]) elif ch in self._PAIRS.values(): if not stack or stack[-1] != ch: return f"unbalanced delimiter '{ch}' in code lemma" stack.pop() if stack: return f"unclosed delimiter(s) in code lemma: {''.join(stack)}" # Minimum span length total_bytes = sum(s.byte_end - s.byte_start for s in packet.provenance) if total_bytes < 1: return "total span length is 0 bytes" return None class GovernanceGate: """ Gate 3: ReviewLevel and DeterminismClass consistency. Checks: - AUTO_ACCEPT_ELIGIBLE is only claimed by D0/D1 frontends (enforced at packet construction; double-checked here) - AUTO_REJECT packets are always rejected - D2–D4 packets not covered by a ReviewDecision land in review_required """ def check( self, packet: CandidateGeometricPressure, authorized_ids: frozenset[str], ) -> GateDisposition: if packet.review_level == ReviewLevel.AUTO_REJECT: return GateDisposition.REJECTED_GOVERNANCE # Override: an authorized ReviewDecision accepts regardless of level if packet.pressure_id in authorized_ids: return GateDisposition.OVERRIDE_ACCEPTED if packet.review_level == ReviewLevel.AUTO_ACCEPT_ELIGIBLE: # Invariant already enforced at construction — D0/D1 only return GateDisposition.ACCEPTED # OPERATOR_REVIEW_REQUIRED or ARCHITECT_REVIEW_REQUIRED without override return GateDisposition.REVIEW_REQUIRED # --------------------------------------------------------------------------- # Compiler # --------------------------------------------------------------------------- class IngestCompiler: """ Processes batches of CandidateGeometricPressure packets through the three-gate pipeline and produces a ValidationReport. Usage ----- compiler = IngestCompiler() report, artifacts = compiler.compile(packets, review_decision=decision) """ def __init__(self) -> None: self._provenance_gate = ProvenanceGate() self._semantic_gate = SemanticGate() self._governance_gate = GovernanceGate() def compile( self, packets: Sequence[CandidateGeometricPressure], review_decision: ReviewDecision | None = None, ) -> tuple[ValidationReport, list[LearningArtifact]]: """ Validate a batch of candidate packets. Parameters ---------- packets : The batch to validate (any sequence). review_decision : Optional operator/architect authorization. Packets whose pressure_id appears in review_decision.authorized_ids are accepted regardless of their ReviewLevel. Returns ------- (ValidationReport, list[LearningArtifact]) """ authorized_ids: frozenset[str] = ( review_decision.authorized_ids if review_decision is not None else frozenset() ) # Deduplication: track seen pressure_ids (structural) and semantic_keys seen_pressure_ids: set[str] = set() semantic_counts: dict[str, int] = defaultdict(int) results: list[ValidationResult] = [] artifacts: list[LearningArtifact] = [] accepted_ids: set[str] = set() rejected_ids: set[str] = set() review_ids: set[str] = set() for packet in packets: pid = packet.pressure_id sk = packet.semantic_key # Structural deduplication if pid in seen_pressure_ids: results.append(ValidationResult( pressure_id=pid, semantic_key=sk, disposition=GateDisposition.REJECTED_PROVENANCE, gate_failed="provenance", failure_reason="duplicate pressure_id — structural duplicate", )) rejected_ids.add(pid) continue seen_pressure_ids.add(pid) # Convergent-evidence tracking prior_count = semantic_counts[sk] semantic_counts[sk] += 1 convergence_warning: tuple[str, ...] = () if prior_count > 0: convergence_warning = ( f"semantic_convergence:{prior_count}_prior_sources", ) # Gate 1: Provenance prov_failure = self._provenance_gate.check(packet) if prov_failure is not None: results.append(ValidationResult( pressure_id=pid, semantic_key=sk, disposition=GateDisposition.REJECTED_PROVENANCE, gate_failed="provenance", failure_reason=prov_failure, warnings=convergence_warning, )) rejected_ids.add(pid) continue # Gate 2: Semantic sem_failure = self._semantic_gate.check(packet) if sem_failure is not None: results.append(ValidationResult( pressure_id=pid, semantic_key=sk, disposition=GateDisposition.REJECTED_SEMANTIC, gate_failed="semantic", failure_reason=sem_failure, warnings=convergence_warning, )) rejected_ids.add(pid) continue # Gate 3: Governance disposition = self._governance_gate.check(packet, authorized_ids) if disposition == GateDisposition.REVIEW_REQUIRED: results.append(ValidationResult( pressure_id=pid, semantic_key=sk, disposition=disposition, warnings=convergence_warning, )) review_ids.add(pid) continue if disposition == GateDisposition.REJECTED_GOVERNANCE: results.append(ValidationResult( pressure_id=pid, semantic_key=sk, disposition=disposition, gate_failed="governance", failure_reason="AUTO_REJECT review level", warnings=convergence_warning, )) rejected_ids.add(pid) continue # ACCEPTED or OVERRIDE_ACCEPTED result = ValidationResult( pressure_id=pid, semantic_key=sk, disposition=disposition, warnings=convergence_warning, ) results.append(result) accepted_ids.add(pid) artifacts.append(LearningArtifact(packet=packet, result=result)) report = ValidationReport( results=tuple(results), accepted_ids=frozenset(accepted_ids), rejected_ids=frozenset(rejected_ids), review_ids=frozenset(review_ids), ) return report, artifacts