303 lines
11 KiB
Python
303 lines
11 KiB
Python
"""
|
||
IngestCompiler — deterministic three-gate validation pipeline.
|
||
|
||
Gate order (sequential; a packet failing any gate does not proceed):
|
||
1. ProvenanceGate — SourceSpan integrity: non-empty text, valid SHA-256,
|
||
byte offsets non-negative and ordered
|
||
2. SemanticGate — span completeness: non-empty payload, balanced
|
||
delimiters (for code/math), minimum length
|
||
3. GovernanceGate — ReviewLevel vs DeterminismClass consistency;
|
||
ReviewDecision override check
|
||
|
||
The compiler:
|
||
- performs structural deduplication by pressure_id (exact structural identity)
|
||
- tracks semantic convergence by semantic_key and annotates packets that
|
||
assert the same claim as prior packets with a warning
|
||
- applies ReviewDecision overrides without mutating the original packet
|
||
- produces a ValidationReport alongside the original immutable packets
|
||
- exports accepted packets as LearningArtifact objects
|
||
|
||
Ingest/gate.py is NOT imported or called here.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from collections import defaultdict
|
||
from typing import Sequence
|
||
|
||
from core_ingest.types import (
|
||
CandidateGeometricPressure,
|
||
DeterminismClass,
|
||
GateDisposition,
|
||
LearningArtifact,
|
||
ReviewDecision,
|
||
ReviewLevel,
|
||
ValidationReport,
|
||
ValidationResult,
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Individual gates
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class ProvenanceGate:
|
||
"""
|
||
Gate 1: SourceSpan integrity.
|
||
|
||
Checks:
|
||
- At least one SourceSpan (enforced at packet construction; double-checked here)
|
||
- source_sha256 is 64 hex characters
|
||
- byte_start >= 0
|
||
- byte_end > byte_start
|
||
"""
|
||
|
||
def check(self, packet: CandidateGeometricPressure) -> str | None:
|
||
"""Return a failure reason string, or None if the packet passes."""
|
||
if not packet.provenance:
|
||
return "provenance is empty"
|
||
for span in packet.provenance:
|
||
if span.byte_start < 0:
|
||
return f"negative byte_start ({span.byte_start}) in SourceSpan"
|
||
if span.byte_end <= span.byte_start:
|
||
return (
|
||
f"byte_end ({span.byte_end}) <= byte_start ({span.byte_start})"
|
||
)
|
||
if len(span.source_sha256) != 64:
|
||
return (
|
||
f"source_sha256 is {len(span.source_sha256)} chars, expected 64"
|
||
)
|
||
return None
|
||
|
||
|
||
class SemanticGate:
|
||
"""
|
||
Gate 2: Span completeness.
|
||
|
||
Checks:
|
||
- payload_json is non-empty (not just '{}')
|
||
- lemma is non-empty for TEXT and SCRIPTURE modalities
|
||
- For CODE modality: no unbalanced braces/brackets/parens in lemma
|
||
- Minimum combined span length of 1 byte
|
||
"""
|
||
_PAIRS = {"{": "}", "[": "]", "(": ")", "<": ">"}
|
||
|
||
def check(self, packet: CandidateGeometricPressure) -> str | None:
|
||
import json
|
||
from core_ingest.types import Modality
|
||
|
||
if packet.payload_json in ("{}", ""):
|
||
return "payload_json is empty — no content to ingest"
|
||
|
||
# Require non-empty lemma for text / scripture
|
||
if packet.modality in (Modality.TEXT, Modality.SCRIPTURE) and not packet.lemma:
|
||
return (
|
||
f"modality {packet.modality.value} requires a non-empty lemma"
|
||
)
|
||
|
||
# CODE: balanced delimiter check on lemma
|
||
if packet.modality == Modality.CODE and packet.lemma:
|
||
stack: list[str] = []
|
||
for ch in packet.lemma:
|
||
if ch in self._PAIRS:
|
||
stack.append(self._PAIRS[ch])
|
||
elif ch in self._PAIRS.values():
|
||
if not stack or stack[-1] != ch:
|
||
return f"unbalanced delimiter '{ch}' in code lemma"
|
||
stack.pop()
|
||
if stack:
|
||
return f"unclosed delimiter(s) in code lemma: {''.join(stack)}"
|
||
|
||
# Minimum span length
|
||
total_bytes = sum(s.byte_end - s.byte_start for s in packet.provenance)
|
||
if total_bytes < 1:
|
||
return "total span length is 0 bytes"
|
||
|
||
return None
|
||
|
||
|
||
class GovernanceGate:
|
||
"""
|
||
Gate 3: ReviewLevel and DeterminismClass consistency.
|
||
|
||
Checks:
|
||
- AUTO_ACCEPT_ELIGIBLE is only claimed by D0/D1 frontends
|
||
(enforced at packet construction; double-checked here)
|
||
- AUTO_REJECT packets are always rejected
|
||
- D2–D4 packets not covered by a ReviewDecision land in review_required
|
||
"""
|
||
|
||
def check(
|
||
self,
|
||
packet: CandidateGeometricPressure,
|
||
authorized_ids: frozenset[str],
|
||
) -> GateDisposition:
|
||
if packet.review_level == ReviewLevel.AUTO_REJECT:
|
||
return GateDisposition.REJECTED_GOVERNANCE
|
||
|
||
# Override: an authorized ReviewDecision accepts regardless of level
|
||
if packet.pressure_id in authorized_ids:
|
||
return GateDisposition.OVERRIDE_ACCEPTED
|
||
|
||
if packet.review_level == ReviewLevel.AUTO_ACCEPT_ELIGIBLE:
|
||
# Invariant already enforced at construction — D0/D1 only
|
||
return GateDisposition.ACCEPTED
|
||
|
||
# OPERATOR_REVIEW_REQUIRED or ARCHITECT_REVIEW_REQUIRED without override
|
||
return GateDisposition.REVIEW_REQUIRED
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Compiler
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class IngestCompiler:
|
||
"""
|
||
Processes batches of CandidateGeometricPressure packets through the
|
||
three-gate pipeline and produces a ValidationReport.
|
||
|
||
Usage
|
||
-----
|
||
compiler = IngestCompiler()
|
||
report, artifacts = compiler.compile(packets, review_decision=decision)
|
||
"""
|
||
|
||
def __init__(self) -> None:
|
||
self._provenance_gate = ProvenanceGate()
|
||
self._semantic_gate = SemanticGate()
|
||
self._governance_gate = GovernanceGate()
|
||
|
||
def compile(
|
||
self,
|
||
packets: Sequence[CandidateGeometricPressure],
|
||
review_decision: ReviewDecision | None = None,
|
||
) -> tuple[ValidationReport, list[LearningArtifact]]:
|
||
"""
|
||
Validate a batch of candidate packets.
|
||
|
||
Parameters
|
||
----------
|
||
packets : The batch to validate (any sequence).
|
||
review_decision : Optional operator/architect authorization. Packets
|
||
whose pressure_id appears in
|
||
review_decision.authorized_ids are accepted
|
||
regardless of their ReviewLevel.
|
||
|
||
Returns
|
||
-------
|
||
(ValidationReport, list[LearningArtifact])
|
||
"""
|
||
authorized_ids: frozenset[str] = (
|
||
review_decision.authorized_ids
|
||
if review_decision is not None
|
||
else frozenset()
|
||
)
|
||
|
||
# Deduplication: track seen pressure_ids (structural) and semantic_keys
|
||
seen_pressure_ids: set[str] = set()
|
||
semantic_counts: dict[str, int] = defaultdict(int)
|
||
|
||
results: list[ValidationResult] = []
|
||
artifacts: list[LearningArtifact] = []
|
||
|
||
accepted_ids: set[str] = set()
|
||
rejected_ids: set[str] = set()
|
||
review_ids: set[str] = set()
|
||
|
||
for packet in packets:
|
||
pid = packet.pressure_id
|
||
sk = packet.semantic_key
|
||
|
||
# Structural deduplication
|
||
if pid in seen_pressure_ids:
|
||
results.append(ValidationResult(
|
||
pressure_id=pid,
|
||
semantic_key=sk,
|
||
disposition=GateDisposition.REJECTED_PROVENANCE,
|
||
gate_failed="provenance",
|
||
failure_reason="duplicate pressure_id — structural duplicate",
|
||
))
|
||
rejected_ids.add(pid)
|
||
continue
|
||
seen_pressure_ids.add(pid)
|
||
|
||
# Convergent-evidence tracking
|
||
prior_count = semantic_counts[sk]
|
||
semantic_counts[sk] += 1
|
||
convergence_warning: tuple[str, ...] = ()
|
||
if prior_count > 0:
|
||
convergence_warning = (
|
||
f"semantic_convergence:{prior_count}_prior_sources",
|
||
)
|
||
|
||
# Gate 1: Provenance
|
||
prov_failure = self._provenance_gate.check(packet)
|
||
if prov_failure is not None:
|
||
results.append(ValidationResult(
|
||
pressure_id=pid,
|
||
semantic_key=sk,
|
||
disposition=GateDisposition.REJECTED_PROVENANCE,
|
||
gate_failed="provenance",
|
||
failure_reason=prov_failure,
|
||
warnings=convergence_warning,
|
||
))
|
||
rejected_ids.add(pid)
|
||
continue
|
||
|
||
# Gate 2: Semantic
|
||
sem_failure = self._semantic_gate.check(packet)
|
||
if sem_failure is not None:
|
||
results.append(ValidationResult(
|
||
pressure_id=pid,
|
||
semantic_key=sk,
|
||
disposition=GateDisposition.REJECTED_SEMANTIC,
|
||
gate_failed="semantic",
|
||
failure_reason=sem_failure,
|
||
warnings=convergence_warning,
|
||
))
|
||
rejected_ids.add(pid)
|
||
continue
|
||
|
||
# Gate 3: Governance
|
||
disposition = self._governance_gate.check(packet, authorized_ids)
|
||
|
||
if disposition == GateDisposition.REVIEW_REQUIRED:
|
||
results.append(ValidationResult(
|
||
pressure_id=pid,
|
||
semantic_key=sk,
|
||
disposition=disposition,
|
||
warnings=convergence_warning,
|
||
))
|
||
review_ids.add(pid)
|
||
continue
|
||
|
||
if disposition == GateDisposition.REJECTED_GOVERNANCE:
|
||
results.append(ValidationResult(
|
||
pressure_id=pid,
|
||
semantic_key=sk,
|
||
disposition=disposition,
|
||
gate_failed="governance",
|
||
failure_reason="AUTO_REJECT review level",
|
||
warnings=convergence_warning,
|
||
))
|
||
rejected_ids.add(pid)
|
||
continue
|
||
|
||
# ACCEPTED or OVERRIDE_ACCEPTED
|
||
result = ValidationResult(
|
||
pressure_id=pid,
|
||
semantic_key=sk,
|
||
disposition=disposition,
|
||
warnings=convergence_warning,
|
||
)
|
||
results.append(result)
|
||
accepted_ids.add(pid)
|
||
artifacts.append(LearningArtifact(packet=packet, result=result))
|
||
|
||
report = ValidationReport(
|
||
results=tuple(results),
|
||
accepted_ids=frozenset(accepted_ids),
|
||
rejected_ids=frozenset(rejected_ids),
|
||
review_ids=frozenset(review_ids),
|
||
)
|
||
return report, artifacts
|