feat(contemplation): ADR-0080 read-only speculative loop (#55)
* docs(adr): ADR-0080 contemplation loop boundary * feat(contemplation): add read-only contemplation package * feat(contemplation): add immutable speculative finding schema * feat(contemplation): add deterministic substrate snapshot * feat(contemplation): add frontier report miner package * feat(contemplation): mine frontier compare failures as speculative findings * feat(contemplation): add read-only contemplation runner * feat(contemplation): add read-only contemplation CLI * test(contemplation): prove read-only speculative loop invariants
This commit is contained in:
parent
6761fc0974
commit
06bbac86e1
9 changed files with 842 additions and 0 deletions
23
core/contemplation/__init__.py
Normal file
23
core/contemplation/__init__.py
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
"""Read-only contemplation loop primitives.
|
||||||
|
|
||||||
|
ADR-0080: contemplation can emit speculative findings about current
|
||||||
|
substrate/report evidence, but it cannot ratify, promote, or mutate packs.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .runner import contemplate_frontier_reports
|
||||||
|
from .schema import (
|
||||||
|
ContemplationEvidenceRef,
|
||||||
|
ContemplationFinding,
|
||||||
|
ContemplationRun,
|
||||||
|
FindingKind,
|
||||||
|
)
|
||||||
|
from .snapshot import ContemplationSubstrate
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"ContemplationEvidenceRef",
|
||||||
|
"ContemplationFinding",
|
||||||
|
"ContemplationRun",
|
||||||
|
"ContemplationSubstrate",
|
||||||
|
"FindingKind",
|
||||||
|
"contemplate_frontier_reports",
|
||||||
|
]
|
||||||
60
core/contemplation/__main__.py
Normal file
60
core/contemplation/__main__.py
Normal file
|
|
@ -0,0 +1,60 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from core.contemplation.runner import (
|
||||||
|
contemplate_frontier_reports,
|
||||||
|
write_contemplation_run,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
prog="python -m core.contemplation",
|
||||||
|
description="Run ADR-0080 read-only contemplation over explicit evidence files.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"reports",
|
||||||
|
nargs="+",
|
||||||
|
type=Path,
|
||||||
|
help="frontier_compare JSON report path(s) to contemplate.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--pack-id",
|
||||||
|
action="append",
|
||||||
|
default=(),
|
||||||
|
help="Optional pack id to include in the substrate snapshot. May repeat.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--note",
|
||||||
|
action="append",
|
||||||
|
default=(),
|
||||||
|
help="Optional operator note included in the substrate snapshot. May repeat.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--report",
|
||||||
|
type=Path,
|
||||||
|
default=None,
|
||||||
|
help="Optional output path for the contemplation run JSON.",
|
||||||
|
)
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
args = build_parser().parse_args(argv)
|
||||||
|
run = contemplate_frontier_reports(
|
||||||
|
args.reports,
|
||||||
|
pack_ids=tuple(args.pack_id or ()),
|
||||||
|
notes=tuple(args.note or ()),
|
||||||
|
)
|
||||||
|
if args.report is not None:
|
||||||
|
write_contemplation_run(run, args.report)
|
||||||
|
print(json.dumps(run.as_dict(), ensure_ascii=False, indent=2, sort_keys=True))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__": # pragma: no cover
|
||||||
|
raise SystemExit(main(sys.argv[1:]))
|
||||||
5
core/contemplation/miners/__init__.py
Normal file
5
core/contemplation/miners/__init__.py
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
"""Read-only contemplation miners."""
|
||||||
|
|
||||||
|
from .frontier_compare import mine_frontier_compare_report
|
||||||
|
|
||||||
|
__all__ = ["mine_frontier_compare_report"]
|
||||||
75
core/contemplation/miners/frontier_compare.py
Normal file
75
core/contemplation/miners/frontier_compare.py
Normal file
|
|
@ -0,0 +1,75 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from core.contemplation.schema import (
|
||||||
|
ContemplationEvidenceRef,
|
||||||
|
ContemplationFinding,
|
||||||
|
FindingKind,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _suite_iter(report: dict[str, Any]) -> tuple[dict[str, Any], ...]:
|
||||||
|
if "suites" in report:
|
||||||
|
return tuple(s for s in report.get("suites", ()) if isinstance(s, dict))
|
||||||
|
if "suite" in report:
|
||||||
|
return (report,)
|
||||||
|
return ()
|
||||||
|
|
||||||
|
|
||||||
|
def _case_iter(suite: dict[str, Any]) -> tuple[dict[str, Any], ...]:
|
||||||
|
return tuple(c for c in suite.get("cases", ()) if isinstance(c, dict))
|
||||||
|
|
||||||
|
|
||||||
|
def mine_frontier_compare_report(
|
||||||
|
report_path: str | Path,
|
||||||
|
*,
|
||||||
|
substrate_hash: str,
|
||||||
|
) -> tuple[ContemplationFinding, ...]:
|
||||||
|
"""Convert failed frontier-compare cases into speculative findings.
|
||||||
|
|
||||||
|
Read-only: this function only parses *report_path* and returns immutable
|
||||||
|
findings. It does not write reports, packs, teaching examples, or runtime
|
||||||
|
state.
|
||||||
|
"""
|
||||||
|
|
||||||
|
path = Path(report_path)
|
||||||
|
report = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
findings: list[ContemplationFinding] = []
|
||||||
|
source_id = str(path)
|
||||||
|
|
||||||
|
for suite in _suite_iter(report):
|
||||||
|
suite_name = str(suite.get("suite", "unknown_suite"))
|
||||||
|
for case in _case_iter(suite):
|
||||||
|
if bool(case.get("passed", False)):
|
||||||
|
continue
|
||||||
|
case_id = str(case.get("case_id", "unknown_case"))
|
||||||
|
prompt = str(case.get("prompt", ""))
|
||||||
|
failures = tuple(str(f) for f in case.get("failures", ()) or ())
|
||||||
|
evidence = ContemplationEvidenceRef(
|
||||||
|
source_type="frontier_compare_report",
|
||||||
|
source_id=source_id,
|
||||||
|
pointer=f"suite={suite_name};case={case_id}",
|
||||||
|
summary=", ".join(failures) if failures else "case failed",
|
||||||
|
)
|
||||||
|
findings.append(
|
||||||
|
ContemplationFinding(
|
||||||
|
kind=FindingKind.BENCHMARK_CASE,
|
||||||
|
subject=f"{suite_name}/{case_id}",
|
||||||
|
predicate="failed_case",
|
||||||
|
object=prompt,
|
||||||
|
evidence_refs=(evidence,),
|
||||||
|
proposed_action=(
|
||||||
|
"Review this failed benchmark case; either repair the runtime/pack "
|
||||||
|
"behavior or promote the case into a focused regression/curriculum item."
|
||||||
|
),
|
||||||
|
substrate_hash=substrate_hash,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return tuple(findings)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["mine_frontier_compare_report"]
|
||||||
74
core/contemplation/runner.py
Normal file
74
core/contemplation/runner.py
Normal file
|
|
@ -0,0 +1,74 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Iterable
|
||||||
|
|
||||||
|
from core.contemplation.miners.frontier_compare import mine_frontier_compare_report
|
||||||
|
from core.contemplation.schema import ContemplationFinding, ContemplationRun
|
||||||
|
from core.contemplation.snapshot import ContemplationSubstrate
|
||||||
|
|
||||||
|
|
||||||
|
def _config_hash(payload: dict[str, object]) -> str:
|
||||||
|
encoded = json.dumps(
|
||||||
|
payload,
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
)
|
||||||
|
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def contemplate_frontier_reports(
|
||||||
|
report_paths: Iterable[str | Path],
|
||||||
|
*,
|
||||||
|
pack_ids: Iterable[str] = (),
|
||||||
|
notes: Iterable[str] = (),
|
||||||
|
) -> ContemplationRun:
|
||||||
|
"""Run ADR-0080 Phase 1 over explicit frontier-compare reports.
|
||||||
|
|
||||||
|
The runner is read-only. It does not discover files implicitly, does not
|
||||||
|
mutate packs, does not write teaching examples, and does not promote any
|
||||||
|
finding beyond SPECULATIVE.
|
||||||
|
"""
|
||||||
|
|
||||||
|
paths = tuple(Path(p) for p in report_paths)
|
||||||
|
substrate = ContemplationSubstrate.from_report_paths(
|
||||||
|
paths,
|
||||||
|
pack_ids=tuple(pack_ids),
|
||||||
|
notes=tuple(notes),
|
||||||
|
)
|
||||||
|
findings: list[ContemplationFinding] = []
|
||||||
|
for path in paths:
|
||||||
|
findings.extend(
|
||||||
|
mine_frontier_compare_report(
|
||||||
|
path,
|
||||||
|
substrate_hash=substrate.substrate_hash,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
config_hash = _config_hash(
|
||||||
|
{
|
||||||
|
"runner": "contemplate_frontier_reports",
|
||||||
|
"report_paths": [str(p) for p in paths],
|
||||||
|
"pack_ids": tuple(sorted(set(pack_ids))),
|
||||||
|
"notes": tuple(notes),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return ContemplationRun(
|
||||||
|
substrate_hash=substrate.substrate_hash,
|
||||||
|
config_hash=config_hash,
|
||||||
|
findings=tuple(findings),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def write_contemplation_run(run: ContemplationRun, path: str | Path) -> None:
|
||||||
|
target = Path(path)
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
target.write_text(
|
||||||
|
json.dumps(run.as_dict(), ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["contemplate_frontier_reports", "write_contemplation_run"]
|
||||||
166
core/contemplation/schema.py
Normal file
166
core/contemplation/schema.py
Normal file
|
|
@ -0,0 +1,166 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from enum import Enum, unique
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from teaching.epistemic import EpistemicStatus
|
||||||
|
|
||||||
|
|
||||||
|
@unique
|
||||||
|
class FindingKind(Enum):
|
||||||
|
"""Kinds of read-only contemplation findings.
|
||||||
|
|
||||||
|
These are diagnostic/curricular signals, not learned truths.
|
||||||
|
"""
|
||||||
|
|
||||||
|
COVERAGE_GAP = "coverage_gap"
|
||||||
|
CONTRADICTION = "contradiction"
|
||||||
|
WEAK_SURFACE = "weak_surface"
|
||||||
|
UNPROVED_RELATION = "unproved_relation"
|
||||||
|
DERIVABLE_RELATION = "derivable_relation"
|
||||||
|
BENCHMARK_CASE = "benchmark_case"
|
||||||
|
OOV_GAP = "oov_gap"
|
||||||
|
PLANNER_GAP = "planner_gap"
|
||||||
|
PACK_MUTATION_CANDIDATE = "pack_mutation_candidate"
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical_json(payload: dict[str, Any]) -> str:
|
||||||
|
return json.dumps(
|
||||||
|
payload,
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256_16(payload: dict[str, Any]) -> str:
|
||||||
|
return hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest()[:16]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ContemplationEvidenceRef:
|
||||||
|
"""Pointer to evidence supporting a contemplation finding."""
|
||||||
|
|
||||||
|
source_type: str
|
||||||
|
source_id: str
|
||||||
|
pointer: str
|
||||||
|
summary: str = ""
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
if not self.source_type.strip():
|
||||||
|
raise ValueError("ContemplationEvidenceRef.source_type is required")
|
||||||
|
if not self.source_id.strip():
|
||||||
|
raise ValueError("ContemplationEvidenceRef.source_id is required")
|
||||||
|
if not self.pointer.strip():
|
||||||
|
raise ValueError("ContemplationEvidenceRef.pointer is required")
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, str]:
|
||||||
|
return {
|
||||||
|
"source_type": self.source_type,
|
||||||
|
"source_id": self.source_id,
|
||||||
|
"pointer": self.pointer,
|
||||||
|
"summary": self.summary,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ContemplationFinding:
|
||||||
|
"""One speculative self-contemplation output.
|
||||||
|
|
||||||
|
ADR-0080 invariant: contemplation findings are never COHERENT. The
|
||||||
|
loop may propose reviewable evidence, but it may not ratify itself.
|
||||||
|
"""
|
||||||
|
|
||||||
|
kind: FindingKind
|
||||||
|
subject: str
|
||||||
|
predicate: str
|
||||||
|
object: str | None
|
||||||
|
evidence_refs: tuple[ContemplationEvidenceRef, ...]
|
||||||
|
proposed_action: str
|
||||||
|
substrate_hash: str
|
||||||
|
epistemic_status: EpistemicStatus = EpistemicStatus.SPECULATIVE
|
||||||
|
finding_id: str = field(default="")
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
if self.epistemic_status is not EpistemicStatus.SPECULATIVE:
|
||||||
|
raise ValueError(
|
||||||
|
"ContemplationFinding must remain SPECULATIVE; "
|
||||||
|
f"got {self.epistemic_status.value}"
|
||||||
|
)
|
||||||
|
if not self.subject.strip():
|
||||||
|
raise ValueError("ContemplationFinding.subject is required")
|
||||||
|
if not self.predicate.strip():
|
||||||
|
raise ValueError("ContemplationFinding.predicate is required")
|
||||||
|
if not self.proposed_action.strip():
|
||||||
|
raise ValueError("ContemplationFinding.proposed_action is required")
|
||||||
|
if not self.substrate_hash.strip():
|
||||||
|
raise ValueError("ContemplationFinding.substrate_hash is required")
|
||||||
|
if not self.evidence_refs:
|
||||||
|
raise ValueError("ContemplationFinding requires at least one evidence ref")
|
||||||
|
if not self.finding_id:
|
||||||
|
object.__setattr__(self, "finding_id", _sha256_16(self._identity_dict()))
|
||||||
|
|
||||||
|
def _identity_dict(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"kind": self.kind.value,
|
||||||
|
"subject": self.subject,
|
||||||
|
"predicate": self.predicate,
|
||||||
|
"object": self.object,
|
||||||
|
"evidence_refs": [e.as_dict() for e in self.evidence_refs],
|
||||||
|
"proposed_action": self.proposed_action,
|
||||||
|
"substrate_hash": self.substrate_hash,
|
||||||
|
"epistemic_status": self.epistemic_status.value,
|
||||||
|
}
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, Any]:
|
||||||
|
payload = self._identity_dict()
|
||||||
|
payload["finding_id"] = self.finding_id
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ContemplationRun:
|
||||||
|
"""Deterministic result of a read-only contemplation pass."""
|
||||||
|
|
||||||
|
substrate_hash: str
|
||||||
|
config_hash: str
|
||||||
|
findings: tuple[ContemplationFinding, ...]
|
||||||
|
run_id: str = field(default="")
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
if not self.substrate_hash.strip():
|
||||||
|
raise ValueError("ContemplationRun.substrate_hash is required")
|
||||||
|
if not self.config_hash.strip():
|
||||||
|
raise ValueError("ContemplationRun.config_hash is required")
|
||||||
|
for finding in self.findings:
|
||||||
|
if finding.epistemic_status is not EpistemicStatus.SPECULATIVE:
|
||||||
|
raise ValueError("ContemplationRun cannot contain non-SPECULATIVE findings")
|
||||||
|
if not self.run_id:
|
||||||
|
object.__setattr__(self, "run_id", _sha256_16(self._identity_dict()))
|
||||||
|
|
||||||
|
def _identity_dict(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"substrate_hash": self.substrate_hash,
|
||||||
|
"config_hash": self.config_hash,
|
||||||
|
"findings": [f.as_dict() for f in self.findings],
|
||||||
|
}
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"run_id": self.run_id,
|
||||||
|
"substrate_hash": self.substrate_hash,
|
||||||
|
"config_hash": self.config_hash,
|
||||||
|
"finding_count": len(self.findings),
|
||||||
|
"findings": [f.as_dict() for f in self.findings],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"ContemplationEvidenceRef",
|
||||||
|
"ContemplationFinding",
|
||||||
|
"ContemplationRun",
|
||||||
|
"FindingKind",
|
||||||
|
]
|
||||||
77
core/contemplation/snapshot.py
Normal file
77
core/contemplation/snapshot.py
Normal file
|
|
@ -0,0 +1,77 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Iterable
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical_json(payload: dict[str, Any]) -> str:
|
||||||
|
return json.dumps(
|
||||||
|
payload,
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256(payload: str) -> str:
|
||||||
|
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _file_digest(path: Path) -> str:
|
||||||
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ContemplationSubstrate:
|
||||||
|
"""Explicit read-only substrate contemplated by ADR-0080.
|
||||||
|
|
||||||
|
Phase 1 intentionally records identifiers and report hashes only. It does
|
||||||
|
not crawl mutable runtime state and does not treat ambient files as hidden
|
||||||
|
evidence.
|
||||||
|
"""
|
||||||
|
|
||||||
|
pack_ids: tuple[str, ...] = ()
|
||||||
|
report_hashes: tuple[str, ...] = ()
|
||||||
|
notes: tuple[str, ...] = ()
|
||||||
|
substrate_hash: str = field(default="")
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
object.__setattr__(self, "pack_ids", tuple(sorted(set(self.pack_ids))))
|
||||||
|
object.__setattr__(self, "report_hashes", tuple(sorted(set(self.report_hashes))))
|
||||||
|
object.__setattr__(self, "notes", tuple(self.notes))
|
||||||
|
if not self.substrate_hash:
|
||||||
|
object.__setattr__(self, "substrate_hash", _sha256(_canonical_json(self._identity_dict())))
|
||||||
|
|
||||||
|
def _identity_dict(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"pack_ids": list(self.pack_ids),
|
||||||
|
"report_hashes": list(self.report_hashes),
|
||||||
|
"notes": list(self.notes),
|
||||||
|
}
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, Any]:
|
||||||
|
payload = self._identity_dict()
|
||||||
|
payload["substrate_hash"] = self.substrate_hash
|
||||||
|
return payload
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_report_paths(
|
||||||
|
cls,
|
||||||
|
report_paths: Iterable[str | Path],
|
||||||
|
*,
|
||||||
|
pack_ids: Iterable[str] = (),
|
||||||
|
notes: Iterable[str] = (),
|
||||||
|
) -> "ContemplationSubstrate":
|
||||||
|
paths = tuple(Path(p) for p in report_paths)
|
||||||
|
report_hashes = tuple(_file_digest(path) for path in paths)
|
||||||
|
return cls(
|
||||||
|
pack_ids=tuple(pack_ids),
|
||||||
|
report_hashes=report_hashes,
|
||||||
|
notes=tuple(notes),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["ContemplationSubstrate"]
|
||||||
201
docs/decisions/ADR-0080-contemplation-loop.md
Normal file
201
docs/decisions/ADR-0080-contemplation-loop.md
Normal file
|
|
@ -0,0 +1,201 @@
|
||||||
|
# ADR-0080 — Contemplation Loop: self-interrogation without self-ratification
|
||||||
|
|
||||||
|
**Status:** Proposed
|
||||||
|
**Date:** 2026-05-20
|
||||||
|
**Author:** Shay
|
||||||
|
**Builds on:** [ADR-0021](./ADR-0021-epistemic-grade-policy.md) (epistemic grade policy), [ADR-0055](./ADR-0055-teaching-corpus-audit.md) (reviewed teaching corpus), [ADR-0078](./ADR-0078-composer-graph-atom-equivalence.md) (composer/graph atom telemetry)
|
||||||
|
**Series:** L0 — learning safety / self-contemplation boundary
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
CORE already has a conservative learning boundary:
|
||||||
|
|
||||||
|
* reviewed corrections become `ReviewedTeachingExample` records;
|
||||||
|
* accepted examples emit `PackMutationProposal` objects;
|
||||||
|
* proposals do not mutate packs directly;
|
||||||
|
* epistemic status starts as `SPECULATIVE` and only coherence review may promote a claim;
|
||||||
|
* only `COHERENT` claims are admissible as downstream evidence.
|
||||||
|
|
||||||
|
That boundary is correct. Self-contemplation must not bypass it.
|
||||||
|
|
||||||
|
The architectural temptation is to let the model think about its current knowledge, generate new statements, and then treat those statements as learned. That is not learning; it is recursive self-confirmation. It would reintroduce the same failure mode CORE is designed to avoid: ungrounded output hardening into future substrate.
|
||||||
|
|
||||||
|
The correct form is narrower:
|
||||||
|
|
||||||
|
> Self-contemplation may interrogate the current learned substrate and emit evidence-bearing `SPECULATIVE` findings. It may not ratify, promote, or apply its own conclusions.
|
||||||
|
|
||||||
|
Contemplation is therefore a read-only scientific instrument, not an autonomous training loop.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
Introduce a deterministic, sandboxed contemplation loop whose first phase is read-only and report-driven.
|
||||||
|
|
||||||
|
The loop consumes explicit substrate snapshots and structured reports, then emits `ContemplationFinding` records. A finding can represent a coverage gap, contradiction, weak surface, unproved relation, derivable relation, benchmark case, OOV gap, planner gap, or pack mutation candidate. In ADR-0080 Phase 1, every emitted finding is `SPECULATIVE` by construction.
|
||||||
|
|
||||||
|
The initial implementation is intentionally small:
|
||||||
|
|
||||||
|
```text
|
||||||
|
core/contemplation/
|
||||||
|
schema.py
|
||||||
|
snapshot.py
|
||||||
|
runner.py
|
||||||
|
miners/frontier_compare.py
|
||||||
|
```
|
||||||
|
|
||||||
|
Phase 1 mines existing `frontier_compare` benchmark JSON reports and turns failed cases into reviewable findings. This creates a safe path for self-contemplation to improve the test curriculum without mutating knowledge.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Doctrine
|
||||||
|
|
||||||
|
### Contemplation may produce
|
||||||
|
|
||||||
|
```text
|
||||||
|
coverage gaps
|
||||||
|
contradiction candidates
|
||||||
|
weak-surface findings
|
||||||
|
unproved-relation findings
|
||||||
|
derivable-relation candidates
|
||||||
|
benchmark-case candidates
|
||||||
|
OOV-gap findings
|
||||||
|
planner-gap findings
|
||||||
|
pack-mutation candidates
|
||||||
|
```
|
||||||
|
|
||||||
|
### Contemplation may not produce
|
||||||
|
|
||||||
|
```text
|
||||||
|
COHERENT claims
|
||||||
|
applied pack mutations
|
||||||
|
ratified packs
|
||||||
|
runtime behavior changes
|
||||||
|
durable truth writes
|
||||||
|
silent teaching corpus writes
|
||||||
|
```
|
||||||
|
|
||||||
|
### Learning remains externalized
|
||||||
|
|
||||||
|
The only admissible durable-learning path remains:
|
||||||
|
|
||||||
|
```text
|
||||||
|
finding / proposal
|
||||||
|
-> review
|
||||||
|
-> coherence judgment
|
||||||
|
-> pack mutation proposal
|
||||||
|
-> ratification
|
||||||
|
-> replay / eval
|
||||||
|
```
|
||||||
|
|
||||||
|
Contemplation can discover what should be reviewed. It cannot review itself.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Invariants
|
||||||
|
|
||||||
|
ADR-0080 Phase 1 requires the following hard invariants:
|
||||||
|
|
||||||
|
### 1. No self-ratification
|
||||||
|
|
||||||
|
Every `ContemplationFinding` emitted by the v1 loop must have:
|
||||||
|
|
||||||
|
```text
|
||||||
|
epistemic_status == SPECULATIVE
|
||||||
|
```
|
||||||
|
|
||||||
|
Constructing a finding with `COHERENT`, `CONTESTED`, or `FALSIFIED` is rejected.
|
||||||
|
|
||||||
|
### 2. No pack mutation
|
||||||
|
|
||||||
|
The contemplation runner must not write to `language_packs/`, ratification outputs, teaching corpora, or runtime pack manifests.
|
||||||
|
|
||||||
|
### 3. Replay determinism
|
||||||
|
|
||||||
|
The same substrate snapshot and the same report bytes must produce the same run hash and finding hashes.
|
||||||
|
|
||||||
|
### 4. Evidence-bearing output
|
||||||
|
|
||||||
|
Every finding must carry at least one `ContemplationEvidenceRef`. A model may not emit an unsupported finding merely because it generated an interesting thought.
|
||||||
|
|
||||||
|
### 5. Read-only first wave
|
||||||
|
|
||||||
|
Phase 1 only reads explicit files passed by the operator. It does not crawl the repo, call external APIs, or inspect live runtime mutable state.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Phase 1 implementation boundary
|
||||||
|
|
||||||
|
Phase 1 may implement:
|
||||||
|
|
||||||
|
* immutable schema dataclasses;
|
||||||
|
* canonical JSON and stable hashes;
|
||||||
|
* substrate snapshot hashing;
|
||||||
|
* a `frontier_compare` report miner;
|
||||||
|
* a read-only runner;
|
||||||
|
* unit tests proving no promotion, no mutation, deterministic replay, and evidence requirements.
|
||||||
|
|
||||||
|
Phase 1 must not implement:
|
||||||
|
|
||||||
|
* autonomous curriculum generation;
|
||||||
|
* pack mutation writing;
|
||||||
|
* teaching-store writes;
|
||||||
|
* graph/proposition mutation;
|
||||||
|
* runtime integration;
|
||||||
|
* long-running self-play;
|
||||||
|
* external model calls;
|
||||||
|
* hidden background jobs.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Report-mining behavior
|
||||||
|
|
||||||
|
Given a `frontier_compare` report JSON, the v1 miner emits one finding for every failed case.
|
||||||
|
|
||||||
|
A failed case becomes:
|
||||||
|
|
||||||
|
```text
|
||||||
|
kind: benchmark_case
|
||||||
|
subject: <suite>/<case_id>
|
||||||
|
predicate: failed_case
|
||||||
|
object: <prompt>
|
||||||
|
epistemic_status: speculative
|
||||||
|
evidence_refs: report path, suite, case id, failures
|
||||||
|
proposed_action: add or repair a benchmark/training case for the observed failure
|
||||||
|
```
|
||||||
|
|
||||||
|
The finding is not a claim about world truth. It is a claim about system behavior under a recorded benchmark report.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
### Positive
|
||||||
|
|
||||||
|
* Gives CORE a safe self-contemplation surface.
|
||||||
|
* Converts eval failures into structured, reviewable findings.
|
||||||
|
* Preserves the existing epistemic lattice.
|
||||||
|
* Prevents self-generated text from hardening into knowledge.
|
||||||
|
* Creates the first bridge from benchmarks to weakness-driven curriculum.
|
||||||
|
|
||||||
|
### Negative / costs
|
||||||
|
|
||||||
|
* Does not make CORE autonomously learn yet.
|
||||||
|
* Requires operator review before any durable learning occurs.
|
||||||
|
* Starts with benchmark-report mining only; deeper contradiction/derivation miners are deferred.
|
||||||
|
|
||||||
|
### Deferred follow-up
|
||||||
|
|
||||||
|
Later ADRs may add miners for:
|
||||||
|
|
||||||
|
* pack coverage gaps;
|
||||||
|
* teaching-chain gaps;
|
||||||
|
* graph contradiction candidates;
|
||||||
|
* derivable-relation candidates;
|
||||||
|
* OOV queue consolidation;
|
||||||
|
* long-form workbench failures;
|
||||||
|
* curriculum task generation.
|
||||||
|
|
||||||
|
Each follow-up must preserve the no-self-ratification invariant.
|
||||||
161
tests/test_contemplation_loop.py
Normal file
161
tests/test_contemplation_loop.py
Normal file
|
|
@ -0,0 +1,161 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from core.contemplation.__main__ import main
|
||||||
|
from core.contemplation.miners.frontier_compare import mine_frontier_compare_report
|
||||||
|
from core.contemplation.runner import contemplate_frontier_reports, write_contemplation_run
|
||||||
|
from core.contemplation.schema import (
|
||||||
|
ContemplationEvidenceRef,
|
||||||
|
ContemplationFinding,
|
||||||
|
FindingKind,
|
||||||
|
)
|
||||||
|
from core.contemplation.snapshot import ContemplationSubstrate
|
||||||
|
from teaching.epistemic import EpistemicStatus
|
||||||
|
|
||||||
|
|
||||||
|
def _sample_frontier_report(path: Path) -> None:
|
||||||
|
payload = {
|
||||||
|
"benchmark_family": "frontier_compare_wave1",
|
||||||
|
"model": "core",
|
||||||
|
"mode": "native",
|
||||||
|
"summary": {"suite_count": 1, "case_count": 2, "primary_score": 0.5, "passed": False},
|
||||||
|
"suites": [
|
||||||
|
{
|
||||||
|
"suite": "truth_lock",
|
||||||
|
"case_count": 2,
|
||||||
|
"primary_score": 0.5,
|
||||||
|
"passed": False,
|
||||||
|
"cases": [
|
||||||
|
{
|
||||||
|
"suite": "truth_lock",
|
||||||
|
"case_id": "known_truth",
|
||||||
|
"prompt": "What is truth?",
|
||||||
|
"passed": True,
|
||||||
|
"score": 1.0,
|
||||||
|
"elapsed_ms": 1.0,
|
||||||
|
"details": {},
|
||||||
|
"failures": [],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"suite": "truth_lock",
|
||||||
|
"case_id": "unknown_relation",
|
||||||
|
"prompt": "Why does xylomorphic matter?",
|
||||||
|
"passed": False,
|
||||||
|
"score": 0.0,
|
||||||
|
"elapsed_ms": 1.0,
|
||||||
|
"details": {},
|
||||||
|
"failures": ["unexpected_grounding_source:vault"],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def test_contemplation_finding_rejects_non_speculative_status() -> None:
|
||||||
|
evidence = ContemplationEvidenceRef(
|
||||||
|
source_type="test",
|
||||||
|
source_id="unit",
|
||||||
|
pointer="case=1",
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="SPECULATIVE"):
|
||||||
|
ContemplationFinding(
|
||||||
|
kind=FindingKind.BENCHMARK_CASE,
|
||||||
|
subject="suite/case",
|
||||||
|
predicate="failed_case",
|
||||||
|
object="prompt",
|
||||||
|
evidence_refs=(evidence,),
|
||||||
|
proposed_action="review failure",
|
||||||
|
substrate_hash="substrate",
|
||||||
|
epistemic_status=EpistemicStatus.COHERENT,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_contemplation_finding_requires_evidence() -> None:
|
||||||
|
with pytest.raises(ValueError, match="evidence"):
|
||||||
|
ContemplationFinding(
|
||||||
|
kind=FindingKind.BENCHMARK_CASE,
|
||||||
|
subject="suite/case",
|
||||||
|
predicate="failed_case",
|
||||||
|
object="prompt",
|
||||||
|
evidence_refs=(),
|
||||||
|
proposed_action="review failure",
|
||||||
|
substrate_hash="substrate",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_frontier_report_miner_emits_failed_cases_only(tmp_path: Path) -> None:
|
||||||
|
report = tmp_path / "frontier_wave1.json"
|
||||||
|
_sample_frontier_report(report)
|
||||||
|
substrate = ContemplationSubstrate.from_report_paths((report,))
|
||||||
|
|
||||||
|
findings = mine_frontier_compare_report(report, substrate_hash=substrate.substrate_hash)
|
||||||
|
|
||||||
|
assert len(findings) == 1
|
||||||
|
finding = findings[0]
|
||||||
|
assert finding.kind is FindingKind.BENCHMARK_CASE
|
||||||
|
assert finding.subject == "truth_lock/unknown_relation"
|
||||||
|
assert finding.predicate == "failed_case"
|
||||||
|
assert finding.object == "Why does xylomorphic matter?"
|
||||||
|
assert finding.epistemic_status is EpistemicStatus.SPECULATIVE
|
||||||
|
assert finding.evidence_refs[0].summary == "unexpected_grounding_source:vault"
|
||||||
|
|
||||||
|
|
||||||
|
def test_contemplation_runner_is_replay_deterministic(tmp_path: Path) -> None:
|
||||||
|
report = tmp_path / "frontier_wave1.json"
|
||||||
|
_sample_frontier_report(report)
|
||||||
|
|
||||||
|
first = contemplate_frontier_reports((report,), pack_ids=("en_core_cognition_v1",))
|
||||||
|
second = contemplate_frontier_reports((report,), pack_ids=("en_core_cognition_v1",))
|
||||||
|
|
||||||
|
assert first.run_id == second.run_id
|
||||||
|
assert first.as_dict() == second.as_dict()
|
||||||
|
assert all(
|
||||||
|
finding.epistemic_status is EpistemicStatus.SPECULATIVE
|
||||||
|
for finding in first.findings
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_contemplation_runner_does_not_mutate_pack_tree(tmp_path: Path) -> None:
|
||||||
|
report = tmp_path / "frontier_wave1.json"
|
||||||
|
_sample_frontier_report(report)
|
||||||
|
pack_root = Path("language_packs")
|
||||||
|
before = sorted(
|
||||||
|
(p.relative_to(pack_root).as_posix(), p.stat().st_mtime_ns, p.stat().st_size)
|
||||||
|
for p in pack_root.rglob("*")
|
||||||
|
if p.is_file()
|
||||||
|
)
|
||||||
|
|
||||||
|
contemplate_frontier_reports((report,))
|
||||||
|
|
||||||
|
after = sorted(
|
||||||
|
(p.relative_to(pack_root).as_posix(), p.stat().st_mtime_ns, p.stat().st_size)
|
||||||
|
for p in pack_root.rglob("*")
|
||||||
|
if p.is_file()
|
||||||
|
)
|
||||||
|
assert before == after
|
||||||
|
|
||||||
|
|
||||||
|
def test_contemplation_write_and_cli(tmp_path: Path, capsys) -> None:
|
||||||
|
source_report = tmp_path / "frontier_wave1.json"
|
||||||
|
output_report = tmp_path / "contemplation.json"
|
||||||
|
_sample_frontier_report(source_report)
|
||||||
|
|
||||||
|
run = contemplate_frontier_reports((source_report,))
|
||||||
|
write_contemplation_run(run, output_report)
|
||||||
|
written = json.loads(output_report.read_text(encoding="utf-8"))
|
||||||
|
assert written["run_id"] == run.run_id
|
||||||
|
assert written["finding_count"] == 1
|
||||||
|
|
||||||
|
code = main([str(source_report), "--report", str(output_report)])
|
||||||
|
out = capsys.readouterr().out
|
||||||
|
printed = json.loads(out)
|
||||||
|
assert code == 0
|
||||||
|
assert printed["finding_count"] == 1
|
||||||
|
assert json.loads(output_report.read_text(encoding="utf-8")) == printed
|
||||||
Loading…
Reference in a new issue