feat(contemplation): ADR-0080 read-only speculative loop (#55)
* docs(adr): ADR-0080 contemplation loop boundary * feat(contemplation): add read-only contemplation package * feat(contemplation): add immutable speculative finding schema * feat(contemplation): add deterministic substrate snapshot * feat(contemplation): add frontier report miner package * feat(contemplation): mine frontier compare failures as speculative findings * feat(contemplation): add read-only contemplation runner * feat(contemplation): add read-only contemplation CLI * test(contemplation): prove read-only speculative loop invariants
This commit is contained in:
parent
6761fc0974
commit
06bbac86e1
9 changed files with 842 additions and 0 deletions
23
core/contemplation/__init__.py
Normal file
23
core/contemplation/__init__.py
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
"""Read-only contemplation loop primitives.
|
||||
|
||||
ADR-0080: contemplation can emit speculative findings about current
|
||||
substrate/report evidence, but it cannot ratify, promote, or mutate packs.
|
||||
"""
|
||||
|
||||
from .runner import contemplate_frontier_reports
|
||||
from .schema import (
|
||||
ContemplationEvidenceRef,
|
||||
ContemplationFinding,
|
||||
ContemplationRun,
|
||||
FindingKind,
|
||||
)
|
||||
from .snapshot import ContemplationSubstrate
|
||||
|
||||
__all__ = [
|
||||
"ContemplationEvidenceRef",
|
||||
"ContemplationFinding",
|
||||
"ContemplationRun",
|
||||
"ContemplationSubstrate",
|
||||
"FindingKind",
|
||||
"contemplate_frontier_reports",
|
||||
]
|
||||
60
core/contemplation/__main__.py
Normal file
60
core/contemplation/__main__.py
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from core.contemplation.runner import (
|
||||
contemplate_frontier_reports,
|
||||
write_contemplation_run,
|
||||
)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="python -m core.contemplation",
|
||||
description="Run ADR-0080 read-only contemplation over explicit evidence files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"reports",
|
||||
nargs="+",
|
||||
type=Path,
|
||||
help="frontier_compare JSON report path(s) to contemplate.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pack-id",
|
||||
action="append",
|
||||
default=(),
|
||||
help="Optional pack id to include in the substrate snapshot. May repeat.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--note",
|
||||
action="append",
|
||||
default=(),
|
||||
help="Optional operator note included in the substrate snapshot. May repeat.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--report",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Optional output path for the contemplation run JSON.",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
run = contemplate_frontier_reports(
|
||||
args.reports,
|
||||
pack_ids=tuple(args.pack_id or ()),
|
||||
notes=tuple(args.note or ()),
|
||||
)
|
||||
if args.report is not None:
|
||||
write_contemplation_run(run, args.report)
|
||||
print(json.dumps(run.as_dict(), ensure_ascii=False, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
5
core/contemplation/miners/__init__.py
Normal file
5
core/contemplation/miners/__init__.py
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
"""Read-only contemplation miners."""
|
||||
|
||||
from .frontier_compare import mine_frontier_compare_report
|
||||
|
||||
__all__ = ["mine_frontier_compare_report"]
|
||||
75
core/contemplation/miners/frontier_compare.py
Normal file
75
core/contemplation/miners/frontier_compare.py
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from core.contemplation.schema import (
|
||||
ContemplationEvidenceRef,
|
||||
ContemplationFinding,
|
||||
FindingKind,
|
||||
)
|
||||
|
||||
|
||||
def _suite_iter(report: dict[str, Any]) -> tuple[dict[str, Any], ...]:
|
||||
if "suites" in report:
|
||||
return tuple(s for s in report.get("suites", ()) if isinstance(s, dict))
|
||||
if "suite" in report:
|
||||
return (report,)
|
||||
return ()
|
||||
|
||||
|
||||
def _case_iter(suite: dict[str, Any]) -> tuple[dict[str, Any], ...]:
|
||||
return tuple(c for c in suite.get("cases", ()) if isinstance(c, dict))
|
||||
|
||||
|
||||
def mine_frontier_compare_report(
|
||||
report_path: str | Path,
|
||||
*,
|
||||
substrate_hash: str,
|
||||
) -> tuple[ContemplationFinding, ...]:
|
||||
"""Convert failed frontier-compare cases into speculative findings.
|
||||
|
||||
Read-only: this function only parses *report_path* and returns immutable
|
||||
findings. It does not write reports, packs, teaching examples, or runtime
|
||||
state.
|
||||
"""
|
||||
|
||||
path = Path(report_path)
|
||||
report = json.loads(path.read_text(encoding="utf-8"))
|
||||
findings: list[ContemplationFinding] = []
|
||||
source_id = str(path)
|
||||
|
||||
for suite in _suite_iter(report):
|
||||
suite_name = str(suite.get("suite", "unknown_suite"))
|
||||
for case in _case_iter(suite):
|
||||
if bool(case.get("passed", False)):
|
||||
continue
|
||||
case_id = str(case.get("case_id", "unknown_case"))
|
||||
prompt = str(case.get("prompt", ""))
|
||||
failures = tuple(str(f) for f in case.get("failures", ()) or ())
|
||||
evidence = ContemplationEvidenceRef(
|
||||
source_type="frontier_compare_report",
|
||||
source_id=source_id,
|
||||
pointer=f"suite={suite_name};case={case_id}",
|
||||
summary=", ".join(failures) if failures else "case failed",
|
||||
)
|
||||
findings.append(
|
||||
ContemplationFinding(
|
||||
kind=FindingKind.BENCHMARK_CASE,
|
||||
subject=f"{suite_name}/{case_id}",
|
||||
predicate="failed_case",
|
||||
object=prompt,
|
||||
evidence_refs=(evidence,),
|
||||
proposed_action=(
|
||||
"Review this failed benchmark case; either repair the runtime/pack "
|
||||
"behavior or promote the case into a focused regression/curriculum item."
|
||||
),
|
||||
substrate_hash=substrate_hash,
|
||||
)
|
||||
)
|
||||
|
||||
return tuple(findings)
|
||||
|
||||
|
||||
__all__ = ["mine_frontier_compare_report"]
|
||||
74
core/contemplation/runner.py
Normal file
74
core/contemplation/runner.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
from core.contemplation.miners.frontier_compare import mine_frontier_compare_report
|
||||
from core.contemplation.schema import ContemplationFinding, ContemplationRun
|
||||
from core.contemplation.snapshot import ContemplationSubstrate
|
||||
|
||||
|
||||
def _config_hash(payload: dict[str, object]) -> str:
|
||||
encoded = json.dumps(
|
||||
payload,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def contemplate_frontier_reports(
|
||||
report_paths: Iterable[str | Path],
|
||||
*,
|
||||
pack_ids: Iterable[str] = (),
|
||||
notes: Iterable[str] = (),
|
||||
) -> ContemplationRun:
|
||||
"""Run ADR-0080 Phase 1 over explicit frontier-compare reports.
|
||||
|
||||
The runner is read-only. It does not discover files implicitly, does not
|
||||
mutate packs, does not write teaching examples, and does not promote any
|
||||
finding beyond SPECULATIVE.
|
||||
"""
|
||||
|
||||
paths = tuple(Path(p) for p in report_paths)
|
||||
substrate = ContemplationSubstrate.from_report_paths(
|
||||
paths,
|
||||
pack_ids=tuple(pack_ids),
|
||||
notes=tuple(notes),
|
||||
)
|
||||
findings: list[ContemplationFinding] = []
|
||||
for path in paths:
|
||||
findings.extend(
|
||||
mine_frontier_compare_report(
|
||||
path,
|
||||
substrate_hash=substrate.substrate_hash,
|
||||
)
|
||||
)
|
||||
config_hash = _config_hash(
|
||||
{
|
||||
"runner": "contemplate_frontier_reports",
|
||||
"report_paths": [str(p) for p in paths],
|
||||
"pack_ids": tuple(sorted(set(pack_ids))),
|
||||
"notes": tuple(notes),
|
||||
}
|
||||
)
|
||||
return ContemplationRun(
|
||||
substrate_hash=substrate.substrate_hash,
|
||||
config_hash=config_hash,
|
||||
findings=tuple(findings),
|
||||
)
|
||||
|
||||
|
||||
def write_contemplation_run(run: ContemplationRun, path: str | Path) -> None:
|
||||
target = Path(path)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(
|
||||
json.dumps(run.as_dict(), ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["contemplate_frontier_reports", "write_contemplation_run"]
|
||||
166
core/contemplation/schema.py
Normal file
166
core/contemplation/schema.py
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum, unique
|
||||
from typing import Any
|
||||
|
||||
from teaching.epistemic import EpistemicStatus
|
||||
|
||||
|
||||
@unique
|
||||
class FindingKind(Enum):
|
||||
"""Kinds of read-only contemplation findings.
|
||||
|
||||
These are diagnostic/curricular signals, not learned truths.
|
||||
"""
|
||||
|
||||
COVERAGE_GAP = "coverage_gap"
|
||||
CONTRADICTION = "contradiction"
|
||||
WEAK_SURFACE = "weak_surface"
|
||||
UNPROVED_RELATION = "unproved_relation"
|
||||
DERIVABLE_RELATION = "derivable_relation"
|
||||
BENCHMARK_CASE = "benchmark_case"
|
||||
OOV_GAP = "oov_gap"
|
||||
PLANNER_GAP = "planner_gap"
|
||||
PACK_MUTATION_CANDIDATE = "pack_mutation_candidate"
|
||||
|
||||
|
||||
def _canonical_json(payload: dict[str, Any]) -> str:
|
||||
return json.dumps(
|
||||
payload,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
|
||||
|
||||
def _sha256_16(payload: dict[str, Any]) -> str:
|
||||
return hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ContemplationEvidenceRef:
|
||||
"""Pointer to evidence supporting a contemplation finding."""
|
||||
|
||||
source_type: str
|
||||
source_id: str
|
||||
pointer: str
|
||||
summary: str = ""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.source_type.strip():
|
||||
raise ValueError("ContemplationEvidenceRef.source_type is required")
|
||||
if not self.source_id.strip():
|
||||
raise ValueError("ContemplationEvidenceRef.source_id is required")
|
||||
if not self.pointer.strip():
|
||||
raise ValueError("ContemplationEvidenceRef.pointer is required")
|
||||
|
||||
def as_dict(self) -> dict[str, str]:
|
||||
return {
|
||||
"source_type": self.source_type,
|
||||
"source_id": self.source_id,
|
||||
"pointer": self.pointer,
|
||||
"summary": self.summary,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ContemplationFinding:
|
||||
"""One speculative self-contemplation output.
|
||||
|
||||
ADR-0080 invariant: contemplation findings are never COHERENT. The
|
||||
loop may propose reviewable evidence, but it may not ratify itself.
|
||||
"""
|
||||
|
||||
kind: FindingKind
|
||||
subject: str
|
||||
predicate: str
|
||||
object: str | None
|
||||
evidence_refs: tuple[ContemplationEvidenceRef, ...]
|
||||
proposed_action: str
|
||||
substrate_hash: str
|
||||
epistemic_status: EpistemicStatus = EpistemicStatus.SPECULATIVE
|
||||
finding_id: str = field(default="")
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.epistemic_status is not EpistemicStatus.SPECULATIVE:
|
||||
raise ValueError(
|
||||
"ContemplationFinding must remain SPECULATIVE; "
|
||||
f"got {self.epistemic_status.value}"
|
||||
)
|
||||
if not self.subject.strip():
|
||||
raise ValueError("ContemplationFinding.subject is required")
|
||||
if not self.predicate.strip():
|
||||
raise ValueError("ContemplationFinding.predicate is required")
|
||||
if not self.proposed_action.strip():
|
||||
raise ValueError("ContemplationFinding.proposed_action is required")
|
||||
if not self.substrate_hash.strip():
|
||||
raise ValueError("ContemplationFinding.substrate_hash is required")
|
||||
if not self.evidence_refs:
|
||||
raise ValueError("ContemplationFinding requires at least one evidence ref")
|
||||
if not self.finding_id:
|
||||
object.__setattr__(self, "finding_id", _sha256_16(self._identity_dict()))
|
||||
|
||||
def _identity_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"kind": self.kind.value,
|
||||
"subject": self.subject,
|
||||
"predicate": self.predicate,
|
||||
"object": self.object,
|
||||
"evidence_refs": [e.as_dict() for e in self.evidence_refs],
|
||||
"proposed_action": self.proposed_action,
|
||||
"substrate_hash": self.substrate_hash,
|
||||
"epistemic_status": self.epistemic_status.value,
|
||||
}
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
payload = self._identity_dict()
|
||||
payload["finding_id"] = self.finding_id
|
||||
return payload
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ContemplationRun:
|
||||
"""Deterministic result of a read-only contemplation pass."""
|
||||
|
||||
substrate_hash: str
|
||||
config_hash: str
|
||||
findings: tuple[ContemplationFinding, ...]
|
||||
run_id: str = field(default="")
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.substrate_hash.strip():
|
||||
raise ValueError("ContemplationRun.substrate_hash is required")
|
||||
if not self.config_hash.strip():
|
||||
raise ValueError("ContemplationRun.config_hash is required")
|
||||
for finding in self.findings:
|
||||
if finding.epistemic_status is not EpistemicStatus.SPECULATIVE:
|
||||
raise ValueError("ContemplationRun cannot contain non-SPECULATIVE findings")
|
||||
if not self.run_id:
|
||||
object.__setattr__(self, "run_id", _sha256_16(self._identity_dict()))
|
||||
|
||||
def _identity_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"substrate_hash": self.substrate_hash,
|
||||
"config_hash": self.config_hash,
|
||||
"findings": [f.as_dict() for f in self.findings],
|
||||
}
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"run_id": self.run_id,
|
||||
"substrate_hash": self.substrate_hash,
|
||||
"config_hash": self.config_hash,
|
||||
"finding_count": len(self.findings),
|
||||
"findings": [f.as_dict() for f in self.findings],
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ContemplationEvidenceRef",
|
||||
"ContemplationFinding",
|
||||
"ContemplationRun",
|
||||
"FindingKind",
|
||||
]
|
||||
77
core/contemplation/snapshot.py
Normal file
77
core/contemplation/snapshot.py
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
def _canonical_json(payload: dict[str, Any]) -> str:
|
||||
return json.dumps(
|
||||
payload,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
|
||||
|
||||
def _sha256(payload: str) -> str:
|
||||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _file_digest(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ContemplationSubstrate:
|
||||
"""Explicit read-only substrate contemplated by ADR-0080.
|
||||
|
||||
Phase 1 intentionally records identifiers and report hashes only. It does
|
||||
not crawl mutable runtime state and does not treat ambient files as hidden
|
||||
evidence.
|
||||
"""
|
||||
|
||||
pack_ids: tuple[str, ...] = ()
|
||||
report_hashes: tuple[str, ...] = ()
|
||||
notes: tuple[str, ...] = ()
|
||||
substrate_hash: str = field(default="")
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "pack_ids", tuple(sorted(set(self.pack_ids))))
|
||||
object.__setattr__(self, "report_hashes", tuple(sorted(set(self.report_hashes))))
|
||||
object.__setattr__(self, "notes", tuple(self.notes))
|
||||
if not self.substrate_hash:
|
||||
object.__setattr__(self, "substrate_hash", _sha256(_canonical_json(self._identity_dict())))
|
||||
|
||||
def _identity_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"pack_ids": list(self.pack_ids),
|
||||
"report_hashes": list(self.report_hashes),
|
||||
"notes": list(self.notes),
|
||||
}
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
payload = self._identity_dict()
|
||||
payload["substrate_hash"] = self.substrate_hash
|
||||
return payload
|
||||
|
||||
@classmethod
|
||||
def from_report_paths(
|
||||
cls,
|
||||
report_paths: Iterable[str | Path],
|
||||
*,
|
||||
pack_ids: Iterable[str] = (),
|
||||
notes: Iterable[str] = (),
|
||||
) -> "ContemplationSubstrate":
|
||||
paths = tuple(Path(p) for p in report_paths)
|
||||
report_hashes = tuple(_file_digest(path) for path in paths)
|
||||
return cls(
|
||||
pack_ids=tuple(pack_ids),
|
||||
report_hashes=report_hashes,
|
||||
notes=tuple(notes),
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["ContemplationSubstrate"]
|
||||
201
docs/decisions/ADR-0080-contemplation-loop.md
Normal file
201
docs/decisions/ADR-0080-contemplation-loop.md
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
# ADR-0080 — Contemplation Loop: self-interrogation without self-ratification
|
||||
|
||||
**Status:** Proposed
|
||||
**Date:** 2026-05-20
|
||||
**Author:** Shay
|
||||
**Builds on:** [ADR-0021](./ADR-0021-epistemic-grade-policy.md) (epistemic grade policy), [ADR-0055](./ADR-0055-teaching-corpus-audit.md) (reviewed teaching corpus), [ADR-0078](./ADR-0078-composer-graph-atom-equivalence.md) (composer/graph atom telemetry)
|
||||
**Series:** L0 — learning safety / self-contemplation boundary
|
||||
|
||||
---
|
||||
|
||||
## Context
|
||||
|
||||
CORE already has a conservative learning boundary:
|
||||
|
||||
* reviewed corrections become `ReviewedTeachingExample` records;
|
||||
* accepted examples emit `PackMutationProposal` objects;
|
||||
* proposals do not mutate packs directly;
|
||||
* epistemic status starts as `SPECULATIVE` and only coherence review may promote a claim;
|
||||
* only `COHERENT` claims are admissible as downstream evidence.
|
||||
|
||||
That boundary is correct. Self-contemplation must not bypass it.
|
||||
|
||||
The architectural temptation is to let the model think about its current knowledge, generate new statements, and then treat those statements as learned. That is not learning; it is recursive self-confirmation. It would reintroduce the same failure mode CORE is designed to avoid: ungrounded output hardening into future substrate.
|
||||
|
||||
The correct form is narrower:
|
||||
|
||||
> Self-contemplation may interrogate the current learned substrate and emit evidence-bearing `SPECULATIVE` findings. It may not ratify, promote, or apply its own conclusions.
|
||||
|
||||
Contemplation is therefore a read-only scientific instrument, not an autonomous training loop.
|
||||
|
||||
---
|
||||
|
||||
## Decision
|
||||
|
||||
Introduce a deterministic, sandboxed contemplation loop whose first phase is read-only and report-driven.
|
||||
|
||||
The loop consumes explicit substrate snapshots and structured reports, then emits `ContemplationFinding` records. A finding can represent a coverage gap, contradiction, weak surface, unproved relation, derivable relation, benchmark case, OOV gap, planner gap, or pack mutation candidate. In ADR-0080 Phase 1, every emitted finding is `SPECULATIVE` by construction.
|
||||
|
||||
The initial implementation is intentionally small:
|
||||
|
||||
```text
|
||||
core/contemplation/
|
||||
schema.py
|
||||
snapshot.py
|
||||
runner.py
|
||||
miners/frontier_compare.py
|
||||
```
|
||||
|
||||
Phase 1 mines existing `frontier_compare` benchmark JSON reports and turns failed cases into reviewable findings. This creates a safe path for self-contemplation to improve the test curriculum without mutating knowledge.
|
||||
|
||||
---
|
||||
|
||||
## Doctrine
|
||||
|
||||
### Contemplation may produce
|
||||
|
||||
```text
|
||||
coverage gaps
|
||||
contradiction candidates
|
||||
weak-surface findings
|
||||
unproved-relation findings
|
||||
derivable-relation candidates
|
||||
benchmark-case candidates
|
||||
OOV-gap findings
|
||||
planner-gap findings
|
||||
pack-mutation candidates
|
||||
```
|
||||
|
||||
### Contemplation may not produce
|
||||
|
||||
```text
|
||||
COHERENT claims
|
||||
applied pack mutations
|
||||
ratified packs
|
||||
runtime behavior changes
|
||||
durable truth writes
|
||||
silent teaching corpus writes
|
||||
```
|
||||
|
||||
### Learning remains externalized
|
||||
|
||||
The only admissible durable-learning path remains:
|
||||
|
||||
```text
|
||||
finding / proposal
|
||||
-> review
|
||||
-> coherence judgment
|
||||
-> pack mutation proposal
|
||||
-> ratification
|
||||
-> replay / eval
|
||||
```
|
||||
|
||||
Contemplation can discover what should be reviewed. It cannot review itself.
|
||||
|
||||
---
|
||||
|
||||
## Invariants
|
||||
|
||||
ADR-0080 Phase 1 requires the following hard invariants:
|
||||
|
||||
### 1. No self-ratification
|
||||
|
||||
Every `ContemplationFinding` emitted by the v1 loop must have:
|
||||
|
||||
```text
|
||||
epistemic_status == SPECULATIVE
|
||||
```
|
||||
|
||||
Constructing a finding with `COHERENT`, `CONTESTED`, or `FALSIFIED` is rejected.
|
||||
|
||||
### 2. No pack mutation
|
||||
|
||||
The contemplation runner must not write to `language_packs/`, ratification outputs, teaching corpora, or runtime pack manifests.
|
||||
|
||||
### 3. Replay determinism
|
||||
|
||||
The same substrate snapshot and the same report bytes must produce the same run hash and finding hashes.
|
||||
|
||||
### 4. Evidence-bearing output
|
||||
|
||||
Every finding must carry at least one `ContemplationEvidenceRef`. A model may not emit an unsupported finding merely because it generated an interesting thought.
|
||||
|
||||
### 5. Read-only first wave
|
||||
|
||||
Phase 1 only reads explicit files passed by the operator. It does not crawl the repo, call external APIs, or inspect live runtime mutable state.
|
||||
|
||||
---
|
||||
|
||||
## Phase 1 implementation boundary
|
||||
|
||||
Phase 1 may implement:
|
||||
|
||||
* immutable schema dataclasses;
|
||||
* canonical JSON and stable hashes;
|
||||
* substrate snapshot hashing;
|
||||
* a `frontier_compare` report miner;
|
||||
* a read-only runner;
|
||||
* unit tests proving no promotion, no mutation, deterministic replay, and evidence requirements.
|
||||
|
||||
Phase 1 must not implement:
|
||||
|
||||
* autonomous curriculum generation;
|
||||
* pack mutation writing;
|
||||
* teaching-store writes;
|
||||
* graph/proposition mutation;
|
||||
* runtime integration;
|
||||
* long-running self-play;
|
||||
* external model calls;
|
||||
* hidden background jobs.
|
||||
|
||||
---
|
||||
|
||||
## Report-mining behavior
|
||||
|
||||
Given a `frontier_compare` report JSON, the v1 miner emits one finding for every failed case.
|
||||
|
||||
A failed case becomes:
|
||||
|
||||
```text
|
||||
kind: benchmark_case
|
||||
subject: <suite>/<case_id>
|
||||
predicate: failed_case
|
||||
object: <prompt>
|
||||
epistemic_status: speculative
|
||||
evidence_refs: report path, suite, case id, failures
|
||||
proposed_action: add or repair a benchmark/training case for the observed failure
|
||||
```
|
||||
|
||||
The finding is not a claim about world truth. It is a claim about system behavior under a recorded benchmark report.
|
||||
|
||||
---
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
* Gives CORE a safe self-contemplation surface.
|
||||
* Converts eval failures into structured, reviewable findings.
|
||||
* Preserves the existing epistemic lattice.
|
||||
* Prevents self-generated text from hardening into knowledge.
|
||||
* Creates the first bridge from benchmarks to weakness-driven curriculum.
|
||||
|
||||
### Negative / costs
|
||||
|
||||
* Does not make CORE autonomously learn yet.
|
||||
* Requires operator review before any durable learning occurs.
|
||||
* Starts with benchmark-report mining only; deeper contradiction/derivation miners are deferred.
|
||||
|
||||
### Deferred follow-up
|
||||
|
||||
Later ADRs may add miners for:
|
||||
|
||||
* pack coverage gaps;
|
||||
* teaching-chain gaps;
|
||||
* graph contradiction candidates;
|
||||
* derivable-relation candidates;
|
||||
* OOV queue consolidation;
|
||||
* long-form workbench failures;
|
||||
* curriculum task generation.
|
||||
|
||||
Each follow-up must preserve the no-self-ratification invariant.
|
||||
161
tests/test_contemplation_loop.py
Normal file
161
tests/test_contemplation_loop.py
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from core.contemplation.__main__ import main
|
||||
from core.contemplation.miners.frontier_compare import mine_frontier_compare_report
|
||||
from core.contemplation.runner import contemplate_frontier_reports, write_contemplation_run
|
||||
from core.contemplation.schema import (
|
||||
ContemplationEvidenceRef,
|
||||
ContemplationFinding,
|
||||
FindingKind,
|
||||
)
|
||||
from core.contemplation.snapshot import ContemplationSubstrate
|
||||
from teaching.epistemic import EpistemicStatus
|
||||
|
||||
|
||||
def _sample_frontier_report(path: Path) -> None:
|
||||
payload = {
|
||||
"benchmark_family": "frontier_compare_wave1",
|
||||
"model": "core",
|
||||
"mode": "native",
|
||||
"summary": {"suite_count": 1, "case_count": 2, "primary_score": 0.5, "passed": False},
|
||||
"suites": [
|
||||
{
|
||||
"suite": "truth_lock",
|
||||
"case_count": 2,
|
||||
"primary_score": 0.5,
|
||||
"passed": False,
|
||||
"cases": [
|
||||
{
|
||||
"suite": "truth_lock",
|
||||
"case_id": "known_truth",
|
||||
"prompt": "What is truth?",
|
||||
"passed": True,
|
||||
"score": 1.0,
|
||||
"elapsed_ms": 1.0,
|
||||
"details": {},
|
||||
"failures": [],
|
||||
},
|
||||
{
|
||||
"suite": "truth_lock",
|
||||
"case_id": "unknown_relation",
|
||||
"prompt": "Why does xylomorphic matter?",
|
||||
"passed": False,
|
||||
"score": 0.0,
|
||||
"elapsed_ms": 1.0,
|
||||
"details": {},
|
||||
"failures": ["unexpected_grounding_source:vault"],
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
|
||||
|
||||
|
||||
def test_contemplation_finding_rejects_non_speculative_status() -> None:
|
||||
evidence = ContemplationEvidenceRef(
|
||||
source_type="test",
|
||||
source_id="unit",
|
||||
pointer="case=1",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="SPECULATIVE"):
|
||||
ContemplationFinding(
|
||||
kind=FindingKind.BENCHMARK_CASE,
|
||||
subject="suite/case",
|
||||
predicate="failed_case",
|
||||
object="prompt",
|
||||
evidence_refs=(evidence,),
|
||||
proposed_action="review failure",
|
||||
substrate_hash="substrate",
|
||||
epistemic_status=EpistemicStatus.COHERENT,
|
||||
)
|
||||
|
||||
|
||||
def test_contemplation_finding_requires_evidence() -> None:
|
||||
with pytest.raises(ValueError, match="evidence"):
|
||||
ContemplationFinding(
|
||||
kind=FindingKind.BENCHMARK_CASE,
|
||||
subject="suite/case",
|
||||
predicate="failed_case",
|
||||
object="prompt",
|
||||
evidence_refs=(),
|
||||
proposed_action="review failure",
|
||||
substrate_hash="substrate",
|
||||
)
|
||||
|
||||
|
||||
def test_frontier_report_miner_emits_failed_cases_only(tmp_path: Path) -> None:
|
||||
report = tmp_path / "frontier_wave1.json"
|
||||
_sample_frontier_report(report)
|
||||
substrate = ContemplationSubstrate.from_report_paths((report,))
|
||||
|
||||
findings = mine_frontier_compare_report(report, substrate_hash=substrate.substrate_hash)
|
||||
|
||||
assert len(findings) == 1
|
||||
finding = findings[0]
|
||||
assert finding.kind is FindingKind.BENCHMARK_CASE
|
||||
assert finding.subject == "truth_lock/unknown_relation"
|
||||
assert finding.predicate == "failed_case"
|
||||
assert finding.object == "Why does xylomorphic matter?"
|
||||
assert finding.epistemic_status is EpistemicStatus.SPECULATIVE
|
||||
assert finding.evidence_refs[0].summary == "unexpected_grounding_source:vault"
|
||||
|
||||
|
||||
def test_contemplation_runner_is_replay_deterministic(tmp_path: Path) -> None:
|
||||
report = tmp_path / "frontier_wave1.json"
|
||||
_sample_frontier_report(report)
|
||||
|
||||
first = contemplate_frontier_reports((report,), pack_ids=("en_core_cognition_v1",))
|
||||
second = contemplate_frontier_reports((report,), pack_ids=("en_core_cognition_v1",))
|
||||
|
||||
assert first.run_id == second.run_id
|
||||
assert first.as_dict() == second.as_dict()
|
||||
assert all(
|
||||
finding.epistemic_status is EpistemicStatus.SPECULATIVE
|
||||
for finding in first.findings
|
||||
)
|
||||
|
||||
|
||||
def test_contemplation_runner_does_not_mutate_pack_tree(tmp_path: Path) -> None:
|
||||
report = tmp_path / "frontier_wave1.json"
|
||||
_sample_frontier_report(report)
|
||||
pack_root = Path("language_packs")
|
||||
before = sorted(
|
||||
(p.relative_to(pack_root).as_posix(), p.stat().st_mtime_ns, p.stat().st_size)
|
||||
for p in pack_root.rglob("*")
|
||||
if p.is_file()
|
||||
)
|
||||
|
||||
contemplate_frontier_reports((report,))
|
||||
|
||||
after = sorted(
|
||||
(p.relative_to(pack_root).as_posix(), p.stat().st_mtime_ns, p.stat().st_size)
|
||||
for p in pack_root.rglob("*")
|
||||
if p.is_file()
|
||||
)
|
||||
assert before == after
|
||||
|
||||
|
||||
def test_contemplation_write_and_cli(tmp_path: Path, capsys) -> None:
|
||||
source_report = tmp_path / "frontier_wave1.json"
|
||||
output_report = tmp_path / "contemplation.json"
|
||||
_sample_frontier_report(source_report)
|
||||
|
||||
run = contemplate_frontier_reports((source_report,))
|
||||
write_contemplation_run(run, output_report)
|
||||
written = json.loads(output_report.read_text(encoding="utf-8"))
|
||||
assert written["run_id"] == run.run_id
|
||||
assert written["finding_count"] == 1
|
||||
|
||||
code = main([str(source_report), "--report", str(output_report)])
|
||||
out = capsys.readouterr().out
|
||||
printed = json.loads(out)
|
||||
assert code == 0
|
||||
assert printed["finding_count"] == 1
|
||||
assert json.loads(output_report.read_text(encoding="utf-8")) == printed
|
||||
Loading…
Reference in a new issue