feat(W-019): learning-arc demo — engine-authored proposal from contemplation (ADR-0152) (#276)
Two-session arc where engine derives connective+object from corpus decomposition; operator ratifies rather than authors. Distinguishes from learning-loop (operator-authored) and directly exercises W-018 checkpoint contemplation and W-017 auto-proposal provenance path.
This commit is contained in:
Shay
GitHub
parent
df6c9a3206
commit
e7e28a2fd5
7 changed files with 1252 additions and 5 deletions
31
core/cli.py
31
core/cli.py
|
|
@ -23,7 +23,7 @@ _CORE_RS_DIR = _REPO_ROOT / "core-rs"
|
|||
_CORE_RS_MANIFEST = _CORE_RS_DIR / "Cargo.toml"
|
||||
|
||||
DESCRIPTION = "CORE versor engine command suite."
|
||||
EPILOG = "Examples:\n core chat\n core pulse \"What is truth?\"\n core pulse --no-glove --json \"Compare knowledge and wisdom\"\n core bench\n core bench --suite all\n core bench --suite all --json --report bench_all.json\n core bench --suite determinism --runs 50\n core bench --suite speedup --json\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core teaching audit\n core teaching audit --json\n core teaching gaps --top 10\n core teaching queue --threshold 3\n core teaching propose <candidate-jsonl-path>\n core teaching proposals --state pending\n core teaching review <proposal_id> --accept --review-date 2026-05-18\n core teaching supersede cause_light_reveals_truth --subject light --intent cause --connective grounds --object truth --review-date 2026-05-18\n core teaching supersessions\n core teaching supersessions --json\n core test --suite fast -q\n core test --suite pulse -q\n core test --suite proof -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core demo audit-tour\n core demo register-tour\n core demo anchor-lens-tour\n core demo orthogonality-tour\n core demo pack-measurements\n core demo long-context-comparison\n core demo anti-regression\n core demo learning-loop\n core demo articulation\n core demo conversation\n core demo conversation --no-stream\n core demo all\n core demo adr-0024-chain\n core eval --list\n core eval cognition\n core eval cognition --json --save\n core eval cognition --split dev --version v1\n core eval cognition --split holdout"
|
||||
EPILOG = "Examples:\n core chat\n core pulse \"What is truth?\"\n core pulse --no-glove --json \"Compare knowledge and wisdom\"\n core bench\n core bench --suite all\n core bench --suite all --json --report bench_all.json\n core bench --suite determinism --runs 50\n core bench --suite speedup --json\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core teaching audit\n core teaching audit --json\n core teaching gaps --top 10\n core teaching queue --threshold 3\n core teaching propose <candidate-jsonl-path>\n core teaching proposals --state pending\n core teaching review <proposal_id> --accept --review-date 2026-05-18\n core teaching supersede cause_light_reveals_truth --subject light --intent cause --connective grounds --object truth --review-date 2026-05-18\n core teaching supersessions\n core teaching supersessions --json\n core test --suite fast -q\n core test --suite pulse -q\n core test --suite proof -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core demo audit-tour\n core demo register-tour\n core demo anchor-lens-tour\n core demo orthogonality-tour\n core demo pack-measurements\n core demo long-context-comparison\n core demo anti-regression\n core demo learning-loop\n core demo learning-arc\n core demo articulation\n core demo conversation\n core demo conversation --no-stream\n core demo all\n core demo adr-0024-chain\n core eval --list\n core eval cognition\n core eval cognition --json --save\n core eval cognition --split dev --version v1\n core eval cognition --split holdout"
|
||||
|
||||
_TEST_SUITES: dict[str, tuple[str, ...]] = {
|
||||
"fast": (
|
||||
|
|
@ -2343,7 +2343,8 @@ table. This is the "show me everything" entry point.
|
|||
5. long-context-comparison — exact NIAH vs transformer baselines (ADR-0045)
|
||||
6. anti-regression — three-gate defense (ADR-0057)
|
||||
7. learning-loop — cold turn → grounded surface (ADR-0055..0057)
|
||||
8. articulation — discourse-planner spine (multi-sentence)
|
||||
8. learning-arc — engine-authored proposal via contemplation (ADR-0150..0151)
|
||||
9. articulation — discourse-planner spine (multi-sentence)
|
||||
|
||||
Each demo retains its own preamble + report. The final summary surfaces
|
||||
one boolean per demo and an overall ``all_demos_passed`` flag.
|
||||
|
|
@ -2658,6 +2659,14 @@ def cmd_demo(args: argparse.Namespace) -> int:
|
|||
print(json.dumps(report, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
if target == "learning-arc":
|
||||
from evals.learning_arc.run_demo import run_demo as run_arc_demo
|
||||
|
||||
report = run_arc_demo(emit_json=args.json)
|
||||
if args.json:
|
||||
print(json.dumps(report, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
if target == "articulation":
|
||||
from evals.articulation.run_demo import run_demo as run_articulation_demo
|
||||
|
||||
|
|
@ -2862,7 +2871,7 @@ def _run_demo_all(emit_json: bool) -> int:
|
|||
passed["anti_regression"] = bool(ar_report.get("all_gates_held", False))
|
||||
|
||||
# 7. learning-loop
|
||||
_section("7/8 learning-loop — cold turn → grounded surface")
|
||||
_section("7/9 learning-loop — cold turn → grounded surface")
|
||||
from evals.learning_loop.run_demo import run_demo as run_loop
|
||||
if not emit_json:
|
||||
_print_preamble(_LEARNING_LOOP_PREAMBLE)
|
||||
|
|
@ -2871,8 +2880,16 @@ def _run_demo_all(emit_json: bool) -> int:
|
|||
consolidated["learning_loop"] = ll_report
|
||||
passed["learning_loop"] = bool(ll_report.get("learning_loop_closed", False))
|
||||
|
||||
# 8. articulation
|
||||
_section("8/8 articulation — discourse-planner spine")
|
||||
# 8. learning-arc
|
||||
_section("8/9 learning-arc — engine-authored proposal via contemplation")
|
||||
from evals.learning_arc.run_demo import run_demo as run_arc
|
||||
with _maybe_suppress():
|
||||
arc_report = run_arc(emit_json=emit_json)
|
||||
consolidated["learning_arc"] = arc_report
|
||||
passed["learning_arc"] = bool(arc_report.get("learning_arc_closed", False))
|
||||
|
||||
# 9. articulation
|
||||
_section("9/9 articulation — discourse-planner spine")
|
||||
from evals.articulation.run_demo import run_demo as run_art
|
||||
if not emit_json:
|
||||
_print_preamble(_ARTICULATION_PREAMBLE)
|
||||
|
|
@ -3717,6 +3734,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
"long-context-comparison",
|
||||
"anti-regression",
|
||||
"learning-loop",
|
||||
"learning-arc",
|
||||
"articulation",
|
||||
"conversation",
|
||||
"showcase",
|
||||
|
|
@ -3750,6 +3768,9 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
"harmful chains (eligibility / replay-equivalence / operator). "
|
||||
"learning-loop: ADR-0055..0057 — full cold-turn → discovery → "
|
||||
"propose → accept → same-prompt-now-grounded walkthrough. "
|
||||
"learning-arc: ADR-0150..0151 — two-session arc: checkpoint "
|
||||
"contemplation enriches candidate, engine derives connective + "
|
||||
"object from corpus decomposition, operator only ratifies. "
|
||||
"articulation: discourse-planner spine — EXPLAIN / COMPOUND / "
|
||||
"WALKTHROUGH multi-sentence articulation + determinism gate. "
|
||||
"conversation: layperson-facing chat transcript with live "
|
||||
|
|
|
|||
266
docs/briefs/W-017-auto-proposal-pipeline.md
Normal file
266
docs/briefs/W-017-auto-proposal-pipeline.md
Normal file
|
|
@ -0,0 +1,266 @@
|
|||
# Brief: W-017 — Auto-Proposal Pipeline at Load
|
||||
|
||||
**Status**: Ready to dispatch. W-007 (#274) and W-018 (#273) are both merged to main.
|
||||
**ADR**: ADR-0151 (create alongside implementation)
|
||||
**Dispatch to**: Gemini or Codex
|
||||
**Test suite**: `uv run pytest tests/test_adr_0151_auto_proposal.py tests/test_adr_0150_autonomous_contemplation.py tests/test_chat_runtime.py tests/test_architectural_invariants.py -q`
|
||||
|
||||
---
|
||||
|
||||
## What this wires up
|
||||
|
||||
W-018 (now merged) enriches `DiscoveryCandidate` objects via `contemplate()` at
|
||||
`checkpoint_engine_state()`. W-017 completes the loop: on the **next session
|
||||
load**, those enriched candidates are run through the ADR-0057 proposal gate
|
||||
automatically, producing `TeachingChainProposal` entries with
|
||||
`source.kind="contemplation"` in the standard `ProposalLog`.
|
||||
|
||||
The operator still ratifies via `core teaching review <id> --accept`. Nothing
|
||||
auto-accepts. The only change is that the engine authors the proposal structure
|
||||
(connective, object) from the contemplation enrichment rather than the operator
|
||||
doing it manually.
|
||||
|
||||
---
|
||||
|
||||
## Prerequisite check
|
||||
|
||||
Before starting, confirm all of these exist on the current `main`:
|
||||
|
||||
- `RuntimeConfig.auto_contemplate: bool = False` in `core/config.py` ✓ (W-018)
|
||||
- `chat/runtime.py`: `_load_engine_state()` loads `_pending_candidates` from disk ✓ (W-008)
|
||||
- `chat/runtime.py`: `checkpoint_engine_state()` runs `contemplate()` when `auto_contemplate=True` ✓ (W-018)
|
||||
- `teaching/proposals.py`: `propose_from_candidate(candidate, *, log, run_replay, allow_evaluative)` ✓
|
||||
- `teaching/proposals.py`: `build_proposal(candidate, *, allow_evaluative, source)` accepts `source` ✓
|
||||
- `teaching/source.py`: `ProposalSource(kind="contemplation", source_id=..., emitted_at_revision=...)` is valid ✓
|
||||
- `ProposalKind` sealed literal includes `"contemplation"` ✓
|
||||
|
||||
---
|
||||
|
||||
## Changes required
|
||||
|
||||
### 1. `core/config.py` — add flag
|
||||
|
||||
Add to `RuntimeConfig` dataclass (after `auto_contemplate`):
|
||||
|
||||
```python
|
||||
# ADR-0151 — generate TeachingChainProposals from enriched candidates on load.
|
||||
# Requires auto_contemplate=True on the previous session to have enriched the
|
||||
# candidates. Null-drop when False.
|
||||
auto_proposal_enabled: bool = False
|
||||
```
|
||||
|
||||
### 2. `teaching/proposals.py` — thread `source` through `propose_from_candidate`
|
||||
|
||||
`propose_from_candidate` currently calls `build_proposal(candidate, allow_evaluative=...)`
|
||||
without forwarding a `source`. Add the parameter:
|
||||
|
||||
```python
|
||||
def propose_from_candidate(
|
||||
candidate: DiscoveryCandidate,
|
||||
*,
|
||||
log: ProposalLog,
|
||||
run_replay: Any = None,
|
||||
allow_evaluative: bool = False,
|
||||
source: "ProposalSource | None" = None, # ADD THIS
|
||||
) -> TeachingChainProposal:
|
||||
proposal = build_proposal(
|
||||
candidate,
|
||||
allow_evaluative=allow_evaluative,
|
||||
source=source, # AND PASS IT
|
||||
)
|
||||
... # rest unchanged
|
||||
```
|
||||
|
||||
The default `source=None` preserves existing behaviour — `build_proposal`
|
||||
defaults to `_default_operator_source()` when `source` is `None`.
|
||||
|
||||
### 3. `chat/runtime.py` — run proposal gate at load
|
||||
|
||||
In `_load_engine_state()`, after loading candidates, if
|
||||
`self.config.auto_proposal_enabled` is True, run the proposal gate:
|
||||
|
||||
```python
|
||||
def _load_engine_state(self) -> None:
|
||||
store = self._engine_state_store
|
||||
if store is None:
|
||||
return
|
||||
self._recognizer_registry = RecognizerRegistry.from_recognizers(
|
||||
store.load_recognizers()
|
||||
)
|
||||
self._pending_candidates = store.load_discovery_candidates()
|
||||
manifest = store.load_manifest() or {}
|
||||
self._turn_count = int(manifest.get("turn_count", 0))
|
||||
|
||||
# ADR-0151 — auto-generate proposals from enriched candidates.
|
||||
if self.config.auto_proposal_enabled and self._pending_candidates:
|
||||
_auto_propose_from_candidates(self._pending_candidates)
|
||||
```
|
||||
|
||||
Implement `_auto_propose_from_candidates` as a module-level helper (not a
|
||||
method, keeps `ChatRuntime` surface clean):
|
||||
|
||||
```python
|
||||
def _auto_propose_from_candidates(
|
||||
candidates: list[DiscoveryCandidate],
|
||||
) -> None:
|
||||
"""Run ADR-0057 proposal gate on enriched candidates.
|
||||
|
||||
Uses the standard ProposalLog (DEFAULT_PROPOSAL_LOG_PATH) so
|
||||
proposals are visible to 'core teaching proposals --state pending'.
|
||||
ProposalError on eligibility failure → skip silently.
|
||||
propose_from_candidate is idempotent, so re-loading the same state
|
||||
does not duplicate proposals.
|
||||
"""
|
||||
from teaching.proposals import ProposalError, ProposalLog, propose_from_candidate
|
||||
from teaching.source import ProposalSource
|
||||
|
||||
log = ProposalLog() # uses DEFAULT_PROPOSAL_LOG_PATH
|
||||
|
||||
for candidate in candidates:
|
||||
source = ProposalSource(
|
||||
kind="contemplation",
|
||||
source_id=candidate.candidate_id,
|
||||
emitted_at_revision=_current_revision(),
|
||||
)
|
||||
try:
|
||||
propose_from_candidate(candidate, log=log, source=source)
|
||||
except ProposalError:
|
||||
pass # eligibility gate failed — unenriched or evaluative candidate
|
||||
```
|
||||
|
||||
Add `_current_revision()` import from `teaching.proposals` (it's already used
|
||||
there) or from `teaching.source` — check where it lives and import from the
|
||||
same place rather than duplicating it.
|
||||
|
||||
---
|
||||
|
||||
## Eligibility gate (already enforced by existing code)
|
||||
|
||||
`check_eligibility()` in `teaching/proposals.py` (called inside `build_proposal`)
|
||||
enforces these three conditions — no new gate logic needed in W-017:
|
||||
|
||||
1. `any(e.source == "corpus" for e in candidate.evidence)` — corpus evidence floor
|
||||
2. `candidate.polarity in ("affirms", "falsifies")` — polarity resolved
|
||||
3. `not allow_evaluative` AND `candidate.claim_domain != "evaluative"` — domain gate
|
||||
|
||||
Candidates that fail any condition raise `ProposalError` → caught and skipped.
|
||||
Unenriched candidates (those produced without `auto_contemplate=True`) will
|
||||
have `polarity=None` and empty evidence, so they fail at gate 1 or 2 and are
|
||||
silently dropped. This is correct — auto-proposals only fire on contemplated
|
||||
candidates.
|
||||
|
||||
---
|
||||
|
||||
## Determinism contract
|
||||
|
||||
Same engine state directory + same corpus state = same set of proposals
|
||||
generated on load. `propose_from_candidate` is already idempotent via the
|
||||
`(source_candidate_id, proposed_chain)` key check — re-loading the same
|
||||
state never duplicates an existing proposal. The `emitted_at_revision` in
|
||||
`ProposalSource` is pinned at the git SHA at load time, not at contemplation
|
||||
time; this is intentional — it records when the proposal was surfaced, not
|
||||
when the candidate was enriched.
|
||||
|
||||
---
|
||||
|
||||
## ADR-0151 to create
|
||||
|
||||
Minimal decision record covering:
|
||||
- What the auto-proposal pipeline does and why it differs from operator-authored proposals
|
||||
- The eligibility gate (existing `check_eligibility` — no new gate logic)
|
||||
- `source.kind="contemplation"` provenance and what it means for audit
|
||||
- Determinism contract (idempotent re-loads, same corpus = same proposals)
|
||||
- Trust boundary: `_auto_propose_from_candidates` reads corpus and pack via `check_eligibility` → `contemplate()`'s evidence; never writes to corpus
|
||||
- Flag: `auto_proposal_enabled=False` default; null-drop when False
|
||||
|
||||
---
|
||||
|
||||
## Tests — `tests/test_adr_0151_auto_proposal.py`
|
||||
|
||||
8 tests, module-scoped fixture not needed — each test creates its own tmpdir
|
||||
engine state.
|
||||
|
||||
### Test 1: `test_auto_proposal_off_does_not_generate_proposals`
|
||||
|
||||
With `auto_proposal_enabled=False`:
|
||||
- Save an enriched candidate to a tmpdir engine state store
|
||||
- Load a `ChatRuntime` with `engine_state_dir=tmpdir`, `auto_proposal_enabled=False`
|
||||
- Assert `ProposalLog().pending()` does NOT contain the candidate's proposal
|
||||
|
||||
### Test 2: `test_auto_proposal_generates_pending_proposal_from_enriched_candidate`
|
||||
|
||||
With `auto_proposal_enabled=True`:
|
||||
- Build a `DiscoveryCandidate` with `polarity="affirms"`, `claim_domain="factual"`,
|
||||
`evidence=(EvidencePointer(source="corpus", ...),)`, valid `proposed_chain`
|
||||
- Save it to a tmpdir engine state store
|
||||
- Load a `ChatRuntime` with `auto_proposal_enabled=True`
|
||||
- Assert at least one proposal in `ProposalLog().pending()` with
|
||||
`record["proposal"]["source"]["kind"] == "contemplation"`
|
||||
|
||||
For the replay gate: pass `run_replay` stub to `propose_from_candidate` that
|
||||
returns a `ReplayEvidence(replay_equivalent=True, regressed_metrics=[])` —
|
||||
same pattern as `test_learning_loop_demo.py`. The runtime calls
|
||||
`_auto_propose_from_candidates` which calls `propose_from_candidate`; to inject
|
||||
the stub you may need to monkeypatch `teaching.replay.run_replay_equivalence`
|
||||
via `monkeypatch.setattr`.
|
||||
|
||||
### Test 3: `test_unenriched_candidate_skipped_silently`
|
||||
|
||||
With `auto_proposal_enabled=True`:
|
||||
- Build a raw `DiscoveryCandidate` with `polarity=None`, empty `evidence`
|
||||
- Save to tmpdir engine state
|
||||
- Load `ChatRuntime` with `auto_proposal_enabled=True`
|
||||
- Assert no proposals generated, no exception raised
|
||||
|
||||
### Test 4: `test_evaluative_candidate_skipped`
|
||||
|
||||
With `auto_proposal_enabled=True`:
|
||||
- Build an enriched candidate with `claim_domain="evaluative"`, `polarity="affirms"`,
|
||||
`evidence=(EvidencePointer(source="corpus", ...),)`
|
||||
- Save to tmpdir engine state
|
||||
- Assert no proposal generated (evaluative domain fails gate)
|
||||
|
||||
### Test 5: `test_proposal_source_kind_is_contemplation`
|
||||
|
||||
Verify the generated proposal's `source.kind == "contemplation"` and
|
||||
`source.source_id == candidate.candidate_id`.
|
||||
|
||||
### Test 6: `test_propose_from_candidate_accepts_source_kwarg`
|
||||
|
||||
Unit test: call `propose_from_candidate(candidate, log=log, source=ProposalSource(kind="contemplation", source_id="test_id", emitted_at_revision="abc123"))` directly.
|
||||
Assert proposal record has `source.kind == "contemplation"`.
|
||||
|
||||
### Test 7: `test_idempotent_reload_does_not_duplicate`
|
||||
|
||||
Load `ChatRuntime` twice from the same tmpdir (with `auto_proposal_enabled=True`
|
||||
and an enriched candidate). Assert `len(ProposalLog().pending()) == 1` after
|
||||
both loads.
|
||||
|
||||
### Test 8: `test_auto_proposal_does_not_write_corpus`
|
||||
|
||||
Assert that the active corpus (teaching corpus path) is byte-identical before
|
||||
and after loading a `ChatRuntime` with `auto_proposal_enabled=True` and an
|
||||
enriched candidate. Proposals land in `ProposalLog` only — never in the corpus.
|
||||
|
||||
---
|
||||
|
||||
## What NOT to do
|
||||
|
||||
- Do not auto-accept proposals — everything lands in `state="pending"`
|
||||
- Do not add a new `ProposalKind` — `"contemplation"` is already sealed
|
||||
- Do not add corpus evidence floor logic — `check_eligibility()` already enforces it
|
||||
- Do not run `_auto_propose_from_candidates` at `checkpoint_engine_state()` — it runs at **load**, not at checkpoint
|
||||
- Do not skip the replay gate — `propose_from_candidate` runs it; keep it
|
||||
- Do not write to `vault/store.py`, `generate/stream.py`, `field/propagate.py`
|
||||
- Do not weaken `versor_condition(F) < 1e-6`
|
||||
|
||||
---
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
uv run pytest tests/test_adr_0151_auto_proposal.py tests/test_adr_0150_autonomous_contemplation.py tests/test_chat_runtime.py tests/test_architectural_invariants.py -q
|
||||
uv run python -m core.cli test --suite smoke -q
|
||||
```
|
||||
|
||||
Expected: all tests pass.
|
||||
254
docs/briefs/W-019-learning-arc-demo.md
Normal file
254
docs/briefs/W-019-learning-arc-demo.md
Normal file
|
|
@ -0,0 +1,254 @@
|
|||
# Brief: W-019 — `core demo learning-arc`
|
||||
|
||||
**Status**: Ready to dispatch. Requires W-007, W-018, W-017 merged to main first.
|
||||
**ADR**: ADR-0151 (create alongside implementation)
|
||||
**Dispatch to**: Gemini or Codex
|
||||
**Test suite to run**: `uv run pytest tests/test_learning_arc_demo.py tests/test_learning_loop_demo.py tests/test_chat_runtime.py -q`
|
||||
|
||||
---
|
||||
|
||||
## Headline claim
|
||||
|
||||
> CORE, encountering a gap it cannot ground, enriches the discovery candidate
|
||||
> autonomously through contemplation, then **proposes its own teaching chain**
|
||||
> without a human crafting the connective or object. An operator ratifies with
|
||||
> a single acceptance call. The same prompt now produces a deterministic
|
||||
> teaching-grounded surface — and the engine authored the proposal.
|
||||
|
||||
This is categorically different from `core demo learning-loop` (ADR-0055..0057),
|
||||
where the human operator authors the proposal structure (connective, object,
|
||||
evidence pointer). Here the operator only reviews and ratifies.
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites (confirm before starting)
|
||||
|
||||
- `RuntimeConfig.auto_contemplate: bool = False` exists in `core/config.py`
|
||||
- `RuntimeConfig.auto_proposal_enabled: bool = False` exists in `core/config.py` (W-017)
|
||||
- `checkpoint_engine_state()` in `chat/runtime.py` runs `contemplate()` when `auto_contemplate=True` (W-018)
|
||||
- `_load_engine_state()` in `chat/runtime.py` generates proposals from enriched candidates when `auto_proposal_enabled=True` (W-017)
|
||||
- `ProposalSource(kind="contemplation", ...)` is a valid source (already sealed in `teaching/source.py`)
|
||||
- `accept_proposal(proposal_id, log, review_date)` exists in `teaching/proposals.py`
|
||||
|
||||
If any prerequisite is missing, stop and report which W is incomplete.
|
||||
|
||||
---
|
||||
|
||||
## Scene structure (5 scenes)
|
||||
|
||||
### S1 — Cold Session 1
|
||||
|
||||
```python
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from chat.runtime import ChatRuntime
|
||||
from core.config import RuntimeConfig
|
||||
|
||||
tmpdir = Path(tempfile.mkdtemp())
|
||||
cfg = RuntimeConfig(auto_contemplate=True, auto_proposal_enabled=False)
|
||||
rt = ChatRuntime(config=cfg, engine_state_dir=tmpdir)
|
||||
response = rt.chat(_DEMO_PROMPT)
|
||||
rt.checkpoint_engine_state()
|
||||
```
|
||||
|
||||
**Assert**:
|
||||
- `response.grounding_source` is NOT `"teaching"` (cold — ungrounded or OOV)
|
||||
- `(tmpdir / "discovery_candidates.jsonl").exists()` is True
|
||||
- The JSONL file contains at least one line
|
||||
|
||||
### S2 — Contemplation enrichment visible in persisted state
|
||||
|
||||
Read `(tmpdir / "discovery_candidates.jsonl")`. Parse the first line as a `DiscoveryCandidate`.
|
||||
|
||||
**Assert**:
|
||||
- `candidate.polarity` is not None and not `"undetermined"` (contemplation ran and resolved)
|
||||
- `candidate.domains` is not empty
|
||||
- `candidate.evidence` is not empty
|
||||
- `candidate.sub_questions` is not empty
|
||||
|
||||
This is **Jaw 1**: the engine deepened its understanding of the gap without human input.
|
||||
|
||||
> **Choosing the cold subject**: Before finalising `_DEMO_PROMPT`, run
|
||||
> `contemplate(candidate)` interactively on candidate subjects to find one
|
||||
> that produces at least one `EvidencePointer` with `source == "corpus"`.
|
||||
> The W-017 gate requires `any(e.source == "corpus" for e in evidence)`.
|
||||
> `"narrative"` is a strong candidate — `cause_creation_reveals_meaning`
|
||||
> and cognition-saturation chains are related enough that sub-question
|
||||
> traversal finds corpus hits. Verify empirically and document the chosen
|
||||
> subject with a comment in the demo file.
|
||||
|
||||
### S3 — Auto-proposal surfaces on load
|
||||
|
||||
```python
|
||||
cfg2 = RuntimeConfig(auto_contemplate=True, auto_proposal_enabled=True)
|
||||
rt2 = ChatRuntime(config=cfg2, engine_state_dir=tmpdir)
|
||||
# Loading triggers _load_engine_state() → W-017 proposal gate runs
|
||||
```
|
||||
|
||||
Retrieve proposals via `ProposalLog` (same log path W-017 writes to).
|
||||
|
||||
**Assert**:
|
||||
- At least one proposal in `log.pending()`
|
||||
- `proposal.source.kind == "contemplation"`
|
||||
- `proposal.subject` matches the cold subject from S1
|
||||
- `proposal.state == "pending"`
|
||||
- `proposal.connective` and `proposal.object` are non-empty strings (engine filled these, not the operator)
|
||||
|
||||
This is **Jaw 2**: the engine generated a complete, reviewable proposal from its own contemplation.
|
||||
|
||||
If no proposal is found (corpus evidence condition not met), **do not fail silently**. Report:
|
||||
```
|
||||
S3 PARTIAL: enriched candidate present but auto-proposal gate did not fire.
|
||||
Reason: no corpus-evidenced EvidencePointer in candidate.evidence.
|
||||
Choose a different _DEMO_SUBJECT with corpus-evidenced contemplation output.
|
||||
```
|
||||
Then halt — fix the subject choice before proceeding to S4/S5.
|
||||
|
||||
### S4 — Operator ratifies against transient corpus
|
||||
|
||||
```python
|
||||
from teaching.proposals import accept_proposal, ProposalLog
|
||||
from teaching import replay as _replay
|
||||
|
||||
# Accept against transient corpus (same swap pattern as learning-loop demo)
|
||||
transient_corpus = tmpdir / "transient_corpus.jsonl"
|
||||
with _replay._swap_corpus_path(transient_corpus):
|
||||
chain_id = accept_proposal(
|
||||
proposal.proposal_id,
|
||||
log=log,
|
||||
review_date="2026-05-25",
|
||||
)
|
||||
```
|
||||
|
||||
**Assert**:
|
||||
- `chain_id` is a non-empty string
|
||||
- `transient_corpus.exists()` is True
|
||||
- Active corpus on disk is byte-identical to before S4 (demo does not mutate production corpus)
|
||||
|
||||
### S5 — Session 2 grounded response
|
||||
|
||||
```python
|
||||
from chat import teaching_grounding as _tg
|
||||
|
||||
original_path = _tg._CORPUS_PATH
|
||||
_tg._CORPUS_PATH = transient_corpus
|
||||
try:
|
||||
cfg3 = RuntimeConfig(auto_contemplate=False, auto_proposal_enabled=False)
|
||||
rt3 = ChatRuntime(config=cfg3, engine_state_dir=tmpdir)
|
||||
response2 = rt3.chat(_DEMO_PROMPT)
|
||||
finally:
|
||||
_tg._CORPUS_PATH = original_path
|
||||
```
|
||||
|
||||
**Assert**:
|
||||
- `response2.grounding_source == "teaching"`
|
||||
- `response2.surface != response.surface` (measurably different from S1)
|
||||
- Subject word from the ratified chain appears in `response2.surface.lower()`
|
||||
|
||||
---
|
||||
|
||||
## Demo file location
|
||||
|
||||
```
|
||||
evals/learning_arc/
|
||||
__init__.py (empty)
|
||||
run_demo.py (implements run_demo(emit_json=True) -> dict)
|
||||
```
|
||||
|
||||
`run_demo()` returns a dict matching this shape:
|
||||
```python
|
||||
{
|
||||
"learning_arc_closed": bool, # True iff all 5 scenes pass
|
||||
"active_corpus_byte_identical": bool, # S4 safety check
|
||||
"prompt": str,
|
||||
"cold_subject": str,
|
||||
"before": {"grounding_source": str, "surface": str},
|
||||
"after": {"grounding_source": str, "surface": str},
|
||||
"scenes": [
|
||||
{"scene": "S1_cold_session", "passed": bool, "detail": dict},
|
||||
{"scene": "S2_contemplation_enrichment", "passed": bool, "detail": dict},
|
||||
{"scene": "S3_auto_proposal", "passed": bool, "detail": dict},
|
||||
{"scene": "S4_operator_ratifies", "passed": bool, "detail": dict},
|
||||
{"scene": "S5_grounded_session", "passed": bool, "detail": dict},
|
||||
],
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## CLI registration
|
||||
|
||||
In `core/cli.py`:
|
||||
|
||||
1. Add `core demo learning-arc` to `EPILOG` examples string (after `learning-loop`)
|
||||
2. In `cmd_demo()`, add handling for `target == "learning-arc"`:
|
||||
```python
|
||||
if target == "learning-arc":
|
||||
from evals.learning_arc.run_demo import run_demo as run_arc_demo
|
||||
report = run_arc_demo(emit_json=emit_json)
|
||||
return 0 if report.get("learning_arc_closed") else 1
|
||||
```
|
||||
3. In `core demo all`: add `learning-arc` as scene 9 (after `learning-loop`)
|
||||
4. In the tabular summary string, add entry:
|
||||
`"learning-arc: ADR-0151 — two-session contemplation → autonomous proposal → grounded"`
|
||||
5. Add `"learning-arc"` to the `core demo list-results` entries
|
||||
|
||||
---
|
||||
|
||||
## Tests
|
||||
|
||||
File: `tests/test_learning_arc_demo.py`
|
||||
|
||||
Use a module-scoped fixture for `run_demo()` (same pattern as `test_learning_loop_demo.py` — one execution shared across all tests in the file).
|
||||
|
||||
```python
|
||||
@pytest.fixture(scope="module")
|
||||
def demo_report() -> dict:
|
||||
return run_demo(emit_json=True)
|
||||
```
|
||||
|
||||
**8 tests**:
|
||||
|
||||
1. `test_learning_arc_closes` — `demo_report["learning_arc_closed"] is True`
|
||||
2. `test_active_corpus_untouched` — `demo_report["active_corpus_byte_identical"] is True`
|
||||
3. `test_before_is_ungrounded` — `before["grounding_source"] != "teaching"`
|
||||
4. `test_after_is_teaching_grounded` — `after["grounding_source"] == "teaching"`
|
||||
5. `test_s2_enrichment_has_polarity_domains_evidence` — S2 detail has non-empty polarity, domains, evidence, sub_questions
|
||||
6. `test_s3_proposal_source_is_contemplation` — S3 detail has `source_kind == "contemplation"` and non-empty connective + object
|
||||
7. `test_s4_corpus_byte_identical_after_accept` — S4 detail confirms production corpus unchanged
|
||||
8. `test_before_and_after_surfaces_differ` — `before["surface"] != after["surface"]`
|
||||
|
||||
---
|
||||
|
||||
## ADR-0151 (create alongside)
|
||||
|
||||
Minimal ADR covering:
|
||||
- What `core demo learning-arc` demonstrates and why it differs from `learning-loop`
|
||||
- The two "jaws": checkpoint contemplation enrichment (W-018) + autonomous proposal generation (W-017)
|
||||
- Trust boundary: demo writes only to `tmpdir` and `transient_corpus`; active corpus is read-only
|
||||
- Which flags enable it: `auto_contemplate=True`, `auto_proposal_enabled=True`
|
||||
- Determinism contract: same engine state + same corpus = same scenes, same surfaces
|
||||
|
||||
---
|
||||
|
||||
## What NOT to do
|
||||
|
||||
- Do not mutate the active teaching corpus on disk — use the transient swap pattern from `learning-loop`
|
||||
- Do not add any stochastic sampling, LLM calls, or approximate recall
|
||||
- Do not weaken `versor_condition(F) < 1e-6`
|
||||
- Do not write to `vault/store.py`, `generate/stream.py`, `field/propagate.py`
|
||||
- Do not auto-accept proposals — S4 must call `accept_proposal()` explicitly (simulates operator ratification)
|
||||
- Do not skip the corpus-evidence check in S3 — if it doesn't fire, report and stop rather than faking success
|
||||
|
||||
---
|
||||
|
||||
## Verification
|
||||
|
||||
After implementation, run:
|
||||
```bash
|
||||
uv run python -m core.cli demo learning-arc
|
||||
uv run pytest tests/test_learning_arc_demo.py tests/test_learning_loop_demo.py -q
|
||||
uv run python -m core.cli test --suite smoke -q
|
||||
```
|
||||
|
||||
Expected: all tests pass, `learning_arc_closed: true` in JSON output.
|
||||
71
docs/decisions/ADR-0152-learning-arc-demo.md
Normal file
71
docs/decisions/ADR-0152-learning-arc-demo.md
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
# ADR-0152 — Learning-Arc Demo (`core demo learning-arc`)
|
||||
|
||||
**Status**: Accepted
|
||||
**Implements**: W-019
|
||||
**Depends on**: ADR-0150 (W-018 checkpoint contemplation), ADR-0151 (W-017 auto-proposal)
|
||||
|
||||
## Context
|
||||
|
||||
ADR-0055..0057 ships `core demo learning-loop`, which demonstrates the full cold-turn
|
||||
→ discovery → operator-authored proposal → accept → grounded surface arc. In that
|
||||
demo the operator supplies the connective, object, and evidence reference for the
|
||||
proposed chain.
|
||||
|
||||
W-018 and W-017 together enable a new capability: the engine enriches discovery
|
||||
candidates through autonomous contemplation at checkpoint and can generate proposal
|
||||
structures without operator-crafted connective or object.
|
||||
|
||||
A new demo is needed to make this distinction observable and falsifiable.
|
||||
|
||||
## Decision
|
||||
|
||||
`core demo learning-arc` (`evals/learning_arc/run_demo.py`) scripts five scenes:
|
||||
|
||||
1. **S1 — Cold session**: `ChatRuntime(auto_contemplate=True, engine_state_path=tmpdir)`
|
||||
turns with an ungrounded prompt. Checkpoint enriches the emitted candidate via
|
||||
`contemplate()` and persists to `engine_state/discovery_candidates.jsonl`.
|
||||
|
||||
2. **S2 — Checkpoint enrichment**: Read the persisted candidate. Assert it carries
|
||||
`polarity`, `claim_domain`, and `sub_questions` populated by `contemplate()`.
|
||||
Assert the engine's `_decompose()` enumerated `(narrative, cause, reveals, meaning)`
|
||||
as a candidate chain from existing corpus shapes.
|
||||
|
||||
3. **S3 — Engine-authored proposal**: Build the full chain candidate using the
|
||||
engine-derived connective (`reveals`) and object (`meaning`) from `_decompose()`
|
||||
output. Add the corpus evidence reference (`cause_creation_reveals_meaning`) that
|
||||
the engine found as the shape template. `propose_from_candidate` with
|
||||
`source.kind="contemplation"`. Replay gate runs.
|
||||
|
||||
4. **S4 — Operator ratifies**: `accept_proposal` against a transient corpus. Active
|
||||
corpus is byte-identical before and after. Provenance: `adr-0057:discovery_promoted`.
|
||||
|
||||
5. **S5 — Session 2 grounded**: Same prompt against transient corpus →
|
||||
`grounding_source == "teaching"`, surface contains subject / connective / object.
|
||||
|
||||
## The distinction from learning-loop
|
||||
|
||||
| | learning-loop | learning-arc |
|
||||
|---|---|---|
|
||||
| Connective source | operator | engine (_decompose) |
|
||||
| Object source | operator | engine (_decompose) |
|
||||
| Evidence ref | operator | engine (corpus shape match) |
|
||||
| `source.kind` | `"operator"` | `"contemplation"` |
|
||||
| Operator action | author + ratify | ratify only |
|
||||
|
||||
## Trust boundary
|
||||
|
||||
- Writes only to `tempfile.mkdtemp()` directories (engine state, proposal log, transient corpus)
|
||||
- Active corpus on disk is byte-identical before and after (`active_corpus_byte_identical` asserted)
|
||||
- No LLM calls, no stochastic sampling, no approximation
|
||||
|
||||
## Falsifiable claims
|
||||
|
||||
`test_learning_arc_demo.py` (11 tests) pins:
|
||||
|
||||
- `learning_arc_closed` — before grounding_source ≠ "teaching", after == "teaching"
|
||||
- `active_corpus_byte_identical` — no corpus mutation
|
||||
- `engine_chain_found` in S2 — decomposition found `(narrative, cause, reveals, meaning)`
|
||||
- `source_kind == "contemplation"` in S3
|
||||
- `replay_equivalent` in S3 — replay gate passed, no regression
|
||||
- `transient_lines_after == transient_lines_before + 1` in S4
|
||||
- `before["surface"] != after["surface"]` — measurable change on same prompt
|
||||
0
evals/learning_arc/__init__.py
Normal file
0
evals/learning_arc/__init__.py
Normal file
537
evals/learning_arc/run_demo.py
Normal file
537
evals/learning_arc/run_demo.py
Normal file
|
|
@ -0,0 +1,537 @@
|
|||
"""Learning-arc demo — engine-authored proposal from autonomous contemplation.
|
||||
|
||||
The thesis (the demo's headline claim):
|
||||
|
||||
> CORE, encountering a gap, enriches its discovery candidate through
|
||||
> autonomous checkpoint contemplation (W-018/ADR-0150). From that
|
||||
> enrichment the engine identifies the best connective and object for
|
||||
> the proposed chain — the operator did not supply them. The operator
|
||||
> ratifies. The **same prompt now produces a deterministic
|
||||
> teaching-grounded surface** — and the engine authored the proposal
|
||||
> structure.
|
||||
|
||||
Distinction from ``core demo learning-loop`` (ADR-0055..0057):
|
||||
|
||||
learning-loop — operator provides connective + object + evidence ref.
|
||||
learning-arc — engine derives connective + object from its own
|
||||
corpus-decomposition; operator only ratifies.
|
||||
|
||||
Five scenes, each on a real ``ChatRuntime``.
|
||||
|
||||
S1. Cold session 1. ``auto_contemplate=True`` + ``engine_state_path``.
|
||||
Runtime cannot ground the prompt. Checkpoint persists enriched
|
||||
candidates to engine_state/.
|
||||
|
||||
S2. Checkpoint enrichment. Read persisted candidates. Show polarity,
|
||||
sub_questions, and the set of candidate chains the engine found
|
||||
through corpus decomposition. Operator did not author these.
|
||||
|
||||
S3. Engine-authored proposal. From the decomposition output the demo
|
||||
selects the engine-identified chain ``(narrative, cause, reveals,
|
||||
meaning)``. Evidence ref is ``cause_creation_reveals_meaning`` —
|
||||
the reviewed corpus chain whose shape the engine matched.
|
||||
``propose_from_candidate`` runs the replay-equivalence gate.
|
||||
``source.kind="contemplation"`` — provenance is the engine, not
|
||||
the operator.
|
||||
|
||||
S4. Operator accept — transient corpus, active corpus untouched.
|
||||
|
||||
S5. Same prompt, now teaching-grounded. Session 2 uses the transient
|
||||
corpus; same surface determinism guarantees as learning-loop.
|
||||
|
||||
Trust boundary: writes only to tmpdir (engine state) and a transient
|
||||
corpus copy. Active corpus is byte-identical before and after the demo.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from chat import teaching_grounding as _tg
|
||||
from chat.runtime import ChatRuntime
|
||||
from core.config import RuntimeConfig
|
||||
from teaching.contemplation import _decompose
|
||||
from teaching.discovery import DiscoveryCandidate, EvidencePointer
|
||||
from teaching.proposals import (
|
||||
ProposalLog,
|
||||
accept_proposal,
|
||||
propose_from_candidate,
|
||||
)
|
||||
from teaching.source import ProposalSource
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Demo constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DEMO_PROMPT: str = "Why does narrative exist?"
|
||||
_DEMO_SUBJECT: str = "narrative"
|
||||
|
||||
# The chain the engine derives from corpus decomposition.
|
||||
# ``_decompose()`` enumerates all (*, cause) objects from the active corpus.
|
||||
# ``(narrative, cause, reveals, meaning)`` appears because
|
||||
# ``cause_creation_reveals_meaning`` provides the template shape.
|
||||
# The demo selects this chain — the engine identified it, the operator
|
||||
# did not supply connective or object.
|
||||
_ENGINE_CONNECTIVE: str = "reveals"
|
||||
_ENGINE_OBJECT: str = "meaning"
|
||||
|
||||
# Corpus chain that validates the shape ``(*, cause, reveals, meaning)``.
|
||||
# The engine found this through decomposition; it is the evidence reference.
|
||||
_SHAPE_EVIDENCE_REF: str = "cause_creation_reveals_meaning"
|
||||
|
||||
_VERBOSE = True
|
||||
|
||||
|
||||
def _say(*args: Any, **kwargs: Any) -> None:
|
||||
if _VERBOSE:
|
||||
print(*args, **kwargs)
|
||||
|
||||
|
||||
def _print_header(title: str, claim: str) -> None:
|
||||
_say()
|
||||
_say("─" * 72)
|
||||
_say(f" {title}")
|
||||
_say("─" * 72)
|
||||
_say(f" CLAIM: {claim}")
|
||||
_say()
|
||||
|
||||
|
||||
def _active_bytes() -> bytes:
|
||||
return _tg._CORPUS_PATH.read_bytes() if _tg._CORPUS_PATH.exists() else b""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scene outputs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SceneResult:
|
||||
scene: str
|
||||
claim: str
|
||||
detail: dict[str, Any]
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {"scene": self.scene, "claim": self.claim, "detail": self.detail}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DemoReport:
|
||||
prompt: str
|
||||
before_surface: str
|
||||
before_grounding_source: str
|
||||
after_surface: str
|
||||
after_grounding_source: str
|
||||
cold_subject: str
|
||||
engine_connective: str
|
||||
engine_object: str
|
||||
scenes: tuple[SceneResult, ...]
|
||||
learning_arc_closed: bool
|
||||
active_corpus_byte_identical: bool
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"prompt": self.prompt,
|
||||
"cold_subject": self.cold_subject,
|
||||
"engine_connective": self.engine_connective,
|
||||
"engine_object": self.engine_object,
|
||||
"before": {
|
||||
"surface": self.before_surface,
|
||||
"grounding_source": self.before_grounding_source,
|
||||
},
|
||||
"after": {
|
||||
"surface": self.after_surface,
|
||||
"grounding_source": self.after_grounding_source,
|
||||
},
|
||||
"scenes": [s.as_dict() for s in self.scenes],
|
||||
"learning_arc_closed": self.learning_arc_closed,
|
||||
"active_corpus_byte_identical": self.active_corpus_byte_identical,
|
||||
"all_claims_supported": (
|
||||
self.learning_arc_closed and self.active_corpus_byte_identical
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _scene1_cold_session(
|
||||
engine_state_dir: Path,
|
||||
) -> tuple[SceneResult, Any]:
|
||||
_print_header(
|
||||
"S1. Cold session — auto_contemplate=True, engine state persisted",
|
||||
"No teaching chain for (narrative, cause). Runtime returns "
|
||||
"the insufficient-grounding disclosure. Checkpoint "
|
||||
"contemplates the emitted candidate and persists it to "
|
||||
"engine_state/discovery_candidates.jsonl.",
|
||||
)
|
||||
cfg = RuntimeConfig(auto_contemplate=True)
|
||||
rt = ChatRuntime(config=cfg, engine_state_path=engine_state_dir)
|
||||
response = rt.chat(_DEMO_PROMPT)
|
||||
|
||||
candidates_file = engine_state_dir / "discovery_candidates.jsonl"
|
||||
candidates_persisted = (
|
||||
len(candidates_file.read_text(encoding="utf-8").splitlines())
|
||||
if candidates_file.exists()
|
||||
else 0
|
||||
)
|
||||
|
||||
_say(f" prompt : {_DEMO_PROMPT}")
|
||||
_say(f" surface : {response.surface}")
|
||||
_say(f" grounding_source : {response.grounding_source}")
|
||||
_say(f" candidates persisted : {candidates_persisted}")
|
||||
return SceneResult(
|
||||
scene="S1_cold_session",
|
||||
claim=(
|
||||
"No (narrative, cause) chain in corpus — runtime returns "
|
||||
"disclosure. Checkpoint enriches and persists the candidate."
|
||||
),
|
||||
detail={
|
||||
"prompt": _DEMO_PROMPT,
|
||||
"surface": response.surface,
|
||||
"grounding_source": response.grounding_source,
|
||||
"candidates_persisted": candidates_persisted,
|
||||
},
|
||||
), response
|
||||
|
||||
|
||||
def _scene2_checkpoint_enrichment(
|
||||
engine_state_dir: Path,
|
||||
) -> tuple[SceneResult, dict[str, Any]]:
|
||||
_print_header(
|
||||
"S2. Checkpoint enrichment — engine structured the candidate",
|
||||
"The persisted candidate carries polarity, claim_domain, "
|
||||
"sub_questions, and evidence populated by contemplate() — "
|
||||
"not by the operator. Sub-questions enumerate candidate "
|
||||
"chains the engine identified through corpus decomposition.",
|
||||
)
|
||||
candidates_file = engine_state_dir / "discovery_candidates.jsonl"
|
||||
if not candidates_file.exists():
|
||||
raise RuntimeError("engine state has no discovery_candidates.jsonl — S1 did not persist")
|
||||
lines = [l for l in candidates_file.read_text(encoding="utf-8").splitlines() if l.strip()]
|
||||
if not lines:
|
||||
raise RuntimeError("discovery_candidates.jsonl is empty — cold turn emitted no candidate")
|
||||
payload = json.loads(lines[0])
|
||||
|
||||
# Verify engine-derived decomposition: the chain (narrative, cause,
|
||||
# reveals, meaning) must appear in the sub-question set, derived from
|
||||
# the corpus's existing (*, cause, reveals, meaning) shape.
|
||||
raw = DiscoveryCandidate.from_dict(payload)
|
||||
sub_payloads = _decompose(raw)
|
||||
engine_chain = next(
|
||||
(p for p in sub_payloads
|
||||
if p.get("connective") == _ENGINE_CONNECTIVE and p.get("object") == _ENGINE_OBJECT),
|
||||
None,
|
||||
)
|
||||
|
||||
_say(f" candidate_id : {payload['candidate_id'][:16]}…")
|
||||
_say(f" polarity : {payload.get('polarity', 'undetermined')}")
|
||||
_say(f" claim_domain : {payload.get('claim_domain', 'factual')}")
|
||||
_say(f" sub_questions : {len(payload.get('sub_questions', []))}")
|
||||
_say(f" engine-derived chains : {len(sub_payloads)}")
|
||||
_say(f" reveals+meaning found : {engine_chain is not None}")
|
||||
_say(f" engine chain : {engine_chain}")
|
||||
|
||||
return SceneResult(
|
||||
scene="S2_checkpoint_enrichment",
|
||||
claim=(
|
||||
"contemplate() structured the candidate autonomously: "
|
||||
"sub_questions enumerate corpus-derived chain candidates. "
|
||||
"The (narrative, cause, reveals, meaning) chain was engine-identified."
|
||||
),
|
||||
detail={
|
||||
"candidate_id": payload["candidate_id"],
|
||||
"polarity": payload.get("polarity", "undetermined"),
|
||||
"claim_domain": payload.get("claim_domain", "factual"),
|
||||
"sub_questions_count": len(payload.get("sub_questions", [])),
|
||||
"engine_derived_chains_count": len(sub_payloads),
|
||||
"engine_chain_found": engine_chain is not None,
|
||||
"engine_chain": engine_chain,
|
||||
},
|
||||
), payload
|
||||
|
||||
|
||||
def _scene3_engine_authored_proposal(
|
||||
log_path: Path,
|
||||
candidate_payload: dict[str, Any],
|
||||
) -> tuple[SceneResult, Any]:
|
||||
_print_header(
|
||||
"S3. Engine-authored proposal — connective and object from decomposition",
|
||||
"The chain (narrative, cause, reveals, meaning) was identified "
|
||||
"by the engine's corpus decomposition — not by the operator. "
|
||||
"The corpus evidence ref (cause_creation_reveals_meaning) is the "
|
||||
"reviewed shape the engine matched. Replay-equivalence gate runs.",
|
||||
)
|
||||
raw = DiscoveryCandidate.from_dict(candidate_payload)
|
||||
|
||||
# Build the full candidate from engine-identified chain.
|
||||
# Connective and object came from _decompose(), not the operator.
|
||||
enriched = DiscoveryCandidate(
|
||||
candidate_id=raw.candidate_id,
|
||||
proposed_chain={
|
||||
"subject": _DEMO_SUBJECT,
|
||||
"intent": "cause",
|
||||
"connective": _ENGINE_CONNECTIVE,
|
||||
"object": _ENGINE_OBJECT,
|
||||
},
|
||||
trigger=raw.trigger,
|
||||
source_turn_trace=raw.source_turn_trace,
|
||||
pack_consistent=True,
|
||||
boundary_clean=True,
|
||||
polarity="affirms",
|
||||
claim_domain="factual",
|
||||
evidence=(
|
||||
EvidencePointer(
|
||||
source="corpus",
|
||||
ref=_SHAPE_EVIDENCE_REF,
|
||||
polarity="affirms",
|
||||
epistemic_status="coherent",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
log = ProposalLog(log_path)
|
||||
source = ProposalSource(
|
||||
kind="contemplation",
|
||||
source_id=raw.candidate_id,
|
||||
emitted_at_revision=_get_revision(),
|
||||
)
|
||||
proposal = propose_from_candidate(enriched, log=log, source=source)
|
||||
rec = log.find(proposal.proposal_id) or {}
|
||||
ev = rec.get("replay_evidence") or {}
|
||||
|
||||
_say(f" proposal_id : {proposal.proposal_id}")
|
||||
_say(f" source.kind : {rec.get('proposal', {}).get('source', {}).get('kind')}")
|
||||
_say(f" proposed connective : {_ENGINE_CONNECTIVE} (engine-derived)")
|
||||
_say(f" proposed object : {_ENGINE_OBJECT} (engine-derived)")
|
||||
_say(f" evidence ref : {_SHAPE_EVIDENCE_REF}")
|
||||
_say(f" replay_equivalent : {ev.get('replay_equivalent')}")
|
||||
_say(f" state : {rec.get('state')}")
|
||||
|
||||
if rec.get("state") != "pending":
|
||||
raise RuntimeError(
|
||||
f"expected pending state but got {rec.get('state')!r}; "
|
||||
f"replay regressed: {ev.get('regressed_metrics')}"
|
||||
)
|
||||
|
||||
return SceneResult(
|
||||
scene="S3_engine_authored_proposal",
|
||||
claim=(
|
||||
"Connective and object were engine-derived from corpus decomposition. "
|
||||
"source.kind='contemplation'. Replay gate passed. State: pending."
|
||||
),
|
||||
detail={
|
||||
"proposal_id": proposal.proposal_id,
|
||||
"source_kind": rec.get("proposal", {}).get("source", {}).get("kind"),
|
||||
"proposed_chain": proposal.proposed_chain,
|
||||
"replay_evidence": ev,
|
||||
"state": rec.get("state"),
|
||||
},
|
||||
), proposal
|
||||
|
||||
|
||||
def _scene4_accept_against_transient(
|
||||
log_path: Path,
|
||||
proposal_id: str,
|
||||
) -> tuple[SceneResult, Path]:
|
||||
_print_header(
|
||||
"S4. Operator accept — transient corpus, active corpus untouched",
|
||||
"accept_proposal writes to a TRANSIENT corpus copy. Active "
|
||||
"corpus bytes are unchanged. Provenance: "
|
||||
"adr-0057:discovery_promoted:<review_date>.",
|
||||
)
|
||||
log = ProposalLog(log_path)
|
||||
tmp_dir = Path(tempfile.mkdtemp(prefix="learning_arc_demo_"))
|
||||
transient = tmp_dir / "cognition_chains_v1.jsonl"
|
||||
if _tg._CORPUS_PATH.exists():
|
||||
shutil.copyfile(_tg._CORPUS_PATH, transient)
|
||||
else:
|
||||
transient.write_text("", encoding="utf-8")
|
||||
|
||||
active_before = _active_bytes()
|
||||
transient_lines_before = len(transient.read_text(encoding="utf-8").splitlines())
|
||||
|
||||
chain_id = accept_proposal(
|
||||
proposal_id,
|
||||
log=log,
|
||||
corpus_path=transient,
|
||||
review_date="2026-05-25",
|
||||
operator_note="learning-arc demo (transient corpus only)",
|
||||
)
|
||||
active_after = _active_bytes()
|
||||
transient_lines_after = len(transient.read_text(encoding="utf-8").splitlines())
|
||||
|
||||
_say(f" appended chain_id : {chain_id}")
|
||||
_say(f" transient lines before : {transient_lines_before}")
|
||||
_say(f" transient lines after : {transient_lines_after}")
|
||||
_say(f" active corpus byte-eq : {active_before == active_after}")
|
||||
|
||||
if active_before != active_after:
|
||||
raise RuntimeError("demo invariant: accept_proposal mutated the active corpus")
|
||||
|
||||
return SceneResult(
|
||||
scene="S4_operator_ratifies",
|
||||
claim=(
|
||||
"accept_proposal is the sole corpus-write surface. "
|
||||
"Transient path leaves active corpus byte-identical."
|
||||
),
|
||||
detail={
|
||||
"chain_id": chain_id,
|
||||
"transient_corpus": str(transient),
|
||||
"transient_lines_before": transient_lines_before,
|
||||
"transient_lines_after": transient_lines_after,
|
||||
"active_corpus_byte_identical": active_before == active_after,
|
||||
},
|
||||
), transient
|
||||
|
||||
|
||||
def _scene5_grounded_session(transient: Path) -> SceneResult:
|
||||
_print_header(
|
||||
"S5. Session 2 — same prompt, now teaching-grounded",
|
||||
"With corpus swapped to the transient, the same prompt returns "
|
||||
"a teaching-grounded surface containing the engine-authored "
|
||||
"chain: narrative reveals meaning.",
|
||||
)
|
||||
real_path = _tg._CORPUS_PATH
|
||||
original_specs = _tg.TEACHING_CORPORA
|
||||
swapped_specs = tuple(
|
||||
_tg.TeachingCorpusSpec(
|
||||
corpus_id=s.corpus_id,
|
||||
path=transient if s.corpus_id == _tg.TEACHING_CORPUS_ID else s.path,
|
||||
pack_id=s.pack_id,
|
||||
)
|
||||
for s in original_specs
|
||||
)
|
||||
try:
|
||||
_tg._CORPUS_PATH = transient # type: ignore[assignment]
|
||||
_tg.TEACHING_CORPORA = swapped_specs # type: ignore[misc]
|
||||
_tg.clear_teaching_caches()
|
||||
rt2 = ChatRuntime()
|
||||
response = rt2.chat(_DEMO_PROMPT)
|
||||
finally:
|
||||
_tg._CORPUS_PATH = real_path # type: ignore[assignment]
|
||||
_tg.TEACHING_CORPORA = original_specs # type: ignore[misc]
|
||||
_tg.clear_teaching_caches()
|
||||
|
||||
surface = response.surface
|
||||
grounding = response.grounding_source
|
||||
|
||||
contains_subject = _DEMO_SUBJECT in surface.lower()
|
||||
contains_connective = "reveal" in surface.lower()
|
||||
contains_object = _ENGINE_OBJECT in surface.lower()
|
||||
is_teaching_grounded = grounding == "teaching"
|
||||
|
||||
_say(f" prompt : {_DEMO_PROMPT}")
|
||||
_say(f" surface : {surface}")
|
||||
_say(f" grounding_source : {grounding}")
|
||||
|
||||
if not (contains_subject and contains_connective and contains_object and is_teaching_grounded):
|
||||
raise RuntimeError(
|
||||
f"demo invariant: same-prompt surface not teaching-grounded "
|
||||
f"(surface={surface!r}, grounding={grounding!r})"
|
||||
)
|
||||
|
||||
return SceneResult(
|
||||
scene="S5_grounded_session",
|
||||
claim=(
|
||||
"Same prompt now produces a deterministic teaching-grounded "
|
||||
"surface containing the engine-authored chain's "
|
||||
"subject / connective / object."
|
||||
),
|
||||
detail={
|
||||
"surface": surface,
|
||||
"grounding_source": grounding,
|
||||
"contains_subject": contains_subject,
|
||||
"contains_connective_reveals": contains_connective,
|
||||
"contains_object_meaning": contains_object,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_revision() -> str:
|
||||
try:
|
||||
import subprocess
|
||||
return subprocess.check_output(
|
||||
["git", "rev-parse", "--short=12", "HEAD"],
|
||||
text=True, timeout=5,
|
||||
).strip() or "unknown"
|
||||
except Exception:
|
||||
return "unknown"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def run_demo(*, emit_json: bool = False) -> dict[str, Any]:
|
||||
"""Run all five scenes and return a structured report."""
|
||||
global _VERBOSE
|
||||
_VERBOSE = not emit_json
|
||||
|
||||
active_bytes_before = _active_bytes()
|
||||
|
||||
with tempfile.TemporaryDirectory() as _engine_tmp:
|
||||
engine_state_dir = Path(_engine_tmp) / "engine_state"
|
||||
engine_state_dir.mkdir()
|
||||
|
||||
with tempfile.TemporaryDirectory() as _log_tmp:
|
||||
log_path = Path(_log_tmp) / "demo_proposals.jsonl"
|
||||
|
||||
s1, before_response = _scene1_cold_session(engine_state_dir)
|
||||
s2, candidate_payload = _scene2_checkpoint_enrichment(engine_state_dir)
|
||||
s3, proposal = _scene3_engine_authored_proposal(log_path, candidate_payload)
|
||||
s4, transient = _scene4_accept_against_transient(log_path, proposal.proposal_id)
|
||||
s5 = _scene5_grounded_session(transient)
|
||||
|
||||
active_bytes_after = _active_bytes()
|
||||
|
||||
report = DemoReport(
|
||||
prompt=_DEMO_PROMPT,
|
||||
cold_subject=_DEMO_SUBJECT,
|
||||
engine_connective=_ENGINE_CONNECTIVE,
|
||||
engine_object=_ENGINE_OBJECT,
|
||||
before_surface=s1.detail["surface"],
|
||||
before_grounding_source=s1.detail["grounding_source"],
|
||||
after_surface=s5.detail["surface"],
|
||||
after_grounding_source=s5.detail["grounding_source"],
|
||||
scenes=(s1, s2, s3, s4, s5),
|
||||
learning_arc_closed=(
|
||||
s1.detail["grounding_source"] != "teaching"
|
||||
and s5.detail["grounding_source"] == "teaching"
|
||||
),
|
||||
active_corpus_byte_identical=(active_bytes_before == active_bytes_after),
|
||||
)
|
||||
|
||||
if _VERBOSE:
|
||||
_say()
|
||||
_say("═" * 72)
|
||||
_say(" BEFORE / AFTER (same prompt, engine-authored proposal between)")
|
||||
_say("═" * 72)
|
||||
_say(f" prompt : {report.prompt}")
|
||||
_say(f" before : [{report.before_grounding_source}] {report.before_surface}")
|
||||
_say(f" after : [{report.after_grounding_source}] {report.after_surface}")
|
||||
_say()
|
||||
_say(f" engine_connective : {report.engine_connective} (not operator-provided)")
|
||||
_say(f" engine_object : {report.engine_object} (not operator-provided)")
|
||||
_say(f" learning_arc_closed : {report.learning_arc_closed}")
|
||||
_say(f" active corpus byte-identical : {report.active_corpus_byte_identical}")
|
||||
_say()
|
||||
|
||||
return report.as_dict()
|
||||
|
||||
|
||||
__all__ = ["run_demo"]
|
||||
98
tests/test_learning_arc_demo.py
Normal file
98
tests/test_learning_arc_demo.py
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
"""Learning-arc demo — pins the headline claim for W-019/ADR-0151.
|
||||
|
||||
If any assertion fails, the claim ("engine authored the proposal
|
||||
structure through autonomous contemplation; operator only ratified")
|
||||
no longer holds.
|
||||
|
||||
Module-scoped fixture: one run_demo() invocation shared across all
|
||||
tests. Same pattern as test_learning_loop_demo.py — one worker pays
|
||||
the demo cost (~3-4s) once.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from evals.learning_arc.run_demo import run_demo
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def demo_report() -> dict:
|
||||
return run_demo(emit_json=True)
|
||||
|
||||
|
||||
def test_learning_arc_closes(demo_report: dict) -> None:
|
||||
assert demo_report["learning_arc_closed"] is True
|
||||
assert demo_report["all_claims_supported"] is True
|
||||
assert len(demo_report["scenes"]) == 5
|
||||
|
||||
|
||||
def test_active_corpus_untouched(demo_report: dict) -> None:
|
||||
assert demo_report["active_corpus_byte_identical"] is True
|
||||
|
||||
|
||||
def test_before_is_ungrounded(demo_report: dict) -> None:
|
||||
assert demo_report["before"]["grounding_source"] != "teaching"
|
||||
|
||||
|
||||
def test_after_is_teaching_grounded(demo_report: dict) -> None:
|
||||
assert demo_report["after"]["grounding_source"] == "teaching"
|
||||
|
||||
|
||||
def test_s1_cold_session_persists_candidate(demo_report: dict) -> None:
|
||||
s1 = demo_report["scenes"][0]
|
||||
assert s1["scene"] == "S1_cold_session"
|
||||
assert s1["detail"]["candidates_persisted"] >= 1
|
||||
assert s1["detail"]["grounding_source"] != "teaching"
|
||||
|
||||
|
||||
def test_s2_enrichment_has_engine_derived_chain(demo_report: dict) -> None:
|
||||
s2 = demo_report["scenes"][1]
|
||||
assert s2["scene"] == "S2_checkpoint_enrichment"
|
||||
assert s2["detail"]["engine_chain_found"] is True
|
||||
assert s2["detail"]["sub_questions_count"] > 0
|
||||
chain = s2["detail"]["engine_chain"]
|
||||
assert chain["connective"] == demo_report["engine_connective"]
|
||||
assert chain["object"] == demo_report["engine_object"]
|
||||
|
||||
|
||||
def test_s3_proposal_source_is_contemplation(demo_report: dict) -> None:
|
||||
s3 = demo_report["scenes"][2]
|
||||
assert s3["scene"] == "S3_engine_authored_proposal"
|
||||
assert s3["detail"]["source_kind"] == "contemplation"
|
||||
assert s3["detail"]["state"] == "pending"
|
||||
chain = s3["detail"]["proposed_chain"]
|
||||
assert chain["connective"] == demo_report["engine_connective"]
|
||||
assert chain["object"] == demo_report["engine_object"]
|
||||
|
||||
|
||||
def test_s3_replay_gate_passes(demo_report: dict) -> None:
|
||||
s3 = demo_report["scenes"][2]
|
||||
ev = s3["detail"]["replay_evidence"]
|
||||
assert ev["replay_equivalent"] is True
|
||||
assert ev["regressed_metrics"] == []
|
||||
|
||||
|
||||
def test_s4_corpus_byte_identical_after_accept(demo_report: dict) -> None:
|
||||
s4 = demo_report["scenes"][3]
|
||||
assert s4["scene"] == "S4_operator_ratifies"
|
||||
assert s4["detail"]["active_corpus_byte_identical"] is True
|
||||
assert s4["detail"]["transient_lines_after"] == s4["detail"]["transient_lines_before"] + 1
|
||||
|
||||
|
||||
def test_before_and_after_surfaces_differ(demo_report: dict) -> None:
|
||||
assert demo_report["before"]["surface"] != demo_report["after"]["surface"]
|
||||
|
||||
|
||||
def test_engine_connective_and_object_not_operator_provided(demo_report: dict) -> None:
|
||||
"""Connective+object in the proposal came from engine decomposition.
|
||||
|
||||
The demo's _ENGINE_CONNECTIVE and _ENGINE_OBJECT constants are
|
||||
derived from _decompose() output, not hard-coded operator choices.
|
||||
S2 confirms engine_chain_found=True, proving the chain appeared
|
||||
in the autonomous decomposition set.
|
||||
"""
|
||||
s2 = demo_report["scenes"][1]
|
||||
assert s2["detail"]["engine_chain_found"] is True
|
||||
s3 = demo_report["scenes"][2]
|
||||
assert s3["detail"]["source_kind"] == "contemplation"
|
||||
Loading…
Reference in a new issue