From e7e28a2fd5d6c2d60fda96442cfe9d9dd8259d09 Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 25 May 2026 13:03:10 -0700 Subject: [PATCH] =?UTF-8?q?feat(W-019):=20learning-arc=20demo=20=E2=80=94?= =?UTF-8?q?=20engine-authored=20proposal=20from=20contemplation=20(ADR-015?= =?UTF-8?q?2)=20(#276)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two-session arc where engine derives connective+object from corpus decomposition; operator ratifies rather than authors. Distinguishes from learning-loop (operator-authored) and directly exercises W-018 checkpoint contemplation and W-017 auto-proposal provenance path. --- core/cli.py | 31 +- docs/briefs/W-017-auto-proposal-pipeline.md | 266 +++++++++ docs/briefs/W-019-learning-arc-demo.md | 254 +++++++++ docs/decisions/ADR-0152-learning-arc-demo.md | 71 +++ evals/learning_arc/__init__.py | 0 evals/learning_arc/run_demo.py | 537 +++++++++++++++++++ tests/test_learning_arc_demo.py | 98 ++++ 7 files changed, 1252 insertions(+), 5 deletions(-) create mode 100644 docs/briefs/W-017-auto-proposal-pipeline.md create mode 100644 docs/briefs/W-019-learning-arc-demo.md create mode 100644 docs/decisions/ADR-0152-learning-arc-demo.md create mode 100644 evals/learning_arc/__init__.py create mode 100644 evals/learning_arc/run_demo.py create mode 100644 tests/test_learning_arc_demo.py diff --git a/core/cli.py b/core/cli.py index dc0ba2ae..032f0726 100644 --- a/core/cli.py +++ b/core/cli.py @@ -23,7 +23,7 @@ _CORE_RS_DIR = _REPO_ROOT / "core-rs" _CORE_RS_MANIFEST = _CORE_RS_DIR / "Cargo.toml" DESCRIPTION = "CORE versor engine command suite." -EPILOG = "Examples:\n core chat\n core pulse \"What is truth?\"\n core pulse --no-glove --json \"Compare knowledge and wisdom\"\n core bench\n core bench --suite all\n core bench --suite all --json --report bench_all.json\n core bench --suite determinism --runs 50\n core bench --suite speedup --json\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core teaching audit\n core teaching audit --json\n core teaching gaps --top 10\n core teaching queue --threshold 3\n core teaching propose \n core teaching proposals --state pending\n core teaching review --accept --review-date 2026-05-18\n core teaching supersede cause_light_reveals_truth --subject light --intent cause --connective grounds --object truth --review-date 2026-05-18\n core teaching supersessions\n core teaching supersessions --json\n core test --suite fast -q\n core test --suite pulse -q\n core test --suite proof -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core demo audit-tour\n core demo register-tour\n core demo anchor-lens-tour\n core demo orthogonality-tour\n core demo pack-measurements\n core demo long-context-comparison\n core demo anti-regression\n core demo learning-loop\n core demo articulation\n core demo conversation\n core demo conversation --no-stream\n core demo all\n core demo adr-0024-chain\n core eval --list\n core eval cognition\n core eval cognition --json --save\n core eval cognition --split dev --version v1\n core eval cognition --split holdout" +EPILOG = "Examples:\n core chat\n core pulse \"What is truth?\"\n core pulse --no-glove --json \"Compare knowledge and wisdom\"\n core bench\n core bench --suite all\n core bench --suite all --json --report bench_all.json\n core bench --suite determinism --runs 50\n core bench --suite speedup --json\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core teaching audit\n core teaching audit --json\n core teaching gaps --top 10\n core teaching queue --threshold 3\n core teaching propose \n core teaching proposals --state pending\n core teaching review --accept --review-date 2026-05-18\n core teaching supersede cause_light_reveals_truth --subject light --intent cause --connective grounds --object truth --review-date 2026-05-18\n core teaching supersessions\n core teaching supersessions --json\n core test --suite fast -q\n core test --suite pulse -q\n core test --suite proof -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core demo audit-tour\n core demo register-tour\n core demo anchor-lens-tour\n core demo orthogonality-tour\n core demo pack-measurements\n core demo long-context-comparison\n core demo anti-regression\n core demo learning-loop\n core demo learning-arc\n core demo articulation\n core demo conversation\n core demo conversation --no-stream\n core demo all\n core demo adr-0024-chain\n core eval --list\n core eval cognition\n core eval cognition --json --save\n core eval cognition --split dev --version v1\n core eval cognition --split holdout" _TEST_SUITES: dict[str, tuple[str, ...]] = { "fast": ( @@ -2343,7 +2343,8 @@ table. This is the "show me everything" entry point. 5. long-context-comparison — exact NIAH vs transformer baselines (ADR-0045) 6. anti-regression — three-gate defense (ADR-0057) 7. learning-loop — cold turn → grounded surface (ADR-0055..0057) - 8. articulation — discourse-planner spine (multi-sentence) + 8. learning-arc — engine-authored proposal via contemplation (ADR-0150..0151) + 9. articulation — discourse-planner spine (multi-sentence) Each demo retains its own preamble + report. The final summary surfaces one boolean per demo and an overall ``all_demos_passed`` flag. @@ -2658,6 +2659,14 @@ def cmd_demo(args: argparse.Namespace) -> int: print(json.dumps(report, indent=2, sort_keys=True)) return 0 + if target == "learning-arc": + from evals.learning_arc.run_demo import run_demo as run_arc_demo + + report = run_arc_demo(emit_json=args.json) + if args.json: + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + if target == "articulation": from evals.articulation.run_demo import run_demo as run_articulation_demo @@ -2862,7 +2871,7 @@ def _run_demo_all(emit_json: bool) -> int: passed["anti_regression"] = bool(ar_report.get("all_gates_held", False)) # 7. learning-loop - _section("7/8 learning-loop — cold turn → grounded surface") + _section("7/9 learning-loop — cold turn → grounded surface") from evals.learning_loop.run_demo import run_demo as run_loop if not emit_json: _print_preamble(_LEARNING_LOOP_PREAMBLE) @@ -2871,8 +2880,16 @@ def _run_demo_all(emit_json: bool) -> int: consolidated["learning_loop"] = ll_report passed["learning_loop"] = bool(ll_report.get("learning_loop_closed", False)) - # 8. articulation - _section("8/8 articulation — discourse-planner spine") + # 8. learning-arc + _section("8/9 learning-arc — engine-authored proposal via contemplation") + from evals.learning_arc.run_demo import run_demo as run_arc + with _maybe_suppress(): + arc_report = run_arc(emit_json=emit_json) + consolidated["learning_arc"] = arc_report + passed["learning_arc"] = bool(arc_report.get("learning_arc_closed", False)) + + # 9. articulation + _section("9/9 articulation — discourse-planner spine") from evals.articulation.run_demo import run_demo as run_art if not emit_json: _print_preamble(_ARTICULATION_PREAMBLE) @@ -3717,6 +3734,7 @@ def build_parser() -> argparse.ArgumentParser: "long-context-comparison", "anti-regression", "learning-loop", + "learning-arc", "articulation", "conversation", "showcase", @@ -3750,6 +3768,9 @@ def build_parser() -> argparse.ArgumentParser: "harmful chains (eligibility / replay-equivalence / operator). " "learning-loop: ADR-0055..0057 — full cold-turn → discovery → " "propose → accept → same-prompt-now-grounded walkthrough. " + "learning-arc: ADR-0150..0151 — two-session arc: checkpoint " + "contemplation enriches candidate, engine derives connective + " + "object from corpus decomposition, operator only ratifies. " "articulation: discourse-planner spine — EXPLAIN / COMPOUND / " "WALKTHROUGH multi-sentence articulation + determinism gate. " "conversation: layperson-facing chat transcript with live " diff --git a/docs/briefs/W-017-auto-proposal-pipeline.md b/docs/briefs/W-017-auto-proposal-pipeline.md new file mode 100644 index 00000000..c8fa328f --- /dev/null +++ b/docs/briefs/W-017-auto-proposal-pipeline.md @@ -0,0 +1,266 @@ +# Brief: W-017 — Auto-Proposal Pipeline at Load + +**Status**: Ready to dispatch. W-007 (#274) and W-018 (#273) are both merged to main. +**ADR**: ADR-0151 (create alongside implementation) +**Dispatch to**: Gemini or Codex +**Test suite**: `uv run pytest tests/test_adr_0151_auto_proposal.py tests/test_adr_0150_autonomous_contemplation.py tests/test_chat_runtime.py tests/test_architectural_invariants.py -q` + +--- + +## What this wires up + +W-018 (now merged) enriches `DiscoveryCandidate` objects via `contemplate()` at +`checkpoint_engine_state()`. W-017 completes the loop: on the **next session +load**, those enriched candidates are run through the ADR-0057 proposal gate +automatically, producing `TeachingChainProposal` entries with +`source.kind="contemplation"` in the standard `ProposalLog`. + +The operator still ratifies via `core teaching review --accept`. Nothing +auto-accepts. The only change is that the engine authors the proposal structure +(connective, object) from the contemplation enrichment rather than the operator +doing it manually. + +--- + +## Prerequisite check + +Before starting, confirm all of these exist on the current `main`: + +- `RuntimeConfig.auto_contemplate: bool = False` in `core/config.py` ✓ (W-018) +- `chat/runtime.py`: `_load_engine_state()` loads `_pending_candidates` from disk ✓ (W-008) +- `chat/runtime.py`: `checkpoint_engine_state()` runs `contemplate()` when `auto_contemplate=True` ✓ (W-018) +- `teaching/proposals.py`: `propose_from_candidate(candidate, *, log, run_replay, allow_evaluative)` ✓ +- `teaching/proposals.py`: `build_proposal(candidate, *, allow_evaluative, source)` accepts `source` ✓ +- `teaching/source.py`: `ProposalSource(kind="contemplation", source_id=..., emitted_at_revision=...)` is valid ✓ +- `ProposalKind` sealed literal includes `"contemplation"` ✓ + +--- + +## Changes required + +### 1. `core/config.py` — add flag + +Add to `RuntimeConfig` dataclass (after `auto_contemplate`): + +```python +# ADR-0151 — generate TeachingChainProposals from enriched candidates on load. +# Requires auto_contemplate=True on the previous session to have enriched the +# candidates. Null-drop when False. +auto_proposal_enabled: bool = False +``` + +### 2. `teaching/proposals.py` — thread `source` through `propose_from_candidate` + +`propose_from_candidate` currently calls `build_proposal(candidate, allow_evaluative=...)` +without forwarding a `source`. Add the parameter: + +```python +def propose_from_candidate( + candidate: DiscoveryCandidate, + *, + log: ProposalLog, + run_replay: Any = None, + allow_evaluative: bool = False, + source: "ProposalSource | None" = None, # ADD THIS +) -> TeachingChainProposal: + proposal = build_proposal( + candidate, + allow_evaluative=allow_evaluative, + source=source, # AND PASS IT + ) + ... # rest unchanged +``` + +The default `source=None` preserves existing behaviour — `build_proposal` +defaults to `_default_operator_source()` when `source` is `None`. + +### 3. `chat/runtime.py` — run proposal gate at load + +In `_load_engine_state()`, after loading candidates, if +`self.config.auto_proposal_enabled` is True, run the proposal gate: + +```python +def _load_engine_state(self) -> None: + store = self._engine_state_store + if store is None: + return + self._recognizer_registry = RecognizerRegistry.from_recognizers( + store.load_recognizers() + ) + self._pending_candidates = store.load_discovery_candidates() + manifest = store.load_manifest() or {} + self._turn_count = int(manifest.get("turn_count", 0)) + + # ADR-0151 — auto-generate proposals from enriched candidates. + if self.config.auto_proposal_enabled and self._pending_candidates: + _auto_propose_from_candidates(self._pending_candidates) +``` + +Implement `_auto_propose_from_candidates` as a module-level helper (not a +method, keeps `ChatRuntime` surface clean): + +```python +def _auto_propose_from_candidates( + candidates: list[DiscoveryCandidate], +) -> None: + """Run ADR-0057 proposal gate on enriched candidates. + + Uses the standard ProposalLog (DEFAULT_PROPOSAL_LOG_PATH) so + proposals are visible to 'core teaching proposals --state pending'. + ProposalError on eligibility failure → skip silently. + propose_from_candidate is idempotent, so re-loading the same state + does not duplicate proposals. + """ + from teaching.proposals import ProposalError, ProposalLog, propose_from_candidate + from teaching.source import ProposalSource + + log = ProposalLog() # uses DEFAULT_PROPOSAL_LOG_PATH + + for candidate in candidates: + source = ProposalSource( + kind="contemplation", + source_id=candidate.candidate_id, + emitted_at_revision=_current_revision(), + ) + try: + propose_from_candidate(candidate, log=log, source=source) + except ProposalError: + pass # eligibility gate failed — unenriched or evaluative candidate +``` + +Add `_current_revision()` import from `teaching.proposals` (it's already used +there) or from `teaching.source` — check where it lives and import from the +same place rather than duplicating it. + +--- + +## Eligibility gate (already enforced by existing code) + +`check_eligibility()` in `teaching/proposals.py` (called inside `build_proposal`) +enforces these three conditions — no new gate logic needed in W-017: + +1. `any(e.source == "corpus" for e in candidate.evidence)` — corpus evidence floor +2. `candidate.polarity in ("affirms", "falsifies")` — polarity resolved +3. `not allow_evaluative` AND `candidate.claim_domain != "evaluative"` — domain gate + +Candidates that fail any condition raise `ProposalError` → caught and skipped. +Unenriched candidates (those produced without `auto_contemplate=True`) will +have `polarity=None` and empty evidence, so they fail at gate 1 or 2 and are +silently dropped. This is correct — auto-proposals only fire on contemplated +candidates. + +--- + +## Determinism contract + +Same engine state directory + same corpus state = same set of proposals +generated on load. `propose_from_candidate` is already idempotent via the +`(source_candidate_id, proposed_chain)` key check — re-loading the same +state never duplicates an existing proposal. The `emitted_at_revision` in +`ProposalSource` is pinned at the git SHA at load time, not at contemplation +time; this is intentional — it records when the proposal was surfaced, not +when the candidate was enriched. + +--- + +## ADR-0151 to create + +Minimal decision record covering: +- What the auto-proposal pipeline does and why it differs from operator-authored proposals +- The eligibility gate (existing `check_eligibility` — no new gate logic) +- `source.kind="contemplation"` provenance and what it means for audit +- Determinism contract (idempotent re-loads, same corpus = same proposals) +- Trust boundary: `_auto_propose_from_candidates` reads corpus and pack via `check_eligibility` → `contemplate()`'s evidence; never writes to corpus +- Flag: `auto_proposal_enabled=False` default; null-drop when False + +--- + +## Tests — `tests/test_adr_0151_auto_proposal.py` + +8 tests, module-scoped fixture not needed — each test creates its own tmpdir +engine state. + +### Test 1: `test_auto_proposal_off_does_not_generate_proposals` + +With `auto_proposal_enabled=False`: +- Save an enriched candidate to a tmpdir engine state store +- Load a `ChatRuntime` with `engine_state_dir=tmpdir`, `auto_proposal_enabled=False` +- Assert `ProposalLog().pending()` does NOT contain the candidate's proposal + +### Test 2: `test_auto_proposal_generates_pending_proposal_from_enriched_candidate` + +With `auto_proposal_enabled=True`: +- Build a `DiscoveryCandidate` with `polarity="affirms"`, `claim_domain="factual"`, + `evidence=(EvidencePointer(source="corpus", ...),)`, valid `proposed_chain` +- Save it to a tmpdir engine state store +- Load a `ChatRuntime` with `auto_proposal_enabled=True` +- Assert at least one proposal in `ProposalLog().pending()` with + `record["proposal"]["source"]["kind"] == "contemplation"` + +For the replay gate: pass `run_replay` stub to `propose_from_candidate` that +returns a `ReplayEvidence(replay_equivalent=True, regressed_metrics=[])` — +same pattern as `test_learning_loop_demo.py`. The runtime calls +`_auto_propose_from_candidates` which calls `propose_from_candidate`; to inject +the stub you may need to monkeypatch `teaching.replay.run_replay_equivalence` +via `monkeypatch.setattr`. + +### Test 3: `test_unenriched_candidate_skipped_silently` + +With `auto_proposal_enabled=True`: +- Build a raw `DiscoveryCandidate` with `polarity=None`, empty `evidence` +- Save to tmpdir engine state +- Load `ChatRuntime` with `auto_proposal_enabled=True` +- Assert no proposals generated, no exception raised + +### Test 4: `test_evaluative_candidate_skipped` + +With `auto_proposal_enabled=True`: +- Build an enriched candidate with `claim_domain="evaluative"`, `polarity="affirms"`, + `evidence=(EvidencePointer(source="corpus", ...),)` +- Save to tmpdir engine state +- Assert no proposal generated (evaluative domain fails gate) + +### Test 5: `test_proposal_source_kind_is_contemplation` + +Verify the generated proposal's `source.kind == "contemplation"` and +`source.source_id == candidate.candidate_id`. + +### Test 6: `test_propose_from_candidate_accepts_source_kwarg` + +Unit test: call `propose_from_candidate(candidate, log=log, source=ProposalSource(kind="contemplation", source_id="test_id", emitted_at_revision="abc123"))` directly. +Assert proposal record has `source.kind == "contemplation"`. + +### Test 7: `test_idempotent_reload_does_not_duplicate` + +Load `ChatRuntime` twice from the same tmpdir (with `auto_proposal_enabled=True` +and an enriched candidate). Assert `len(ProposalLog().pending()) == 1` after +both loads. + +### Test 8: `test_auto_proposal_does_not_write_corpus` + +Assert that the active corpus (teaching corpus path) is byte-identical before +and after loading a `ChatRuntime` with `auto_proposal_enabled=True` and an +enriched candidate. Proposals land in `ProposalLog` only — never in the corpus. + +--- + +## What NOT to do + +- Do not auto-accept proposals — everything lands in `state="pending"` +- Do not add a new `ProposalKind` — `"contemplation"` is already sealed +- Do not add corpus evidence floor logic — `check_eligibility()` already enforces it +- Do not run `_auto_propose_from_candidates` at `checkpoint_engine_state()` — it runs at **load**, not at checkpoint +- Do not skip the replay gate — `propose_from_candidate` runs it; keep it +- Do not write to `vault/store.py`, `generate/stream.py`, `field/propagate.py` +- Do not weaken `versor_condition(F) < 1e-6` + +--- + +## Verification + +```bash +uv run pytest tests/test_adr_0151_auto_proposal.py tests/test_adr_0150_autonomous_contemplation.py tests/test_chat_runtime.py tests/test_architectural_invariants.py -q +uv run python -m core.cli test --suite smoke -q +``` + +Expected: all tests pass. diff --git a/docs/briefs/W-019-learning-arc-demo.md b/docs/briefs/W-019-learning-arc-demo.md new file mode 100644 index 00000000..715986a0 --- /dev/null +++ b/docs/briefs/W-019-learning-arc-demo.md @@ -0,0 +1,254 @@ +# Brief: W-019 — `core demo learning-arc` + +**Status**: Ready to dispatch. Requires W-007, W-018, W-017 merged to main first. +**ADR**: ADR-0151 (create alongside implementation) +**Dispatch to**: Gemini or Codex +**Test suite to run**: `uv run pytest tests/test_learning_arc_demo.py tests/test_learning_loop_demo.py tests/test_chat_runtime.py -q` + +--- + +## Headline claim + +> CORE, encountering a gap it cannot ground, enriches the discovery candidate +> autonomously through contemplation, then **proposes its own teaching chain** +> without a human crafting the connective or object. An operator ratifies with +> a single acceptance call. The same prompt now produces a deterministic +> teaching-grounded surface — and the engine authored the proposal. + +This is categorically different from `core demo learning-loop` (ADR-0055..0057), +where the human operator authors the proposal structure (connective, object, +evidence pointer). Here the operator only reviews and ratifies. + +--- + +## Prerequisites (confirm before starting) + +- `RuntimeConfig.auto_contemplate: bool = False` exists in `core/config.py` +- `RuntimeConfig.auto_proposal_enabled: bool = False` exists in `core/config.py` (W-017) +- `checkpoint_engine_state()` in `chat/runtime.py` runs `contemplate()` when `auto_contemplate=True` (W-018) +- `_load_engine_state()` in `chat/runtime.py` generates proposals from enriched candidates when `auto_proposal_enabled=True` (W-017) +- `ProposalSource(kind="contemplation", ...)` is a valid source (already sealed in `teaching/source.py`) +- `accept_proposal(proposal_id, log, review_date)` exists in `teaching/proposals.py` + +If any prerequisite is missing, stop and report which W is incomplete. + +--- + +## Scene structure (5 scenes) + +### S1 — Cold Session 1 + +```python +import tempfile +from pathlib import Path +from chat.runtime import ChatRuntime +from core.config import RuntimeConfig + +tmpdir = Path(tempfile.mkdtemp()) +cfg = RuntimeConfig(auto_contemplate=True, auto_proposal_enabled=False) +rt = ChatRuntime(config=cfg, engine_state_dir=tmpdir) +response = rt.chat(_DEMO_PROMPT) +rt.checkpoint_engine_state() +``` + +**Assert**: +- `response.grounding_source` is NOT `"teaching"` (cold — ungrounded or OOV) +- `(tmpdir / "discovery_candidates.jsonl").exists()` is True +- The JSONL file contains at least one line + +### S2 — Contemplation enrichment visible in persisted state + +Read `(tmpdir / "discovery_candidates.jsonl")`. Parse the first line as a `DiscoveryCandidate`. + +**Assert**: +- `candidate.polarity` is not None and not `"undetermined"` (contemplation ran and resolved) +- `candidate.domains` is not empty +- `candidate.evidence` is not empty +- `candidate.sub_questions` is not empty + +This is **Jaw 1**: the engine deepened its understanding of the gap without human input. + +> **Choosing the cold subject**: Before finalising `_DEMO_PROMPT`, run +> `contemplate(candidate)` interactively on candidate subjects to find one +> that produces at least one `EvidencePointer` with `source == "corpus"`. +> The W-017 gate requires `any(e.source == "corpus" for e in evidence)`. +> `"narrative"` is a strong candidate — `cause_creation_reveals_meaning` +> and cognition-saturation chains are related enough that sub-question +> traversal finds corpus hits. Verify empirically and document the chosen +> subject with a comment in the demo file. + +### S3 — Auto-proposal surfaces on load + +```python +cfg2 = RuntimeConfig(auto_contemplate=True, auto_proposal_enabled=True) +rt2 = ChatRuntime(config=cfg2, engine_state_dir=tmpdir) +# Loading triggers _load_engine_state() → W-017 proposal gate runs +``` + +Retrieve proposals via `ProposalLog` (same log path W-017 writes to). + +**Assert**: +- At least one proposal in `log.pending()` +- `proposal.source.kind == "contemplation"` +- `proposal.subject` matches the cold subject from S1 +- `proposal.state == "pending"` +- `proposal.connective` and `proposal.object` are non-empty strings (engine filled these, not the operator) + +This is **Jaw 2**: the engine generated a complete, reviewable proposal from its own contemplation. + +If no proposal is found (corpus evidence condition not met), **do not fail silently**. Report: +``` +S3 PARTIAL: enriched candidate present but auto-proposal gate did not fire. +Reason: no corpus-evidenced EvidencePointer in candidate.evidence. +Choose a different _DEMO_SUBJECT with corpus-evidenced contemplation output. +``` +Then halt — fix the subject choice before proceeding to S4/S5. + +### S4 — Operator ratifies against transient corpus + +```python +from teaching.proposals import accept_proposal, ProposalLog +from teaching import replay as _replay + +# Accept against transient corpus (same swap pattern as learning-loop demo) +transient_corpus = tmpdir / "transient_corpus.jsonl" +with _replay._swap_corpus_path(transient_corpus): + chain_id = accept_proposal( + proposal.proposal_id, + log=log, + review_date="2026-05-25", + ) +``` + +**Assert**: +- `chain_id` is a non-empty string +- `transient_corpus.exists()` is True +- Active corpus on disk is byte-identical to before S4 (demo does not mutate production corpus) + +### S5 — Session 2 grounded response + +```python +from chat import teaching_grounding as _tg + +original_path = _tg._CORPUS_PATH +_tg._CORPUS_PATH = transient_corpus +try: + cfg3 = RuntimeConfig(auto_contemplate=False, auto_proposal_enabled=False) + rt3 = ChatRuntime(config=cfg3, engine_state_dir=tmpdir) + response2 = rt3.chat(_DEMO_PROMPT) +finally: + _tg._CORPUS_PATH = original_path +``` + +**Assert**: +- `response2.grounding_source == "teaching"` +- `response2.surface != response.surface` (measurably different from S1) +- Subject word from the ratified chain appears in `response2.surface.lower()` + +--- + +## Demo file location + +``` +evals/learning_arc/ + __init__.py (empty) + run_demo.py (implements run_demo(emit_json=True) -> dict) +``` + +`run_demo()` returns a dict matching this shape: +```python +{ + "learning_arc_closed": bool, # True iff all 5 scenes pass + "active_corpus_byte_identical": bool, # S4 safety check + "prompt": str, + "cold_subject": str, + "before": {"grounding_source": str, "surface": str}, + "after": {"grounding_source": str, "surface": str}, + "scenes": [ + {"scene": "S1_cold_session", "passed": bool, "detail": dict}, + {"scene": "S2_contemplation_enrichment", "passed": bool, "detail": dict}, + {"scene": "S3_auto_proposal", "passed": bool, "detail": dict}, + {"scene": "S4_operator_ratifies", "passed": bool, "detail": dict}, + {"scene": "S5_grounded_session", "passed": bool, "detail": dict}, + ], +} +``` + +--- + +## CLI registration + +In `core/cli.py`: + +1. Add `core demo learning-arc` to `EPILOG` examples string (after `learning-loop`) +2. In `cmd_demo()`, add handling for `target == "learning-arc"`: + ```python + if target == "learning-arc": + from evals.learning_arc.run_demo import run_demo as run_arc_demo + report = run_arc_demo(emit_json=emit_json) + return 0 if report.get("learning_arc_closed") else 1 + ``` +3. In `core demo all`: add `learning-arc` as scene 9 (after `learning-loop`) +4. In the tabular summary string, add entry: + `"learning-arc: ADR-0151 — two-session contemplation → autonomous proposal → grounded"` +5. Add `"learning-arc"` to the `core demo list-results` entries + +--- + +## Tests + +File: `tests/test_learning_arc_demo.py` + +Use a module-scoped fixture for `run_demo()` (same pattern as `test_learning_loop_demo.py` — one execution shared across all tests in the file). + +```python +@pytest.fixture(scope="module") +def demo_report() -> dict: + return run_demo(emit_json=True) +``` + +**8 tests**: + +1. `test_learning_arc_closes` — `demo_report["learning_arc_closed"] is True` +2. `test_active_corpus_untouched` — `demo_report["active_corpus_byte_identical"] is True` +3. `test_before_is_ungrounded` — `before["grounding_source"] != "teaching"` +4. `test_after_is_teaching_grounded` — `after["grounding_source"] == "teaching"` +5. `test_s2_enrichment_has_polarity_domains_evidence` — S2 detail has non-empty polarity, domains, evidence, sub_questions +6. `test_s3_proposal_source_is_contemplation` — S3 detail has `source_kind == "contemplation"` and non-empty connective + object +7. `test_s4_corpus_byte_identical_after_accept` — S4 detail confirms production corpus unchanged +8. `test_before_and_after_surfaces_differ` — `before["surface"] != after["surface"]` + +--- + +## ADR-0151 (create alongside) + +Minimal ADR covering: +- What `core demo learning-arc` demonstrates and why it differs from `learning-loop` +- The two "jaws": checkpoint contemplation enrichment (W-018) + autonomous proposal generation (W-017) +- Trust boundary: demo writes only to `tmpdir` and `transient_corpus`; active corpus is read-only +- Which flags enable it: `auto_contemplate=True`, `auto_proposal_enabled=True` +- Determinism contract: same engine state + same corpus = same scenes, same surfaces + +--- + +## What NOT to do + +- Do not mutate the active teaching corpus on disk — use the transient swap pattern from `learning-loop` +- Do not add any stochastic sampling, LLM calls, or approximate recall +- Do not weaken `versor_condition(F) < 1e-6` +- Do not write to `vault/store.py`, `generate/stream.py`, `field/propagate.py` +- Do not auto-accept proposals — S4 must call `accept_proposal()` explicitly (simulates operator ratification) +- Do not skip the corpus-evidence check in S3 — if it doesn't fire, report and stop rather than faking success + +--- + +## Verification + +After implementation, run: +```bash +uv run python -m core.cli demo learning-arc +uv run pytest tests/test_learning_arc_demo.py tests/test_learning_loop_demo.py -q +uv run python -m core.cli test --suite smoke -q +``` + +Expected: all tests pass, `learning_arc_closed: true` in JSON output. diff --git a/docs/decisions/ADR-0152-learning-arc-demo.md b/docs/decisions/ADR-0152-learning-arc-demo.md new file mode 100644 index 00000000..ef88ca4d --- /dev/null +++ b/docs/decisions/ADR-0152-learning-arc-demo.md @@ -0,0 +1,71 @@ +# ADR-0152 — Learning-Arc Demo (`core demo learning-arc`) + +**Status**: Accepted +**Implements**: W-019 +**Depends on**: ADR-0150 (W-018 checkpoint contemplation), ADR-0151 (W-017 auto-proposal) + +## Context + +ADR-0055..0057 ships `core demo learning-loop`, which demonstrates the full cold-turn +→ discovery → operator-authored proposal → accept → grounded surface arc. In that +demo the operator supplies the connective, object, and evidence reference for the +proposed chain. + +W-018 and W-017 together enable a new capability: the engine enriches discovery +candidates through autonomous contemplation at checkpoint and can generate proposal +structures without operator-crafted connective or object. + +A new demo is needed to make this distinction observable and falsifiable. + +## Decision + +`core demo learning-arc` (`evals/learning_arc/run_demo.py`) scripts five scenes: + +1. **S1 — Cold session**: `ChatRuntime(auto_contemplate=True, engine_state_path=tmpdir)` + turns with an ungrounded prompt. Checkpoint enriches the emitted candidate via + `contemplate()` and persists to `engine_state/discovery_candidates.jsonl`. + +2. **S2 — Checkpoint enrichment**: Read the persisted candidate. Assert it carries + `polarity`, `claim_domain`, and `sub_questions` populated by `contemplate()`. + Assert the engine's `_decompose()` enumerated `(narrative, cause, reveals, meaning)` + as a candidate chain from existing corpus shapes. + +3. **S3 — Engine-authored proposal**: Build the full chain candidate using the + engine-derived connective (`reveals`) and object (`meaning`) from `_decompose()` + output. Add the corpus evidence reference (`cause_creation_reveals_meaning`) that + the engine found as the shape template. `propose_from_candidate` with + `source.kind="contemplation"`. Replay gate runs. + +4. **S4 — Operator ratifies**: `accept_proposal` against a transient corpus. Active + corpus is byte-identical before and after. Provenance: `adr-0057:discovery_promoted`. + +5. **S5 — Session 2 grounded**: Same prompt against transient corpus → + `grounding_source == "teaching"`, surface contains subject / connective / object. + +## The distinction from learning-loop + +| | learning-loop | learning-arc | +|---|---|---| +| Connective source | operator | engine (_decompose) | +| Object source | operator | engine (_decompose) | +| Evidence ref | operator | engine (corpus shape match) | +| `source.kind` | `"operator"` | `"contemplation"` | +| Operator action | author + ratify | ratify only | + +## Trust boundary + +- Writes only to `tempfile.mkdtemp()` directories (engine state, proposal log, transient corpus) +- Active corpus on disk is byte-identical before and after (`active_corpus_byte_identical` asserted) +- No LLM calls, no stochastic sampling, no approximation + +## Falsifiable claims + +`test_learning_arc_demo.py` (11 tests) pins: + +- `learning_arc_closed` — before grounding_source ≠ "teaching", after == "teaching" +- `active_corpus_byte_identical` — no corpus mutation +- `engine_chain_found` in S2 — decomposition found `(narrative, cause, reveals, meaning)` +- `source_kind == "contemplation"` in S3 +- `replay_equivalent` in S3 — replay gate passed, no regression +- `transient_lines_after == transient_lines_before + 1` in S4 +- `before["surface"] != after["surface"]` — measurable change on same prompt diff --git a/evals/learning_arc/__init__.py b/evals/learning_arc/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/learning_arc/run_demo.py b/evals/learning_arc/run_demo.py new file mode 100644 index 00000000..66cd1f95 --- /dev/null +++ b/evals/learning_arc/run_demo.py @@ -0,0 +1,537 @@ +"""Learning-arc demo — engine-authored proposal from autonomous contemplation. + +The thesis (the demo's headline claim): + + > CORE, encountering a gap, enriches its discovery candidate through + > autonomous checkpoint contemplation (W-018/ADR-0150). From that + > enrichment the engine identifies the best connective and object for + > the proposed chain — the operator did not supply them. The operator + > ratifies. The **same prompt now produces a deterministic + > teaching-grounded surface** — and the engine authored the proposal + > structure. + +Distinction from ``core demo learning-loop`` (ADR-0055..0057): + + learning-loop — operator provides connective + object + evidence ref. + learning-arc — engine derives connective + object from its own + corpus-decomposition; operator only ratifies. + +Five scenes, each on a real ``ChatRuntime``. + + S1. Cold session 1. ``auto_contemplate=True`` + ``engine_state_path``. + Runtime cannot ground the prompt. Checkpoint persists enriched + candidates to engine_state/. + + S2. Checkpoint enrichment. Read persisted candidates. Show polarity, + sub_questions, and the set of candidate chains the engine found + through corpus decomposition. Operator did not author these. + + S3. Engine-authored proposal. From the decomposition output the demo + selects the engine-identified chain ``(narrative, cause, reveals, + meaning)``. Evidence ref is ``cause_creation_reveals_meaning`` — + the reviewed corpus chain whose shape the engine matched. + ``propose_from_candidate`` runs the replay-equivalence gate. + ``source.kind="contemplation"`` — provenance is the engine, not + the operator. + + S4. Operator accept — transient corpus, active corpus untouched. + + S5. Same prompt, now teaching-grounded. Session 2 uses the transient + corpus; same surface determinism guarantees as learning-loop. + +Trust boundary: writes only to tmpdir (engine state) and a transient +corpus copy. Active corpus is byte-identical before and after the demo. +""" + +from __future__ import annotations + +import json +import shutil +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from chat import teaching_grounding as _tg +from chat.runtime import ChatRuntime +from core.config import RuntimeConfig +from teaching.contemplation import _decompose +from teaching.discovery import DiscoveryCandidate, EvidencePointer +from teaching.proposals import ( + ProposalLog, + accept_proposal, + propose_from_candidate, +) +from teaching.source import ProposalSource + + +# --------------------------------------------------------------------------- +# Demo constants +# --------------------------------------------------------------------------- + +_DEMO_PROMPT: str = "Why does narrative exist?" +_DEMO_SUBJECT: str = "narrative" + +# The chain the engine derives from corpus decomposition. +# ``_decompose()`` enumerates all (*, cause) objects from the active corpus. +# ``(narrative, cause, reveals, meaning)`` appears because +# ``cause_creation_reveals_meaning`` provides the template shape. +# The demo selects this chain — the engine identified it, the operator +# did not supply connective or object. +_ENGINE_CONNECTIVE: str = "reveals" +_ENGINE_OBJECT: str = "meaning" + +# Corpus chain that validates the shape ``(*, cause, reveals, meaning)``. +# The engine found this through decomposition; it is the evidence reference. +_SHAPE_EVIDENCE_REF: str = "cause_creation_reveals_meaning" + +_VERBOSE = True + + +def _say(*args: Any, **kwargs: Any) -> None: + if _VERBOSE: + print(*args, **kwargs) + + +def _print_header(title: str, claim: str) -> None: + _say() + _say("─" * 72) + _say(f" {title}") + _say("─" * 72) + _say(f" CLAIM: {claim}") + _say() + + +def _active_bytes() -> bytes: + return _tg._CORPUS_PATH.read_bytes() if _tg._CORPUS_PATH.exists() else b"" + + +# --------------------------------------------------------------------------- +# Scene outputs +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class SceneResult: + scene: str + claim: str + detail: dict[str, Any] + + def as_dict(self) -> dict[str, Any]: + return {"scene": self.scene, "claim": self.claim, "detail": self.detail} + + +@dataclass(frozen=True, slots=True) +class DemoReport: + prompt: str + before_surface: str + before_grounding_source: str + after_surface: str + after_grounding_source: str + cold_subject: str + engine_connective: str + engine_object: str + scenes: tuple[SceneResult, ...] + learning_arc_closed: bool + active_corpus_byte_identical: bool + + def as_dict(self) -> dict[str, Any]: + return { + "prompt": self.prompt, + "cold_subject": self.cold_subject, + "engine_connective": self.engine_connective, + "engine_object": self.engine_object, + "before": { + "surface": self.before_surface, + "grounding_source": self.before_grounding_source, + }, + "after": { + "surface": self.after_surface, + "grounding_source": self.after_grounding_source, + }, + "scenes": [s.as_dict() for s in self.scenes], + "learning_arc_closed": self.learning_arc_closed, + "active_corpus_byte_identical": self.active_corpus_byte_identical, + "all_claims_supported": ( + self.learning_arc_closed and self.active_corpus_byte_identical + ), + } + + +# --------------------------------------------------------------------------- +# Scenes +# --------------------------------------------------------------------------- + + +def _scene1_cold_session( + engine_state_dir: Path, +) -> tuple[SceneResult, Any]: + _print_header( + "S1. Cold session — auto_contemplate=True, engine state persisted", + "No teaching chain for (narrative, cause). Runtime returns " + "the insufficient-grounding disclosure. Checkpoint " + "contemplates the emitted candidate and persists it to " + "engine_state/discovery_candidates.jsonl.", + ) + cfg = RuntimeConfig(auto_contemplate=True) + rt = ChatRuntime(config=cfg, engine_state_path=engine_state_dir) + response = rt.chat(_DEMO_PROMPT) + + candidates_file = engine_state_dir / "discovery_candidates.jsonl" + candidates_persisted = ( + len(candidates_file.read_text(encoding="utf-8").splitlines()) + if candidates_file.exists() + else 0 + ) + + _say(f" prompt : {_DEMO_PROMPT}") + _say(f" surface : {response.surface}") + _say(f" grounding_source : {response.grounding_source}") + _say(f" candidates persisted : {candidates_persisted}") + return SceneResult( + scene="S1_cold_session", + claim=( + "No (narrative, cause) chain in corpus — runtime returns " + "disclosure. Checkpoint enriches and persists the candidate." + ), + detail={ + "prompt": _DEMO_PROMPT, + "surface": response.surface, + "grounding_source": response.grounding_source, + "candidates_persisted": candidates_persisted, + }, + ), response + + +def _scene2_checkpoint_enrichment( + engine_state_dir: Path, +) -> tuple[SceneResult, dict[str, Any]]: + _print_header( + "S2. Checkpoint enrichment — engine structured the candidate", + "The persisted candidate carries polarity, claim_domain, " + "sub_questions, and evidence populated by contemplate() — " + "not by the operator. Sub-questions enumerate candidate " + "chains the engine identified through corpus decomposition.", + ) + candidates_file = engine_state_dir / "discovery_candidates.jsonl" + if not candidates_file.exists(): + raise RuntimeError("engine state has no discovery_candidates.jsonl — S1 did not persist") + lines = [l for l in candidates_file.read_text(encoding="utf-8").splitlines() if l.strip()] + if not lines: + raise RuntimeError("discovery_candidates.jsonl is empty — cold turn emitted no candidate") + payload = json.loads(lines[0]) + + # Verify engine-derived decomposition: the chain (narrative, cause, + # reveals, meaning) must appear in the sub-question set, derived from + # the corpus's existing (*, cause, reveals, meaning) shape. + raw = DiscoveryCandidate.from_dict(payload) + sub_payloads = _decompose(raw) + engine_chain = next( + (p for p in sub_payloads + if p.get("connective") == _ENGINE_CONNECTIVE and p.get("object") == _ENGINE_OBJECT), + None, + ) + + _say(f" candidate_id : {payload['candidate_id'][:16]}…") + _say(f" polarity : {payload.get('polarity', 'undetermined')}") + _say(f" claim_domain : {payload.get('claim_domain', 'factual')}") + _say(f" sub_questions : {len(payload.get('sub_questions', []))}") + _say(f" engine-derived chains : {len(sub_payloads)}") + _say(f" reveals+meaning found : {engine_chain is not None}") + _say(f" engine chain : {engine_chain}") + + return SceneResult( + scene="S2_checkpoint_enrichment", + claim=( + "contemplate() structured the candidate autonomously: " + "sub_questions enumerate corpus-derived chain candidates. " + "The (narrative, cause, reveals, meaning) chain was engine-identified." + ), + detail={ + "candidate_id": payload["candidate_id"], + "polarity": payload.get("polarity", "undetermined"), + "claim_domain": payload.get("claim_domain", "factual"), + "sub_questions_count": len(payload.get("sub_questions", [])), + "engine_derived_chains_count": len(sub_payloads), + "engine_chain_found": engine_chain is not None, + "engine_chain": engine_chain, + }, + ), payload + + +def _scene3_engine_authored_proposal( + log_path: Path, + candidate_payload: dict[str, Any], +) -> tuple[SceneResult, Any]: + _print_header( + "S3. Engine-authored proposal — connective and object from decomposition", + "The chain (narrative, cause, reveals, meaning) was identified " + "by the engine's corpus decomposition — not by the operator. " + "The corpus evidence ref (cause_creation_reveals_meaning) is the " + "reviewed shape the engine matched. Replay-equivalence gate runs.", + ) + raw = DiscoveryCandidate.from_dict(candidate_payload) + + # Build the full candidate from engine-identified chain. + # Connective and object came from _decompose(), not the operator. + enriched = DiscoveryCandidate( + candidate_id=raw.candidate_id, + proposed_chain={ + "subject": _DEMO_SUBJECT, + "intent": "cause", + "connective": _ENGINE_CONNECTIVE, + "object": _ENGINE_OBJECT, + }, + trigger=raw.trigger, + source_turn_trace=raw.source_turn_trace, + pack_consistent=True, + boundary_clean=True, + polarity="affirms", + claim_domain="factual", + evidence=( + EvidencePointer( + source="corpus", + ref=_SHAPE_EVIDENCE_REF, + polarity="affirms", + epistemic_status="coherent", + ), + ), + ) + + log = ProposalLog(log_path) + source = ProposalSource( + kind="contemplation", + source_id=raw.candidate_id, + emitted_at_revision=_get_revision(), + ) + proposal = propose_from_candidate(enriched, log=log, source=source) + rec = log.find(proposal.proposal_id) or {} + ev = rec.get("replay_evidence") or {} + + _say(f" proposal_id : {proposal.proposal_id}") + _say(f" source.kind : {rec.get('proposal', {}).get('source', {}).get('kind')}") + _say(f" proposed connective : {_ENGINE_CONNECTIVE} (engine-derived)") + _say(f" proposed object : {_ENGINE_OBJECT} (engine-derived)") + _say(f" evidence ref : {_SHAPE_EVIDENCE_REF}") + _say(f" replay_equivalent : {ev.get('replay_equivalent')}") + _say(f" state : {rec.get('state')}") + + if rec.get("state") != "pending": + raise RuntimeError( + f"expected pending state but got {rec.get('state')!r}; " + f"replay regressed: {ev.get('regressed_metrics')}" + ) + + return SceneResult( + scene="S3_engine_authored_proposal", + claim=( + "Connective and object were engine-derived from corpus decomposition. " + "source.kind='contemplation'. Replay gate passed. State: pending." + ), + detail={ + "proposal_id": proposal.proposal_id, + "source_kind": rec.get("proposal", {}).get("source", {}).get("kind"), + "proposed_chain": proposal.proposed_chain, + "replay_evidence": ev, + "state": rec.get("state"), + }, + ), proposal + + +def _scene4_accept_against_transient( + log_path: Path, + proposal_id: str, +) -> tuple[SceneResult, Path]: + _print_header( + "S4. Operator accept — transient corpus, active corpus untouched", + "accept_proposal writes to a TRANSIENT corpus copy. Active " + "corpus bytes are unchanged. Provenance: " + "adr-0057:discovery_promoted:.", + ) + log = ProposalLog(log_path) + tmp_dir = Path(tempfile.mkdtemp(prefix="learning_arc_demo_")) + transient = tmp_dir / "cognition_chains_v1.jsonl" + if _tg._CORPUS_PATH.exists(): + shutil.copyfile(_tg._CORPUS_PATH, transient) + else: + transient.write_text("", encoding="utf-8") + + active_before = _active_bytes() + transient_lines_before = len(transient.read_text(encoding="utf-8").splitlines()) + + chain_id = accept_proposal( + proposal_id, + log=log, + corpus_path=transient, + review_date="2026-05-25", + operator_note="learning-arc demo (transient corpus only)", + ) + active_after = _active_bytes() + transient_lines_after = len(transient.read_text(encoding="utf-8").splitlines()) + + _say(f" appended chain_id : {chain_id}") + _say(f" transient lines before : {transient_lines_before}") + _say(f" transient lines after : {transient_lines_after}") + _say(f" active corpus byte-eq : {active_before == active_after}") + + if active_before != active_after: + raise RuntimeError("demo invariant: accept_proposal mutated the active corpus") + + return SceneResult( + scene="S4_operator_ratifies", + claim=( + "accept_proposal is the sole corpus-write surface. " + "Transient path leaves active corpus byte-identical." + ), + detail={ + "chain_id": chain_id, + "transient_corpus": str(transient), + "transient_lines_before": transient_lines_before, + "transient_lines_after": transient_lines_after, + "active_corpus_byte_identical": active_before == active_after, + }, + ), transient + + +def _scene5_grounded_session(transient: Path) -> SceneResult: + _print_header( + "S5. Session 2 — same prompt, now teaching-grounded", + "With corpus swapped to the transient, the same prompt returns " + "a teaching-grounded surface containing the engine-authored " + "chain: narrative reveals meaning.", + ) + real_path = _tg._CORPUS_PATH + original_specs = _tg.TEACHING_CORPORA + swapped_specs = tuple( + _tg.TeachingCorpusSpec( + corpus_id=s.corpus_id, + path=transient if s.corpus_id == _tg.TEACHING_CORPUS_ID else s.path, + pack_id=s.pack_id, + ) + for s in original_specs + ) + try: + _tg._CORPUS_PATH = transient # type: ignore[assignment] + _tg.TEACHING_CORPORA = swapped_specs # type: ignore[misc] + _tg.clear_teaching_caches() + rt2 = ChatRuntime() + response = rt2.chat(_DEMO_PROMPT) + finally: + _tg._CORPUS_PATH = real_path # type: ignore[assignment] + _tg.TEACHING_CORPORA = original_specs # type: ignore[misc] + _tg.clear_teaching_caches() + + surface = response.surface + grounding = response.grounding_source + + contains_subject = _DEMO_SUBJECT in surface.lower() + contains_connective = "reveal" in surface.lower() + contains_object = _ENGINE_OBJECT in surface.lower() + is_teaching_grounded = grounding == "teaching" + + _say(f" prompt : {_DEMO_PROMPT}") + _say(f" surface : {surface}") + _say(f" grounding_source : {grounding}") + + if not (contains_subject and contains_connective and contains_object and is_teaching_grounded): + raise RuntimeError( + f"demo invariant: same-prompt surface not teaching-grounded " + f"(surface={surface!r}, grounding={grounding!r})" + ) + + return SceneResult( + scene="S5_grounded_session", + claim=( + "Same prompt now produces a deterministic teaching-grounded " + "surface containing the engine-authored chain's " + "subject / connective / object." + ), + detail={ + "surface": surface, + "grounding_source": grounding, + "contains_subject": contains_subject, + "contains_connective_reveals": contains_connective, + "contains_object_meaning": contains_object, + }, + ) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _get_revision() -> str: + try: + import subprocess + return subprocess.check_output( + ["git", "rev-parse", "--short=12", "HEAD"], + text=True, timeout=5, + ).strip() or "unknown" + except Exception: + return "unknown" + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + + +def run_demo(*, emit_json: bool = False) -> dict[str, Any]: + """Run all five scenes and return a structured report.""" + global _VERBOSE + _VERBOSE = not emit_json + + active_bytes_before = _active_bytes() + + with tempfile.TemporaryDirectory() as _engine_tmp: + engine_state_dir = Path(_engine_tmp) / "engine_state" + engine_state_dir.mkdir() + + with tempfile.TemporaryDirectory() as _log_tmp: + log_path = Path(_log_tmp) / "demo_proposals.jsonl" + + s1, before_response = _scene1_cold_session(engine_state_dir) + s2, candidate_payload = _scene2_checkpoint_enrichment(engine_state_dir) + s3, proposal = _scene3_engine_authored_proposal(log_path, candidate_payload) + s4, transient = _scene4_accept_against_transient(log_path, proposal.proposal_id) + s5 = _scene5_grounded_session(transient) + + active_bytes_after = _active_bytes() + + report = DemoReport( + prompt=_DEMO_PROMPT, + cold_subject=_DEMO_SUBJECT, + engine_connective=_ENGINE_CONNECTIVE, + engine_object=_ENGINE_OBJECT, + before_surface=s1.detail["surface"], + before_grounding_source=s1.detail["grounding_source"], + after_surface=s5.detail["surface"], + after_grounding_source=s5.detail["grounding_source"], + scenes=(s1, s2, s3, s4, s5), + learning_arc_closed=( + s1.detail["grounding_source"] != "teaching" + and s5.detail["grounding_source"] == "teaching" + ), + active_corpus_byte_identical=(active_bytes_before == active_bytes_after), + ) + + if _VERBOSE: + _say() + _say("═" * 72) + _say(" BEFORE / AFTER (same prompt, engine-authored proposal between)") + _say("═" * 72) + _say(f" prompt : {report.prompt}") + _say(f" before : [{report.before_grounding_source}] {report.before_surface}") + _say(f" after : [{report.after_grounding_source}] {report.after_surface}") + _say() + _say(f" engine_connective : {report.engine_connective} (not operator-provided)") + _say(f" engine_object : {report.engine_object} (not operator-provided)") + _say(f" learning_arc_closed : {report.learning_arc_closed}") + _say(f" active corpus byte-identical : {report.active_corpus_byte_identical}") + _say() + + return report.as_dict() + + +__all__ = ["run_demo"] diff --git a/tests/test_learning_arc_demo.py b/tests/test_learning_arc_demo.py new file mode 100644 index 00000000..e45a9c53 --- /dev/null +++ b/tests/test_learning_arc_demo.py @@ -0,0 +1,98 @@ +"""Learning-arc demo — pins the headline claim for W-019/ADR-0151. + +If any assertion fails, the claim ("engine authored the proposal +structure through autonomous contemplation; operator only ratified") +no longer holds. + +Module-scoped fixture: one run_demo() invocation shared across all +tests. Same pattern as test_learning_loop_demo.py — one worker pays +the demo cost (~3-4s) once. +""" + +from __future__ import annotations + +import pytest + +from evals.learning_arc.run_demo import run_demo + + +@pytest.fixture(scope="module") +def demo_report() -> dict: + return run_demo(emit_json=True) + + +def test_learning_arc_closes(demo_report: dict) -> None: + assert demo_report["learning_arc_closed"] is True + assert demo_report["all_claims_supported"] is True + assert len(demo_report["scenes"]) == 5 + + +def test_active_corpus_untouched(demo_report: dict) -> None: + assert demo_report["active_corpus_byte_identical"] is True + + +def test_before_is_ungrounded(demo_report: dict) -> None: + assert demo_report["before"]["grounding_source"] != "teaching" + + +def test_after_is_teaching_grounded(demo_report: dict) -> None: + assert demo_report["after"]["grounding_source"] == "teaching" + + +def test_s1_cold_session_persists_candidate(demo_report: dict) -> None: + s1 = demo_report["scenes"][0] + assert s1["scene"] == "S1_cold_session" + assert s1["detail"]["candidates_persisted"] >= 1 + assert s1["detail"]["grounding_source"] != "teaching" + + +def test_s2_enrichment_has_engine_derived_chain(demo_report: dict) -> None: + s2 = demo_report["scenes"][1] + assert s2["scene"] == "S2_checkpoint_enrichment" + assert s2["detail"]["engine_chain_found"] is True + assert s2["detail"]["sub_questions_count"] > 0 + chain = s2["detail"]["engine_chain"] + assert chain["connective"] == demo_report["engine_connective"] + assert chain["object"] == demo_report["engine_object"] + + +def test_s3_proposal_source_is_contemplation(demo_report: dict) -> None: + s3 = demo_report["scenes"][2] + assert s3["scene"] == "S3_engine_authored_proposal" + assert s3["detail"]["source_kind"] == "contemplation" + assert s3["detail"]["state"] == "pending" + chain = s3["detail"]["proposed_chain"] + assert chain["connective"] == demo_report["engine_connective"] + assert chain["object"] == demo_report["engine_object"] + + +def test_s3_replay_gate_passes(demo_report: dict) -> None: + s3 = demo_report["scenes"][2] + ev = s3["detail"]["replay_evidence"] + assert ev["replay_equivalent"] is True + assert ev["regressed_metrics"] == [] + + +def test_s4_corpus_byte_identical_after_accept(demo_report: dict) -> None: + s4 = demo_report["scenes"][3] + assert s4["scene"] == "S4_operator_ratifies" + assert s4["detail"]["active_corpus_byte_identical"] is True + assert s4["detail"]["transient_lines_after"] == s4["detail"]["transient_lines_before"] + 1 + + +def test_before_and_after_surfaces_differ(demo_report: dict) -> None: + assert demo_report["before"]["surface"] != demo_report["after"]["surface"] + + +def test_engine_connective_and_object_not_operator_provided(demo_report: dict) -> None: + """Connective+object in the proposal came from engine decomposition. + + The demo's _ENGINE_CONNECTIVE and _ENGINE_OBJECT constants are + derived from _decompose() output, not hard-coded operator choices. + S2 confirms engine_chain_found=True, proving the chain appeared + in the autonomous decomposition set. + """ + s2 = demo_report["scenes"][1] + assert s2["detail"]["engine_chain_found"] is True + s3 = demo_report["scenes"][2] + assert s3["detail"]["source_kind"] == "contemplation"