Compare commits
2 commits
add-ask-ac
...
docs/q1-d-
| Author | SHA1 | Date | |
|---|---|---|---|
|
fc70b9a8f5 | ||
|
|
57a0e02148 |
16 changed files with 6 additions and 2242 deletions
|
|
@ -335,15 +335,6 @@ class RuntimeConfig:
|
|||
# default); the engine never raises its own ceiling.
|
||||
estimation_enabled: bool = False
|
||||
|
||||
# ASK serving gate enable flag. When True, ASK serving is allowed.
|
||||
# Default False (dark).
|
||||
ask_serving_enabled: bool = False
|
||||
|
||||
# VERIFIED serving gate enable flag. When True, VERIFIED serving is allowed.
|
||||
# Default False (dark).
|
||||
verified_serving_enabled: bool = False
|
||||
|
||||
|
||||
|
||||
DEFAULT_IDENTITY_PACK: str = "default_general_v1"
|
||||
DEFAULT_ETHICS_PACK: str = "default_general_ethics_v1"
|
||||
|
|
|
|||
|
|
@ -19,9 +19,6 @@ Shipped so far (all off-serving — nothing here imports ``generate.derivation``
|
|||
``EpistemicState × LimitationAssessment × DisclosureClaim → ServedDisposition``.
|
||||
Mapping scaffold only — no rendering, no bus, no ``verify.py``; nothing consumes
|
||||
it yet.
|
||||
* :mod:`~core.epistemic_disclosure.ask_serving` — a narrow Q1-D served-ASK artifact
|
||||
adapter. It validates already-rendered question artifacts and returns a typed
|
||||
decision; it does not render prose and does not acquire runtime contemplation.
|
||||
* :mod:`~core.epistemic_disclosure.verified_contract` (P1-A) — the VERIFIED contract:
|
||||
the obligation, the proof shape, the validator, and the single sanctioned route to
|
||||
``EpistemicState.VERIFIED`` / ``DisclosureClaim.VERIFIED``. Contract only — no
|
||||
|
|
@ -30,10 +27,6 @@ Shipped so far (all off-serving — nothing here imports ``generate.derivation``
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from core.epistemic_disclosure.ask_serving import (
|
||||
ServedAskDecision,
|
||||
evaluate_served_ask,
|
||||
)
|
||||
from core.epistemic_disclosure.disclosure_claim import (
|
||||
DEFAULT_DISCLOSURE_CLAIM,
|
||||
DisclosureClaim,
|
||||
|
|
@ -71,7 +64,6 @@ __all__ = [
|
|||
"LimitationKind",
|
||||
"MissingSlot",
|
||||
"ResolutionAction",
|
||||
"ServedAskDecision",
|
||||
"ServedDisposition",
|
||||
"VerificationObligation",
|
||||
"VerificationProof",
|
||||
|
|
@ -81,7 +73,6 @@ __all__ = [
|
|||
"assess_from_family",
|
||||
"choose_served_disposition",
|
||||
"disclosure_for_verification",
|
||||
"evaluate_served_ask",
|
||||
"evaluate_verification",
|
||||
"terminal_for_action",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,171 +0,0 @@
|
|||
"""Stage 2 ASK served-surface artifact adapter.
|
||||
|
||||
This module is intentionally narrow: it validates a pre-rendered Q1-D
|
||||
``DeliveredQuestion`` artifact and decides whether that artifact is eligible to
|
||||
be exposed as a served ASK/QUESTION_NEEDED surface. It does not acquire
|
||||
contemplation results from runtime and does not render question prose.
|
||||
|
||||
Validation enforces the Q1-D artifact contract:
|
||||
|
||||
- top-level JSON object only;
|
||||
- ``status == "question_only"``;
|
||||
- ``requires_review is True``;
|
||||
- ``served is False``;
|
||||
- ``answer_binding`` is absent or ``None``;
|
||||
- ``question`` is an object;
|
||||
- ``question.text`` is a non-empty string;
|
||||
- ``question.slot_name`` is a non-empty string;
|
||||
- ``question_path`` exists on disk and differs from ``proposal_path``.
|
||||
|
||||
Any validation failure fails closed to the caller's fallback surface and
|
||||
standing disposition. The served text is consumed from the artifact exactly; no
|
||||
runtime prose construction or mutation happens here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from core.epistemic_disclosure.disposition import ServedDisposition, choose_served_disposition
|
||||
from core.epistemic_disclosure.limitation import LimitationAssessment
|
||||
from core.epistemic_questions.serving_gate import ask_serving_enabled
|
||||
from core.epistemic_state import EpistemicState
|
||||
|
||||
_MISSING = object()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ServedAskDecision:
|
||||
"""The adapter's served-ASK decision."""
|
||||
|
||||
served: bool
|
||||
terminal: str
|
||||
surface: str
|
||||
disposition: ServedDisposition
|
||||
|
||||
|
||||
def _terminal_value(contemplation_result: Any) -> str:
|
||||
terminal = getattr(contemplation_result, "terminal", None)
|
||||
if terminal is None:
|
||||
return "NO_PROGRESS"
|
||||
return str(getattr(terminal, "value", terminal))
|
||||
|
||||
|
||||
def _fallback_disposition(terminal: str) -> ServedDisposition:
|
||||
if terminal == "PROPOSAL_EMITTED":
|
||||
return ServedDisposition.PROPOSE
|
||||
if terminal == "SOLVED_VERIFIED":
|
||||
return ServedDisposition.COMMIT
|
||||
return ServedDisposition.REFUSE
|
||||
|
||||
|
||||
def _fallback_decision(contemplation_result: Any, fallback_surface: str) -> ServedAskDecision:
|
||||
terminal = _terminal_value(contemplation_result)
|
||||
return ServedAskDecision(
|
||||
served=False,
|
||||
terminal=terminal,
|
||||
surface=fallback_surface,
|
||||
disposition=_fallback_disposition(terminal),
|
||||
)
|
||||
|
||||
|
||||
def _validate_question_artifact(data: Any, *, question_path: Path, proposal_path: Any) -> str | None:
|
||||
"""Return the valid question text, or ``None`` for any contract violation."""
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
if data.get("status") != "question_only":
|
||||
return None
|
||||
if data.get("requires_review") is not True:
|
||||
return None
|
||||
served = data.get("served", _MISSING)
|
||||
if served is _MISSING or served is not False:
|
||||
return None
|
||||
answer_binding = data.get("answer_binding", _MISSING)
|
||||
if answer_binding is not _MISSING and answer_binding is not None:
|
||||
return None
|
||||
|
||||
question = data.get("question")
|
||||
if not isinstance(question, dict):
|
||||
return None
|
||||
|
||||
text = question.get("text")
|
||||
if not isinstance(text, str) or not text.strip():
|
||||
return None
|
||||
|
||||
slot_name = question.get("slot_name")
|
||||
if not isinstance(slot_name, str) or not slot_name.strip():
|
||||
return None
|
||||
|
||||
if proposal_path is not None and str(question_path) == str(proposal_path):
|
||||
return None
|
||||
|
||||
return text.strip()
|
||||
|
||||
|
||||
def evaluate_served_ask(
|
||||
config: Any,
|
||||
contemplation_result: Any,
|
||||
fallback_surface: str,
|
||||
) -> ServedAskDecision:
|
||||
"""Evaluate whether a Q1-D question artifact may be surfaced as ASK.
|
||||
|
||||
This is a bus/disposition adapter, not a renderer and not the runtime
|
||||
acquisition path. The caller supplies a contemplation result that already
|
||||
points to a delivered question artifact. When the gate is disabled or any
|
||||
artifact invariant fails, the adapter returns the fallback surface and the
|
||||
standing fallback disposition.
|
||||
"""
|
||||
|
||||
if not ask_serving_enabled(config):
|
||||
return _fallback_decision(contemplation_result, fallback_surface)
|
||||
|
||||
if _terminal_value(contemplation_result) != "QUESTION_NEEDED":
|
||||
return _fallback_decision(contemplation_result, fallback_surface)
|
||||
|
||||
question_path_value = getattr(contemplation_result, "question_path", None)
|
||||
proposal_path_value = getattr(contemplation_result, "proposal_path", None)
|
||||
if not question_path_value or question_path_value == proposal_path_value:
|
||||
return _fallback_decision(contemplation_result, fallback_surface)
|
||||
|
||||
question_path = Path(question_path_value)
|
||||
if not question_path.is_file():
|
||||
return _fallback_decision(contemplation_result, fallback_surface)
|
||||
|
||||
try:
|
||||
payload = json.loads(question_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return _fallback_decision(contemplation_result, fallback_surface)
|
||||
|
||||
question_text = _validate_question_artifact(
|
||||
payload,
|
||||
question_path=question_path,
|
||||
proposal_path=proposal_path_value,
|
||||
)
|
||||
if question_text is None:
|
||||
return _fallback_decision(contemplation_result, fallback_surface)
|
||||
|
||||
limitation = LimitationAssessment(
|
||||
limitation_kind="missing_information",
|
||||
resolution_action="ask_question",
|
||||
epistemic_state=EpistemicState.UNDETERMINED,
|
||||
owner_organ=payload.get("owner_organ"),
|
||||
blocking_reason=str(payload.get("blocking_reason", "")),
|
||||
)
|
||||
disposition = choose_served_disposition(
|
||||
epistemic_state=EpistemicState.UNDETERMINED,
|
||||
limitation=limitation,
|
||||
)
|
||||
|
||||
return ServedAskDecision(
|
||||
served=True,
|
||||
terminal="QUESTION_NEEDED",
|
||||
surface=question_text,
|
||||
disposition=disposition,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["ServedAskDecision", "evaluate_served_ask"]
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
"""VERIFIED serving gate helper — default-dark, no served-surface wiring.
|
||||
|
||||
This module centralizes the future kill-switch predicate for VERIFIED serving.
|
||||
It is default-dark / fail-closed: if the config field is missing or malformed,
|
||||
it must evaluate to False.
|
||||
|
||||
This helper only centralizes the future kill-switch predicate so that future serving code
|
||||
has one audited predicate. It does not wire any served-surface or implement served
|
||||
VERIFIED behavior. Missing field means False.
|
||||
|
||||
Note that eval-gold-backed producers (such as the verification producer in
|
||||
evals/constraint_oracle/verified_producer.py) are not serving-eligible.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from core.config import DEFAULT_CONFIG, RuntimeConfig
|
||||
|
||||
|
||||
def verified_serving_enabled(config: RuntimeConfig | Any | None = None) -> bool:
|
||||
"""Return whether served VERIFIED delivery is explicitly enabled.
|
||||
|
||||
Missing attribute means False. This is the load-bearing dark-gate invariant:
|
||||
the served VERIFIED path cannot light merely because the helper exists or because
|
||||
an older RuntimeConfig instance lacks the future field.
|
||||
"""
|
||||
cfg = DEFAULT_CONFIG if config is None else config
|
||||
return bool(getattr(cfg, "verified_serving_enabled", False))
|
||||
|
||||
|
||||
__all__ = ["verified_serving_enabled"]
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
"""ASK serving gate helper — default-dark, no served-surface wiring.
|
||||
|
||||
This module is the first code slice after the ASK serving-integration scoping brief.
|
||||
It intentionally does **not** call ``deliver_ask``/``emit_question``, does not import
|
||||
``chat.runtime``, and does not expose any user-facing surface. It only centralizes the
|
||||
kill-switch read so future serving code has one audited predicate.
|
||||
|
||||
The planned config field is ``RuntimeConfig.ask_serving_enabled``. During this dark-gate
|
||||
slice the predicate is conservative: absent field == ``False``. That lets the helper land
|
||||
without widening behavior and preserves the current default for every existing
|
||||
``RuntimeConfig`` instance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from core.config import DEFAULT_CONFIG, RuntimeConfig
|
||||
|
||||
|
||||
def ask_serving_enabled(config: RuntimeConfig | Any | None = None) -> bool:
|
||||
"""Return whether served ASK delivery is explicitly enabled.
|
||||
|
||||
Missing attribute means ``False``. That is the load-bearing dark-gate invariant:
|
||||
the served ASK path cannot light merely because the helper exists or because an
|
||||
older ``RuntimeConfig`` instance lacks the future field.
|
||||
"""
|
||||
cfg = DEFAULT_CONFIG if config is None else config
|
||||
return bool(getattr(cfg, "ask_serving_enabled", False))
|
||||
|
||||
|
||||
__all__ = ["ask_serving_enabled"]
|
||||
|
|
@ -1,199 +0,0 @@
|
|||
# ASK serving-integration — `ask_serving_enabled` — scoping brief
|
||||
|
||||
**Date:** 2026-06-08 · **Status:** scoping (NO CODE) · **HOLD for review** ·
|
||||
**Branch:** `docs/serving-integration-scoping`
|
||||
|
||||
**What this brief is.** The scope for the *one* served-surface decision the ASK lane
|
||||
deferred. The off-serving ASK lane is complete and on main:
|
||||
|
||||
```text
|
||||
Q1-B typed residue + ask classification (LimitationAssessment.missing_slots)
|
||||
Q1-C grounded-only rendering (render_question → EpistemicQuestion)
|
||||
Q1-D off-serving delivery artifact (deliver_ask → QUESTION_NEEDED + sink)
|
||||
```
|
||||
|
||||
`deliver_ask` / `emit_question` exist and are tested, but **nothing calls them from a
|
||||
live path** — they are produced-but-not-emitted, exactly as P1-B's verified producer
|
||||
is built-but-not-served. This brief scopes the step that closes that gap: letting a
|
||||
`QUESTION_NEEDED` actually reach a user, behind a named gate. **No code here** — this
|
||||
is the decision surface for review.
|
||||
|
||||
> Companions: [[q1-d-ask-bus-delivery-scoping-2026-06-08]] (the off-serving rung, §7
|
||||
> defers exactly this), [[verified-serving-wiring-scoping-2026-06-08]] (the VERIFIED
|
||||
> half of the same "where off-serving stops" line). Design of record: the session doc
|
||||
> §5 / §1.5.8 (the disclosure bus).
|
||||
|
||||
---
|
||||
|
||||
## 1. The five things this decision must pin
|
||||
|
||||
The steer named five; each is grounded against shipped code below.
|
||||
|
||||
```text
|
||||
1. ask_serving_enabled — the kill switch
|
||||
2. the pass_manager integration point — where deliver_ask is actually called
|
||||
3. the Q1B_ASK_CARVE_OUT retirement gate + registry flip
|
||||
4. served-surface behaviour for a QUESTION_NEEDED reaching a user
|
||||
5. the no-question/no-proposal dead-zone proof
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. The integration point (grounded)
|
||||
|
||||
`generate/contemplation/pass_manager.py::contemplate(text, *, proposal_root=...)` is
|
||||
the live contemplation entry. When every attempt is refused it calls
|
||||
`_classify_all_refused(text, attempts, findings, proposal_root)`, which is where
|
||||
`emit_proposal` fires for a `proposal_allowed` family and terminates
|
||||
`PROPOSAL_EMITTED`. **That is the exact analogous site for ASK:** when a refused
|
||||
attempt's family maps to `ask_question` (via `assess_from_attempt`), call `deliver_ask`
|
||||
and — on a renderable result — `emit_question` to the sink and terminate
|
||||
`QUESTION_NEEDED`; on the D2 fallback, fall through to today's proposal/refuse path.
|
||||
|
||||
`contemplate()` is called from `chat/runtime.py` at three sites (827, 901, 1364) — so
|
||||
the contemplation `Terminal` is already on the served path. The served question text
|
||||
travels the same way `PROPOSAL_EMITTED` already does.
|
||||
|
||||
**Two-layer split (the recommendation), mirroring VERIFIED:**
|
||||
|
||||
- **Layer A — pass-manager emission (off-serving still).** `contemplate` emits
|
||||
`QUESTION_NEEDED` + writes the `teaching/questions/` artifact, exactly as it emits
|
||||
`PROPOSAL_EMITTED` today. This changes the *contemplation* terminal but **reaches no
|
||||
user** — the teaching loop is off-serving. This layer is buildable behind no gate
|
||||
(it only adds a terminal the pass can reach), but see §3: it interacts with the
|
||||
carve-out and so should still wait for the gate, to avoid double-emission churn.
|
||||
- **Layer B — served delivery (the gated surface).** `chat/runtime.py`, gated by
|
||||
`ask_serving_enabled`, renders the `DeliveredQuestion.question.text` to the user as
|
||||
the served response when the terminal is `QUESTION_NEEDED`. This is the only layer
|
||||
that actually asks the user anything.
|
||||
|
||||
Decision to confirm: **do Layer A and Layer B land together behind one gate, or does
|
||||
Layer A land first (pass emits, nothing served) and Layer B follow?** Recommend
|
||||
**together, behind `ask_serving_enabled`** — Layer A alone creates the double-emission
|
||||
state in §3 with no compensating benefit.
|
||||
|
||||
---
|
||||
|
||||
## 3. The kill switch + the carve-out retirement gate (the coupled core)
|
||||
|
||||
### 3.1 `ask_serving_enabled`
|
||||
|
||||
Add `ask_serving_enabled: bool = False` to `core/config.py`, the sibling of the
|
||||
existing `estimation_enabled = False` kill-switch pattern. Default **off**: the
|
||||
served question path is dark until deliberately enabled, holdout-gated (§5).
|
||||
|
||||
### 3.2 Why the carve-out flip is *coupled* to the gate (not to Q1-D)
|
||||
|
||||
Q1-B introduced `Q1B_ASK_CARVE_OUT` for `missing_total_count` / `missing_weighted_total`:
|
||||
the disclosure layer classifies them `ask_question`, but the shipped `REGISTRY` keeps
|
||||
`proposal_allowed = True` so the proposal pile keeps working. The carve-out's
|
||||
*retirement condition* is written into `limitation.py`: *"Once ASK is serving, flip
|
||||
`proposal_allowed = False` on these two families, drop the carve-out set, amend the
|
||||
test."* The operative word is **serving** — not "delivery exists" (Q1-D already shipped
|
||||
that off-serving). So:
|
||||
|
||||
```text
|
||||
carve-out retires ⟺ ask_serving_enabled is the ruling
|
||||
AND a QUESTION_NEEDED is actually served for these families
|
||||
AND the §4 dead-zone proof holds
|
||||
```
|
||||
|
||||
Until then, `proposal_allowed` stays `True`. During the gate's "off" state both signals
|
||||
coexist (the off-serving question artifact + the proposal) — intentional, no loss.
|
||||
|
||||
### 3.3 The flip, as a single reviewed act
|
||||
|
||||
When `ask_serving_enabled` is turned on for these families, in **one** change:
|
||||
1. `proposal_allowed = False` for `missing_total_count` / `missing_weighted_total`.
|
||||
2. Drop them from `Q1B_ASK_CARVE_OUT` (empty the set, or remove the constant).
|
||||
3. Amend the `proposal_allowed` invariant test + the carve-out test.
|
||||
4. The §4 dead-zone proof test must already be green.
|
||||
|
||||
This is the "conscious act, not a silent re-key" the carve-out was built to force.
|
||||
|
||||
---
|
||||
|
||||
## 4. The no-question/no-proposal dead-zone proof (the wrong=0-adjacent guard)
|
||||
|
||||
**The hazard.** The flip removes the proposal signal for these families. If, for some
|
||||
input class, the family classifies `ask_question` BUT the question is unrenderable
|
||||
(D2), AND the proposal is now off, the family would terminate `NO_PROGRESS` with **no
|
||||
served question and no proposal** — a dead zone where a user-resolvable gap produces
|
||||
*nothing*. That is the ASK-side wrong=0 hazard: not a false answer, but a silent loss
|
||||
of a capability that previously at least proposed.
|
||||
|
||||
**The proof obligation (before any flip).** For every family being flipped, prove that
|
||||
**no input class lands in the dead zone** — i.e. for every reachable assessment of that
|
||||
family, the question renders (so `QUESTION_NEEDED` is served), OR the proposal is still
|
||||
on. Concretely:
|
||||
|
||||
- The `missing_*` families have pinned single slots in `_FAMILY_TO_MISSING_SLOTS` with
|
||||
mapped types (`count_int` / `measured_unit_int`) → they **always render** today, so
|
||||
the dead zone is currently empty. The proof must show this is *structural*, not
|
||||
incidental: a test that asserts `deliver_ask` returns `QUESTION_NEEDED` (never a
|
||||
fallback) for every reachable assessment of the flipped families.
|
||||
- If any future residue change could make one of these unrenderable (multi-slot,
|
||||
unmapped type), the flip must be blocked for that family until either the renderer
|
||||
covers it or the proposal stays on.
|
||||
|
||||
**The rule:** `proposal_allowed` may flip `True → False` for a family **only** if a
|
||||
test proves every reachable ask of that family renders. The dead-zone proof is a
|
||||
precondition of the flip, enforced like the D2 guard (it must *fail* if a fallback path
|
||||
is reachable for a flipped family).
|
||||
|
||||
---
|
||||
|
||||
## 5. Served-surface behaviour + holdout gating
|
||||
|
||||
### 5.1 What a served `QUESTION_NEEDED` looks like
|
||||
|
||||
When `ask_serving_enabled` and the terminal is `QUESTION_NEEDED`, the served surface
|
||||
returns the `DeliveredQuestion.question.text` (the grounded-only rendered question) as
|
||||
the response — distinct from a committed answer, an `[approximate]` disclosure, or a
|
||||
refusal. It is an **intake request**: the disposition is `ServedDisposition.ASK`
|
||||
(already mapped in `disposition.py`). The question names nothing ungrounded (Q1-C
|
||||
guarantee), so it cannot leak a fabricated entity even on the served path.
|
||||
|
||||
Open sub-decision: **the served prefix/marker.** VERIFIED gets a distinct `[verified]`
|
||||
prefix; APPROXIMATE gets `[approximate]`. ASK should get its own surface marker (a
|
||||
question is neither). Recommend a distinct, tested marker; pin the exact string at
|
||||
build, not here.
|
||||
|
||||
### 5.2 Holdout gate (no quiet widening)
|
||||
|
||||
Like VERIFIED, ASK serving must be proven on a holdout before it widens live: a
|
||||
validate-first probe over a held-out set confirming (a) served questions are
|
||||
grounded-only (no fabrication escapes on the served path), (b) no family in the flip
|
||||
set hits the dead zone, (c) the GSM8K serving seal is byte-identical (ASK is
|
||||
off-the-metric — it asks, it does not answer — but the probe proves it). Only then does
|
||||
`ask_serving_enabled` go on, one surface at a time.
|
||||
|
||||
---
|
||||
|
||||
## 6. What this is NOT
|
||||
|
||||
- **Not** a dialogue manager / multi-turn state machine — one grounded question, then
|
||||
the existing flow; the answer round-trip is Q2 (`AnswerBinding`, a separate batch).
|
||||
- **Not** a re-render — the served path emits the Q1-D `DeliveredQuestion.question.text`
|
||||
verbatim; no second prose surface.
|
||||
- **Not** a GSM8K-metric move — ASK asks, it never answers; the pinned SHAs and
|
||||
`CLAIMS.md` are untouched. The holdout probe proves it.
|
||||
- **Not** the carve-out flip *yet* — the flip is the terminal act of *this* decision,
|
||||
gated on `ask_serving_enabled` + the §4 dead-zone proof, not on Q1-D.
|
||||
|
||||
---
|
||||
|
||||
## 7. The questions for the ruling
|
||||
|
||||
1. **Gate:** add `ask_serving_enabled = False` (sibling of `estimation_enabled`)? (rec: yes)
|
||||
2. **Layering:** land pass-emission (Layer A) and served delivery (Layer B) together
|
||||
behind the one gate? (rec: yes — Layer A alone only adds double-emission churn)
|
||||
3. **Carve-out flip:** retire `Q1B_ASK_CARVE_OUT` + flip `proposal_allowed` as a single
|
||||
reviewed act, gated on the gate AND the §4 dead-zone proof? (rec: yes)
|
||||
4. **Dead-zone proof:** require a passing "every reachable ask of a flipped family
|
||||
renders" test as a precondition of any flip? (rec: yes — this is the ASK wrong=0 guard)
|
||||
5. **Served marker:** a distinct ASK surface marker (not `[verified]` / `[approximate]`)? (rec: yes)
|
||||
6. **Holdout:** validate-first probe (grounded-only on served path + no dead zone +
|
||||
GSM8K seal byte-identical) before the gate goes on? (rec: yes)
|
||||
|
||||
No served-surface code until this brief is reviewed.
|
||||
|
|
@ -1,144 +0,0 @@
|
|||
# ASK Serving Integration Scoping — ask_serving_enabled, QUESTION_NEEDED, and Carve-Out Retirement
|
||||
|
||||
## 1. Current State
|
||||
|
||||
Currently, the ASK capability exists strictly off-serving.
|
||||
|
||||
- **Residue Capture (Q1-B):** The `core/epistemic_disclosure/limitation.py` module captures `LimitationAssessment` with typed ASK residue including `MissingSlot` and `grounded_terms`. Specifically, `missing_total_count` and `missing_weighted_total` are classified as ask-oriented inside the disclosure layer.
|
||||
- **Carve-Out Safety (Q1-B):** The transitional carve-out constant `Q1B_ASK_CARVE_OUT` is defined in `core/epistemic_disclosure/limitation.py`. The registry in `core/comprehension_attempt/failure_family.py` preserves `proposal_allowed=True` for these carve-out families. This guarantees that the proposal-pile signal remains active and uninterrupted until served ASK is fully wired and verified.
|
||||
- **Grounded Rendering (Q1-C):** The `core/epistemic_questions/render.py` module safely renders `EpistemicQuestion` structures. It enforces structural rendering, ensuring no ungrounded problem-entity names or internal `snake_case` tokens escape to the user. Multi-slot, unmapped, or unsafe cases fall back to `question_unrenderable`.
|
||||
- **Off-serving Delivery (Q1-D):** The delivery infrastructure in `core/epistemic_questions/delivery.py` defines `DeliveredQuestion` and ships the `Terminal.QUESTION_NEEDED` tenant. The resulting off-serving artifacts are written directly to the `teaching/questions/` sink. No served/user-facing surfaces are exposed.
|
||||
- **Default-Dark Gate (Helper):** The helper function `ask_serving_enabled` is implemented in `core/epistemic_questions/serving_gate.py`. It operates in a fail-closed, default-dark manner. If the config field `ask_serving_enabled` is absent, it returns `False`. An explicit, truthy configuration is required to allow served ASK.
|
||||
- **Integration gaps:** Currently, `generate/contemplation/pass_manager.py` does not emit served ASK, `chat/runtime.py` does not expose ASK, and the `Q1B_ASK_CARVE_OUT` remains active (unretired).
|
||||
|
||||
## 2. Non-Negotiable Boundary
|
||||
|
||||
Before ASK can be delivered to any user-facing surface, the following boundaries must be strictly enforced:
|
||||
|
||||
- **Strict Gate Guard:** No served question may be shown without an explicit, active `ask_serving_enabled` configuration check.
|
||||
- **Fail-Closed Default:** The `ask_serving_enabled` helper must default to `False`. A missing or `None` config attribute must evaluate to `False`.
|
||||
- **Prose Encapsulation:** The serving layer must not construct or mutate question prose directly. It may only consume pre-rendered `EpistemicQuestion` / `DeliveredQuestion` structures produced by Q1-C/Q1-D.
|
||||
- **No Contentless Delivery:** An unrenderable ASK must never be promoted to `QUESTION_NEEDED`. Contentless `QUESTION_NEEDED` outcomes are strictly forbidden.
|
||||
- **Carve-Out Preservation:** The `Q1B_ASK_CARVE_OUT` must remain active and unchanged until served ASK is fully verified. No proposal signal may be lost before a served `QUESTION_NEEDED` is verified live.
|
||||
- **Sink Distinction:** The `question_only` (teaching/questions) sink must remain logically and physically distinct from the `proposal_only` (teaching/proposals) sink.
|
||||
- **Zero Impact on Claims:** No benchmark, `CLAIMS.md`, or performance metrics may be modified by this scoping documentation.
|
||||
|
||||
## 3. Proposed Served Gate: ask_serving_enabled
|
||||
|
||||
The serving gate helper exists under `core/epistemic_questions/serving_gate.py`. This document scoping defines how future served-surface code must interact with the gate:
|
||||
|
||||
- **Helper Invariant:** Future code in the served-surface layer (e.g., `chat/runtime.py`) must verify `ask_serving_enabled(config)` before delivering any `QUESTION_NEEDED` response to a user.
|
||||
- **Off-Serving Isolation:** The gate controls served (user-visible) output only. It must not disable or interfere with off-serving artifacts written to the `teaching/questions/` directory.
|
||||
|
||||
### Serving Gate Policy
|
||||
|
||||
| Config State | ask_serving_enabled(...) | Served ASK Allowed? |
|
||||
| --- | --- | --- |
|
||||
| Missing field | `False` | No |
|
||||
| `None` / Default config | `False` | No |
|
||||
| Explicit `False` | `False` | No |
|
||||
| Explicit `True` | `True` | Only if rendering and delivery obligations pass |
|
||||
|
||||
## 4. pass_manager Integration Boundary
|
||||
|
||||
This section defines the future integration interface for `generate/contemplation/pass_manager.py`. This is scoping only; no execution is performed here.
|
||||
|
||||
1. **Evaluation:** The refusal/comprehension flow yields a `ComprehensionAttempt`.
|
||||
2. **Assessment:** A `LimitationAssessment` is derived from the attempt.
|
||||
3. **Resolution Action:** If `resolution_action == "ask_question"`:
|
||||
- The pipeline invokes `deliver_ask(assessment)`.
|
||||
- `deliver_ask` calls `render_question` exactly once.
|
||||
- If the question is renderable, the final `DeliveryOutcome` terminal becomes `QUESTION_NEEDED`.
|
||||
- If the question is unrenderable, the outcome falls back to the standing disposition (e.g., proposal or refusal).
|
||||
4. **Gate Enforced downstream:** `generate/contemplation/pass_manager.py` may produce/record ASK delivery outcomes later, but user-visible exposure remains gated downstream by `ask_serving_enabled` (e.g., in `chat/runtime.py`).
|
||||
|
||||
- **Constraints:**
|
||||
- `pass_manager` must not contain prose templates or formatting rules.
|
||||
- `pass_manager` must not construct `DeliveredQuestion` manually (it must delegate to `deliver_ask`).
|
||||
- `pass_manager` must never bypass `deliver_ask`.
|
||||
- `pass_manager` must never emit a contentless `QUESTION_NEEDED`.
|
||||
|
||||
## 5. Served Behavior Matrix
|
||||
|
||||
| Assessment / Rendering | Gate Disabled | Gate Enabled |
|
||||
| --- | --- | --- |
|
||||
| Renderable `ask_question` | No served ASK; existing refusal/proposal behavior preserved | Served `QUESTION_NEEDED` is allowed |
|
||||
| Unrenderable `ask_question` | Standing fallback; no `QUESTION_NEEDED` | Standing fallback; no `QUESTION_NEEDED` |
|
||||
| Non-ASK limitation | Unaffected | Unaffected |
|
||||
| `missing_total_count` / `missing_weighted_total` (carve-out) | Proposal signal preserved | Served ASK only after carve-out retirement proof |
|
||||
| Multi-slot ASK | Unrenderable fallback | Unrenderable fallback |
|
||||
|
||||
*Note: Gate enabled is a necessary but not sufficient condition for serving. The renderable checks and delivery invariants must also pass.*
|
||||
|
||||
## 6. Q1B_ASK_CARVE_OUT Retirement Conditions
|
||||
|
||||
The transitional carve-out constant `Q1B_ASK_CARVE_OUT` can only be retired and removed from `core/epistemic_disclosure/limitation.py` when the following milestones are met and verified:
|
||||
|
||||
1. The `ask_serving_enabled` helper is tested and confirmed default-dark.
|
||||
2. The `pass_manager.py` ASK integration is fully implemented behind the gate.
|
||||
3. Served `QUESTION_NEEDED` terminal behavior is successfully tested using renderable `EpistemicQuestion` instances.
|
||||
4. Unrenderable ASK paths are verified to fall back correctly, never emitting `QUESTION_NEEDED`.
|
||||
5. Carve-out keys `missing_total_count` and `missing_weighted_total` are proven to yield safe renderable questions (or safe standing fallbacks).
|
||||
6. A dedicated no-question/no-proposal dead-zone validation test is introduced and passes.
|
||||
7. The registry in `core/comprehension_attempt/failure_family.py` flips `proposal_allowed=False` for the carve-out families without dropping signal.
|
||||
8. The `teaching/questions/` (`question_only`) sink remains physically distinct from the `teaching/proposals/` (`proposal_only`) sink.
|
||||
9. Smoke, contemplation, proposal, and disclosure test suites remain fully green.
|
||||
|
||||
*Until all 9 conditions are met, the registry's `proposal_allowed=True` setting for these families must remain.*
|
||||
|
||||
## 7. No-Dead-Zone Proof Obligation
|
||||
|
||||
### The Dead-Zone Hazard
|
||||
If `proposal_allowed` is flipped to `False` on a failure family before ASK serving is enabled/renderable, an input falling into that family would yield no proposal signal AND no served question. This results in a silent loss of capability (a dead zone), violating the `wrong=0` refusal-first discipline.
|
||||
|
||||
### Required Proof Before Retirement
|
||||
Before the carve-out is retired, tests must prove that for each carve-out family:
|
||||
- If `proposal_allowed` is removed, either a served `QUESTION_NEEDED` is emitted safely, or the standing fallback preserves an admissible signal.
|
||||
- The test suite must assert failure if an evaluation result yields neither a proposal nor a valid served question.
|
||||
|
||||
### Suggested Test Names
|
||||
- `test_ask_serving_disabled_preserves_existing_proposal_signal`
|
||||
- `test_carveout_retirement_has_no_question_no_proposal_dead_zone`
|
||||
- `test_unrenderable_ask_never_emits_question_needed`
|
||||
- `test_pass_manager_uses_deliver_ask_not_direct_rendering`
|
||||
- `test_question_only_not_proposal_only`
|
||||
|
||||
## 8. Required Future Tests Before Wiring
|
||||
|
||||
The implementation PR must include tests asserting the following behaviors:
|
||||
|
||||
- **Gate Default:** `ask_serving_enabled` defaults to `False`.
|
||||
- **Config Completeness:** A missing config field evaluates to `False`.
|
||||
- **Opt-in Activation:** An explicit `True` configuration is required to allow served ASK.
|
||||
- **Gate Override:** If the gate is disabled, no served ASK is emitted even if `deliver_ask` produces a `QUESTION_NEEDED` outcome.
|
||||
- **Rendering Obligation:** If the gate is enabled, the output still requires a renderable `EpistemicQuestion`.
|
||||
- **Unrenderable Fallback:** Unrenderable ASK instances are never served.
|
||||
- **Prose Isolation:** `pass_manager` does not construct prose templates or strings.
|
||||
- **Delegated Delivery:** `pass_manager` delegates entirely to `deliver_ask` and does not call `render_question` directly.
|
||||
- **No Empty Terminals:** No contentless `QUESTION_NEEDED` outcomes are permitted.
|
||||
- **Sink Safety:** The `question_only` artifact is written to the correct, distinct location.
|
||||
- **Carve-out Lock:** `Q1B_ASK_CARVE_OUT` is preserved until both the gate and the dead-zone proofs are active.
|
||||
- **Registry Guard:** Flipped registries are blocked until all retirement conditions are met.
|
||||
|
||||
## 9. Non-Claims
|
||||
|
||||
This scoping document establishes architectural boundaries only:
|
||||
|
||||
- **No Implementation:** It does not implement served ASK.
|
||||
- **No Wiring:** It does not wire `generate/contemplation/pass_manager.py` or `chat/runtime.py`.
|
||||
- **No Retirement:** It does not retire `Q1B_ASK_CARVE_OUT` or flip any registry `proposal_allowed` flags.
|
||||
- **No Metric Changes:** It does not alter GSM8K benchmark claims or refusal behaviors.
|
||||
- **No General Intelligence:** It does not make ASK generally intelligent; it simply bounds the served-surface gate.
|
||||
|
||||
## 10. Recommended Next Slice
|
||||
|
||||
To safely approach the implementation, the next sequential slices are recommended:
|
||||
|
||||
### Slice 1: Configuration Definition (No Wiring)
|
||||
- Add a concrete configuration field `RuntimeConfig.ask_serving_enabled: bool = False` (if not already present).
|
||||
- Ensure the helper `ask_serving_enabled(...)` references this configuration.
|
||||
- Write configuration tests verifying the default and override values.
|
||||
- Do not wire `pass_manager` or modify runtime loops.
|
||||
|
||||
### Slice 2: pass_manager Integration
|
||||
- Wire `pass_manager` to call `deliver_ask` without constructing prose; any user-visible ASK exposure remains behind `ask_serving_enabled`; preserve `Q1B_ASK_CARVE_OUT`.
|
||||
|
|
@ -1,317 +0,0 @@
|
|||
# Stage 2 Disclosure Bus Implementation Map — session pivot to current slices
|
||||
|
||||
**Date:** 2026-06-09
|
||||
**Status:** implementation map / controlling checklist — docs only
|
||||
|
||||
This document reconciles the June 8 session-design pivot with the implementation slices that have now landed. It is not a new architecture. It is the map that prevents the work from drifting into isolated feature patches.
|
||||
|
||||
## 0. Why this map exists
|
||||
|
||||
The recent implementation work touched ASK and VERIFIED serving foundations in small slices. That was correct, but the controlling plan is larger than any one PR:
|
||||
|
||||
- `docs/sessions/2026-06-08-practice-attempts-and-servability-blade.md`
|
||||
- `docs/sessions/2026-06-08-epistemic-question-articulation-first-skill-of-contemplation.md`
|
||||
- `docs/analysis/stage2-epistemic-disclosure-bus-verified-v1-scoping-2026-06-08.md`
|
||||
- `docs/analysis/q1-epistemic-question-articulation-v1-scoping-2026-06-08.md`
|
||||
- `docs/analysis/ask-serving-integration-scoping-2026-06-09.md`
|
||||
- `docs/analysis/verified-serving-wiring-scoping-2026-06-09.md`
|
||||
|
||||
The governing pivot is:
|
||||
|
||||
```text
|
||||
wrong=0 is not a binary answer/refuse command.
|
||||
wrong=0 is no false presentation of epistemic status.
|
||||
practice/contemplation may explore typed, isolated candidates.
|
||||
serving must disclose only what is truthfully labelable.
|
||||
```
|
||||
|
||||
Therefore the served surface should not grow by one-off feature branches. It should grow by activating tenants on the already-scoped Epistemic Disclosure Bus.
|
||||
|
||||
## 1. Controlling doctrine from the session docs
|
||||
|
||||
### 1.1 Practice versus serving
|
||||
|
||||
The practice/servability session separates two lanes:
|
||||
|
||||
| Lane | Purpose | License | Primary danger |
|
||||
| --- | --- | --- | --- |
|
||||
| practice / contemplation | explore, attempt, eliminate, learn | typed and isolated candidates may exist | getting stuck / never learning |
|
||||
| serving | emit to a user/downstream surface | no false presentation of epistemic status | misrepresenting uncertainty as truth |
|
||||
|
||||
The practical consequence is that internal attempts, ASK artifacts, proposals, and verification proofs may exist off-serving without being user-visible. Serving requires an explicit governance decision.
|
||||
|
||||
### 1.2 The servability blade is not a new parallel system
|
||||
|
||||
The session document explicitly amends the initial sketch: the servability blade must reuse the already-shipped ADR-0206 response-governance seam (`ReachPolicy`, `govern_response`, `shape_surface`) rather than invent a new parallel object.
|
||||
|
||||
For Stage 2 work, every served-surface slice must therefore ask:
|
||||
|
||||
```text
|
||||
Does this activate the existing governance seam, or is it bypassing it?
|
||||
```
|
||||
|
||||
A bypass is architectural drift.
|
||||
|
||||
### 1.3 ASK is typed intake, not chat clarification
|
||||
|
||||
The question-articulation session defines a question as a typed request for missing state. The first contemplative move is not “ask a question”; it is:
|
||||
|
||||
```text
|
||||
What is preventing resolution — and what KIND of limitation is it?
|
||||
```
|
||||
|
||||
Only limitations of kind `missing_information` or `ambiguous_structure` may become ASK. Capability gaps propose. Hard boundaries refuse. Contradictions report. Input-shape cases step aside.
|
||||
|
||||
### 1.4 QUESTION_NEEDED and PROPOSAL_EMITTED are siblings
|
||||
|
||||
The session doc draws a hard line:
|
||||
|
||||
| Terminal | Meaning |
|
||||
| --- | --- |
|
||||
| `QUESTION_NEEDED` | the input is under-specified but the problem is potentially knowable after one missing datum is supplied |
|
||||
| `PROPOSAL_EMITTED` | the input is sufficiently specified but CORE lacks the transform/capability |
|
||||
|
||||
This distinction must be preserved in artifacts, paths, tests, and served surfaces.
|
||||
|
||||
## 2. Stage 2 bus frame
|
||||
|
||||
The Stage 2 scoping doc reframes the frontier:
|
||||
|
||||
```text
|
||||
What may CORE disclose through the served surface — and under what governed disposition?
|
||||
```
|
||||
|
||||
The bus is the consolidating view:
|
||||
|
||||
```text
|
||||
EpistemicState + LimitationAssessment + proof/license evidence
|
||||
-> ServedDisposition / disclosure decision
|
||||
-> shaped surface through the governance seam
|
||||
```
|
||||
|
||||
VERIFIED is not the whole system. It is one tenant. ASK is another. Scope-boundary explanation, contradiction reporting, proposal-only, partial progress, multiple candidates, and provisional working answers are later tenants or modes on the same governance surface.
|
||||
|
||||
## 3. Current implementation state
|
||||
|
||||
### 3.1 ASK tenant — landed foundation
|
||||
|
||||
The following ASK foundation is now present:
|
||||
|
||||
```text
|
||||
Q1-B typed ASK residue / MissingSlot / LimitationAssessment ask_question
|
||||
Q1-C grounded-only renderer
|
||||
Q1-D off-serving DeliveredQuestion / deliver_ask / teaching/questions sink
|
||||
ask_serving_enabled helper, default-dark
|
||||
RuntimeConfig.ask_serving_enabled = False
|
||||
pass_manager off-serving ASK integration behind exercise_ask
|
||||
ContemplationResult.question_path separated from proposal_path
|
||||
```
|
||||
|
||||
Important preserved boundaries:
|
||||
|
||||
```text
|
||||
no chat/runtime.py serving path
|
||||
no served ASK surface
|
||||
no Q1B_ASK_CARVE_OUT retirement
|
||||
no proposal_allowed registry flip
|
||||
no CLAIMS or benchmark movement
|
||||
boundary-first remains before ASK
|
||||
question_only sink remains distinct from proposal_only sink
|
||||
```
|
||||
|
||||
### 3.2 ASK tenant — not yet done
|
||||
|
||||
The following are still open:
|
||||
|
||||
```text
|
||||
served ASK / QUESTION_NEEDED through the governance bus
|
||||
Q2 AnswerBinding and re-run through the owner organ
|
||||
no-question/no-proposal dead-zone proof
|
||||
Q1B_ASK_CARVE_OUT retirement proof
|
||||
registry flip for missing_total_count / missing_weighted_total
|
||||
broader ASK families beyond the current typed-residue subset
|
||||
```
|
||||
|
||||
### 3.3 VERIFIED tenant — landed foundation
|
||||
|
||||
The following VERIFIED foundation is now present:
|
||||
|
||||
```text
|
||||
P1-A VERIFIED contract
|
||||
P1-B off-serving gold-setup-backed R2 producer
|
||||
P1-C bound_slots_digest proof hardening
|
||||
verified_serving_enabled helper, default-dark
|
||||
RuntimeConfig.verified_serving_enabled = False
|
||||
verified serving scoping / gold-free independence doc
|
||||
```
|
||||
|
||||
Important preserved boundaries:
|
||||
|
||||
```text
|
||||
no served VERIFIED surface
|
||||
no verify.py consumption
|
||||
no eval-gold-backed proof in serving
|
||||
no CLAIMS or benchmark movement
|
||||
no gold-free independent reader yet
|
||||
```
|
||||
|
||||
### 3.4 VERIFIED tenant — not yet done
|
||||
|
||||
The following are still open:
|
||||
|
||||
```text
|
||||
gold-free independent reader/proof source
|
||||
poison fixture harness
|
||||
holdout-gated verification harness
|
||||
deterministic replay digest proof for served verification traces
|
||||
verify.py consumption of contract verdict
|
||||
served [verified] surface behind verified_serving_enabled
|
||||
```
|
||||
|
||||
## 4. Recent PRs in plan terms
|
||||
|
||||
| PR | Plan role | Status |
|
||||
| --- | --- | --- |
|
||||
| #664 | Q1-B typed ASK residue + carve-out | merged |
|
||||
| #666 | Q1-C grounded-only renderer | merged |
|
||||
| #667 | Q1-D decision record | merged |
|
||||
| #668 | Q1-D off-serving delivery | merged |
|
||||
| #670 | ASK default-dark gate helper | merged |
|
||||
| #671 | ASK serving / carve-out retirement scoping | merged |
|
||||
| #672 | VERIFIED serving / gold-free independence scoping | merged |
|
||||
| #673 | VERIFIED default-dark gate helper | merged |
|
||||
| #674 | explicit ASK + VERIFIED config fields | merged |
|
||||
| #675 | off-serving ASK pass_manager integration | merged |
|
||||
| #676 | split question_path from proposal_path | merged |
|
||||
|
||||
This sequence was mostly faithful to the plan, but the controlling label should be:
|
||||
|
||||
```text
|
||||
ASK tenant implementation on the Epistemic Disclosure Bus foundation
|
||||
```
|
||||
|
||||
not merely:
|
||||
|
||||
```text
|
||||
question feature / pass_manager feature
|
||||
```
|
||||
|
||||
## 5. Correct next-slice ordering
|
||||
|
||||
### 5.1 ASK next code slice — bus activation, not runtime bypass
|
||||
|
||||
The next ASK code slice should not be described as “wire chat/runtime.” It should be:
|
||||
|
||||
```text
|
||||
Activate ASK/clarify as a served tenant through the disclosure/governance bus.
|
||||
```
|
||||
|
||||
Minimum requirements:
|
||||
|
||||
- use `ask_serving_enabled(config)` as a necessary gate;
|
||||
- consume `ContemplationResult.question_path` / delivered question artifact;
|
||||
- do not construct question prose in serving;
|
||||
- do not serve unrenderable ASK;
|
||||
- preserve `Q1B_ASK_CARVE_OUT`;
|
||||
- preserve proposal signal when gate is disabled;
|
||||
- do not flip `proposal_allowed`;
|
||||
- do not bypass `govern_response` / `shape_surface` if that seam is applicable;
|
||||
- if the current governance seam cannot carry ASK yet, stop and scope the exact missing adapter first.
|
||||
|
||||
Expected test names or equivalents:
|
||||
|
||||
```text
|
||||
test_ask_serving_disabled_preserves_existing_proposal_signal
|
||||
test_ask_serving_enabled_surfaces_question_needed_from_artifact
|
||||
test_unrenderable_ask_never_serves_question_needed
|
||||
test_question_only_not_proposal_only
|
||||
test_served_ask_does_not_construct_question_prose
|
||||
test_served_ask_uses_governance_bus_not_parallel_runtime_path
|
||||
```
|
||||
|
||||
### 5.2 ASK following slice — no-dead-zone proof
|
||||
|
||||
After served ASK exists behind the gate, prove that the carve-out families cannot fall into a no-question/no-proposal dead zone.
|
||||
|
||||
Required proof shape:
|
||||
|
||||
```text
|
||||
for missing_total_count / missing_weighted_total:
|
||||
gate disabled -> proposal signal preserved
|
||||
gate enabled + renderable -> QUESTION_NEEDED served safely
|
||||
gate enabled + unrenderable -> standing fallback, never contentless QUESTION_NEEDED
|
||||
no case yields neither proposal nor valid question
|
||||
```
|
||||
|
||||
### 5.3 ASK later slice — Q1B_ASK_CARVE_OUT retirement
|
||||
|
||||
Only after the no-dead-zone proof passes may a later PR consider:
|
||||
|
||||
```text
|
||||
proposal_allowed=False for missing_total_count / missing_weighted_total
|
||||
remove/retire Q1B_ASK_CARVE_OUT
|
||||
```
|
||||
|
||||
That PR must be explicit, separate, and guarded by tests.
|
||||
|
||||
### 5.4 VERIFIED next code slice — gold-free independent reader
|
||||
|
||||
The next VERIFIED code slice is not served VERIFIED. It is:
|
||||
|
||||
```text
|
||||
Design/prototype a gold-free independent reader/proof source.
|
||||
```
|
||||
|
||||
Requirements:
|
||||
|
||||
- no eval-gold setup in serving;
|
||||
- no gold answer;
|
||||
- no benchmark fixture;
|
||||
- distinct primary/independent reader lineages;
|
||||
- convergent canonical read digests;
|
||||
- strict rejection of same-reader-twice;
|
||||
- strict rejection of second-solver-over-one-read;
|
||||
- off-serving only.
|
||||
|
||||
### 5.5 VERIFIED later slices
|
||||
|
||||
Only after gold-free independence exists:
|
||||
|
||||
```text
|
||||
poison fixture harness
|
||||
holdout-gated verification harness
|
||||
verify.py consumption scoping
|
||||
served [verified] behind verified_serving_enabled
|
||||
```
|
||||
|
||||
## 6. Explicit anti-drift rules
|
||||
|
||||
The following are forbidden unless a future ADR/decision record explicitly reopens them:
|
||||
|
||||
```text
|
||||
no standalone chat/runtime ASK patch that bypasses the bus
|
||||
no served ASK without ask_serving_enabled
|
||||
no served ASK that constructs prose instead of consuming DeliveredQuestion
|
||||
no contentless QUESTION_NEEDED
|
||||
no question artifact under proposal_path
|
||||
no proposal artifact under question_path
|
||||
no Q1B carve-out retirement before no-dead-zone proof
|
||||
no served VERIFIED from eval-gold producer
|
||||
no verify.py VERIFIED consumption before gold-free independent reader + poison/holdout harness
|
||||
no direct EpistemicState.VERIFIED construction outside the contract route
|
||||
no CLAIMS or benchmark movement from off-serving artifacts
|
||||
```
|
||||
|
||||
## 7. Current status checkpoint
|
||||
|
||||
As of the `question_path` cleanup landing, the project is here:
|
||||
|
||||
```text
|
||||
Stage 2 bus doctrine: scoped, not fully active
|
||||
ASK tenant: off-serving foundation complete enough for served-bus scoping
|
||||
VERIFIED tenant: contract/gate foundation complete, serving blocked on gold-free independence
|
||||
Next ASK move: bus-governed served ASK, no carve-out retirement
|
||||
Next VERIFIED move: gold-free independent reader, no served VERIFIED
|
||||
```
|
||||
|
||||
Use this document as the checklist before issuing the next implementation brief.
|
||||
|
|
@ -1,183 +0,0 @@
|
|||
# VERIFIED serving-wiring — `verified_serving_enabled` — scoping brief
|
||||
|
||||
**Date:** 2026-06-08 · **Status:** scoping (NO CODE) · **HOLD for review** ·
|
||||
**Branch:** `docs/serving-integration-scoping`
|
||||
|
||||
**What this brief is.** The scope for the served-surface decision the VERIFIED lane
|
||||
deferred. The off-serving VERIFIED lane is complete and on main:
|
||||
|
||||
```text
|
||||
P1-A the VERIFIED contract (verified_contract.py — two independent reads converge)
|
||||
P1-B gold-setup-backed producer (evals/constraint_oracle/verified_producer.py — OFF-SERVING)
|
||||
P1-C bound_slots_digest (a separable, load-bearing proof obligation)
|
||||
```
|
||||
|
||||
P1-B verifies 7/13 real R2 problems with wrong=0 — but it is **gold-setup-backed**, so
|
||||
it is structurally off-serving: the independent read is the INV-25 hand-authored gold
|
||||
SETUP, which is not available at serving time. This brief scopes what a *serving-time*
|
||||
VERIFIED would require — and why it cannot reuse P1-B. **No code here.**
|
||||
|
||||
> Companions: [[ask-serving-integration-scoping-2026-06-08]] (the ASK half of the same
|
||||
> "where off-serving stops" line), [[VERIFIED-canonical-comparison-scoping-2026-06-06]]
|
||||
> (the validate-first probe that already KILLED the naive fold-reader producer),
|
||||
> [[stage2-epistemic-disclosure-bus-verified-v1-scoping-2026-06-08]] (Doc 1).
|
||||
|
||||
---
|
||||
|
||||
## 1. The seam (grounded) — and why it is still inert by design
|
||||
|
||||
`generate/derivation/verify.py::_canonically_verified(verified, problem_text, policy)`
|
||||
is the ADR-0206 §5 VERIFIED gate — *the only thing that may license a math answer past
|
||||
gold* (resolve a disagreement STRICT refuses). It **returns `None` today**, so the
|
||||
widening is structurally inert: disagreement refuses regardless of `policy`, preserving
|
||||
absolute `wrong == 0`. Its own docstring states the bright line this brief must honour:
|
||||
|
||||
> "A reliability *license* (statistical) must NEVER substitute here: math serving is
|
||||
> absolute-wrong=0, not disclosed like the cognition path."
|
||||
|
||||
So VERIFIED serving = replacing that `return None` with a producer that returns a
|
||||
derivation **only when it is canonically VERIFIED** (proven correct, not merely sound),
|
||||
behind a kill switch, proven on a holdout. Everything below is the eligibility bar for
|
||||
that producer.
|
||||
|
||||
---
|
||||
|
||||
## 2. The five things this decision must pin
|
||||
|
||||
```text
|
||||
1. the gold-free independent-reader requirement — the crux
|
||||
2. verified_serving_enabled — the kill switch
|
||||
3. holdout gates — validate-first, no quiet widening
|
||||
4. proof-producer eligibility — what may plug into _canonically_verified
|
||||
5. the explicit ban on eval-gold-backed serving — why P1-B cannot serve
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. The gold-free independent-reader requirement (the crux)
|
||||
|
||||
VERIFIED means **two independent reads that converge on one canonical structure**
|
||||
(P1-A): a faithful solve of a *wrong read* is caught because the independent read
|
||||
disagrees. P1-B's two reads are `read_constraint_problem` (the engine reader) vs the
|
||||
**gold-authored setup**. At serving time there is no gold. So serving-time VERIFIED
|
||||
needs a **second, gold-free reader** whose disagreement is the safety mechanism.
|
||||
|
||||
The hard constraints, from the killed-probe doc and CLAUDE.md:
|
||||
|
||||
- **Independence must be in the READING, not the solving.** Back-substitution catches
|
||||
solve-errors, never read-errors. Two solvers over one reading is *fake* independence.
|
||||
The second reader must parse the problem into the same `ConstraintProblem` structure
|
||||
by a **genuinely different route** (different lineage, asserted by the
|
||||
`SAME_READER_LINEAGE` firewall / INV-27 reader-disjointness), and converge on the
|
||||
same canonical signature.
|
||||
- **No eval gold in the read** (§7). The second reader may not consult, hash, or be
|
||||
derived from any gold artifact — not the answer, not the setup. If it needs gold to
|
||||
read, it is P1-B, and P1-B does not serve.
|
||||
- **Conservative refuse-on-doubt carries wrong=0.** The second reader, like the first,
|
||||
refuses when uncertain. VERIFIED fires only on *agreement of two confident,
|
||||
independent reads*; any refusal or disagreement → STRICT refuses (today's behaviour).
|
||||
|
||||
**Eligibility, stated as a gate:** a serving-time VERIFIED producer is eligible **only
|
||||
if** it exhibits two reads with (a) distinct, firewall-asserted lineages, (b) neither
|
||||
read derived from gold, (c) convergence on one canonical `ConstraintProblem` signature,
|
||||
(d) back-substitution + boundary-clear + bound-slots (the P1-A/P1-C obligations), and
|
||||
(e) refuse-preferring failure. Absent any one, it stays `None`.
|
||||
|
||||
Whether such a second R2 reader *exists yet* is the open empirical question — the killed
|
||||
fold-reader probe is the cautionary precedent: complementary readers were ~98% wrong on
|
||||
the refused set. **This brief does not assume one exists; it sets the bar a candidate
|
||||
must clear, validate-first, before any wiring.**
|
||||
|
||||
---
|
||||
|
||||
## 4. `verified_serving_enabled` — the kill switch
|
||||
|
||||
Add `verified_serving_enabled: bool = False` to `core/config.py` (sibling of
|
||||
`estimation_enabled`). Default **off**. When off, `_canonically_verified` returns `None`
|
||||
unconditionally (today's inert state) regardless of any producer being present. The
|
||||
switch is the single audited place that lights the seam, and it stays off until §5.
|
||||
|
||||
---
|
||||
|
||||
## 5. Holdout gates — validate-first, no quiet widening
|
||||
|
||||
VERIFIED may not widen live until proven on a **held-out** set it never trained or
|
||||
tuned on (INV-25 discipline; the killed-probe doc's validate-first rule). The gate, in
|
||||
order:
|
||||
|
||||
1. **Holdout probe (off-serving):** run the candidate producer over a held-out R2 set
|
||||
with gold answers withheld; require **wrong == 0** on everything it marks VERIFIED,
|
||||
and that everything it cannot independently verify it *refuses* (over-refusal is
|
||||
acceptable; one wrong is disqualifying).
|
||||
2. **Seal byte-identity:** the GSM8K candidate-graph serving seal (pinned SHAs) stays
|
||||
byte-identical — VERIFIED widens a *different* surface (R2 constraint answers), it
|
||||
must not perturb the sealed lane.
|
||||
3. **Disagreement-still-refuses:** a test proving that with the producer wired and the
|
||||
switch ON, a faithful solve of a deliberately wrong read still refuses (the P1-A
|
||||
poison test, now on the served path).
|
||||
4. Only then does `verified_serving_enabled` go on, **one surface at a time** (R2
|
||||
first; R4 second), each with its own holdout pass.
|
||||
|
||||
---
|
||||
|
||||
## 6. The served surface — a distinct `[verified]` disclosure
|
||||
|
||||
When wired and enabled, a VERIFIED R2 answer is served under its **own** disclosure
|
||||
claim/marker — the locked decision from Doc 1's review: *VERIFIED gets a distinct
|
||||
served disclosure mode + `[verified]` prefix, NEVER reused from `[approximate]`*.
|
||||
VERIFIED is a *license* claim (more licensed than gold-strict), not a *speculation*
|
||||
claim. The route to it is the only sanctioned one: `disclosure_for_verification(result)`
|
||||
→ `(EpistemicState.VERIFIED, DisclosureClaim.VERIFIED)` → `ServedDisposition.DISCLOSE`
|
||||
under the P0-3 guard (which already degrades an unbacked VERIFIED claim to COMMIT).
|
||||
|
||||
---
|
||||
|
||||
## 7. The explicit ban on eval-gold-backed serving (the load-bearing non-claim)
|
||||
|
||||
**P1-B must never become a serving producer.** It is gold-setup-backed: its independent
|
||||
read is the hand-authored gold SETUP. Wiring it into `_canonically_verified` would mean
|
||||
the engine "verifies" by consulting the answer key's structure — circular, and a
|
||||
wrong=0 fiction (it would serve VERIFIED on exactly the problems it was handed the setup
|
||||
for). The ban, stated so a test can enforce it:
|
||||
|
||||
- `_canonically_verified` (and any serving producer behind it) may import **nothing**
|
||||
from `evals/` — no gold loader, no `gold_to_problem`, no `r2_gold`. An AST/import test
|
||||
asserts the serving path is gold-free, the mirror of the off-serving AST tests.
|
||||
- P1-B stays in `evals/` precisely so this boundary is structural: `evals → core` is the
|
||||
allowed direction; `core/generate serving → evals` is forbidden.
|
||||
- "Verified on a holdout" (§5) is **not** the same as "verified by gold at serving" —
|
||||
the holdout *measures* a gold-free producer; it never *feeds* gold into one.
|
||||
|
||||
This is the VERIFIED analogue of the GSM8K "candidate-graph owns the metric; no bridge
|
||||
re-enables without sealed/independent wrong=0" discipline.
|
||||
|
||||
---
|
||||
|
||||
## 8. What this is NOT
|
||||
|
||||
- **Not** a claim that a serving-time VERIFIED producer exists — §3 sets the eligibility
|
||||
bar; whether an R2 second reader clears it is an open, validate-first question.
|
||||
- **Not** a reliability/statistical license at the math seam — that path is the
|
||||
cognition `[approximate]` disclosure; math serving is absolute wrong=0 (§1 bright line).
|
||||
- **Not** a GSM8K-seal move — VERIFIED widens R2 constraint answers, seal stays
|
||||
byte-identical (§5.2).
|
||||
- **Not** P1-B promoted to serving — explicitly banned (§7); P1-B stays off-serving.
|
||||
|
||||
---
|
||||
|
||||
## 9. The questions for the ruling
|
||||
|
||||
1. **Eligibility bar:** adopt §3 (a)–(e) as the gate any serving VERIFIED producer must
|
||||
clear — gold-free second reader, independence-in-the-reading, refuse-preferring? (rec: yes)
|
||||
2. **Gate:** add `verified_serving_enabled = False` (sibling of `estimation_enabled`),
|
||||
`_canonically_verified` stays `None` while off? (rec: yes)
|
||||
3. **Holdout:** require the §5 validate-first sequence (wrong=0 on holdout + seal
|
||||
byte-identity + disagreement-still-refuses) before the switch, R2-first? (rec: yes)
|
||||
4. **Gold ban:** enforce the §7 import ban (serving path imports nothing from `evals/`)
|
||||
with a test, keeping P1-B off-serving forever? (rec: yes)
|
||||
5. **Surface:** serve VERIFIED under its distinct `[verified]` marker via
|
||||
`disclosure_for_verification` only? (rec: yes — the locked Doc 1 decision)
|
||||
|
||||
No served-surface code until this brief is reviewed. Pairs with
|
||||
[[ask-serving-integration-scoping-2026-06-08]] — together they draw the full line where
|
||||
off-serving stops.
|
||||
|
|
@ -1,240 +0,0 @@
|
|||
# VERIFIED Serving Wiring Scoping — verified_serving_enabled and Gold-Free Independence
|
||||
|
||||
## 1. Current State
|
||||
|
||||
Currently, the VERIFIED verification pipeline exists off-serving only. No served [verified] surface exists, no serving-time config or gate exists, and no `verify.py` serving wiring is implemented.
|
||||
|
||||
The existing off-serving spine consists of:
|
||||
- **P1-A**: Defines the contract in `core/epistemic_disclosure/verified_contract.py`. This contract includes:
|
||||
- `VerificationProof` (the data structure containing digests and lineages)
|
||||
- `VerificationObligation` (the strict checklist of obligations)
|
||||
- `evaluate_verification` (the pure-logic evaluation function enforcing the contract)
|
||||
- `disclosure_for_verification` (the single sanctioned route to transition a result to the verified state)
|
||||
- Defines the meaning of **VERIFIED**: two independent reads must converge on a single canonical structure. This requires:
|
||||
- Independent reads (`primary_reader_lineage` must not equal `independent_reader_lineage`)
|
||||
- Convergent canonical read digests (`primary_read_digest` must equal `independent_read_digest`)
|
||||
- `derivation_digest` present
|
||||
- `bound_slots_digest` present
|
||||
- `back_substitution_digest` present
|
||||
- `boundary_clear` is `True`
|
||||
- `contradiction_clear` is `True`
|
||||
- `limitation` is `None`
|
||||
- **P1-B**: Adds an off-serving R2 verification producer in `evals/constraint_oracle/verified_producer.py`:
|
||||
- Extracts the primary read from the problem text via `read_constraint_problem(text)`.
|
||||
- Obtains the independent read from a hand-authored gold setup (`gold_setup`).
|
||||
- The gold answer itself never enters the verification process (the gold structure setup signature is matched, not the answer).
|
||||
- The producer is strictly for evaluation and runs off-serving only.
|
||||
- **P1-C**: Introduces `bound_slots_digest` as a distinct, separable proof obligation to ensure the answer binds only to the stated slots (asked unknowns) and not phantom slots.
|
||||
- **Serving Isolation**:
|
||||
- No served VERIFIED surface exists.
|
||||
- No `verified_serving_enabled` gate exists.
|
||||
- No `verify.py` serving integration exists.
|
||||
|
||||
## 2. Hard Non-Claim
|
||||
|
||||
The gold-setup-backed producer implemented in `evals/constraint_oracle/verified_producer.py` **cannot** be used for serving.
|
||||
|
||||
What P1-B proves:
|
||||
- The VERIFIED contract logic is sound and functional.
|
||||
- The verification producer can successfully assemble valid `VerificationProof` objects.
|
||||
- Wrong reader structures diverge and correctly fail the independent verification check.
|
||||
- The actual gold answer is not required to verify the canonical structural solve.
|
||||
|
||||
What P1-B does **NOT** prove:
|
||||
- Serving-time independence (since gold setups do not exist for runtime user requests).
|
||||
- Gold-free verification.
|
||||
- User-visible served `[verified]` status.
|
||||
- Benchmark or `CLAIMS.md` movement.
|
||||
- AGI capabilities.
|
||||
|
||||
Any served `[verified]` status requires a gold-free independent read source at serving time.
|
||||
|
||||
## 3. Proposed Served Gate: verified_serving_enabled
|
||||
|
||||
To control the rollout of serving-time verification, a future configuration gate must be defined:
|
||||
|
||||
- The configuration field `verified_serving_enabled` does not exist yet.
|
||||
- This document does not add or implement this configuration gate.
|
||||
- The future gate must be default-false and fail-closed: if the config field is missing or malformed, it must evaluate to `False`.
|
||||
- The gate must only control the served/user-visible `[verified]` surface.
|
||||
- The gate must never affect off-serving evaluation proof generation or off-serving validation runs.
|
||||
- The gate must strictly prohibit eval-gold-backed verification pipelines from touching serving code.
|
||||
- Enabling the gate is necessary but not sufficient: even if `verified_serving_enabled` is `True`, a served VERIFIED verdict is only allowed if all contract proof obligations successfully pass.
|
||||
|
||||
| Config State | verified_serving_enabled | Served VERIFIED Allowed? |
|
||||
| --- | --- | --- |
|
||||
| missing field | False | No |
|
||||
| default config | False | No |
|
||||
| explicit false | False | No |
|
||||
| explicit true | True | Only if gold-free proof obligations pass |
|
||||
|
||||
## 4. Gold-Free Independent Read Requirement
|
||||
|
||||
Before any served `[verified]` surface can exist, a serving proof must use a gold-free independent read source.
|
||||
|
||||
A served proof must adhere to the following:
|
||||
- Primary reader lineage is recorded.
|
||||
- Independent reader lineage is recorded.
|
||||
- The lineages must use distinct lineage identifiers.
|
||||
- The primary read digest and independent read digest must converge.
|
||||
- No gold setup is used.
|
||||
- No gold answer is used.
|
||||
- No benchmark fixture is used.
|
||||
- No eval-lane imports are performed.
|
||||
- Same reader twice (invoking the exact same reader lineage twice on the same text) is strictly rejected.
|
||||
- Second solver over one read (running two solvers over the same read signature) is strictly rejected.
|
||||
|
||||
### Comparison
|
||||
|
||||
* **Invalid (Pseudo-Independence):**
|
||||
```text
|
||||
primary read ──> solver A
|
||||
└───> solver B ──> same answer ──> VERIFIED (INVALID)
|
||||
```
|
||||
* **Valid (True Independence):**
|
||||
```text
|
||||
primary read lineage ───────────> canonical digest C ──┐
|
||||
├──> convergent digests ──> VERIFIED (VALID)
|
||||
gold-free independent lineage ──> canonical digest C ──┘
|
||||
```
|
||||
|
||||
The valid workflow requires:
|
||||
- Primary read lineage converges to canonical digest `C`.
|
||||
- Gold-free independent read lineage converges to canonical digest `C`.
|
||||
- Derivation is computed from stated quantities.
|
||||
- Bound slots are present.
|
||||
- Back-substitution succeeds.
|
||||
- No boundary has fired.
|
||||
- No contradiction is present.
|
||||
- No unresolved limitation exists.
|
||||
|
||||
The primary unsolved technical milestone is designing and implementing a gold-free independent reader/verifier source.
|
||||
|
||||
## 5. Serving Eligibility Criteria
|
||||
|
||||
For any response to be eligible for served `[verified]` status, all of the following conditions must be met:
|
||||
|
||||
- `source_problem_digest` is present.
|
||||
- `primary_reader_lineage` is present.
|
||||
- `independent_reader_lineage` is present.
|
||||
- `primary_reader_lineage` and `independent_reader_lineage` are distinct.
|
||||
- `primary_read_digest` is present.
|
||||
- `independent_read_digest` is present.
|
||||
- `primary_read_digest` and `independent_read_digest` are identical (converge).
|
||||
- `derivation_digest` is present.
|
||||
- `bound_slots_digest` is present.
|
||||
- `back_substitution_digest` is present.
|
||||
- `boundary_clear` is `True`.
|
||||
- `contradiction_clear` is `True`.
|
||||
- `limitation` is `None`.
|
||||
- `evaluate_verification(...)` evaluates to `VerificationVerdict.VERIFIED`.
|
||||
- `disclosure_for_verification(...)` is the only functional pathway used to transition the result to `(EpistemicState.VERIFIED, DisclosureClaim.VERIFIED)`.
|
||||
- `verified_serving_enabled` is explicitly configured to `True`.
|
||||
- The proof producer is gold-free and serving-eligible.
|
||||
|
||||
## 6. Holdout and Kill-Switch Gates
|
||||
|
||||
Before any served `[verified]` behavior can be wired to production, the serving-time verifier must clear a set of holdout and kill-switch gates:
|
||||
- **Sealed Holdout Lane**: A dedicated evaluation run on a sealed, off-distribution holdout dataset.
|
||||
- **Wrong=0 Requirement**: Zero incorrect answers are permitted for any result designated as VERIFIED.
|
||||
- **Poison Fixtures**: Comprehensive verification tests using poison test cases to prove that:
|
||||
- Wrong-read structures diverge and fail.
|
||||
- Same-reader-twice reads fail verification.
|
||||
- Answer matches gold, but missing proof fails verification.
|
||||
- Absence of refusal alone without proof fails verification.
|
||||
- Missing `bound_slots_digest` fails verification.
|
||||
- Triggered boundaries fail verification.
|
||||
- Contradictions present fail verification.
|
||||
- Unresolved limitations fail verification.
|
||||
- **Deterministic Replay Digest**: The verification trace must be replayable, yielding identical digests.
|
||||
- **Kill-Switch**: Default off / fail-closed behavior must be validated.
|
||||
- **Gate Disabled Test**: A test must verify that disabling `verified_serving_enabled` suppresses the served `[verified]` label even when a fully valid proof is present.
|
||||
- **Gate Enabled Test**: A test must verify that enabling `verified_serving_enabled` still requires all proof obligations to pass before serving.
|
||||
|
||||
## 7. verify.py Boundary
|
||||
|
||||
The transition of the verification verdict to serving must enforce strict boundaries at the `verify.py` layer:
|
||||
- `verify.py` may eventually consume a verified verdict from the contract.
|
||||
- `verify.py` must not define or redefine the semantic meaning of VERIFIED.
|
||||
- The formal meaning of VERIFIED remains exclusively defined in `core/epistemic_disclosure/verified_contract.py`.
|
||||
- `verify.py` must never attempt to repair failed proofs.
|
||||
- `verify.py` must not treat raw answer correctness (matching gold) as verification.
|
||||
- `verify.py` must not import or use eval-lane gold datasets.
|
||||
- No hot-path repair or silent corrections may occur in `verify.py`.
|
||||
|
||||
## 8. Served Behavior Matrix
|
||||
|
||||
| Proof / Gate State | Served Behavior |
|
||||
| --- | --- |
|
||||
| Valid off-serving eval-gold proof | Never served |
|
||||
| Valid gold-free proof + gate disabled | No served `[verified]` |
|
||||
| Valid gold-free proof + gate enabled | Served `[verified]` may be allowed |
|
||||
| Wrong read divergence | No served `[verified]` |
|
||||
| Same reader twice | No served `[verified]` |
|
||||
| Answer matches gold but proof missing | No served `[verified]` |
|
||||
| No refusal but no proof | No served `[verified]` |
|
||||
| Boundary fired | No served `[verified]` |
|
||||
| Contradiction present | No served `[verified]` |
|
||||
| Unresolved limitation | No served `[verified]` |
|
||||
|
||||
## 9. Required Future Tests Before Wiring
|
||||
|
||||
The following test suite must be implemented and pass before any serving-time code can land:
|
||||
- `verified_serving_enabled` defaults to `False`.
|
||||
- A missing config field for `verified_serving_enabled` resolves to `False`.
|
||||
- Explicit `True` configuration is required to serve `[verified]`.
|
||||
- A disabled gate suppresses served `[verified]` even when a valid proof exists.
|
||||
- An enabled gate still enforces and requires all proof obligations to pass.
|
||||
- Eval-gold verification producers cannot be imported or accessed by serving modules (AST / dependency check).
|
||||
- Eval-gold-backed proofs are rejected from being served.
|
||||
- Same reader twice fails verification.
|
||||
- Running a second solver over one reading fails verification.
|
||||
- Answer-gold match without verification proof does not serve `[verified]`.
|
||||
- Absence of refusal alone does not serve `[verified]`.
|
||||
- Direct construction of `EpistemicState.VERIFIED` is blocked outside of `disclosure_for_verification`.
|
||||
- Poison wrong-read inputs do not serve `[verified]`.
|
||||
- Missing `bound_slots_digest` does not serve `[verified]`.
|
||||
- Triggered boundary does not serve `[verified]`.
|
||||
- Contradiction present does not serve `[verified]`.
|
||||
- Unresolved limitation does not serve `[verified]`.
|
||||
- `verify.py` consumes the contract outcome but does not define its rules.
|
||||
|
||||
## 10. Non-Claims
|
||||
|
||||
This scoping document establishes architectural boundaries only.
|
||||
|
||||
- This document does not implement served VERIFIED.
|
||||
- This document does not add the `verified_serving_enabled` config gate.
|
||||
- This document does not wire `verify.py` to serving.
|
||||
- This document does not move the evaluation producer (`verified_producer.py`) into the core serving layer.
|
||||
- This document does not make gold-backed verification serving-safe.
|
||||
- This document does not change benchmark metrics.
|
||||
- This document does not update `CLAIMS.md`.
|
||||
- This document does not solve the design of the gold-free independent reader.
|
||||
- This document does not claim AGI progress.
|
||||
|
||||
## 11. Recommended Next Slices
|
||||
|
||||
The implementation of serving-time verification should proceed in isolated, review-gated slices:
|
||||
|
||||
### Slice 1: Configuration Gate
|
||||
- Add default-dark `verified_serving_enabled` helper only.
|
||||
- No runtime wiring, no `verify.py` wiring.
|
||||
- Strict import checks preventing eval producer imports.
|
||||
- Tests demonstrating default-dark behavior and config defaults.
|
||||
|
||||
### Slice 2: Gold-Free Independent Reader
|
||||
- Design and prototype a gold-free independent reader/verifier source.
|
||||
- Maintain off-serving isolation.
|
||||
- Generate independent reader lineage and canonical digests.
|
||||
- No integration with the served path.
|
||||
|
||||
### Slice 3: Verification Harness & Poison Fixtures
|
||||
- Implement a holdout-gated verification harness.
|
||||
- Integrate poison fixtures and test cases.
|
||||
- Validate deterministic replay digests.
|
||||
- Do not expose any served surface.
|
||||
|
||||
### Slice 4: verify.py Consumption
|
||||
- Only after all prior slices are approved and pass tests, scope the consumption of the verified verdict within `verify.py`.
|
||||
- No implementation without separate architectural review.
|
||||
|
|
@ -28,7 +28,7 @@ from __future__ import annotations
|
|||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, TYPE_CHECKING
|
||||
from typing import Any
|
||||
|
||||
from core.comprehension_attempt import (
|
||||
ComprehensionAttempt,
|
||||
|
|
@ -49,10 +49,6 @@ from generate.rate_comprehension.solver import solve_rate
|
|||
from generate.contemplation.findings import Finding, Terminal
|
||||
from generate.meaning_graph.reader import Refusal
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from core.epistemic_disclosure.limitation import LimitationAssessment
|
||||
from core.epistemic_questions.delivery import DeliveryOutcome
|
||||
|
||||
#: Substantive boundaries that are *recognized-but-unsupported* capabilities (not hard errors).
|
||||
_UNSUPPORTED_FAMILIES = frozenset(
|
||||
{
|
||||
|
|
@ -78,76 +74,16 @@ class ContemplationResult:
|
|||
answer: int | None = None
|
||||
family: str | None = None
|
||||
proposal_path: str | None = None
|
||||
question_path: str | None = None
|
||||
message: str | None = None
|
||||
|
||||
|
||||
def _delivery_outcome_for_limitation(assessment: LimitationAssessment) -> DeliveryOutcome:
|
||||
"""Helper to delegate to deliver_ask, pure and testable."""
|
||||
from core.epistemic_questions.delivery import deliver_ask
|
||||
return deliver_ask(assessment)
|
||||
|
||||
|
||||
def _handle_ask_delivery(
|
||||
assessment: LimitationAssessment,
|
||||
family_name: str,
|
||||
findings: list[Finding],
|
||||
attempts: tuple[ComprehensionAttempt, ...],
|
||||
text: str,
|
||||
proposal_root: Path | None,
|
||||
question_root: Path | None,
|
||||
exercise_ask: bool,
|
||||
selected_organ: str | None = None,
|
||||
) -> ContemplationResult:
|
||||
outcome = _delivery_outcome_for_limitation(assessment)
|
||||
if outcome.terminal == Terminal.QUESTION_NEEDED:
|
||||
assert outcome.question is not None
|
||||
import json
|
||||
from core.epistemic_questions.delivery import question_path
|
||||
|
||||
path = question_path(outcome.question, question_root)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(
|
||||
json.dumps(outcome.question.to_json_dict(), indent=2, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
findings.append(Finding("ask", f"emitted question-only {assessment.blocking_reason}"))
|
||||
findings.append(Finding("terminal", Terminal.QUESTION_NEEDED.value))
|
||||
return ContemplationResult(
|
||||
Terminal.QUESTION_NEEDED, tuple(findings), attempts,
|
||||
selected_organ=selected_organ, family=family_name,
|
||||
proposal_path=None,
|
||||
question_path=str(path),
|
||||
)
|
||||
else:
|
||||
findings.append(Finding("ask", f"unrenderable ask: {outcome.fallback_reason}"))
|
||||
findings.append(Finding("terminal", outcome.terminal.value))
|
||||
if outcome.terminal == Terminal.PROPOSAL_EMITTED:
|
||||
from core.comprehension_attempt.failure_family import family_by_name
|
||||
family_obj = family_by_name(family_name)
|
||||
if family_obj is not None:
|
||||
path = emit_proposal(text, family_obj, attempts, root=proposal_root)
|
||||
return ContemplationResult(
|
||||
Terminal.PROPOSAL_EMITTED, tuple(findings), attempts,
|
||||
selected_organ=selected_organ, family=family_name,
|
||||
proposal_path=str(path) if path else None,
|
||||
)
|
||||
return ContemplationResult(
|
||||
outcome.terminal, tuple(findings), attempts,
|
||||
selected_organ=selected_organ, family=family_name,
|
||||
)
|
||||
|
||||
|
||||
def contemplate(
|
||||
text: str,
|
||||
*,
|
||||
options: dict[str, Any] | None = None,
|
||||
answer_key: str | None = None,
|
||||
proposal_root: Path | None = None,
|
||||
question_root: Path | None = None,
|
||||
case_id: str | None = None,
|
||||
exercise_ask: bool = False,
|
||||
) -> ContemplationResult:
|
||||
"""Run one bounded contemplation pass over *text*."""
|
||||
findings: list[Finding] = []
|
||||
|
|
@ -175,11 +111,11 @@ def contemplate(
|
|||
if route.status == "routed":
|
||||
assert route.selected is not None
|
||||
if route.selected.organ == "r2_constraints":
|
||||
return _solve_and_verify_r2(text, options, answer_key, findings, attempts, proposal_root, question_root, exercise_ask)
|
||||
return _solve_and_verify_r2(text, options, answer_key, findings, attempts)
|
||||
if route.selected.organ == "r3_rate":
|
||||
return _solve_and_verify_r3(text, options, answer_key, findings, attempts, proposal_root, question_root, exercise_ask)
|
||||
return _solve_and_verify_r3(text, options, answer_key, findings, attempts)
|
||||
if route.selected.organ == "r4_combined_rate":
|
||||
return _solve_and_verify_cmb(text, options, answer_key, findings, attempts, proposal_root, question_root, exercise_ask)
|
||||
return _solve_and_verify_cmb(text, options, answer_key, findings, attempts)
|
||||
findings.append(Finding("solve", "r1 admissible setup (numeric answer is the eval lane in v0)"))
|
||||
findings.append(Finding("terminal", Terminal.SOLVED_VERIFIED.value))
|
||||
return ContemplationResult(
|
||||
|
|
@ -187,7 +123,7 @@ def contemplate(
|
|||
)
|
||||
|
||||
# route.status == "all_refused"
|
||||
return _classify_all_refused(text, attempts, findings, proposal_root, question_root, exercise_ask)
|
||||
return _classify_all_refused(text, attempts, findings, proposal_root)
|
||||
|
||||
|
||||
def _solve_and_verify_r2(
|
||||
|
|
@ -196,26 +132,12 @@ def _solve_and_verify_r2(
|
|||
answer_key: str | None,
|
||||
findings: list[Finding],
|
||||
attempts: tuple[ComprehensionAttempt, ...],
|
||||
proposal_root: Path | None,
|
||||
question_root: Path | None,
|
||||
exercise_ask: bool,
|
||||
) -> ContemplationResult:
|
||||
problem = read_constraint_problem(text)
|
||||
assert not isinstance(problem, Refusal) # routed => the reader admitted a setup
|
||||
value = answer_constraint_problem(problem)
|
||||
if isinstance(value, Refusal):
|
||||
findings.append(Finding("solve", f"solver refused: {value.reason}"))
|
||||
from core.comprehension_attempt.failure_family import family_by_name
|
||||
from core.epistemic_disclosure.limitation import assess_from_family
|
||||
family_obj = family_by_name(value.reason)
|
||||
if family_obj is not None:
|
||||
assessment = assess_from_family(family_obj)
|
||||
if assessment.resolution_action == "ask_question":
|
||||
if exercise_ask:
|
||||
return _handle_ask_delivery(
|
||||
assessment, family_obj.name, findings, attempts, text, proposal_root, question_root, exercise_ask,
|
||||
selected_organ="r2_constraints"
|
||||
)
|
||||
findings.append(Finding("terminal", Terminal.REFUSED_KNOWN_BOUNDARY.value))
|
||||
return ContemplationResult(
|
||||
Terminal.REFUSED_KNOWN_BOUNDARY, tuple(findings), attempts,
|
||||
|
|
@ -253,26 +175,12 @@ def _solve_and_verify_r3(
|
|||
answer_key: str | None,
|
||||
findings: list[Finding],
|
||||
attempts: tuple[ComprehensionAttempt, ...],
|
||||
proposal_root: Path | None,
|
||||
question_root: Path | None,
|
||||
exercise_ask: bool,
|
||||
) -> ContemplationResult:
|
||||
problem = read_rate_problem(text)
|
||||
assert not isinstance(problem, Refusal) # routed => the reader admitted a setup
|
||||
value = solve_rate(problem)
|
||||
if isinstance(value, Refusal):
|
||||
findings.append(Finding("solve", f"solver refused: {value.reason}"))
|
||||
from core.comprehension_attempt.failure_family import family_by_name
|
||||
from core.epistemic_disclosure.limitation import assess_from_family
|
||||
family_obj = family_by_name(value.reason)
|
||||
if family_obj is not None:
|
||||
assessment = assess_from_family(family_obj)
|
||||
if assessment.resolution_action == "ask_question":
|
||||
if exercise_ask:
|
||||
return _handle_ask_delivery(
|
||||
assessment, family_obj.name, findings, attempts, text, proposal_root, question_root, exercise_ask,
|
||||
selected_organ="r3_rate"
|
||||
)
|
||||
findings.append(Finding("terminal", Terminal.REFUSED_KNOWN_BOUNDARY.value))
|
||||
return ContemplationResult(
|
||||
Terminal.REFUSED_KNOWN_BOUNDARY, tuple(findings), attempts,
|
||||
|
|
@ -310,9 +218,6 @@ def _solve_and_verify_cmb(
|
|||
answer_key: str | None,
|
||||
findings: list[Finding],
|
||||
attempts: tuple[ComprehensionAttempt, ...],
|
||||
proposal_root: Path | None,
|
||||
question_root: Path | None,
|
||||
exercise_ask: bool,
|
||||
) -> ContemplationResult:
|
||||
problem = read_combined_rate_problem(text)
|
||||
assert not isinstance(problem, Refusal) # routed => the reader admitted a setup
|
||||
|
|
@ -322,18 +227,6 @@ def _solve_and_verify_cmb(
|
|||
# answerable boundary. A solver refusal is a terminal boundary, never a proposal — and the
|
||||
# reason is namespaced cmb_* so it resolves to the CMB solver family, not R2/R3's.
|
||||
findings.append(Finding("solve", f"solver refused: {value.reason}"))
|
||||
from core.comprehension_attempt.failure_family import family_by_name
|
||||
from core.epistemic_disclosure.limitation import assess_from_family
|
||||
reason = cmb_reason(value.reason)
|
||||
family_obj = family_by_name(reason)
|
||||
if family_obj is not None:
|
||||
assessment = assess_from_family(family_obj)
|
||||
if assessment.resolution_action == "ask_question":
|
||||
if exercise_ask:
|
||||
return _handle_ask_delivery(
|
||||
assessment, family_obj.name, findings, attempts, text, proposal_root, question_root, exercise_ask,
|
||||
selected_organ="r4_combined_rate"
|
||||
)
|
||||
findings.append(Finding("terminal", Terminal.REFUSED_KNOWN_BOUNDARY.value))
|
||||
return ContemplationResult(
|
||||
Terminal.REFUSED_KNOWN_BOUNDARY, tuple(findings), attempts,
|
||||
|
|
@ -370,8 +263,6 @@ def _classify_all_refused(
|
|||
attempts: tuple[ComprehensionAttempt, ...],
|
||||
findings: list[Finding],
|
||||
proposal_root: Path | None,
|
||||
question_root: Path | None,
|
||||
exercise_ask: bool,
|
||||
) -> ContemplationResult:
|
||||
# CMB-over-R3 precedence (family side): when CMB substantively recognized the text, R3's broader
|
||||
# partial classification is suppressed, so CMB's sharper diagnosis owns the terminal/proposal
|
||||
|
|
@ -379,10 +270,9 @@ def _classify_all_refused(
|
|||
considered = attempts
|
||||
if cmb_is_authoritative(attempts):
|
||||
considered = tuple(a for a in attempts if a.organ != "r3_rate")
|
||||
|
||||
families = [(a, family_for_reason(a.refusal_reason)) for a in considered]
|
||||
|
||||
# Boundary-first: a substantive recognized boundary blocks any proposal or ASK.
|
||||
# Boundary-first: a substantive recognized boundary blocks any proposal.
|
||||
for _attempt, family in families:
|
||||
if family is not None and family.must_remain_refused and family.name != _NOT_MY_DOMAIN:
|
||||
terminal = (
|
||||
|
|
@ -393,16 +283,6 @@ def _classify_all_refused(
|
|||
findings.append(Finding("terminal", f"{terminal.value} via {family.name}"))
|
||||
return ContemplationResult(terminal, tuple(findings), attempts, family=family.name)
|
||||
|
||||
# Check for ASK delivery only after substantive boundaries are ruled out.
|
||||
for attempt in considered:
|
||||
from core.epistemic_disclosure.limitation import assess_from_attempt
|
||||
assessment = assess_from_attempt(attempt)
|
||||
if assessment is not None and assessment.resolution_action == "ask_question":
|
||||
if exercise_ask:
|
||||
return _handle_ask_delivery(
|
||||
assessment, assessment.blocking_reason, findings, attempts, text, proposal_root, question_root, exercise_ask
|
||||
)
|
||||
|
||||
# No substantive boundary: a genuine growth surface may emit a proposal-only artifact.
|
||||
for _attempt, family in families:
|
||||
if family is not None and family.proposal_allowed:
|
||||
|
|
|
|||
|
|
@ -1,336 +0,0 @@
|
|||
"""Focused tests for off-serving ASK delivery integration in the contemplation pass manager (N6).
|
||||
|
||||
Ensures that the pass manager seam delegates to deliver_ask, correctly handles fallback dispositions,
|
||||
avoids direct rendering imports/text construction, and preserves the Q1-B carve-out.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from core.epistemic_disclosure.limitation import (
|
||||
Q1B_ASK_CARVE_OUT,
|
||||
LimitationAssessment,
|
||||
MissingSlot,
|
||||
)
|
||||
from core.epistemic_state import EpistemicState
|
||||
from generate.contemplation import Terminal, contemplate
|
||||
from generate.contemplation.pass_manager import (
|
||||
_delivery_outcome_for_limitation,
|
||||
)
|
||||
from core.comprehension_attempt.failure_family import family_by_name
|
||||
|
||||
|
||||
def _make_assessment(
|
||||
*,
|
||||
blocking_reason: str,
|
||||
slots: tuple[MissingSlot, ...],
|
||||
resolution_action: str = "ask_question",
|
||||
) -> LimitationAssessment:
|
||||
return LimitationAssessment(
|
||||
limitation_kind="missing_information",
|
||||
resolution_action=resolution_action, # type: ignore[arg-type]
|
||||
epistemic_state=EpistemicState.UNDETERMINED,
|
||||
owner_organ="r2_constraint",
|
||||
blocking_reason=blocking_reason,
|
||||
missing_slots=slots,
|
||||
)
|
||||
|
||||
|
||||
_TOTAL_COUNT_SLOT = MissingSlot(
|
||||
slot_name="total_count",
|
||||
expected_unit_or_type="count_int",
|
||||
binding_target="collective_unit_total",
|
||||
)
|
||||
_WEIGHTED_SLOT = MissingSlot(
|
||||
slot_name="weighted_total",
|
||||
expected_unit_or_type="measured_unit_int",
|
||||
binding_target="weighted_total_value",
|
||||
)
|
||||
|
||||
|
||||
def test_pass_manager_uses_deliver_ask_for_renderable_ask() -> None:
|
||||
assessment = _make_assessment(
|
||||
blocking_reason="missing_total_count", slots=(_TOTAL_COUNT_SLOT,)
|
||||
)
|
||||
outcome = _delivery_outcome_for_limitation(assessment)
|
||||
assert outcome.terminal == Terminal.QUESTION_NEEDED
|
||||
assert outcome.question is not None
|
||||
assert outcome.fallback_reason is None
|
||||
|
||||
|
||||
def test_pass_manager_unrenderable_ask_falls_back_without_question_needed() -> None:
|
||||
# Multi-slot => unrenderable, falls back to standing disposition (PROPOSAL_EMITTED since missing_total_count is carve-out)
|
||||
assessment = _make_assessment(
|
||||
blocking_reason="missing_total_count",
|
||||
slots=(_TOTAL_COUNT_SLOT, _WEIGHTED_SLOT),
|
||||
)
|
||||
outcome = _delivery_outcome_for_limitation(assessment)
|
||||
assert outcome.terminal == Terminal.PROPOSAL_EMITTED
|
||||
assert outcome.terminal is not Terminal.QUESTION_NEEDED
|
||||
assert outcome.question is None
|
||||
assert outcome.fallback_reason == "multi_slot_not_supported"
|
||||
|
||||
|
||||
def test_pass_manager_does_not_import_or_call_render_question_directly() -> None:
|
||||
path = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "generate"
|
||||
/ "contemplation"
|
||||
/ "pass_manager.py"
|
||||
)
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
for node in ast.walk(tree):
|
||||
# Ensure render_question is not imported, and chat/chat.runtime is not imported
|
||||
if isinstance(node, ast.ImportFrom):
|
||||
assert node.module != "core.epistemic_questions.render"
|
||||
assert node.module != "chat.runtime"
|
||||
assert node.module != "chat"
|
||||
if node.names:
|
||||
for alias in node.names:
|
||||
assert alias.name != "render_question"
|
||||
elif isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
assert alias.name != "core.epistemic_questions.render"
|
||||
assert alias.name != "chat.runtime"
|
||||
assert alias.name != "chat"
|
||||
|
||||
# Ensure render_question is not called directly
|
||||
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
|
||||
assert node.func.id != "render_question"
|
||||
|
||||
|
||||
def test_pass_manager_does_not_construct_question_text() -> None:
|
||||
path = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "generate"
|
||||
/ "contemplation"
|
||||
/ "pass_manager.py"
|
||||
)
|
||||
content = path.read_text(encoding="utf-8")
|
||||
forbidden_templates = ["What ", "Which ", "How many", "Please provide"]
|
||||
for template in forbidden_templates:
|
||||
assert template not in content, (
|
||||
f"pass_manager.py must not construct prose templates like {template!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_q1b_carveout_preserved_during_pass_manager_ask_integration() -> None:
|
||||
assert "missing_total_count" in Q1B_ASK_CARVE_OUT
|
||||
assert "missing_weighted_total" in Q1B_ASK_CARVE_OUT
|
||||
|
||||
for name in Q1B_ASK_CARVE_OUT:
|
||||
family = family_by_name(name)
|
||||
assert family is not None
|
||||
assert family.proposal_allowed is True
|
||||
|
||||
|
||||
def test_renderable_ask_path_returns_question_needed_under_exercise_ask(monkeypatch, tmp_path) -> None:
|
||||
from core.comprehension_attempt import ComprehensionAttempt, RouteResult
|
||||
import generate.contemplation.pass_manager as pm
|
||||
from generate.binding_graph.model import SourceSpanLink
|
||||
|
||||
span = SourceSpanLink(source_id="src", start=0, end=8, text="chickens")
|
||||
attempt = ComprehensionAttempt(
|
||||
organ="r2_constraints",
|
||||
outcome="setup_refused",
|
||||
refusal_reason="missing_total_count",
|
||||
evidence=(span,),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(pm, "route_setup", lambda text, case_id=None: RouteResult((attempt,), None, "all_refused"))
|
||||
|
||||
question_root = tmp_path / "teaching" / "questions"
|
||||
proposal_root = tmp_path / "teaching" / "proposals"
|
||||
|
||||
repo_questions_dir = Path(__file__).resolve().parents[1] / "teaching" / "questions"
|
||||
before_files = set(repo_questions_dir.glob("**/*")) if repo_questions_dir.exists() else set()
|
||||
|
||||
# 1. Assert that without exercise_ask, it falls back to PROPOSAL_EMITTED (due to carve-out)
|
||||
res_normal = contemplate(
|
||||
"chickens",
|
||||
proposal_root=proposal_root,
|
||||
question_root=question_root,
|
||||
)
|
||||
assert res_normal.terminal == Terminal.PROPOSAL_EMITTED
|
||||
assert res_normal.proposal_path is not None
|
||||
assert res_normal.question_path is None
|
||||
assert proposal_root in Path(res_normal.proposal_path).parents
|
||||
|
||||
# 2. Assert that with exercise_ask=True, it returns QUESTION_NEEDED
|
||||
calls = []
|
||||
orig = pm._delivery_outcome_for_limitation
|
||||
def wrapped(assessment):
|
||||
calls.append(assessment)
|
||||
return orig(assessment)
|
||||
monkeypatch.setattr(pm, "_delivery_outcome_for_limitation", wrapped)
|
||||
|
||||
res_ask = contemplate(
|
||||
"chickens",
|
||||
proposal_root=proposal_root,
|
||||
question_root=question_root,
|
||||
exercise_ask=True,
|
||||
)
|
||||
assert res_ask.terminal == Terminal.QUESTION_NEEDED
|
||||
assert len(calls) == 1 # Verify it is called exactly once! No double delivery / double render!
|
||||
|
||||
# Verify the question artifact path
|
||||
assert res_ask.proposal_path is None
|
||||
assert res_ask.question_path is not None
|
||||
artifact_path = Path(res_ask.question_path)
|
||||
assert artifact_path.exists()
|
||||
|
||||
# Assert question artifact is under question_root
|
||||
assert question_root in artifact_path.parents
|
||||
# Assert question artifact is not under proposal_root
|
||||
assert proposal_root not in artifact_path.parents
|
||||
|
||||
# Assert no repo-local teaching/questions artifact is created during tests
|
||||
after_files = set(repo_questions_dir.glob("**/*")) if repo_questions_dir.exists() else set()
|
||||
assert before_files == after_files
|
||||
|
||||
|
||||
def test_unrenderable_ask_falls_back_in_pass_manager(monkeypatch, tmp_path) -> None:
|
||||
from core.comprehension_attempt import ComprehensionAttempt, RouteResult
|
||||
import generate.contemplation.pass_manager as pm
|
||||
from core.epistemic_questions.delivery import DeliveryOutcome
|
||||
|
||||
attempt = ComprehensionAttempt(
|
||||
organ="r2_constraints",
|
||||
outcome="setup_refused",
|
||||
refusal_reason="missing_total_count",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(pm, "route_setup", lambda text, case_id=None: RouteResult((attempt,), None, "all_refused"))
|
||||
monkeypatch.setattr(
|
||||
pm,
|
||||
"_delivery_outcome_for_limitation",
|
||||
lambda assessment: DeliveryOutcome(Terminal.PROPOSAL_EMITTED, None, "multi_slot_not_supported")
|
||||
)
|
||||
|
||||
question_root = tmp_path / "teaching" / "questions"
|
||||
proposal_root = tmp_path / "teaching" / "proposals"
|
||||
|
||||
res = contemplate(
|
||||
"chickens",
|
||||
proposal_root=proposal_root,
|
||||
question_root=question_root,
|
||||
exercise_ask=True,
|
||||
)
|
||||
assert res.terminal == Terminal.PROPOSAL_EMITTED
|
||||
assert res.terminal is not Terminal.QUESTION_NEEDED
|
||||
assert res.proposal_path is not None
|
||||
assert Path(res.proposal_path).exists()
|
||||
assert res.question_path is None
|
||||
|
||||
|
||||
def test_family_none_does_not_crash_ask_branch(monkeypatch, tmp_path) -> None:
|
||||
from core.comprehension_attempt import ComprehensionAttempt, RouteResult
|
||||
import generate.contemplation.pass_manager as pm
|
||||
from core.epistemic_disclosure.limitation import LimitationAssessment
|
||||
import core.epistemic_disclosure.limitation as lim_mod
|
||||
|
||||
fake_assessment = LimitationAssessment(
|
||||
limitation_kind="missing_information",
|
||||
resolution_action="ask_question",
|
||||
epistemic_state=EpistemicState.UNDETERMINED,
|
||||
owner_organ="r2_constraint",
|
||||
blocking_reason="nonexistent_family_name",
|
||||
)
|
||||
|
||||
attempt = ComprehensionAttempt(
|
||||
organ="r2_constraints",
|
||||
outcome="setup_refused",
|
||||
refusal_reason="nonexistent_family_name",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(pm, "route_setup", lambda text, case_id=None: RouteResult((attempt,), None, "all_refused"))
|
||||
monkeypatch.setattr(lim_mod, "assess_from_attempt", lambda att: fake_assessment)
|
||||
|
||||
question_root = tmp_path / "teaching" / "questions"
|
||||
proposal_root = tmp_path / "teaching" / "proposals"
|
||||
|
||||
res = contemplate(
|
||||
"chickens",
|
||||
proposal_root=proposal_root,
|
||||
question_root=question_root,
|
||||
exercise_ask=True,
|
||||
)
|
||||
assert res.terminal == Terminal.NO_PROGRESS
|
||||
|
||||
|
||||
def test_boundary_wins_over_ask_in_pass_manager(monkeypatch, tmp_path) -> None:
|
||||
from core.comprehension_attempt import ComprehensionAttempt, RouteResult
|
||||
import generate.contemplation.pass_manager as pm
|
||||
from generate.binding_graph.model import SourceSpanLink
|
||||
|
||||
span = SourceSpanLink(source_id="src", start=0, end=8, text="chickens")
|
||||
attempt_boundary = ComprehensionAttempt(
|
||||
organ="r2_constraints",
|
||||
outcome="setup_refused",
|
||||
refusal_reason="too_many_categories", # maps to unsupported_system_size (must_remain_refused = True)
|
||||
evidence=(span,),
|
||||
)
|
||||
attempt_ask = ComprehensionAttempt(
|
||||
organ="r2_constraints",
|
||||
outcome="setup_refused",
|
||||
refusal_reason="missing_total_count", # ask carve-out
|
||||
evidence=(span,),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
pm,
|
||||
"route_setup",
|
||||
lambda text, case_id=None: RouteResult((attempt_boundary, attempt_ask), None, "all_refused"),
|
||||
)
|
||||
|
||||
question_root = tmp_path / "teaching" / "questions"
|
||||
proposal_root = tmp_path / "teaching" / "proposals"
|
||||
|
||||
res = contemplate(
|
||||
"chickens",
|
||||
proposal_root=proposal_root,
|
||||
question_root=question_root,
|
||||
exercise_ask=True,
|
||||
)
|
||||
# Assert terminal remains REFUSED_UNSUPPORTED_FAMILY or REFUSED_KNOWN_BOUNDARY, not QUESTION_NEEDED
|
||||
assert res.terminal == Terminal.REFUSED_UNSUPPORTED_FAMILY
|
||||
assert res.question_path is None
|
||||
assert res.proposal_path is None
|
||||
# Ensure no question is written under question_root
|
||||
assert not question_root.exists() or len(list(question_root.glob("**/*"))) == 0
|
||||
|
||||
|
||||
def test_unrenderable_ask_falls_back_to_no_progress_in_pass_manager(monkeypatch, tmp_path) -> None:
|
||||
from core.comprehension_attempt import ComprehensionAttempt, RouteResult
|
||||
import generate.contemplation.pass_manager as pm
|
||||
from core.epistemic_questions.delivery import DeliveryOutcome
|
||||
|
||||
attempt = ComprehensionAttempt(
|
||||
organ="r2_constraints",
|
||||
outcome="setup_refused",
|
||||
refusal_reason="missing_total_count",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(pm, "route_setup", lambda text, case_id=None: RouteResult((attempt,), None, "all_refused"))
|
||||
monkeypatch.setattr(
|
||||
pm,
|
||||
"_delivery_outcome_for_limitation",
|
||||
lambda assessment: DeliveryOutcome(Terminal.NO_PROGRESS, None, "some_fallback_reason")
|
||||
)
|
||||
|
||||
question_root = tmp_path / "teaching" / "questions"
|
||||
proposal_root = tmp_path / "teaching" / "proposals"
|
||||
|
||||
res = contemplate(
|
||||
"chickens",
|
||||
proposal_root=proposal_root,
|
||||
question_root=question_root,
|
||||
exercise_ask=True,
|
||||
)
|
||||
assert res.terminal == Terminal.NO_PROGRESS
|
||||
assert res.proposal_path is None
|
||||
assert res.question_path is None
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
"""ASK serving gate — default-dark invariant.
|
||||
|
||||
This is deliberately narrower than serving integration. It proves the post-scoping
|
||||
kill-switch predicate is dark unless an operator/config object explicitly opts in.
|
||||
No chat/runtime wiring, no pass-manager emission, no carve-out retirement.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from core.config import DEFAULT_CONFIG, RuntimeConfig
|
||||
from core.epistemic_questions.serving_gate import ask_serving_enabled
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _LegacyConfig:
|
||||
"""A pre-field config shape: absence of the flag must mean dark."""
|
||||
|
||||
unrelated: bool = True
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _OptInConfig:
|
||||
ask_serving_enabled: bool
|
||||
|
||||
|
||||
def test_default_runtime_config_keeps_ask_serving_dark() -> None:
|
||||
assert hasattr(RuntimeConfig(), "ask_serving_enabled")
|
||||
assert RuntimeConfig().ask_serving_enabled is False
|
||||
assert DEFAULT_CONFIG.ask_serving_enabled is False
|
||||
assert ask_serving_enabled(DEFAULT_CONFIG) is False
|
||||
assert ask_serving_enabled(RuntimeConfig()) is False
|
||||
|
||||
|
||||
|
||||
def test_missing_flag_is_dark_for_legacy_config_shape() -> None:
|
||||
assert ask_serving_enabled(_LegacyConfig()) is False
|
||||
|
||||
|
||||
def test_gate_only_lights_on_explicit_truthy_opt_in() -> None:
|
||||
assert ask_serving_enabled(_OptInConfig(False)) is False
|
||||
assert ask_serving_enabled(_OptInConfig(True)) is True
|
||||
|
||||
|
||||
def test_none_uses_default_config_and_stays_dark() -> None:
|
||||
assert ask_serving_enabled() is False
|
||||
|
|
@ -1,287 +0,0 @@
|
|||
"""Focused tests for the Stage 2 served ASK artifact adapter.
|
||||
|
||||
These tests intentionally avoid ``chat.runtime``. This slice is adapter-only:
|
||||
it validates Q1-D question artifacts and returns a typed decision, but does not
|
||||
wire runtime acquisition of ``ContemplationResult``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from core.config import RuntimeConfig
|
||||
from core.epistemic_disclosure import ServedAskDecision, evaluate_served_ask
|
||||
from core.epistemic_disclosure.disposition import ServedDisposition, choose_served_disposition
|
||||
|
||||
|
||||
class DummyTerminal:
|
||||
def __init__(self, value: str):
|
||||
self.value = value
|
||||
|
||||
|
||||
class DummyContemplationResult:
|
||||
def __init__(
|
||||
self,
|
||||
terminal: str,
|
||||
*,
|
||||
question_path: str | None = None,
|
||||
proposal_path: str | None = None,
|
||||
family: str | None = None,
|
||||
) -> None:
|
||||
self.terminal = DummyTerminal(terminal)
|
||||
self.question_path = question_path
|
||||
self.proposal_path = proposal_path
|
||||
self.family = family
|
||||
|
||||
|
||||
def _write_artifact(path: Path, data: dict) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data), encoding="utf-8")
|
||||
|
||||
|
||||
def _valid_payload(text: str = "How many crates are left?") -> dict:
|
||||
return {
|
||||
"status": "question_only",
|
||||
"blocking_reason": "missing_total_count",
|
||||
"owner_organ": "r2_constraint",
|
||||
"question": {
|
||||
"text": text,
|
||||
"reason": "missing_total_count",
|
||||
"slot_name": "total_count",
|
||||
"expected_unit_or_type": "count_int",
|
||||
"binding_target": "collective_unit_total",
|
||||
},
|
||||
"answer_binding": None,
|
||||
"requires_review": True,
|
||||
"served": False,
|
||||
}
|
||||
|
||||
|
||||
def _question_result(q_path: Path, p_path: Path | None = None) -> DummyContemplationResult:
|
||||
return DummyContemplationResult(
|
||||
"QUESTION_NEEDED",
|
||||
question_path=str(q_path),
|
||||
proposal_path=str(p_path) if p_path is not None else None,
|
||||
)
|
||||
|
||||
|
||||
def test_ask_serving_disabled_preserves_existing_fallback_surface(tmp_path: Path) -> None:
|
||||
q_path = tmp_path / "questions" / "q.json"
|
||||
_write_artifact(q_path, _valid_payload())
|
||||
|
||||
decision = evaluate_served_ask(
|
||||
RuntimeConfig(ask_serving_enabled=False),
|
||||
_question_result(q_path),
|
||||
"fallback",
|
||||
)
|
||||
|
||||
assert isinstance(decision, ServedAskDecision)
|
||||
assert decision.served is False
|
||||
assert decision.terminal == "QUESTION_NEEDED"
|
||||
assert decision.surface == "fallback"
|
||||
assert decision.disposition is ServedDisposition.REFUSE
|
||||
|
||||
|
||||
def test_ask_serving_enabled_surfaces_question_needed_from_artifact(tmp_path: Path) -> None:
|
||||
q_path = tmp_path / "questions" / "q.json"
|
||||
p_path = tmp_path / "proposals" / "p.json"
|
||||
_write_artifact(q_path, _valid_payload("How many crates are left?"))
|
||||
|
||||
decision = evaluate_served_ask(
|
||||
RuntimeConfig(ask_serving_enabled=True),
|
||||
_question_result(q_path, p_path),
|
||||
"fallback",
|
||||
)
|
||||
|
||||
assert decision.served is True
|
||||
assert decision.terminal == "QUESTION_NEEDED"
|
||||
assert decision.surface == "How many crates are left?"
|
||||
assert decision.disposition is ServedDisposition.ASK
|
||||
|
||||
|
||||
def test_served_ask_uses_governance_disposition_bus(monkeypatch, tmp_path: Path) -> None:
|
||||
q_path = tmp_path / "questions" / "q.json"
|
||||
_write_artifact(q_path, _valid_payload())
|
||||
calls = []
|
||||
|
||||
def traced_choose_served_disposition(*args, **kwargs):
|
||||
calls.append((args, kwargs))
|
||||
return choose_served_disposition(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"core.epistemic_disclosure.ask_serving.choose_served_disposition",
|
||||
traced_choose_served_disposition,
|
||||
)
|
||||
|
||||
decision = evaluate_served_ask(
|
||||
RuntimeConfig(ask_serving_enabled=True),
|
||||
_question_result(q_path),
|
||||
"fallback",
|
||||
)
|
||||
|
||||
assert decision.served is True
|
||||
assert decision.disposition is ServedDisposition.ASK
|
||||
assert len(calls) == 1
|
||||
assert calls[0][1]["limitation"].resolution_action == "ask_question"
|
||||
|
||||
|
||||
def test_question_path_must_not_equal_proposal_path(tmp_path: Path) -> None:
|
||||
q_path = tmp_path / "questions" / "same.json"
|
||||
_write_artifact(q_path, _valid_payload())
|
||||
|
||||
decision = evaluate_served_ask(
|
||||
RuntimeConfig(ask_serving_enabled=True),
|
||||
_question_result(q_path, q_path),
|
||||
"fallback",
|
||||
)
|
||||
|
||||
assert decision.served is False
|
||||
assert decision.surface == "fallback"
|
||||
|
||||
|
||||
def test_non_question_needed_terminal_preserves_proposal_signal(tmp_path: Path) -> None:
|
||||
q_path = tmp_path / "questions" / "q.json"
|
||||
_write_artifact(q_path, _valid_payload())
|
||||
result = DummyContemplationResult(
|
||||
"PROPOSAL_EMITTED",
|
||||
question_path=str(q_path),
|
||||
proposal_path=str(tmp_path / "proposals" / "p.json"),
|
||||
)
|
||||
|
||||
decision = evaluate_served_ask(
|
||||
RuntimeConfig(ask_serving_enabled=True),
|
||||
result,
|
||||
"fallback",
|
||||
)
|
||||
|
||||
assert decision.served is False
|
||||
assert decision.terminal == "PROPOSAL_EMITTED"
|
||||
assert decision.surface == "fallback"
|
||||
assert decision.disposition is ServedDisposition.PROPOSE
|
||||
|
||||
|
||||
def test_missing_or_unreadable_question_artifact_fails_closed(tmp_path: Path) -> None:
|
||||
missing = tmp_path / "questions" / "missing.json"
|
||||
|
||||
decision = evaluate_served_ask(
|
||||
RuntimeConfig(ask_serving_enabled=True),
|
||||
_question_result(missing),
|
||||
"fallback",
|
||||
)
|
||||
|
||||
assert decision.served is False
|
||||
assert decision.surface == "fallback"
|
||||
|
||||
|
||||
def test_malformed_question_artifact_fails_closed(tmp_path: Path) -> None:
|
||||
q_path = tmp_path / "questions" / "q.json"
|
||||
q_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
q_path.write_text("{bad json", encoding="utf-8")
|
||||
|
||||
decision = evaluate_served_ask(
|
||||
RuntimeConfig(ask_serving_enabled=True),
|
||||
_question_result(q_path),
|
||||
"fallback",
|
||||
)
|
||||
|
||||
assert decision.served is False
|
||||
assert decision.surface == "fallback"
|
||||
|
||||
|
||||
def test_rejects_artifact_status_proposal_only(tmp_path: Path) -> None:
|
||||
q_path = tmp_path / "questions" / "q.json"
|
||||
data = _valid_payload()
|
||||
data["status"] = "proposal_only"
|
||||
_write_artifact(q_path, data)
|
||||
|
||||
decision = evaluate_served_ask(RuntimeConfig(ask_serving_enabled=True), _question_result(q_path), "fallback")
|
||||
|
||||
assert decision.served is False
|
||||
assert decision.surface == "fallback"
|
||||
|
||||
|
||||
def test_rejects_artifact_served_true(tmp_path: Path) -> None:
|
||||
q_path = tmp_path / "questions" / "q.json"
|
||||
data = _valid_payload()
|
||||
data["served"] = True
|
||||
_write_artifact(q_path, data)
|
||||
|
||||
decision = evaluate_served_ask(RuntimeConfig(ask_serving_enabled=True), _question_result(q_path), "fallback")
|
||||
|
||||
assert decision.served is False
|
||||
|
||||
|
||||
def test_rejects_artifact_requires_review_false(tmp_path: Path) -> None:
|
||||
q_path = tmp_path / "questions" / "q.json"
|
||||
data = _valid_payload()
|
||||
data["requires_review"] = False
|
||||
_write_artifact(q_path, data)
|
||||
|
||||
decision = evaluate_served_ask(RuntimeConfig(ask_serving_enabled=True), _question_result(q_path), "fallback")
|
||||
|
||||
assert decision.served is False
|
||||
|
||||
|
||||
def test_rejects_artifact_non_null_answer_binding(tmp_path: Path) -> None:
|
||||
q_path = tmp_path / "questions" / "q.json"
|
||||
data = _valid_payload()
|
||||
data["answer_binding"] = {"slot": "total_count", "value": 12}
|
||||
_write_artifact(q_path, data)
|
||||
|
||||
decision = evaluate_served_ask(RuntimeConfig(ask_serving_enabled=True), _question_result(q_path), "fallback")
|
||||
|
||||
assert decision.served is False
|
||||
|
||||
|
||||
def test_rejects_missing_or_blank_question_text(tmp_path: Path) -> None:
|
||||
for value in (None, ""):
|
||||
q_path = tmp_path / f"questions_{value!r}" / "q.json"
|
||||
data = _valid_payload()
|
||||
if value is None:
|
||||
del data["question"]["text"]
|
||||
else:
|
||||
data["question"]["text"] = " "
|
||||
_write_artifact(q_path, data)
|
||||
|
||||
decision = evaluate_served_ask(RuntimeConfig(ask_serving_enabled=True), _question_result(q_path), "fallback")
|
||||
|
||||
assert decision.served is False
|
||||
assert decision.surface == "fallback"
|
||||
|
||||
|
||||
def test_rejects_missing_or_blank_slot_name(tmp_path: Path) -> None:
|
||||
for value in (None, ""):
|
||||
q_path = tmp_path / f"questions_slot_{value!r}" / "q.json"
|
||||
data = _valid_payload()
|
||||
if value is None:
|
||||
del data["question"]["slot_name"]
|
||||
else:
|
||||
data["question"]["slot_name"] = " "
|
||||
_write_artifact(q_path, data)
|
||||
|
||||
decision = evaluate_served_ask(RuntimeConfig(ask_serving_enabled=True), _question_result(q_path), "fallback")
|
||||
|
||||
assert decision.served is False
|
||||
assert decision.surface == "fallback"
|
||||
|
||||
|
||||
def test_adapter_does_not_import_question_renderer_or_construct_prose() -> None:
|
||||
path = Path(__file__).resolve().parents[1] / "core" / "epistemic_disclosure" / "ask_serving.py"
|
||||
source = path.read_text(encoding="utf-8")
|
||||
tree = ast.parse(source, filename=str(path))
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom) and node.module:
|
||||
assert node.module != "core.epistemic_questions.render"
|
||||
assert not node.module.startswith("core.epistemic_questions.render.")
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
assert alias.name != "core.epistemic_questions.render"
|
||||
assert not alias.name.startswith("core.epistemic_questions.render.")
|
||||
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
|
||||
assert node.func.id != "render_question"
|
||||
|
||||
for forbidden_template in ("How many", "What ", "Which ", "Please provide"):
|
||||
assert forbidden_template not in source
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
"""Cross-gate configuration field tests.
|
||||
|
||||
Verifies that RuntimeConfig has both ask_serving_enabled and
|
||||
verified_serving_enabled, that both are default false, and that setting
|
||||
one does not implicitly enable the other.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
|
||||
from core.config import RuntimeConfig
|
||||
|
||||
|
||||
def test_cross_serving_gates_independence() -> None:
|
||||
# 1. Verify both are present on a fresh config and default to False
|
||||
config = RuntimeConfig()
|
||||
assert hasattr(config, "ask_serving_enabled")
|
||||
assert hasattr(config, "verified_serving_enabled")
|
||||
assert config.ask_serving_enabled is False
|
||||
assert config.verified_serving_enabled is False
|
||||
|
||||
# 2. Verify setting ask_serving_enabled to True does not enable verified_serving_enabled
|
||||
config_ask = dataclasses.replace(config, ask_serving_enabled=True)
|
||||
assert config_ask.ask_serving_enabled is True
|
||||
assert config_ask.verified_serving_enabled is False
|
||||
|
||||
# 3. Verify setting verified_serving_enabled to True does not enable ask_serving_enabled
|
||||
config_verified = dataclasses.replace(config, verified_serving_enabled=True)
|
||||
assert config_verified.ask_serving_enabled is False
|
||||
assert config_verified.verified_serving_enabled is True
|
||||
|
|
@ -1,78 +0,0 @@
|
|||
"""VERIFIED serving gate — default-dark invariant.
|
||||
|
||||
This proves the post-scoping kill-switch predicate is dark unless an
|
||||
operator/config object explicitly opts in.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from core.config import DEFAULT_CONFIG, RuntimeConfig
|
||||
from core.epistemic_disclosure.serving_gate import verified_serving_enabled
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _LegacyConfig:
|
||||
"""A pre-field config shape: absence of the flag must mean dark."""
|
||||
|
||||
unrelated: bool = True
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _OptInConfig:
|
||||
"""Config with the verified_serving_enabled field."""
|
||||
|
||||
verified_serving_enabled: bool | None
|
||||
|
||||
|
||||
def test_default_runtime_config_keeps_verified_serving_dark() -> None:
|
||||
assert hasattr(RuntimeConfig(), "verified_serving_enabled")
|
||||
assert RuntimeConfig().verified_serving_enabled is False
|
||||
assert DEFAULT_CONFIG.verified_serving_enabled is False
|
||||
assert verified_serving_enabled(DEFAULT_CONFIG) is False
|
||||
assert verified_serving_enabled(RuntimeConfig()) is False
|
||||
|
||||
|
||||
|
||||
def test_missing_flag_is_dark_for_legacy_config_shape() -> None:
|
||||
assert verified_serving_enabled(_LegacyConfig()) is False
|
||||
|
||||
|
||||
def test_gate_only_lights_on_explicit_truthy_opt_in() -> None:
|
||||
assert verified_serving_enabled(_OptInConfig(False)) is False
|
||||
assert verified_serving_enabled(_OptInConfig(True)) is True
|
||||
assert verified_serving_enabled(_OptInConfig(None)) is False
|
||||
|
||||
|
||||
def test_none_uses_default_config_and_stays_dark() -> None:
|
||||
assert verified_serving_enabled() is False
|
||||
|
||||
|
||||
def test_verified_serving_gate_has_no_eval_or_runtime_imports() -> None:
|
||||
path = Path(__file__).parent.parent / "core/epistemic_disclosure/serving_gate.py"
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"))
|
||||
|
||||
forbidden = {
|
||||
"evals",
|
||||
"evals.constraint_oracle",
|
||||
"evals.constraint_oracle.verified_producer",
|
||||
"verify",
|
||||
"chat.runtime",
|
||||
"generate.contemplation.pass_manager",
|
||||
}
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for name in node.names:
|
||||
for banned in forbidden:
|
||||
assert name.name != banned, f"Forbidden import: {name.name}"
|
||||
assert not name.name.startswith(banned + "."), f"Forbidden import: {name.name}"
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
if node.module:
|
||||
for banned in forbidden:
|
||||
assert node.module != banned, f"Forbidden import from: {node.module}"
|
||||
assert not node.module.startswith(banned + "."), f"Forbidden import from: {node.module}"
|
||||
Loading…
Reference in a new issue