Compare commits
21 commits
feat/p1-c-
...
add-ask-ac
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4308519ca9 | ||
|
|
426adbd197 | ||
|
|
9cff9755cd | ||
|
|
b687f62cf0 | ||
|
|
3cfaaf9153 | ||
|
|
45835987b9 | ||
|
|
1fff7dad59 | ||
|
|
ec9102a8d6 | ||
|
|
ffbaf0f27c | ||
|
|
4a6153d53e | ||
|
|
17e36acdc0 | ||
|
|
297349fd8d | ||
|
|
9a54374048 | ||
|
|
28cb42d0dd | ||
|
|
07149c208a | ||
|
|
65413c415f | ||
|
|
f88e03e53a | ||
|
|
0401ec4b39 | ||
|
|
5f74351729 | ||
|
|
b186f07382 | ||
|
|
710743c38f |
29 changed files with 3572 additions and 31 deletions
|
|
@ -335,6 +335,15 @@ class RuntimeConfig:
|
|||
# default); the engine never raises its own ceiling.
|
||||
estimation_enabled: bool = False
|
||||
|
||||
# ASK serving gate enable flag. When True, ASK serving is allowed.
|
||||
# Default False (dark).
|
||||
ask_serving_enabled: bool = False
|
||||
|
||||
# VERIFIED serving gate enable flag. When True, VERIFIED serving is allowed.
|
||||
# Default False (dark).
|
||||
verified_serving_enabled: bool = False
|
||||
|
||||
|
||||
|
||||
DEFAULT_IDENTITY_PACK: str = "default_general_v1"
|
||||
DEFAULT_ETHICS_PACK: str = "default_general_ethics_v1"
|
||||
|
|
|
|||
|
|
@ -19,6 +19,9 @@ Shipped so far (all off-serving — nothing here imports ``generate.derivation``
|
|||
``EpistemicState × LimitationAssessment × DisclosureClaim → ServedDisposition``.
|
||||
Mapping scaffold only — no rendering, no bus, no ``verify.py``; nothing consumes
|
||||
it yet.
|
||||
* :mod:`~core.epistemic_disclosure.ask_serving` — a narrow Q1-D served-ASK artifact
|
||||
adapter. It validates already-rendered question artifacts and returns a typed
|
||||
decision; it does not render prose and does not acquire runtime contemplation.
|
||||
* :mod:`~core.epistemic_disclosure.verified_contract` (P1-A) — the VERIFIED contract:
|
||||
the obligation, the proof shape, the validator, and the single sanctioned route to
|
||||
``EpistemicState.VERIFIED`` / ``DisclosureClaim.VERIFIED``. Contract only — no
|
||||
|
|
@ -27,6 +30,10 @@ Shipped so far (all off-serving — nothing here imports ``generate.derivation``
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from core.epistemic_disclosure.ask_serving import (
|
||||
ServedAskDecision,
|
||||
evaluate_served_ask,
|
||||
)
|
||||
from core.epistemic_disclosure.disclosure_claim import (
|
||||
DEFAULT_DISCLOSURE_CLAIM,
|
||||
DisclosureClaim,
|
||||
|
|
@ -64,6 +71,7 @@ __all__ = [
|
|||
"LimitationKind",
|
||||
"MissingSlot",
|
||||
"ResolutionAction",
|
||||
"ServedAskDecision",
|
||||
"ServedDisposition",
|
||||
"VerificationObligation",
|
||||
"VerificationProof",
|
||||
|
|
@ -73,6 +81,7 @@ __all__ = [
|
|||
"assess_from_family",
|
||||
"choose_served_disposition",
|
||||
"disclosure_for_verification",
|
||||
"evaluate_served_ask",
|
||||
"evaluate_verification",
|
||||
"terminal_for_action",
|
||||
]
|
||||
|
|
|
|||
171
core/epistemic_disclosure/ask_serving.py
Normal file
171
core/epistemic_disclosure/ask_serving.py
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
"""Stage 2 ASK served-surface artifact adapter.
|
||||
|
||||
This module is intentionally narrow: it validates a pre-rendered Q1-D
|
||||
``DeliveredQuestion`` artifact and decides whether that artifact is eligible to
|
||||
be exposed as a served ASK/QUESTION_NEEDED surface. It does not acquire
|
||||
contemplation results from runtime and does not render question prose.
|
||||
|
||||
Validation enforces the Q1-D artifact contract:
|
||||
|
||||
- top-level JSON object only;
|
||||
- ``status == "question_only"``;
|
||||
- ``requires_review is True``;
|
||||
- ``served is False``;
|
||||
- ``answer_binding`` is absent or ``None``;
|
||||
- ``question`` is an object;
|
||||
- ``question.text`` is a non-empty string;
|
||||
- ``question.slot_name`` is a non-empty string;
|
||||
- ``question_path`` exists on disk and differs from ``proposal_path``.
|
||||
|
||||
Any validation failure fails closed to the caller's fallback surface and
|
||||
standing disposition. The served text is consumed from the artifact exactly; no
|
||||
runtime prose construction or mutation happens here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from core.epistemic_disclosure.disposition import ServedDisposition, choose_served_disposition
|
||||
from core.epistemic_disclosure.limitation import LimitationAssessment
|
||||
from core.epistemic_questions.serving_gate import ask_serving_enabled
|
||||
from core.epistemic_state import EpistemicState
|
||||
|
||||
_MISSING = object()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ServedAskDecision:
|
||||
"""The adapter's served-ASK decision."""
|
||||
|
||||
served: bool
|
||||
terminal: str
|
||||
surface: str
|
||||
disposition: ServedDisposition
|
||||
|
||||
|
||||
def _terminal_value(contemplation_result: Any) -> str:
|
||||
terminal = getattr(contemplation_result, "terminal", None)
|
||||
if terminal is None:
|
||||
return "NO_PROGRESS"
|
||||
return str(getattr(terminal, "value", terminal))
|
||||
|
||||
|
||||
def _fallback_disposition(terminal: str) -> ServedDisposition:
|
||||
if terminal == "PROPOSAL_EMITTED":
|
||||
return ServedDisposition.PROPOSE
|
||||
if terminal == "SOLVED_VERIFIED":
|
||||
return ServedDisposition.COMMIT
|
||||
return ServedDisposition.REFUSE
|
||||
|
||||
|
||||
def _fallback_decision(contemplation_result: Any, fallback_surface: str) -> ServedAskDecision:
|
||||
terminal = _terminal_value(contemplation_result)
|
||||
return ServedAskDecision(
|
||||
served=False,
|
||||
terminal=terminal,
|
||||
surface=fallback_surface,
|
||||
disposition=_fallback_disposition(terminal),
|
||||
)
|
||||
|
||||
|
||||
def _validate_question_artifact(data: Any, *, question_path: Path, proposal_path: Any) -> str | None:
|
||||
"""Return the valid question text, or ``None`` for any contract violation."""
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
if data.get("status") != "question_only":
|
||||
return None
|
||||
if data.get("requires_review") is not True:
|
||||
return None
|
||||
served = data.get("served", _MISSING)
|
||||
if served is _MISSING or served is not False:
|
||||
return None
|
||||
answer_binding = data.get("answer_binding", _MISSING)
|
||||
if answer_binding is not _MISSING and answer_binding is not None:
|
||||
return None
|
||||
|
||||
question = data.get("question")
|
||||
if not isinstance(question, dict):
|
||||
return None
|
||||
|
||||
text = question.get("text")
|
||||
if not isinstance(text, str) or not text.strip():
|
||||
return None
|
||||
|
||||
slot_name = question.get("slot_name")
|
||||
if not isinstance(slot_name, str) or not slot_name.strip():
|
||||
return None
|
||||
|
||||
if proposal_path is not None and str(question_path) == str(proposal_path):
|
||||
return None
|
||||
|
||||
return text.strip()
|
||||
|
||||
|
||||
def evaluate_served_ask(
|
||||
config: Any,
|
||||
contemplation_result: Any,
|
||||
fallback_surface: str,
|
||||
) -> ServedAskDecision:
|
||||
"""Evaluate whether a Q1-D question artifact may be surfaced as ASK.
|
||||
|
||||
This is a bus/disposition adapter, not a renderer and not the runtime
|
||||
acquisition path. The caller supplies a contemplation result that already
|
||||
points to a delivered question artifact. When the gate is disabled or any
|
||||
artifact invariant fails, the adapter returns the fallback surface and the
|
||||
standing fallback disposition.
|
||||
"""
|
||||
|
||||
if not ask_serving_enabled(config):
|
||||
return _fallback_decision(contemplation_result, fallback_surface)
|
||||
|
||||
if _terminal_value(contemplation_result) != "QUESTION_NEEDED":
|
||||
return _fallback_decision(contemplation_result, fallback_surface)
|
||||
|
||||
question_path_value = getattr(contemplation_result, "question_path", None)
|
||||
proposal_path_value = getattr(contemplation_result, "proposal_path", None)
|
||||
if not question_path_value or question_path_value == proposal_path_value:
|
||||
return _fallback_decision(contemplation_result, fallback_surface)
|
||||
|
||||
question_path = Path(question_path_value)
|
||||
if not question_path.is_file():
|
||||
return _fallback_decision(contemplation_result, fallback_surface)
|
||||
|
||||
try:
|
||||
payload = json.loads(question_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return _fallback_decision(contemplation_result, fallback_surface)
|
||||
|
||||
question_text = _validate_question_artifact(
|
||||
payload,
|
||||
question_path=question_path,
|
||||
proposal_path=proposal_path_value,
|
||||
)
|
||||
if question_text is None:
|
||||
return _fallback_decision(contemplation_result, fallback_surface)
|
||||
|
||||
limitation = LimitationAssessment(
|
||||
limitation_kind="missing_information",
|
||||
resolution_action="ask_question",
|
||||
epistemic_state=EpistemicState.UNDETERMINED,
|
||||
owner_organ=payload.get("owner_organ"),
|
||||
blocking_reason=str(payload.get("blocking_reason", "")),
|
||||
)
|
||||
disposition = choose_served_disposition(
|
||||
epistemic_state=EpistemicState.UNDETERMINED,
|
||||
limitation=limitation,
|
||||
)
|
||||
|
||||
return ServedAskDecision(
|
||||
served=True,
|
||||
terminal="QUESTION_NEEDED",
|
||||
surface=question_text,
|
||||
disposition=disposition,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["ServedAskDecision", "evaluate_served_ask"]
|
||||
|
|
@ -15,8 +15,10 @@ DERIVES each from the already-shipped failure-family registry
|
|||
|
||||
**Hard invariant (no fourth taxonomy).** Every assessment is mechanically derived
|
||||
from an existing ``FailureFamily``; the *only* genuinely new resolution action is
|
||||
``ask_question`` (the future Q1/ASK tenant — it is the one action with no shipped
|
||||
terminal, see :func:`terminal_for_action`).
|
||||
``ask_question`` (the Q1/ASK tenant). Through Q1-C it was the one action with no
|
||||
shipped terminal; Q1-D adds :attr:`~generate.contemplation.findings.Terminal.QUESTION_NEEDED`
|
||||
(sibling of ``PROPOSAL_EMITTED``), so :func:`terminal_for_action` is now total — every
|
||||
action maps onto a shipped terminal, which is what makes this a consolidating view.
|
||||
|
||||
**Q1-B transitional carve-out (this slice).** Two shipped families —
|
||||
``missing_total_count`` and ``missing_weighted_total`` — are classified here as
|
||||
|
|
@ -140,17 +142,24 @@ _KIND_TO_STATE: dict[LimitationKind, EpistemicState] = {
|
|||
"input_shape": EpistemicState.UNDETERMINED,
|
||||
}
|
||||
|
||||
#: The shipped contemplation ``Terminal`` each action corresponds to. ``ask_question``
|
||||
#: is ``None`` — it is the ONE action the spine adds that has no terminal yet (Q1/ASK).
|
||||
#: This map is the proof that the limitation pass is a consolidating view, not a new
|
||||
#: universe: five of six actions already exist as terminals.
|
||||
_ACTION_TO_TERMINAL: dict[ResolutionAction, Terminal | None] = {
|
||||
#: The shipped contemplation ``Terminal`` each action corresponds to. Through Q1-C
|
||||
#: ``ask_question`` mapped to ``None`` (the one action the spine added with no terminal
|
||||
#: yet); Q1-D ships ``QUESTION_NEEDED`` (sibling of ``PROPOSAL_EMITTED``), so the map is
|
||||
#: now TOTAL — all six actions correspond to shipped terminals. This totality is the
|
||||
#: proof the limitation pass is a consolidating view, not a new universe.
|
||||
#:
|
||||
#: NOTE: this is the action's *home* terminal "in principle". The terminal a Q1-D
|
||||
#: *delivery* actually emits depends on renderability — an unrenderable ``ask_question``
|
||||
#: falls back to the family's standing disposition (the D2 guard in
|
||||
#: :mod:`core.epistemic_questions.delivery`), it does NOT emit a contentless
|
||||
#: ``QUESTION_NEEDED``.
|
||||
_ACTION_TO_TERMINAL: dict[ResolutionAction, Terminal] = {
|
||||
"answer": Terminal.SOLVED_VERIFIED,
|
||||
"emit_proposal": Terminal.PROPOSAL_EMITTED,
|
||||
"refuse_known_boundary": Terminal.REFUSED_KNOWN_BOUNDARY,
|
||||
"report_contradiction": Terminal.CONTRADICTION_DETECTED,
|
||||
"step_aside": Terminal.NO_PROGRESS,
|
||||
"ask_question": None,
|
||||
"ask_question": Terminal.QUESTION_NEEDED,
|
||||
}
|
||||
|
||||
#: **Transitional carve-out (Q1-B).** Families this slice classifies as
|
||||
|
|
@ -344,12 +353,15 @@ def assess_from_attempt(attempt: ComprehensionAttempt) -> LimitationAssessment |
|
|||
)
|
||||
|
||||
|
||||
def terminal_for_action(action: ResolutionAction) -> Terminal | None:
|
||||
"""The shipped contemplation ``Terminal`` an action corresponds to.
|
||||
def terminal_for_action(action: ResolutionAction) -> Terminal:
|
||||
"""The shipped contemplation ``Terminal`` an action corresponds to — total.
|
||||
|
||||
``None`` only for ``ask_question`` — the one action this spine adds that has no
|
||||
shipped terminal yet (the Q1/ASK tenant). Every other action maps to an existing
|
||||
terminal, which is what makes this a consolidating view rather than a new taxonomy.
|
||||
Every action maps to a shipped terminal; ``ask_question`` resolves to
|
||||
``QUESTION_NEEDED`` (added by Q1-D, sibling of ``PROPOSAL_EMITTED``). That totality
|
||||
is what makes this a consolidating view rather than a new taxonomy. This is the
|
||||
action's *home* terminal; the terminal a Q1-D delivery actually emits may differ
|
||||
when the question is unrenderable (the D2 fallback — see
|
||||
:mod:`core.epistemic_questions.delivery`).
|
||||
"""
|
||||
return _ACTION_TO_TERMINAL[action]
|
||||
|
||||
|
|
|
|||
33
core/epistemic_disclosure/serving_gate.py
Normal file
33
core/epistemic_disclosure/serving_gate.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
"""VERIFIED serving gate helper — default-dark, no served-surface wiring.
|
||||
|
||||
This module centralizes the future kill-switch predicate for VERIFIED serving.
|
||||
It is default-dark / fail-closed: if the config field is missing or malformed,
|
||||
it must evaluate to False.
|
||||
|
||||
This helper only centralizes the future kill-switch predicate so that future serving code
|
||||
has one audited predicate. It does not wire any served-surface or implement served
|
||||
VERIFIED behavior. Missing field means False.
|
||||
|
||||
Note that eval-gold-backed producers (such as the verification producer in
|
||||
evals/constraint_oracle/verified_producer.py) are not serving-eligible.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from core.config import DEFAULT_CONFIG, RuntimeConfig
|
||||
|
||||
|
||||
def verified_serving_enabled(config: RuntimeConfig | Any | None = None) -> bool:
|
||||
"""Return whether served VERIFIED delivery is explicitly enabled.
|
||||
|
||||
Missing attribute means False. This is the load-bearing dark-gate invariant:
|
||||
the served VERIFIED path cannot light merely because the helper exists or because
|
||||
an older RuntimeConfig instance lacks the future field.
|
||||
"""
|
||||
cfg = DEFAULT_CONFIG if config is None else config
|
||||
return bool(getattr(cfg, "verified_serving_enabled", False))
|
||||
|
||||
|
||||
__all__ = ["verified_serving_enabled"]
|
||||
|
|
@ -64,6 +64,7 @@ class VerificationObligation:
|
|||
|
||||
requires_independent_read: bool # two DISTINCT reader lineages — not the same read twice
|
||||
rejects_wrong_read_even_if_solved: bool # the two reads must CONVERGE — catches a wrong read
|
||||
requires_bound_slots: bool # the answer binds to the STATED slots (query/unknowns), not phantom ones
|
||||
requires_back_substitution: bool # the answer back-substitutes into the canonical structure
|
||||
requires_boundary_clear: bool # no organ boundary fired in the chain
|
||||
|
||||
|
|
@ -72,6 +73,7 @@ class VerificationObligation:
|
|||
VERIFICATION_OBLIGATION: VerificationObligation = VerificationObligation(
|
||||
requires_independent_read=True,
|
||||
rejects_wrong_read_even_if_solved=True,
|
||||
requires_bound_slots=True,
|
||||
requires_back_substitution=True,
|
||||
requires_boundary_clear=True,
|
||||
)
|
||||
|
|
@ -86,6 +88,14 @@ class VerificationProof:
|
|||
``independent_reader_lineage`` must DIFFER (independence), and ``primary_read_digest``
|
||||
/ ``independent_read_digest`` must MATCH (convergence on one canonical structure).
|
||||
Independence + convergence together are what reject a faithful solve of a wrong read.
|
||||
|
||||
The "from the stated quantities" obligation is split into THREE separable digests
|
||||
(P1-C hardening — for replay, audit, and failure localization):
|
||||
``derivation_digest`` (a solve happened from the structure), ``bound_slots_digest``
|
||||
(the answer binds to a STATED slot — the asked unknown — not a phantom), and
|
||||
``back_substitution_digest`` (the answer satisfies the constraints). A complete
|
||||
derivation does NOT imply the answer bound to a declared slot, so the two are
|
||||
distinct obligations (``test_derivation_digest_alone_is_insufficient_without_bound_slots``).
|
||||
"""
|
||||
|
||||
source_problem_digest: str # provenance: hash of the problem text
|
||||
|
|
@ -93,7 +103,8 @@ class VerificationProof:
|
|||
independent_reader_lineage: str # identity of the independent cross-check reader
|
||||
primary_read_digest: str # canonical structure the primary read produced
|
||||
independent_read_digest: str # canonical structure the independent read produced
|
||||
derivation_digest: str # the derivation from the STATED quantities
|
||||
derivation_digest: str # the derivation (solve) from the STATED quantities
|
||||
bound_slots_digest: str # the answer binds to the STATED slots (asked unknown), not a phantom
|
||||
back_substitution_digest: str # back-substitution into the canonical structure
|
||||
boundary_clear: bool # no organ boundary fired
|
||||
contradiction_clear: bool # no contradiction family fired
|
||||
|
|
@ -102,6 +113,7 @@ class VerificationProof:
|
|||
# Reason codes for a failed obligation — each names exactly one violated rule.
|
||||
REASON_READS_NOT_INDEPENDENT = "reads_not_independent" # same reader lineage twice
|
||||
REASON_READS_DISAGREE = "reads_disagree" # the wrong-read catcher
|
||||
REASON_NO_BOUND_SLOTS = "no_bound_slots" # answer did not bind to a stated slot
|
||||
REASON_NO_BACK_SUBSTITUTION = "no_back_substitution"
|
||||
REASON_BOUNDARY_FIRED = "boundary_fired"
|
||||
REASON_CONTRADICTION_PRESENT = "contradiction_present"
|
||||
|
|
@ -143,6 +155,9 @@ def evaluate_verification(
|
|||
):
|
||||
failed.append(REASON_READS_DISAGREE)
|
||||
|
||||
if obligation.requires_bound_slots and not proof.bound_slots_digest:
|
||||
failed.append(REASON_NO_BOUND_SLOTS)
|
||||
|
||||
if obligation.requires_back_substitution and not proof.back_substitution_digest:
|
||||
failed.append(REASON_NO_BACK_SUBSTITUTION)
|
||||
|
||||
|
|
|
|||
40
core/epistemic_questions/__init__.py
Normal file
40
core/epistemic_questions/__init__.py
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
"""The epistemic question organ — render (Q1-C) and off-serving delivery (Q1-D).
|
||||
|
||||
Q1-C (:mod:`core.epistemic_questions.render`) turns an ASK
|
||||
:class:`~core.epistemic_disclosure.limitation.LimitationAssessment` into an
|
||||
:class:`~core.epistemic_questions.render.EpistemicQuestion` under the wrong=0
|
||||
grounded-rendering invariant (scoping §2) — render only, no fabrication.
|
||||
|
||||
Q1-D (:mod:`core.epistemic_questions.delivery`) routes that rendered question onto
|
||||
the contemplation bus as the ``QUESTION_NEEDED`` tenant and writes a proposal-only
|
||||
artifact to the ``teaching/questions`` sink — consuming the renderer verbatim, never
|
||||
rendering. Off-serving: no served surface, no ``ask_serving_enabled`` yet.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from core.epistemic_questions.delivery import (
|
||||
AnswerBinding,
|
||||
DeliveredQuestion,
|
||||
DeliveryOutcome,
|
||||
default_question_root,
|
||||
deliver_ask,
|
||||
emit_question,
|
||||
question_path,
|
||||
)
|
||||
from core.epistemic_questions.render import (
|
||||
EpistemicQuestion,
|
||||
render_question,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AnswerBinding",
|
||||
"DeliveredQuestion",
|
||||
"DeliveryOutcome",
|
||||
"EpistemicQuestion",
|
||||
"default_question_root",
|
||||
"deliver_ask",
|
||||
"emit_question",
|
||||
"question_path",
|
||||
"render_question",
|
||||
]
|
||||
272
core/epistemic_questions/delivery.py
Normal file
272
core/epistemic_questions/delivery.py
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
"""Q1-D — off-serving ASK delivery: route a rendered question onto the bus.
|
||||
|
||||
The fourth and final Q1 rung. Given an ``ask_question``
|
||||
:class:`~core.epistemic_disclosure.limitation.LimitationAssessment`, decide the
|
||||
contemplation :class:`~generate.contemplation.findings.Terminal` and, when a question
|
||||
can honestly be asked, produce a :class:`DeliveredQuestion` artifact for the
|
||||
review-gated, **question-only** ``teaching/questions`` sink. A question is an *intake
|
||||
request*, NOT a capability proposal — ``question_only`` is its own lane, distinct from
|
||||
the ``proposal_only`` lane of :mod:`core.comprehension_attempt.proposal`. This is the
|
||||
ASK *analogue* of that emitter (and just as toothless: it never serves, never mounts,
|
||||
always requires review), but the artifacts must never be conflated.
|
||||
|
||||
**The rung separation (the steer).** Q1-D *consumes* the Q1-C
|
||||
:class:`~core.epistemic_questions.render.EpistemicQuestion` — it does NOT render.
|
||||
:func:`render_question` is called exactly once (in :func:`deliver_ask`) and its
|
||||
result is wrapped verbatim; Q1-D constructs no user-facing prose of its own, so the
|
||||
Q1-C grounded-rendering wrong=0 guard is never bypassed by a second surface.
|
||||
|
||||
Q1-B: typed residue (what is missing, as typed slots)
|
||||
Q1-C: renderability (can it be asked without fabricating? → EpistemicQuestion)
|
||||
Q1-D: delivery (route the rendered question onto the bus) ← here
|
||||
|
||||
**D2 — the delivery-side wrong=0 guard.** A contentless ``QUESTION_NEEDED`` is worse
|
||||
than useless: it is a *false intake surface* (the user is invited to answer a question
|
||||
that names nothing). So when :func:`render_question` returns ``unrenderable``
|
||||
(``renderability_gap`` / ``multi_slot_not_supported`` / ``no_missing_slot`` / the
|
||||
fabrication backstop), :func:`deliver_ask` does NOT emit ``QUESTION_NEEDED``. It falls
|
||||
back to the family's *standing disposition*:
|
||||
|
||||
family still proposes (``proposal_allowed``) → ``PROPOSAL_EMITTED``
|
||||
otherwise → ``NO_PROGRESS``
|
||||
|
||||
This guard is enforced twice: in :func:`deliver_ask`'s branch, and structurally — a
|
||||
:class:`DeliveredQuestion` *cannot wrap an unrenderable question* (its
|
||||
``__post_init__`` refuses), so ``QUESTION_NEEDED`` is unreachable without a real,
|
||||
rendered question.
|
||||
|
||||
**D3 — the carve-out stays.** Q1-D delivery is OFF-SERVING. It does not flip the
|
||||
``Q1B_ASK_CARVE_OUT`` (``missing_total_count`` / ``missing_weighted_total`` keep
|
||||
``proposal_allowed = True`` in the registry, so the proposal pile keeps working). Both
|
||||
signals coexist during the carve-out — the off-serving question artifact AND the
|
||||
existing proposal — so no operational signal is lost. The flip to ``ask`` waits on a
|
||||
future ``ask_serving_enabled`` gate, not on this rung.
|
||||
|
||||
**D5 — single-slot only.** Q1-C refuses multi-slot rendering, so Q1-D delivers at most
|
||||
one ``DeliveredQuestion`` per assessment; a multi-slot assessment is ``unrenderable``
|
||||
(``multi_slot_not_supported``) and takes the D2 fallback. No fan-out here.
|
||||
|
||||
**Off-serving.** Imports nothing from ``generate.derivation`` / ``core.reliability_gate``;
|
||||
it cannot move the sealed GSM8K metric, and there is NO served surface — no
|
||||
``ask_serving_enabled``, no ``chat/runtime.py`` wiring. The artifacts land in the
|
||||
review-gated teaching sink and nowhere else.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from core.comprehension_attempt.failure_family import family_by_name
|
||||
from core.epistemic_disclosure.limitation import LimitationAssessment
|
||||
from core.epistemic_questions.render import EpistemicQuestion, render_question
|
||||
from generate.contemplation.findings import Terminal
|
||||
|
||||
#: The sink status — the ASK analogue of the proposal emitter's ``"proposal_only"``.
|
||||
#: A delivered question is review-gated intake, never a served answer.
|
||||
_QUESTION_STATUS = "question_only"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AnswerBinding:
|
||||
"""RESERVED for Q2 — the typed seat a future answer round-trip binds into.
|
||||
|
||||
Q1-D produces NO ``AnswerBinding`` (every :class:`DeliveredQuestion` carries
|
||||
``answer_binding = None``; see that class's ``__post_init__``). The seat exists so
|
||||
the Q2 binder — which must re-enter the limitation gate with augmented input, never
|
||||
mutate the model mid-flight (scoping §4) — is wireable without reshaping the
|
||||
artifact. ``target_slot`` / ``binding_target`` mirror the
|
||||
:class:`~core.epistemic_disclosure.limitation.MissingSlot` the answer fills.
|
||||
"""
|
||||
|
||||
target_organ: str
|
||||
target_slot: str
|
||||
binding_target: str
|
||||
parser: str
|
||||
unit: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DeliveredQuestion:
|
||||
"""A review-gated, question-only ASK artifact wrapping a rendered Q1-C question.
|
||||
|
||||
``question_only`` is its OWN lane (an intake request), not the ``proposal_only``
|
||||
lane — the analogy to :class:`~core.comprehension_attempt.proposal.FailureProposal`
|
||||
is structural (toothless, review-gated, never served), never semantic.
|
||||
|
||||
The invariant fields are enforced in ``__post_init__`` so even a hand-constructed
|
||||
instance cannot become a contentless question, a served question, or an
|
||||
answer-bound (Q2) question — illegal states are unrepresentable:
|
||||
|
||||
- it can never wrap an ``unrenderable`` question (the D2 guard, structurally) —
|
||||
so a ``QUESTION_NEEDED`` terminal always carries real, rendered text;
|
||||
- it can never be ``served`` (off-serving; ``ask_serving_enabled`` does not exist);
|
||||
- it always ``requires_review``;
|
||||
- its ``answer_binding`` is always ``None`` in Q1-D (the Q2 seat is reserved, unbound).
|
||||
"""
|
||||
|
||||
question: EpistemicQuestion
|
||||
owner_organ: str | None
|
||||
blocking_reason: str
|
||||
answer_binding: AnswerBinding | None = None
|
||||
status: str = _QUESTION_STATUS
|
||||
requires_review: bool = True
|
||||
served: bool = False
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.question.unrenderable or self.question.text is None:
|
||||
raise ValueError(
|
||||
"a DeliveredQuestion cannot wrap an unrenderable question "
|
||||
f"(reason={self.question.reason!r}); the D2 fallback handles it"
|
||||
)
|
||||
if self.status != _QUESTION_STATUS:
|
||||
raise ValueError(
|
||||
f"question status must be {_QUESTION_STATUS!r}; got {self.status!r}"
|
||||
)
|
||||
if self.served:
|
||||
raise ValueError("a Q1-D delivered question is never served")
|
||||
if not self.requires_review:
|
||||
raise ValueError("a delivered question always requires review")
|
||||
if self.answer_binding is not None:
|
||||
raise ValueError("answer_binding is reserved for Q2; Q1-D emits None")
|
||||
|
||||
def content_hash(self) -> str:
|
||||
"""Deterministic content address: same question on the same blocking family and
|
||||
slot always yields the same hash (idempotent sink writes). No clock, no
|
||||
randomness. The rendered ``text`` is included so a template change re-addresses.
|
||||
"""
|
||||
slot_name = self.question.slot.slot_name if self.question.slot else ""
|
||||
payload = f"{self.blocking_reason}:{slot_name}:{self.question.text}"
|
||||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||
|
||||
def to_json_dict(self) -> dict[str, Any]:
|
||||
slot = self.question.slot
|
||||
return {
|
||||
"status": self.status,
|
||||
"blocking_reason": self.blocking_reason,
|
||||
"owner_organ": self.owner_organ,
|
||||
"question": {
|
||||
"text": self.question.text,
|
||||
"reason": self.question.reason,
|
||||
"slot_name": slot.slot_name if slot else None,
|
||||
"expected_unit_or_type": slot.expected_unit_or_type if slot else None,
|
||||
"binding_target": slot.binding_target if slot else None,
|
||||
},
|
||||
"answer_binding": None, # reserved (Q2)
|
||||
"requires_review": self.requires_review,
|
||||
"served": self.served,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DeliveryOutcome:
|
||||
"""The result of routing one ``ask_question`` assessment.
|
||||
|
||||
``question`` is the artifact iff ``terminal is QUESTION_NEEDED`` (a renderable ask);
|
||||
otherwise it is ``None`` and ``fallback_reason`` carries the unrenderable reason that
|
||||
triggered the D2 standing-disposition fallback.
|
||||
"""
|
||||
|
||||
terminal: Terminal
|
||||
question: DeliveredQuestion | None
|
||||
fallback_reason: str | None
|
||||
|
||||
|
||||
def _standing_disposition(blocking_reason: str) -> Terminal:
|
||||
"""The D2 fallback terminal for an unrenderable ask: the family's standing move.
|
||||
|
||||
A family that still proposes (``proposal_allowed``) falls back to
|
||||
``PROPOSAL_EMITTED`` — its existing operational signal, preserved (D3). Anything
|
||||
else (an unknown reason, or a family that does not propose) falls back to
|
||||
``NO_PROGRESS``. Never a contentless ``QUESTION_NEEDED``.
|
||||
"""
|
||||
family = family_by_name(blocking_reason)
|
||||
if family is not None and family.proposal_allowed:
|
||||
return Terminal.PROPOSAL_EMITTED
|
||||
return Terminal.NO_PROGRESS
|
||||
|
||||
|
||||
def deliver_ask(assessment: LimitationAssessment) -> DeliveryOutcome:
|
||||
"""Route an ``ask_question`` assessment to a terminal — render via Q1-C, never here.
|
||||
|
||||
Renderable → ``QUESTION_NEEDED`` + a :class:`DeliveredQuestion` wrapping the Q1-C
|
||||
result verbatim. Unrenderable → the D2 standing-disposition fallback (no artifact).
|
||||
|
||||
Raises ``ValueError`` if called on a non-ASK assessment: the bus routes only
|
||||
``ask_question`` resolutions here, so any other action is a wiring error, not a
|
||||
runtime input — fail loudly rather than silently mis-deliver.
|
||||
"""
|
||||
if assessment.resolution_action != "ask_question":
|
||||
raise ValueError(
|
||||
"deliver_ask is only valid for ask_question assessments; got "
|
||||
f"{assessment.resolution_action!r}"
|
||||
)
|
||||
|
||||
question = render_question(assessment)
|
||||
if question.unrenderable:
|
||||
return DeliveryOutcome(
|
||||
terminal=_standing_disposition(assessment.blocking_reason),
|
||||
question=None,
|
||||
fallback_reason=question.reason,
|
||||
)
|
||||
|
||||
delivered = DeliveredQuestion(
|
||||
question=question,
|
||||
owner_organ=assessment.owner_organ,
|
||||
blocking_reason=assessment.blocking_reason,
|
||||
)
|
||||
return DeliveryOutcome(
|
||||
terminal=Terminal.QUESTION_NEEDED,
|
||||
question=delivered,
|
||||
fallback_reason=None,
|
||||
)
|
||||
|
||||
|
||||
def default_question_root() -> Path:
|
||||
"""``<repo>/teaching/questions`` — the write-only, review-gated ASK sink.
|
||||
|
||||
A sibling of ``teaching/proposals`` (D4): questions are intake requests, not
|
||||
capability proposals, so they do not overload the proposal pile.
|
||||
"""
|
||||
return Path(__file__).resolve().parents[2] / "teaching" / "questions"
|
||||
|
||||
|
||||
def question_path(delivered: DeliveredQuestion, root: Path | None = None) -> Path:
|
||||
base = root if root is not None else default_question_root()
|
||||
return base / f"{delivered.content_hash()}.json"
|
||||
|
||||
|
||||
def emit_question(
|
||||
assessment: LimitationAssessment, *, root: Path | None = None
|
||||
) -> Path | None:
|
||||
"""Deliver an ask assessment and, iff it renders, write the artifact to the sink.
|
||||
|
||||
Returns the artifact path for a ``QUESTION_NEEDED`` delivery, or ``None`` when the
|
||||
ask was unrenderable and fell back (D2) — no contentless artifact is ever written.
|
||||
Idempotent: the same question writes the same content-addressed path with
|
||||
byte-identical content (``sort_keys``). Creates the sink directory on demand.
|
||||
"""
|
||||
outcome = deliver_ask(assessment)
|
||||
if outcome.question is None:
|
||||
return None
|
||||
path = question_path(outcome.question, root)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(
|
||||
json.dumps(outcome.question.to_json_dict(), indent=2, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AnswerBinding",
|
||||
"DeliveredQuestion",
|
||||
"DeliveryOutcome",
|
||||
"default_question_root",
|
||||
"deliver_ask",
|
||||
"emit_question",
|
||||
"question_path",
|
||||
]
|
||||
195
core/epistemic_questions/render.py
Normal file
195
core/epistemic_questions/render.py
Normal file
|
|
@ -0,0 +1,195 @@
|
|||
"""The grounded-only question renderer (Q1-C) — wrong=0-safe by construction.
|
||||
|
||||
This is the renderer rung of the Epistemic Disclosure ASK spine: given a
|
||||
:class:`~core.epistemic_disclosure.limitation.LimitationAssessment` whose
|
||||
``resolution_action == "ask_question"`` and which carries at least one typed
|
||||
:class:`~core.epistemic_disclosure.limitation.MissingSlot`, produce an
|
||||
:class:`EpistemicQuestion` — a rendered user-facing question, or an explicit
|
||||
``question_unrenderable`` verdict. Nothing here delivers, serves, or chooses a
|
||||
disposition (that is Q1-D); this is *only* surface realization of the residue.
|
||||
|
||||
**The wrong=0 invariant (scoping §2 / session §1.5.7) — the whole point.**
|
||||
|
||||
A question may name an entity, slot, unit, or relation only if it appears
|
||||
*verbatim* in the assessment's ``grounded_terms``. When the grounded terms
|
||||
lack what a question needs, degrade to a generic question or emit
|
||||
``question_unrenderable`` — never a named guess.
|
||||
|
||||
Two substrate facts force the conservative policy below:
|
||||
|
||||
1. ``grounded_terms`` is empty for every assessment produced today — the readers
|
||||
do not yet emit verbatim evidence on refusal (scoping §3, the substrate gap
|
||||
Q1 must close first). So there is no grounded problem-entity to name.
|
||||
2. A *missing* slot's referent is, by definition, absent from the comprehension
|
||||
trace — the missing thing can never appear in ``grounded_terms`` even once
|
||||
readers do emit evidence. ``grounded_terms`` can only supply *context*
|
||||
entities, and binding a slot to its context entity needs an alignment step
|
||||
that Q1-C does not have (a later rung).
|
||||
|
||||
**Chosen rendering policy — generic-structural, names zero problem entities.**
|
||||
|
||||
Because Q1-C can neither (today) read grounded context nor (ever, for the slot
|
||||
itself) name the missing referent from grounded text, the only wrong=0-safe
|
||||
artifact it can render is a *generic* question whose sole variable content is a
|
||||
controlled English phrase for the slot's structural *type*
|
||||
(``expected_unit_or_type``), drawn from the closed, audited
|
||||
:data:`_CLOSED_TYPE_PHRASES` map below. Concretely the renderer:
|
||||
|
||||
- NEVER surfaces ``slot_name`` or ``binding_target`` — these are snake_case
|
||||
structural identifiers, and user-facing prose must never come from them
|
||||
(limitation.py ``MissingSlot`` docstring; session §1.5.7). ``slot_name`` is
|
||||
also the field most likely to *read* like a fabricated entity (``ben_rate`` →
|
||||
the forbidden "Ben"), so it is never touched.
|
||||
- NEVER prettifies a snake_case identifier into a natural-language entity — no
|
||||
capitalization, no possessive, no splitting on underscores.
|
||||
- Translates ``expected_unit_or_type`` through the closed map only; an unmapped
|
||||
type degrades to ``question_unrenderable`` (a ``renderability_gap``) rather
|
||||
than dumping raw snake_case or guessing.
|
||||
- Names no problem-specific entity at all. The closed type phrases ("a
|
||||
whole-number count") are generic structural descriptors that assert nothing
|
||||
about *this* problem — distinct from problem-specific names, which would need
|
||||
grounding. This is the line scoping §2 draws ("generic, all terms grounded"
|
||||
vs. the fabricated "Ben").
|
||||
|
||||
A post-render fabrication guard (:func:`_names_only_grounded`) re-checks that
|
||||
every word in the rendered text is closed-vocabulary scaffold or appears in
|
||||
``grounded_terms`` — defense in depth so a fabricated token can never escape even
|
||||
if the template were later edited carelessly.
|
||||
|
||||
**Off-serving.** Imports nothing from ``generate.derivation`` or
|
||||
``core.reliability_gate``; it cannot move the sealed GSM8K metric.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
from core.epistemic_disclosure.limitation import (
|
||||
LimitationAssessment,
|
||||
MissingSlot,
|
||||
)
|
||||
|
||||
#: Closed, audited map from a family-pinned ``expected_unit_or_type`` to a
|
||||
#: controlled English phrase. Keys are exactly the slot types that ship today
|
||||
#: (``core.epistemic_disclosure.limitation._FAMILY_TO_MISSING_SLOTS``). A slot
|
||||
#: whose type is absent here is NOT rendered — the renderer refuses with a
|
||||
#: ``renderability_gap`` rather than surfacing raw snake_case. New slot types
|
||||
#: earn a phrase here (with a test) when their family lands, never by guessing.
|
||||
_CLOSED_TYPE_PHRASES: dict[str, str] = {
|
||||
"count_int": "a whole-number count",
|
||||
"measured_unit_int": "a whole-number quantity",
|
||||
}
|
||||
|
||||
#: The fixed question scaffold. The only hole is the closed type phrase; every
|
||||
#: other word is constant, problem-independent English. It names no entity.
|
||||
_TEMPLATE = (
|
||||
"To answer this, one more value is still needed — {phrase} — that the "
|
||||
"problem does not state. What is it?"
|
||||
)
|
||||
|
||||
#: Machine reasons for an :class:`EpistemicQuestion`. Closed set.
|
||||
_REASON_RENDERED = "rendered"
|
||||
_REASON_NOT_ASK = "not_ask"
|
||||
_REASON_NO_SLOT = "no_missing_slot"
|
||||
_REASON_MULTI_SLOT = "multi_slot_not_supported"
|
||||
_REASON_RENDERABILITY_GAP = "renderability_gap"
|
||||
_REASON_FABRICATION_GUARD = "fabrication_guard"
|
||||
|
||||
|
||||
def _tokens(text: str) -> set[str]:
|
||||
"""Lowercased maximal alphabetic runs — the unit the fabrication guard checks."""
|
||||
return set(re.findall(r"[a-z]+", text.lower()))
|
||||
|
||||
|
||||
#: Every word the renderer is allowed to emit *without* grounding: the scaffold
|
||||
#: words plus the words of every closed type phrase. Built once at import.
|
||||
_ALLOWED_SCAFFOLD_WORDS: frozenset[str] = frozenset(
|
||||
_tokens(_TEMPLATE.replace("{phrase}", " "))
|
||||
| {w for phrase in _CLOSED_TYPE_PHRASES.values() for w in _tokens(phrase)}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class EpistemicQuestion:
|
||||
"""The rendered ASK artifact, or an explicit unrenderable verdict.
|
||||
|
||||
``slot`` is the bound :class:`MissingSlot` the question is about — present
|
||||
whenever a slot was selected, ``None`` when the assessment carried no slot to
|
||||
bind (non-ASK, or zero slots). ``text`` is the rendered question, or ``None``
|
||||
when ``unrenderable``. ``reason`` is a closed-set machine string (one of the
|
||||
``_REASON_*`` constants) explaining the verdict; for a renderable question it
|
||||
is :data:`_REASON_RENDERED`.
|
||||
"""
|
||||
|
||||
slot: MissingSlot | None
|
||||
text: str | None
|
||||
unrenderable: bool
|
||||
reason: str
|
||||
|
||||
|
||||
def _unrenderable(reason: str, slot: MissingSlot | None = None) -> EpistemicQuestion:
|
||||
"""A ``question_unrenderable`` verdict with no text."""
|
||||
return EpistemicQuestion(slot=slot, text=None, unrenderable=True, reason=reason)
|
||||
|
||||
|
||||
def _names_only_grounded(text: str, grounded_terms: tuple[str, ...]) -> bool:
|
||||
"""True iff every word in ``text`` is closed-vocab scaffold or grounded.
|
||||
|
||||
The wrong=0 guard, enforced post-render as defense in depth: a fabricated
|
||||
entity (a word neither in the closed scaffold/phrase vocabulary nor verbatim
|
||||
in ``grounded_terms``) makes this return ``False``, and the renderer refuses.
|
||||
With today's empty ``grounded_terms`` and a fully closed-vocab template this
|
||||
holds by construction; the guard exists so that can never silently change.
|
||||
"""
|
||||
allowed = _ALLOWED_SCAFFOLD_WORDS | _tokens(" ".join(grounded_terms))
|
||||
return _tokens(text) <= allowed
|
||||
|
||||
|
||||
def render_question(assessment: LimitationAssessment) -> EpistemicQuestion:
|
||||
"""Render a single-slot generic ASK question, or refuse to render.
|
||||
|
||||
Strictly single-slot: Q1-C renders only when the assessment carries
|
||||
*exactly one* missing slot. The fixed template asserts "one more value is
|
||||
still needed" — a globally-quantified claim that exactly one value is
|
||||
missing — so rendering the first of several slots and ignoring the rest
|
||||
would make that sentence subtly false (it would imply the single rendered
|
||||
value closes the gap when others remain). Rather than weaken the template to
|
||||
an honest-but-vaguer plural, a multi-slot assessment refuses with
|
||||
``multi_slot_not_supported`` (slot ``None``); one-question-per-slot fan-out
|
||||
is a later rung. The renderer also refuses (``question_unrenderable``) when
|
||||
the assessment is not an ASK, carries no slot, or the slot's structural type
|
||||
is outside the closed phrase map. It NEVER fabricates a natural-language
|
||||
entity name — see the module docstring for the policy and the wrong=0
|
||||
rationale.
|
||||
"""
|
||||
if assessment.resolution_action != "ask_question":
|
||||
return _unrenderable(_REASON_NOT_ASK)
|
||||
if not assessment.missing_slots:
|
||||
return _unrenderable(_REASON_NO_SLOT)
|
||||
if len(assessment.missing_slots) > 1:
|
||||
# Strictly single-slot: the template claims exactly one value is
|
||||
# missing, which is false when several slots remain. Refuse rather than
|
||||
# render the first and silently drop the rest.
|
||||
return _unrenderable(_REASON_MULTI_SLOT, slot=None)
|
||||
|
||||
slot = assessment.missing_slots[0]
|
||||
phrase = _CLOSED_TYPE_PHRASES.get(slot.expected_unit_or_type)
|
||||
if phrase is None:
|
||||
# Unknown structural type: refuse rather than surface raw snake_case.
|
||||
return _unrenderable(_REASON_RENDERABILITY_GAP, slot=slot)
|
||||
|
||||
text = _TEMPLATE.format(phrase=phrase)
|
||||
if not _names_only_grounded(text, assessment.grounded_terms):
|
||||
# Unreachable by construction; the guard is the wrong=0 backstop.
|
||||
return _unrenderable(_REASON_FABRICATION_GUARD, slot=slot)
|
||||
|
||||
return EpistemicQuestion(
|
||||
slot=slot, text=text, unrenderable=False, reason=_REASON_RENDERED
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"EpistemicQuestion",
|
||||
"render_question",
|
||||
]
|
||||
32
core/epistemic_questions/serving_gate.py
Normal file
32
core/epistemic_questions/serving_gate.py
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
"""ASK serving gate helper — default-dark, no served-surface wiring.
|
||||
|
||||
This module is the first code slice after the ASK serving-integration scoping brief.
|
||||
It intentionally does **not** call ``deliver_ask``/``emit_question``, does not import
|
||||
``chat.runtime``, and does not expose any user-facing surface. It only centralizes the
|
||||
kill-switch read so future serving code has one audited predicate.
|
||||
|
||||
The planned config field is ``RuntimeConfig.ask_serving_enabled``. During this dark-gate
|
||||
slice the predicate is conservative: absent field == ``False``. That lets the helper land
|
||||
without widening behavior and preserves the current default for every existing
|
||||
``RuntimeConfig`` instance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from core.config import DEFAULT_CONFIG, RuntimeConfig
|
||||
|
||||
|
||||
def ask_serving_enabled(config: RuntimeConfig | Any | None = None) -> bool:
|
||||
"""Return whether served ASK delivery is explicitly enabled.
|
||||
|
||||
Missing attribute means ``False``. That is the load-bearing dark-gate invariant:
|
||||
the served ASK path cannot light merely because the helper exists or because an
|
||||
older ``RuntimeConfig`` instance lacks the future field.
|
||||
"""
|
||||
cfg = DEFAULT_CONFIG if config is None else config
|
||||
return bool(getattr(cfg, "ask_serving_enabled", False))
|
||||
|
||||
|
||||
__all__ = ["ask_serving_enabled"]
|
||||
199
docs/analysis/ask-serving-integration-scoping-2026-06-08.md
Normal file
199
docs/analysis/ask-serving-integration-scoping-2026-06-08.md
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
# ASK serving-integration — `ask_serving_enabled` — scoping brief
|
||||
|
||||
**Date:** 2026-06-08 · **Status:** scoping (NO CODE) · **HOLD for review** ·
|
||||
**Branch:** `docs/serving-integration-scoping`
|
||||
|
||||
**What this brief is.** The scope for the *one* served-surface decision the ASK lane
|
||||
deferred. The off-serving ASK lane is complete and on main:
|
||||
|
||||
```text
|
||||
Q1-B typed residue + ask classification (LimitationAssessment.missing_slots)
|
||||
Q1-C grounded-only rendering (render_question → EpistemicQuestion)
|
||||
Q1-D off-serving delivery artifact (deliver_ask → QUESTION_NEEDED + sink)
|
||||
```
|
||||
|
||||
`deliver_ask` / `emit_question` exist and are tested, but **nothing calls them from a
|
||||
live path** — they are produced-but-not-emitted, exactly as P1-B's verified producer
|
||||
is built-but-not-served. This brief scopes the step that closes that gap: letting a
|
||||
`QUESTION_NEEDED` actually reach a user, behind a named gate. **No code here** — this
|
||||
is the decision surface for review.
|
||||
|
||||
> Companions: [[q1-d-ask-bus-delivery-scoping-2026-06-08]] (the off-serving rung, §7
|
||||
> defers exactly this), [[verified-serving-wiring-scoping-2026-06-08]] (the VERIFIED
|
||||
> half of the same "where off-serving stops" line). Design of record: the session doc
|
||||
> §5 / §1.5.8 (the disclosure bus).
|
||||
|
||||
---
|
||||
|
||||
## 1. The five things this decision must pin
|
||||
|
||||
The steer named five; each is grounded against shipped code below.
|
||||
|
||||
```text
|
||||
1. ask_serving_enabled — the kill switch
|
||||
2. the pass_manager integration point — where deliver_ask is actually called
|
||||
3. the Q1B_ASK_CARVE_OUT retirement gate + registry flip
|
||||
4. served-surface behaviour for a QUESTION_NEEDED reaching a user
|
||||
5. the no-question/no-proposal dead-zone proof
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. The integration point (grounded)
|
||||
|
||||
`generate/contemplation/pass_manager.py::contemplate(text, *, proposal_root=...)` is
|
||||
the live contemplation entry. When every attempt is refused it calls
|
||||
`_classify_all_refused(text, attempts, findings, proposal_root)`, which is where
|
||||
`emit_proposal` fires for a `proposal_allowed` family and terminates
|
||||
`PROPOSAL_EMITTED`. **That is the exact analogous site for ASK:** when a refused
|
||||
attempt's family maps to `ask_question` (via `assess_from_attempt`), call `deliver_ask`
|
||||
and — on a renderable result — `emit_question` to the sink and terminate
|
||||
`QUESTION_NEEDED`; on the D2 fallback, fall through to today's proposal/refuse path.
|
||||
|
||||
`contemplate()` is called from `chat/runtime.py` at three sites (827, 901, 1364) — so
|
||||
the contemplation `Terminal` is already on the served path. The served question text
|
||||
travels the same way `PROPOSAL_EMITTED` already does.
|
||||
|
||||
**Two-layer split (the recommendation), mirroring VERIFIED:**
|
||||
|
||||
- **Layer A — pass-manager emission (off-serving still).** `contemplate` emits
|
||||
`QUESTION_NEEDED` + writes the `teaching/questions/` artifact, exactly as it emits
|
||||
`PROPOSAL_EMITTED` today. This changes the *contemplation* terminal but **reaches no
|
||||
user** — the teaching loop is off-serving. This layer is buildable behind no gate
|
||||
(it only adds a terminal the pass can reach), but see §3: it interacts with the
|
||||
carve-out and so should still wait for the gate, to avoid double-emission churn.
|
||||
- **Layer B — served delivery (the gated surface).** `chat/runtime.py`, gated by
|
||||
`ask_serving_enabled`, renders the `DeliveredQuestion.question.text` to the user as
|
||||
the served response when the terminal is `QUESTION_NEEDED`. This is the only layer
|
||||
that actually asks the user anything.
|
||||
|
||||
Decision to confirm: **do Layer A and Layer B land together behind one gate, or does
|
||||
Layer A land first (pass emits, nothing served) and Layer B follow?** Recommend
|
||||
**together, behind `ask_serving_enabled`** — Layer A alone creates the double-emission
|
||||
state in §3 with no compensating benefit.
|
||||
|
||||
---
|
||||
|
||||
## 3. The kill switch + the carve-out retirement gate (the coupled core)
|
||||
|
||||
### 3.1 `ask_serving_enabled`
|
||||
|
||||
Add `ask_serving_enabled: bool = False` to `core/config.py`, the sibling of the
|
||||
existing `estimation_enabled = False` kill-switch pattern. Default **off**: the
|
||||
served question path is dark until deliberately enabled, holdout-gated (§5).
|
||||
|
||||
### 3.2 Why the carve-out flip is *coupled* to the gate (not to Q1-D)
|
||||
|
||||
Q1-B introduced `Q1B_ASK_CARVE_OUT` for `missing_total_count` / `missing_weighted_total`:
|
||||
the disclosure layer classifies them `ask_question`, but the shipped `REGISTRY` keeps
|
||||
`proposal_allowed = True` so the proposal pile keeps working. The carve-out's
|
||||
*retirement condition* is written into `limitation.py`: *"Once ASK is serving, flip
|
||||
`proposal_allowed = False` on these two families, drop the carve-out set, amend the
|
||||
test."* The operative word is **serving** — not "delivery exists" (Q1-D already shipped
|
||||
that off-serving). So:
|
||||
|
||||
```text
|
||||
carve-out retires ⟺ ask_serving_enabled is the ruling
|
||||
AND a QUESTION_NEEDED is actually served for these families
|
||||
AND the §4 dead-zone proof holds
|
||||
```
|
||||
|
||||
Until then, `proposal_allowed` stays `True`. During the gate's "off" state both signals
|
||||
coexist (the off-serving question artifact + the proposal) — intentional, no loss.
|
||||
|
||||
### 3.3 The flip, as a single reviewed act
|
||||
|
||||
When `ask_serving_enabled` is turned on for these families, in **one** change:
|
||||
1. `proposal_allowed = False` for `missing_total_count` / `missing_weighted_total`.
|
||||
2. Drop them from `Q1B_ASK_CARVE_OUT` (empty the set, or remove the constant).
|
||||
3. Amend the `proposal_allowed` invariant test + the carve-out test.
|
||||
4. The §4 dead-zone proof test must already be green.
|
||||
|
||||
This is the "conscious act, not a silent re-key" the carve-out was built to force.
|
||||
|
||||
---
|
||||
|
||||
## 4. The no-question/no-proposal dead-zone proof (the wrong=0-adjacent guard)
|
||||
|
||||
**The hazard.** The flip removes the proposal signal for these families. If, for some
|
||||
input class, the family classifies `ask_question` BUT the question is unrenderable
|
||||
(D2), AND the proposal is now off, the family would terminate `NO_PROGRESS` with **no
|
||||
served question and no proposal** — a dead zone where a user-resolvable gap produces
|
||||
*nothing*. That is the ASK-side wrong=0 hazard: not a false answer, but a silent loss
|
||||
of a capability that previously at least proposed.
|
||||
|
||||
**The proof obligation (before any flip).** For every family being flipped, prove that
|
||||
**no input class lands in the dead zone** — i.e. for every reachable assessment of that
|
||||
family, the question renders (so `QUESTION_NEEDED` is served), OR the proposal is still
|
||||
on. Concretely:
|
||||
|
||||
- The `missing_*` families have pinned single slots in `_FAMILY_TO_MISSING_SLOTS` with
|
||||
mapped types (`count_int` / `measured_unit_int`) → they **always render** today, so
|
||||
the dead zone is currently empty. The proof must show this is *structural*, not
|
||||
incidental: a test that asserts `deliver_ask` returns `QUESTION_NEEDED` (never a
|
||||
fallback) for every reachable assessment of the flipped families.
|
||||
- If any future residue change could make one of these unrenderable (multi-slot,
|
||||
unmapped type), the flip must be blocked for that family until either the renderer
|
||||
covers it or the proposal stays on.
|
||||
|
||||
**The rule:** `proposal_allowed` may flip `True → False` for a family **only** if a
|
||||
test proves every reachable ask of that family renders. The dead-zone proof is a
|
||||
precondition of the flip, enforced like the D2 guard (it must *fail* if a fallback path
|
||||
is reachable for a flipped family).
|
||||
|
||||
---
|
||||
|
||||
## 5. Served-surface behaviour + holdout gating
|
||||
|
||||
### 5.1 What a served `QUESTION_NEEDED` looks like
|
||||
|
||||
When `ask_serving_enabled` and the terminal is `QUESTION_NEEDED`, the served surface
|
||||
returns the `DeliveredQuestion.question.text` (the grounded-only rendered question) as
|
||||
the response — distinct from a committed answer, an `[approximate]` disclosure, or a
|
||||
refusal. It is an **intake request**: the disposition is `ServedDisposition.ASK`
|
||||
(already mapped in `disposition.py`). The question names nothing ungrounded (Q1-C
|
||||
guarantee), so it cannot leak a fabricated entity even on the served path.
|
||||
|
||||
Open sub-decision: **the served prefix/marker.** VERIFIED gets a distinct `[verified]`
|
||||
prefix; APPROXIMATE gets `[approximate]`. ASK should get its own surface marker (a
|
||||
question is neither). Recommend a distinct, tested marker; pin the exact string at
|
||||
build, not here.
|
||||
|
||||
### 5.2 Holdout gate (no quiet widening)
|
||||
|
||||
Like VERIFIED, ASK serving must be proven on a holdout before it widens live: a
|
||||
validate-first probe over a held-out set confirming (a) served questions are
|
||||
grounded-only (no fabrication escapes on the served path), (b) no family in the flip
|
||||
set hits the dead zone, (c) the GSM8K serving seal is byte-identical (ASK is
|
||||
off-the-metric — it asks, it does not answer — but the probe proves it). Only then does
|
||||
`ask_serving_enabled` go on, one surface at a time.
|
||||
|
||||
---
|
||||
|
||||
## 6. What this is NOT
|
||||
|
||||
- **Not** a dialogue manager / multi-turn state machine — one grounded question, then
|
||||
the existing flow; the answer round-trip is Q2 (`AnswerBinding`, a separate batch).
|
||||
- **Not** a re-render — the served path emits the Q1-D `DeliveredQuestion.question.text`
|
||||
verbatim; no second prose surface.
|
||||
- **Not** a GSM8K-metric move — ASK asks, it never answers; the pinned SHAs and
|
||||
`CLAIMS.md` are untouched. The holdout probe proves it.
|
||||
- **Not** the carve-out flip *yet* — the flip is the terminal act of *this* decision,
|
||||
gated on `ask_serving_enabled` + the §4 dead-zone proof, not on Q1-D.
|
||||
|
||||
---
|
||||
|
||||
## 7. The questions for the ruling
|
||||
|
||||
1. **Gate:** add `ask_serving_enabled = False` (sibling of `estimation_enabled`)? (rec: yes)
|
||||
2. **Layering:** land pass-emission (Layer A) and served delivery (Layer B) together
|
||||
behind the one gate? (rec: yes — Layer A alone only adds double-emission churn)
|
||||
3. **Carve-out flip:** retire `Q1B_ASK_CARVE_OUT` + flip `proposal_allowed` as a single
|
||||
reviewed act, gated on the gate AND the §4 dead-zone proof? (rec: yes)
|
||||
4. **Dead-zone proof:** require a passing "every reachable ask of a flipped family
|
||||
renders" test as a precondition of any flip? (rec: yes — this is the ASK wrong=0 guard)
|
||||
5. **Served marker:** a distinct ASK surface marker (not `[verified]` / `[approximate]`)? (rec: yes)
|
||||
6. **Holdout:** validate-first probe (grounded-only on served path + no dead zone +
|
||||
GSM8K seal byte-identical) before the gate goes on? (rec: yes)
|
||||
|
||||
No served-surface code until this brief is reviewed.
|
||||
144
docs/analysis/ask-serving-integration-scoping-2026-06-09.md
Normal file
144
docs/analysis/ask-serving-integration-scoping-2026-06-09.md
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
# ASK Serving Integration Scoping — ask_serving_enabled, QUESTION_NEEDED, and Carve-Out Retirement
|
||||
|
||||
## 1. Current State
|
||||
|
||||
Currently, the ASK capability exists strictly off-serving.
|
||||
|
||||
- **Residue Capture (Q1-B):** The `core/epistemic_disclosure/limitation.py` module captures `LimitationAssessment` with typed ASK residue including `MissingSlot` and `grounded_terms`. Specifically, `missing_total_count` and `missing_weighted_total` are classified as ask-oriented inside the disclosure layer.
|
||||
- **Carve-Out Safety (Q1-B):** The transitional carve-out constant `Q1B_ASK_CARVE_OUT` is defined in `core/epistemic_disclosure/limitation.py`. The registry in `core/comprehension_attempt/failure_family.py` preserves `proposal_allowed=True` for these carve-out families. This guarantees that the proposal-pile signal remains active and uninterrupted until served ASK is fully wired and verified.
|
||||
- **Grounded Rendering (Q1-C):** The `core/epistemic_questions/render.py` module safely renders `EpistemicQuestion` structures. It enforces structural rendering, ensuring no ungrounded problem-entity names or internal `snake_case` tokens escape to the user. Multi-slot, unmapped, or unsafe cases fall back to `question_unrenderable`.
|
||||
- **Off-serving Delivery (Q1-D):** The delivery infrastructure in `core/epistemic_questions/delivery.py` defines `DeliveredQuestion` and ships the `Terminal.QUESTION_NEEDED` tenant. The resulting off-serving artifacts are written directly to the `teaching/questions/` sink. No served/user-facing surfaces are exposed.
|
||||
- **Default-Dark Gate (Helper):** The helper function `ask_serving_enabled` is implemented in `core/epistemic_questions/serving_gate.py`. It operates in a fail-closed, default-dark manner. If the config field `ask_serving_enabled` is absent, it returns `False`. An explicit, truthy configuration is required to allow served ASK.
|
||||
- **Integration gaps:** Currently, `generate/contemplation/pass_manager.py` does not emit served ASK, `chat/runtime.py` does not expose ASK, and the `Q1B_ASK_CARVE_OUT` remains active (unretired).
|
||||
|
||||
## 2. Non-Negotiable Boundary
|
||||
|
||||
Before ASK can be delivered to any user-facing surface, the following boundaries must be strictly enforced:
|
||||
|
||||
- **Strict Gate Guard:** No served question may be shown without an explicit, active `ask_serving_enabled` configuration check.
|
||||
- **Fail-Closed Default:** The `ask_serving_enabled` helper must default to `False`. A missing or `None` config attribute must evaluate to `False`.
|
||||
- **Prose Encapsulation:** The serving layer must not construct or mutate question prose directly. It may only consume pre-rendered `EpistemicQuestion` / `DeliveredQuestion` structures produced by Q1-C/Q1-D.
|
||||
- **No Contentless Delivery:** An unrenderable ASK must never be promoted to `QUESTION_NEEDED`. Contentless `QUESTION_NEEDED` outcomes are strictly forbidden.
|
||||
- **Carve-Out Preservation:** The `Q1B_ASK_CARVE_OUT` must remain active and unchanged until served ASK is fully verified. No proposal signal may be lost before a served `QUESTION_NEEDED` is verified live.
|
||||
- **Sink Distinction:** The `question_only` (teaching/questions) sink must remain logically and physically distinct from the `proposal_only` (teaching/proposals) sink.
|
||||
- **Zero Impact on Claims:** No benchmark, `CLAIMS.md`, or performance metrics may be modified by this scoping documentation.
|
||||
|
||||
## 3. Proposed Served Gate: ask_serving_enabled
|
||||
|
||||
The serving gate helper exists under `core/epistemic_questions/serving_gate.py`. This document scoping defines how future served-surface code must interact with the gate:
|
||||
|
||||
- **Helper Invariant:** Future code in the served-surface layer (e.g., `chat/runtime.py`) must verify `ask_serving_enabled(config)` before delivering any `QUESTION_NEEDED` response to a user.
|
||||
- **Off-Serving Isolation:** The gate controls served (user-visible) output only. It must not disable or interfere with off-serving artifacts written to the `teaching/questions/` directory.
|
||||
|
||||
### Serving Gate Policy
|
||||
|
||||
| Config State | ask_serving_enabled(...) | Served ASK Allowed? |
|
||||
| --- | --- | --- |
|
||||
| Missing field | `False` | No |
|
||||
| `None` / Default config | `False` | No |
|
||||
| Explicit `False` | `False` | No |
|
||||
| Explicit `True` | `True` | Only if rendering and delivery obligations pass |
|
||||
|
||||
## 4. pass_manager Integration Boundary
|
||||
|
||||
This section defines the future integration interface for `generate/contemplation/pass_manager.py`. This is scoping only; no execution is performed here.
|
||||
|
||||
1. **Evaluation:** The refusal/comprehension flow yields a `ComprehensionAttempt`.
|
||||
2. **Assessment:** A `LimitationAssessment` is derived from the attempt.
|
||||
3. **Resolution Action:** If `resolution_action == "ask_question"`:
|
||||
- The pipeline invokes `deliver_ask(assessment)`.
|
||||
- `deliver_ask` calls `render_question` exactly once.
|
||||
- If the question is renderable, the final `DeliveryOutcome` terminal becomes `QUESTION_NEEDED`.
|
||||
- If the question is unrenderable, the outcome falls back to the standing disposition (e.g., proposal or refusal).
|
||||
4. **Gate Enforced downstream:** `generate/contemplation/pass_manager.py` may produce/record ASK delivery outcomes later, but user-visible exposure remains gated downstream by `ask_serving_enabled` (e.g., in `chat/runtime.py`).
|
||||
|
||||
- **Constraints:**
|
||||
- `pass_manager` must not contain prose templates or formatting rules.
|
||||
- `pass_manager` must not construct `DeliveredQuestion` manually (it must delegate to `deliver_ask`).
|
||||
- `pass_manager` must never bypass `deliver_ask`.
|
||||
- `pass_manager` must never emit a contentless `QUESTION_NEEDED`.
|
||||
|
||||
## 5. Served Behavior Matrix
|
||||
|
||||
| Assessment / Rendering | Gate Disabled | Gate Enabled |
|
||||
| --- | --- | --- |
|
||||
| Renderable `ask_question` | No served ASK; existing refusal/proposal behavior preserved | Served `QUESTION_NEEDED` is allowed |
|
||||
| Unrenderable `ask_question` | Standing fallback; no `QUESTION_NEEDED` | Standing fallback; no `QUESTION_NEEDED` |
|
||||
| Non-ASK limitation | Unaffected | Unaffected |
|
||||
| `missing_total_count` / `missing_weighted_total` (carve-out) | Proposal signal preserved | Served ASK only after carve-out retirement proof |
|
||||
| Multi-slot ASK | Unrenderable fallback | Unrenderable fallback |
|
||||
|
||||
*Note: Gate enabled is a necessary but not sufficient condition for serving. The renderable checks and delivery invariants must also pass.*
|
||||
|
||||
## 6. Q1B_ASK_CARVE_OUT Retirement Conditions
|
||||
|
||||
The transitional carve-out constant `Q1B_ASK_CARVE_OUT` can only be retired and removed from `core/epistemic_disclosure/limitation.py` when the following milestones are met and verified:
|
||||
|
||||
1. The `ask_serving_enabled` helper is tested and confirmed default-dark.
|
||||
2. The `pass_manager.py` ASK integration is fully implemented behind the gate.
|
||||
3. Served `QUESTION_NEEDED` terminal behavior is successfully tested using renderable `EpistemicQuestion` instances.
|
||||
4. Unrenderable ASK paths are verified to fall back correctly, never emitting `QUESTION_NEEDED`.
|
||||
5. Carve-out keys `missing_total_count` and `missing_weighted_total` are proven to yield safe renderable questions (or safe standing fallbacks).
|
||||
6. A dedicated no-question/no-proposal dead-zone validation test is introduced and passes.
|
||||
7. The registry in `core/comprehension_attempt/failure_family.py` flips `proposal_allowed=False` for the carve-out families without dropping signal.
|
||||
8. The `teaching/questions/` (`question_only`) sink remains physically distinct from the `teaching/proposals/` (`proposal_only`) sink.
|
||||
9. Smoke, contemplation, proposal, and disclosure test suites remain fully green.
|
||||
|
||||
*Until all 9 conditions are met, the registry's `proposal_allowed=True` setting for these families must remain.*
|
||||
|
||||
## 7. No-Dead-Zone Proof Obligation
|
||||
|
||||
### The Dead-Zone Hazard
|
||||
If `proposal_allowed` is flipped to `False` on a failure family before ASK serving is enabled/renderable, an input falling into that family would yield no proposal signal AND no served question. This results in a silent loss of capability (a dead zone), violating the `wrong=0` refusal-first discipline.
|
||||
|
||||
### Required Proof Before Retirement
|
||||
Before the carve-out is retired, tests must prove that for each carve-out family:
|
||||
- If `proposal_allowed` is removed, either a served `QUESTION_NEEDED` is emitted safely, or the standing fallback preserves an admissible signal.
|
||||
- The test suite must assert failure if an evaluation result yields neither a proposal nor a valid served question.
|
||||
|
||||
### Suggested Test Names
|
||||
- `test_ask_serving_disabled_preserves_existing_proposal_signal`
|
||||
- `test_carveout_retirement_has_no_question_no_proposal_dead_zone`
|
||||
- `test_unrenderable_ask_never_emits_question_needed`
|
||||
- `test_pass_manager_uses_deliver_ask_not_direct_rendering`
|
||||
- `test_question_only_not_proposal_only`
|
||||
|
||||
## 8. Required Future Tests Before Wiring
|
||||
|
||||
The implementation PR must include tests asserting the following behaviors:
|
||||
|
||||
- **Gate Default:** `ask_serving_enabled` defaults to `False`.
|
||||
- **Config Completeness:** A missing config field evaluates to `False`.
|
||||
- **Opt-in Activation:** An explicit `True` configuration is required to allow served ASK.
|
||||
- **Gate Override:** If the gate is disabled, no served ASK is emitted even if `deliver_ask` produces a `QUESTION_NEEDED` outcome.
|
||||
- **Rendering Obligation:** If the gate is enabled, the output still requires a renderable `EpistemicQuestion`.
|
||||
- **Unrenderable Fallback:** Unrenderable ASK instances are never served.
|
||||
- **Prose Isolation:** `pass_manager` does not construct prose templates or strings.
|
||||
- **Delegated Delivery:** `pass_manager` delegates entirely to `deliver_ask` and does not call `render_question` directly.
|
||||
- **No Empty Terminals:** No contentless `QUESTION_NEEDED` outcomes are permitted.
|
||||
- **Sink Safety:** The `question_only` artifact is written to the correct, distinct location.
|
||||
- **Carve-out Lock:** `Q1B_ASK_CARVE_OUT` is preserved until both the gate and the dead-zone proofs are active.
|
||||
- **Registry Guard:** Flipped registries are blocked until all retirement conditions are met.
|
||||
|
||||
## 9. Non-Claims
|
||||
|
||||
This scoping document establishes architectural boundaries only:
|
||||
|
||||
- **No Implementation:** It does not implement served ASK.
|
||||
- **No Wiring:** It does not wire `generate/contemplation/pass_manager.py` or `chat/runtime.py`.
|
||||
- **No Retirement:** It does not retire `Q1B_ASK_CARVE_OUT` or flip any registry `proposal_allowed` flags.
|
||||
- **No Metric Changes:** It does not alter GSM8K benchmark claims or refusal behaviors.
|
||||
- **No General Intelligence:** It does not make ASK generally intelligent; it simply bounds the served-surface gate.
|
||||
|
||||
## 10. Recommended Next Slice
|
||||
|
||||
To safely approach the implementation, the next sequential slices are recommended:
|
||||
|
||||
### Slice 1: Configuration Definition (No Wiring)
|
||||
- Add a concrete configuration field `RuntimeConfig.ask_serving_enabled: bool = False` (if not already present).
|
||||
- Ensure the helper `ask_serving_enabled(...)` references this configuration.
|
||||
- Write configuration tests verifying the default and override values.
|
||||
- Do not wire `pass_manager` or modify runtime loops.
|
||||
|
||||
### Slice 2: pass_manager Integration
|
||||
- Wire `pass_manager` to call `deliver_ask` without constructing prose; any user-visible ASK exposure remains behind `ask_serving_enabled`; preserve `Q1B_ASK_CARVE_OUT`.
|
||||
274
docs/analysis/q1-d-ask-bus-delivery-scoping-2026-06-08.md
Normal file
274
docs/analysis/q1-d-ask-bus-delivery-scoping-2026-06-08.md
Normal file
|
|
@ -0,0 +1,274 @@
|
|||
# Q1-D — ASK bus delivery (`QUESTION_NEEDED` tenant) — decision record
|
||||
|
||||
**Date:** 2026-06-08 · **Status:** ACCEPTED + IMPLEMENTED (PR #668, merge commit
|
||||
`07149c20`) · **Branch:** `docs/q1-d-ask-bus-delivery-scoping`
|
||||
|
||||
> **Decision record, not an open brief.** All five decisions below (D1–D5) were
|
||||
> ruled and are implemented in PR #668. This doc is retained as the durable record of
|
||||
> *what was decided and why*; the implementation lives in
|
||||
> `core/epistemic_questions/delivery.py`, `generate/contemplation/findings.py`
|
||||
> (`Terminal.QUESTION_NEEDED`), `core/epistemic_disclosure/limitation.py` (the now-total
|
||||
> `terminal_for_action`), and `tests/test_question_delivery.py`. Where this doc and the
|
||||
> code disagree, the code is authoritative. Two *served-surface* decisions remain open
|
||||
> and are deferred to dedicated scoping docs (see §7): `ask_serving_enabled` and the
|
||||
> registry/carve-out flip.
|
||||
|
||||
**What this records.** The decisions for the **fourth and last Q1 rung** — *delivery*.
|
||||
Q1-A (limitation pass), Q1-B (typed residue + `missing_*` reclassification,
|
||||
`Q1B_ASK_CARVE_OUT`), and Q1-C (the grounded-only renderer, strictly single-slot) all
|
||||
shipped **off-serving**; Q1-D (also off-serving) takes the rendered question and routes
|
||||
it onto Doc 1's Epistemic Disclosure Bus as the **second tenant** (`QUESTION_NEEDED`),
|
||||
behind the first (`VERIFIED`). Each decision is pinned against the shipped code it
|
||||
landed on.
|
||||
|
||||
> Design of record: the session doc
|
||||
> `docs/sessions/2026-06-08-epistemic-question-articulation-first-skill-of-contemplation.md`
|
||||
> (§4 the `QUESTION_NEEDED` terminal, §5/§1.5.8 bus integration).
|
||||
> Frontier companions: [[q1-epistemic-question-articulation-v1-scoping-2026-06-08]]
|
||||
> §5/§7 (Q1-D = bus delivery), [[stage2-epistemic-disclosure-bus-verified-v1-scoping-2026-06-08]]
|
||||
> (the bus VERIFIED is first tenant of).
|
||||
|
||||
**The one-line shape.**
|
||||
|
||||
```text
|
||||
LimitationAssessment(ask_question, missing_slots)
|
||||
→ choose_served_disposition(...) = ServedDisposition.ASK [P0-3, shipped]
|
||||
→ render_question(assessment) = EpistemicQuestion [Q1-C, shipped]
|
||||
→ Q1-D: package → Terminal.QUESTION_NEEDED + DeliveredQuestion [THIS RUNG]
|
||||
→ (off-serving sink: written like a proposal, never auto-served)
|
||||
```
|
||||
|
||||
**The hard constraint from the steer.** Q1-D **does not render**. It consumes only
|
||||
the `EpistemicQuestion` the Q1-C renderer produced. This preserves the rung
|
||||
separation that makes each layer independently auditable:
|
||||
|
||||
```text
|
||||
Q1-B: typed residue (what is missing, as typed slots)
|
||||
Q1-C: renderability (can it be asked without fabricating? → EpistemicQuestion)
|
||||
Q1-D: delivery (route the rendered question onto the bus)
|
||||
```
|
||||
|
||||
If Q1-D re-derived or re-rendered, the wrong=0 grounded-rendering guard (Q1-C
|
||||
`_names_only_grounded`) would be bypassed by a second, ungoverned surface — the
|
||||
exact parallel-path anti-pattern the consolidation discipline forbids.
|
||||
|
||||
---
|
||||
|
||||
## 1. State of the substrate Q1-D lands on (verified on branch)
|
||||
|
||||
| Piece | Status | Q1-D dependency |
|
||||
|---|---|---|
|
||||
| `ServedDisposition` + `choose_served_disposition` (`core/epistemic_disclosure/disposition.py`) | **shipped (P0-3)** | `ask_question → ASK` already mapped; **zero runtime consumers** |
|
||||
| `EpistemicQuestion` + `render_question` (`core/epistemic_questions/render.py`) | **shipped (Q1-C)** | the *only* input Q1-D consumes; strictly single-slot |
|
||||
| `Terminal` enum (`generate/contemplation/findings.py`) | shipped, **7 members** | `QUESTION_NEEDED` **absent** — Q1-D adds it (sibling to `PROPOSAL_EMITTED`) |
|
||||
| `LimitationAssessment` / `MissingSlot` (`core/epistemic_disclosure/limitation.py`) | shipped (Q1-A/B) | source of the assessment; `Q1B_ASK_CARVE_OUT` still live |
|
||||
| failure-family `REGISTRY` (`core/comprehension_attempt/failure_family.py`) | shipped | `missing_*` families still `proposal_allowed=True` (carve-out) |
|
||||
| contemplation proposal sink (`teaching/proposals/`) | shipped | the **template** for the off-serving question sink |
|
||||
|
||||
Two facts set the altitude:
|
||||
|
||||
1. **Nothing consumes `ServedDisposition.ASK` or `EpistemicQuestion` today.** The bus
|
||||
decision table is a pure mapping with no caller. Q1-D is therefore the **first**
|
||||
consumer of the ASK disposition — which is exactly why it must be scoped, not
|
||||
improvised.
|
||||
2. **The bus is itself still off-serving.** The VERIFIED tenant (P1) was built as a
|
||||
proof-object producer in `evals/`; the served-surface wiring was deferred to a
|
||||
named decision (`verified_serving_enabled`). Q1-D inherits the same two-layer
|
||||
split: an off-serving delivery layer buildable now, and a served-surface wiring
|
||||
layer that waits on a ruling.
|
||||
|
||||
---
|
||||
|
||||
## 2. What Q1-D shipped (off-serving) — ACCEPTED
|
||||
|
||||
**Decision: build the off-serving delivery layer; defer served-surface wiring to a
|
||||
named gate.** Implemented in #668. This mirrors the VERIFIED lane exactly (P1-B
|
||||
produced proof objects off-serving; serving wiring is a separate, gated decision).
|
||||
One scope refinement landed with the code: the delivery layer ships **standalone** —
|
||||
`deliver_ask`/`emit_question` are tested but **not yet called from `pass_manager`**
|
||||
(mirrors P1-B shipping the producer without wiring `verify.py`); pass-manager wiring is
|
||||
a deliberate follow-up, folded into the §7 ASK serving-integration scope.
|
||||
|
||||
### 2.1 `Terminal.QUESTION_NEEDED` — the off-serving sink
|
||||
|
||||
Add `QUESTION_NEEDED` as an eighth contemplation terminal, a **sibling of
|
||||
`PROPOSAL_EMITTED`** (the session doc §4 already specifies this: "not a subtype of
|
||||
`PROPOSAL_EMITTED`; sibling terminals"). The contemplation pass, when an
|
||||
`ask_question` assessment renders successfully, terminates `QUESTION_NEEDED` carrying
|
||||
the `DeliveredQuestion` artifact — **just as `PROPOSAL_EMITTED` carries the proposal,
|
||||
and just as proposal-only never auto-installs**. The artifact is written to a sink
|
||||
(`teaching/proposals/`-analogue, e.g. `teaching/questions/` or a
|
||||
`questions/` JSONL), reviewable, never delivered to a user by this rung.
|
||||
|
||||
### 2.2 `DeliveredQuestion` — the delivery artifact (wraps, never re-renders)
|
||||
|
||||
A new frozen dataclass that **wraps** the Q1-C `EpistemicQuestion` and adds the
|
||||
delivery-level fields the assessment already carries — it does **not** promote the
|
||||
renderer to the session-doc's richer aspirational model:
|
||||
|
||||
```text
|
||||
DeliveredQuestion (frozen):
|
||||
question: EpistemicQuestion # verbatim from render_question — never rebuilt
|
||||
owner_organ: str # from LimitationAssessment.owner_organ
|
||||
blocking_reason: str # from LimitationAssessment.blocking_reason
|
||||
answer_binding: AnswerBinding | None # RESERVED (Q2); None in Q1-D
|
||||
source_attempt_id: str | None # provenance, if the attempt supplied one
|
||||
```
|
||||
|
||||
`AnswerBinding` is **reserved, not implemented** — the seat exists so the Q2
|
||||
round-trip is wireable without reshaping, but Q1-D binds no answers (session doc §4:
|
||||
"answer-binding must re-enter the gate", a later batch).
|
||||
|
||||
### 2.3 Off-serving guarantee
|
||||
|
||||
The delivery module imports **no** `generate.derivation` / `core.reliability_gate`
|
||||
(AST-checked, same test shape as Q1-C `test_renderer_is_off_serving`). It cannot move
|
||||
the sealed GSM8K metric or any pinned SHA. **No served surface, no `chat/runtime.py`
|
||||
wiring, no `CLAIMS.md` change.**
|
||||
|
||||
---
|
||||
|
||||
## 3. The decisions — ALL ACCEPTED + IMPLEMENTED (#668)
|
||||
|
||||
These were the load-bearing choices Q1-D forced. Each was ruled as its recommendation
|
||||
and is implemented in #668; the rulings are recorded inline (**Ruling:**) under each.
|
||||
|
||||
### D1 — Off-serving delivery now, or wait for the bus to be served? — **ACCEPTED: now**
|
||||
|
||||
The Q1 scoping doc (§7.4) said *"Q1-D requires the bus to be live (Stage 2 S2-A..C)."*
|
||||
But Stage 2 shipped its first tenant **off-serving** (P1 is a proof-object producer;
|
||||
serving wiring deferred). So the literal precondition ("bus live") never materialised
|
||||
as *served* — it materialised as *off-serving infrastructure*.
|
||||
|
||||
- **Recommend:** build Q1-D's off-serving delivery layer now (matches the VERIFIED
|
||||
precedent), and defer served delivery to a named gate **`ask_serving_enabled`** —
|
||||
the ASK analogue of `verified_serving_enabled`. Q1-D off-serving produces and tests
|
||||
`QUESTION_NEEDED` + `DeliveredQuestion`; **nothing reaches a user.**
|
||||
- **Alternative:** hold Q1-D entirely until the bus has a served surface. Costs the
|
||||
off-serving proof object that the VERIFIED lane found valuable; gains nothing the
|
||||
gate doesn't already protect.
|
||||
|
||||
### D2 — The unrenderable fallback (the wrong=0-adjacent decision) — **ACCEPTED**
|
||||
|
||||
When Q1-C returns `unrenderable` (`renderability_gap`, `multi_slot_not_supported`,
|
||||
`fabrication_guard`, `no_missing_slot`), Q1-D must **not** terminate `QUESTION_NEEDED`
|
||||
with no question — that would be a contentless ASK, the delivery-side equivalent of a
|
||||
fabricated answer.
|
||||
|
||||
- **Recommend:** an unrenderable ASK **falls back to the family's standing
|
||||
disposition** — for the `missing_*` carve-out families that is `PROPOSAL_EMITTED`
|
||||
(they are `proposal_allowed=True` today); for an ambiguous-structure family with no
|
||||
renderable slot it is `NO_PROGRESS`. The rule: *an `ask` classification that cannot
|
||||
be rendered never produces a dead end and never produces an empty question.* This is
|
||||
the delivery-layer guard, and it gets a test that **fails** if an unrenderable
|
||||
assessment ever reaches `QUESTION_NEEDED`.
|
||||
- This is why the carve-out (D3) must not flip yet — see below.
|
||||
|
||||
### D3 — Does Q1-D resolve the `Q1B_ASK_CARVE_OUT`? — **ACCEPTED: no, carve-out stays**
|
||||
|
||||
Q1-B deliberately **kept** `REGISTRY` `proposal_allowed=True` for `missing_total_count`
|
||||
/ `missing_weighted_total` and added the disclosure-layer `Q1B_ASK_CARVE_OUT`, with the
|
||||
stated condition: *no silent re-key, no proposal-signal dead zone "until ASK delivery
|
||||
lands."* Q1-D **is** ASK delivery landing — so does the registry flip now?
|
||||
|
||||
- **Recommend: no — the carve-out persists until `ask_serving_enabled`.** Off-serving
|
||||
`QUESTION_NEEDED` does **not** yet replace the proposal channel a real user would
|
||||
see. If the registry flipped `proposal_allowed → ask` while serving still proposes,
|
||||
the carve-out's own dead-zone hazard reappears (a family classified `ask` with no
|
||||
served ask path). The flip is therefore gated on **serving** delivery, not
|
||||
off-serving delivery. Q1-D records this explicitly so the carve-out's resolution
|
||||
condition is unambiguous: *the carve-out closes when `ask_serving_enabled` is the
|
||||
ruling, not when off-serving `QUESTION_NEEDED` ships.*
|
||||
- The D2 fallback (unrenderable ASK → standing disposition) is what makes this safe:
|
||||
while the carve-out stands, the registry's `proposal_allowed=True` remains the
|
||||
truthful fallback target.
|
||||
|
||||
### D4 — Where does the off-serving sink write? — **ACCEPTED: `teaching/questions/`**
|
||||
|
||||
`PROPOSAL_EMITTED` writes to `teaching/proposals/`. The question sink is a sibling.
|
||||
|
||||
- **Recommend:** `teaching/questions/` (or a `questions.jsonl` beside `proposals.jsonl`)
|
||||
— same proposal-only, review-gated, HITL-visible discipline; never auto-served. The
|
||||
artifact is the `DeliveredQuestion` serialized deterministically (frozen → hashable →
|
||||
replayable). Confirm the path/name; it is a one-line decision, not architecture.
|
||||
|
||||
### D5 — Single-slot carries through (no fan-out in Q1-D) — **ACCEPTED**
|
||||
|
||||
Q1-C is **strictly single-slot** (the `multi_slot_not_supported` refusal shipped in
|
||||
this same review). Q1-D therefore delivers **at most one** `DeliveredQuestion` per
|
||||
assessment; a multi-slot assessment renders unrenderable and takes the D2 fallback.
|
||||
One-question-per-slot fan-out is **not** Q1-D — it is a later rung that must first
|
||||
solve the "which slot first / minimal-sufficient" ranking (session doc §1.4, `rank.py`,
|
||||
Q2+). Stated here so delivery does not silently grow a fan-out.
|
||||
|
||||
---
|
||||
|
||||
## 4. What Q1-D is NOT (non-claims)
|
||||
|
||||
- **Not** a served surface. No question reaches a user; `chat/runtime.py` is untouched;
|
||||
the served-surface wiring is the deferred `ask_serving_enabled` gate (D1).
|
||||
- **Not** a re-renderer. Q1-D consumes the Q1-C `EpistemicQuestion` verbatim; it never
|
||||
builds prose, so the grounded-rendering wrong=0 guard is never bypassed.
|
||||
- **Not** the registry flip. `missing_*` stays `proposal_allowed=True`; the
|
||||
`Q1B_ASK_CARVE_OUT` persists until `ask_serving_enabled` (D3).
|
||||
- **Not** Q2 answer-binding. `AnswerBinding` is a reserved seat, bound to nothing; the
|
||||
answer round-trip re-enters the limitation gate in a later batch (session doc §4).
|
||||
- **Not** a fan-out. One slot, one question — multi-slot takes the unrenderable
|
||||
fallback (D5).
|
||||
- **Not** a metric/SHA/`CLAIMS.md` move — off-serving, AST-verified.
|
||||
|
||||
---
|
||||
|
||||
## 5. Verification lanes (as shipped in #668)
|
||||
|
||||
- `tests/test_question_delivery.py` (14 tests) — `ask_question` + renderable assessment
|
||||
→ `Terminal.QUESTION_NEEDED` + a `DeliveredQuestion` wrapping the *exact* Q1-C
|
||||
`EpistemicQuestion`; both **D2 fallback branches** (proposing → `PROPOSAL_EMITTED`,
|
||||
non-proposing → `NO_PROGRESS`); the structural guards (a `DeliveredQuestion` cannot
|
||||
wrap an unrenderable question / be served / carry an `AnswerBinding`); the idempotent
|
||||
content-addressed sink + no-artifact-on-fallback; the off-serving AST test.
|
||||
- `tests/test_limitation_assessment.py` — the two consolidation tests updated for the
|
||||
now-total `terminal_for_action` (`ask_question → QUESTION_NEEDED`).
|
||||
- Schema-obligation discipline (CLAUDE.md): the D2 guard is enforced **twice** — in
|
||||
`deliver_ask` and structurally in `DeliveredQuestion.__post_init__` — so an
|
||||
unrenderable assessment can never terminate `QUESTION_NEEDED`; the off-serving test
|
||||
fails if a forbidden import is added. Results: smoke 90/0, affected suites 128/0.
|
||||
|
||||
---
|
||||
|
||||
## 6. Build order (Q1-D, off-serving, one PR — as shipped in #668)
|
||||
|
||||
1. `Terminal.QUESTION_NEEDED` added (sibling of `PROPOSAL_EMITTED`).
|
||||
2. `DeliveredQuestion` dataclass (+ reserved `AnswerBinding`); wraps `EpistemicQuestion`.
|
||||
3. The delivery function `deliver_ask`: `ask_question` assessment → render (consume
|
||||
Q1-C) → on renderable, `QUESTION_NEEDED` + artifact; on unrenderable, the **D2
|
||||
fallback**. Plus the `emit_question` sink writer.
|
||||
4. Off-serving AST test + the D2 guard tests + the sink-write tests.
|
||||
|
||||
The served-surface wiring (`ask_serving_enabled`), the `pass_manager` call site, and the
|
||||
registry flip (D3) are a **separate, later, ruling-gated** step — deferred to the §7
|
||||
ASK serving-integration scoping doc, NOT this PR. Q2 answer-binding is a separate batch.
|
||||
|
||||
---
|
||||
|
||||
## 7. Outcome — RESOLVED, and what remains open
|
||||
|
||||
**Resolved (#668):** Q1-D's **off-serving** delivery layer was built —
|
||||
`Terminal.QUESTION_NEEDED` + `DeliveredQuestion`, consuming Q1-C, with the D2
|
||||
unrenderable-fallback guard and the D3 carve-out left standing — matching the VERIFIED
|
||||
precedent, keeping wrong=0 honest, moving no metric. The off-serving ASK lane (residue →
|
||||
renderability → delivery) is now complete.
|
||||
|
||||
**Open — deferred to dedicated scoping docs (no served-surface code until both are
|
||||
reviewed):**
|
||||
|
||||
1. **ASK serving-integration** (`ask_serving_enabled`): the `pass_manager` integration
|
||||
point (where `deliver_ask` is actually called), the served-surface behaviour for a
|
||||
`QUESTION_NEEDED` reaching a user, the `Q1B_ASK_CARVE_OUT` retirement gate + registry
|
||||
flip (`proposal_allowed → ask`), and the no-question/no-proposal **dead-zone proof**
|
||||
(a family must never lose both signals across the flip).
|
||||
2. **VERIFIED serving-wiring** (`verified_serving_enabled`): the gold-free independent
|
||||
reader requirement, holdout gates, proof-producer eligibility, and the explicit ban
|
||||
on eval-gold-backed serving.
|
||||
|
||||
Both are the VERIFIED/ASK halves of the same "this is where off-serving stops" line.
|
||||
|
|
@ -0,0 +1,317 @@
|
|||
# Stage 2 Disclosure Bus Implementation Map — session pivot to current slices
|
||||
|
||||
**Date:** 2026-06-09
|
||||
**Status:** implementation map / controlling checklist — docs only
|
||||
|
||||
This document reconciles the June 8 session-design pivot with the implementation slices that have now landed. It is not a new architecture. It is the map that prevents the work from drifting into isolated feature patches.
|
||||
|
||||
## 0. Why this map exists
|
||||
|
||||
The recent implementation work touched ASK and VERIFIED serving foundations in small slices. That was correct, but the controlling plan is larger than any one PR:
|
||||
|
||||
- `docs/sessions/2026-06-08-practice-attempts-and-servability-blade.md`
|
||||
- `docs/sessions/2026-06-08-epistemic-question-articulation-first-skill-of-contemplation.md`
|
||||
- `docs/analysis/stage2-epistemic-disclosure-bus-verified-v1-scoping-2026-06-08.md`
|
||||
- `docs/analysis/q1-epistemic-question-articulation-v1-scoping-2026-06-08.md`
|
||||
- `docs/analysis/ask-serving-integration-scoping-2026-06-09.md`
|
||||
- `docs/analysis/verified-serving-wiring-scoping-2026-06-09.md`
|
||||
|
||||
The governing pivot is:
|
||||
|
||||
```text
|
||||
wrong=0 is not a binary answer/refuse command.
|
||||
wrong=0 is no false presentation of epistemic status.
|
||||
practice/contemplation may explore typed, isolated candidates.
|
||||
serving must disclose only what is truthfully labelable.
|
||||
```
|
||||
|
||||
Therefore the served surface should not grow by one-off feature branches. It should grow by activating tenants on the already-scoped Epistemic Disclosure Bus.
|
||||
|
||||
## 1. Controlling doctrine from the session docs
|
||||
|
||||
### 1.1 Practice versus serving
|
||||
|
||||
The practice/servability session separates two lanes:
|
||||
|
||||
| Lane | Purpose | License | Primary danger |
|
||||
| --- | --- | --- | --- |
|
||||
| practice / contemplation | explore, attempt, eliminate, learn | typed and isolated candidates may exist | getting stuck / never learning |
|
||||
| serving | emit to a user/downstream surface | no false presentation of epistemic status | misrepresenting uncertainty as truth |
|
||||
|
||||
The practical consequence is that internal attempts, ASK artifacts, proposals, and verification proofs may exist off-serving without being user-visible. Serving requires an explicit governance decision.
|
||||
|
||||
### 1.2 The servability blade is not a new parallel system
|
||||
|
||||
The session document explicitly amends the initial sketch: the servability blade must reuse the already-shipped ADR-0206 response-governance seam (`ReachPolicy`, `govern_response`, `shape_surface`) rather than invent a new parallel object.
|
||||
|
||||
For Stage 2 work, every served-surface slice must therefore ask:
|
||||
|
||||
```text
|
||||
Does this activate the existing governance seam, or is it bypassing it?
|
||||
```
|
||||
|
||||
A bypass is architectural drift.
|
||||
|
||||
### 1.3 ASK is typed intake, not chat clarification
|
||||
|
||||
The question-articulation session defines a question as a typed request for missing state. The first contemplative move is not “ask a question”; it is:
|
||||
|
||||
```text
|
||||
What is preventing resolution — and what KIND of limitation is it?
|
||||
```
|
||||
|
||||
Only limitations of kind `missing_information` or `ambiguous_structure` may become ASK. Capability gaps propose. Hard boundaries refuse. Contradictions report. Input-shape cases step aside.
|
||||
|
||||
### 1.4 QUESTION_NEEDED and PROPOSAL_EMITTED are siblings
|
||||
|
||||
The session doc draws a hard line:
|
||||
|
||||
| Terminal | Meaning |
|
||||
| --- | --- |
|
||||
| `QUESTION_NEEDED` | the input is under-specified but the problem is potentially knowable after one missing datum is supplied |
|
||||
| `PROPOSAL_EMITTED` | the input is sufficiently specified but CORE lacks the transform/capability |
|
||||
|
||||
This distinction must be preserved in artifacts, paths, tests, and served surfaces.
|
||||
|
||||
## 2. Stage 2 bus frame
|
||||
|
||||
The Stage 2 scoping doc reframes the frontier:
|
||||
|
||||
```text
|
||||
What may CORE disclose through the served surface — and under what governed disposition?
|
||||
```
|
||||
|
||||
The bus is the consolidating view:
|
||||
|
||||
```text
|
||||
EpistemicState + LimitationAssessment + proof/license evidence
|
||||
-> ServedDisposition / disclosure decision
|
||||
-> shaped surface through the governance seam
|
||||
```
|
||||
|
||||
VERIFIED is not the whole system. It is one tenant. ASK is another. Scope-boundary explanation, contradiction reporting, proposal-only, partial progress, multiple candidates, and provisional working answers are later tenants or modes on the same governance surface.
|
||||
|
||||
## 3. Current implementation state
|
||||
|
||||
### 3.1 ASK tenant — landed foundation
|
||||
|
||||
The following ASK foundation is now present:
|
||||
|
||||
```text
|
||||
Q1-B typed ASK residue / MissingSlot / LimitationAssessment ask_question
|
||||
Q1-C grounded-only renderer
|
||||
Q1-D off-serving DeliveredQuestion / deliver_ask / teaching/questions sink
|
||||
ask_serving_enabled helper, default-dark
|
||||
RuntimeConfig.ask_serving_enabled = False
|
||||
pass_manager off-serving ASK integration behind exercise_ask
|
||||
ContemplationResult.question_path separated from proposal_path
|
||||
```
|
||||
|
||||
Important preserved boundaries:
|
||||
|
||||
```text
|
||||
no chat/runtime.py serving path
|
||||
no served ASK surface
|
||||
no Q1B_ASK_CARVE_OUT retirement
|
||||
no proposal_allowed registry flip
|
||||
no CLAIMS or benchmark movement
|
||||
boundary-first remains before ASK
|
||||
question_only sink remains distinct from proposal_only sink
|
||||
```
|
||||
|
||||
### 3.2 ASK tenant — not yet done
|
||||
|
||||
The following are still open:
|
||||
|
||||
```text
|
||||
served ASK / QUESTION_NEEDED through the governance bus
|
||||
Q2 AnswerBinding and re-run through the owner organ
|
||||
no-question/no-proposal dead-zone proof
|
||||
Q1B_ASK_CARVE_OUT retirement proof
|
||||
registry flip for missing_total_count / missing_weighted_total
|
||||
broader ASK families beyond the current typed-residue subset
|
||||
```
|
||||
|
||||
### 3.3 VERIFIED tenant — landed foundation
|
||||
|
||||
The following VERIFIED foundation is now present:
|
||||
|
||||
```text
|
||||
P1-A VERIFIED contract
|
||||
P1-B off-serving gold-setup-backed R2 producer
|
||||
P1-C bound_slots_digest proof hardening
|
||||
verified_serving_enabled helper, default-dark
|
||||
RuntimeConfig.verified_serving_enabled = False
|
||||
verified serving scoping / gold-free independence doc
|
||||
```
|
||||
|
||||
Important preserved boundaries:
|
||||
|
||||
```text
|
||||
no served VERIFIED surface
|
||||
no verify.py consumption
|
||||
no eval-gold-backed proof in serving
|
||||
no CLAIMS or benchmark movement
|
||||
no gold-free independent reader yet
|
||||
```
|
||||
|
||||
### 3.4 VERIFIED tenant — not yet done
|
||||
|
||||
The following are still open:
|
||||
|
||||
```text
|
||||
gold-free independent reader/proof source
|
||||
poison fixture harness
|
||||
holdout-gated verification harness
|
||||
deterministic replay digest proof for served verification traces
|
||||
verify.py consumption of contract verdict
|
||||
served [verified] surface behind verified_serving_enabled
|
||||
```
|
||||
|
||||
## 4. Recent PRs in plan terms
|
||||
|
||||
| PR | Plan role | Status |
|
||||
| --- | --- | --- |
|
||||
| #664 | Q1-B typed ASK residue + carve-out | merged |
|
||||
| #666 | Q1-C grounded-only renderer | merged |
|
||||
| #667 | Q1-D decision record | merged |
|
||||
| #668 | Q1-D off-serving delivery | merged |
|
||||
| #670 | ASK default-dark gate helper | merged |
|
||||
| #671 | ASK serving / carve-out retirement scoping | merged |
|
||||
| #672 | VERIFIED serving / gold-free independence scoping | merged |
|
||||
| #673 | VERIFIED default-dark gate helper | merged |
|
||||
| #674 | explicit ASK + VERIFIED config fields | merged |
|
||||
| #675 | off-serving ASK pass_manager integration | merged |
|
||||
| #676 | split question_path from proposal_path | merged |
|
||||
|
||||
This sequence was mostly faithful to the plan, but the controlling label should be:
|
||||
|
||||
```text
|
||||
ASK tenant implementation on the Epistemic Disclosure Bus foundation
|
||||
```
|
||||
|
||||
not merely:
|
||||
|
||||
```text
|
||||
question feature / pass_manager feature
|
||||
```
|
||||
|
||||
## 5. Correct next-slice ordering
|
||||
|
||||
### 5.1 ASK next code slice — bus activation, not runtime bypass
|
||||
|
||||
The next ASK code slice should not be described as “wire chat/runtime.” It should be:
|
||||
|
||||
```text
|
||||
Activate ASK/clarify as a served tenant through the disclosure/governance bus.
|
||||
```
|
||||
|
||||
Minimum requirements:
|
||||
|
||||
- use `ask_serving_enabled(config)` as a necessary gate;
|
||||
- consume `ContemplationResult.question_path` / delivered question artifact;
|
||||
- do not construct question prose in serving;
|
||||
- do not serve unrenderable ASK;
|
||||
- preserve `Q1B_ASK_CARVE_OUT`;
|
||||
- preserve proposal signal when gate is disabled;
|
||||
- do not flip `proposal_allowed`;
|
||||
- do not bypass `govern_response` / `shape_surface` if that seam is applicable;
|
||||
- if the current governance seam cannot carry ASK yet, stop and scope the exact missing adapter first.
|
||||
|
||||
Expected test names or equivalents:
|
||||
|
||||
```text
|
||||
test_ask_serving_disabled_preserves_existing_proposal_signal
|
||||
test_ask_serving_enabled_surfaces_question_needed_from_artifact
|
||||
test_unrenderable_ask_never_serves_question_needed
|
||||
test_question_only_not_proposal_only
|
||||
test_served_ask_does_not_construct_question_prose
|
||||
test_served_ask_uses_governance_bus_not_parallel_runtime_path
|
||||
```
|
||||
|
||||
### 5.2 ASK following slice — no-dead-zone proof
|
||||
|
||||
After served ASK exists behind the gate, prove that the carve-out families cannot fall into a no-question/no-proposal dead zone.
|
||||
|
||||
Required proof shape:
|
||||
|
||||
```text
|
||||
for missing_total_count / missing_weighted_total:
|
||||
gate disabled -> proposal signal preserved
|
||||
gate enabled + renderable -> QUESTION_NEEDED served safely
|
||||
gate enabled + unrenderable -> standing fallback, never contentless QUESTION_NEEDED
|
||||
no case yields neither proposal nor valid question
|
||||
```
|
||||
|
||||
### 5.3 ASK later slice — Q1B_ASK_CARVE_OUT retirement
|
||||
|
||||
Only after the no-dead-zone proof passes may a later PR consider:
|
||||
|
||||
```text
|
||||
proposal_allowed=False for missing_total_count / missing_weighted_total
|
||||
remove/retire Q1B_ASK_CARVE_OUT
|
||||
```
|
||||
|
||||
That PR must be explicit, separate, and guarded by tests.
|
||||
|
||||
### 5.4 VERIFIED next code slice — gold-free independent reader
|
||||
|
||||
The next VERIFIED code slice is not served VERIFIED. It is:
|
||||
|
||||
```text
|
||||
Design/prototype a gold-free independent reader/proof source.
|
||||
```
|
||||
|
||||
Requirements:
|
||||
|
||||
- no eval-gold setup in serving;
|
||||
- no gold answer;
|
||||
- no benchmark fixture;
|
||||
- distinct primary/independent reader lineages;
|
||||
- convergent canonical read digests;
|
||||
- strict rejection of same-reader-twice;
|
||||
- strict rejection of second-solver-over-one-read;
|
||||
- off-serving only.
|
||||
|
||||
### 5.5 VERIFIED later slices
|
||||
|
||||
Only after gold-free independence exists:
|
||||
|
||||
```text
|
||||
poison fixture harness
|
||||
holdout-gated verification harness
|
||||
verify.py consumption scoping
|
||||
served [verified] behind verified_serving_enabled
|
||||
```
|
||||
|
||||
## 6. Explicit anti-drift rules
|
||||
|
||||
The following are forbidden unless a future ADR/decision record explicitly reopens them:
|
||||
|
||||
```text
|
||||
no standalone chat/runtime ASK patch that bypasses the bus
|
||||
no served ASK without ask_serving_enabled
|
||||
no served ASK that constructs prose instead of consuming DeliveredQuestion
|
||||
no contentless QUESTION_NEEDED
|
||||
no question artifact under proposal_path
|
||||
no proposal artifact under question_path
|
||||
no Q1B carve-out retirement before no-dead-zone proof
|
||||
no served VERIFIED from eval-gold producer
|
||||
no verify.py VERIFIED consumption before gold-free independent reader + poison/holdout harness
|
||||
no direct EpistemicState.VERIFIED construction outside the contract route
|
||||
no CLAIMS or benchmark movement from off-serving artifacts
|
||||
```
|
||||
|
||||
## 7. Current status checkpoint
|
||||
|
||||
As of the `question_path` cleanup landing, the project is here:
|
||||
|
||||
```text
|
||||
Stage 2 bus doctrine: scoped, not fully active
|
||||
ASK tenant: off-serving foundation complete enough for served-bus scoping
|
||||
VERIFIED tenant: contract/gate foundation complete, serving blocked on gold-free independence
|
||||
Next ASK move: bus-governed served ASK, no carve-out retirement
|
||||
Next VERIFIED move: gold-free independent reader, no served VERIFIED
|
||||
```
|
||||
|
||||
Use this document as the checklist before issuing the next implementation brief.
|
||||
183
docs/analysis/verified-serving-wiring-scoping-2026-06-08.md
Normal file
183
docs/analysis/verified-serving-wiring-scoping-2026-06-08.md
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
# VERIFIED serving-wiring — `verified_serving_enabled` — scoping brief
|
||||
|
||||
**Date:** 2026-06-08 · **Status:** scoping (NO CODE) · **HOLD for review** ·
|
||||
**Branch:** `docs/serving-integration-scoping`
|
||||
|
||||
**What this brief is.** The scope for the served-surface decision the VERIFIED lane
|
||||
deferred. The off-serving VERIFIED lane is complete and on main:
|
||||
|
||||
```text
|
||||
P1-A the VERIFIED contract (verified_contract.py — two independent reads converge)
|
||||
P1-B gold-setup-backed producer (evals/constraint_oracle/verified_producer.py — OFF-SERVING)
|
||||
P1-C bound_slots_digest (a separable, load-bearing proof obligation)
|
||||
```
|
||||
|
||||
P1-B verifies 7/13 real R2 problems with wrong=0 — but it is **gold-setup-backed**, so
|
||||
it is structurally off-serving: the independent read is the INV-25 hand-authored gold
|
||||
SETUP, which is not available at serving time. This brief scopes what a *serving-time*
|
||||
VERIFIED would require — and why it cannot reuse P1-B. **No code here.**
|
||||
|
||||
> Companions: [[ask-serving-integration-scoping-2026-06-08]] (the ASK half of the same
|
||||
> "where off-serving stops" line), [[VERIFIED-canonical-comparison-scoping-2026-06-06]]
|
||||
> (the validate-first probe that already KILLED the naive fold-reader producer),
|
||||
> [[stage2-epistemic-disclosure-bus-verified-v1-scoping-2026-06-08]] (Doc 1).
|
||||
|
||||
---
|
||||
|
||||
## 1. The seam (grounded) — and why it is still inert by design
|
||||
|
||||
`generate/derivation/verify.py::_canonically_verified(verified, problem_text, policy)`
|
||||
is the ADR-0206 §5 VERIFIED gate — *the only thing that may license a math answer past
|
||||
gold* (resolve a disagreement STRICT refuses). It **returns `None` today**, so the
|
||||
widening is structurally inert: disagreement refuses regardless of `policy`, preserving
|
||||
absolute `wrong == 0`. Its own docstring states the bright line this brief must honour:
|
||||
|
||||
> "A reliability *license* (statistical) must NEVER substitute here: math serving is
|
||||
> absolute-wrong=0, not disclosed like the cognition path."
|
||||
|
||||
So VERIFIED serving = replacing that `return None` with a producer that returns a
|
||||
derivation **only when it is canonically VERIFIED** (proven correct, not merely sound),
|
||||
behind a kill switch, proven on a holdout. Everything below is the eligibility bar for
|
||||
that producer.
|
||||
|
||||
---
|
||||
|
||||
## 2. The five things this decision must pin
|
||||
|
||||
```text
|
||||
1. the gold-free independent-reader requirement — the crux
|
||||
2. verified_serving_enabled — the kill switch
|
||||
3. holdout gates — validate-first, no quiet widening
|
||||
4. proof-producer eligibility — what may plug into _canonically_verified
|
||||
5. the explicit ban on eval-gold-backed serving — why P1-B cannot serve
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. The gold-free independent-reader requirement (the crux)
|
||||
|
||||
VERIFIED means **two independent reads that converge on one canonical structure**
|
||||
(P1-A): a faithful solve of a *wrong read* is caught because the independent read
|
||||
disagrees. P1-B's two reads are `read_constraint_problem` (the engine reader) vs the
|
||||
**gold-authored setup**. At serving time there is no gold. So serving-time VERIFIED
|
||||
needs a **second, gold-free reader** whose disagreement is the safety mechanism.
|
||||
|
||||
The hard constraints, from the killed-probe doc and CLAUDE.md:
|
||||
|
||||
- **Independence must be in the READING, not the solving.** Back-substitution catches
|
||||
solve-errors, never read-errors. Two solvers over one reading is *fake* independence.
|
||||
The second reader must parse the problem into the same `ConstraintProblem` structure
|
||||
by a **genuinely different route** (different lineage, asserted by the
|
||||
`SAME_READER_LINEAGE` firewall / INV-27 reader-disjointness), and converge on the
|
||||
same canonical signature.
|
||||
- **No eval gold in the read** (§7). The second reader may not consult, hash, or be
|
||||
derived from any gold artifact — not the answer, not the setup. If it needs gold to
|
||||
read, it is P1-B, and P1-B does not serve.
|
||||
- **Conservative refuse-on-doubt carries wrong=0.** The second reader, like the first,
|
||||
refuses when uncertain. VERIFIED fires only on *agreement of two confident,
|
||||
independent reads*; any refusal or disagreement → STRICT refuses (today's behaviour).
|
||||
|
||||
**Eligibility, stated as a gate:** a serving-time VERIFIED producer is eligible **only
|
||||
if** it exhibits two reads with (a) distinct, firewall-asserted lineages, (b) neither
|
||||
read derived from gold, (c) convergence on one canonical `ConstraintProblem` signature,
|
||||
(d) back-substitution + boundary-clear + bound-slots (the P1-A/P1-C obligations), and
|
||||
(e) refuse-preferring failure. Absent any one, it stays `None`.
|
||||
|
||||
Whether such a second R2 reader *exists yet* is the open empirical question — the killed
|
||||
fold-reader probe is the cautionary precedent: complementary readers were ~98% wrong on
|
||||
the refused set. **This brief does not assume one exists; it sets the bar a candidate
|
||||
must clear, validate-first, before any wiring.**
|
||||
|
||||
---
|
||||
|
||||
## 4. `verified_serving_enabled` — the kill switch
|
||||
|
||||
Add `verified_serving_enabled: bool = False` to `core/config.py` (sibling of
|
||||
`estimation_enabled`). Default **off**. When off, `_canonically_verified` returns `None`
|
||||
unconditionally (today's inert state) regardless of any producer being present. The
|
||||
switch is the single audited place that lights the seam, and it stays off until §5.
|
||||
|
||||
---
|
||||
|
||||
## 5. Holdout gates — validate-first, no quiet widening
|
||||
|
||||
VERIFIED may not widen live until proven on a **held-out** set it never trained or
|
||||
tuned on (INV-25 discipline; the killed-probe doc's validate-first rule). The gate, in
|
||||
order:
|
||||
|
||||
1. **Holdout probe (off-serving):** run the candidate producer over a held-out R2 set
|
||||
with gold answers withheld; require **wrong == 0** on everything it marks VERIFIED,
|
||||
and that everything it cannot independently verify it *refuses* (over-refusal is
|
||||
acceptable; one wrong is disqualifying).
|
||||
2. **Seal byte-identity:** the GSM8K candidate-graph serving seal (pinned SHAs) stays
|
||||
byte-identical — VERIFIED widens a *different* surface (R2 constraint answers), it
|
||||
must not perturb the sealed lane.
|
||||
3. **Disagreement-still-refuses:** a test proving that with the producer wired and the
|
||||
switch ON, a faithful solve of a deliberately wrong read still refuses (the P1-A
|
||||
poison test, now on the served path).
|
||||
4. Only then does `verified_serving_enabled` go on, **one surface at a time** (R2
|
||||
first; R4 second), each with its own holdout pass.
|
||||
|
||||
---
|
||||
|
||||
## 6. The served surface — a distinct `[verified]` disclosure
|
||||
|
||||
When wired and enabled, a VERIFIED R2 answer is served under its **own** disclosure
|
||||
claim/marker — the locked decision from Doc 1's review: *VERIFIED gets a distinct
|
||||
served disclosure mode + `[verified]` prefix, NEVER reused from `[approximate]`*.
|
||||
VERIFIED is a *license* claim (more licensed than gold-strict), not a *speculation*
|
||||
claim. The route to it is the only sanctioned one: `disclosure_for_verification(result)`
|
||||
→ `(EpistemicState.VERIFIED, DisclosureClaim.VERIFIED)` → `ServedDisposition.DISCLOSE`
|
||||
under the P0-3 guard (which already degrades an unbacked VERIFIED claim to COMMIT).
|
||||
|
||||
---
|
||||
|
||||
## 7. The explicit ban on eval-gold-backed serving (the load-bearing non-claim)
|
||||
|
||||
**P1-B must never become a serving producer.** It is gold-setup-backed: its independent
|
||||
read is the hand-authored gold SETUP. Wiring it into `_canonically_verified` would mean
|
||||
the engine "verifies" by consulting the answer key's structure — circular, and a
|
||||
wrong=0 fiction (it would serve VERIFIED on exactly the problems it was handed the setup
|
||||
for). The ban, stated so a test can enforce it:
|
||||
|
||||
- `_canonically_verified` (and any serving producer behind it) may import **nothing**
|
||||
from `evals/` — no gold loader, no `gold_to_problem`, no `r2_gold`. An AST/import test
|
||||
asserts the serving path is gold-free, the mirror of the off-serving AST tests.
|
||||
- P1-B stays in `evals/` precisely so this boundary is structural: `evals → core` is the
|
||||
allowed direction; `core/generate serving → evals` is forbidden.
|
||||
- "Verified on a holdout" (§5) is **not** the same as "verified by gold at serving" —
|
||||
the holdout *measures* a gold-free producer; it never *feeds* gold into one.
|
||||
|
||||
This is the VERIFIED analogue of the GSM8K "candidate-graph owns the metric; no bridge
|
||||
re-enables without sealed/independent wrong=0" discipline.
|
||||
|
||||
---
|
||||
|
||||
## 8. What this is NOT
|
||||
|
||||
- **Not** a claim that a serving-time VERIFIED producer exists — §3 sets the eligibility
|
||||
bar; whether an R2 second reader clears it is an open, validate-first question.
|
||||
- **Not** a reliability/statistical license at the math seam — that path is the
|
||||
cognition `[approximate]` disclosure; math serving is absolute wrong=0 (§1 bright line).
|
||||
- **Not** a GSM8K-seal move — VERIFIED widens R2 constraint answers, seal stays
|
||||
byte-identical (§5.2).
|
||||
- **Not** P1-B promoted to serving — explicitly banned (§7); P1-B stays off-serving.
|
||||
|
||||
---
|
||||
|
||||
## 9. The questions for the ruling
|
||||
|
||||
1. **Eligibility bar:** adopt §3 (a)–(e) as the gate any serving VERIFIED producer must
|
||||
clear — gold-free second reader, independence-in-the-reading, refuse-preferring? (rec: yes)
|
||||
2. **Gate:** add `verified_serving_enabled = False` (sibling of `estimation_enabled`),
|
||||
`_canonically_verified` stays `None` while off? (rec: yes)
|
||||
3. **Holdout:** require the §5 validate-first sequence (wrong=0 on holdout + seal
|
||||
byte-identity + disagreement-still-refuses) before the switch, R2-first? (rec: yes)
|
||||
4. **Gold ban:** enforce the §7 import ban (serving path imports nothing from `evals/`)
|
||||
with a test, keeping P1-B off-serving forever? (rec: yes)
|
||||
5. **Surface:** serve VERIFIED under its distinct `[verified]` marker via
|
||||
`disclosure_for_verification` only? (rec: yes — the locked Doc 1 decision)
|
||||
|
||||
No served-surface code until this brief is reviewed. Pairs with
|
||||
[[ask-serving-integration-scoping-2026-06-08]] — together they draw the full line where
|
||||
off-serving stops.
|
||||
240
docs/analysis/verified-serving-wiring-scoping-2026-06-09.md
Normal file
240
docs/analysis/verified-serving-wiring-scoping-2026-06-09.md
Normal file
|
|
@ -0,0 +1,240 @@
|
|||
# VERIFIED Serving Wiring Scoping — verified_serving_enabled and Gold-Free Independence
|
||||
|
||||
## 1. Current State
|
||||
|
||||
Currently, the VERIFIED verification pipeline exists off-serving only. No served [verified] surface exists, no serving-time config or gate exists, and no `verify.py` serving wiring is implemented.
|
||||
|
||||
The existing off-serving spine consists of:
|
||||
- **P1-A**: Defines the contract in `core/epistemic_disclosure/verified_contract.py`. This contract includes:
|
||||
- `VerificationProof` (the data structure containing digests and lineages)
|
||||
- `VerificationObligation` (the strict checklist of obligations)
|
||||
- `evaluate_verification` (the pure-logic evaluation function enforcing the contract)
|
||||
- `disclosure_for_verification` (the single sanctioned route to transition a result to the verified state)
|
||||
- Defines the meaning of **VERIFIED**: two independent reads must converge on a single canonical structure. This requires:
|
||||
- Independent reads (`primary_reader_lineage` must not equal `independent_reader_lineage`)
|
||||
- Convergent canonical read digests (`primary_read_digest` must equal `independent_read_digest`)
|
||||
- `derivation_digest` present
|
||||
- `bound_slots_digest` present
|
||||
- `back_substitution_digest` present
|
||||
- `boundary_clear` is `True`
|
||||
- `contradiction_clear` is `True`
|
||||
- `limitation` is `None`
|
||||
- **P1-B**: Adds an off-serving R2 verification producer in `evals/constraint_oracle/verified_producer.py`:
|
||||
- Extracts the primary read from the problem text via `read_constraint_problem(text)`.
|
||||
- Obtains the independent read from a hand-authored gold setup (`gold_setup`).
|
||||
- The gold answer itself never enters the verification process (the gold structure setup signature is matched, not the answer).
|
||||
- The producer is strictly for evaluation and runs off-serving only.
|
||||
- **P1-C**: Introduces `bound_slots_digest` as a distinct, separable proof obligation to ensure the answer binds only to the stated slots (asked unknowns) and not phantom slots.
|
||||
- **Serving Isolation**:
|
||||
- No served VERIFIED surface exists.
|
||||
- No `verified_serving_enabled` gate exists.
|
||||
- No `verify.py` serving integration exists.
|
||||
|
||||
## 2. Hard Non-Claim
|
||||
|
||||
The gold-setup-backed producer implemented in `evals/constraint_oracle/verified_producer.py` **cannot** be used for serving.
|
||||
|
||||
What P1-B proves:
|
||||
- The VERIFIED contract logic is sound and functional.
|
||||
- The verification producer can successfully assemble valid `VerificationProof` objects.
|
||||
- Wrong reader structures diverge and correctly fail the independent verification check.
|
||||
- The actual gold answer is not required to verify the canonical structural solve.
|
||||
|
||||
What P1-B does **NOT** prove:
|
||||
- Serving-time independence (since gold setups do not exist for runtime user requests).
|
||||
- Gold-free verification.
|
||||
- User-visible served `[verified]` status.
|
||||
- Benchmark or `CLAIMS.md` movement.
|
||||
- AGI capabilities.
|
||||
|
||||
Any served `[verified]` status requires a gold-free independent read source at serving time.
|
||||
|
||||
## 3. Proposed Served Gate: verified_serving_enabled
|
||||
|
||||
To control the rollout of serving-time verification, a future configuration gate must be defined:
|
||||
|
||||
- The configuration field `verified_serving_enabled` does not exist yet.
|
||||
- This document does not add or implement this configuration gate.
|
||||
- The future gate must be default-false and fail-closed: if the config field is missing or malformed, it must evaluate to `False`.
|
||||
- The gate must only control the served/user-visible `[verified]` surface.
|
||||
- The gate must never affect off-serving evaluation proof generation or off-serving validation runs.
|
||||
- The gate must strictly prohibit eval-gold-backed verification pipelines from touching serving code.
|
||||
- Enabling the gate is necessary but not sufficient: even if `verified_serving_enabled` is `True`, a served VERIFIED verdict is only allowed if all contract proof obligations successfully pass.
|
||||
|
||||
| Config State | verified_serving_enabled | Served VERIFIED Allowed? |
|
||||
| --- | --- | --- |
|
||||
| missing field | False | No |
|
||||
| default config | False | No |
|
||||
| explicit false | False | No |
|
||||
| explicit true | True | Only if gold-free proof obligations pass |
|
||||
|
||||
## 4. Gold-Free Independent Read Requirement
|
||||
|
||||
Before any served `[verified]` surface can exist, a serving proof must use a gold-free independent read source.
|
||||
|
||||
A served proof must adhere to the following:
|
||||
- Primary reader lineage is recorded.
|
||||
- Independent reader lineage is recorded.
|
||||
- The lineages must use distinct lineage identifiers.
|
||||
- The primary read digest and independent read digest must converge.
|
||||
- No gold setup is used.
|
||||
- No gold answer is used.
|
||||
- No benchmark fixture is used.
|
||||
- No eval-lane imports are performed.
|
||||
- Same reader twice (invoking the exact same reader lineage twice on the same text) is strictly rejected.
|
||||
- Second solver over one read (running two solvers over the same read signature) is strictly rejected.
|
||||
|
||||
### Comparison
|
||||
|
||||
* **Invalid (Pseudo-Independence):**
|
||||
```text
|
||||
primary read ──> solver A
|
||||
└───> solver B ──> same answer ──> VERIFIED (INVALID)
|
||||
```
|
||||
* **Valid (True Independence):**
|
||||
```text
|
||||
primary read lineage ───────────> canonical digest C ──┐
|
||||
├──> convergent digests ──> VERIFIED (VALID)
|
||||
gold-free independent lineage ──> canonical digest C ──┘
|
||||
```
|
||||
|
||||
The valid workflow requires:
|
||||
- Primary read lineage converges to canonical digest `C`.
|
||||
- Gold-free independent read lineage converges to canonical digest `C`.
|
||||
- Derivation is computed from stated quantities.
|
||||
- Bound slots are present.
|
||||
- Back-substitution succeeds.
|
||||
- No boundary has fired.
|
||||
- No contradiction is present.
|
||||
- No unresolved limitation exists.
|
||||
|
||||
The primary unsolved technical milestone is designing and implementing a gold-free independent reader/verifier source.
|
||||
|
||||
## 5. Serving Eligibility Criteria
|
||||
|
||||
For any response to be eligible for served `[verified]` status, all of the following conditions must be met:
|
||||
|
||||
- `source_problem_digest` is present.
|
||||
- `primary_reader_lineage` is present.
|
||||
- `independent_reader_lineage` is present.
|
||||
- `primary_reader_lineage` and `independent_reader_lineage` are distinct.
|
||||
- `primary_read_digest` is present.
|
||||
- `independent_read_digest` is present.
|
||||
- `primary_read_digest` and `independent_read_digest` are identical (converge).
|
||||
- `derivation_digest` is present.
|
||||
- `bound_slots_digest` is present.
|
||||
- `back_substitution_digest` is present.
|
||||
- `boundary_clear` is `True`.
|
||||
- `contradiction_clear` is `True`.
|
||||
- `limitation` is `None`.
|
||||
- `evaluate_verification(...)` evaluates to `VerificationVerdict.VERIFIED`.
|
||||
- `disclosure_for_verification(...)` is the only functional pathway used to transition the result to `(EpistemicState.VERIFIED, DisclosureClaim.VERIFIED)`.
|
||||
- `verified_serving_enabled` is explicitly configured to `True`.
|
||||
- The proof producer is gold-free and serving-eligible.
|
||||
|
||||
## 6. Holdout and Kill-Switch Gates
|
||||
|
||||
Before any served `[verified]` behavior can be wired to production, the serving-time verifier must clear a set of holdout and kill-switch gates:
|
||||
- **Sealed Holdout Lane**: A dedicated evaluation run on a sealed, off-distribution holdout dataset.
|
||||
- **Wrong=0 Requirement**: Zero incorrect answers are permitted for any result designated as VERIFIED.
|
||||
- **Poison Fixtures**: Comprehensive verification tests using poison test cases to prove that:
|
||||
- Wrong-read structures diverge and fail.
|
||||
- Same-reader-twice reads fail verification.
|
||||
- Answer matches gold, but missing proof fails verification.
|
||||
- Absence of refusal alone without proof fails verification.
|
||||
- Missing `bound_slots_digest` fails verification.
|
||||
- Triggered boundaries fail verification.
|
||||
- Contradictions present fail verification.
|
||||
- Unresolved limitations fail verification.
|
||||
- **Deterministic Replay Digest**: The verification trace must be replayable, yielding identical digests.
|
||||
- **Kill-Switch**: Default off / fail-closed behavior must be validated.
|
||||
- **Gate Disabled Test**: A test must verify that disabling `verified_serving_enabled` suppresses the served `[verified]` label even when a fully valid proof is present.
|
||||
- **Gate Enabled Test**: A test must verify that enabling `verified_serving_enabled` still requires all proof obligations to pass before serving.
|
||||
|
||||
## 7. verify.py Boundary
|
||||
|
||||
The transition of the verification verdict to serving must enforce strict boundaries at the `verify.py` layer:
|
||||
- `verify.py` may eventually consume a verified verdict from the contract.
|
||||
- `verify.py` must not define or redefine the semantic meaning of VERIFIED.
|
||||
- The formal meaning of VERIFIED remains exclusively defined in `core/epistemic_disclosure/verified_contract.py`.
|
||||
- `verify.py` must never attempt to repair failed proofs.
|
||||
- `verify.py` must not treat raw answer correctness (matching gold) as verification.
|
||||
- `verify.py` must not import or use eval-lane gold datasets.
|
||||
- No hot-path repair or silent corrections may occur in `verify.py`.
|
||||
|
||||
## 8. Served Behavior Matrix
|
||||
|
||||
| Proof / Gate State | Served Behavior |
|
||||
| --- | --- |
|
||||
| Valid off-serving eval-gold proof | Never served |
|
||||
| Valid gold-free proof + gate disabled | No served `[verified]` |
|
||||
| Valid gold-free proof + gate enabled | Served `[verified]` may be allowed |
|
||||
| Wrong read divergence | No served `[verified]` |
|
||||
| Same reader twice | No served `[verified]` |
|
||||
| Answer matches gold but proof missing | No served `[verified]` |
|
||||
| No refusal but no proof | No served `[verified]` |
|
||||
| Boundary fired | No served `[verified]` |
|
||||
| Contradiction present | No served `[verified]` |
|
||||
| Unresolved limitation | No served `[verified]` |
|
||||
|
||||
## 9. Required Future Tests Before Wiring
|
||||
|
||||
The following test suite must be implemented and pass before any serving-time code can land:
|
||||
- `verified_serving_enabled` defaults to `False`.
|
||||
- A missing config field for `verified_serving_enabled` resolves to `False`.
|
||||
- Explicit `True` configuration is required to serve `[verified]`.
|
||||
- A disabled gate suppresses served `[verified]` even when a valid proof exists.
|
||||
- An enabled gate still enforces and requires all proof obligations to pass.
|
||||
- Eval-gold verification producers cannot be imported or accessed by serving modules (AST / dependency check).
|
||||
- Eval-gold-backed proofs are rejected from being served.
|
||||
- Same reader twice fails verification.
|
||||
- Running a second solver over one reading fails verification.
|
||||
- Answer-gold match without verification proof does not serve `[verified]`.
|
||||
- Absence of refusal alone does not serve `[verified]`.
|
||||
- Direct construction of `EpistemicState.VERIFIED` is blocked outside of `disclosure_for_verification`.
|
||||
- Poison wrong-read inputs do not serve `[verified]`.
|
||||
- Missing `bound_slots_digest` does not serve `[verified]`.
|
||||
- Triggered boundary does not serve `[verified]`.
|
||||
- Contradiction present does not serve `[verified]`.
|
||||
- Unresolved limitation does not serve `[verified]`.
|
||||
- `verify.py` consumes the contract outcome but does not define its rules.
|
||||
|
||||
## 10. Non-Claims
|
||||
|
||||
This scoping document establishes architectural boundaries only.
|
||||
|
||||
- This document does not implement served VERIFIED.
|
||||
- This document does not add the `verified_serving_enabled` config gate.
|
||||
- This document does not wire `verify.py` to serving.
|
||||
- This document does not move the evaluation producer (`verified_producer.py`) into the core serving layer.
|
||||
- This document does not make gold-backed verification serving-safe.
|
||||
- This document does not change benchmark metrics.
|
||||
- This document does not update `CLAIMS.md`.
|
||||
- This document does not solve the design of the gold-free independent reader.
|
||||
- This document does not claim AGI progress.
|
||||
|
||||
## 11. Recommended Next Slices
|
||||
|
||||
The implementation of serving-time verification should proceed in isolated, review-gated slices:
|
||||
|
||||
### Slice 1: Configuration Gate
|
||||
- Add default-dark `verified_serving_enabled` helper only.
|
||||
- No runtime wiring, no `verify.py` wiring.
|
||||
- Strict import checks preventing eval producer imports.
|
||||
- Tests demonstrating default-dark behavior and config defaults.
|
||||
|
||||
### Slice 2: Gold-Free Independent Reader
|
||||
- Design and prototype a gold-free independent reader/verifier source.
|
||||
- Maintain off-serving isolation.
|
||||
- Generate independent reader lineage and canonical digests.
|
||||
- No integration with the served path.
|
||||
|
||||
### Slice 3: Verification Harness & Poison Fixtures
|
||||
- Implement a holdout-gated verification harness.
|
||||
- Integrate poison fixtures and test cases.
|
||||
- Validate deterministic replay digests.
|
||||
- Do not expose any served surface.
|
||||
|
||||
### Slice 4: verify.py Consumption
|
||||
- Only after all prior slices are approved and pass tests, scope the consumption of the verified verdict within `verify.py`.
|
||||
- No implementation without separate architectural review.
|
||||
|
|
@ -52,6 +52,8 @@ from core.epistemic_state import EpistemicState
|
|||
from evals.constraint_oracle.signature import (
|
||||
constraint_setup_signature,
|
||||
constraints_signature,
|
||||
query_signature,
|
||||
unknowns_signature,
|
||||
)
|
||||
from generate.constraint_comprehension.model import ConstraintProblem
|
||||
from generate.constraint_comprehension.reader import read_constraint_problem
|
||||
|
|
@ -139,6 +141,7 @@ def verify_r2(text: str, gold_setup: ConstraintProblem) -> R2VerificationOutcome
|
|||
if isinstance(solution, Refusal):
|
||||
boundary_clear = False
|
||||
derivation_digest = ""
|
||||
bound_slots_digest = ""
|
||||
back_substitution_digest = ""
|
||||
else:
|
||||
boundary_clear = True
|
||||
|
|
@ -146,6 +149,23 @@ def verify_r2(text: str, gold_setup: ConstraintProblem) -> R2VerificationOutcome
|
|||
derivation_digest = _sha(
|
||||
repr(("derivation", constraints_signature(primary.constraints), bound))
|
||||
)
|
||||
# bound_slots: the answer binds to a STATED slot — the asked unknown — among the
|
||||
# declared unknowns. Empty if the asked unknown was not solved (a phantom answer);
|
||||
# a complete derivation does NOT by itself prove this (P1-C: separable obligation).
|
||||
bound_slots_digest = (
|
||||
_sha(
|
||||
repr(
|
||||
(
|
||||
"bound_slots",
|
||||
unknowns_signature(primary.unknowns),
|
||||
query_signature(primary.query),
|
||||
bound,
|
||||
)
|
||||
)
|
||||
)
|
||||
if primary.query.symbol in solution
|
||||
else ""
|
||||
)
|
||||
back_substitution_digest = (
|
||||
_sha(repr(("back_substitution", bound)))
|
||||
if _back_substitutes(primary, solution)
|
||||
|
|
@ -159,6 +179,7 @@ def verify_r2(text: str, gold_setup: ConstraintProblem) -> R2VerificationOutcome
|
|||
primary_read_digest=_sha(repr(constraint_setup_signature(primary))),
|
||||
independent_read_digest=_sha(repr(constraint_setup_signature(gold_setup))),
|
||||
derivation_digest=derivation_digest,
|
||||
bound_slots_digest=bound_slots_digest,
|
||||
back_substitution_digest=back_substitution_digest,
|
||||
boundary_clear=boundary_clear,
|
||||
# Setup-only verification has no answer-key path, so no contradiction can arise
|
||||
|
|
|
|||
|
|
@ -19,6 +19,12 @@ class Terminal(str, Enum):
|
|||
REFUSED_UNSUPPORTED_FAMILY = "REFUSED_UNSUPPORTED_FAMILY"
|
||||
CONTRADICTION_DETECTED = "CONTRADICTION_DETECTED"
|
||||
PROPOSAL_EMITTED = "PROPOSAL_EMITTED"
|
||||
# The ASK tenant (Q1-D): a solvable attempt blocked on missing/ambiguous *input*
|
||||
# that a grounded question can intake. A SIBLING of PROPOSAL_EMITTED, never a
|
||||
# subtype — a proposal offers a capability for review; a question requests a datum
|
||||
# from the user. Off-serving: produced by the Q1-D delivery layer into the
|
||||
# teaching/questions sink, never served until a future ask_serving_enabled gate.
|
||||
QUESTION_NEEDED = "QUESTION_NEEDED"
|
||||
AMBIGUOUS_ORGAN = "AMBIGUOUS_ORGAN"
|
||||
NO_PROGRESS = "NO_PROGRESS"
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ from __future__ import annotations
|
|||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, TYPE_CHECKING
|
||||
|
||||
from core.comprehension_attempt import (
|
||||
ComprehensionAttempt,
|
||||
|
|
@ -49,6 +49,10 @@ from generate.rate_comprehension.solver import solve_rate
|
|||
from generate.contemplation.findings import Finding, Terminal
|
||||
from generate.meaning_graph.reader import Refusal
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from core.epistemic_disclosure.limitation import LimitationAssessment
|
||||
from core.epistemic_questions.delivery import DeliveryOutcome
|
||||
|
||||
#: Substantive boundaries that are *recognized-but-unsupported* capabilities (not hard errors).
|
||||
_UNSUPPORTED_FAMILIES = frozenset(
|
||||
{
|
||||
|
|
@ -74,16 +78,76 @@ class ContemplationResult:
|
|||
answer: int | None = None
|
||||
family: str | None = None
|
||||
proposal_path: str | None = None
|
||||
question_path: str | None = None
|
||||
message: str | None = None
|
||||
|
||||
|
||||
def _delivery_outcome_for_limitation(assessment: LimitationAssessment) -> DeliveryOutcome:
|
||||
"""Helper to delegate to deliver_ask, pure and testable."""
|
||||
from core.epistemic_questions.delivery import deliver_ask
|
||||
return deliver_ask(assessment)
|
||||
|
||||
|
||||
def _handle_ask_delivery(
|
||||
assessment: LimitationAssessment,
|
||||
family_name: str,
|
||||
findings: list[Finding],
|
||||
attempts: tuple[ComprehensionAttempt, ...],
|
||||
text: str,
|
||||
proposal_root: Path | None,
|
||||
question_root: Path | None,
|
||||
exercise_ask: bool,
|
||||
selected_organ: str | None = None,
|
||||
) -> ContemplationResult:
|
||||
outcome = _delivery_outcome_for_limitation(assessment)
|
||||
if outcome.terminal == Terminal.QUESTION_NEEDED:
|
||||
assert outcome.question is not None
|
||||
import json
|
||||
from core.epistemic_questions.delivery import question_path
|
||||
|
||||
path = question_path(outcome.question, question_root)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(
|
||||
json.dumps(outcome.question.to_json_dict(), indent=2, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
findings.append(Finding("ask", f"emitted question-only {assessment.blocking_reason}"))
|
||||
findings.append(Finding("terminal", Terminal.QUESTION_NEEDED.value))
|
||||
return ContemplationResult(
|
||||
Terminal.QUESTION_NEEDED, tuple(findings), attempts,
|
||||
selected_organ=selected_organ, family=family_name,
|
||||
proposal_path=None,
|
||||
question_path=str(path),
|
||||
)
|
||||
else:
|
||||
findings.append(Finding("ask", f"unrenderable ask: {outcome.fallback_reason}"))
|
||||
findings.append(Finding("terminal", outcome.terminal.value))
|
||||
if outcome.terminal == Terminal.PROPOSAL_EMITTED:
|
||||
from core.comprehension_attempt.failure_family import family_by_name
|
||||
family_obj = family_by_name(family_name)
|
||||
if family_obj is not None:
|
||||
path = emit_proposal(text, family_obj, attempts, root=proposal_root)
|
||||
return ContemplationResult(
|
||||
Terminal.PROPOSAL_EMITTED, tuple(findings), attempts,
|
||||
selected_organ=selected_organ, family=family_name,
|
||||
proposal_path=str(path) if path else None,
|
||||
)
|
||||
return ContemplationResult(
|
||||
outcome.terminal, tuple(findings), attempts,
|
||||
selected_organ=selected_organ, family=family_name,
|
||||
)
|
||||
|
||||
|
||||
def contemplate(
|
||||
text: str,
|
||||
*,
|
||||
options: dict[str, Any] | None = None,
|
||||
answer_key: str | None = None,
|
||||
proposal_root: Path | None = None,
|
||||
question_root: Path | None = None,
|
||||
case_id: str | None = None,
|
||||
exercise_ask: bool = False,
|
||||
) -> ContemplationResult:
|
||||
"""Run one bounded contemplation pass over *text*."""
|
||||
findings: list[Finding] = []
|
||||
|
|
@ -111,11 +175,11 @@ def contemplate(
|
|||
if route.status == "routed":
|
||||
assert route.selected is not None
|
||||
if route.selected.organ == "r2_constraints":
|
||||
return _solve_and_verify_r2(text, options, answer_key, findings, attempts)
|
||||
return _solve_and_verify_r2(text, options, answer_key, findings, attempts, proposal_root, question_root, exercise_ask)
|
||||
if route.selected.organ == "r3_rate":
|
||||
return _solve_and_verify_r3(text, options, answer_key, findings, attempts)
|
||||
return _solve_and_verify_r3(text, options, answer_key, findings, attempts, proposal_root, question_root, exercise_ask)
|
||||
if route.selected.organ == "r4_combined_rate":
|
||||
return _solve_and_verify_cmb(text, options, answer_key, findings, attempts)
|
||||
return _solve_and_verify_cmb(text, options, answer_key, findings, attempts, proposal_root, question_root, exercise_ask)
|
||||
findings.append(Finding("solve", "r1 admissible setup (numeric answer is the eval lane in v0)"))
|
||||
findings.append(Finding("terminal", Terminal.SOLVED_VERIFIED.value))
|
||||
return ContemplationResult(
|
||||
|
|
@ -123,7 +187,7 @@ def contemplate(
|
|||
)
|
||||
|
||||
# route.status == "all_refused"
|
||||
return _classify_all_refused(text, attempts, findings, proposal_root)
|
||||
return _classify_all_refused(text, attempts, findings, proposal_root, question_root, exercise_ask)
|
||||
|
||||
|
||||
def _solve_and_verify_r2(
|
||||
|
|
@ -132,12 +196,26 @@ def _solve_and_verify_r2(
|
|||
answer_key: str | None,
|
||||
findings: list[Finding],
|
||||
attempts: tuple[ComprehensionAttempt, ...],
|
||||
proposal_root: Path | None,
|
||||
question_root: Path | None,
|
||||
exercise_ask: bool,
|
||||
) -> ContemplationResult:
|
||||
problem = read_constraint_problem(text)
|
||||
assert not isinstance(problem, Refusal) # routed => the reader admitted a setup
|
||||
value = answer_constraint_problem(problem)
|
||||
if isinstance(value, Refusal):
|
||||
findings.append(Finding("solve", f"solver refused: {value.reason}"))
|
||||
from core.comprehension_attempt.failure_family import family_by_name
|
||||
from core.epistemic_disclosure.limitation import assess_from_family
|
||||
family_obj = family_by_name(value.reason)
|
||||
if family_obj is not None:
|
||||
assessment = assess_from_family(family_obj)
|
||||
if assessment.resolution_action == "ask_question":
|
||||
if exercise_ask:
|
||||
return _handle_ask_delivery(
|
||||
assessment, family_obj.name, findings, attempts, text, proposal_root, question_root, exercise_ask,
|
||||
selected_organ="r2_constraints"
|
||||
)
|
||||
findings.append(Finding("terminal", Terminal.REFUSED_KNOWN_BOUNDARY.value))
|
||||
return ContemplationResult(
|
||||
Terminal.REFUSED_KNOWN_BOUNDARY, tuple(findings), attempts,
|
||||
|
|
@ -175,12 +253,26 @@ def _solve_and_verify_r3(
|
|||
answer_key: str | None,
|
||||
findings: list[Finding],
|
||||
attempts: tuple[ComprehensionAttempt, ...],
|
||||
proposal_root: Path | None,
|
||||
question_root: Path | None,
|
||||
exercise_ask: bool,
|
||||
) -> ContemplationResult:
|
||||
problem = read_rate_problem(text)
|
||||
assert not isinstance(problem, Refusal) # routed => the reader admitted a setup
|
||||
value = solve_rate(problem)
|
||||
if isinstance(value, Refusal):
|
||||
findings.append(Finding("solve", f"solver refused: {value.reason}"))
|
||||
from core.comprehension_attempt.failure_family import family_by_name
|
||||
from core.epistemic_disclosure.limitation import assess_from_family
|
||||
family_obj = family_by_name(value.reason)
|
||||
if family_obj is not None:
|
||||
assessment = assess_from_family(family_obj)
|
||||
if assessment.resolution_action == "ask_question":
|
||||
if exercise_ask:
|
||||
return _handle_ask_delivery(
|
||||
assessment, family_obj.name, findings, attempts, text, proposal_root, question_root, exercise_ask,
|
||||
selected_organ="r3_rate"
|
||||
)
|
||||
findings.append(Finding("terminal", Terminal.REFUSED_KNOWN_BOUNDARY.value))
|
||||
return ContemplationResult(
|
||||
Terminal.REFUSED_KNOWN_BOUNDARY, tuple(findings), attempts,
|
||||
|
|
@ -218,6 +310,9 @@ def _solve_and_verify_cmb(
|
|||
answer_key: str | None,
|
||||
findings: list[Finding],
|
||||
attempts: tuple[ComprehensionAttempt, ...],
|
||||
proposal_root: Path | None,
|
||||
question_root: Path | None,
|
||||
exercise_ask: bool,
|
||||
) -> ContemplationResult:
|
||||
problem = read_combined_rate_problem(text)
|
||||
assert not isinstance(problem, Refusal) # routed => the reader admitted a setup
|
||||
|
|
@ -227,6 +322,18 @@ def _solve_and_verify_cmb(
|
|||
# answerable boundary. A solver refusal is a terminal boundary, never a proposal — and the
|
||||
# reason is namespaced cmb_* so it resolves to the CMB solver family, not R2/R3's.
|
||||
findings.append(Finding("solve", f"solver refused: {value.reason}"))
|
||||
from core.comprehension_attempt.failure_family import family_by_name
|
||||
from core.epistemic_disclosure.limitation import assess_from_family
|
||||
reason = cmb_reason(value.reason)
|
||||
family_obj = family_by_name(reason)
|
||||
if family_obj is not None:
|
||||
assessment = assess_from_family(family_obj)
|
||||
if assessment.resolution_action == "ask_question":
|
||||
if exercise_ask:
|
||||
return _handle_ask_delivery(
|
||||
assessment, family_obj.name, findings, attempts, text, proposal_root, question_root, exercise_ask,
|
||||
selected_organ="r4_combined_rate"
|
||||
)
|
||||
findings.append(Finding("terminal", Terminal.REFUSED_KNOWN_BOUNDARY.value))
|
||||
return ContemplationResult(
|
||||
Terminal.REFUSED_KNOWN_BOUNDARY, tuple(findings), attempts,
|
||||
|
|
@ -263,6 +370,8 @@ def _classify_all_refused(
|
|||
attempts: tuple[ComprehensionAttempt, ...],
|
||||
findings: list[Finding],
|
||||
proposal_root: Path | None,
|
||||
question_root: Path | None,
|
||||
exercise_ask: bool,
|
||||
) -> ContemplationResult:
|
||||
# CMB-over-R3 precedence (family side): when CMB substantively recognized the text, R3's broader
|
||||
# partial classification is suppressed, so CMB's sharper diagnosis owns the terminal/proposal
|
||||
|
|
@ -270,9 +379,10 @@ def _classify_all_refused(
|
|||
considered = attempts
|
||||
if cmb_is_authoritative(attempts):
|
||||
considered = tuple(a for a in attempts if a.organ != "r3_rate")
|
||||
|
||||
families = [(a, family_for_reason(a.refusal_reason)) for a in considered]
|
||||
|
||||
# Boundary-first: a substantive recognized boundary blocks any proposal.
|
||||
# Boundary-first: a substantive recognized boundary blocks any proposal or ASK.
|
||||
for _attempt, family in families:
|
||||
if family is not None and family.must_remain_refused and family.name != _NOT_MY_DOMAIN:
|
||||
terminal = (
|
||||
|
|
@ -283,6 +393,16 @@ def _classify_all_refused(
|
|||
findings.append(Finding("terminal", f"{terminal.value} via {family.name}"))
|
||||
return ContemplationResult(terminal, tuple(findings), attempts, family=family.name)
|
||||
|
||||
# Check for ASK delivery only after substantive boundaries are ruled out.
|
||||
for attempt in considered:
|
||||
from core.epistemic_disclosure.limitation import assess_from_attempt
|
||||
assessment = assess_from_attempt(attempt)
|
||||
if assessment is not None and assessment.resolution_action == "ask_question":
|
||||
if exercise_ask:
|
||||
return _handle_ask_delivery(
|
||||
assessment, assessment.blocking_reason, findings, attempts, text, proposal_root, question_root, exercise_ask
|
||||
)
|
||||
|
||||
# No substantive boundary: a genuine growth surface may emit a proposal-only artifact.
|
||||
for _attempt, family in families:
|
||||
if family is not None and family.proposal_allowed:
|
||||
|
|
|
|||
336
tests/test_ask_pass_manager_delivery.py
Normal file
336
tests/test_ask_pass_manager_delivery.py
Normal file
|
|
@ -0,0 +1,336 @@
|
|||
"""Focused tests for off-serving ASK delivery integration in the contemplation pass manager (N6).
|
||||
|
||||
Ensures that the pass manager seam delegates to deliver_ask, correctly handles fallback dispositions,
|
||||
avoids direct rendering imports/text construction, and preserves the Q1-B carve-out.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from core.epistemic_disclosure.limitation import (
|
||||
Q1B_ASK_CARVE_OUT,
|
||||
LimitationAssessment,
|
||||
MissingSlot,
|
||||
)
|
||||
from core.epistemic_state import EpistemicState
|
||||
from generate.contemplation import Terminal, contemplate
|
||||
from generate.contemplation.pass_manager import (
|
||||
_delivery_outcome_for_limitation,
|
||||
)
|
||||
from core.comprehension_attempt.failure_family import family_by_name
|
||||
|
||||
|
||||
def _make_assessment(
|
||||
*,
|
||||
blocking_reason: str,
|
||||
slots: tuple[MissingSlot, ...],
|
||||
resolution_action: str = "ask_question",
|
||||
) -> LimitationAssessment:
|
||||
return LimitationAssessment(
|
||||
limitation_kind="missing_information",
|
||||
resolution_action=resolution_action, # type: ignore[arg-type]
|
||||
epistemic_state=EpistemicState.UNDETERMINED,
|
||||
owner_organ="r2_constraint",
|
||||
blocking_reason=blocking_reason,
|
||||
missing_slots=slots,
|
||||
)
|
||||
|
||||
|
||||
_TOTAL_COUNT_SLOT = MissingSlot(
|
||||
slot_name="total_count",
|
||||
expected_unit_or_type="count_int",
|
||||
binding_target="collective_unit_total",
|
||||
)
|
||||
_WEIGHTED_SLOT = MissingSlot(
|
||||
slot_name="weighted_total",
|
||||
expected_unit_or_type="measured_unit_int",
|
||||
binding_target="weighted_total_value",
|
||||
)
|
||||
|
||||
|
||||
def test_pass_manager_uses_deliver_ask_for_renderable_ask() -> None:
|
||||
assessment = _make_assessment(
|
||||
blocking_reason="missing_total_count", slots=(_TOTAL_COUNT_SLOT,)
|
||||
)
|
||||
outcome = _delivery_outcome_for_limitation(assessment)
|
||||
assert outcome.terminal == Terminal.QUESTION_NEEDED
|
||||
assert outcome.question is not None
|
||||
assert outcome.fallback_reason is None
|
||||
|
||||
|
||||
def test_pass_manager_unrenderable_ask_falls_back_without_question_needed() -> None:
|
||||
# Multi-slot => unrenderable, falls back to standing disposition (PROPOSAL_EMITTED since missing_total_count is carve-out)
|
||||
assessment = _make_assessment(
|
||||
blocking_reason="missing_total_count",
|
||||
slots=(_TOTAL_COUNT_SLOT, _WEIGHTED_SLOT),
|
||||
)
|
||||
outcome = _delivery_outcome_for_limitation(assessment)
|
||||
assert outcome.terminal == Terminal.PROPOSAL_EMITTED
|
||||
assert outcome.terminal is not Terminal.QUESTION_NEEDED
|
||||
assert outcome.question is None
|
||||
assert outcome.fallback_reason == "multi_slot_not_supported"
|
||||
|
||||
|
||||
def test_pass_manager_does_not_import_or_call_render_question_directly() -> None:
|
||||
path = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "generate"
|
||||
/ "contemplation"
|
||||
/ "pass_manager.py"
|
||||
)
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
for node in ast.walk(tree):
|
||||
# Ensure render_question is not imported, and chat/chat.runtime is not imported
|
||||
if isinstance(node, ast.ImportFrom):
|
||||
assert node.module != "core.epistemic_questions.render"
|
||||
assert node.module != "chat.runtime"
|
||||
assert node.module != "chat"
|
||||
if node.names:
|
||||
for alias in node.names:
|
||||
assert alias.name != "render_question"
|
||||
elif isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
assert alias.name != "core.epistemic_questions.render"
|
||||
assert alias.name != "chat.runtime"
|
||||
assert alias.name != "chat"
|
||||
|
||||
# Ensure render_question is not called directly
|
||||
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
|
||||
assert node.func.id != "render_question"
|
||||
|
||||
|
||||
def test_pass_manager_does_not_construct_question_text() -> None:
|
||||
path = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "generate"
|
||||
/ "contemplation"
|
||||
/ "pass_manager.py"
|
||||
)
|
||||
content = path.read_text(encoding="utf-8")
|
||||
forbidden_templates = ["What ", "Which ", "How many", "Please provide"]
|
||||
for template in forbidden_templates:
|
||||
assert template not in content, (
|
||||
f"pass_manager.py must not construct prose templates like {template!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_q1b_carveout_preserved_during_pass_manager_ask_integration() -> None:
|
||||
assert "missing_total_count" in Q1B_ASK_CARVE_OUT
|
||||
assert "missing_weighted_total" in Q1B_ASK_CARVE_OUT
|
||||
|
||||
for name in Q1B_ASK_CARVE_OUT:
|
||||
family = family_by_name(name)
|
||||
assert family is not None
|
||||
assert family.proposal_allowed is True
|
||||
|
||||
|
||||
def test_renderable_ask_path_returns_question_needed_under_exercise_ask(monkeypatch, tmp_path) -> None:
|
||||
from core.comprehension_attempt import ComprehensionAttempt, RouteResult
|
||||
import generate.contemplation.pass_manager as pm
|
||||
from generate.binding_graph.model import SourceSpanLink
|
||||
|
||||
span = SourceSpanLink(source_id="src", start=0, end=8, text="chickens")
|
||||
attempt = ComprehensionAttempt(
|
||||
organ="r2_constraints",
|
||||
outcome="setup_refused",
|
||||
refusal_reason="missing_total_count",
|
||||
evidence=(span,),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(pm, "route_setup", lambda text, case_id=None: RouteResult((attempt,), None, "all_refused"))
|
||||
|
||||
question_root = tmp_path / "teaching" / "questions"
|
||||
proposal_root = tmp_path / "teaching" / "proposals"
|
||||
|
||||
repo_questions_dir = Path(__file__).resolve().parents[1] / "teaching" / "questions"
|
||||
before_files = set(repo_questions_dir.glob("**/*")) if repo_questions_dir.exists() else set()
|
||||
|
||||
# 1. Assert that without exercise_ask, it falls back to PROPOSAL_EMITTED (due to carve-out)
|
||||
res_normal = contemplate(
|
||||
"chickens",
|
||||
proposal_root=proposal_root,
|
||||
question_root=question_root,
|
||||
)
|
||||
assert res_normal.terminal == Terminal.PROPOSAL_EMITTED
|
||||
assert res_normal.proposal_path is not None
|
||||
assert res_normal.question_path is None
|
||||
assert proposal_root in Path(res_normal.proposal_path).parents
|
||||
|
||||
# 2. Assert that with exercise_ask=True, it returns QUESTION_NEEDED
|
||||
calls = []
|
||||
orig = pm._delivery_outcome_for_limitation
|
||||
def wrapped(assessment):
|
||||
calls.append(assessment)
|
||||
return orig(assessment)
|
||||
monkeypatch.setattr(pm, "_delivery_outcome_for_limitation", wrapped)
|
||||
|
||||
res_ask = contemplate(
|
||||
"chickens",
|
||||
proposal_root=proposal_root,
|
||||
question_root=question_root,
|
||||
exercise_ask=True,
|
||||
)
|
||||
assert res_ask.terminal == Terminal.QUESTION_NEEDED
|
||||
assert len(calls) == 1 # Verify it is called exactly once! No double delivery / double render!
|
||||
|
||||
# Verify the question artifact path
|
||||
assert res_ask.proposal_path is None
|
||||
assert res_ask.question_path is not None
|
||||
artifact_path = Path(res_ask.question_path)
|
||||
assert artifact_path.exists()
|
||||
|
||||
# Assert question artifact is under question_root
|
||||
assert question_root in artifact_path.parents
|
||||
# Assert question artifact is not under proposal_root
|
||||
assert proposal_root not in artifact_path.parents
|
||||
|
||||
# Assert no repo-local teaching/questions artifact is created during tests
|
||||
after_files = set(repo_questions_dir.glob("**/*")) if repo_questions_dir.exists() else set()
|
||||
assert before_files == after_files
|
||||
|
||||
|
||||
def test_unrenderable_ask_falls_back_in_pass_manager(monkeypatch, tmp_path) -> None:
|
||||
from core.comprehension_attempt import ComprehensionAttempt, RouteResult
|
||||
import generate.contemplation.pass_manager as pm
|
||||
from core.epistemic_questions.delivery import DeliveryOutcome
|
||||
|
||||
attempt = ComprehensionAttempt(
|
||||
organ="r2_constraints",
|
||||
outcome="setup_refused",
|
||||
refusal_reason="missing_total_count",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(pm, "route_setup", lambda text, case_id=None: RouteResult((attempt,), None, "all_refused"))
|
||||
monkeypatch.setattr(
|
||||
pm,
|
||||
"_delivery_outcome_for_limitation",
|
||||
lambda assessment: DeliveryOutcome(Terminal.PROPOSAL_EMITTED, None, "multi_slot_not_supported")
|
||||
)
|
||||
|
||||
question_root = tmp_path / "teaching" / "questions"
|
||||
proposal_root = tmp_path / "teaching" / "proposals"
|
||||
|
||||
res = contemplate(
|
||||
"chickens",
|
||||
proposal_root=proposal_root,
|
||||
question_root=question_root,
|
||||
exercise_ask=True,
|
||||
)
|
||||
assert res.terminal == Terminal.PROPOSAL_EMITTED
|
||||
assert res.terminal is not Terminal.QUESTION_NEEDED
|
||||
assert res.proposal_path is not None
|
||||
assert Path(res.proposal_path).exists()
|
||||
assert res.question_path is None
|
||||
|
||||
|
||||
def test_family_none_does_not_crash_ask_branch(monkeypatch, tmp_path) -> None:
|
||||
from core.comprehension_attempt import ComprehensionAttempt, RouteResult
|
||||
import generate.contemplation.pass_manager as pm
|
||||
from core.epistemic_disclosure.limitation import LimitationAssessment
|
||||
import core.epistemic_disclosure.limitation as lim_mod
|
||||
|
||||
fake_assessment = LimitationAssessment(
|
||||
limitation_kind="missing_information",
|
||||
resolution_action="ask_question",
|
||||
epistemic_state=EpistemicState.UNDETERMINED,
|
||||
owner_organ="r2_constraint",
|
||||
blocking_reason="nonexistent_family_name",
|
||||
)
|
||||
|
||||
attempt = ComprehensionAttempt(
|
||||
organ="r2_constraints",
|
||||
outcome="setup_refused",
|
||||
refusal_reason="nonexistent_family_name",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(pm, "route_setup", lambda text, case_id=None: RouteResult((attempt,), None, "all_refused"))
|
||||
monkeypatch.setattr(lim_mod, "assess_from_attempt", lambda att: fake_assessment)
|
||||
|
||||
question_root = tmp_path / "teaching" / "questions"
|
||||
proposal_root = tmp_path / "teaching" / "proposals"
|
||||
|
||||
res = contemplate(
|
||||
"chickens",
|
||||
proposal_root=proposal_root,
|
||||
question_root=question_root,
|
||||
exercise_ask=True,
|
||||
)
|
||||
assert res.terminal == Terminal.NO_PROGRESS
|
||||
|
||||
|
||||
def test_boundary_wins_over_ask_in_pass_manager(monkeypatch, tmp_path) -> None:
|
||||
from core.comprehension_attempt import ComprehensionAttempt, RouteResult
|
||||
import generate.contemplation.pass_manager as pm
|
||||
from generate.binding_graph.model import SourceSpanLink
|
||||
|
||||
span = SourceSpanLink(source_id="src", start=0, end=8, text="chickens")
|
||||
attempt_boundary = ComprehensionAttempt(
|
||||
organ="r2_constraints",
|
||||
outcome="setup_refused",
|
||||
refusal_reason="too_many_categories", # maps to unsupported_system_size (must_remain_refused = True)
|
||||
evidence=(span,),
|
||||
)
|
||||
attempt_ask = ComprehensionAttempt(
|
||||
organ="r2_constraints",
|
||||
outcome="setup_refused",
|
||||
refusal_reason="missing_total_count", # ask carve-out
|
||||
evidence=(span,),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
pm,
|
||||
"route_setup",
|
||||
lambda text, case_id=None: RouteResult((attempt_boundary, attempt_ask), None, "all_refused"),
|
||||
)
|
||||
|
||||
question_root = tmp_path / "teaching" / "questions"
|
||||
proposal_root = tmp_path / "teaching" / "proposals"
|
||||
|
||||
res = contemplate(
|
||||
"chickens",
|
||||
proposal_root=proposal_root,
|
||||
question_root=question_root,
|
||||
exercise_ask=True,
|
||||
)
|
||||
# Assert terminal remains REFUSED_UNSUPPORTED_FAMILY or REFUSED_KNOWN_BOUNDARY, not QUESTION_NEEDED
|
||||
assert res.terminal == Terminal.REFUSED_UNSUPPORTED_FAMILY
|
||||
assert res.question_path is None
|
||||
assert res.proposal_path is None
|
||||
# Ensure no question is written under question_root
|
||||
assert not question_root.exists() or len(list(question_root.glob("**/*"))) == 0
|
||||
|
||||
|
||||
def test_unrenderable_ask_falls_back_to_no_progress_in_pass_manager(monkeypatch, tmp_path) -> None:
|
||||
from core.comprehension_attempt import ComprehensionAttempt, RouteResult
|
||||
import generate.contemplation.pass_manager as pm
|
||||
from core.epistemic_questions.delivery import DeliveryOutcome
|
||||
|
||||
attempt = ComprehensionAttempt(
|
||||
organ="r2_constraints",
|
||||
outcome="setup_refused",
|
||||
refusal_reason="missing_total_count",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(pm, "route_setup", lambda text, case_id=None: RouteResult((attempt,), None, "all_refused"))
|
||||
monkeypatch.setattr(
|
||||
pm,
|
||||
"_delivery_outcome_for_limitation",
|
||||
lambda assessment: DeliveryOutcome(Terminal.NO_PROGRESS, None, "some_fallback_reason")
|
||||
)
|
||||
|
||||
question_root = tmp_path / "teaching" / "questions"
|
||||
proposal_root = tmp_path / "teaching" / "proposals"
|
||||
|
||||
res = contemplate(
|
||||
"chickens",
|
||||
proposal_root=proposal_root,
|
||||
question_root=question_root,
|
||||
exercise_ask=True,
|
||||
)
|
||||
assert res.terminal == Terminal.NO_PROGRESS
|
||||
assert res.proposal_path is None
|
||||
assert res.question_path is None
|
||||
47
tests/test_ask_serving_gate.py
Normal file
47
tests/test_ask_serving_gate.py
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
"""ASK serving gate — default-dark invariant.
|
||||
|
||||
This is deliberately narrower than serving integration. It proves the post-scoping
|
||||
kill-switch predicate is dark unless an operator/config object explicitly opts in.
|
||||
No chat/runtime wiring, no pass-manager emission, no carve-out retirement.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from core.config import DEFAULT_CONFIG, RuntimeConfig
|
||||
from core.epistemic_questions.serving_gate import ask_serving_enabled
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _LegacyConfig:
|
||||
"""A pre-field config shape: absence of the flag must mean dark."""
|
||||
|
||||
unrelated: bool = True
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _OptInConfig:
|
||||
ask_serving_enabled: bool
|
||||
|
||||
|
||||
def test_default_runtime_config_keeps_ask_serving_dark() -> None:
|
||||
assert hasattr(RuntimeConfig(), "ask_serving_enabled")
|
||||
assert RuntimeConfig().ask_serving_enabled is False
|
||||
assert DEFAULT_CONFIG.ask_serving_enabled is False
|
||||
assert ask_serving_enabled(DEFAULT_CONFIG) is False
|
||||
assert ask_serving_enabled(RuntimeConfig()) is False
|
||||
|
||||
|
||||
|
||||
def test_missing_flag_is_dark_for_legacy_config_shape() -> None:
|
||||
assert ask_serving_enabled(_LegacyConfig()) is False
|
||||
|
||||
|
||||
def test_gate_only_lights_on_explicit_truthy_opt_in() -> None:
|
||||
assert ask_serving_enabled(_OptInConfig(False)) is False
|
||||
assert ask_serving_enabled(_OptInConfig(True)) is True
|
||||
|
||||
|
||||
def test_none_uses_default_config_and_stays_dark() -> None:
|
||||
assert ask_serving_enabled() is False
|
||||
287
tests/test_ask_serving_integration.py
Normal file
287
tests/test_ask_serving_integration.py
Normal file
|
|
@ -0,0 +1,287 @@
|
|||
"""Focused tests for the Stage 2 served ASK artifact adapter.
|
||||
|
||||
These tests intentionally avoid ``chat.runtime``. This slice is adapter-only:
|
||||
it validates Q1-D question artifacts and returns a typed decision, but does not
|
||||
wire runtime acquisition of ``ContemplationResult``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from core.config import RuntimeConfig
|
||||
from core.epistemic_disclosure import ServedAskDecision, evaluate_served_ask
|
||||
from core.epistemic_disclosure.disposition import ServedDisposition, choose_served_disposition
|
||||
|
||||
|
||||
class DummyTerminal:
|
||||
def __init__(self, value: str):
|
||||
self.value = value
|
||||
|
||||
|
||||
class DummyContemplationResult:
|
||||
def __init__(
|
||||
self,
|
||||
terminal: str,
|
||||
*,
|
||||
question_path: str | None = None,
|
||||
proposal_path: str | None = None,
|
||||
family: str | None = None,
|
||||
) -> None:
|
||||
self.terminal = DummyTerminal(terminal)
|
||||
self.question_path = question_path
|
||||
self.proposal_path = proposal_path
|
||||
self.family = family
|
||||
|
||||
|
||||
def _write_artifact(path: Path, data: dict) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data), encoding="utf-8")
|
||||
|
||||
|
||||
def _valid_payload(text: str = "How many crates are left?") -> dict:
|
||||
return {
|
||||
"status": "question_only",
|
||||
"blocking_reason": "missing_total_count",
|
||||
"owner_organ": "r2_constraint",
|
||||
"question": {
|
||||
"text": text,
|
||||
"reason": "missing_total_count",
|
||||
"slot_name": "total_count",
|
||||
"expected_unit_or_type": "count_int",
|
||||
"binding_target": "collective_unit_total",
|
||||
},
|
||||
"answer_binding": None,
|
||||
"requires_review": True,
|
||||
"served": False,
|
||||
}
|
||||
|
||||
|
||||
def _question_result(q_path: Path, p_path: Path | None = None) -> DummyContemplationResult:
|
||||
return DummyContemplationResult(
|
||||
"QUESTION_NEEDED",
|
||||
question_path=str(q_path),
|
||||
proposal_path=str(p_path) if p_path is not None else None,
|
||||
)
|
||||
|
||||
|
||||
def test_ask_serving_disabled_preserves_existing_fallback_surface(tmp_path: Path) -> None:
|
||||
q_path = tmp_path / "questions" / "q.json"
|
||||
_write_artifact(q_path, _valid_payload())
|
||||
|
||||
decision = evaluate_served_ask(
|
||||
RuntimeConfig(ask_serving_enabled=False),
|
||||
_question_result(q_path),
|
||||
"fallback",
|
||||
)
|
||||
|
||||
assert isinstance(decision, ServedAskDecision)
|
||||
assert decision.served is False
|
||||
assert decision.terminal == "QUESTION_NEEDED"
|
||||
assert decision.surface == "fallback"
|
||||
assert decision.disposition is ServedDisposition.REFUSE
|
||||
|
||||
|
||||
def test_ask_serving_enabled_surfaces_question_needed_from_artifact(tmp_path: Path) -> None:
|
||||
q_path = tmp_path / "questions" / "q.json"
|
||||
p_path = tmp_path / "proposals" / "p.json"
|
||||
_write_artifact(q_path, _valid_payload("How many crates are left?"))
|
||||
|
||||
decision = evaluate_served_ask(
|
||||
RuntimeConfig(ask_serving_enabled=True),
|
||||
_question_result(q_path, p_path),
|
||||
"fallback",
|
||||
)
|
||||
|
||||
assert decision.served is True
|
||||
assert decision.terminal == "QUESTION_NEEDED"
|
||||
assert decision.surface == "How many crates are left?"
|
||||
assert decision.disposition is ServedDisposition.ASK
|
||||
|
||||
|
||||
def test_served_ask_uses_governance_disposition_bus(monkeypatch, tmp_path: Path) -> None:
|
||||
q_path = tmp_path / "questions" / "q.json"
|
||||
_write_artifact(q_path, _valid_payload())
|
||||
calls = []
|
||||
|
||||
def traced_choose_served_disposition(*args, **kwargs):
|
||||
calls.append((args, kwargs))
|
||||
return choose_served_disposition(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"core.epistemic_disclosure.ask_serving.choose_served_disposition",
|
||||
traced_choose_served_disposition,
|
||||
)
|
||||
|
||||
decision = evaluate_served_ask(
|
||||
RuntimeConfig(ask_serving_enabled=True),
|
||||
_question_result(q_path),
|
||||
"fallback",
|
||||
)
|
||||
|
||||
assert decision.served is True
|
||||
assert decision.disposition is ServedDisposition.ASK
|
||||
assert len(calls) == 1
|
||||
assert calls[0][1]["limitation"].resolution_action == "ask_question"
|
||||
|
||||
|
||||
def test_question_path_must_not_equal_proposal_path(tmp_path: Path) -> None:
|
||||
q_path = tmp_path / "questions" / "same.json"
|
||||
_write_artifact(q_path, _valid_payload())
|
||||
|
||||
decision = evaluate_served_ask(
|
||||
RuntimeConfig(ask_serving_enabled=True),
|
||||
_question_result(q_path, q_path),
|
||||
"fallback",
|
||||
)
|
||||
|
||||
assert decision.served is False
|
||||
assert decision.surface == "fallback"
|
||||
|
||||
|
||||
def test_non_question_needed_terminal_preserves_proposal_signal(tmp_path: Path) -> None:
|
||||
q_path = tmp_path / "questions" / "q.json"
|
||||
_write_artifact(q_path, _valid_payload())
|
||||
result = DummyContemplationResult(
|
||||
"PROPOSAL_EMITTED",
|
||||
question_path=str(q_path),
|
||||
proposal_path=str(tmp_path / "proposals" / "p.json"),
|
||||
)
|
||||
|
||||
decision = evaluate_served_ask(
|
||||
RuntimeConfig(ask_serving_enabled=True),
|
||||
result,
|
||||
"fallback",
|
||||
)
|
||||
|
||||
assert decision.served is False
|
||||
assert decision.terminal == "PROPOSAL_EMITTED"
|
||||
assert decision.surface == "fallback"
|
||||
assert decision.disposition is ServedDisposition.PROPOSE
|
||||
|
||||
|
||||
def test_missing_or_unreadable_question_artifact_fails_closed(tmp_path: Path) -> None:
|
||||
missing = tmp_path / "questions" / "missing.json"
|
||||
|
||||
decision = evaluate_served_ask(
|
||||
RuntimeConfig(ask_serving_enabled=True),
|
||||
_question_result(missing),
|
||||
"fallback",
|
||||
)
|
||||
|
||||
assert decision.served is False
|
||||
assert decision.surface == "fallback"
|
||||
|
||||
|
||||
def test_malformed_question_artifact_fails_closed(tmp_path: Path) -> None:
|
||||
q_path = tmp_path / "questions" / "q.json"
|
||||
q_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
q_path.write_text("{bad json", encoding="utf-8")
|
||||
|
||||
decision = evaluate_served_ask(
|
||||
RuntimeConfig(ask_serving_enabled=True),
|
||||
_question_result(q_path),
|
||||
"fallback",
|
||||
)
|
||||
|
||||
assert decision.served is False
|
||||
assert decision.surface == "fallback"
|
||||
|
||||
|
||||
def test_rejects_artifact_status_proposal_only(tmp_path: Path) -> None:
|
||||
q_path = tmp_path / "questions" / "q.json"
|
||||
data = _valid_payload()
|
||||
data["status"] = "proposal_only"
|
||||
_write_artifact(q_path, data)
|
||||
|
||||
decision = evaluate_served_ask(RuntimeConfig(ask_serving_enabled=True), _question_result(q_path), "fallback")
|
||||
|
||||
assert decision.served is False
|
||||
assert decision.surface == "fallback"
|
||||
|
||||
|
||||
def test_rejects_artifact_served_true(tmp_path: Path) -> None:
|
||||
q_path = tmp_path / "questions" / "q.json"
|
||||
data = _valid_payload()
|
||||
data["served"] = True
|
||||
_write_artifact(q_path, data)
|
||||
|
||||
decision = evaluate_served_ask(RuntimeConfig(ask_serving_enabled=True), _question_result(q_path), "fallback")
|
||||
|
||||
assert decision.served is False
|
||||
|
||||
|
||||
def test_rejects_artifact_requires_review_false(tmp_path: Path) -> None:
|
||||
q_path = tmp_path / "questions" / "q.json"
|
||||
data = _valid_payload()
|
||||
data["requires_review"] = False
|
||||
_write_artifact(q_path, data)
|
||||
|
||||
decision = evaluate_served_ask(RuntimeConfig(ask_serving_enabled=True), _question_result(q_path), "fallback")
|
||||
|
||||
assert decision.served is False
|
||||
|
||||
|
||||
def test_rejects_artifact_non_null_answer_binding(tmp_path: Path) -> None:
|
||||
q_path = tmp_path / "questions" / "q.json"
|
||||
data = _valid_payload()
|
||||
data["answer_binding"] = {"slot": "total_count", "value": 12}
|
||||
_write_artifact(q_path, data)
|
||||
|
||||
decision = evaluate_served_ask(RuntimeConfig(ask_serving_enabled=True), _question_result(q_path), "fallback")
|
||||
|
||||
assert decision.served is False
|
||||
|
||||
|
||||
def test_rejects_missing_or_blank_question_text(tmp_path: Path) -> None:
|
||||
for value in (None, ""):
|
||||
q_path = tmp_path / f"questions_{value!r}" / "q.json"
|
||||
data = _valid_payload()
|
||||
if value is None:
|
||||
del data["question"]["text"]
|
||||
else:
|
||||
data["question"]["text"] = " "
|
||||
_write_artifact(q_path, data)
|
||||
|
||||
decision = evaluate_served_ask(RuntimeConfig(ask_serving_enabled=True), _question_result(q_path), "fallback")
|
||||
|
||||
assert decision.served is False
|
||||
assert decision.surface == "fallback"
|
||||
|
||||
|
||||
def test_rejects_missing_or_blank_slot_name(tmp_path: Path) -> None:
|
||||
for value in (None, ""):
|
||||
q_path = tmp_path / f"questions_slot_{value!r}" / "q.json"
|
||||
data = _valid_payload()
|
||||
if value is None:
|
||||
del data["question"]["slot_name"]
|
||||
else:
|
||||
data["question"]["slot_name"] = " "
|
||||
_write_artifact(q_path, data)
|
||||
|
||||
decision = evaluate_served_ask(RuntimeConfig(ask_serving_enabled=True), _question_result(q_path), "fallback")
|
||||
|
||||
assert decision.served is False
|
||||
assert decision.surface == "fallback"
|
||||
|
||||
|
||||
def test_adapter_does_not_import_question_renderer_or_construct_prose() -> None:
|
||||
path = Path(__file__).resolve().parents[1] / "core" / "epistemic_disclosure" / "ask_serving.py"
|
||||
source = path.read_text(encoding="utf-8")
|
||||
tree = ast.parse(source, filename=str(path))
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom) and node.module:
|
||||
assert node.module != "core.epistemic_questions.render"
|
||||
assert not node.module.startswith("core.epistemic_questions.render.")
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
assert alias.name != "core.epistemic_questions.render"
|
||||
assert not alias.name.startswith("core.epistemic_questions.render.")
|
||||
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
|
||||
assert node.func.id != "render_question"
|
||||
|
||||
for forbidden_template in ("How many", "What ", "Which ", "Please provide"):
|
||||
assert forbidden_template not in source
|
||||
31
tests/test_cross_serving_gates.py
Normal file
31
tests/test_cross_serving_gates.py
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
"""Cross-gate configuration field tests.
|
||||
|
||||
Verifies that RuntimeConfig has both ask_serving_enabled and
|
||||
verified_serving_enabled, that both are default false, and that setting
|
||||
one does not implicitly enable the other.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
|
||||
from core.config import RuntimeConfig
|
||||
|
||||
|
||||
def test_cross_serving_gates_independence() -> None:
|
||||
# 1. Verify both are present on a fresh config and default to False
|
||||
config = RuntimeConfig()
|
||||
assert hasattr(config, "ask_serving_enabled")
|
||||
assert hasattr(config, "verified_serving_enabled")
|
||||
assert config.ask_serving_enabled is False
|
||||
assert config.verified_serving_enabled is False
|
||||
|
||||
# 2. Verify setting ask_serving_enabled to True does not enable verified_serving_enabled
|
||||
config_ask = dataclasses.replace(config, ask_serving_enabled=True)
|
||||
assert config_ask.ask_serving_enabled is True
|
||||
assert config_ask.verified_serving_enabled is False
|
||||
|
||||
# 3. Verify setting verified_serving_enabled to True does not enable ask_serving_enabled
|
||||
config_verified = dataclasses.replace(config, verified_serving_enabled=True)
|
||||
assert config_verified.ask_serving_enabled is False
|
||||
assert config_verified.verified_serving_enabled is True
|
||||
206
tests/test_epistemic_question_render.py
Normal file
206
tests/test_epistemic_question_render.py
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
"""Q1-C — the grounded-only question renderer (off-serving).
|
||||
|
||||
Pins the wrong=0 grounded-rendering invariant (scoping §2 / session §1.5.7): a
|
||||
rendered question may name a problem entity only if it appears verbatim in
|
||||
``grounded_terms``; otherwise the renderer degrades to a generic structural
|
||||
question or emits ``question_unrenderable``. ``grounded_terms`` is empty
|
||||
everywhere in production today, so the renderable path is exercised here with the
|
||||
closed structural-type vocabulary only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
from core.epistemic_disclosure.limitation import (
|
||||
LimitationAssessment,
|
||||
MissingSlot,
|
||||
)
|
||||
from core.epistemic_questions import EpistemicQuestion, render_question
|
||||
from core.epistemic_questions.render import (
|
||||
_REASON_MULTI_SLOT,
|
||||
_REASON_NO_SLOT,
|
||||
_REASON_NOT_ASK,
|
||||
_REASON_RENDERABILITY_GAP,
|
||||
_names_only_grounded,
|
||||
)
|
||||
from core.epistemic_state import EpistemicState
|
||||
|
||||
|
||||
def _ask_assessment(
|
||||
slots: tuple[MissingSlot, ...],
|
||||
grounded_terms: tuple[str, ...] = (),
|
||||
) -> LimitationAssessment:
|
||||
"""An ``ask_question`` assessment carrying the given typed residue."""
|
||||
return LimitationAssessment(
|
||||
limitation_kind="missing_information",
|
||||
resolution_action="ask_question",
|
||||
epistemic_state=EpistemicState.UNDETERMINED,
|
||||
owner_organ="r2_constraint",
|
||||
blocking_reason="missing_total_count",
|
||||
missing_slots=slots,
|
||||
grounded_terms=grounded_terms,
|
||||
)
|
||||
|
||||
|
||||
_TOTAL_COUNT_SLOT = MissingSlot(
|
||||
slot_name="total_count",
|
||||
expected_unit_or_type="count_int",
|
||||
binding_target="collective_unit_total",
|
||||
)
|
||||
|
||||
|
||||
# --- the wrong=0 invariant: nothing named that is not grounded -----------------
|
||||
|
||||
|
||||
def test_ask_with_slot_and_empty_grounded_renders_generic_without_ungrounded_tokens() -> None:
|
||||
"""ASK + one slot + empty grounded_terms → a generic question whose every
|
||||
token is closed-vocab scaffold (nothing absent from grounded_terms is named).
|
||||
"""
|
||||
q = render_question(_ask_assessment((_TOTAL_COUNT_SLOT,)))
|
||||
|
||||
assert isinstance(q, EpistemicQuestion)
|
||||
assert not q.unrenderable
|
||||
assert q.slot == _TOTAL_COUNT_SLOT
|
||||
assert q.text is not None
|
||||
# The whole point: with empty grounded_terms, the text names nothing beyond
|
||||
# the closed structural vocabulary.
|
||||
assert _names_only_grounded(q.text, ())
|
||||
# The closed type phrase for ``count_int`` is conveyed.
|
||||
assert "whole-number count" in q.text
|
||||
|
||||
|
||||
def test_rendered_text_never_surfaces_snake_case_identifiers() -> None:
|
||||
"""Neither ``slot_name`` nor ``binding_target`` (snake_case) reaches prose."""
|
||||
q = render_question(_ask_assessment((_TOTAL_COUNT_SLOT,)))
|
||||
|
||||
assert q.text is not None
|
||||
assert "total_count" not in q.text
|
||||
assert "collective_unit_total" not in q.text
|
||||
assert "count_int" not in q.text # the type is translated, never raw
|
||||
|
||||
|
||||
# --- the fabrication guard: snake_case must never become an entity -------------
|
||||
|
||||
|
||||
def test_fabrication_guard_slot_name_ben_rate_yields_no_named_ben() -> None:
|
||||
"""A slot whose ``slot_name``/``binding_target`` is ``ben_rate`` must NOT
|
||||
produce a question naming "Ben" — slot_name and binding_target are never
|
||||
surfaced, prettified, or capitalized.
|
||||
"""
|
||||
ben_slot = MissingSlot(
|
||||
slot_name="ben_rate",
|
||||
expected_unit_or_type="count_int", # mapped → renders
|
||||
binding_target="ben_rate",
|
||||
)
|
||||
q = render_question(_ask_assessment((ben_slot,)))
|
||||
|
||||
assert q.text is not None # renders (type is mapped)
|
||||
assert "Ben" not in q.text
|
||||
assert "ben" not in _tokens_lower(q.text)
|
||||
assert "ben_rate" not in q.text
|
||||
|
||||
|
||||
def test_fabrication_guard_predicate_rejects_ungrounded_entity() -> None:
|
||||
"""The guard predicate itself flags a fabricated name when not grounded, and
|
||||
admits it once it is grounded verbatim.
|
||||
"""
|
||||
assert not _names_only_grounded("What is Ben rate?", ())
|
||||
assert _names_only_grounded("What is Ben rate?", ("Ben", "rate"))
|
||||
|
||||
|
||||
# --- non-ASK / no-slot / multi-slot / unmapped-type ---------------------------
|
||||
|
||||
|
||||
def test_non_ask_assessment_is_unrenderable() -> None:
|
||||
refuse = LimitationAssessment(
|
||||
limitation_kind="hard_boundary",
|
||||
resolution_action="refuse_known_boundary",
|
||||
epistemic_state=EpistemicState.UNDETERMINED,
|
||||
owner_organ="r2_constraint",
|
||||
blocking_reason="non_integer_solution",
|
||||
missing_slots=(_TOTAL_COUNT_SLOT,), # present, but action is not ask
|
||||
)
|
||||
q = render_question(refuse)
|
||||
|
||||
assert q.unrenderable
|
||||
assert q.text is None
|
||||
assert q.slot is None
|
||||
assert q.reason == _REASON_NOT_ASK
|
||||
|
||||
|
||||
def test_ask_with_zero_slots_is_unrenderable() -> None:
|
||||
q = render_question(_ask_assessment(()))
|
||||
|
||||
assert q.unrenderable
|
||||
assert q.text is None
|
||||
assert q.slot is None
|
||||
assert q.reason == _REASON_NO_SLOT
|
||||
|
||||
|
||||
def test_multi_slot_does_not_claim_all_missing_information_was_asked() -> None:
|
||||
"""Two missing slots → refuse with ``multi_slot_not_supported``.
|
||||
|
||||
The template asserts "one more value is still needed" (exactly one). With two
|
||||
slots that claim is false, so the renderer must NOT render the first and drop
|
||||
the second — it refuses outright, naming no slot. This is the wrong=0-honest
|
||||
choice: Q1-C is strictly single-slot, not first-of-many.
|
||||
"""
|
||||
second = MissingSlot(
|
||||
slot_name="weighted_total",
|
||||
expected_unit_or_type="measured_unit_int",
|
||||
binding_target="weighted_total_value",
|
||||
)
|
||||
q = render_question(_ask_assessment((_TOTAL_COUNT_SLOT, second)))
|
||||
|
||||
assert q.unrenderable
|
||||
assert q.text is None
|
||||
assert q.slot is None
|
||||
assert q.reason == _REASON_MULTI_SLOT
|
||||
|
||||
|
||||
def test_unmapped_structural_type_degrades_to_renderability_gap() -> None:
|
||||
"""An unknown ``expected_unit_or_type`` refuses rather than dumping raw
|
||||
snake_case — the slot is bound but no text is produced.
|
||||
"""
|
||||
exotic = MissingSlot(
|
||||
slot_name="mystery",
|
||||
expected_unit_or_type="some_unmapped_type",
|
||||
binding_target="mystery_target",
|
||||
)
|
||||
q = render_question(_ask_assessment((exotic,)))
|
||||
|
||||
assert q.unrenderable
|
||||
assert q.text is None
|
||||
assert q.slot == exotic
|
||||
assert q.reason == _REASON_RENDERABILITY_GAP
|
||||
|
||||
|
||||
# --- off-serving structural guard ---------------------------------------------
|
||||
|
||||
|
||||
def test_renderer_is_off_serving() -> None:
|
||||
"""The renderer must not import the sealed GSM8K serving substrate."""
|
||||
src = Path(__file__).resolve().parents[1] / "core" / "epistemic_questions"
|
||||
forbidden = ("generate.derivation", "core.reliability_gate")
|
||||
for path in src.glob("*.py"):
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom) and node.module:
|
||||
assert not any(
|
||||
node.module == f or node.module.startswith(f + ".")
|
||||
for f in forbidden
|
||||
), f"{path.name} imports forbidden serving module {node.module}"
|
||||
elif isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
assert not any(
|
||||
alias.name == f or alias.name.startswith(f + ".")
|
||||
for f in forbidden
|
||||
), f"{path.name} imports forbidden serving module {alias.name}"
|
||||
|
||||
|
||||
def _tokens_lower(text: str) -> set[str]:
|
||||
import re
|
||||
|
||||
return set(re.findall(r"[a-z]+", text.lower()))
|
||||
|
|
@ -368,22 +368,22 @@ def test_epistemic_state_axis():
|
|||
assert by_name["cmb_underdetermined"].epistemic_state == EpistemicState.UNDETERMINED
|
||||
|
||||
|
||||
# --- the consolidation proof: actions live on shipped terminals (except ask) --------- #
|
||||
# --- the consolidation proof: every action lives on a shipped terminal --------------- #
|
||||
|
||||
def test_actions_consolidate_onto_terminals_except_ask():
|
||||
def test_actions_consolidate_onto_terminals():
|
||||
"""The proof of 'consolidating view, not a fourth taxonomy': every action maps to a
|
||||
shipped Terminal except ask_question (the one genuinely new action / Q1 tenant)."""
|
||||
shipped Terminal. Through Q1-C ask_question was the lone exception (no terminal yet);
|
||||
Q1-D ships QUESTION_NEEDED, so the map is now total — no action lacks a terminal."""
|
||||
for action in VALID_ACTIONS:
|
||||
terminal = terminal_for_action(action)
|
||||
if action == "ask_question":
|
||||
assert terminal is None
|
||||
else:
|
||||
assert isinstance(terminal, Terminal), action
|
||||
assert isinstance(terminal_for_action(action), Terminal), action
|
||||
|
||||
|
||||
def test_only_ask_question_is_new():
|
||||
new_actions = sorted(a for a in VALID_ACTIONS if terminal_for_action(a) is None)
|
||||
assert new_actions == ["ask_question"]
|
||||
def test_ask_question_resolves_to_question_needed():
|
||||
"""Q1-D: ask_question's home terminal is QUESTION_NEEDED (the ASK tenant), the one
|
||||
terminal the spine added to complete the consolidation."""
|
||||
assert terminal_for_action("ask_question") == Terminal.QUESTION_NEEDED
|
||||
# And no action is left without a terminal.
|
||||
assert [a for a in VALID_ACTIONS if terminal_for_action(a) is None] == []
|
||||
|
||||
|
||||
# --- attempt-level classification ---------------------------------------------------- #
|
||||
|
|
|
|||
239
tests/test_question_delivery.py
Normal file
239
tests/test_question_delivery.py
Normal file
|
|
@ -0,0 +1,239 @@
|
|||
"""Q1-D — off-serving ASK delivery (`QUESTION_NEEDED` tenant).
|
||||
|
||||
Pins the delivery rung: a renderable ``ask_question`` assessment routes to
|
||||
``Terminal.QUESTION_NEEDED`` carrying the Q1-C question *verbatim*; an unrenderable
|
||||
one takes the D2 standing-disposition fallback (``PROPOSAL_EMITTED`` if the family
|
||||
still proposes, else ``NO_PROGRESS``) and writes NO artifact. Each guard is written so
|
||||
it FAILS under the violation it nominally proves (CLAUDE.md schema-obligation
|
||||
discipline): a contentless ``QUESTION_NEEDED`` is structurally impossible, a delivered
|
||||
question is never served, and ``answer_binding`` is always the reserved ``None``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from core.epistemic_disclosure.limitation import (
|
||||
LimitationAssessment,
|
||||
MissingSlot,
|
||||
)
|
||||
from core.epistemic_questions import (
|
||||
AnswerBinding,
|
||||
DeliveredQuestion,
|
||||
deliver_ask,
|
||||
emit_question,
|
||||
question_path,
|
||||
render_question,
|
||||
)
|
||||
from core.epistemic_state import EpistemicState
|
||||
from generate.contemplation.findings import Terminal
|
||||
|
||||
|
||||
def _ask(
|
||||
*,
|
||||
blocking_reason: str,
|
||||
slots: tuple[MissingSlot, ...],
|
||||
kind: str = "missing_information",
|
||||
) -> LimitationAssessment:
|
||||
"""An ``ask_question`` assessment with a chosen blocking family + typed residue."""
|
||||
return LimitationAssessment(
|
||||
limitation_kind=kind, # type: ignore[arg-type]
|
||||
resolution_action="ask_question",
|
||||
epistemic_state=EpistemicState.UNDETERMINED,
|
||||
owner_organ="r2_constraint",
|
||||
blocking_reason=blocking_reason,
|
||||
missing_slots=slots,
|
||||
)
|
||||
|
||||
|
||||
_TOTAL_COUNT_SLOT = MissingSlot(
|
||||
slot_name="total_count",
|
||||
expected_unit_or_type="count_int",
|
||||
binding_target="collective_unit_total",
|
||||
)
|
||||
_WEIGHTED_SLOT = MissingSlot(
|
||||
slot_name="weighted_total",
|
||||
expected_unit_or_type="measured_unit_int",
|
||||
binding_target="weighted_total_value",
|
||||
)
|
||||
|
||||
|
||||
# --- the happy path: renderable ask → QUESTION_NEEDED, verbatim ----------------------- #
|
||||
|
||||
|
||||
def test_renderable_ask_delivers_question_needed() -> None:
|
||||
outcome = deliver_ask(_ask(blocking_reason="missing_total_count", slots=(_TOTAL_COUNT_SLOT,)))
|
||||
|
||||
assert outcome.terminal == Terminal.QUESTION_NEEDED
|
||||
assert outcome.fallback_reason is None
|
||||
assert outcome.question is not None
|
||||
assert outcome.question.blocking_reason == "missing_total_count"
|
||||
assert outcome.question.owner_organ == "r2_constraint"
|
||||
assert outcome.question.answer_binding is None # reserved (Q2)
|
||||
|
||||
|
||||
def test_delivered_question_wraps_the_q1c_render_verbatim() -> None:
|
||||
"""Q1-D consumes — it does not re-render. The wrapped question must be exactly what
|
||||
the Q1-C renderer returns for the same assessment (no second prose surface)."""
|
||||
assessment = _ask(blocking_reason="missing_total_count", slots=(_TOTAL_COUNT_SLOT,))
|
||||
outcome = deliver_ask(assessment)
|
||||
|
||||
assert outcome.question is not None
|
||||
assert outcome.question.question == render_question(assessment)
|
||||
|
||||
|
||||
# --- D2: the unrenderable fallback (never a contentless QUESTION_NEEDED) --------------- #
|
||||
|
||||
|
||||
def test_unrenderable_ask_falls_back_to_proposal_for_proposing_family() -> None:
|
||||
"""Multi-slot ⇒ Q1-C refuses (multi_slot_not_supported). The family still proposes
|
||||
(missing_total_count is a carve-out, proposal_allowed=True), so the standing
|
||||
disposition is PROPOSAL_EMITTED — NOT a contentless QUESTION_NEEDED."""
|
||||
outcome = deliver_ask(
|
||||
_ask(blocking_reason="missing_total_count", slots=(_TOTAL_COUNT_SLOT, _WEIGHTED_SLOT))
|
||||
)
|
||||
|
||||
assert outcome.terminal == Terminal.PROPOSAL_EMITTED
|
||||
assert outcome.question is None
|
||||
assert outcome.fallback_reason == "multi_slot_not_supported"
|
||||
|
||||
|
||||
def test_unrenderable_ask_falls_back_to_no_progress_for_nonproposing_family() -> None:
|
||||
"""Zero slots ⇒ Q1-C refuses (no_missing_slot). cmb_underdetermined does not propose
|
||||
(must_remain_refused), so the standing disposition is NO_PROGRESS."""
|
||||
outcome = deliver_ask(_ask(blocking_reason="cmb_underdetermined", slots=()))
|
||||
|
||||
assert outcome.terminal == Terminal.NO_PROGRESS
|
||||
assert outcome.question is None
|
||||
assert outcome.fallback_reason == "no_missing_slot"
|
||||
|
||||
|
||||
def test_unmapped_type_unrenderable_also_falls_back() -> None:
|
||||
"""An unmapped structural type ⇒ Q1-C renderability_gap ⇒ fallback, never delivered."""
|
||||
exotic = MissingSlot(
|
||||
slot_name="mystery", expected_unit_or_type="some_unmapped_type", binding_target="x"
|
||||
)
|
||||
outcome = deliver_ask(_ask(blocking_reason="missing_total_count", slots=(exotic,)))
|
||||
|
||||
assert outcome.terminal == Terminal.PROPOSAL_EMITTED # proposing family
|
||||
assert outcome.question is None
|
||||
assert outcome.fallback_reason == "renderability_gap"
|
||||
|
||||
|
||||
# --- structural guards: illegal DeliveredQuestion states are unrepresentable ----------- #
|
||||
|
||||
|
||||
def test_delivered_question_cannot_wrap_unrenderable() -> None:
|
||||
"""The D2 guard, structurally: a DeliveredQuestion can NEVER wrap an unrenderable
|
||||
question — so a QUESTION_NEEDED terminal always carries real rendered text."""
|
||||
unrenderable = render_question(_ask(blocking_reason="cmb_underdetermined", slots=()))
|
||||
assert unrenderable.unrenderable # precondition
|
||||
|
||||
with pytest.raises(ValueError, match="unrenderable"):
|
||||
DeliveredQuestion(
|
||||
question=unrenderable, owner_organ="r2_constraint", blocking_reason="x"
|
||||
)
|
||||
|
||||
|
||||
def test_delivered_question_is_never_served() -> None:
|
||||
rendered = render_question(_ask(blocking_reason="missing_total_count", slots=(_TOTAL_COUNT_SLOT,)))
|
||||
with pytest.raises(ValueError, match="never served"):
|
||||
DeliveredQuestion(
|
||||
question=rendered, owner_organ="o", blocking_reason="missing_total_count", served=True
|
||||
)
|
||||
|
||||
|
||||
def test_answer_binding_is_reserved_and_rejected_in_q1d() -> None:
|
||||
rendered = render_question(_ask(blocking_reason="missing_total_count", slots=(_TOTAL_COUNT_SLOT,)))
|
||||
binding = AnswerBinding(
|
||||
target_organ="o", target_slot="total_count", binding_target="collective_unit_total", parser="int"
|
||||
)
|
||||
with pytest.raises(ValueError, match="reserved for Q2"):
|
||||
DeliveredQuestion(
|
||||
question=rendered,
|
||||
owner_organ="o",
|
||||
blocking_reason="missing_total_count",
|
||||
answer_binding=binding,
|
||||
)
|
||||
|
||||
|
||||
def test_deliver_ask_rejects_non_ask_assessment() -> None:
|
||||
refuse = LimitationAssessment(
|
||||
limitation_kind="hard_boundary",
|
||||
resolution_action="refuse_known_boundary",
|
||||
epistemic_state=EpistemicState.UNDETERMINED,
|
||||
owner_organ="r2_constraint",
|
||||
blocking_reason="non_integer_solution",
|
||||
)
|
||||
with pytest.raises(ValueError, match="only valid for ask_question"):
|
||||
deliver_ask(refuse)
|
||||
|
||||
|
||||
# --- the sink: proposal-only, idempotent, no artifact on fallback ---------------------- #
|
||||
|
||||
|
||||
def test_emit_question_writes_question_only_artifact_idempotently(tmp_path: Path) -> None:
|
||||
assessment = _ask(blocking_reason="missing_total_count", slots=(_TOTAL_COUNT_SLOT,))
|
||||
path1 = emit_question(assessment, root=tmp_path)
|
||||
|
||||
assert path1 is not None and path1.exists()
|
||||
doc = json.loads(path1.read_text(encoding="utf-8"))
|
||||
assert doc["status"] == "question_only"
|
||||
assert doc["served"] is False
|
||||
assert doc["requires_review"] is True
|
||||
assert doc["answer_binding"] is None
|
||||
assert doc["blocking_reason"] == "missing_total_count"
|
||||
assert doc["question"]["text"] # real rendered text, not empty
|
||||
|
||||
first_bytes = path1.read_bytes()
|
||||
path2 = emit_question(assessment, root=tmp_path) # idempotent
|
||||
assert path2 == path1
|
||||
assert path2.read_bytes() == first_bytes
|
||||
|
||||
|
||||
def test_emit_question_writes_nothing_on_unrenderable_fallback(tmp_path: Path) -> None:
|
||||
"""No contentless artifact: an unrenderable ask writes no file and returns None."""
|
||||
path = emit_question(_ask(blocking_reason="cmb_underdetermined", slots=()), root=tmp_path)
|
||||
|
||||
assert path is None
|
||||
assert list(tmp_path.glob("*.json")) == []
|
||||
|
||||
|
||||
def test_question_path_is_content_addressed(tmp_path: Path) -> None:
|
||||
assessment = _ask(blocking_reason="missing_total_count", slots=(_TOTAL_COUNT_SLOT,))
|
||||
outcome = deliver_ask(assessment)
|
||||
assert outcome.question is not None
|
||||
p = question_path(outcome.question, tmp_path)
|
||||
assert p.parent == tmp_path
|
||||
assert p.name.endswith(".json")
|
||||
assert len(p.stem) == 64 # sha256 hex
|
||||
|
||||
|
||||
# --- off-serving structural guard ----------------------------------------------------- #
|
||||
|
||||
|
||||
def test_delivery_is_off_serving() -> None:
|
||||
"""The delivery module must not import the sealed GSM8K serving substrate."""
|
||||
path = Path(__file__).resolve().parents[1] / "core" / "epistemic_questions" / "delivery.py"
|
||||
forbidden = ("generate.derivation", "core.reliability_gate")
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom) and node.module:
|
||||
assert not any(
|
||||
node.module == f or node.module.startswith(f + ".") for f in forbidden
|
||||
), f"delivery.py imports forbidden serving module {node.module}"
|
||||
elif isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
assert not any(
|
||||
alias.name == f or alias.name.startswith(f + ".") for f in forbidden
|
||||
), f"delivery.py imports forbidden serving module {alias.name}"
|
||||
|
||||
|
||||
def test_question_needed_is_a_distinct_terminal() -> None:
|
||||
"""QUESTION_NEEDED is a real, distinct terminal — sibling of PROPOSAL_EMITTED."""
|
||||
assert Terminal.QUESTION_NEEDED.value == "QUESTION_NEEDED"
|
||||
assert Terminal.QUESTION_NEEDED is not Terminal.PROPOSAL_EMITTED
|
||||
|
|
@ -23,6 +23,7 @@ from core.epistemic_disclosure.verified_contract import (
|
|||
REASON_CONTRADICTION_PRESENT,
|
||||
REASON_INCOMPLETE_PROOF,
|
||||
REASON_NO_BACK_SUBSTITUTION,
|
||||
REASON_NO_BOUND_SLOTS,
|
||||
REASON_READS_DISAGREE,
|
||||
REASON_READS_NOT_INDEPENDENT,
|
||||
REASON_UNRESOLVED_LIMITATION,
|
||||
|
|
@ -46,6 +47,7 @@ def _valid_proof(**overrides) -> VerificationProof:
|
|||
primary_read_digest="canonical_structure_C",
|
||||
independent_read_digest="canonical_structure_C",
|
||||
derivation_digest="deriv#1",
|
||||
bound_slots_digest="bound#1",
|
||||
back_substitution_digest="backsub#1",
|
||||
boundary_clear=True,
|
||||
contradiction_clear=True,
|
||||
|
|
@ -96,6 +98,27 @@ def test_verified_requires_back_substitution():
|
|||
assert REASON_NO_BACK_SUBSTITUTION in result.failed_checks
|
||||
|
||||
|
||||
def test_verified_requires_bound_slots():
|
||||
result = evaluate_verification(_valid_proof(bound_slots_digest=""), limitation=None)
|
||||
assert result.verdict is VerificationVerdict.NOT_VERIFIED
|
||||
assert REASON_NO_BOUND_SLOTS in result.failed_checks
|
||||
|
||||
|
||||
def test_derivation_digest_alone_is_insufficient_without_bound_slots():
|
||||
"""P1-C hardening: a complete derivation does NOT imply the answer bound to a STATED
|
||||
slot — they are SEPARATE obligations. A proof with a derivation but no bound-slots must
|
||||
not verify; and relaxing exactly the bound-slots obligation stops that check firing
|
||||
(so the obligation is load-bearing, not decoration)."""
|
||||
proof = _valid_proof(derivation_digest="deriv#1", bound_slots_digest="")
|
||||
canonical = evaluate_verification(proof, limitation=None)
|
||||
assert canonical.verdict is VerificationVerdict.NOT_VERIFIED
|
||||
assert REASON_NO_BOUND_SLOTS in canonical.failed_checks
|
||||
|
||||
relaxed = replace(VERIFICATION_OBLIGATION, requires_bound_slots=False)
|
||||
relaxed_result = evaluate_verification(proof, limitation=None, obligation=relaxed)
|
||||
assert REASON_NO_BOUND_SLOTS not in relaxed_result.failed_checks
|
||||
|
||||
|
||||
def test_verified_requires_boundary_clear():
|
||||
result = evaluate_verification(_valid_proof(boundary_clear=False), limitation=None)
|
||||
assert result.verdict is VerificationVerdict.NOT_VERIFIED
|
||||
|
|
@ -133,6 +156,7 @@ def test_verified_rejects_any_limitation_assessment():
|
|||
def test_canonical_obligation_is_fully_strict():
|
||||
assert VERIFICATION_OBLIGATION.requires_independent_read
|
||||
assert VERIFICATION_OBLIGATION.rejects_wrong_read_even_if_solved
|
||||
assert VERIFICATION_OBLIGATION.requires_bound_slots
|
||||
assert VERIFICATION_OBLIGATION.requires_back_substitution
|
||||
assert VERIFICATION_OBLIGATION.requires_boundary_clear
|
||||
|
||||
|
|
|
|||
|
|
@ -173,6 +173,7 @@ def test_all_proof_digests_populated_on_verified():
|
|||
p.primary_read_digest,
|
||||
p.independent_read_digest,
|
||||
p.derivation_digest,
|
||||
p.bound_slots_digest,
|
||||
p.back_substitution_digest,
|
||||
):
|
||||
assert digest, "every replay-critical digest must be populated on a VERIFIED proof"
|
||||
|
|
|
|||
78
tests/test_verified_serving_gate.py
Normal file
78
tests/test_verified_serving_gate.py
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
"""VERIFIED serving gate — default-dark invariant.
|
||||
|
||||
This proves the post-scoping kill-switch predicate is dark unless an
|
||||
operator/config object explicitly opts in.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from core.config import DEFAULT_CONFIG, RuntimeConfig
|
||||
from core.epistemic_disclosure.serving_gate import verified_serving_enabled
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _LegacyConfig:
|
||||
"""A pre-field config shape: absence of the flag must mean dark."""
|
||||
|
||||
unrelated: bool = True
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _OptInConfig:
|
||||
"""Config with the verified_serving_enabled field."""
|
||||
|
||||
verified_serving_enabled: bool | None
|
||||
|
||||
|
||||
def test_default_runtime_config_keeps_verified_serving_dark() -> None:
|
||||
assert hasattr(RuntimeConfig(), "verified_serving_enabled")
|
||||
assert RuntimeConfig().verified_serving_enabled is False
|
||||
assert DEFAULT_CONFIG.verified_serving_enabled is False
|
||||
assert verified_serving_enabled(DEFAULT_CONFIG) is False
|
||||
assert verified_serving_enabled(RuntimeConfig()) is False
|
||||
|
||||
|
||||
|
||||
def test_missing_flag_is_dark_for_legacy_config_shape() -> None:
|
||||
assert verified_serving_enabled(_LegacyConfig()) is False
|
||||
|
||||
|
||||
def test_gate_only_lights_on_explicit_truthy_opt_in() -> None:
|
||||
assert verified_serving_enabled(_OptInConfig(False)) is False
|
||||
assert verified_serving_enabled(_OptInConfig(True)) is True
|
||||
assert verified_serving_enabled(_OptInConfig(None)) is False
|
||||
|
||||
|
||||
def test_none_uses_default_config_and_stays_dark() -> None:
|
||||
assert verified_serving_enabled() is False
|
||||
|
||||
|
||||
def test_verified_serving_gate_has_no_eval_or_runtime_imports() -> None:
|
||||
path = Path(__file__).parent.parent / "core/epistemic_disclosure/serving_gate.py"
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"))
|
||||
|
||||
forbidden = {
|
||||
"evals",
|
||||
"evals.constraint_oracle",
|
||||
"evals.constraint_oracle.verified_producer",
|
||||
"verify",
|
||||
"chat.runtime",
|
||||
"generate.contemplation.pass_manager",
|
||||
}
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for name in node.names:
|
||||
for banned in forbidden:
|
||||
assert name.name != banned, f"Forbidden import: {name.name}"
|
||||
assert not name.name.startswith(banned + "."), f"Forbidden import: {name.name}"
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
if node.module:
|
||||
for banned in forbidden:
|
||||
assert node.module != banned, f"Forbidden import from: {node.module}"
|
||||
assert not node.module.startswith(banned + "."), f"Forbidden import from: {node.module}"
|
||||
Loading…
Reference in a new issue