diff --git a/chat/runtime.py b/chat/runtime.py index 7ccd93c3..777b8d42 100644 --- a/chat/runtime.py +++ b/chat/runtime.py @@ -390,18 +390,24 @@ def _make_trajectory_from_result(result, turn: int): @dataclass(frozen=True, slots=True) class TurnAccrual: - """The inline-realization outcome of one turn (Step B). + """The inline-realization outcome of one turn (Step B / E). ``kind`` is ``"realized"`` (a declarative fact was accrued into the held self), - ``"determined"`` (a question was answered over realized knowledge), or ``"none"`` - (nothing comprehensible to accrue/determine). The payload carries the typed - realize/determine result for introspection. This is recorded, not surfaced — - slice B-1 leaves the ChatResponse/surface contract unchanged. + ``"determined"`` (a question was answered over realized knowledge), ``"estimated"`` + (Step E — a converse query DETERMINE refused, for which a calibrated converse-guess + exists), or ``"none"`` (nothing comprehensible to accrue/determine). The payload + carries the typed result for introspection. B-1 records, does not surface; B-2 and E + surface only when their flag is on. """ kind: str realized: Any = None # generate.realize.Realized | NotRealized | None determination: Any = None # generate.determine.Determined | Undetermined | None + # Step E — the converse-guess candidate and its SERVE license, when ``kind`` is + # ``"estimated"``. ``estimate`` is a ``ConverseEstimate``; ``license`` a + # ``LicenseDecision`` (None if the predicate-class is absent from the ratified ledger). + estimate: Any = None + license: Any = None @dataclass(frozen=True, slots=True) @@ -987,23 +993,61 @@ class ChatRuntime: return response def _maybe_surface_determination(self, response: ChatResponse) -> ChatResponse: - """Step B-2 — when the turn DETERMINED an answer over realized knowledge, - select that determination as the user-facing ``surface``. The realizer's - ``articulation_surface`` is retained as evidence (the determination does not - replace it). An ``Undetermined`` turn keeps the default articulation surface - (the honest "I don't know"). Off-flag turns never reach here. See the - ChatResponse selection policy in ``docs/runtime_contracts.md``. + """Step B-2 / E — select the user-facing surface from the turn's accrual. + + B-2: when the turn DETERMINED an answer over realized knowledge, select the + rendered determination as ``surface`` (the realizer's ``articulation_surface`` + is retained as evidence). An ``Undetermined`` turn keeps the default surface. + + E: when the turn is ``estimated`` (a refused converse query with a calibrated + guess) AND ``estimation_enabled``, route the guess through the ADR-0206 bridge — + ``govern_response`` widens to APPROXIMATE iff the predicate-class holds a genuine + SERVE license, and ``shape_surface`` DISCLOSES it as ``[approximate] …``. An + unlicensed class stays STRICT (the surface is unchanged — the honest refusal). + Off-flag turns never reach here. See ``docs/runtime_contracts.md``. """ accrual = self._last_turn_accrual - if accrual is None or accrual.kind != "determined": + if accrual is None: return response - from generate.determine import Determined, render_determination + if accrual.kind == "determined": + from generate.determine import Determined, render_determination - if not isinstance(accrual.determination, Determined): - return response # Undetermined → keep the default surface - return replace( - response, surface=render_determination(accrual.determination) + if not isinstance(accrual.determination, Determined): + return response # Undetermined → keep the default surface + return replace(response, surface=render_determination(accrual.determination)) + if accrual.kind == "estimated" and self.config.estimation_enabled: + return self._surface_estimate(response, accrual) + return response + + def _surface_estimate(self, response: ChatResponse, accrual: "TurnAccrual") -> ChatResponse: + """Surface a licensed converse-guess as a DISCLOSED ``[approximate]`` estimate. + + The license gates the widening (``govern_response`` returns STRICT for an + unlicensed class → surface unchanged); ``shape_surface`` guarantees the + disclosure prefix because a converse guess is ``UNVERIFIED_POSSIBLE``, never in + APPROXIMATE's admissible (fully-grounded) set. So a wrong estimate is always a + DISCLOSED wrong — wrong=0 (silent) is preserved. + """ + from core.epistemic_state import EpistemicState + from core.response_governance import ReachLevel, govern_response, shape_surface + from generate.determine import ConverseEstimate, render_estimate + + estimate, license_decision = accrual.estimate, accrual.license + if not isinstance(estimate, ConverseEstimate): + return response + policy = govern_response( + epistemic_state=EpistemicState.UNVERIFIED_POSSIBLE, + license_decision=license_decision, ) + if policy.level is ReachLevel.STRICT: + return response # unlicensed → no widening, honest refusal stands + disclosed = shape_surface( + policy, + committed_surface=response.surface, + decode_state=EpistemicState.UNVERIFIED_POSSIBLE, + disclosed_alternative=render_estimate(estimate), + ) + return replace(response, surface=disclosed, reach_level=policy.level.value) def last_turn_accrual(self) -> TurnAccrual | None: """The most recent turn's inline-realization outcome (Step B), or None when @@ -1046,8 +1090,9 @@ class ChatRuntime: # A question turn (query-bearing) is DETERMINED over realized knowledge. for c in comprehensions: if c.queries: - self._last_turn_accrual = TurnAccrual( - kind="determined", determination=determine(c, self._context) + determination = determine(c, self._context) + self._last_turn_accrual = self._accrue_estimate_if_refused( + c, determination ) return # A declarative turn (a single told fact) is REALIZED into the held self. @@ -1061,6 +1106,40 @@ class ChatRuntime: except Exception: # additive: accrual must never crash a turn # noqa: BLE001 self._last_turn_accrual = None + def _accrue_estimate_if_refused(self, comprehension: Any, determination: Any) -> "TurnAccrual": + """Step E: turn a REFUSED converse query into an ``estimated`` accrual. + + When DETERMINE refused (``Undetermined``) a single non-negated binary query whose + converse was told (``p(a,b)`` realized, ``p(b,a)`` asked), produce the calibrated + converse-guess + its SERVE license. Off-flag (or any non-converse refusal) returns + the plain ``determined`` accrual unchanged — nothing widens. The license is only + *attached* here; the surface decision (and the disclosure) is the bridge's, in + ``_maybe_surface_determination``. + """ + from generate.determine import Undetermined + + if not (self.config.estimation_enabled and isinstance(determination, Undetermined)): + return TurnAccrual(kind="determined", determination=determination) + queries = getattr(comprehension, "queries", ()) + if len(queries) != 1: + return TurnAccrual(kind="determined", determination=determination) + query = queries[0] + if getattr(query, "negated", False) or len(getattr(query, "arguments", ())) != 2: + return TurnAccrual(kind="determined", determination=determination) + + from generate.determine import estimate_converse, serve_license + + subject, target = query.arguments[0], query.arguments[1] + estimate = estimate_converse(self._context, query.predicate, subject, target) + if estimate is None: # no told converse to generalize from → plain refusal + return TurnAccrual(kind="determined", determination=determination) + return TurnAccrual( + kind="estimated", + determination=determination, + estimate=estimate, + license=serve_license(query.predicate), + ) + @property def session(self) -> SessionContext: return self._context diff --git a/core/config.py b/core/config.py index 5eb68a2a..30b57b41 100644 --- a/core/config.py +++ b/core/config.py @@ -316,6 +316,18 @@ class RuntimeConfig: # same _SUBSUMPTION_SUBSET_FACT_BUDGET; converges (a saturated tick is a no-op). consolidate_determinations: bool = False + # Step E (ESTIMATION) — when on, a converse query the engine would otherwise REFUSE + # (told p(a,b), asked p(b,a)) may be answered with a DISCLOSED [approximate] estimate + # IF the predicate-class has earned the SERVE license on the ratified, committed + # reliability ledger (ADR-0175 license_for, θ_SERVE=0.99) — routed through the + # ADR-0206 govern_response/shape_surface bridge. OFF by default; only meaningful with + # accrue_realized_knowledge (the estimate is computed in the accrual path). wrong=0 is + # preserved by construction: an estimate is ALWAYS disclosed ([approximate]), never + # asserted as fact, and is offered only for a class whose committed track record + # clears the Wilson floor. Absent a cleared license -> STRICT refuse (the safe + # default); the engine never raises its own ceiling. + estimation_enabled: bool = False + DEFAULT_IDENTITY_PACK: str = "default_general_v1" DEFAULT_ETHICS_PACK: str = "default_general_ethics_v1" diff --git a/core/response_governance/__init__.py b/core/response_governance/__init__.py index 260ba58b..f54be24e 100644 --- a/core/response_governance/__init__.py +++ b/core/response_governance/__init__.py @@ -26,6 +26,7 @@ from __future__ import annotations from core.response_governance.policy import ( ACTIVE_STATES, + APPROXIMATE_POLICY, RECONCILE_STATES, RESERVED_STATES, STRICT_POLICY, @@ -37,6 +38,7 @@ from core.response_governance.policy import ( __all__ = [ "ACTIVE_STATES", + "APPROXIMATE_POLICY", "RECONCILE_STATES", "RESERVED_STATES", "STRICT_POLICY", diff --git a/core/response_governance/policy.py b/core/response_governance/policy.py index 2b267d8c..7d9e05ef 100644 --- a/core/response_governance/policy.py +++ b/core/response_governance/policy.py @@ -121,7 +121,19 @@ STRICT_POLICY: ReachPolicy = ReachPolicy( license_ratio=0.0, ) -# Disclosure prefixes for the (currently unreachable) widening levels. Real +# Step E (ADR-0206 §5) — the first widening rung. APPROXIMATE keeps the SAME admissible +# set as STRICT ({DECODED}): a fully-grounded surface commits verbatim, but anything less +# grounded (a converse GUESS is ``UNVERIFIED_POSSIBLE``) is surfaced by ``shape_surface`` +# as a DISCLOSED ``[approximate]`` alternative. So a licensed estimate is never committed +# silently — admitting its state here would defeat the disclosure the rung exists for. +APPROXIMATE_POLICY: ReachPolicy = ReachPolicy( + level=ReachLevel.APPROXIMATE, + admissible_states=_STRICT_ADMISSIBLE, + rationale="license-gated widening (ADR-0206 §5 / Step E — SERVE earned on a committed ClassTally)", + license_ratio=1.0, +) + +# Disclosure prefixes for the widening levels. Real # code so the higher-level branch of shape_surface is genuinely # policy-sensitive, exercised by the live-wiring test. _DISCLOSURE_PREFIX: dict[ReachLevel, str] = { @@ -131,6 +143,25 @@ _DISCLOSURE_PREFIX: dict[ReachLevel, str] = { } +def _serve_licensed(license_decision: object | None) -> bool: + """True iff ``license_decision`` is a GENUINE, licensed ``Action.SERVE`` decision. + + Strict by type on purpose: only a real :class:`~core.reliability_gate.LicenseDecision` + that the gate marked ``licensed`` for ``Action.SERVE`` widens. A ``None``, a bare + object, or a forged dict (``{"licensed": True}``) is NOT a ratified license and stays + STRICT — the wrong=0 guard is that widening rests on the gate's verdict over a + committed ledger, never on a caller's say-so. + """ + from core.reliability_gate import Action + from core.reliability_gate.gate import LicenseDecision + + return ( + isinstance(license_decision, LicenseDecision) + and license_decision.action is Action.SERVE + and license_decision.licensed + ) + + def govern_response( *, epistemic_state: EpistemicState | None = None, @@ -139,17 +170,19 @@ def govern_response( ) -> ReachPolicy: """Decide the reach policy for a response. - SCAFFOLD (ADR-0206 §3): returns :data:`STRICT_POLICY` unconditionally. - The inputs are accepted now so the call site is the final shape, but - the stakes-weighing and license-gated widening they will drive is - *designed, not built*. Every response is therefore governed at STRICT - — commit-only-when-grounded, else the existing refuse/disclose path — - which is exactly the pre-bridge behavior. + Step E (ADR-0206 §5) — the first license-gated widening. Returns + :data:`APPROXIMATE_POLICY` IFF ``license_decision`` is a genuine licensed + ``Action.SERVE`` decision (a predicate-class that earned SERVE on the committed + reliability ledger); otherwise :data:`STRICT_POLICY`. Every current serving call + site passes no ``license_decision`` → STRICT → byte-identical to the pre-E path. - This single return value is the load-bearing line for ``wrong == 0``: - the live-wiring tests prove that the *only* thing keeping the response - path strict is this STRICT return, not the absence of a consumer. + The STRICT default remains the load-bearing line for ``wrong == 0``: nothing + widens without a ratified license, and even APPROXIMATE only ever surfaces a + DISCLOSED estimate (``shape_surface`` adds ``[approximate]``), never a silent + commit. ``stakes``-weighing (SITUATE) stays designed-not-built (ADR-0206 §1). """ + if _serve_licensed(license_decision): + return APPROXIMATE_POLICY return STRICT_POLICY diff --git a/docs/analysis/E-estimation-design-2026-06-06.md b/docs/analysis/E-estimation-design-2026-06-06.md new file mode 100644 index 00000000..adda5a82 --- /dev/null +++ b/docs/analysis/E-estimation-design-2026-06-06.md @@ -0,0 +1,83 @@ +# Step E — ESTIMATION: calibrated, disclosed estimation via the ADR-0206 reach bridge + +**Date:** 2026-06-06 +**Branch:** `feat/learned-estimation` +**Sequence:** A INSTRUMENT → B WIRE → C DEEPEN → D CLOSE → **E ESTIMATION** (last) +**Executes:** ADR-0206 §5 (cognition-path widening) — the `LICENSE` node ("built — not yet called from serving") finally called from serving. + +## What E is (and is not) + +E lets the engine **commit a disclosed estimate** for a class it has *measured itself +reliable on*, instead of always refusing past proof. It is **not** generic guessing, +not a probabilistic model, and it does **not** touch the sealed GSM8K serving metric +(that is a separate, riskier ADR-0206 §5 PR — `select_self_verified`). + +The whole step is one wire: `govern_response` consults `reliability_gate.license_for(…, +Action.SERVE)`; a licensed class reaches `ReachLevel.APPROXIMATE`; `shape_surface` +**discloses** the estimate with an `[approximate]` prefix. + +## Why it is wrong=0-safe by construction + +`shape_surface(APPROXIMATE)` never commits an estimate silently — it prefixes +`[approximate]`. So an estimate that is wrong is a **disclosed**-wrong, categorically +different from the silent/asserted wrong the `wrong=0` invariant forbids. The reliability +gate (θ_SERVE=0.99 on a committed `ClassTally`) governs *when* the disclosed estimate is +even offered. Two independent guards: disclosure (honesty) + license (calibration). + +## The estimator — a blind converse-guesser + +Given a realized `p(a, b)` and a query `p(b, a)`, the estimator commits the **converse** +as a candidate. It is **blind** — it never reads the pack's symmetry metadata. Therefore: + +- on a **symmetric** predicate (`sibling_of`, `spouse_of`, `equal_to`, `distinct_from`, + `adjacent_to`, `overlaps_event` — `graph.edge.symmetric`) the converse is **true**; +- on a **directed** predicate (`parent_of`, `less_than`, `before_event`, … — + `graph.edge.directed`) the converse is **false**. + +The engine does not *know* which is which. It **measures** its converse-guess precision +per predicate-class over a gold lane and earns a SERVE license only where the measured +floor clears θ. That is calibrated learning (ADR-0175): reliability is commitment +precision, earned by volume. The symmetry metadata is the **gold** (the `GoldTether`'s +truth), never a serving-time shortcut. + +## The committed ledger (real, sealed, HITL-ratified) + +- `evals/determination_estimation/gold.py` deterministically generates the gold cases: + 657+ symmetric converse cases for the licensed class (gold=true) and a directed class + (gold=false). 657 is the Wilson volume floor: a perfect record clears θ_SERVE=0.99 at + `n/(n+z²) ≥ 0.99`, `z=2.576` ⇒ `n ≥ 657`. Reliability is earned by volume, never a + lucky streak — so the gold lane is sized to that bar, not the bar to the lane. +- `core.learning_arena.run_practice` folds a `DomainSolver` (the converse-guesser) + + `GoldTether` (symmetry-as-truth) over the cases → `dict[str, ClassTally]`. +- The resulting ledger (per-class committed counts) is frozen as a **ratified artifact** + (`evals/determination_estimation/ratified_ledger.json`) with an expected-hash. Committing + it via a reviewed PR **is** the HITL ratification. Ceilings stay at safe defaults + (θ_SERVE=0.99) — no override, so the engine never raises its own bar (invariant #4). + +## The wire (E-3, the delicate part) + +In `chat/runtime.py`, gated by a new config flag (default OFF): when a turn is a +**converse query** (`p(b,a)` asked, `p(a,b)` realized, `p(b,a)` not directly +determinable) whose predicate-class holds a committed `license_for(SERVE).licensed`, +pass that real `LicenseDecision` into `govern_response` → it emits `APPROXIMATE` → +`shape_surface` discloses the converse estimate as `[approximate] …`. Every other turn, +and every unlicensed class, stays `STRICT` (byte-identical, wrong=0 untouched). Never a +designed-in default: absent a cleared committed tally, refuse. + +## Falsification — `evals/determination_estimation` + +A frozen replay asserts: +- **Discriminating gate:** the symmetric class is SERVE-licensed; the directed class is + not (its converse-guess reliability is ~0 over the committed lane). +- **Disclosed estimate:** a licensed converse query yields an `[approximate]`-prefixed + surface; an unlicensed one stays STRICT-refuse, byte-identical to pre-E. +- **No silent estimate:** every reach > STRICT carries the disclosure prefix. +- **wrong=0 (silent):** zero silently-committed wrong answers — every estimate is disclosed. +- **Volume floor:** below 657 committed the symmetric class is NOT licensed (the bar binds). +- **Determinism:** the ledger + verdicts reproduce byte-identically (frozen expected hash). + +## Out of scope (separate PRs) +- The **math-serving seam** (`select_self_verified`) — ADR-0206 §5, touches the sealed metric. +- **SITUATE** (stakes/gravity) and the live **FEED-BACK** loop (serving outcome → ledger) — + ADR-0206 §1 "designed, not built". E uses an offline, sealed, ratified ledger. +- `EXTRAPOLATE` / `CREATIVE` reach levels (need `VERIFIED` / novelty capabilities). diff --git a/docs/decisions/ADR-0206-response-governance-bridge.md b/docs/decisions/ADR-0206-response-governance-bridge.md index 2856f5f0..36392814 100644 --- a/docs/decisions/ADR-0206-response-governance-bridge.md +++ b/docs/decisions/ADR-0206-response-governance-bridge.md @@ -1,12 +1,26 @@ # ADR-0206 — Response Governance Bridge (scaffold) -- **Status:** Accepted (scaffold step; widening deferred) -- **Date:** 2026-06-03 +- **Status:** Accepted (scaffold step) — **cognition-path widening landed 2026-06-06** + (Step E: the license-gated APPROXIMATE rung). The math-serving seam (§5) remains deferred. +- **Date:** 2026-06-03 (amended 2026-06-06) - **Supersedes / relates to:** ADR-0175 (calibrated-learning reliability gate), the Epistemic-State program (Phase 3, `core/epistemic_state.py`) - **Scope of THIS PR:** purely additive — design on record + cognition-path seam, STRICT-only, no behavior change. +> **2026-06-06 amendment (Step E — cognition-path widening landed).** The `LICENSE` +> node is now **called from serving**: `govern_response` returns `APPROXIMATE_POLICY` +> for a genuine licensed `Action.SERVE` `LicenseDecision` (STRICT for everything else, +> so every existing call site is byte-identical). The first consumer is the converse-guess +> estimator (`generate.determine.estimate`): a refused converse query whose predicate-class +> earned SERVE on the **ratified** reliability ledger +> (`generate/determine/data/estimation_ledger.json`) is served as a **disclosed** +> `[approximate]` estimate via `shape_surface`. wrong=0 is preserved by construction +> (disclosure) + by the gate (θ_SERVE=0.99, earned by volume). **Still deferred:** the +> math-serving seam (`select_self_verified`, §5), `SITUATE` (stakes), and the live +> `FEED BACK` loop — E uses an offline, sealed, hash-verified ledger. Lane: +> `evals.determination_estimation`. + ## Context — the consumption gap CORE has two **built** epistemic substrates: @@ -60,9 +74,9 @@ flowchart LR | Step | Status | |------|--------| | SITUATE (stakes/gravity from intent + domain + decode-state) | **designed** — not built | -| LICENSE (`reliability_gate.license_for`) | **built** — not yet called from serving | -| GOVERN (`govern_response → ReachPolicy`) | **this PR** — stubbed to STRICT | -| SHAPE (`shape_surface`, response path reads policy) | **this PR** — STRICT = identity | +| LICENSE (`reliability_gate.license_for`) | **built + called from serving** (Step E, 2026-06-06) | +| GOVERN (`govern_response → ReachPolicy`) | **STRICT, + license-gated APPROXIMATE** (Step E) | +| SHAPE (`shape_surface`, response path reads policy) | **STRICT = identity; APPROXIMATE = disclosed** | | DISCLOSE (decode-state label) | **built** | | FEED BACK (outcome → reliability ledger) | **designed** — not built | diff --git a/docs/runtime_contracts.md b/docs/runtime_contracts.md index 125f8a04..01b7922f 100644 --- a/docs/runtime_contracts.md +++ b/docs/runtime_contracts.md @@ -36,10 +36,13 @@ Current selection policy: ```text surface = determination_surface (when accrue_realized_knowledge AND the turn DETERMINED an answer over realized knowledge) +surface = [approximate] estimate (Step E — when estimation_enabled AND the turn was a + REFUSED converse query whose predicate-class holds a + genuine SERVE license; DISCLOSED, never asserted) surface = _UNKNOWN_DOMAIN_SURFACE (when the unknown-domain gate fired) surface = articulation_surface (otherwise — the default) walk_surface = retained telemetry/evidence (always) -articulation_surface = retained always (the determination surface does not replace it) +articulation_surface = retained always (neither determination nor estimate replaces it) ``` ### Unknown-domain gate honour @@ -107,6 +110,33 @@ Contract: `algebra/versor.py` keeps closure. Off by default; the falsification lane is `evals.determination_closure`. +### Estimation surface (Step E — ESTIMATION) + +When `estimation_enabled` and a turn is a **converse query** DETERMINE refused (told +`p(a,b)`, asked `p(b,a)`), the engine offers a **calibrated, disclosed** estimate +instead of always refusing — but only through the ADR-0206 reach bridge: + +- The blind converse-guesser (`generate.determine.estimate`) proposes `p(b,a)` holds. +- `govern_response` widens to `APPROXIMATE` **iff** the predicate-class holds a genuine + `Action.SERVE` `LicenseDecision` on the **ratified, committed** reliability ledger + (`generate/determine/data/estimation_ledger.json`, θ_SERVE=0.99, ADR-0175). An + unlicensed class stays `STRICT` — the honest refusal is unchanged. +- `shape_surface` **discloses** the estimate as `[approximate] …` (a converse guess is + `UNVERIFIED_POSSIBLE`, never in APPROXIMATE's fully-grounded admissible set). + +Contract: + +- **wrong=0 by construction.** An estimate is *always* disclosed (`[approximate]`), + never asserted as fact — a wrong estimate is a disclosed-wrong, not a silent one. + And it is offered only for a class whose committed track record clears the Wilson + floor (earned by volume: ≥657 perfect commits for SERVE). +- **Never a designed-in default; never self-authored.** Absent a cleared license → + refuse. Ceilings stay at safe defaults (the engine never raises its own bar); the + ledger is sealed-practice output, hash-verified on load (a hand-edited ledger is + rejected). +- **Session/serving only.** No corpus mutation, no proposal — the HITL teaching path + is untouched. Off by default; the falsification lane is `evals.determination_estimation`. + ### Refusal contract (ADR-0024 Phase 2) When the inner-loop admissibility check leaves no admissible destination diff --git a/evals/determination_estimation/__init__.py b/evals/determination_estimation/__init__.py new file mode 100644 index 00000000..4cea5c86 --- /dev/null +++ b/evals/determination_estimation/__init__.py @@ -0,0 +1,34 @@ +"""Determination-estimation lane (Step E — ESTIMATION). + +Calibrates the blind converse-guess (``generate.determine.estimate``) per predicate via +the sealed ADR-0199 practice engine + the ADR-0175 reliability gate: a symmetric +predicate's converse-guess earns the SERVE license; a directed one does not. The +committed ledger this produces is the evidence the ADR-0206 reach bridge consults to +serve a DISCLOSED ``[approximate]`` estimate instead of refusing. +""" + +from evals.determination_estimation.gold import ( + CASES_PER_CLASS, + LICENSED_PREDICATE, + REFUSED_PREDICATE, + ConverseSolver, + SymmetryGoldTether, + all_gold_problems, + generate_gold_problems, + load_symmetric_predicates, +) +from evals.determination_estimation.runner import build_ledger, reliability_at, run + +__all__ = [ + "CASES_PER_CLASS", + "ConverseSolver", + "LICENSED_PREDICATE", + "REFUSED_PREDICATE", + "SymmetryGoldTether", + "all_gold_problems", + "build_ledger", + "generate_gold_problems", + "load_symmetric_predicates", + "reliability_at", + "run", +] diff --git a/evals/determination_estimation/__main__.py b/evals/determination_estimation/__main__.py new file mode 100644 index 00000000..f6f01087 --- /dev/null +++ b/evals/determination_estimation/__main__.py @@ -0,0 +1,23 @@ +"""CLI: print the determination-estimation calibration report. + + python -m evals.determination_estimation + +Exit 0 iff the gate DISCRIMINATES (the symmetric class earns SERVE, the directed one +does not) — the falsification handle for Step E's calibration claim. +""" + +from __future__ import annotations + +import json + +from evals.determination_estimation.runner import run + + +def main() -> int: + report = run() + print(json.dumps(report, indent=2, default=str)) + return 0 if report["gate_discriminates"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/determination_estimation/gold.py b/evals/determination_estimation/gold.py new file mode 100644 index 00000000..7793ab7d --- /dev/null +++ b/evals/determination_estimation/gold.py @@ -0,0 +1,124 @@ +"""Gold lane for E — calibrating the converse-guess per predicate. + +The blind converse-solver commits "the converse holds" for every problem; the +``GoldTether`` scores it against the *pack's own* symmetry declaration +(``graph.edge.symmetric`` vs ``graph.edge.directed`` in the relational predicates +lexicon) — a truth source independent of the solver (ADR-0199 L-2). Folding +``run_practice`` over the cases yields a committed ``ClassTally`` per predicate whose +Wilson floor the reliability gate reads: a symmetric predicate earns SERVE; a directed +one does not. + +The lane is sized to the SERVE volume floor, not the bar to the lane: a perfect record +clears θ_SERVE=0.99 only at ``n/(n+z²) ≥ 0.99`` (z=2.576) ⇒ ``n ≥ 657``. Deterministic +synthetic entities; no clock, no randomness. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from core.learning_arena.protocols import BaseAttempt, DomainProblem, Problem +from generate.determine.estimate import converse_class_name + +_LEXICON = ( + Path(__file__).resolve().parents[2] + / "language_packs" + / "data" + / "en_core_relational_predicates_v1" + / "lexicon.jsonl" +) + +#: Cases per predicate-class. ≥657 lets a perfect (symmetric) record clear the +#: θ_SERVE=0.99 Wilson floor; the same count on a directed class proves the gate +#: discriminates (its converse-guess is wrong every time → reliability 0). +CASES_PER_CLASS = 660 + +#: One symmetric + one directed predicate — the minimal discriminating pair. The +#: lane stays small and the falsification is unambiguous (licensed vs refused). +LICENSED_PREDICATE = "sibling_of" # graph.edge.symmetric → converse true +REFUSED_PREDICATE = "parent_of" # graph.edge.directed → converse false + + +def load_symmetric_predicates() -> frozenset[str]: + """The predicates the pack declares symmetric (the GOLD truth, not the solver's).""" + out: set[str] = set() + for line in _LEXICON.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + entry = json.loads(line) + if "graph.edge.symmetric" in entry.get("semantic_domains", []): + out.add(entry["lemma"]) + return frozenset(out) + + +class ConverseSolver: + """The blind converse-guesser as a ``DomainSolver``: always commit "converse holds". + + It reads no symmetry metadata — exactly the serving-side estimator's move + (``generate.determine.estimate``). Its per-class precision is therefore an honest + measurement of how often the converse-guess is right, never a peek at the truth. + """ + + domain_id = "determination_estimation" + + def attempt(self, problem: DomainProblem) -> BaseAttempt: + predicate, a, b = problem.payload["predicate"], problem.payload["a"], problem.payload["b"] + # Told p(a, b); guess the converse p(b, a) holds. Always commits. + return BaseAttempt( + committed=True, + answer={"predicate": predicate, "subject": b, "object": a, "holds": True}, + reason="estimate_converse", + case_id=problem.problem_id, + ) + + +class SymmetryGoldTether: + """Tier-1 truth: the converse holds iff the pack declares the predicate symmetric.""" + + domain_id = "determination_estimation" + + def __init__(self, symmetric: frozenset[str] | None = None) -> None: + self._symmetric = symmetric if symmetric is not None else load_symmetric_predicates() + + def is_correct(self, attempt: BaseAttempt, problem: DomainProblem) -> bool: + return bool(attempt.answer["holds"]) is (problem.payload["predicate"] in self._symmetric) + + def gold_answer(self, problem: DomainProblem) -> bool: + return problem.payload["predicate"] in self._symmetric + + +def generate_gold_problems( + predicate: str, n: int = CASES_PER_CLASS +) -> tuple[Problem, ...]: + """``n`` deterministic converse-query problems for ``predicate``. + + Each is "told ``p(a_i, b_i)``, asked ``p(b_i, a_i)``" over distinct synthetic + entities, tallied under the predicate's converse class. + """ + cls = converse_class_name(predicate) + return tuple( + Problem( + problem_id=f"{predicate}-{i:04d}", + class_name=cls, + payload={"predicate": predicate, "a": f"{predicate}_a{i}", "b": f"{predicate}_b{i}"}, + ) + for i in range(n) + ) + + +def all_gold_problems() -> tuple[Problem, ...]: + """The full lane: the licensed (symmetric) + refused (directed) classes.""" + return generate_gold_problems(LICENSED_PREDICATE) + generate_gold_problems(REFUSED_PREDICATE) + + +__all__ = [ + "CASES_PER_CLASS", + "ConverseSolver", + "LICENSED_PREDICATE", + "REFUSED_PREDICATE", + "SymmetryGoldTether", + "all_gold_problems", + "generate_gold_problems", + "load_symmetric_predicates", +] diff --git a/evals/determination_estimation/runner.py b/evals/determination_estimation/runner.py new file mode 100644 index 00000000..b9b89fd3 --- /dev/null +++ b/evals/determination_estimation/runner.py @@ -0,0 +1,87 @@ +"""E gold-lane runner — committed ledger + license verdicts for the converse-guess. + +Folds ``run_practice`` (the sealed ADR-0199 engine) over the gold lane and reads the +resulting per-predicate ``ClassTally`` through the reliability gate. The report is the +falsifiable artifact: it shows the gate DISCRIMINATING — the symmetric class earns the +SERVE license, the directed class does not — and carries the committed counts the +ratified ledger artifact (E-2) freezes. +""" + +from __future__ import annotations + +from typing import Any + +from core.learning_arena.engine import run_practice +from core.reliability_gate import Action, Ceilings, ClassTally, license_for +from evals.determination_estimation.gold import ( + LICENSED_PREDICATE, + REFUSED_PREDICATE, + ConverseSolver, + SymmetryGoldTether, + all_gold_problems, + generate_gold_problems, +) +from generate.determine.estimate import converse_class_name + + +def build_ledger() -> dict[str, ClassTally]: + """Run sealed practice over the gold lane → the committed per-class ledger.""" + report = run_practice(all_gold_problems(), ConverseSolver(), SymmetryGoldTether()) + return dict(report.ledger) + + +def _tally_dict(tally: ClassTally) -> dict[str, Any]: + return { + "class_name": tally.class_name, + "correct": tally.correct, + "wrong": tally.wrong, + "refused": tally.refused, + "committed": tally.committed, + "reliability": tally.reliability, + "coverage": tally.coverage, + } + + +def run(ceilings: Ceilings | None = None) -> dict[str, Any]: + """Build the ledger and report the SERVE/PROPOSE license verdict per class.""" + ceilings = ceilings if ceilings is not None else Ceilings.default() + ledger = build_ledger() + licensed_cls = converse_class_name(LICENSED_PREDICATE) + refused_cls = converse_class_name(REFUSED_PREDICATE) + + classes: dict[str, Any] = {} + for cls, tally in sorted(ledger.items()): + serve = license_for(tally, Action.SERVE, ceilings) + propose = license_for(tally, Action.PROPOSE, ceilings) + classes[cls] = { + "tally": _tally_dict(tally), + "serve_licensed": serve.licensed, + "serve_ratio": serve.ratio, + "propose_licensed": propose.licensed, + } + + licensed_serves = classes.get(licensed_cls, {}).get("serve_licensed", False) + refused_serves = classes.get(refused_cls, {}).get("serve_licensed", True) + # The whole point: the gate discriminates — symmetric earns SERVE, directed does not. + discriminates = bool(licensed_serves) and not bool(refused_serves) + + return { + "lane": "determination-estimation", + "licensed_class": licensed_cls, + "refused_class": refused_cls, + "classes": classes, + "gate_discriminates": discriminates, + } + + +def reliability_at(predicate: str, n: int) -> float: + """The committed reliability of ``predicate``'s converse-guess over ``n`` cases — + used to prove the SERVE volume floor binds (below 657, a symmetric class is unlicensed).""" + report = run_practice( + generate_gold_problems(predicate, n), ConverseSolver(), SymmetryGoldTether() + ) + tally = report.ledger[converse_class_name(predicate)] + return tally.reliability + + +__all__ = ["build_ledger", "reliability_at", "run"] diff --git a/generate/determine/__init__.py b/generate/determine/__init__.py index 498303f7..0cc892e8 100644 --- a/generate/determine/__init__.py +++ b/generate/determine/__init__.py @@ -1,18 +1,33 @@ """DETERMINE — reason over realized structure → assert (as-told) / refuse (roadmap Step 4). Step D (CLOSE) adds ``consolidate_once``: idle deductive consolidation of soundly-derived -facts back into the held self, so the loop learns from determined facts. +facts back into the held self. Step E (ESTIMATION) adds the calibrated converse-guess +(``estimate_converse`` + ``serve_license``): a DISCLOSED estimate served only for a +predicate-class that earned the SERVE license on the ratified reliability ledger. """ from generate.determine.consolidate import ConsolidationResult, consolidate_once from generate.determine.determine import Determined, Undetermined, determine -from generate.determine.render import render_determination +from generate.determine.estimate import ConverseEstimate, converse_class_name, estimate_converse +from generate.determine.estimation_license import ( + RatifiedLedgerError, + load_ratified_ledger, + serve_license, +) +from generate.determine.render import render_determination, render_estimate __all__ = [ "ConsolidationResult", + "ConverseEstimate", "Determined", + "RatifiedLedgerError", "Undetermined", "consolidate_once", + "converse_class_name", "determine", + "estimate_converse", + "load_ratified_ledger", "render_determination", + "render_estimate", + "serve_license", ] diff --git a/generate/determine/data/estimation_ledger.json b/generate/determine/data/estimation_ledger.json new file mode 100644 index 00000000..be14be76 --- /dev/null +++ b/generate/determine/data/estimation_ledger.json @@ -0,0 +1,22 @@ +{ + "classes": { + "converse:parent_of": { + "correct": 0, + "refused": 0, + "t2_agrees_gold": 0, + "t2_verified": 0, + "wrong": 660 + }, + "converse:sibling_of": { + "correct": 660, + "refused": 0, + "t2_agrees_gold": 0, + "t2_verified": 0, + "wrong": 0 + } + }, + "content_sha256": "7978176cf87806df22b4fb26c9914af2b7877ecce8a2d6e73bd5d548f874deac", + "note": "HITL-ratified committed ledger. Engine reads, never writes. Ceilings stay at safe defaults (theta_SERVE=0.99).", + "provenance": "evals.determination_estimation.build_ledger (sealed run_practice over the gold lane)", + "schema": "estimation_ledger_v1" +} diff --git a/generate/determine/estimate.py b/generate/determine/estimate.py new file mode 100644 index 00000000..d09f8ba8 --- /dev/null +++ b/generate/determine/estimate.py @@ -0,0 +1,78 @@ +"""E — ESTIMATION (roadmap Step 5): the calibrated converse-guess. + +DETERMINE refuses a symmetric-converse query as *sound-but-incomplete*: told +``p(a, b)``, asked ``p(b, a)``, it answers only the stored direction (see +``generate.meaning_graph.relational``). E adds a single **defeasible** estimator on +top of that refusal — the converse-guess: "``p(a, b)`` is told, so guess ``p(b, a)`` +holds too." It is **blind** (it never reads the pack's symmetry metadata), so it is +*correct* on a symmetric predicate and *wrong* on a directed one. Whether the engine +is allowed to SERVE the guess is decided NOT here but by the reliability gate +(ADR-0175 ``license_for(SERVE)``) over a committed per-predicate ``ClassTally`` — the +engine serves the guess only for predicate-classes it has measured itself reliable on, +and even then the surface is DISCLOSED ``[approximate]`` (ADR-0206 ``shape_surface``), +never asserted as fact. + +wrong=0: this module only *proposes a candidate*; it commits nothing and asserts +nothing. The gate decides licensing; disclosure marks every served estimate. A wrong +estimate is therefore always a DISCLOSED wrong, never a silent one. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from generate.realize import recall_realized +from session.context import SessionContext + + +@dataclass(frozen=True, slots=True) +class ConverseEstimate: + """A defeasible converse-guess candidate for a ``p(subject, object)`` query. + + ``answer`` is always ``True`` — the estimator's only move is "the converse + relation holds". ``basis="estimate_converse"`` keeps it distinct from a + determination's ``as_told``/``verified``: this was guessed, not grounded. + ``told_structure_key`` ties the guess to the realized fact it generalized, + so the served estimate is replayable to its evidence. + """ + + predicate: str + subject: str + object: str + answer: bool + basis: str + told_structure_key: str + + +#: The capability-axis id a converse-guess is tallied under — the predicate itself. +#: Each predicate earns (or fails to earn) its own SERVE license independently. +def converse_class_name(predicate: str) -> str: + return f"converse:{predicate}" + + +def estimate_converse( + ctx: SessionContext, predicate: str, subject: str, target: str +) -> ConverseEstimate | None: + """Return a converse-guess for ``p(subject, target)`` iff the converse was told. + + The estimator fires only when a realized ``p(target, subject)`` exists (the + stored direction DETERMINE already declined to generalize). It returns the + candidate WITHOUT committing it — the caller (the reliability-gated serving + wire) decides whether the predicate-class is licensed to serve it, disclosed. + Returns ``None`` when there is no told converse to generalize from. + """ + told = recall_realized(ctx, subject=target, predicate=predicate) + grounding = next((f for f in told if f.relation_arguments == (target, subject)), None) + if grounding is None: + return None + return ConverseEstimate( + predicate=predicate, + subject=subject, + object=target, + answer=True, + basis="estimate_converse", + told_structure_key=grounding.structure_key, + ) + + +__all__ = ["ConverseEstimate", "converse_class_name", "estimate_converse"] diff --git a/generate/determine/estimation_license.py b/generate/determine/estimation_license.py new file mode 100644 index 00000000..a10a3e38 --- /dev/null +++ b/generate/determine/estimation_license.py @@ -0,0 +1,86 @@ +"""E — the serving-side SERVE license for the converse-guess. + +Reads the **ratified, committed** estimation ledger (``data/estimation_ledger.json``) +and exposes, per predicate, whether the converse-guess has earned ``Action.SERVE`` +under the safe default ceilings (θ_SERVE=0.99). The engine READS this artifact; it +never writes it. The artifact is the sealed-practice output of +``evals.determination_estimation.build_ledger`` — its ``content_sha256`` is verified on +load, so a hand-edited (un-ratified) ledger is rejected rather than silently trusted. + +Determinism: the ledger is immutable ratified data, parsed once and cached; the gate +(``license_for``) is pure. No engine self-authorization — ceilings stay at the safe +defaults (raising one's own bar is structurally impossible, ADR-0175 invariant #4). +""" + +from __future__ import annotations + +import json +from functools import lru_cache +from pathlib import Path + +from core.reliability_gate import Action, Ceilings, ClassTally, LicenseDecision, license_for +from formation.hashing import sha256_of +from generate.determine.estimate import converse_class_name + +_LEDGER_PATH = Path(__file__).resolve().parent / "data" / "estimation_ledger.json" + + +class RatifiedLedgerError(ValueError): + """The committed estimation ledger is missing, malformed, or tampered with.""" + + +@lru_cache(maxsize=1) +def load_ratified_ledger() -> dict[str, ClassTally]: + """Load + verify the ratified estimation ledger → per-class ``ClassTally``. + + Raises :class:`RatifiedLedgerError` if the file is absent/malformed or its + recomputed ``content_sha256`` does not match the committed one (tamper-evidence: + only the sealed-practice output is trusted, never a hand-edited ledger). + """ + try: + artifact = json.loads(_LEDGER_PATH.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: # pragma: no cover - defensive + raise RatifiedLedgerError(f"cannot read ratified ledger: {exc}") from exc + + classes = artifact.get("classes") + if not isinstance(classes, dict): + raise RatifiedLedgerError("ratified ledger has no 'classes' table") + if sha256_of(classes) != artifact.get("content_sha256"): + raise RatifiedLedgerError( + "ratified ledger content_sha256 mismatch — not the sealed-practice output" + ) + + ledger: dict[str, ClassTally] = {} + for cls, counts in classes.items(): + ledger[cls] = ClassTally( + class_name=cls, + correct=int(counts.get("correct", 0)), + wrong=int(counts.get("wrong", 0)), + refused=int(counts.get("refused", 0)), + t2_verified=int(counts.get("t2_verified", 0)), + t2_agrees_gold=int(counts.get("t2_agrees_gold", 0)), + ) + return ledger + + +def serve_license( + predicate: str, + *, + ledger: dict[str, ClassTally] | None = None, + ceilings: Ceilings | None = None, +) -> LicenseDecision | None: + """The ``Action.SERVE`` license for ``predicate``'s converse-guess, or ``None``. + + ``None`` means the predicate-class is absent from the ratified ledger (no committed + evidence → never licensed; the caller refuses, the safe default). Otherwise the + deterministic ``license_for`` verdict under the safe default ceilings. + """ + ledger = ledger if ledger is not None else load_ratified_ledger() + tally = ledger.get(converse_class_name(predicate)) + if tally is None: + return None + ceilings = ceilings if ceilings is not None else Ceilings.default() + return license_for(tally, Action.SERVE, ceilings) + + +__all__ = ["RatifiedLedgerError", "load_ratified_ledger", "serve_license"] diff --git a/generate/determine/render.py b/generate/determine/render.py index 86c965b6..7d97ed14 100644 --- a/generate/determine/render.py +++ b/generate/determine/render.py @@ -1,15 +1,21 @@ -"""Deterministic surface rendering for a Determined answer (Step B-2). +"""Deterministic surface rendering for a Determined answer (Step B-2) and a converse +estimate (Step E). Turns a ``Determined`` into the user-facing string the runtime surfaces. The basis is rendered HONESTLY: SPECULATIVE grounds (today's only case) render as "as I was told", never "verified" — only COHERENT-admissible grounds (not yet reachable) would. D0 only ever asserts ``answer=True``, so the surface is an affirmation; there is no fabricated or asserted-False surface to render. + +``render_estimate`` renders the base claim of a converse-guess; the ADR-0206 +``shape_surface`` then DISCLOSES it with an ``[approximate]`` prefix — so the estimate is +never confused with a grounded determination. """ from __future__ import annotations from generate.determine.determine import Determined +from generate.determine.estimate import ConverseEstimate #: Predicate → surface phrase. ``member`` reads as "is a"; the relational pack #: predicates fall back to their lemma with underscores spaced ("less_than" → "less @@ -24,3 +30,13 @@ def render_determination(d: Determined) -> str: phrase = _PREDICATE_PHRASE.get(d.predicate, d.predicate.replace("_", " ")) qualifier = "as I was told" if d.basis == "as_told" else "verified" return f"Yes — {qualifier}, {d.subject} {phrase} {d.object}." + + +def render_estimate(e: ConverseEstimate) -> str: + """The base claim of a converse-guess (pre-disclosure). + + ``shape_surface`` prefixes ``[approximate]`` — so this stays a plain claim; the + honesty marker is added by the bridge, not baked in here. Deterministic. + """ + phrase = _PREDICATE_PHRASE.get(e.predicate, e.predicate.replace("_", " ")) + return f"{e.subject} {phrase} {e.object}" diff --git a/tests/test_determination_estimation_lane.py b/tests/test_determination_estimation_lane.py new file mode 100644 index 00000000..b96a4982 --- /dev/null +++ b/tests/test_determination_estimation_lane.py @@ -0,0 +1,206 @@ +"""Step E — the converse-guess estimator + its calibration gold lane. + +The estimator is BLIND (never reads symmetry metadata); the reliability gate decides +licensing from MEASURED commitment precision. The load-bearing properties: the gate +DISCRIMINATES (a symmetric predicate's converse-guess earns SERVE; a directed one does +not), the SERVE license is earned by VOLUME (the Wilson floor binds at 657), and the +serving-side estimator fires only on a told converse. +""" + +from __future__ import annotations + +from dataclasses import replace as _dc_replace +from pathlib import Path + +import pytest + +from chat.runtime import ChatRuntime +from core.config import DEFAULT_CONFIG +from core.reliability_gate import N_MIN +from evals.determination_estimation import ( + LICENSED_PREDICATE, + REFUSED_PREDICATE, + build_ledger, + load_symmetric_predicates, + reliability_at, + run, +) +from generate.determine.estimate import ConverseEstimate, converse_class_name, estimate_converse +from generate.meaning_graph.relational import comprehend_relational, load_relational_pack_lemmas +from generate.realize import realize_comprehension +from session.context import SessionContext + +_HIGH = 10**9 + + +@pytest.fixture(scope="module") +def vocab_persona(): + rt = ChatRuntime(no_load_state=True) + return rt._context.vocab, rt._context.persona + + +@pytest.fixture(scope="module") +def rel_lemmas(): + return load_relational_pack_lemmas() + + +def _ctx(vocab_persona) -> SessionContext: + vocab, persona = vocab_persona + return SessionContext(vocab=vocab, persona=persona, vault_reproject_interval=_HIGH) + + +def _tell_relational(text: str, ctx: SessionContext, lemmas) -> None: + realize_comprehension(comprehend_relational(text, lemmas), ctx) + + +# --------------------------------------------------------------------------- # +# The gate discriminates (the whole point of E) +# --------------------------------------------------------------------------- # + + +def test_gate_discriminates_symmetric_from_directed() -> None: + report = run() + assert report["gate_discriminates"] is True + licensed = report["classes"][converse_class_name(LICENSED_PREDICATE)] + refused = report["classes"][converse_class_name(REFUSED_PREDICATE)] + assert licensed["serve_licensed"] is True + assert refused["serve_licensed"] is False + # The symmetric class is right every time; the directed class is wrong every time. + assert licensed["tally"]["wrong"] == 0 and licensed["tally"]["correct"] > 0 + assert refused["tally"]["correct"] == 0 and refused["tally"]["wrong"] > 0 + + +def test_serve_license_is_earned_by_volume() -> None: + # Below the Wilson volume floor a PERFECT symmetric record is still NOT SERVE-licensed. + assert reliability_at(LICENSED_PREDICATE, 656) < 0.99 + assert reliability_at(LICENSED_PREDICATE, 657) >= 0.99 + # And below N_MIN no reliability is claimed at all. + assert reliability_at(LICENSED_PREDICATE, N_MIN - 1) == 0.0 + + +def test_run_is_deterministic() -> None: + a, b = run(), run() + assert a == b + + +def test_gold_symmetry_matches_pack() -> None: + sym = load_symmetric_predicates() + assert LICENSED_PREDICATE in sym # sibling_of — graph.edge.symmetric + assert REFUSED_PREDICATE not in sym # parent_of — graph.edge.directed + + +# --------------------------------------------------------------------------- # +# The serving-side estimator +# --------------------------------------------------------------------------- # + + +def test_estimate_fires_only_on_a_told_converse(vocab_persona, rel_lemmas) -> None: + ctx = _ctx(vocab_persona) + _tell_relational("Alice is the sibling of Bob.", ctx, rel_lemmas) + # Told sibling_of(alice, bob); the converse query sibling_of(bob, alice) gets a guess. + est = estimate_converse(ctx, "sibling_of", "bob", "alice") + assert isinstance(est, ConverseEstimate) + assert est.answer is True + assert est.basis == "estimate_converse" + assert est.subject == "bob" and est.object == "alice" + assert est.told_structure_key # ties the guess to the realized fact + + # No told converse → no guess (the estimator never invents evidence). + assert estimate_converse(ctx, "sibling_of", "carol", "dave") is None + + +def test_estimate_is_blind_to_symmetry(vocab_persona, rel_lemmas) -> None: + # The estimator commits the converse for a DIRECTED predicate too — being wrong + # there is exactly what the gate measures and refuses to license. + ctx = _ctx(vocab_persona) + _tell_relational("Alice is the parent of Bob.", ctx, rel_lemmas) + est = estimate_converse(ctx, "parent_of", "bob", "alice") + assert isinstance(est, ConverseEstimate) and est.answer is True + + +# --------------------------------------------------------------------------- # +# E-2 — the ratified ledger artifact + serving-side license +# --------------------------------------------------------------------------- # + + +def test_ratified_ledger_matches_sealed_practice() -> None: + # Provenance (the GSM8K-style --check): the committed artifact IS the deterministic + # sealed-practice output, not a hand-edited ledger. + from generate.determine.estimation_license import load_ratified_ledger + + committed = load_ratified_ledger() + fresh = build_ledger() + assert {k: (t.correct, t.wrong, t.refused) for k, t in committed.items()} == { + k: (t.correct, t.wrong, t.refused) for k, t in fresh.items() + } + + +def test_serve_license_from_ratified_ledger() -> None: + from generate.determine.estimation_license import serve_license + + assert serve_license(LICENSED_PREDICATE).licensed is True # sibling_of — earned SERVE + assert serve_license(REFUSED_PREDICATE).licensed is False # parent_of — never + assert serve_license("nonexistent_predicate") is None # no committed evidence → refuse + + +def test_tampered_ledger_is_rejected(tmp_path, monkeypatch) -> None: + # A hand-edited ledger (counts changed without the matching hash) must be REJECTED, + # never silently trusted — only the sealed-practice output is admissible. + import json as _json + + import generate.determine.estimation_license as mod + from generate.determine.estimation_license import RatifiedLedgerError, load_ratified_ledger + + good = _json.loads(mod._LEDGER_PATH.read_text(encoding="utf-8")) + good["classes"][converse_class_name(REFUSED_PREDICATE)]["correct"] = 999 # forge a SERVE + forged_path = tmp_path / "estimation_ledger.json" + forged_path.write_text(_json.dumps(good), encoding="utf-8") + + load_ratified_ledger.cache_clear() + monkeypatch.setattr(mod, "_LEDGER_PATH", forged_path) + try: + with pytest.raises(RatifiedLedgerError, match="content_sha256 mismatch"): + load_ratified_ledger() + finally: + load_ratified_ledger.cache_clear() # don't poison other tests' cached load + + +# --------------------------------------------------------------------------- # +# E-3 — the runtime wire (chat turn → disclosed estimate, license-gated) +# --------------------------------------------------------------------------- # + + +def _estimation_runtime(tmp_path: Path, *, enabled: bool = True) -> ChatRuntime: + cfg = _dc_replace( + DEFAULT_CONFIG, + estimation_enabled=enabled, + accrue_realized_knowledge=True, + persist_session_state=True, + ) + return ChatRuntime(config=cfg, engine_state_path=tmp_path) + + +def test_licensed_converse_is_served_disclosed_approximate(tmp_path) -> None: + rt = _estimation_runtime(tmp_path) + rt.chat("Alice is the sibling of Bob.") # told sibling_of(alice, bob) + resp = rt.chat("Is Bob the sibling of Alice?") # converse — DETERMINE refuses + assert resp.reach_level == "approximate" + assert resp.surface.startswith("[approximate]") # DISCLOSED, never asserted as fact + assert "bob" in resp.surface and "alice" in resp.surface + + +def test_unlicensed_converse_stays_strict_refusal(tmp_path) -> None: + rt = _estimation_runtime(tmp_path) + rt.chat("Alice is the parent of Bob.") # told parent_of(alice, bob) — DIRECTED + resp = rt.chat("Is Bob the parent of Alice?") + # parent_of's converse-guess is not SERVE-licensed → no widening, honest refusal. + assert resp.reach_level == "strict" + assert not resp.surface.startswith("[approximate]") + + +def test_estimation_flag_off_is_strict(tmp_path) -> None: + rt = _estimation_runtime(tmp_path, enabled=False) + rt.chat("Alice is the sibling of Bob.") + resp = rt.chat("Is Bob the sibling of Alice?") + assert resp.reach_level == "strict" + assert not resp.surface.startswith("[approximate]") diff --git a/tests/test_response_governance.py b/tests/test_response_governance.py index c6817819..be7c0c16 100644 --- a/tests/test_response_governance.py +++ b/tests/test_response_governance.py @@ -34,25 +34,27 @@ from core.response_governance import ( shape_surface, ) -# A small but representative cross-product of governance inputs. The -# scaffold must return STRICT for ALL of them. +# A small but representative cross-product of governance inputs. None of these +# license standins is a GENUINE licensed Action.SERVE LicenseDecision, so +# govern_response must return STRICT for ALL of them — only a ratified license widens +# (Step E). The forged dict ``{"licensed": True}`` is the load-bearing standin: a +# caller's say-so must NEVER widen. _ALL_STATES = tuple(EpistemicState) _LICENSE_STANDINS = (None, object(), {"licensed": True}, {"licensed": False}) _STAKES_STANDINS = (None, "high", "low", 0.0, 1.0) -# --- govern_response: STRICT-only contract ---------------------------------- +# --- govern_response: STRICT unless a genuine SERVE license ------------------- @pytest.mark.parametrize("state", _ALL_STATES) @pytest.mark.parametrize("license_decision", _LICENSE_STANDINS) @pytest.mark.parametrize("stakes", _STAKES_STANDINS) -def test_govern_response_is_strict_only(state, license_decision, stakes): - """The stub governs every input at STRICT — the wrong==0 load-bearing line. - - If this fails, the scaffold has begun widening before the risk-reward - loop and its proofs exist; that is the exact regression the STRICT-only - contract forbids. +def test_govern_response_strict_without_a_genuine_license(state, license_decision, stakes): + """STRICT for every input that is NOT a genuine licensed SERVE decision — the + wrong==0 load-bearing line. A None, a bare object, or a forged ``{"licensed": True}`` + dict must NOT widen: widening rests on the gate's verdict over a committed ledger, + never on a caller's say-so. """ policy = govern_response( epistemic_state=state, license_decision=license_decision, stakes=stakes @@ -61,6 +63,26 @@ def test_govern_response_is_strict_only(state, license_decision, stakes): assert policy is STRICT_POLICY +def test_govern_response_widens_only_on_genuine_serve_license(): + """Step E (ADR-0206 §5): a REAL licensed Action.SERVE LicenseDecision widens to + APPROXIMATE; an unlicensed one, or a PROPOSE license, stays STRICT.""" + from core.reliability_gate import Action, Ceilings, ClassTally, license_for + from core.response_governance import APPROXIMATE_POLICY + + licensed = license_for(ClassTally("c", correct=660), Action.SERVE, Ceilings.default()) + assert licensed.licensed is True + assert govern_response(license_decision=licensed) is APPROXIMATE_POLICY + + unlicensed = license_for(ClassTally("c", wrong=660), Action.SERVE, Ceilings.default()) + assert unlicensed.licensed is False + assert govern_response(license_decision=unlicensed) is STRICT_POLICY + + # A PROPOSE license is NOT a SERVE license — it must not widen a served answer. + propose = license_for(ClassTally("c", correct=660), Action.PROPOSE, Ceilings.default()) + assert propose.licensed is True and propose.action is Action.PROPOSE + assert govern_response(license_decision=propose) is STRICT_POLICY + + def test_strict_policy_admits_only_decoded(): """STRICT's admissible set is exactly {DECODED} — fully-grounded only.""" assert STRICT_POLICY.admissible_states == frozenset({EpistemicState.DECODED})