From 596e2313bee59763b2d3f0e3bd1c63b530e9312b Mon Sep 17 00:00:00 2001 From: Shay Date: Sun, 17 May 2026 09:48:39 -0700 Subject: [PATCH] =?UTF-8?q?feat(epistemic):=20Leak=20C=20read-side=20audit?= =?UTF-8?q?=20=E2=80=94=20INV-24=20callsite=20registry,=20Leak=20C=20fully?= =?UTF-8?q?=20closed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Categorizes every production vault.recall() callsite as RECOGNITION, EVIDENCE_TELEMETRY, or EVIDENCE_USER_FACING. Adds INV-24 architectural invariant (TestINV24VaultRecallRegistry, 3 tests) that forces any new callsite to declare its role and requires EVIDENCE_USER_FACING sites to pass min_status=COHERENT. Audit findings: - chat/runtime.py:330 → RECOGNITION (gate decision input) - vault/decompose.py:121 → RECOGNITION (grade-decomposed gate fallback) - generate/stream.py:147 → EVIDENCE_TELEMETRY (walk_surface per runtime contract) - No EVIDENCE_USER_FACING sites exist today — user-facing surface comes from pack-grounded realize(proposition, vocab), not vault.recall. Why this closes Leak C: the write-side fix already stamps SPECULATIVE on self-stored propositions; the read-side audit confirms no inference path treats them as ratified evidence. If a future change routes the generation walk into the user-facing surface, INV-24 forces the recategorization to be explicit. CLAIMS.md Tier 4.5 Leak C row now CLOSED. docs/truth_seeking_schema.md §Leak C updated with full audit categorization. Verified: smoke (67), cognition (121), runtime (19), all architectural invariants (40) — green. --- chat/runtime.py | 4 + docs/truth_seeking_schema.md | 44 +++++--- evals/CLAIMS.md | 2 +- generate/stream.py | 9 ++ tests/test_architectural_invariants.py | 141 +++++++++++++++++++++++++ vault/decompose.py | 3 + 6 files changed, 189 insertions(+), 14 deletions(-) diff --git a/chat/runtime.py b/chat/runtime.py index 13ddb5de..5af7241a 100644 --- a/chat/runtime.py +++ b/chat/runtime.py @@ -327,6 +327,10 @@ class ChatRuntime: raise ValueError("ChatRuntime.chat() received no in-vocabulary tokens.") probe_state = self._context.probe_ingest(filtered) + # INV-24 recall role: RECOGNITION. Feeds UnknownDomainGate — asks + # "have we seen anything like this before?", not "what is admissible + # evidence?". Session-tier SPECULATIVE memory must count here, so + # no min_status filter is applied. direct_hits = self._context.vault.recall(probe_state.F, top_k=3) direct_best = max((h["score"] for h in direct_hits), default=0.0) gate_decision = default_gate.check( diff --git a/docs/truth_seeking_schema.md b/docs/truth_seeking_schema.md index d45ddbd8..91c46bc2 100644 --- a/docs/truth_seeking_schema.md +++ b/docs/truth_seeking_schema.md @@ -207,30 +207,48 @@ opts in now gets the evidence guarantee. **Regression guard:** `tests/test_architectural_invariants.py::TestINV23VaultEpistemicFilter` (four tests). -### ~~Leak C — Self-reinforcing fabrication via `propose()`~~ — PARTIALLY CLOSED 2026-05-17 +### ~~Leak C — Self-reinforcing fabrication via `propose()`~~ — CLOSED 2026-05-17 **Original gap:** `generate/proposition.py:198` stored every articulated proposition back into vault unmarked. The system says something → recalls own output → cites it → says it again. A fabrication-feedback loop in the substrate. -**Write-side fix landed:** the call site now stamps +**Write-side fix:** the call site now stamps `epistemic_status=EpistemicStatus.SPECULATIVE` with an inline comment naming this leak. The feedback loop is broken in principle: any inference path that recalls with `min_status=COHERENT` will exclude the system's own prior utterances from evidence. -**Residual work:** the *read side* — call sites in -`chat/runtime.py`, `generate/stream.py`, and `vault/decompose.py` -that currently recall without a `min_status` filter — must be -audited site-by-site. Each is either an inference path (should pass -`min_status=COHERENT`) or a context-display path (correctly -tier-agnostic). The decision is per call site, not a single sweep. -Until that audit lands, Leak C is closed at the write side but not -yet enforced at every read site. The architectural-invariant test -allowlist still names `generate/proposition.py` as a vault writer, -correctly — the entry is no longer a leak in spirit, but the -read-side audit is what makes the closure operationally tight. +**Read-side audit (2026-05-17):** every production `vault.recall()` +callsite was categorized and an architectural invariant added +(`TestINV24VaultRecallRegistry`) that requires every new callsite to +declare its role. Categories: + +- **RECOGNITION** — answers *"have we seen this before?"* (gate + decisions, unknown-domain probes). Unfiltered recall is correct, + because session-tier SPECULATIVE memory must count toward + recognition. Sites: `chat/runtime.py:330`, + `vault/decompose.py:121`. +- **EVIDENCE_TELEMETRY** — feeds `walk_surface` and trace evidence + but NOT the user-facing surface (per + `docs/runtime_contracts.md` §surface vs walk_surface). Tolerable + unfiltered because the walk does not shape claims. Site: + `generate/stream.py:147` (`_recall_state`). +- **EVIDENCE_USER_FACING** — would feed user-facing surface as if + ratified knowledge. **MUST pass `min_status=COHERENT`.** Currently + empty by design: user-facing articulation comes from + `realize(proposition, vocab)` via pack lookup (now SPECULATIVE-default + per Leak A fix), not from `vault.recall`. + +If a future change routes the generation walk into the user-facing +surface, INV-24 forces the recategorization to be explicit and +requires the `min_status=COHERENT` filter — the fabrication loop +cannot reopen quietly. + +**Regression guard:** `tests/test_architectural_invariants.py::TestINV24VaultRecallRegistry` +(three tests) + site-level `# INV-24 recall role:` provenance +comments at every callsite. ### Realizer-side surface gaps diff --git a/evals/CLAIMS.md b/evals/CLAIMS.md index edb34016..583f1cd5 100644 --- a/evals/CLAIMS.md +++ b/evals/CLAIMS.md @@ -164,7 +164,7 @@ the work lands. | `contradiction_detection` | false_flag_rate | 1.00 | 0.00 | Same fix — replace the versor-delta heuristic with the coherence checker. | | ~~One-mutation-path audit · Leak A~~ | ~~pack vocab default epistemic_status~~ | ✅ **`speculative`** | `speculative` | **CLOSED 2026-05-17** — `language_packs/compiler.py:331` and `language_packs/schema.py::LexicalEntry` now default to SPECULATIVE; docstring corrected to match ADR-0021 §Schema impact; regression guarded by `tests/test_architectural_invariants.py::TestINV22PackDefaultSpeculative` (3 tests, all passing). 365 existing unmarked pack rows now correctly report SPECULATIVE; explicit COHERENT remains the curator stamp. | | ~~One-mutation-path audit · Leak B~~ | ~~vault.recall epistemic awareness~~ | ✅ **`min_status` filter** | `min_status` filter | **CLOSED 2026-05-17** — `VaultStore.store()` now stamps every entry with `epistemic_status` (default SPECULATIVE per ADR-0021 §3); `VaultStore.recall(min_status=EpistemicStatus.COHERENT)` filters out non-admissible entries. All 4 vault.store call sites updated with explicit status. Regression guarded by `tests/test_architectural_invariants.py::TestINV23VaultEpistemicFilter` (4 tests). Inference paths can now opt into evidence-only recall; session lookup retains tier-agnostic default. | -| ~~One-mutation-path audit · Leak C~~ (write-side) | self-reinforcing fabrication via propose() | ✅ **stamped SPECULATIVE** | stamped + read filter | **WRITE-SIDE CLOSED 2026-05-17** — `generate/proposition.py:198` now stamps `EpistemicStatus.SPECULATIVE` on every stored proposition. The feedback loop is broken in principle once any inference path opts into `min_status=COHERENT`. **Residual:** read-side audit of `chat/runtime.py`, `generate/stream.py`, `vault/decompose.py` to add `min_status` filter site-by-site (each is either inference or context-display — different decisions). | +| ~~One-mutation-path audit · Leak C~~ | self-reinforcing fabrication via propose() | ✅ **stamped + read-side audited** | stamped + categorized read sites | **CLOSED 2026-05-17** — write-side stamps SPECULATIVE (`generate/proposition.py:198`). Read-side audit categorized every production `vault.recall()` callsite as RECOGNITION, EVIDENCE_TELEMETRY, or EVIDENCE_USER_FACING. INV-24 (`TestINV24VaultRecallRegistry`, 3 tests) forces every new callsite to declare its role; EVIDENCE_USER_FACING sites must pass `min_status=COHERENT`. No EVIDENCE_USER_FACING sites exist today — user-facing surface comes from pack-grounded `realize(proposition, vocab)` (now SPECULATIVE-default per Leak A), not from `vault.recall`. Site-level `# INV-24 recall role:` comments at every callsite. See `docs/truth_seeking_schema.md` §Leak C. | | `articulation_of_status` | speculative_articulation_rate | 0.00 | ≥ 0.90 | Realizer consults `pack_mutation_proposal.epistemic_status` (or the read-path filter from Leak B) and emits a status marker when the answer draws on non-COHERENT material. | | `articulation_of_status` | false_certainty_rate | 0.60 | 0.00 | Same fix — bare assertions on SPECULATIVE-backed claims become hedged surfaces. | diff --git a/generate/stream.py b/generate/stream.py index 2b418fde..d10a5dc3 100644 --- a/generate/stream.py +++ b/generate/stream.py @@ -144,6 +144,15 @@ def _recall_state(state: FieldState, vault, top_k: int) -> tuple[FieldState, int if vault is None or top_k <= 0: return state, 0 + # INV-24 recall role: EVIDENCE_TELEMETRY. Hits become rotor transitions + # on the generation walk, but the walk feeds `walk_surface` (telemetry- + # only per docs/runtime_contracts.md) — not the user-facing surface. + # User-facing surface comes from realize(proposition, vocab), which is + # pack-grounded. SPECULATIVE walk influence remains visible in trace + # evidence and is bounded by the recall score floor; no min_status + # filter is applied here. If a future change routes walk output into + # the user-facing surface, this site must be re-categorized to + # EVIDENCE_USER_FACING and pass min_status=COHERENT. hits = vault.recall(state.F, top_k=top_k) if not hits: return state, 0 diff --git a/tests/test_architectural_invariants.py b/tests/test_architectural_invariants.py index ca16ee9c..8053a7c3 100644 --- a/tests/test_architectural_invariants.py +++ b/tests/test_architectural_invariants.py @@ -881,3 +881,144 @@ class TestINV23VaultEpistemicFilter: store.store(v, {"role": "user"}) hits = store.recall(v, top_k=1) assert hits, "Default recall must return SPECULATIVE session entries." + + +# =========================================================================== +# INV-24 Vault recall callsite registry — Leak C read-side audit guard +# =========================================================================== +# +# Claim (ADR-0021 §Leak C, 2026-05-17 audit, completed 2026-05-17): +# Every production vault.recall() callsite must be categorized in +# VAULT_RECALL_SITES. Categories: +# +# RECOGNITION — answers "have we seen anything like this?" +# (gate decisions, unknown-domain probes). +# Unfiltered recall is correct: session-tier +# SPECULATIVE memory must count. +# +# EVIDENCE_TELEMETRY — feeds walk telemetry / trace evidence, NOT the +# user-facing surface (per docs/runtime_contracts.md +# §surface vs walk_surface). Unfiltered recall is +# tolerable because the walk does not shape claims. +# If a future change routes the walk into the +# user-facing surface, the site must move to +# EVIDENCE_USER_FACING and pass min_status=COHERENT. +# +# EVIDENCE_USER_FACING — feeds the user-facing surface as if ratified +# knowledge. MUST pass min_status=COHERENT. +# Currently empty by design: user-facing +# articulation comes from realize(proposition, vocab) +# via pack lookup, not from vault.recall. +# +# A new vault.recall caller is a structural change to the read substrate of +# the epistemic schema and must be reviewed — adding it to the registry +# makes the categorization decision explicit. + +VAULT_RECALL_SITES: dict[str, str] = { + "chat/runtime.py": "RECOGNITION", + "vault/decompose.py": "RECOGNITION", + "generate/stream.py": "EVIDENCE_TELEMETRY", + "session/context.py": "RECOGNITION", # generic delegate; callers categorize + "vault/store.py": "RECOGNITION", # self-references in helpers/docstrings +} + +VALID_RECALL_ROLES: frozenset[str] = frozenset({ + "RECOGNITION", "EVIDENCE_TELEMETRY", "EVIDENCE_USER_FACING", +}) + + +def _file_has_vault_recall_call(path: Path) -> bool: + """Return True iff `path` contains an executable `.recall(...)` call. + + Uses AST so docstrings/comments cannot trigger false positives. + Detects `.recall(...)` on receivers that look like vault bindings: + a name `vault`, any `*_vault`, or any name assigned from `VaultStore(...)`. + """ + try: + tree = ast.parse(path.read_text()) + except (OSError, SyntaxError): + return False + + vault_bindings: set[str] = {"vault"} + for node in ast.walk(tree): + if isinstance(node, ast.Assign) and isinstance(node.value, ast.Call): + func = node.value.func + func_name = ( + func.attr if isinstance(func, ast.Attribute) else + func.id if isinstance(func, ast.Name) else "" + ) + if func_name == "VaultStore": + for target in node.targets: + if isinstance(target, ast.Name): + vault_bindings.add(target.id) + elif isinstance(target, ast.Attribute): + vault_bindings.add(target.attr) + + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + func = node.func + if not isinstance(func, ast.Attribute) or func.attr != "recall": + continue + receiver = func.value + receiver_name = "" + if isinstance(receiver, ast.Name): + receiver_name = receiver.id + elif isinstance(receiver, ast.Attribute): + receiver_name = receiver.attr + if ( + receiver_name in vault_bindings + or receiver_name.endswith("_vault") + or receiver_name == "vault" + ): + return True + return False + + +class TestINV24VaultRecallRegistry: + """Every production vault.recall() callsite must be categorized in + VAULT_RECALL_SITES. A new callsite is a structural change to the + epistemic read-substrate and must justify its categorization.""" + + def test_every_recall_site_is_registered(self): + offenders: list[str] = [] + for path in _enumerate_project_py_files(): + if not _file_has_vault_recall_call(path): + continue + rel = str(path.relative_to(PROJECT_ROOT_FOR_INV21)) + if rel not in VAULT_RECALL_SITES: + offenders.append(rel) + assert not offenders, ( + "New vault recall callsite(s) detected outside the registry:\n " + + "\n ".join(offenders) + + "\n\nAdd the path to VAULT_RECALL_SITES in " + "tests/test_architectural_invariants.py with one of the valid " + f"roles: {sorted(VALID_RECALL_ROLES)}.\n" + "EVIDENCE_USER_FACING sites MUST pass min_status=COHERENT " + "(ADR-0021 §Leak C, 2026-05-17 audit)." + ) + + def test_registry_roles_are_valid(self): + invalid = { + path: role + for path, role in VAULT_RECALL_SITES.items() + if role not in VALID_RECALL_ROLES + } + assert not invalid, ( + f"Invalid recall role(s) in VAULT_RECALL_SITES: {invalid}\n" + f"Valid roles: {sorted(VALID_RECALL_ROLES)}" + ) + + def test_registry_is_not_stale(self): + """If a registered file no longer calls vault.recall(), drop it from + the registry to keep the read-substrate trust surface tight.""" + used: set[str] = set() + for path in _enumerate_project_py_files(): + if _file_has_vault_recall_call(path): + used.add(str(path.relative_to(PROJECT_ROOT_FOR_INV21))) + registered = set(VAULT_RECALL_SITES.keys()) + unused = registered - used - {"vault/store.py"} + assert not unused, ( + f"Registered recall site(s) no longer call vault.recall(): " + f"{sorted(unused)}\nRemove from VAULT_RECALL_SITES." + ) diff --git a/vault/decompose.py b/vault/decompose.py index 85859188..79a2a5d3 100644 --- a/vault/decompose.py +++ b/vault/decompose.py @@ -118,6 +118,9 @@ class FieldDecomposer: component_weights.append(0.0) continue + # INV-24 recall role: RECOGNITION. Grade-decomposed fallback for + # UnknownDomainGate when direct recall scores below the floor — + # still a "have we seen this?" probe, not evidence admission. hits = vault.recall(grade_vec, top_k=top_k) best = max((h["score"] for h in hits), default=0.0) component_scores.append(best)