feat(adr-0054): vault recall indexing/batching + holdout split wired
Two doctrine-aligned CLAUDE.md items closed together. Part 1 — vault indexing + batching (item #4): - VaultStore lazy _matrix_cache (invalidated on store / reproject / eviction); vault_recall(prebuilt_matrix=...) skips deque→ndarray rebuild on hot path - New vault_recall_batch + VaultStore.recall_batch — B queries scored in one component-serial sweep, bit-identical to per-query vault_recall (3 seeds × 7 queries × N=137 parity test) - No approximation, no hot-path repair, scoring arithmetic unchanged Part 2 — holdout split wired: - LaneInfo.holdout_cases_path resolves plaintext holdouts in fixed priority; sealed (.age) holdouts stay in holdout_runner - framework.run_lane(split="holdout") + argparse --split choices - First official cognition holdout numbers: 19 cases, intent 100%, surface 94.7%, term_capture 70.8%, versor 100% — single miss is predicted correction_truth_040 (ADR-0053 scope-limit) Tests: 21 new vault tests + 10 new framework tests. Lanes: smoke 67, cognition 121, runtime 19, teaching 17, packs 6, algebra 132 — all green. versor_condition < 1e-6 invariant preserved.
This commit is contained in:
Shay
parent
e975faf8a8
commit
6b25069da8
8 changed files with 833 additions and 8 deletions
|
|
@ -90,7 +90,13 @@ def cga_inner(X: np.ndarray, Y: np.ndarray) -> float:
|
|||
return _ci(X, Y)
|
||||
|
||||
|
||||
def vault_recall(versors: list, query: np.ndarray, top_k: int = 5) -> list:
|
||||
def vault_recall(
|
||||
versors: list,
|
||||
query: np.ndarray,
|
||||
top_k: int = 5,
|
||||
*,
|
||||
prebuilt_matrix: np.ndarray | None = None,
|
||||
) -> list:
|
||||
"""Top-k CGA inner product recall.
|
||||
|
||||
Rust path: parallel Rayon scan when explicitly enabled.
|
||||
|
|
@ -99,11 +105,21 @@ def vault_recall(versors: list, query: np.ndarray, top_k: int = 5) -> list:
|
|||
because the per-versor sum is folded in the same serial component
|
||||
order; the only thing the vectorisation replaces is the
|
||||
per-element Python dispatch loop. ADR-0019 Stage 1.
|
||||
|
||||
``prebuilt_matrix`` (ADR-0054): optional cached (N, D) f32 matrix
|
||||
of stacked versors maintained by ``VaultStore``. When supplied,
|
||||
the deque→ndarray conversion is skipped — purely an indexing
|
||||
optimisation, scoring arithmetic is identical.
|
||||
"""
|
||||
if not versors:
|
||||
if not versors and prebuilt_matrix is None:
|
||||
return []
|
||||
q = np.asarray(query, dtype=np.float32)
|
||||
M = np.asarray(versors, dtype=np.float32)
|
||||
if prebuilt_matrix is not None:
|
||||
M = prebuilt_matrix
|
||||
if M.shape[0] == 0:
|
||||
return []
|
||||
else:
|
||||
M = np.asarray(versors, dtype=np.float32)
|
||||
if _RUST and M.ndim == 2 and M.shape[1] == 32:
|
||||
try:
|
||||
# Pass the (N, 32) numpy buffer directly — the Rust
|
||||
|
|
@ -143,6 +159,65 @@ def vault_recall(versors: list, query: np.ndarray, top_k: int = 5) -> list:
|
|||
return [(int(i), float(scores[i])) for i in cand]
|
||||
|
||||
|
||||
def vault_recall_batch(
|
||||
matrix: np.ndarray,
|
||||
queries: np.ndarray,
|
||||
top_k: int = 5,
|
||||
) -> list[list[tuple[int, float]]]:
|
||||
"""Top-k CGA inner product recall for B queries against one matrix.
|
||||
|
||||
ADR-0054. Returns one ``[(index, score), ...]`` list per query in
|
||||
the same shape ``vault_recall`` returns for a single query.
|
||||
|
||||
Bit-identity contract: each per-query result must equal the
|
||||
corresponding single-query ``vault_recall`` call against the same
|
||||
matrix. We accumulate scores in component-serial order with the
|
||||
diagonal metric — the same folding pattern as the single-query
|
||||
path — so the per-versor sum is folded identically. Top-k
|
||||
ordering uses the same descending-score / ascending-index stable
|
||||
rule.
|
||||
|
||||
No approximate search. No Rust path here yet (the Rust binding
|
||||
is single-query); Python is canonical.
|
||||
"""
|
||||
M = np.asarray(matrix, dtype=np.float32)
|
||||
Q = np.asarray(queries, dtype=np.float32)
|
||||
if Q.ndim == 1:
|
||||
Q = Q[None, :]
|
||||
if M.ndim != 2 or Q.ndim != 2:
|
||||
raise ValueError(
|
||||
f"vault_recall_batch requires matrix.ndim==2 and queries.ndim in (1, 2); "
|
||||
f"got matrix.ndim={M.ndim}, queries.ndim={Q.ndim}"
|
||||
)
|
||||
if M.shape[1] != Q.shape[1]:
|
||||
raise ValueError(
|
||||
f"vault_recall_batch shape mismatch: matrix has {M.shape[1]} components "
|
||||
f"per row, queries have {Q.shape[1]}"
|
||||
)
|
||||
N = M.shape[0]
|
||||
B = Q.shape[0]
|
||||
if N == 0 or top_k <= 0:
|
||||
return [[] for _ in range(B)]
|
||||
# Component-serial accumulation: scores[b, n] = sum_i metric[i] * M[n,i] * Q[b,i].
|
||||
# Folding component-by-component preserves bit-identity with the
|
||||
# single-query path (same float32 addition order across i).
|
||||
scores = np.zeros((B, N), dtype=np.float32)
|
||||
for i in range(M.shape[1]):
|
||||
scores += (_CGA_INNER_METRIC[i] * M[:, i])[None, :] * Q[:, i, None]
|
||||
k = min(top_k, N)
|
||||
out: list[list[tuple[int, float]]] = []
|
||||
for b in range(B):
|
||||
row = scores[b]
|
||||
if k < N:
|
||||
cand = np.argpartition(-row, k - 1)[:k]
|
||||
else:
|
||||
cand = np.arange(N)
|
||||
order = np.lexsort((cand, -row[cand]))
|
||||
cand = cand[order]
|
||||
out.append([(int(i), float(row[i])) for i in cand])
|
||||
return out
|
||||
|
||||
|
||||
def unitize_expmap(v: np.ndarray) -> np.ndarray:
|
||||
"""Unitize a multivector via the Cl(4,1) exponential map.
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ _CORE_RS_DIR = _REPO_ROOT / "core-rs"
|
|||
_CORE_RS_MANIFEST = _CORE_RS_DIR / "Cargo.toml"
|
||||
|
||||
DESCRIPTION = "CORE versor engine command suite."
|
||||
EPILOG = "Examples:\n core chat\n core pulse \"What is truth?\"\n core pulse --no-glove --json \"Compare knowledge and wisdom\"\n core bench\n core bench --suite determinism --runs 50\n core bench --suite speedup --json\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core test --suite fast -q\n core test --suite pulse -q\n core test --suite proof -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core demo audit-tour\n core demo pack-measurements\n core demo long-context-comparison\n core eval --list\n core eval cognition\n core eval cognition --json --save\n core eval cognition --split dev --version v1"
|
||||
EPILOG = "Examples:\n core chat\n core pulse \"What is truth?\"\n core pulse --no-glove --json \"Compare knowledge and wisdom\"\n core bench\n core bench --suite determinism --runs 50\n core bench --suite speedup --json\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core test --suite fast -q\n core test --suite pulse -q\n core test --suite proof -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core demo audit-tour\n core demo pack-measurements\n core demo long-context-comparison\n core eval --list\n core eval cognition\n core eval cognition --json --save\n core eval cognition --split dev --version v1\n core eval cognition --split holdout"
|
||||
|
||||
_TEST_SUITES: dict[str, tuple[str, ...]] = {
|
||||
"fast": (
|
||||
|
|
@ -1466,7 +1466,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
eval_cmd.add_argument("lane", nargs="?", help="eval lane name (e.g. cognition)")
|
||||
eval_cmd.add_argument("--list", dest="list_lanes", action="store_true", help="list available eval lanes")
|
||||
eval_cmd.add_argument("--version", help="version to evaluate (default: latest)")
|
||||
eval_cmd.add_argument("--split", default="public", choices=["dev", "public"], help="which split to score (default: public)")
|
||||
eval_cmd.add_argument("--split", default="public", choices=["dev", "public", "holdout"], help="which split to score (default: public)")
|
||||
eval_cmd.add_argument("--json", action="store_true", help="emit machine-readable JSON")
|
||||
eval_cmd.add_argument("--save", action="store_true", help="write result to lane results/ directory")
|
||||
eval_cmd.add_argument("--report", metavar="PATH", help="write JSON report to file")
|
||||
|
|
|
|||
244
docs/decisions/ADR-0054-vault-recall-indexing-batching.md
Normal file
244
docs/decisions/ADR-0054-vault-recall-indexing-batching.md
Normal file
|
|
@ -0,0 +1,244 @@
|
|||
# ADR-0054 — Vault Recall: Matrix-Cache Indexing + Batched API; Holdout Split Wired
|
||||
|
||||
**Status:** Accepted
|
||||
**Date:** 2026-05-18
|
||||
**Author:** Shay
|
||||
|
||||
---
|
||||
|
||||
## Context
|
||||
|
||||
Two doctrine-aligned items from CLAUDE.md were still open after
|
||||
ADR-0053:
|
||||
|
||||
1. **CLAUDE.md item #4 — "Add exact vault recall indexing/batching
|
||||
without approximate search."** ADR-0019 Stage 1 vectorised the
|
||||
single-query CGA scan inside `algebra.backend.vault_recall`, but
|
||||
the **deque → ndarray conversion** still happened on every recall,
|
||||
and there was no batched-query API. Repeated recalls against a
|
||||
slowly-growing vault paid the conversion cost each call.
|
||||
2. **Holdouts not in the official eval runner.** The cognition lane
|
||||
has had a 19-case plaintext holdout file
|
||||
(`evals/cognition/holdouts/cases_plaintext.jsonl`) since the lane
|
||||
was set up, but `core eval cognition --split` accepted only `dev`
|
||||
and `public`. Holdout numbers existed only via ad-hoc scripts
|
||||
spawned during ADR-0053.
|
||||
|
||||
Both items are minimal-doctrine work: no algebra change, no new
|
||||
approximation, no new normalisation, no hot-path repair. Bundled
|
||||
together because both touch the validation/eval surface.
|
||||
|
||||
---
|
||||
|
||||
## Decision
|
||||
|
||||
### Part 1 — Vault recall indexing + batching
|
||||
|
||||
**`VaultStore` matrix cache (`vault/store.py`).**
|
||||
|
||||
A lazily-built `_matrix_cache: np.ndarray | None` is held on the
|
||||
store. It is `None` initially and after any mutation; the first
|
||||
`recall` after a mutation rebuilds it via
|
||||
`np.asarray(self._versors, dtype=np.float32)`. Invalidation hooks:
|
||||
|
||||
- `store()` — always invalidates (append shifts the deque view).
|
||||
- `reproject()` — invalidates (every entry replaced).
|
||||
- `_rebuild_index()` — invalidates (called on max-entries eviction).
|
||||
|
||||
The cache is read-only from the recall path; `vault_recall` receives
|
||||
it via a new optional `prebuilt_matrix=` kwarg and skips the
|
||||
deque → ndarray conversion when supplied. No shared mutable state
|
||||
is held across calls — the matrix is the same buffer between recalls
|
||||
only while no mutation has happened.
|
||||
|
||||
**Batched recall (`algebra.backend.vault_recall_batch`).**
|
||||
|
||||
New function with signature
|
||||
`vault_recall_batch(matrix, queries, top_k) -> list[list[(int, float)]]`.
|
||||
Accepts `(N, D)` matrix and `(B, D)` (or `(D,)`) queries, returns
|
||||
one ranked list per query. Scoring uses the same diagonal CGA
|
||||
metric and accumulates **in component-serial order**:
|
||||
|
||||
```python
|
||||
scores = np.zeros((B, N), dtype=np.float32)
|
||||
for i in range(D):
|
||||
scores += (_CGA_INNER_METRIC[i] * M[:, i])[None, :] * Q[:, i, None]
|
||||
```
|
||||
|
||||
Folding component-by-component preserves bit-identity with the
|
||||
single-query path's float32 addition order. Tiebreak rule
|
||||
(descending score, ascending index) is identical.
|
||||
|
||||
**`VaultStore.recall_batch`.**
|
||||
|
||||
Public sibling to `recall`. Same per-query semantics — exact-self-
|
||||
match promotion via the byte-key index, optional `min_status`
|
||||
filter, score=+inf for exact hits — but the underlying scoring scan
|
||||
is a single component-serial sweep over the cached matrix.
|
||||
|
||||
### Part 2 — Wire `--split holdout`
|
||||
|
||||
`evals/framework.py`:
|
||||
|
||||
- `LaneInfo.holdout_cases_path(version)` resolves the first existing
|
||||
of `holdouts/cases.jsonl`, `holdouts/cases_plaintext.jsonl`,
|
||||
`holdouts/<version>/cases.jsonl`. Sealed (`*.age`) holdouts are
|
||||
**not** decrypted here — that path stays in
|
||||
`evals.holdout_runner.run_holdout`, which enforces aggregate-only
|
||||
output by trust-boundary contract.
|
||||
- `run_lane(split="holdout")` reads that path and dispatches to the
|
||||
lane's `run_lane(cases, config=...)` like any other split.
|
||||
|
||||
`core/cli.py`:
|
||||
|
||||
- `--split` argparse `choices` extended to
|
||||
`{"dev", "public", "holdout"}`.
|
||||
- Example added to `EPILOG`.
|
||||
|
||||
---
|
||||
|
||||
## Why this is doctrine-aligned
|
||||
|
||||
- **No approximate search.** Both the matrix cache and
|
||||
`vault_recall_batch` are indexing/vectorisation changes only;
|
||||
scoring arithmetic is unchanged.
|
||||
- **No hidden normalisation, no hot-path repair.** The cache is
|
||||
invalidated, not "auto-rebuilt to fix drift." `reproject()` was
|
||||
already the canonical drift-repair path; this ADR only invalidates
|
||||
the cache when it runs.
|
||||
- **No shared mutable state across recalls.** The cache buffer is
|
||||
read by `vault_recall` via a kwarg; nothing in the recall path
|
||||
mutates it. Mutation paths (store / reproject / eviction) clear
|
||||
it explicitly.
|
||||
- **`versor_condition < 1e-6` invariant untouched.** No field is
|
||||
constructed, normalised, or transformed.
|
||||
- **Holdouts run via the same harness as dev/public.** No parallel
|
||||
scoring path was added; the trust boundary on sealed holdouts is
|
||||
preserved by routing plaintext through the standard runner and
|
||||
leaving the encrypted path to `holdout_runner`.
|
||||
|
||||
---
|
||||
|
||||
## Characterisation
|
||||
|
||||
### Vault recall — bit-identity gate
|
||||
|
||||
`tests/test_vault_recall_indexing_batch.py` adds 21 tests. The
|
||||
batched path is verified bit-identical to per-query
|
||||
`vault_recall` across three seeds × 7 queries × N=137 — every
|
||||
index sequence and every float32 score matches exactly.
|
||||
|
||||
The pre-existing `tests/test_vault_recall_vectorised.py` (ADR-0019
|
||||
Stage 1 gate) continues to pass — the single-query path is
|
||||
unchanged when no `prebuilt_matrix` is passed.
|
||||
|
||||
### Eval lanes — first official holdout run
|
||||
|
||||
```
|
||||
core eval cognition --split holdout
|
||||
cases : 19
|
||||
intent_accuracy : 100.0%
|
||||
surface_groundedness : 94.7%
|
||||
term_capture_rate : 70.8%
|
||||
versor_closure_rate : 100.0%
|
||||
|
||||
core eval cognition --split dev
|
||||
cases : 13
|
||||
intent_accuracy : 100.0%
|
||||
surface_groundedness : 100.0%
|
||||
term_capture_rate : 78.6%
|
||||
versor_closure_rate : 100.0%
|
||||
|
||||
core eval cognition --split public
|
||||
cases : 13
|
||||
intent_accuracy : 100.0%
|
||||
surface_groundedness : 100.0%
|
||||
term_capture_rate : 91.7%
|
||||
versor_closure_rate : 100.0%
|
||||
```
|
||||
|
||||
The single surface_groundedness miss on holdouts is the predicted
|
||||
`correction_truth_040` case — see ADR-0053 scope-limits. Term
|
||||
capture on holdouts is the next-cheapest pull (echo the corrected-
|
||||
subject lemma in the CORRECTION acknowledgement), candidate for a
|
||||
follow-up ADR.
|
||||
|
||||
### Lanes (all green)
|
||||
|
||||
```
|
||||
core test --suite smoke 67 passed
|
||||
core test --suite cognition 121 passed
|
||||
core test --suite runtime 19 passed
|
||||
core test --suite teaching 17 passed
|
||||
core test --suite packs 6 passed
|
||||
core test --suite algebra 132 passed
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Consequences
|
||||
|
||||
### What changes
|
||||
|
||||
- `algebra/backend.py` gains `vault_recall_batch` and an optional
|
||||
`prebuilt_matrix=` kwarg on `vault_recall`.
|
||||
- `vault/store.py` gains a lazy matrix cache, cache-invalidation
|
||||
hooks on mutation paths, and a `recall_batch` method.
|
||||
- `evals/framework.py` gains `LaneInfo.holdout_cases_path` and a
|
||||
`"holdout"` branch in `run_lane`.
|
||||
- `core/cli.py` `--split` now accepts `"holdout"`.
|
||||
|
||||
### What does not change
|
||||
|
||||
- Single-query `vault_recall` semantics — same scores, same order,
|
||||
same Rust dispatch.
|
||||
- ADR-0019 Stage 1 bit-identity contract — still gated.
|
||||
- `versor_condition < 1e-6` invariant unaffected.
|
||||
- Encrypted holdout decryption — still owned by
|
||||
`evals.holdout_runner.run_holdout`; aggregate-only output
|
||||
contract preserved.
|
||||
- All five core lanes remain green.
|
||||
- Cognition eval numbers on dev/public unchanged from ADR-0053.
|
||||
|
||||
### Scope limits
|
||||
|
||||
- **No Rust binding for `vault_recall_batch` yet.** Python is the
|
||||
canonical path; a Rust batched binding can be added under a
|
||||
separate ADR with a parity gate analogous to ADR-0019.
|
||||
- **Holdout case_details are written when run via `--split
|
||||
holdout`** because the standard `LaneResult.case_details` carries
|
||||
the lane runner's output. The trust-boundary doctrine in
|
||||
`evals/holdout_runner.py` applies to **sealed** (encrypted)
|
||||
holdouts — the cognition holdout file is plaintext-in-tree by
|
||||
intent (development), so writing details is consistent. Once a
|
||||
sealed cognition holdout exists, callers must use
|
||||
`holdout_runner.run_holdout` (aggregate-only) instead of
|
||||
`framework.run_lane`.
|
||||
|
||||
---
|
||||
|
||||
## Cross-References
|
||||
|
||||
- [ADR-0019](./ADR-0019-vault-recall-vectorisation.md) — Stage 1
|
||||
vectorised single-query path this ADR builds on (if a file by
|
||||
that name does not exist, the contract lives in
|
||||
`tests/test_vault_recall_vectorised.py`).
|
||||
- [ADR-0053](./ADR-0053-cognition-lane-closure.md) — last cognition
|
||||
lane work; its scope-limits section predicted the holdout
|
||||
number.
|
||||
|
||||
---
|
||||
|
||||
## Verification
|
||||
|
||||
```
|
||||
tests/test_vault_recall_indexing_batch.py — 21 tests, all green
|
||||
tests/test_eval_holdout_split.py — 10 tests, all green
|
||||
tests/test_vault_recall_vectorised.py — 4 tests still green
|
||||
tests/test_vault_recall_rust_parity.py — pre-existing parity gate still green
|
||||
```
|
||||
|
||||
The non-negotiable field invariant (`versor_condition(F) < 1e-6`)
|
||||
is preserved: this ADR adds an indexing cache, a batched scan
|
||||
function, and a CLI flag — no algebra change, no field
|
||||
construction, no normalisation.
|
||||
|
|
@ -63,6 +63,7 @@ ADRs record significant architectural decisions: what was decided, why, what alt
|
|||
| [ADR-0051](ADR-0051-trust-boundary-hardening.md) | Trust-boundary hardening pass | Accepted (2026-05-18) |
|
||||
| [ADR-0052](ADR-0052-teaching-grounded-surface.md) | Teaching-grounded surface for cold-start CAUSE / VERIFICATION | Accepted (2026-05-18) |
|
||||
| [ADR-0053](ADR-0053-cognition-lane-closure.md) | Cognition lane closure: dev-driven corpus expansion + CORRECTION acknowledgement | Accepted (2026-05-18) |
|
||||
| [ADR-0054](ADR-0054-vault-recall-indexing-batching.md) | Vault recall matrix-cache indexing + batched API; holdout split wired into eval CLI | Accepted (2026-05-18) |
|
||||
|
||||
---
|
||||
|
||||
|
|
|
|||
|
|
@ -45,6 +45,32 @@ class LaneInfo:
|
|||
def public_cases_path(self, version: str = "v1") -> Path:
|
||||
return self.root / "public" / version / "cases.jsonl"
|
||||
|
||||
def holdout_cases_path(self, version: str = "v1") -> Path:
|
||||
"""Return the resolved holdout cases path for this lane.
|
||||
|
||||
Resolution order (first existing wins):
|
||||
1. ``holdouts/cases.jsonl`` — flat plaintext
|
||||
2. ``holdouts/cases_plaintext.jsonl`` — cognition convention
|
||||
3. ``holdouts/<version>/cases.jsonl`` — versioned plaintext
|
||||
|
||||
If none exist, returns the versioned path so callers receive a
|
||||
coherent ``FileNotFoundError``.
|
||||
|
||||
This intentionally does NOT decrypt sealed (``*.age``) holdouts —
|
||||
sealed runs must go through ``evals.holdout_runner.run_holdout``,
|
||||
which enforces aggregate-only output per its trust boundary.
|
||||
"""
|
||||
holdouts = self.root / "holdouts"
|
||||
candidates = (
|
||||
holdouts / "cases.jsonl",
|
||||
holdouts / "cases_plaintext.jsonl",
|
||||
holdouts / version / "cases.jsonl",
|
||||
)
|
||||
for path in candidates:
|
||||
if path.exists():
|
||||
return path
|
||||
return candidates[-1]
|
||||
|
||||
def results_dir(self) -> Path:
|
||||
return self.root / "results"
|
||||
|
||||
|
|
@ -133,8 +159,12 @@ def run_lane(
|
|||
cases_path = lane.dev_cases_path()
|
||||
elif split == "public":
|
||||
cases_path = lane.public_cases_path(version)
|
||||
elif split == "holdout":
|
||||
cases_path = lane.holdout_cases_path(version)
|
||||
else:
|
||||
raise ValueError(f"Unsupported split: {split!r}. Use 'dev' or 'public'.")
|
||||
raise ValueError(
|
||||
f"Unsupported split: {split!r}. Use 'dev', 'public', or 'holdout'."
|
||||
)
|
||||
|
||||
if not cases_path.exists():
|
||||
raise FileNotFoundError(f"Cases not found: {cases_path}")
|
||||
|
|
|
|||
134
tests/test_eval_holdout_split.py
Normal file
134
tests/test_eval_holdout_split.py
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
"""ADR-0054 (Part 1) — holdout split wired into the official eval runner.
|
||||
|
||||
Contracts pinned here:
|
||||
|
||||
- ``LaneInfo.holdout_cases_path`` resolves the holdout plaintext file
|
||||
via a fixed priority (cases.jsonl > cases_plaintext.jsonl > v1/cases.jsonl).
|
||||
- ``framework.run_lane(split="holdout")`` reads that path and runs the
|
||||
lane's runner like any other split.
|
||||
- The cognition lane reports a stable holdout metric set (case count,
|
||||
intent_accuracy, surface_groundedness, versor_closure_rate).
|
||||
- Unknown ``split`` values raise ``ValueError`` with a message naming
|
||||
all three accepted values.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from evals.framework import LaneInfo, get_lane, run_lane
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LaneInfo.holdout_cases_path resolution
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_cognition_holdout_path_resolves_to_plaintext() -> None:
|
||||
lane = get_lane("cognition")
|
||||
path = lane.holdout_cases_path()
|
||||
assert path.exists()
|
||||
assert path.name == "cases_plaintext.jsonl"
|
||||
|
||||
|
||||
def test_holdout_path_resolution_prefers_cases_jsonl(tmp_path: Path) -> None:
|
||||
root = tmp_path / "fake_lane"
|
||||
(root / "holdouts" / "v1").mkdir(parents=True)
|
||||
(root / "holdouts" / "cases.jsonl").write_text("{}\n")
|
||||
(root / "holdouts" / "cases_plaintext.jsonl").write_text("{}\n")
|
||||
(root / "holdouts" / "v1" / "cases.jsonl").write_text("{}\n")
|
||||
lane = LaneInfo(name="fake_lane", root=root, versions=("v1",))
|
||||
assert lane.holdout_cases_path().name == "cases.jsonl"
|
||||
assert lane.holdout_cases_path().parent.name == "holdouts"
|
||||
|
||||
|
||||
def test_holdout_path_falls_back_to_plaintext_then_versioned(tmp_path: Path) -> None:
|
||||
root = tmp_path / "fake_lane"
|
||||
(root / "holdouts" / "v1").mkdir(parents=True)
|
||||
(root / "holdouts" / "cases_plaintext.jsonl").write_text("{}\n")
|
||||
(root / "holdouts" / "v1" / "cases.jsonl").write_text("{}\n")
|
||||
lane = LaneInfo(name="fake_lane", root=root, versions=("v1",))
|
||||
assert lane.holdout_cases_path().name == "cases_plaintext.jsonl"
|
||||
|
||||
|
||||
def test_holdout_path_when_nothing_exists_returns_versioned_path(tmp_path: Path) -> None:
|
||||
root = tmp_path / "fake_lane"
|
||||
(root / "holdouts" / "v1").mkdir(parents=True)
|
||||
lane = LaneInfo(name="fake_lane", root=root, versions=("v1",))
|
||||
path = lane.holdout_cases_path()
|
||||
assert path.name == "cases.jsonl"
|
||||
assert path.parent.name == "v1"
|
||||
assert not path.exists()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# framework.run_lane(split="holdout")
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_run_lane_holdout_runs_full_cognition_set() -> None:
|
||||
lane = get_lane("cognition")
|
||||
result = run_lane(lane, split="holdout")
|
||||
assert result.lane == "cognition"
|
||||
assert result.split == "holdout"
|
||||
assert result.metrics["total"] == 19
|
||||
|
||||
|
||||
def test_run_lane_holdout_returns_expected_metric_keys() -> None:
|
||||
lane = get_lane("cognition")
|
||||
result = run_lane(lane, split="holdout")
|
||||
expected = {
|
||||
"total",
|
||||
"intent_accuracy",
|
||||
"term_capture_rate",
|
||||
"surface_groundedness",
|
||||
"versor_closure_rate",
|
||||
}
|
||||
assert expected.issubset(result.metrics.keys())
|
||||
|
||||
|
||||
def test_run_lane_holdout_versor_closure_preserved() -> None:
|
||||
"""The non-negotiable field invariant (versor_condition < 1e-6) must
|
||||
hold on every holdout case — same gate as dev/public."""
|
||||
lane = get_lane("cognition")
|
||||
result = run_lane(lane, split="holdout")
|
||||
assert result.metrics["versor_closure_rate"] == 1.0
|
||||
|
||||
|
||||
def test_run_lane_unknown_split_lists_all_three_values() -> None:
|
||||
lane = get_lane("cognition")
|
||||
with pytest.raises(ValueError) as excinfo:
|
||||
run_lane(lane, split="train")
|
||||
msg = str(excinfo.value)
|
||||
assert "dev" in msg
|
||||
assert "public" in msg
|
||||
assert "holdout" in msg
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Holdout vs dev/public — consistent eval interface
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_holdout_dev_public_share_metric_schema() -> None:
|
||||
lane = get_lane("cognition")
|
||||
dev = run_lane(lane, split="dev").metrics
|
||||
public = run_lane(lane, split="public").metrics
|
||||
holdout = run_lane(lane, split="holdout").metrics
|
||||
assert set(dev.keys()) == set(public.keys()) == set(holdout.keys())
|
||||
|
||||
|
||||
def test_holdout_cases_have_required_fields() -> None:
|
||||
lane = get_lane("cognition")
|
||||
path = lane.holdout_cases_path()
|
||||
for line in path.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
case = json.loads(line)
|
||||
assert "id" in case
|
||||
assert "prompt" in case
|
||||
assert "expected_intent" in case
|
||||
254
tests/test_vault_recall_indexing_batch.py
Normal file
254
tests/test_vault_recall_indexing_batch.py
Normal file
|
|
@ -0,0 +1,254 @@
|
|||
"""ADR-0054 — vault recall indexing + batched API.
|
||||
|
||||
Two doctrine-aligned optimisations on top of ADR-0019 Stage 1:
|
||||
|
||||
1. **Indexing**: ``VaultStore`` keeps a cached ``(N, D)`` f32 matrix
|
||||
view of the deque, rebuilt lazily on the first recall after any
|
||||
mutation. Repeated recalls reuse the cached matrix instead of
|
||||
rebuilding it from a Python list each call.
|
||||
|
||||
2. **Batching**: ``algebra.backend.vault_recall_batch`` scores B
|
||||
queries against one matrix in a single component-serial sweep —
|
||||
bit-identical per-query to ``vault_recall``.
|
||||
|
||||
No approximate search. No hot-path repair. No mutation of shared
|
||||
cached state during recall. ``versor_condition < 1e-6`` invariant is
|
||||
not touched by either change.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from algebra.backend import vault_recall, vault_recall_batch
|
||||
from teaching.epistemic import EpistemicStatus
|
||||
from vault.store import VaultStore
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# vault_recall_batch — bit-identity vs single-query vault_recall
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seed", [0xC07E, 0xBEEF, 0x1234])
|
||||
def test_batch_matches_per_query_bit_identical(seed: int) -> None:
|
||||
rng = np.random.default_rng(seed)
|
||||
N, B = 137, 7
|
||||
versors = [rng.standard_normal(size=(32,), dtype=np.float32) for _ in range(N)]
|
||||
queries = rng.standard_normal(size=(B, 32), dtype=np.float32)
|
||||
matrix = np.asarray(versors, dtype=np.float32)
|
||||
|
||||
batch = vault_recall_batch(matrix, queries, top_k=N)
|
||||
per_query = [vault_recall(versors, queries[b], top_k=N) for b in range(B)]
|
||||
|
||||
assert len(batch) == B == len(per_query)
|
||||
for b in range(B):
|
||||
# Indices must be identical.
|
||||
assert [i for i, _ in batch[b]] == [i for i, _ in per_query[b]]
|
||||
# Scores must be float-equal (bit-identical at f32).
|
||||
b_scores = np.array([s for _, s in batch[b]], dtype=np.float32)
|
||||
q_scores = np.array([s for _, s in per_query[b]], dtype=np.float32)
|
||||
assert np.array_equal(b_scores, q_scores)
|
||||
|
||||
|
||||
def test_batch_handles_1d_query() -> None:
|
||||
rng = np.random.default_rng(0)
|
||||
versors = [rng.standard_normal(size=(32,), dtype=np.float32) for _ in range(10)]
|
||||
matrix = np.asarray(versors, dtype=np.float32)
|
||||
q = rng.standard_normal(size=(32,), dtype=np.float32)
|
||||
batch = vault_recall_batch(matrix, q, top_k=3)
|
||||
assert len(batch) == 1
|
||||
expected = vault_recall(versors, q, top_k=3)
|
||||
assert batch[0] == expected
|
||||
|
||||
|
||||
def test_batch_empty_matrix_returns_empty_per_query() -> None:
|
||||
M = np.zeros((0, 32), dtype=np.float32)
|
||||
Q = np.zeros((3, 32), dtype=np.float32)
|
||||
out = vault_recall_batch(M, Q, top_k=5)
|
||||
assert out == [[], [], []]
|
||||
|
||||
|
||||
def test_batch_zero_top_k_returns_empty_per_query() -> None:
|
||||
rng = np.random.default_rng(0)
|
||||
M = rng.standard_normal(size=(10, 32), dtype=np.float32)
|
||||
Q = rng.standard_normal(size=(2, 32), dtype=np.float32)
|
||||
out = vault_recall_batch(M, Q, top_k=0)
|
||||
assert out == [[], []]
|
||||
|
||||
|
||||
def test_batch_shape_mismatch_raises() -> None:
|
||||
M = np.zeros((5, 32), dtype=np.float32)
|
||||
Q = np.zeros((2, 31), dtype=np.float32)
|
||||
with pytest.raises(ValueError) as excinfo:
|
||||
vault_recall_batch(M, Q, top_k=3)
|
||||
assert "components" in str(excinfo.value)
|
||||
|
||||
|
||||
def test_batch_rejects_higher_dim_matrix() -> None:
|
||||
M = np.zeros((2, 5, 32), dtype=np.float32)
|
||||
Q = np.zeros((1, 32), dtype=np.float32)
|
||||
with pytest.raises(ValueError):
|
||||
vault_recall_batch(M, Q, top_k=1)
|
||||
|
||||
|
||||
def test_batch_top_k_smaller_than_n_preserves_ordering() -> None:
|
||||
rng = np.random.default_rng(0xDEAD)
|
||||
versors = [rng.standard_normal(size=(32,), dtype=np.float32) for _ in range(50)]
|
||||
matrix = np.asarray(versors, dtype=np.float32)
|
||||
queries = rng.standard_normal(size=(4, 32), dtype=np.float32)
|
||||
batch = vault_recall_batch(matrix, queries, top_k=5)
|
||||
for b in range(4):
|
||||
single = vault_recall(versors, queries[b], top_k=5)
|
||||
assert batch[b] == single
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VaultStore matrix cache — invalidation correctness
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _v(seed: int) -> np.ndarray:
|
||||
rng = np.random.default_rng(seed)
|
||||
return rng.standard_normal(size=(32,), dtype=np.float32)
|
||||
|
||||
|
||||
def test_matrix_cache_starts_unbuilt() -> None:
|
||||
store = VaultStore()
|
||||
assert store._matrix_cache is None
|
||||
|
||||
|
||||
def test_matrix_cache_built_on_first_recall() -> None:
|
||||
store = VaultStore()
|
||||
store.store(_v(1))
|
||||
store.store(_v(2))
|
||||
assert store._matrix_cache is None
|
||||
store.recall(_v(3), top_k=1)
|
||||
assert store._matrix_cache is not None
|
||||
assert store._matrix_cache.shape == (2, 32)
|
||||
|
||||
|
||||
def test_matrix_cache_invalidated_on_store() -> None:
|
||||
store = VaultStore()
|
||||
store.store(_v(1))
|
||||
store.recall(_v(2), top_k=1)
|
||||
assert store._matrix_cache is not None
|
||||
store.store(_v(3))
|
||||
assert store._matrix_cache is None
|
||||
|
||||
|
||||
def test_matrix_cache_invalidated_on_reproject() -> None:
|
||||
store = VaultStore()
|
||||
store.store(_v(1))
|
||||
store.recall(_v(2), top_k=1)
|
||||
assert store._matrix_cache is not None
|
||||
store.reproject()
|
||||
assert store._matrix_cache is None
|
||||
|
||||
|
||||
def test_matrix_cache_invalidated_on_eviction() -> None:
|
||||
store = VaultStore(max_entries=2)
|
||||
store.store(_v(1))
|
||||
store.store(_v(2))
|
||||
store.recall(_v(3), top_k=1)
|
||||
assert store._matrix_cache is not None
|
||||
store.store(_v(4)) # triggers eviction → _rebuild_index → invalidate
|
||||
assert store._matrix_cache is None
|
||||
|
||||
|
||||
def test_matrix_cache_does_not_change_recall_results() -> None:
|
||||
"""The cache is an indexing optimisation — results must equal the
|
||||
pre-cache recall behaviour case-for-case."""
|
||||
rng = np.random.default_rng(0xC0DE)
|
||||
store_a = VaultStore(reproject_interval=0)
|
||||
store_b = VaultStore(reproject_interval=0)
|
||||
versors = [rng.standard_normal(size=(32,), dtype=np.float32) for _ in range(20)]
|
||||
for v in versors:
|
||||
store_a.store(v)
|
||||
store_b.store(v)
|
||||
|
||||
for _ in range(5):
|
||||
q = rng.standard_normal(size=(32,), dtype=np.float32)
|
||||
# Force store_a to take fresh non-cached path by clearing cache.
|
||||
store_a._matrix_cache = None
|
||||
r_a = store_a.recall(q, top_k=5)
|
||||
# store_b takes cached path on second+ recalls.
|
||||
store_b.recall(q, top_k=5) # warm cache
|
||||
store_b._matrix_cache = store_b._get_matrix() # ensure cache exists
|
||||
r_b = store_b.recall(q, top_k=5)
|
||||
assert [r["index"] for r in r_a] == [r["index"] for r in r_b]
|
||||
assert [r["score"] for r in r_a] == [r["score"] for r in r_b]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VaultStore.recall_batch — parity with per-query recall
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_recall_batch_matches_per_query_recall() -> None:
|
||||
rng = np.random.default_rng(0xFACE)
|
||||
store = VaultStore(reproject_interval=0)
|
||||
versors = [rng.standard_normal(size=(32,), dtype=np.float32) for _ in range(30)]
|
||||
for v in versors:
|
||||
store.store(v)
|
||||
|
||||
queries = rng.standard_normal(size=(4, 32), dtype=np.float32)
|
||||
batch = store.recall_batch(queries, top_k=5)
|
||||
per_query = [store.recall(queries[b], top_k=5) for b in range(4)]
|
||||
|
||||
assert len(batch) == 4
|
||||
for b in range(4):
|
||||
assert [r["index"] for r in batch[b]] == [r["index"] for r in per_query[b]]
|
||||
assert [r["score"] for r in batch[b]] == [r["score"] for r in per_query[b]]
|
||||
|
||||
|
||||
def test_recall_batch_empty_vault_returns_empty_per_query() -> None:
|
||||
store = VaultStore()
|
||||
Q = np.zeros((3, 32), dtype=np.float32)
|
||||
out = store.recall_batch(Q, top_k=5)
|
||||
assert out == [[], [], []]
|
||||
|
||||
|
||||
def test_recall_batch_zero_top_k_returns_empty_per_query() -> None:
|
||||
store = VaultStore()
|
||||
store.store(_v(1))
|
||||
Q = np.zeros((2, 32), dtype=np.float32)
|
||||
out = store.recall_batch(Q, top_k=0)
|
||||
assert out == [[], []]
|
||||
|
||||
|
||||
def test_recall_batch_accepts_1d_query_as_single_batch() -> None:
|
||||
store = VaultStore(reproject_interval=0)
|
||||
store.store(_v(1))
|
||||
store.store(_v(2))
|
||||
out = store.recall_batch(_v(3), top_k=2)
|
||||
assert len(out) == 1
|
||||
expected = store.recall(_v(3), top_k=2)
|
||||
assert [r["index"] for r in out[0]] == [r["index"] for r in expected]
|
||||
|
||||
|
||||
def test_recall_batch_exact_self_match_promoted() -> None:
|
||||
"""If a query equals a stored versor, its index must appear first
|
||||
with score=+inf — same contract as single-query recall."""
|
||||
store = VaultStore(reproject_interval=0)
|
||||
target = _v(1)
|
||||
store.store(_v(0))
|
||||
store.store(target)
|
||||
store.store(_v(2))
|
||||
Q = np.stack([target, _v(99)])
|
||||
out = store.recall_batch(Q, top_k=3)
|
||||
assert out[0][0]["index"] == 1
|
||||
assert out[0][0]["score"] == float("inf")
|
||||
|
||||
|
||||
def test_recall_batch_min_status_filter_applied_per_query() -> None:
|
||||
store = VaultStore(reproject_interval=0)
|
||||
store.store(_v(1), epistemic_status=EpistemicStatus.COHERENT)
|
||||
store.store(_v(2), epistemic_status=EpistemicStatus.SPECULATIVE)
|
||||
store.store(_v(3), epistemic_status=EpistemicStatus.COHERENT)
|
||||
Q = np.stack([_v(10), _v(11)])
|
||||
out = store.recall_batch(Q, top_k=5, min_status=EpistemicStatus.COHERENT)
|
||||
for per_query in out:
|
||||
for r in per_query:
|
||||
assert r["metadata"]["epistemic_status"] == "coherent"
|
||||
|
|
@ -15,7 +15,7 @@ from __future__ import annotations
|
|||
from collections import deque
|
||||
|
||||
import numpy as np
|
||||
from algebra.backend import vault_recall
|
||||
from algebra.backend import vault_recall, vault_recall_batch
|
||||
from algebra.cga import null_project
|
||||
from teaching.epistemic import ADMISSIBLE_AS_EVIDENCE, EpistemicStatus
|
||||
|
||||
|
|
@ -49,6 +49,10 @@ class VaultStore:
|
|||
self._reproject_interval = reproject_interval
|
||||
self._max_entries = max_entries
|
||||
self._exact_index: dict[bytes, list[int]] = {}
|
||||
# ADR-0054: cached (N, D) f32 matrix view of the deque, rebuilt
|
||||
# lazily on the first recall after any mutation. Indexing
|
||||
# optimisation only — scoring arithmetic is unchanged.
|
||||
self._matrix_cache: np.ndarray | None = None
|
||||
|
||||
def store(
|
||||
self,
|
||||
|
|
@ -80,6 +84,7 @@ class VaultStore:
|
|||
idx = len(self._versors) - 1
|
||||
key = _versor_key(arr)
|
||||
self._exact_index.setdefault(key, []).append(idx)
|
||||
self._matrix_cache = None
|
||||
|
||||
self._store_count += 1
|
||||
if self._reproject_interval > 0 and self._store_count % self._reproject_interval == 0:
|
||||
|
|
@ -117,7 +122,11 @@ class VaultStore:
|
|||
# has a chance at top_k entries. 4x is a generous heuristic;
|
||||
# vault sizes are bounded by max_entries anyway.
|
||||
scan_k = max(top_k * 4, top_k) if min_status is not None else max(top_k, 1)
|
||||
ranked = vault_recall(list(self._versors), query_arr, scan_k)
|
||||
matrix = self._get_matrix()
|
||||
ranked = vault_recall(
|
||||
list(self._versors), query_arr, scan_k,
|
||||
prebuilt_matrix=matrix,
|
||||
)
|
||||
|
||||
key = _versor_key(query_arr)
|
||||
exact_indices = self._exact_index.get(key, [])
|
||||
|
|
@ -148,6 +157,70 @@ class VaultStore:
|
|||
for i, score in ranked[:top_k]
|
||||
]
|
||||
|
||||
def recall_batch(
|
||||
self,
|
||||
queries: np.ndarray,
|
||||
top_k: int = 5,
|
||||
*,
|
||||
min_status: EpistemicStatus | None = None,
|
||||
) -> list[list[dict]]:
|
||||
"""Recall B queries against the stored versors in one sweep.
|
||||
|
||||
Returns one ``list[dict]`` per query in the same shape ``recall``
|
||||
returns. Exact-self-match promotion, ``min_status`` filtering,
|
||||
and the descending-score / ascending-index tiebreak rule are
|
||||
applied per query — semantics are identical to looping
|
||||
``recall(q, top_k=...)`` over each query, but the underlying
|
||||
scoring scan is a single component-serial sweep over the
|
||||
cached matrix. ADR-0054.
|
||||
"""
|
||||
Q = np.asarray(queries, dtype=np.float32)
|
||||
if Q.ndim == 1:
|
||||
Q = Q[None, :]
|
||||
if not self._versors or top_k <= 0:
|
||||
return [[] for _ in range(Q.shape[0])]
|
||||
|
||||
matrix = self._get_matrix()
|
||||
assert matrix is not None # non-empty deque ⇒ matrix is built
|
||||
scan_k = max(top_k * 4, top_k) if min_status is not None else max(top_k, 1)
|
||||
batch_ranked = vault_recall_batch(matrix, Q, scan_k)
|
||||
|
||||
results: list[list[dict]] = []
|
||||
for b, ranked in enumerate(batch_ranked):
|
||||
key = _versor_key(Q[b])
|
||||
exact_indices = self._exact_index.get(key, [])
|
||||
if exact_indices:
|
||||
exact_matches = [(i, float("inf")) for i in exact_indices]
|
||||
seen = set(exact_indices)
|
||||
ranked = exact_matches + [
|
||||
(i, score) for i, score in ranked if i not in seen
|
||||
]
|
||||
|
||||
if min_status is not None:
|
||||
filtered: list[tuple[int, float]] = []
|
||||
for i, score in ranked:
|
||||
raw_status = self._metadata[i].get(
|
||||
"epistemic_status", "speculative",
|
||||
)
|
||||
try:
|
||||
entry_status = EpistemicStatus(raw_status)
|
||||
except ValueError:
|
||||
entry_status = EpistemicStatus.SPECULATIVE
|
||||
if _status_admits(entry_status, min_status):
|
||||
filtered.append((i, score))
|
||||
ranked = filtered
|
||||
|
||||
results.append([
|
||||
{
|
||||
"versor": self._versors[i],
|
||||
"score": float(score),
|
||||
"metadata": self._metadata[i],
|
||||
"index": i,
|
||||
}
|
||||
for i, score in ranked[:top_k]
|
||||
])
|
||||
return results
|
||||
|
||||
def reproject(self) -> None:
|
||||
"""
|
||||
Re-project all stored versors onto the null cone.
|
||||
|
|
@ -162,6 +235,20 @@ class VaultStore:
|
|||
for i, v in enumerate(self._versors):
|
||||
key = _versor_key(v)
|
||||
self._exact_index.setdefault(key, []).append(i)
|
||||
self._matrix_cache = None
|
||||
|
||||
def _get_matrix(self) -> np.ndarray | None:
|
||||
"""Return the cached (N, D) f32 stack of stored versors.
|
||||
|
||||
Rebuilds the cache on first call after any mutation. Returns
|
||||
None when the vault is empty so callers can branch without
|
||||
constructing a 0-row array. ADR-0054.
|
||||
"""
|
||||
if not self._versors:
|
||||
return None
|
||||
if self._matrix_cache is None:
|
||||
self._matrix_cache = np.asarray(self._versors, dtype=np.float32)
|
||||
return self._matrix_cache
|
||||
|
||||
@property
|
||||
def reproject_interval(self) -> int:
|
||||
|
|
|
|||
Loading…
Reference in a new issue