diff --git a/algebra/backend.py b/algebra/backend.py index 7a33106e..922e7b9a 100644 --- a/algebra/backend.py +++ b/algebra/backend.py @@ -90,7 +90,13 @@ def cga_inner(X: np.ndarray, Y: np.ndarray) -> float: return _ci(X, Y) -def vault_recall(versors: list, query: np.ndarray, top_k: int = 5) -> list: +def vault_recall( + versors: list, + query: np.ndarray, + top_k: int = 5, + *, + prebuilt_matrix: np.ndarray | None = None, +) -> list: """Top-k CGA inner product recall. Rust path: parallel Rayon scan when explicitly enabled. @@ -99,11 +105,21 @@ def vault_recall(versors: list, query: np.ndarray, top_k: int = 5) -> list: because the per-versor sum is folded in the same serial component order; the only thing the vectorisation replaces is the per-element Python dispatch loop. ADR-0019 Stage 1. + + ``prebuilt_matrix`` (ADR-0054): optional cached (N, D) f32 matrix + of stacked versors maintained by ``VaultStore``. When supplied, + the deque→ndarray conversion is skipped — purely an indexing + optimisation, scoring arithmetic is identical. """ - if not versors: + if not versors and prebuilt_matrix is None: return [] q = np.asarray(query, dtype=np.float32) - M = np.asarray(versors, dtype=np.float32) + if prebuilt_matrix is not None: + M = prebuilt_matrix + if M.shape[0] == 0: + return [] + else: + M = np.asarray(versors, dtype=np.float32) if _RUST and M.ndim == 2 and M.shape[1] == 32: try: # Pass the (N, 32) numpy buffer directly — the Rust @@ -143,6 +159,65 @@ def vault_recall(versors: list, query: np.ndarray, top_k: int = 5) -> list: return [(int(i), float(scores[i])) for i in cand] +def vault_recall_batch( + matrix: np.ndarray, + queries: np.ndarray, + top_k: int = 5, +) -> list[list[tuple[int, float]]]: + """Top-k CGA inner product recall for B queries against one matrix. + + ADR-0054. Returns one ``[(index, score), ...]`` list per query in + the same shape ``vault_recall`` returns for a single query. + + Bit-identity contract: each per-query result must equal the + corresponding single-query ``vault_recall`` call against the same + matrix. We accumulate scores in component-serial order with the + diagonal metric — the same folding pattern as the single-query + path — so the per-versor sum is folded identically. Top-k + ordering uses the same descending-score / ascending-index stable + rule. + + No approximate search. No Rust path here yet (the Rust binding + is single-query); Python is canonical. + """ + M = np.asarray(matrix, dtype=np.float32) + Q = np.asarray(queries, dtype=np.float32) + if Q.ndim == 1: + Q = Q[None, :] + if M.ndim != 2 or Q.ndim != 2: + raise ValueError( + f"vault_recall_batch requires matrix.ndim==2 and queries.ndim in (1, 2); " + f"got matrix.ndim={M.ndim}, queries.ndim={Q.ndim}" + ) + if M.shape[1] != Q.shape[1]: + raise ValueError( + f"vault_recall_batch shape mismatch: matrix has {M.shape[1]} components " + f"per row, queries have {Q.shape[1]}" + ) + N = M.shape[0] + B = Q.shape[0] + if N == 0 or top_k <= 0: + return [[] for _ in range(B)] + # Component-serial accumulation: scores[b, n] = sum_i metric[i] * M[n,i] * Q[b,i]. + # Folding component-by-component preserves bit-identity with the + # single-query path (same float32 addition order across i). + scores = np.zeros((B, N), dtype=np.float32) + for i in range(M.shape[1]): + scores += (_CGA_INNER_METRIC[i] * M[:, i])[None, :] * Q[:, i, None] + k = min(top_k, N) + out: list[list[tuple[int, float]]] = [] + for b in range(B): + row = scores[b] + if k < N: + cand = np.argpartition(-row, k - 1)[:k] + else: + cand = np.arange(N) + order = np.lexsort((cand, -row[cand])) + cand = cand[order] + out.append([(int(i), float(row[i])) for i in cand]) + return out + + def unitize_expmap(v: np.ndarray) -> np.ndarray: """Unitize a multivector via the Cl(4,1) exponential map. diff --git a/core/cli.py b/core/cli.py index 573b9ea9..02635fcf 100644 --- a/core/cli.py +++ b/core/cli.py @@ -23,7 +23,7 @@ _CORE_RS_DIR = _REPO_ROOT / "core-rs" _CORE_RS_MANIFEST = _CORE_RS_DIR / "Cargo.toml" DESCRIPTION = "CORE versor engine command suite." -EPILOG = "Examples:\n core chat\n core pulse \"What is truth?\"\n core pulse --no-glove --json \"Compare knowledge and wisdom\"\n core bench\n core bench --suite determinism --runs 50\n core bench --suite speedup --json\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core test --suite fast -q\n core test --suite pulse -q\n core test --suite proof -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core demo audit-tour\n core demo pack-measurements\n core demo long-context-comparison\n core eval --list\n core eval cognition\n core eval cognition --json --save\n core eval cognition --split dev --version v1" +EPILOG = "Examples:\n core chat\n core pulse \"What is truth?\"\n core pulse --no-glove --json \"Compare knowledge and wisdom\"\n core bench\n core bench --suite determinism --runs 50\n core bench --suite speedup --json\n core trace \"word beginning truth\"\n core trace --output-language grc --frame-pack grc --json \"logos\"\n core rust status\n core rust build\n core oov covenant\n core pack list\n core pack verify en_minimal_v1\n core test --suite fast -q\n core test --suite pulse -q\n core test --suite proof -q\n core test --suite cognition -q\n core test -- tests/test_alignment_graph.py -q\n core demo audit-tour\n core demo pack-measurements\n core demo long-context-comparison\n core eval --list\n core eval cognition\n core eval cognition --json --save\n core eval cognition --split dev --version v1\n core eval cognition --split holdout" _TEST_SUITES: dict[str, tuple[str, ...]] = { "fast": ( @@ -1466,7 +1466,7 @@ def build_parser() -> argparse.ArgumentParser: eval_cmd.add_argument("lane", nargs="?", help="eval lane name (e.g. cognition)") eval_cmd.add_argument("--list", dest="list_lanes", action="store_true", help="list available eval lanes") eval_cmd.add_argument("--version", help="version to evaluate (default: latest)") - eval_cmd.add_argument("--split", default="public", choices=["dev", "public"], help="which split to score (default: public)") + eval_cmd.add_argument("--split", default="public", choices=["dev", "public", "holdout"], help="which split to score (default: public)") eval_cmd.add_argument("--json", action="store_true", help="emit machine-readable JSON") eval_cmd.add_argument("--save", action="store_true", help="write result to lane results/ directory") eval_cmd.add_argument("--report", metavar="PATH", help="write JSON report to file") diff --git a/docs/decisions/ADR-0054-vault-recall-indexing-batching.md b/docs/decisions/ADR-0054-vault-recall-indexing-batching.md new file mode 100644 index 00000000..18019e89 --- /dev/null +++ b/docs/decisions/ADR-0054-vault-recall-indexing-batching.md @@ -0,0 +1,244 @@ +# ADR-0054 — Vault Recall: Matrix-Cache Indexing + Batched API; Holdout Split Wired + +**Status:** Accepted +**Date:** 2026-05-18 +**Author:** Shay + +--- + +## Context + +Two doctrine-aligned items from CLAUDE.md were still open after +ADR-0053: + +1. **CLAUDE.md item #4 — "Add exact vault recall indexing/batching + without approximate search."** ADR-0019 Stage 1 vectorised the + single-query CGA scan inside `algebra.backend.vault_recall`, but + the **deque → ndarray conversion** still happened on every recall, + and there was no batched-query API. Repeated recalls against a + slowly-growing vault paid the conversion cost each call. +2. **Holdouts not in the official eval runner.** The cognition lane + has had a 19-case plaintext holdout file + (`evals/cognition/holdouts/cases_plaintext.jsonl`) since the lane + was set up, but `core eval cognition --split` accepted only `dev` + and `public`. Holdout numbers existed only via ad-hoc scripts + spawned during ADR-0053. + +Both items are minimal-doctrine work: no algebra change, no new +approximation, no new normalisation, no hot-path repair. Bundled +together because both touch the validation/eval surface. + +--- + +## Decision + +### Part 1 — Vault recall indexing + batching + +**`VaultStore` matrix cache (`vault/store.py`).** + +A lazily-built `_matrix_cache: np.ndarray | None` is held on the +store. It is `None` initially and after any mutation; the first +`recall` after a mutation rebuilds it via +`np.asarray(self._versors, dtype=np.float32)`. Invalidation hooks: + +- `store()` — always invalidates (append shifts the deque view). +- `reproject()` — invalidates (every entry replaced). +- `_rebuild_index()` — invalidates (called on max-entries eviction). + +The cache is read-only from the recall path; `vault_recall` receives +it via a new optional `prebuilt_matrix=` kwarg and skips the +deque → ndarray conversion when supplied. No shared mutable state +is held across calls — the matrix is the same buffer between recalls +only while no mutation has happened. + +**Batched recall (`algebra.backend.vault_recall_batch`).** + +New function with signature +`vault_recall_batch(matrix, queries, top_k) -> list[list[(int, float)]]`. +Accepts `(N, D)` matrix and `(B, D)` (or `(D,)`) queries, returns +one ranked list per query. Scoring uses the same diagonal CGA +metric and accumulates **in component-serial order**: + +```python +scores = np.zeros((B, N), dtype=np.float32) +for i in range(D): + scores += (_CGA_INNER_METRIC[i] * M[:, i])[None, :] * Q[:, i, None] +``` + +Folding component-by-component preserves bit-identity with the +single-query path's float32 addition order. Tiebreak rule +(descending score, ascending index) is identical. + +**`VaultStore.recall_batch`.** + +Public sibling to `recall`. Same per-query semantics — exact-self- +match promotion via the byte-key index, optional `min_status` +filter, score=+inf for exact hits — but the underlying scoring scan +is a single component-serial sweep over the cached matrix. + +### Part 2 — Wire `--split holdout` + +`evals/framework.py`: + +- `LaneInfo.holdout_cases_path(version)` resolves the first existing + of `holdouts/cases.jsonl`, `holdouts/cases_plaintext.jsonl`, + `holdouts//cases.jsonl`. Sealed (`*.age`) holdouts are + **not** decrypted here — that path stays in + `evals.holdout_runner.run_holdout`, which enforces aggregate-only + output by trust-boundary contract. +- `run_lane(split="holdout")` reads that path and dispatches to the + lane's `run_lane(cases, config=...)` like any other split. + +`core/cli.py`: + +- `--split` argparse `choices` extended to + `{"dev", "public", "holdout"}`. +- Example added to `EPILOG`. + +--- + +## Why this is doctrine-aligned + +- **No approximate search.** Both the matrix cache and + `vault_recall_batch` are indexing/vectorisation changes only; + scoring arithmetic is unchanged. +- **No hidden normalisation, no hot-path repair.** The cache is + invalidated, not "auto-rebuilt to fix drift." `reproject()` was + already the canonical drift-repair path; this ADR only invalidates + the cache when it runs. +- **No shared mutable state across recalls.** The cache buffer is + read by `vault_recall` via a kwarg; nothing in the recall path + mutates it. Mutation paths (store / reproject / eviction) clear + it explicitly. +- **`versor_condition < 1e-6` invariant untouched.** No field is + constructed, normalised, or transformed. +- **Holdouts run via the same harness as dev/public.** No parallel + scoring path was added; the trust boundary on sealed holdouts is + preserved by routing plaintext through the standard runner and + leaving the encrypted path to `holdout_runner`. + +--- + +## Characterisation + +### Vault recall — bit-identity gate + +`tests/test_vault_recall_indexing_batch.py` adds 21 tests. The +batched path is verified bit-identical to per-query +`vault_recall` across three seeds × 7 queries × N=137 — every +index sequence and every float32 score matches exactly. + +The pre-existing `tests/test_vault_recall_vectorised.py` (ADR-0019 +Stage 1 gate) continues to pass — the single-query path is +unchanged when no `prebuilt_matrix` is passed. + +### Eval lanes — first official holdout run + +``` +core eval cognition --split holdout + cases : 19 + intent_accuracy : 100.0% + surface_groundedness : 94.7% + term_capture_rate : 70.8% + versor_closure_rate : 100.0% + +core eval cognition --split dev + cases : 13 + intent_accuracy : 100.0% + surface_groundedness : 100.0% + term_capture_rate : 78.6% + versor_closure_rate : 100.0% + +core eval cognition --split public + cases : 13 + intent_accuracy : 100.0% + surface_groundedness : 100.0% + term_capture_rate : 91.7% + versor_closure_rate : 100.0% +``` + +The single surface_groundedness miss on holdouts is the predicted +`correction_truth_040` case — see ADR-0053 scope-limits. Term +capture on holdouts is the next-cheapest pull (echo the corrected- +subject lemma in the CORRECTION acknowledgement), candidate for a +follow-up ADR. + +### Lanes (all green) + +``` +core test --suite smoke 67 passed +core test --suite cognition 121 passed +core test --suite runtime 19 passed +core test --suite teaching 17 passed +core test --suite packs 6 passed +core test --suite algebra 132 passed +``` + +--- + +## Consequences + +### What changes + +- `algebra/backend.py` gains `vault_recall_batch` and an optional + `prebuilt_matrix=` kwarg on `vault_recall`. +- `vault/store.py` gains a lazy matrix cache, cache-invalidation + hooks on mutation paths, and a `recall_batch` method. +- `evals/framework.py` gains `LaneInfo.holdout_cases_path` and a + `"holdout"` branch in `run_lane`. +- `core/cli.py` `--split` now accepts `"holdout"`. + +### What does not change + +- Single-query `vault_recall` semantics — same scores, same order, + same Rust dispatch. +- ADR-0019 Stage 1 bit-identity contract — still gated. +- `versor_condition < 1e-6` invariant unaffected. +- Encrypted holdout decryption — still owned by + `evals.holdout_runner.run_holdout`; aggregate-only output + contract preserved. +- All five core lanes remain green. +- Cognition eval numbers on dev/public unchanged from ADR-0053. + +### Scope limits + +- **No Rust binding for `vault_recall_batch` yet.** Python is the + canonical path; a Rust batched binding can be added under a + separate ADR with a parity gate analogous to ADR-0019. +- **Holdout case_details are written when run via `--split + holdout`** because the standard `LaneResult.case_details` carries + the lane runner's output. The trust-boundary doctrine in + `evals/holdout_runner.py` applies to **sealed** (encrypted) + holdouts — the cognition holdout file is plaintext-in-tree by + intent (development), so writing details is consistent. Once a + sealed cognition holdout exists, callers must use + `holdout_runner.run_holdout` (aggregate-only) instead of + `framework.run_lane`. + +--- + +## Cross-References + +- [ADR-0019](./ADR-0019-vault-recall-vectorisation.md) — Stage 1 + vectorised single-query path this ADR builds on (if a file by + that name does not exist, the contract lives in + `tests/test_vault_recall_vectorised.py`). +- [ADR-0053](./ADR-0053-cognition-lane-closure.md) — last cognition + lane work; its scope-limits section predicted the holdout + number. + +--- + +## Verification + +``` +tests/test_vault_recall_indexing_batch.py — 21 tests, all green +tests/test_eval_holdout_split.py — 10 tests, all green +tests/test_vault_recall_vectorised.py — 4 tests still green +tests/test_vault_recall_rust_parity.py — pre-existing parity gate still green +``` + +The non-negotiable field invariant (`versor_condition(F) < 1e-6`) +is preserved: this ADR adds an indexing cache, a batched scan +function, and a CLI flag — no algebra change, no field +construction, no normalisation. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index eaae6281..e7fa3150 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -63,6 +63,7 @@ ADRs record significant architectural decisions: what was decided, why, what alt | [ADR-0051](ADR-0051-trust-boundary-hardening.md) | Trust-boundary hardening pass | Accepted (2026-05-18) | | [ADR-0052](ADR-0052-teaching-grounded-surface.md) | Teaching-grounded surface for cold-start CAUSE / VERIFICATION | Accepted (2026-05-18) | | [ADR-0053](ADR-0053-cognition-lane-closure.md) | Cognition lane closure: dev-driven corpus expansion + CORRECTION acknowledgement | Accepted (2026-05-18) | +| [ADR-0054](ADR-0054-vault-recall-indexing-batching.md) | Vault recall matrix-cache indexing + batched API; holdout split wired into eval CLI | Accepted (2026-05-18) | --- diff --git a/evals/framework.py b/evals/framework.py index 1e2ddb2d..dbb58cf8 100644 --- a/evals/framework.py +++ b/evals/framework.py @@ -45,6 +45,32 @@ class LaneInfo: def public_cases_path(self, version: str = "v1") -> Path: return self.root / "public" / version / "cases.jsonl" + def holdout_cases_path(self, version: str = "v1") -> Path: + """Return the resolved holdout cases path for this lane. + + Resolution order (first existing wins): + 1. ``holdouts/cases.jsonl`` — flat plaintext + 2. ``holdouts/cases_plaintext.jsonl`` — cognition convention + 3. ``holdouts//cases.jsonl`` — versioned plaintext + + If none exist, returns the versioned path so callers receive a + coherent ``FileNotFoundError``. + + This intentionally does NOT decrypt sealed (``*.age``) holdouts — + sealed runs must go through ``evals.holdout_runner.run_holdout``, + which enforces aggregate-only output per its trust boundary. + """ + holdouts = self.root / "holdouts" + candidates = ( + holdouts / "cases.jsonl", + holdouts / "cases_plaintext.jsonl", + holdouts / version / "cases.jsonl", + ) + for path in candidates: + if path.exists(): + return path + return candidates[-1] + def results_dir(self) -> Path: return self.root / "results" @@ -133,8 +159,12 @@ def run_lane( cases_path = lane.dev_cases_path() elif split == "public": cases_path = lane.public_cases_path(version) + elif split == "holdout": + cases_path = lane.holdout_cases_path(version) else: - raise ValueError(f"Unsupported split: {split!r}. Use 'dev' or 'public'.") + raise ValueError( + f"Unsupported split: {split!r}. Use 'dev', 'public', or 'holdout'." + ) if not cases_path.exists(): raise FileNotFoundError(f"Cases not found: {cases_path}") diff --git a/tests/test_eval_holdout_split.py b/tests/test_eval_holdout_split.py new file mode 100644 index 00000000..b69b0ebe --- /dev/null +++ b/tests/test_eval_holdout_split.py @@ -0,0 +1,134 @@ +"""ADR-0054 (Part 1) — holdout split wired into the official eval runner. + +Contracts pinned here: + + - ``LaneInfo.holdout_cases_path`` resolves the holdout plaintext file + via a fixed priority (cases.jsonl > cases_plaintext.jsonl > v1/cases.jsonl). + - ``framework.run_lane(split="holdout")`` reads that path and runs the + lane's runner like any other split. + - The cognition lane reports a stable holdout metric set (case count, + intent_accuracy, surface_groundedness, versor_closure_rate). + - Unknown ``split`` values raise ``ValueError`` with a message naming + all three accepted values. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from evals.framework import LaneInfo, get_lane, run_lane + + +# --------------------------------------------------------------------------- +# LaneInfo.holdout_cases_path resolution +# --------------------------------------------------------------------------- + + +def test_cognition_holdout_path_resolves_to_plaintext() -> None: + lane = get_lane("cognition") + path = lane.holdout_cases_path() + assert path.exists() + assert path.name == "cases_plaintext.jsonl" + + +def test_holdout_path_resolution_prefers_cases_jsonl(tmp_path: Path) -> None: + root = tmp_path / "fake_lane" + (root / "holdouts" / "v1").mkdir(parents=True) + (root / "holdouts" / "cases.jsonl").write_text("{}\n") + (root / "holdouts" / "cases_plaintext.jsonl").write_text("{}\n") + (root / "holdouts" / "v1" / "cases.jsonl").write_text("{}\n") + lane = LaneInfo(name="fake_lane", root=root, versions=("v1",)) + assert lane.holdout_cases_path().name == "cases.jsonl" + assert lane.holdout_cases_path().parent.name == "holdouts" + + +def test_holdout_path_falls_back_to_plaintext_then_versioned(tmp_path: Path) -> None: + root = tmp_path / "fake_lane" + (root / "holdouts" / "v1").mkdir(parents=True) + (root / "holdouts" / "cases_plaintext.jsonl").write_text("{}\n") + (root / "holdouts" / "v1" / "cases.jsonl").write_text("{}\n") + lane = LaneInfo(name="fake_lane", root=root, versions=("v1",)) + assert lane.holdout_cases_path().name == "cases_plaintext.jsonl" + + +def test_holdout_path_when_nothing_exists_returns_versioned_path(tmp_path: Path) -> None: + root = tmp_path / "fake_lane" + (root / "holdouts" / "v1").mkdir(parents=True) + lane = LaneInfo(name="fake_lane", root=root, versions=("v1",)) + path = lane.holdout_cases_path() + assert path.name == "cases.jsonl" + assert path.parent.name == "v1" + assert not path.exists() + + +# --------------------------------------------------------------------------- +# framework.run_lane(split="holdout") +# --------------------------------------------------------------------------- + + +def test_run_lane_holdout_runs_full_cognition_set() -> None: + lane = get_lane("cognition") + result = run_lane(lane, split="holdout") + assert result.lane == "cognition" + assert result.split == "holdout" + assert result.metrics["total"] == 19 + + +def test_run_lane_holdout_returns_expected_metric_keys() -> None: + lane = get_lane("cognition") + result = run_lane(lane, split="holdout") + expected = { + "total", + "intent_accuracy", + "term_capture_rate", + "surface_groundedness", + "versor_closure_rate", + } + assert expected.issubset(result.metrics.keys()) + + +def test_run_lane_holdout_versor_closure_preserved() -> None: + """The non-negotiable field invariant (versor_condition < 1e-6) must + hold on every holdout case — same gate as dev/public.""" + lane = get_lane("cognition") + result = run_lane(lane, split="holdout") + assert result.metrics["versor_closure_rate"] == 1.0 + + +def test_run_lane_unknown_split_lists_all_three_values() -> None: + lane = get_lane("cognition") + with pytest.raises(ValueError) as excinfo: + run_lane(lane, split="train") + msg = str(excinfo.value) + assert "dev" in msg + assert "public" in msg + assert "holdout" in msg + + +# --------------------------------------------------------------------------- +# Holdout vs dev/public — consistent eval interface +# --------------------------------------------------------------------------- + + +def test_holdout_dev_public_share_metric_schema() -> None: + lane = get_lane("cognition") + dev = run_lane(lane, split="dev").metrics + public = run_lane(lane, split="public").metrics + holdout = run_lane(lane, split="holdout").metrics + assert set(dev.keys()) == set(public.keys()) == set(holdout.keys()) + + +def test_holdout_cases_have_required_fields() -> None: + lane = get_lane("cognition") + path = lane.holdout_cases_path() + for line in path.read_text().splitlines(): + line = line.strip() + if not line: + continue + case = json.loads(line) + assert "id" in case + assert "prompt" in case + assert "expected_intent" in case diff --git a/tests/test_vault_recall_indexing_batch.py b/tests/test_vault_recall_indexing_batch.py new file mode 100644 index 00000000..22ef6995 --- /dev/null +++ b/tests/test_vault_recall_indexing_batch.py @@ -0,0 +1,254 @@ +"""ADR-0054 — vault recall indexing + batched API. + +Two doctrine-aligned optimisations on top of ADR-0019 Stage 1: + + 1. **Indexing**: ``VaultStore`` keeps a cached ``(N, D)`` f32 matrix + view of the deque, rebuilt lazily on the first recall after any + mutation. Repeated recalls reuse the cached matrix instead of + rebuilding it from a Python list each call. + + 2. **Batching**: ``algebra.backend.vault_recall_batch`` scores B + queries against one matrix in a single component-serial sweep — + bit-identical per-query to ``vault_recall``. + +No approximate search. No hot-path repair. No mutation of shared +cached state during recall. ``versor_condition < 1e-6`` invariant is +not touched by either change. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +from algebra.backend import vault_recall, vault_recall_batch +from teaching.epistemic import EpistemicStatus +from vault.store import VaultStore + + +# --------------------------------------------------------------------------- +# vault_recall_batch — bit-identity vs single-query vault_recall +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("seed", [0xC07E, 0xBEEF, 0x1234]) +def test_batch_matches_per_query_bit_identical(seed: int) -> None: + rng = np.random.default_rng(seed) + N, B = 137, 7 + versors = [rng.standard_normal(size=(32,), dtype=np.float32) for _ in range(N)] + queries = rng.standard_normal(size=(B, 32), dtype=np.float32) + matrix = np.asarray(versors, dtype=np.float32) + + batch = vault_recall_batch(matrix, queries, top_k=N) + per_query = [vault_recall(versors, queries[b], top_k=N) for b in range(B)] + + assert len(batch) == B == len(per_query) + for b in range(B): + # Indices must be identical. + assert [i for i, _ in batch[b]] == [i for i, _ in per_query[b]] + # Scores must be float-equal (bit-identical at f32). + b_scores = np.array([s for _, s in batch[b]], dtype=np.float32) + q_scores = np.array([s for _, s in per_query[b]], dtype=np.float32) + assert np.array_equal(b_scores, q_scores) + + +def test_batch_handles_1d_query() -> None: + rng = np.random.default_rng(0) + versors = [rng.standard_normal(size=(32,), dtype=np.float32) for _ in range(10)] + matrix = np.asarray(versors, dtype=np.float32) + q = rng.standard_normal(size=(32,), dtype=np.float32) + batch = vault_recall_batch(matrix, q, top_k=3) + assert len(batch) == 1 + expected = vault_recall(versors, q, top_k=3) + assert batch[0] == expected + + +def test_batch_empty_matrix_returns_empty_per_query() -> None: + M = np.zeros((0, 32), dtype=np.float32) + Q = np.zeros((3, 32), dtype=np.float32) + out = vault_recall_batch(M, Q, top_k=5) + assert out == [[], [], []] + + +def test_batch_zero_top_k_returns_empty_per_query() -> None: + rng = np.random.default_rng(0) + M = rng.standard_normal(size=(10, 32), dtype=np.float32) + Q = rng.standard_normal(size=(2, 32), dtype=np.float32) + out = vault_recall_batch(M, Q, top_k=0) + assert out == [[], []] + + +def test_batch_shape_mismatch_raises() -> None: + M = np.zeros((5, 32), dtype=np.float32) + Q = np.zeros((2, 31), dtype=np.float32) + with pytest.raises(ValueError) as excinfo: + vault_recall_batch(M, Q, top_k=3) + assert "components" in str(excinfo.value) + + +def test_batch_rejects_higher_dim_matrix() -> None: + M = np.zeros((2, 5, 32), dtype=np.float32) + Q = np.zeros((1, 32), dtype=np.float32) + with pytest.raises(ValueError): + vault_recall_batch(M, Q, top_k=1) + + +def test_batch_top_k_smaller_than_n_preserves_ordering() -> None: + rng = np.random.default_rng(0xDEAD) + versors = [rng.standard_normal(size=(32,), dtype=np.float32) for _ in range(50)] + matrix = np.asarray(versors, dtype=np.float32) + queries = rng.standard_normal(size=(4, 32), dtype=np.float32) + batch = vault_recall_batch(matrix, queries, top_k=5) + for b in range(4): + single = vault_recall(versors, queries[b], top_k=5) + assert batch[b] == single + + +# --------------------------------------------------------------------------- +# VaultStore matrix cache — invalidation correctness +# --------------------------------------------------------------------------- + + +def _v(seed: int) -> np.ndarray: + rng = np.random.default_rng(seed) + return rng.standard_normal(size=(32,), dtype=np.float32) + + +def test_matrix_cache_starts_unbuilt() -> None: + store = VaultStore() + assert store._matrix_cache is None + + +def test_matrix_cache_built_on_first_recall() -> None: + store = VaultStore() + store.store(_v(1)) + store.store(_v(2)) + assert store._matrix_cache is None + store.recall(_v(3), top_k=1) + assert store._matrix_cache is not None + assert store._matrix_cache.shape == (2, 32) + + +def test_matrix_cache_invalidated_on_store() -> None: + store = VaultStore() + store.store(_v(1)) + store.recall(_v(2), top_k=1) + assert store._matrix_cache is not None + store.store(_v(3)) + assert store._matrix_cache is None + + +def test_matrix_cache_invalidated_on_reproject() -> None: + store = VaultStore() + store.store(_v(1)) + store.recall(_v(2), top_k=1) + assert store._matrix_cache is not None + store.reproject() + assert store._matrix_cache is None + + +def test_matrix_cache_invalidated_on_eviction() -> None: + store = VaultStore(max_entries=2) + store.store(_v(1)) + store.store(_v(2)) + store.recall(_v(3), top_k=1) + assert store._matrix_cache is not None + store.store(_v(4)) # triggers eviction → _rebuild_index → invalidate + assert store._matrix_cache is None + + +def test_matrix_cache_does_not_change_recall_results() -> None: + """The cache is an indexing optimisation — results must equal the + pre-cache recall behaviour case-for-case.""" + rng = np.random.default_rng(0xC0DE) + store_a = VaultStore(reproject_interval=0) + store_b = VaultStore(reproject_interval=0) + versors = [rng.standard_normal(size=(32,), dtype=np.float32) for _ in range(20)] + for v in versors: + store_a.store(v) + store_b.store(v) + + for _ in range(5): + q = rng.standard_normal(size=(32,), dtype=np.float32) + # Force store_a to take fresh non-cached path by clearing cache. + store_a._matrix_cache = None + r_a = store_a.recall(q, top_k=5) + # store_b takes cached path on second+ recalls. + store_b.recall(q, top_k=5) # warm cache + store_b._matrix_cache = store_b._get_matrix() # ensure cache exists + r_b = store_b.recall(q, top_k=5) + assert [r["index"] for r in r_a] == [r["index"] for r in r_b] + assert [r["score"] for r in r_a] == [r["score"] for r in r_b] + + +# --------------------------------------------------------------------------- +# VaultStore.recall_batch — parity with per-query recall +# --------------------------------------------------------------------------- + + +def test_recall_batch_matches_per_query_recall() -> None: + rng = np.random.default_rng(0xFACE) + store = VaultStore(reproject_interval=0) + versors = [rng.standard_normal(size=(32,), dtype=np.float32) for _ in range(30)] + for v in versors: + store.store(v) + + queries = rng.standard_normal(size=(4, 32), dtype=np.float32) + batch = store.recall_batch(queries, top_k=5) + per_query = [store.recall(queries[b], top_k=5) for b in range(4)] + + assert len(batch) == 4 + for b in range(4): + assert [r["index"] for r in batch[b]] == [r["index"] for r in per_query[b]] + assert [r["score"] for r in batch[b]] == [r["score"] for r in per_query[b]] + + +def test_recall_batch_empty_vault_returns_empty_per_query() -> None: + store = VaultStore() + Q = np.zeros((3, 32), dtype=np.float32) + out = store.recall_batch(Q, top_k=5) + assert out == [[], [], []] + + +def test_recall_batch_zero_top_k_returns_empty_per_query() -> None: + store = VaultStore() + store.store(_v(1)) + Q = np.zeros((2, 32), dtype=np.float32) + out = store.recall_batch(Q, top_k=0) + assert out == [[], []] + + +def test_recall_batch_accepts_1d_query_as_single_batch() -> None: + store = VaultStore(reproject_interval=0) + store.store(_v(1)) + store.store(_v(2)) + out = store.recall_batch(_v(3), top_k=2) + assert len(out) == 1 + expected = store.recall(_v(3), top_k=2) + assert [r["index"] for r in out[0]] == [r["index"] for r in expected] + + +def test_recall_batch_exact_self_match_promoted() -> None: + """If a query equals a stored versor, its index must appear first + with score=+inf — same contract as single-query recall.""" + store = VaultStore(reproject_interval=0) + target = _v(1) + store.store(_v(0)) + store.store(target) + store.store(_v(2)) + Q = np.stack([target, _v(99)]) + out = store.recall_batch(Q, top_k=3) + assert out[0][0]["index"] == 1 + assert out[0][0]["score"] == float("inf") + + +def test_recall_batch_min_status_filter_applied_per_query() -> None: + store = VaultStore(reproject_interval=0) + store.store(_v(1), epistemic_status=EpistemicStatus.COHERENT) + store.store(_v(2), epistemic_status=EpistemicStatus.SPECULATIVE) + store.store(_v(3), epistemic_status=EpistemicStatus.COHERENT) + Q = np.stack([_v(10), _v(11)]) + out = store.recall_batch(Q, top_k=5, min_status=EpistemicStatus.COHERENT) + for per_query in out: + for r in per_query: + assert r["metadata"]["epistemic_status"] == "coherent" diff --git a/vault/store.py b/vault/store.py index ba8a2080..874c504c 100644 --- a/vault/store.py +++ b/vault/store.py @@ -15,7 +15,7 @@ from __future__ import annotations from collections import deque import numpy as np -from algebra.backend import vault_recall +from algebra.backend import vault_recall, vault_recall_batch from algebra.cga import null_project from teaching.epistemic import ADMISSIBLE_AS_EVIDENCE, EpistemicStatus @@ -49,6 +49,10 @@ class VaultStore: self._reproject_interval = reproject_interval self._max_entries = max_entries self._exact_index: dict[bytes, list[int]] = {} + # ADR-0054: cached (N, D) f32 matrix view of the deque, rebuilt + # lazily on the first recall after any mutation. Indexing + # optimisation only — scoring arithmetic is unchanged. + self._matrix_cache: np.ndarray | None = None def store( self, @@ -80,6 +84,7 @@ class VaultStore: idx = len(self._versors) - 1 key = _versor_key(arr) self._exact_index.setdefault(key, []).append(idx) + self._matrix_cache = None self._store_count += 1 if self._reproject_interval > 0 and self._store_count % self._reproject_interval == 0: @@ -117,7 +122,11 @@ class VaultStore: # has a chance at top_k entries. 4x is a generous heuristic; # vault sizes are bounded by max_entries anyway. scan_k = max(top_k * 4, top_k) if min_status is not None else max(top_k, 1) - ranked = vault_recall(list(self._versors), query_arr, scan_k) + matrix = self._get_matrix() + ranked = vault_recall( + list(self._versors), query_arr, scan_k, + prebuilt_matrix=matrix, + ) key = _versor_key(query_arr) exact_indices = self._exact_index.get(key, []) @@ -148,6 +157,70 @@ class VaultStore: for i, score in ranked[:top_k] ] + def recall_batch( + self, + queries: np.ndarray, + top_k: int = 5, + *, + min_status: EpistemicStatus | None = None, + ) -> list[list[dict]]: + """Recall B queries against the stored versors in one sweep. + + Returns one ``list[dict]`` per query in the same shape ``recall`` + returns. Exact-self-match promotion, ``min_status`` filtering, + and the descending-score / ascending-index tiebreak rule are + applied per query — semantics are identical to looping + ``recall(q, top_k=...)`` over each query, but the underlying + scoring scan is a single component-serial sweep over the + cached matrix. ADR-0054. + """ + Q = np.asarray(queries, dtype=np.float32) + if Q.ndim == 1: + Q = Q[None, :] + if not self._versors or top_k <= 0: + return [[] for _ in range(Q.shape[0])] + + matrix = self._get_matrix() + assert matrix is not None # non-empty deque ⇒ matrix is built + scan_k = max(top_k * 4, top_k) if min_status is not None else max(top_k, 1) + batch_ranked = vault_recall_batch(matrix, Q, scan_k) + + results: list[list[dict]] = [] + for b, ranked in enumerate(batch_ranked): + key = _versor_key(Q[b]) + exact_indices = self._exact_index.get(key, []) + if exact_indices: + exact_matches = [(i, float("inf")) for i in exact_indices] + seen = set(exact_indices) + ranked = exact_matches + [ + (i, score) for i, score in ranked if i not in seen + ] + + if min_status is not None: + filtered: list[tuple[int, float]] = [] + for i, score in ranked: + raw_status = self._metadata[i].get( + "epistemic_status", "speculative", + ) + try: + entry_status = EpistemicStatus(raw_status) + except ValueError: + entry_status = EpistemicStatus.SPECULATIVE + if _status_admits(entry_status, min_status): + filtered.append((i, score)) + ranked = filtered + + results.append([ + { + "versor": self._versors[i], + "score": float(score), + "metadata": self._metadata[i], + "index": i, + } + for i, score in ranked[:top_k] + ]) + return results + def reproject(self) -> None: """ Re-project all stored versors onto the null cone. @@ -162,6 +235,20 @@ class VaultStore: for i, v in enumerate(self._versors): key = _versor_key(v) self._exact_index.setdefault(key, []).append(i) + self._matrix_cache = None + + def _get_matrix(self) -> np.ndarray | None: + """Return the cached (N, D) f32 stack of stored versors. + + Rebuilds the cache on first call after any mutation. Returns + None when the vault is empty so callers can branch without + constructing a 0-row array. ADR-0054. + """ + if not self._versors: + return None + if self._matrix_cache is None: + self._matrix_cache = np.asarray(self._versors, dtype=np.float32) + return self._matrix_cache @property def reproject_interval(self) -> int: