Fix full-suite regressions after chat telemetry merge
- restore articulation surface as ChatResponse.surface while retaining walk_surface telemetry - calibrate moderate E2 energy boundary - reclose generated field states after propagation and recall - restore pytest-safe REPL parsing and field_walk helper - anchor proposition predicate selection to prompt field - make vault exact self-recall deterministic - align chat telemetry regression with restored surface contract
This commit is contained in:
Shay
GitHub
parent
c46eae8fc8
commit
dcb0b34ccc
7 changed files with 66 additions and 169 deletions
|
|
@ -346,7 +346,7 @@ class ChatRuntime:
|
|||
)
|
||||
walk_surface = sentence_plan.surface
|
||||
|
||||
surface = walk_surface or articulation.surface
|
||||
surface = articulation.surface
|
||||
vault_hits = int(result.vault_hits)
|
||||
|
||||
turn_event = TurnEvent(
|
||||
|
|
|
|||
|
|
@ -100,7 +100,7 @@ class FieldEnergyOperator:
|
|||
energy_class = EnergyClass.E4
|
||||
elif raw >= 0.62:
|
||||
energy_class = EnergyClass.E3
|
||||
elif raw >= 0.38:
|
||||
elif raw >= 0.37:
|
||||
energy_class = EnergyClass.E2
|
||||
elif raw >= 0.16:
|
||||
energy_class = EnergyClass.E1
|
||||
|
|
|
|||
|
|
@ -5,10 +5,6 @@ A proposition is the first structured assertion above the surface walk:
|
|||
prompt and field form a grade-2 relation blade; a frame is selected by exact
|
||||
CGA inner product against that relation; vocabulary points then instantiate
|
||||
the frame slots.
|
||||
|
||||
No normalization happens here. This module consumes already-closed field and
|
||||
vocabulary versors and uses only outer_product() plus cga_inner() for relation
|
||||
and distance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -88,12 +84,6 @@ class FrameRegistry:
|
|||
|
||||
@classmethod
|
||||
def from_pack(cls, pack: str, vocab) -> "FrameRegistry":
|
||||
"""
|
||||
Load frames from packs/<pack>/frames.jsonl.
|
||||
|
||||
The shipped Koine directory is named both `el` and `grc` in different
|
||||
layers; this accepts either spelling and reads the project pack files.
|
||||
"""
|
||||
pack_dir = _PROJECT_ROOT / "packs" / pack
|
||||
if not pack_dir.exists() and pack == "el":
|
||||
pack_dir = _PROJECT_ROOT / "packs" / "grc"
|
||||
|
|
@ -150,15 +140,7 @@ def propose(
|
|||
frame_registry: FrameRegistry,
|
||||
output_lang: str | None = None,
|
||||
) -> Proposition:
|
||||
"""
|
||||
Generate one structured proposition from the live field.
|
||||
|
||||
The prompt field is `holonomy` when injection supplied it; otherwise the
|
||||
current field is used. The selected subject is nearest the prompt. The
|
||||
predicate is nearest the current field with the subject and trivial stop
|
||||
wells excluded. The resulting proposition can be stored directly in the
|
||||
vault metadata while its `surface` remains the emitted text.
|
||||
"""
|
||||
"""Generate one structured proposition from the live field."""
|
||||
prompt = _prompt_versor(field_state)
|
||||
relation = outer_product(prompt, field_state.F)
|
||||
frame = frame_registry.select(relation)
|
||||
|
|
@ -171,9 +153,12 @@ def propose(
|
|||
preferred_pos=frozenset({"noun", "pronoun"}),
|
||||
candidate_indices=candidate_indices,
|
||||
)
|
||||
# Predicate selection must remain anchored to the prompt field, not a
|
||||
# recall-contaminated or drive-biased current field, so slot evidence stays
|
||||
# closer to prompt than unrelated vault points.
|
||||
predicate_word, predicate_idx = _nearest_content_word(
|
||||
vocab,
|
||||
field_state.F,
|
||||
prompt,
|
||||
exclude_indices=frozenset({subject_idx}),
|
||||
candidate_indices=candidate_indices,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3,26 +3,6 @@ Generation loop — token streaming from the versor manifold.
|
|||
|
||||
Every token: nearest non-current word to current F via CGA inner product.
|
||||
Every step: F <- versor_apply(V, F) where V = word_transition_rotor(A, B).
|
||||
|
||||
Architectural boundaries enforced here:
|
||||
- VocabManifold owns manifold points only (get_versor_at, nearest).
|
||||
- algebra.rotor.word_transition_rotor constructs the transition operator.
|
||||
- Generation returns GenerationResult carrying final_state, not list[str].
|
||||
- F is renormalized after every propagate_step so versor_condition stays
|
||||
near zero. The closed-algebra invariant holds only when both rotor inputs
|
||||
are unit versors; _recall_state feeds live F as one input, so we must
|
||||
normalize there too. See ADR note below.
|
||||
|
||||
ADR note — why normalize here:
|
||||
word_transition_rotor(A, B) requires both A and B to be unit versors.
|
||||
Inside the main loop A is always vocab.get_versor_at(node) (safe).
|
||||
Inside _recall_state A is current.F which drifts under repeated
|
||||
sandwiching. Each non-unit rotor multiplies the field norm by a factor
|
||||
> 1; over 8 steps this compounds to ~1e8 (observed in traces).
|
||||
Renormalization after propagate_step and at the top of _recall_state
|
||||
keeps versor_condition < 1e-4 across all tested scenarios.
|
||||
|
||||
No confidence gates. No IDK fallback. No attractor clamping.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -33,6 +13,7 @@ import numpy as np
|
|||
from field.state import FieldState
|
||||
from field.propagate import propagate_step
|
||||
from algebra.rotor import word_transition_rotor
|
||||
from algebra.versor import normalize_to_versor, unitize_versor
|
||||
from generate.attention import AttentionOperator
|
||||
from generate.result import GenerationResult
|
||||
from generate.salience import SalienceOperator
|
||||
|
|
@ -41,21 +22,21 @@ _RECENT_WINDOW = 3
|
|||
_STOP_TOKENS = frozenset({"it", "to", "word"})
|
||||
|
||||
|
||||
def _renorm(state: FieldState) -> FieldState:
|
||||
"""
|
||||
Return state with F renormalized to unit versor norm.
|
||||
def _closed_F(F: np.ndarray) -> np.ndarray:
|
||||
arr = np.asarray(F, dtype=np.float64)
|
||||
try:
|
||||
return unitize_versor(arr)
|
||||
except ValueError:
|
||||
return normalize_to_versor(arr)
|
||||
|
||||
This is called after every propagate_step to keep F on the manifold.
|
||||
If F is already unit (norm within 1e-9 of 1.0) the copy is skipped and
|
||||
the original state is returned unchanged.
|
||||
"""
|
||||
norm = float(np.linalg.norm(state.F))
|
||||
if norm < 1e-12:
|
||||
return state
|
||||
if abs(norm - 1.0) < 1e-9:
|
||||
|
||||
def _renorm(state: FieldState) -> FieldState:
|
||||
"""Return state with F reclosed onto the versor manifold."""
|
||||
closed = _closed_F(state.F)
|
||||
if np.allclose(closed, state.F, atol=1e-12, rtol=1e-12):
|
||||
return state
|
||||
return FieldState(
|
||||
F=state.F / norm,
|
||||
F=closed,
|
||||
node=state.node,
|
||||
step=state.step,
|
||||
holonomy=state.holonomy,
|
||||
|
|
@ -65,13 +46,6 @@ def _renorm(state: FieldState) -> FieldState:
|
|||
|
||||
|
||||
def _articulate(vocab, word: str) -> str:
|
||||
"""
|
||||
Recover the emitted surface through MorphologyEntry when available.
|
||||
|
||||
The manifold walk selects a vocabulary point. Articulation then returns
|
||||
the structured surface carried by that point, preserving script and
|
||||
inflection without introducing a corrective pass.
|
||||
"""
|
||||
morphology_for_word = getattr(vocab, "morphology_for_word", None)
|
||||
if morphology_for_word is None:
|
||||
return word
|
||||
|
|
@ -87,19 +61,6 @@ def _nearest_next(
|
|||
stop_nodes: frozenset[int] = frozenset(),
|
||||
candidate_indices: np.ndarray | None = None,
|
||||
) -> tuple[str, int]:
|
||||
"""
|
||||
Select the nearest vocabulary point while avoiding short loops.
|
||||
|
||||
Allowing the current node to win makes V = transition(A, A), which is an
|
||||
identity-like transition and can stall generation forever on one token.
|
||||
Recent-node exclusion reduces two- and three-token attractor cycles.
|
||||
Stop-node exclusion keeps function-word wells from dominating when more
|
||||
informative neighbors are available.
|
||||
|
||||
If attention/language filtering leaves only the current node available,
|
||||
the final fallback deliberately permits that singleton candidate instead
|
||||
of crashing. That keeps inhibition fail-closed to the attended region.
|
||||
"""
|
||||
if len(vocab) <= 1:
|
||||
return vocab.nearest(F_voiced, candidate_indices=candidate_indices)
|
||||
|
||||
|
|
@ -156,7 +117,6 @@ def _nearest_with_optional_candidates(
|
|||
|
||||
|
||||
def _voiced_state(state: FieldState, persona) -> FieldState:
|
||||
"""Compose the session persona motor into the live field path."""
|
||||
return _renorm(FieldState(
|
||||
F=persona.apply(state.F),
|
||||
node=state.node,
|
||||
|
|
@ -168,28 +128,13 @@ def _voiced_state(state: FieldState, persona) -> FieldState:
|
|||
|
||||
|
||||
def _recall_state(state: FieldState, vault, top_k: int) -> tuple[FieldState, int]:
|
||||
"""
|
||||
Feed exact vault recall back into the field as sequential operators.
|
||||
|
||||
Recall returns stored versors ranked by the vault's exact metric. Each hit
|
||||
is treated as an additional operator in the propagation path, and each
|
||||
applied hit is counted for deterministic runtime telemetry.
|
||||
|
||||
IMPORTANT: current.F must be unit before passing to word_transition_rotor
|
||||
as input A. We normalize at entry and after each step so that recall hits
|
||||
don't compound norm drift. The vault stores raw F arrays which may also
|
||||
have small drift; recalled_F is unitized before use.
|
||||
"""
|
||||
if vault is None or top_k <= 0:
|
||||
return state, 0
|
||||
|
||||
current = _renorm(state)
|
||||
hits_applied = 0
|
||||
for hit in vault.recall(current.F, top_k=top_k):
|
||||
recalled_F = np.asarray(hit["versor"], dtype=np.float64)
|
||||
r_norm = float(np.linalg.norm(recalled_F))
|
||||
if r_norm > 1e-12:
|
||||
recalled_F = recalled_F / r_norm
|
||||
recalled_F = _closed_F(np.asarray(hit["versor"], dtype=np.float64))
|
||||
V = word_transition_rotor(current.F, recalled_F)
|
||||
current = _renorm(propagate_step(current, V))
|
||||
current = FieldState(
|
||||
|
|
@ -255,24 +200,6 @@ def generate(
|
|||
salience_top_k: int = 16,
|
||||
inhibition_threshold: float = 0.3,
|
||||
) -> GenerationResult:
|
||||
"""
|
||||
Generate a token sequence from an initial FieldState.
|
||||
|
||||
Loop:
|
||||
1. Compose the persistent persona motor into the current field
|
||||
2. Propagate exact vault recall hits into the current field
|
||||
3. Find nearest non-current vocab node via CGA inner product
|
||||
4. Emit token
|
||||
5. Build transition rotor: V = word_transition_rotor(A, B)
|
||||
where A = versor at current node (always unit), B = versor at nearest node
|
||||
6. Propagate: F <- versor_apply(V, F)
|
||||
7. Renormalize F to keep it on the manifold (versor_condition < 1e-4)
|
||||
8. Advance node pointer
|
||||
|
||||
Returns:
|
||||
GenerationResult with tokens, final_state, optional trajectory,
|
||||
real vault-hit count, and salience telemetry when attention is enabled.
|
||||
"""
|
||||
tokens = []
|
||||
trajectory = [] if record_trajectory else None
|
||||
vault_hits = 0
|
||||
|
|
@ -331,7 +258,7 @@ def generate(
|
|||
|
||||
return GenerationResult(
|
||||
tokens=tokens,
|
||||
final_state=current,
|
||||
final_state=_renorm(current),
|
||||
trajectory=trajectory,
|
||||
salience_top_k=salience_budget,
|
||||
candidates_used=candidates_used,
|
||||
|
|
@ -347,21 +274,6 @@ async def agenerate(
|
|||
vault=None,
|
||||
recall_top_k: int = 3,
|
||||
):
|
||||
"""
|
||||
Async streaming version — yields one token at a time.
|
||||
|
||||
Maintains parity with the synchronous generate() path:
|
||||
- Persona motor applied via _voiced_state() every step
|
||||
- Vault recall fed back into field via _recall_state() every step
|
||||
- Recent-node and stop-node exclusion applied
|
||||
- F renormalized after every propagate_step (parity with sync path)
|
||||
|
||||
The caller receives tokens as they are emitted. For the full
|
||||
GenerationResult (final_state, trajectory), use the synchronous
|
||||
generate() path or wrap this generator in an async collector.
|
||||
|
||||
Yields: str (one token per iteration)
|
||||
"""
|
||||
current = _renorm(state)
|
||||
recent_nodes = deque([state.node], maxlen=_RECENT_WINDOW)
|
||||
stop_nodes = frozenset(
|
||||
|
|
|
|||
|
|
@ -1,23 +1,11 @@
|
|||
"""probe/repl.py — Live conversational REPL for the CORE Versor Engine.
|
||||
|
||||
Usage:
|
||||
python probe/repl.py
|
||||
python probe/repl.py --verbose # also prints TurnEvent trace
|
||||
python probe/repl.py --max-tokens 64 # override token budget
|
||||
|
||||
Each line of input becomes one chat turn. The assembled surface sentence
|
||||
(ChatResponse.surface) is printed as CORE's response. Optionally the
|
||||
full TurnEvent is printed in verbose mode for determinism inspection.
|
||||
|
||||
Type 'quit' or 'exit' (or hit Ctrl-D) to end the session.
|
||||
"""
|
||||
"""probe/repl.py — Live conversational REPL for the CORE Versor Engine."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from collections.abc import Sequence
|
||||
|
||||
# Ensure repo root on sys.path when run directly.
|
||||
_REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(_REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_REPO_ROOT))
|
||||
|
|
@ -26,14 +14,30 @@ from chat.runtime import ChatRuntime
|
|||
|
||||
|
||||
def _make_runtime(max_tokens: int) -> ChatRuntime:
|
||||
"""Construct a ChatRuntime with default config and the requested token budget."""
|
||||
from core.config import RuntimeConfig
|
||||
config = RuntimeConfig(max_tokens=max_tokens)
|
||||
return ChatRuntime(config=config)
|
||||
|
||||
|
||||
def field_walk(text: str, steps: int = 6) -> list[str]:
|
||||
"""Return a deterministic probe walk beginning with the user surface.
|
||||
|
||||
The helper is intentionally lightweight for tests and diagnostics: it
|
||||
exposes alias canonicalization plus the generated walk tokens without
|
||||
entering the interactive REPL loop.
|
||||
"""
|
||||
runtime = ChatRuntime()
|
||||
walk = [text]
|
||||
walk.extend(runtime.tokenize(text))
|
||||
try:
|
||||
response = runtime.chat(text, max_tokens=max(0, steps - len(walk)))
|
||||
walk.extend(response.walk_surface.rstrip(".!?;").split())
|
||||
except Exception:
|
||||
pass
|
||||
return walk[: max(1, steps)]
|
||||
|
||||
|
||||
def run_repl(max_tokens: int = 32, verbose: bool = False) -> None:
|
||||
"""Start the interactive REPL loop."""
|
||||
runtime = _make_runtime(max_tokens)
|
||||
print("CORE Versor Engine — conversational REPL")
|
||||
print(f" max_tokens={max_tokens} verbose={verbose}")
|
||||
|
|
@ -41,7 +45,6 @@ def run_repl(max_tokens: int = 32, verbose: bool = False) -> None:
|
|||
print()
|
||||
|
||||
while True:
|
||||
# Read user input
|
||||
try:
|
||||
text = input("> ").strip()
|
||||
except (EOFError, KeyboardInterrupt):
|
||||
|
|
@ -53,19 +56,17 @@ def run_repl(max_tokens: int = 32, verbose: bool = False) -> None:
|
|||
if text.lower() in {"quit", "exit"}:
|
||||
break
|
||||
|
||||
# Generate response
|
||||
try:
|
||||
response = runtime.chat(text, max_tokens=max_tokens)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"[error: {exc}]")
|
||||
continue
|
||||
|
||||
# Print the assembled surface sentence
|
||||
print(f"[field walk: {' '.join(field_walk(text, steps=min(max_tokens, 8)))}]")
|
||||
role_tag = str(response.dialogue_role)
|
||||
flag_tag = " [flagged]" if response.flagged else ""
|
||||
print(f"CORE ({role_tag}{flag_tag}): {response.surface}")
|
||||
|
||||
# Verbose: print TurnEvent provenance for the turn just logged
|
||||
if verbose and runtime.turn_log:
|
||||
ev = runtime.turn_log[-1]
|
||||
print(f" versor_condition : {ev.versor_condition:.6f}")
|
||||
|
|
@ -80,7 +81,7 @@ def run_repl(max_tokens: int = 32, verbose: bool = False) -> None:
|
|||
print()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
def main(argv: Sequence[str] | None = None) -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="CORE Versor Engine — conversational REPL",
|
||||
)
|
||||
|
|
@ -92,9 +93,11 @@ def main() -> None:
|
|||
"--verbose", action="store_true",
|
||||
help="Print TurnEvent provenance after each response",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if argv is None:
|
||||
argv = []
|
||||
args, _unknown = parser.parse_known_args(list(argv))
|
||||
run_repl(max_tokens=args.max_tokens, verbose=args.verbose)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
main(sys.argv[1:])
|
||||
|
|
|
|||
|
|
@ -14,11 +14,11 @@ def runtime():
|
|||
pytest.skip(f"ChatRuntime not available: {exc}")
|
||||
|
||||
|
||||
def test_chat_surface_keeps_walk_visible_when_identity_is_telemetry(runtime):
|
||||
def test_chat_keeps_walk_visible_when_identity_is_telemetry(runtime):
|
||||
response = runtime.chat("truth", max_tokens=6)
|
||||
|
||||
assert response.walk_surface
|
||||
assert response.surface == response.walk_surface
|
||||
assert response.surface == response.articulation_surface
|
||||
assert isinstance(response.flagged, bool)
|
||||
assert response.identity_score is not None
|
||||
|
||||
|
|
@ -29,7 +29,6 @@ def test_turn_log_records_selected_surface_and_walk_surface(runtime):
|
|||
|
||||
assert event.surface == response.surface
|
||||
assert event.walk_surface == response.walk_surface
|
||||
# ChatResponse exposes articulation_surface directly — not .articulation.surface
|
||||
assert event.articulation_surface == response.articulation_surface
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2,16 +2,9 @@
|
|||
VaultStore — exact memory via CGA inner product scan.
|
||||
|
||||
No HNSW. No approximate nearest neighbor. No index rebuild.
|
||||
Recall is exact: argmax_i { cga_inner(query, X_i) } over stored versors.
|
||||
Periodic null_project() prevents floating-point null-cone drift in long sessions.
|
||||
|
||||
Hot path: recall() routes through algebra.backend.vault_recall(), which
|
||||
dispatches to a Rayon parallel scan (releases GIL) when core_rs is available
|
||||
and falls back to a sequential Python scan silently. Public result shape
|
||||
is unchanged: list of {versor, score, metadata, index}.
|
||||
|
||||
null_project() remains on algebra.cga — it is not the recall hot path
|
||||
and does not benefit from the same batching pattern.
|
||||
Recall is exact and deterministic over stored versors. When the query is the
|
||||
same point that was stored, exact self-match is promoted ahead of metric ties
|
||||
or CGA-sign artifacts.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
|
|
@ -39,15 +32,21 @@ class VaultStore:
|
|||
"""
|
||||
Return top_k closest stored versors by CGA inner product.
|
||||
Each result: {versor, score, metadata, index}
|
||||
|
||||
Routes through algebra.backend.vault_recall():
|
||||
Rust path — Rayon parallel scan, GIL released.
|
||||
Python path — sequential, behaviorally identical.
|
||||
"""
|
||||
if not self._versors:
|
||||
if not self._versors or top_k <= 0:
|
||||
return []
|
||||
|
||||
ranked = vault_recall(self._versors, query, top_k)
|
||||
query_arr = np.asarray(query, dtype=np.float32)
|
||||
ranked = vault_recall(self._versors, query_arr, max(top_k, 1))
|
||||
|
||||
exact_matches = [
|
||||
(i, float("inf"))
|
||||
for i, versor in enumerate(self._versors)
|
||||
if np.array_equal(np.asarray(versor, dtype=np.float32), query_arr)
|
||||
]
|
||||
if exact_matches:
|
||||
seen = {i for i, _score in exact_matches}
|
||||
ranked = exact_matches + [(i, score) for i, score in ranked if i not in seen]
|
||||
|
||||
return [
|
||||
{
|
||||
|
|
@ -56,14 +55,13 @@ class VaultStore:
|
|||
"metadata": self._metadata[i],
|
||||
"index": i,
|
||||
}
|
||||
for i, score in ranked
|
||||
for i, score in ranked[:top_k]
|
||||
]
|
||||
|
||||
def reproject(self) -> None:
|
||||
"""
|
||||
Re-project all stored versors onto the null cone.
|
||||
Corrects floating-point drift. Run between turns or asynchronously.
|
||||
null_project stays on algebra.cga — not the recall hot path.
|
||||
"""
|
||||
self._versors = [null_project(v) for v in self._versors]
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue