diff --git a/evals/inference_closure/gaps.md b/evals/inference_closure/gaps.md index 585058fa..a5ecbeaf 100644 --- a/evals/inference_closure/gaps.md +++ b/evals/inference_closure/gaps.md @@ -1,5 +1,30 @@ # inference-closure lane — architectural findings (v1) +## Regression and re-resolution — 2026-05-22 + +Between the 2026-05-17 resolution and 2026-05-22, a regression in +`generate/intent.py` caused correction-prefixed premises ("Actually X +precedes Y.", "Actually X grounds Y.", "Actually X causes Y.") to +route to VERIFICATION instead of CORRECTION. The premise emit path is +gated on CORRECTION intent, so `PackMutationProposal` records stopped +being emitted for any non-`is` relation. The lane fell to +`all_pass_rate=0.4` on public — only the four `transitive_is` cases +passed because `is` isn't in `_DECLARATIVE_RELATION_RE`'s verb list. + +Resolution: added `_CORRECTION_CUE_PREFIX_RE` guard so a leading +"Actually" / "Incorrect, " / "No, " / "Correction" cue skips the +declarative-match branch and falls through to the `_RULES` CORRECTION +rule. Pinned by `tests/test_correction_cue_prefix_routing.py`. + +Post-fix on 2026-05-22: + +| Split | n | derived_recall_rate | premises_stored | replay | overall | +|---|---|---|---|---|---| +| dev/v1 | 5 | **1.0** | 1.0 | 1.0 | ✓ | +| public/v1 | 20 | **1.0** | 1.0 | 1.0 | ✓ | + +The 2026-05-17 resolution narrative below is preserved unchanged. + ## Resolution — 2026-05-17 lane re-run After the typed deterministic operators (ADR-0018: `transitive_walk`, diff --git a/evals/inference_closure/results/v1_dev_20260522T192832Z.json b/evals/inference_closure/results/v1_dev_20260522T192832Z.json new file mode 100644 index 00000000..cdc72849 --- /dev/null +++ b/evals/inference_closure/results/v1_dev_20260522T192832Z.json @@ -0,0 +1,106 @@ +{ + "cases": [ + { + "derived_recall_pass": true, + "entailment_tokens": [ + "truth" + ], + "expected_proposals": 2, + "id": "INF-DEV-001", + "passed": true, + "pattern": "transitive_is", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "e228ea6d2d138d03d7b2111cb98cbea6ba68fb0a7bc05b02831a19dd669007a8", + "trace_hash_replay": "e228ea6d2d138d03d7b2111cb98cbea6ba68fb0a7bc05b02831a19dd669007a8", + "vault_hit": false, + "vault_hits": 9 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "meaning" + ], + "expected_proposals": 2, + "id": "INF-DEV-002", + "passed": true, + "pattern": "transitive_precedes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "ed7232b60c363a369a10400b3ce264d6a84905795b9b197f5b45ddccddf30be5", + "trace_hash_replay": "ed7232b60c363a369a10400b3ce264d6a84905795b9b197f5b45ddccddf30be5", + "vault_hit": false, + "vault_hits": 7 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "judgment" + ], + "expected_proposals": 2, + "id": "INF-DEV-003", + "passed": true, + "pattern": "transitive_grounds", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "8b3e9a44665362ac059ff08c1806b6037ba46479934fe6e8e625d9194fc2850a", + "trace_hash_replay": "8b3e9a44665362ac059ff08c1806b6037ba46479934fe6e8e625d9194fc2850a", + "vault_hit": false, + "vault_hits": 8 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "recognition" + ], + "expected_proposals": 2, + "id": "INF-DEV-004", + "passed": true, + "pattern": "transitive_causes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "ad9ce5470b3ad041e312986a73bdbab5f08f287c53b82786faa691bec17e5699", + "trace_hash_replay": "ad9ce5470b3ad041e312986a73bdbab5f08f287c53b82786faa691bec17e5699", + "vault_hit": false, + "vault_hits": 10 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "thought" + ], + "expected_proposals": 2, + "id": "INF-DEV-005", + "passed": true, + "pattern": "transitive_belongs_to", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "566823c524d52ca919dbf65367e8ce11a7b21288ec8e609f68c91da285fd17a2", + "trace_hash_replay": "566823c524d52ca919dbf65367e8ce11a7b21288ec8e609f68c91da285fd17a2", + "vault_hit": false, + "vault_hits": 9 + } + ], + "lane": "inference_closure", + "metrics": { + "all_pass_rate": 1.0, + "case_count": 5, + "derived_recall_rate": 1.0, + "overall_pass": true, + "premises_stored_rate": 1.0, + "replay_determinism": 1.0 + }, + "split": "dev", + "timestamp": "2026-05-22T19:28:32.992593+00:00", + "version": "v1" +} diff --git a/evals/inference_closure/results/v1_public_20260522T192901Z.json b/evals/inference_closure/results/v1_public_20260522T192901Z.json new file mode 100644 index 00000000..c74b9a1f --- /dev/null +++ b/evals/inference_closure/results/v1_public_20260522T192901Z.json @@ -0,0 +1,376 @@ +{ + "cases": [ + { + "derived_recall_pass": true, + "entailment_tokens": [ + "truth" + ], + "expected_proposals": 2, + "id": "INF-V1-001", + "passed": true, + "pattern": "transitive_is", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "e228ea6d2d138d03d7b2111cb98cbea6ba68fb0a7bc05b02831a19dd669007a8", + "trace_hash_replay": "e228ea6d2d138d03d7b2111cb98cbea6ba68fb0a7bc05b02831a19dd669007a8", + "vault_hit": false, + "vault_hits": 9 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "principle" + ], + "expected_proposals": 2, + "id": "INF-V1-002", + "passed": true, + "pattern": "transitive_is", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "9e80eacaa26d08d54cf537c0fee0d238a93eeb97aeb40e8a0e27bdaef673b3f7", + "trace_hash_replay": "9e80eacaa26d08d54cf537c0fee0d238a93eeb97aeb40e8a0e27bdaef673b3f7", + "vault_hit": false, + "vault_hits": 9 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "wisdom" + ], + "expected_proposals": 2, + "id": "INF-V1-003", + "passed": true, + "pattern": "transitive_is", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "c55631835496b7bdea5dd88b3a07e9c2bfd266baf9bddc982618e309223b5219", + "trace_hash_replay": "c55631835496b7bdea5dd88b3a07e9c2bfd266baf9bddc982618e309223b5219", + "vault_hit": false, + "vault_hits": 8 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "structure" + ], + "expected_proposals": 2, + "id": "INF-V1-004", + "passed": true, + "pattern": "transitive_is", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "478e8aa0bd4a108b568be00f79de90db0bf50d57f78776f5d16625ba5bafd731", + "trace_hash_replay": "478e8aa0bd4a108b568be00f79de90db0bf50d57f78776f5d16625ba5bafd731", + "vault_hit": false, + "vault_hits": 7 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "meaning" + ], + "expected_proposals": 2, + "id": "INF-V1-005", + "passed": true, + "pattern": "transitive_precedes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "ed7232b60c363a369a10400b3ce264d6a84905795b9b197f5b45ddccddf30be5", + "trace_hash_replay": "ed7232b60c363a369a10400b3ce264d6a84905795b9b197f5b45ddccddf30be5", + "vault_hit": false, + "vault_hits": 7 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "recall" + ], + "expected_proposals": 2, + "id": "INF-V1-006", + "passed": true, + "pattern": "transitive_precedes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "3895f56f601b5604a8069b2ef4d97cc039e044e46ed7da1698f71670544c61dd", + "trace_hash_replay": "3895f56f601b5604a8069b2ef4d97cc039e044e46ed7da1698f71670544c61dd", + "vault_hit": false, + "vault_hits": 6 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "recognition" + ], + "expected_proposals": 2, + "id": "INF-V1-007", + "passed": true, + "pattern": "transitive_precedes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "556b62484be30ef6e3d8f757e5f3226ea5c2f2cf84ea8cf46a4c44aae61cd3dd", + "trace_hash_replay": "556b62484be30ef6e3d8f757e5f3226ea5c2f2cf84ea8cf46a4c44aae61cd3dd", + "vault_hit": false, + "vault_hits": 7 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "judgment" + ], + "expected_proposals": 2, + "id": "INF-V1-008", + "passed": true, + "pattern": "transitive_precedes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "649f63f06ffcb16c9f3157a67b72bd3a849e083a9c607d81265334ab66eec9df", + "trace_hash_replay": "649f63f06ffcb16c9f3157a67b72bd3a849e083a9c607d81265334ab66eec9df", + "vault_hit": false, + "vault_hits": 7 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "judgment" + ], + "expected_proposals": 2, + "id": "INF-V1-009", + "passed": true, + "pattern": "transitive_grounds", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "8b3e9a44665362ac059ff08c1806b6037ba46479934fe6e8e625d9194fc2850a", + "trace_hash_replay": "8b3e9a44665362ac059ff08c1806b6037ba46479934fe6e8e625d9194fc2850a", + "vault_hit": false, + "vault_hits": 8 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "inference" + ], + "expected_proposals": 2, + "id": "INF-V1-010", + "passed": true, + "pattern": "transitive_grounds", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "2411f6c620a83b791f9ce531c2cc2e2ab99232dc4106f3d675a722d031d26de8", + "trace_hash_replay": "2411f6c620a83b791f9ce531c2cc2e2ab99232dc4106f3d675a722d031d26de8", + "vault_hit": false, + "vault_hits": 6 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "confidence" + ], + "expected_proposals": 2, + "id": "INF-V1-011", + "passed": true, + "pattern": "transitive_grounds", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "e78cf2211296722f5818acc51e0e27b1bf7078a73126101452fa4e3236f74261", + "trace_hash_replay": "e78cf2211296722f5818acc51e0e27b1bf7078a73126101452fa4e3236f74261", + "vault_hit": false, + "vault_hits": 11 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "meaning" + ], + "expected_proposals": 2, + "id": "INF-V1-012", + "passed": true, + "pattern": "transitive_grounds", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "a054ef36b7d692a6a36655f09d4437b9d375b4f1e8dd07a913be50bec63b8f0f", + "trace_hash_replay": "a054ef36b7d692a6a36655f09d4437b9d375b4f1e8dd07a913be50bec63b8f0f", + "vault_hit": false, + "vault_hits": 8 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "recognition" + ], + "expected_proposals": 2, + "id": "INF-V1-013", + "passed": true, + "pattern": "transitive_causes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "ad9ce5470b3ad041e312986a73bdbab5f08f287c53b82786faa691bec17e5699", + "trace_hash_replay": "ad9ce5470b3ad041e312986a73bdbab5f08f287c53b82786faa691bec17e5699", + "vault_hit": false, + "vault_hits": 10 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "thought" + ], + "expected_proposals": 2, + "id": "INF-V1-014", + "passed": true, + "pattern": "transitive_causes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "dddc38be0922e26c67d8315d2a90b5f50225577b10dc0b223b3d02345c8e482a", + "trace_hash_replay": "dddc38be0922e26c67d8315d2a90b5f50225577b10dc0b223b3d02345c8e482a", + "vault_hit": false, + "vault_hits": 9 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "decision" + ], + "expected_proposals": 2, + "id": "INF-V1-015", + "passed": true, + "pattern": "transitive_causes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "7e2387159559363b524d0dd5ee84a51f32c15a3029cd13e1af443014a70511ed", + "trace_hash_replay": "7e2387159559363b524d0dd5ee84a51f32c15a3029cd13e1af443014a70511ed", + "vault_hit": false, + "vault_hits": 9 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "coherence" + ], + "expected_proposals": 2, + "id": "INF-V1-016", + "passed": true, + "pattern": "transitive_causes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "bbc40a3e50910c23271a86adb4a56b8207a652f6bf3ef54ad715410d2d825376", + "trace_hash_replay": "bbc40a3e50910c23271a86adb4a56b8207a652f6bf3ef54ad715410d2d825376", + "vault_hit": false, + "vault_hits": 8 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "thought" + ], + "expected_proposals": 2, + "id": "INF-V1-017", + "passed": true, + "pattern": "transitive_belongs_to", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "566823c524d52ca919dbf65367e8ce11a7b21288ec8e609f68c91da285fd17a2", + "trace_hash_replay": "566823c524d52ca919dbf65367e8ce11a7b21288ec8e609f68c91da285fd17a2", + "vault_hit": false, + "vault_hits": 9 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "truth" + ], + "expected_proposals": 2, + "id": "INF-V1-018", + "passed": true, + "pattern": "transitive_belongs_to", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "6f7d7900846ce601176022d6feb8bafeedc18065c4ef42c8296f6ee451c5435a", + "trace_hash_replay": "6f7d7900846ce601176022d6feb8bafeedc18065c4ef42c8296f6ee451c5435a", + "vault_hit": false, + "vault_hits": 9 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "cognition" + ], + "expected_proposals": 2, + "id": "INF-V1-019", + "passed": true, + "pattern": "transitive_belongs_to", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "15f87deb2fdc497c6bcab3343a6d781850728b725904623d34518391b503a893", + "trace_hash_replay": "15f87deb2fdc497c6bcab3343a6d781850728b725904623d34518391b503a893", + "vault_hit": false, + "vault_hits": 7 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "truth" + ], + "expected_proposals": 2, + "id": "INF-V1-020", + "passed": true, + "pattern": "transitive_belongs_to", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "4f9a081207336549958a1d0c7105ce20d0a045a09842f182e6335f050c6c9eb4", + "trace_hash_replay": "4f9a081207336549958a1d0c7105ce20d0a045a09842f182e6335f050c6c9eb4", + "vault_hit": false, + "vault_hits": 9 + } + ], + "lane": "inference_closure", + "metrics": { + "all_pass_rate": 1.0, + "case_count": 20, + "derived_recall_rate": 1.0, + "overall_pass": true, + "premises_stored_rate": 1.0, + "replay_determinism": 1.0 + }, + "split": "public", + "timestamp": "2026-05-22T19:29:01.896298+00:00", + "version": "v1" +} diff --git a/generate/intent.py b/generate/intent.py index bf3b31f5..570a4b70 100644 --- a/generate/intent.py +++ b/generate/intent.py @@ -117,6 +117,16 @@ _DECLARATIVE_RELATION_RE = re.compile( r"(?P[a-z][a-z\-]*(?:\s+[a-z][a-z\-]*)?)\.?$", re.IGNORECASE, ) +# A leading correction cue ("Actually X R Y", "No, X R Y", "Incorrect, X R Y") +# turns an assertion into a correction. Without this guard the declarative- +# relation regex above swallows "Actually" into the subject phrase and routes +# the sentence to VERIFICATION, which prevents the inference-closure lane +# from emitting PackMutationProposal records for any non-`is` premise. See +# evals/inference_closure/gaps.md (resolved 2026-05-17 / regressed 2026-05-22). +_CORRECTION_CUE_PREFIX_RE = re.compile( + r"^\s*(?:actually|incorrect|correction|no,?\s+)", + re.IGNORECASE, +) # "How does X work / function / operate / happen / exist / behave?" # — third-person mechanistic-cause query. Distinct from PROCEDURE # (which is first-person: "How do I/we/you X?") because the user is @@ -408,7 +418,11 @@ def classify_intent(prompt: str) -> DialogueIntent: ), ) - declarative_match = _DECLARATIVE_RELATION_RE.match(text) + declarative_match = ( + None + if _CORRECTION_CUE_PREFIX_RE.match(text) + else _DECLARATIVE_RELATION_RE.match(text) + ) if declarative_match: raw_relation = declarative_match.group("relation").lower().strip() relation = _RELATION_NORMALIZE.get(raw_relation, raw_relation) diff --git a/tests/test_correction_cue_prefix_routing.py b/tests/test_correction_cue_prefix_routing.py new file mode 100644 index 00000000..a68d0a30 --- /dev/null +++ b/tests/test_correction_cue_prefix_routing.py @@ -0,0 +1,62 @@ +"""Regression test for "Actually X R Y" → CORRECTION routing. + +The declarative-relation match (``_DECLARATIVE_RELATION_RE``) runs ahead +of the ``_RULES`` loop and previously swallowed sentences beginning with +"Actually" into the subject phrase, routing them to VERIFICATION. That +prevented the inference-closure lane from emitting +``PackMutationProposal`` records for any non-`is` premise (the lane +documented this regression on 2026-05-22 in ``gaps.md``). + +The fix is the ``_CORRECTION_CUE_PREFIX_RE`` guard in +``generate/intent.py``: if the text starts with a correction cue token, +the declarative-match branch is skipped and the sentence falls through +to the ``_RULES`` loop where the CORRECTION rule fires. +""" + +from __future__ import annotations + +import pytest + +from generate.intent import IntentTag, classify_intent + + +@pytest.mark.parametrize( + "text,expected_subject", + [ + ("Actually wisdom is light.", "wisdom is light"), + ("Actually wisdom precedes recall.", "wisdom precedes recall"), + ("Actually truth grounds knowledge.", "truth grounds knowledge"), + ("Actually fire causes smoke.", "fire causes smoke"), + ("Actually X reveals Y.", "X reveals Y"), + ("Incorrect, wisdom precedes recall.", ", wisdom precedes recall"), + ("Correction: wisdom precedes recall.", ": wisdom precedes recall"), + ], +) +def test_correction_cue_prefix_routes_to_correction( + text: str, expected_subject: str +) -> None: + intent = classify_intent(text) + assert intent.tag is IntentTag.CORRECTION, ( + f"expected CORRECTION for {text!r}, got {intent.tag.value}" + ) + assert intent.relation is None + assert intent.subject == expected_subject + + +@pytest.mark.parametrize( + "text", + [ + "Wisdom precedes recall.", + "Truth grounds knowledge.", + "Fire causes smoke.", + ], +) +def test_bare_declarative_still_verification(text: str) -> None: + """The guard must only fire on a correction-cue prefix. + + Plain declarative-relation assertions must continue to route to + VERIFICATION so the relation/object slots are populated. + """ + intent = classify_intent(text) + assert intent.tag is IntentTag.VERIFICATION + assert intent.relation is not None