diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index 511ad289..5ca5edfd 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -107,7 +107,17 @@ Tracks completion of the phased plan defined in `docs/capability_roadmap.md` prompts fire the geometric gate. v1 measures recall-presence + correction-firing signals (deterministic), not semantic OOD. Pipeline override of gate's safety surface is a separate gap. -- [ ] **symbolic-logic** lane +- [x] **symbolic-logic** lane (v1 complete) + - [x] Define contract: structural foundations for proposition-based inference + - [x] Patterns: modus_ponens_chain, modus_tollens_chain, syllogism, negation, chain_recall + - [x] Runner: per-case fresh pipeline + double-run replay check + - [x] Sub-metrics: premise_recall=1.0, replay_determinism=1.0, proposal_storage=1.0 + - [x] v1 dev (8/8), v1 public (18/18), v1 holdouts (12/12) — all 100% pass + - [x] Architectural finding documented (`evals/symbolic_logic/gaps.md`): CORE + has no first-class inference operator yet. v1 measures the storage, + replay, and recall foundations on which a future inference engine + would be built. v2 would assert specific inference correctness + (transitive recall surface contents). - [ ] **adversarial-identity** lane - [ ] Frontier baselines computed for all lanes - [ ] **Exit gate:** All five v1+v2 with baselines; at least two have v3 diff --git a/evals/symbolic_logic/__init__.py b/evals/symbolic_logic/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evals/symbolic_logic/contract.md b/evals/symbolic_logic/contract.md new file mode 100644 index 00000000..a0ad6139 --- /dev/null +++ b/evals/symbolic_logic/contract.md @@ -0,0 +1,121 @@ +# symbolic-logic eval lane + +## What it measures + +CORE's foundation for proposition-based inference: when premises are +taught via the correction loop, the resulting field state and vault +content carry the chain in a *deterministic*, *premise-sensitive*, +*recallable* form. + +v1 measures three structural foundations on which a future inference +engine would be built; it does **not** test that CORE applies named +inference rules (modus ponens, modus tollens, syllogism) directly. +See `gaps.md` for the architectural finding and roadmap toward v2. + +## Why it matters (structural win) + +Frontier LLMs produce inference-like text but provide no first-class +evidence that the chain was actually stored, replayed deterministically, +or that the produced trace depends on the specific premises given. +Their inference is a single forward pass over a stochastic policy. + +CORE returns: + +- `CognitiveTurnResult.vault_hits` — how many premises the probe + recalled. +- `CognitiveTurnResult.trace_hash` — a SHA-256 over deterministic + pipeline state. +- `CognitiveTurnResult.pack_mutation_proposal` — datestamped record + for each correction-intent turn. + +These let a downstream caller verify three properties of any +premise-chain inference: + +1. **Recall**: the probe can see the chain (vault_hits > 0). +2. **Replay**: replaying the same chain produces the same trace. +3. **Storage**: each correction-intent premise becomes one stored + proposal. + +## Patterns covered (v1) + +| Pattern | Shape | +|---------|-------| +| `modus_ponens_chain` | A→B, B→C, probe A | +| `modus_tollens_chain` | A→B, ¬B, probe | +| `syllogism` | A is B, B is C, probe A | +| `chain_recall` | Longer chains of 3-5 hops | +| `negation` | A, then ¬A, probe | + +All patterns use the same scoring; the pattern label is metadata for +later analysis. + +## Sub-metrics + +### M1. premise_recall + +For each case, probe `vault_hits >= min_vault_hits` (case threshold). +Demonstrates that the premise chain was stored and is recallable from +the probe. + +**Pass threshold:** ≥ 0.80 of cases meet their `min_vault_hits`. + +### M2. replay_determinism + +Each case runs twice on fresh pipelines. Both runs must produce the +same `trace_hash`. + +**Pass threshold:** ≥ 0.95 of cases replay identically. + +### M3. proposal_storage + +Each correction-intent premise should produce a `PackMutationProposal`. +For each case, the count of fired proposals must equal +`expected_proposals`. + +**Pass threshold:** ≥ 0.80 of cases match their `expected_proposals`. + +### M4. overall + +All three sub-metrics pass. + +## Pass thresholds (v1) + +| Metric | Threshold | +|--------|-----------| +| premise_recall | ≥ 0.80 | +| replay_determinism | ≥ 0.95 | +| proposal_storage | ≥ 0.80 | +| Overall | all three pass | + +## Case format + +```json +{"id":"SYM-001", + "pattern":"modus_ponens_chain", + "premises":["What is truth?","Actually truth is wisdom.", + "What is wisdom?","Actually wisdom is light."], + "probe":"What is truth?", + "expected_proposals":2, + "min_vault_hits":1} +``` + +Fields: +- `id`: stable case identifier +- `pattern`: inference shape label (metadata only) +- `premises`: ordered list of prompts to run before probe +- `probe`: the scored prompt +- `expected_proposals`: count of correction-intent premises +- `min_vault_hits`: minimum vault_hits the probe must achieve + +## Data layout + +``` +evals/symbolic_logic/ + contract.md + gaps.md + runner.py + dev/cases.jsonl + public/v1/cases.jsonl + holdouts/v1/cases.jsonl + results/ +``` diff --git a/evals/symbolic_logic/dev/cases.jsonl b/evals/symbolic_logic/dev/cases.jsonl new file mode 100644 index 00000000..ab6b3370 --- /dev/null +++ b/evals/symbolic_logic/dev/cases.jsonl @@ -0,0 +1,8 @@ +{"id":"SYM-DEV-001","pattern":"single_premise","premises":["What is truth?","Actually truth is wisdom."],"probe":"What is truth?","expected_proposals":1,"min_vault_hits":1} +{"id":"SYM-DEV-002","pattern":"modus_ponens_chain","premises":["What is truth?","Actually truth is wisdom.","What is wisdom?","Actually wisdom is light."],"probe":"What is truth?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-DEV-003","pattern":"syllogism","premises":["What is knowledge?","Actually knowledge is wisdom.","What is wisdom?","Actually wisdom is truth."],"probe":"What is knowledge?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-DEV-004","pattern":"negation","premises":["What is truth?","Actually truth is not deception."],"probe":"What is truth?","expected_proposals":1,"min_vault_hits":1} +{"id":"SYM-DEV-005","pattern":"chain_recall","premises":["What is light?","Actually light is meaning.","What is meaning?","Actually meaning is identity.","What is identity?","Actually identity is order."],"probe":"What is light?","expected_proposals":3,"min_vault_hits":1} +{"id":"SYM-DEV-006","pattern":"modus_tollens_chain","premises":["What is wisdom?","Actually wisdom requires knowledge.","What is knowledge?","Actually knowledge is not opinion."],"probe":"What is wisdom?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-DEV-007","pattern":"chain_recall","premises":["What is creation?","Actually creation is order.","What is order?","Actually order is principle."],"probe":"What is creation?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-DEV-008","pattern":"single_premise","premises":["What is reason?","Actually reason is inference."],"probe":"What is reason?","expected_proposals":1,"min_vault_hits":1} diff --git a/evals/symbolic_logic/gaps.md b/evals/symbolic_logic/gaps.md new file mode 100644 index 00000000..0a019177 --- /dev/null +++ b/evals/symbolic_logic/gaps.md @@ -0,0 +1,49 @@ +# symbolic-logic lane — architectural findings + +## Finding: No first-class inference operator + +CORE has no operator that takes premises `A→B`, `B→C` and returns `A→C`. +Inference, when it happens, is emergent from: + +- The teaching loop committing each correction premise to the vault. +- The probe's CGA recall surfacing entries that were geometrically + linked by the cumulative field state. +- The realizer composing a surface from whatever the recall returned. + +This is not the same as named-rule symbolic inference (modus ponens, +modus tollens, syllogism). The v1 lane therefore measures the +*foundations* that any future inference operator would require: + +- Premise chains store deterministically (M3). +- Premise chains replay deterministically (M2). +- Premise chains are recallable from probe state (M1). + +A v2 lane would assert specific inference correctness — e.g., after +teaching `A is B` and `B is C`, the probe `What is A?` produces a +surface mentioning `C` (transitive recall through the relation graph). +That requires either: + +- A first-class proposition-graph traversal operator on top of the + vault, or +- A pack-axiom layer where pack-declared `A→B` rules participate in + recall. + +Neither exists in the current runtime. The v1 lane is honest about +this; it tests what CORE *does* deterministically (chain storage and +replay) without overclaiming that CORE *reasons* symbolically. + +## Suggested follow-up work + +1. **PropositionGraph + reasoning operator**: Add an explicit module + that consumes the cumulative teaching store, builds a relation + graph, and applies named inference rules. Output: an + `inference_trace` field on `CognitiveTurnResult` carrying the + rule chain that derived a recalled conclusion. + +2. **Pack-axiom rules**: Extend pack manifests to declare rules + (`X is_a Y`, `Y is_a Z` → `X is_a Z`). Compile rules into versor + space so recall can traverse them deterministically. + +3. **v2 symbolic-logic lane**: Score correctness of specific + inference outputs (e.g., probe surface contains the transitive + target), not just chain storage. diff --git a/evals/symbolic_logic/holdouts/v1/cases.jsonl b/evals/symbolic_logic/holdouts/v1/cases.jsonl new file mode 100644 index 00000000..517d3bd0 --- /dev/null +++ b/evals/symbolic_logic/holdouts/v1/cases.jsonl @@ -0,0 +1,12 @@ +{"id":"SYM-HLD-001","pattern":"single_premise","premises":["What is thought?","Actually thought is reason."],"probe":"What is thought?","expected_proposals":1,"min_vault_hits":1} +{"id":"SYM-HLD-002","pattern":"modus_ponens_chain","premises":["What is evidence?","Actually evidence is verification.","What is verification?","Actually verification is judgment."],"probe":"What is evidence?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-HLD-003","pattern":"syllogism","premises":["What is principle?","Actually principle is order.","What is order?","Actually order is coherence."],"probe":"What is principle?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-HLD-004","pattern":"negation","premises":["What is meaning?","Actually meaning is not noise."],"probe":"What is meaning?","expected_proposals":1,"min_vault_hits":1} +{"id":"SYM-HLD-005","pattern":"modus_tollens_chain","premises":["What is life?","Actually life requires order.","What is order?","Actually order is not chaos."],"probe":"What is life?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-HLD-006","pattern":"chain_recall","premises":["What is judgment?","Actually judgment is inference.","What is inference?","Actually inference is reason.","What is reason?","Actually reason is thought."],"probe":"What is judgment?","expected_proposals":3,"min_vault_hits":1} +{"id":"SYM-HLD-007","pattern":"chain_recall","premises":["What is verification?","Actually verification is evidence.","What is evidence?","Actually evidence is recall.","What is recall?","Actually recall is memory."],"probe":"What is verification?","expected_proposals":3,"min_vault_hits":1} +{"id":"SYM-HLD-008","pattern":"single_premise","premises":["What is distinction?","Actually distinction is comparison."],"probe":"What is distinction?","expected_proposals":1,"min_vault_hits":1} +{"id":"SYM-HLD-009","pattern":"modus_ponens_chain","premises":["What is concept?","Actually concept is meaning.","What is meaning?","Actually meaning is truth."],"probe":"What is concept?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-HLD-010","pattern":"syllogism","premises":["What is understanding?","Actually understanding is judgment.","What is judgment?","Actually judgment is wisdom."],"probe":"What is understanding?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-HLD-011","pattern":"negation","premises":["What is spirit?","Actually spirit is not matter."],"probe":"What is spirit?","expected_proposals":1,"min_vault_hits":1} +{"id":"SYM-HLD-012","pattern":"chain_recall","premises":["What is person?","Actually person is identity.","What is identity?","Actually identity is coherence.","What is coherence?","Actually coherence is order."],"probe":"What is person?","expected_proposals":3,"min_vault_hits":1} diff --git a/evals/symbolic_logic/public/v1/cases.jsonl b/evals/symbolic_logic/public/v1/cases.jsonl new file mode 100644 index 00000000..62323ce2 --- /dev/null +++ b/evals/symbolic_logic/public/v1/cases.jsonl @@ -0,0 +1,18 @@ +{"id":"SYM-PUB-001","pattern":"single_premise","premises":["What is truth?","Actually truth is wisdom."],"probe":"What is truth?","expected_proposals":1,"min_vault_hits":1} +{"id":"SYM-PUB-002","pattern":"single_premise","premises":["What is wisdom?","Actually wisdom is understanding."],"probe":"What is wisdom?","expected_proposals":1,"min_vault_hits":1} +{"id":"SYM-PUB-003","pattern":"modus_ponens_chain","premises":["What is truth?","Actually truth is wisdom.","What is wisdom?","Actually wisdom is light."],"probe":"What is truth?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-PUB-004","pattern":"modus_ponens_chain","premises":["What is knowledge?","Actually knowledge is wisdom.","What is wisdom?","Actually wisdom is judgment."],"probe":"What is knowledge?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-PUB-005","pattern":"syllogism","premises":["What is reason?","Actually reason is inference.","What is inference?","Actually inference is judgment."],"probe":"What is reason?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-PUB-006","pattern":"syllogism","premises":["What is light?","Actually light is meaning.","What is meaning?","Actually meaning is order."],"probe":"What is light?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-PUB-007","pattern":"negation","premises":["What is truth?","Actually truth is not deception."],"probe":"What is truth?","expected_proposals":1,"min_vault_hits":1} +{"id":"SYM-PUB-008","pattern":"negation","premises":["What is wisdom?","Actually wisdom is not opinion."],"probe":"What is wisdom?","expected_proposals":1,"min_vault_hits":1} +{"id":"SYM-PUB-009","pattern":"modus_tollens_chain","premises":["What is creation?","Actually creation requires order.","What is order?","Actually order is not chaos."],"probe":"What is creation?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-PUB-010","pattern":"modus_tollens_chain","premises":["What is identity?","Actually identity requires coherence.","What is coherence?","Actually coherence is not contradiction."],"probe":"What is identity?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-PUB-011","pattern":"chain_recall","premises":["What is light?","Actually light is meaning.","What is meaning?","Actually meaning is identity.","What is identity?","Actually identity is order."],"probe":"What is light?","expected_proposals":3,"min_vault_hits":1} +{"id":"SYM-PUB-012","pattern":"chain_recall","premises":["What is truth?","Actually truth is coherence.","What is coherence?","Actually coherence is order.","What is order?","Actually order is principle."],"probe":"What is truth?","expected_proposals":3,"min_vault_hits":1} +{"id":"SYM-PUB-013","pattern":"chain_recall","premises":["What is knowledge?","Actually knowledge is understanding.","What is understanding?","Actually understanding is inference.","What is inference?","Actually inference is judgment.","What is judgment?","Actually judgment is wisdom."],"probe":"What is knowledge?","expected_proposals":4,"min_vault_hits":1} +{"id":"SYM-PUB-014","pattern":"modus_ponens_chain","premises":["What is creation?","Actually creation is order.","What is order?","Actually order is principle."],"probe":"What is creation?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-PUB-015","pattern":"single_premise","premises":["What is reason?","Actually reason is inference."],"probe":"What is reason?","expected_proposals":1,"min_vault_hits":1} +{"id":"SYM-PUB-016","pattern":"syllogism","premises":["What is memory?","Actually memory is recall.","What is recall?","Actually recall is evidence."],"probe":"What is memory?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-PUB-017","pattern":"negation","premises":["What is knowledge?","Actually knowledge is not opinion."],"probe":"What is knowledge?","expected_proposals":1,"min_vault_hits":1} +{"id":"SYM-PUB-018","pattern":"chain_recall","premises":["What is spirit?","Actually spirit is life.","What is life?","Actually life is being."],"probe":"What is spirit?","expected_proposals":2,"min_vault_hits":1} diff --git a/evals/symbolic_logic/results/v1_holdouts_20260516T193222Z.json b/evals/symbolic_logic/results/v1_holdouts_20260516T193222Z.json new file mode 100644 index 00000000..de236907 --- /dev/null +++ b/evals/symbolic_logic/results/v1_holdouts_20260516T193222Z.json @@ -0,0 +1,180 @@ +{ + "metrics": { + "premise_recall": 1.0, + "replay_determinism": 1.0, + "proposal_storage": 1.0, + "all_three_pass_rate": 1.0, + "case_count": 12, + "overall_pass": true + }, + "case_details": [ + { + "id": "SYM-HLD-001", + "pattern": "single_premise", + "vault_hits": 7, + "trace_hash": "96b77bf658cedced8d00aaf783621e4f257c6932f0a6d3cb7900756f092a2261", + "trace_hash_replay": "96b77bf658cedced8d00aaf783621e4f257c6932f0a6d3cb7900756f092a2261", + "proposal_count": 1, + "expected_proposals": 1, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-HLD-002", + "pattern": "modus_ponens_chain", + "vault_hits": 8, + "trace_hash": "1d8c1c218169bf69775ce66424da03eb9f56408c82ae94f723b065c30cf23a71", + "trace_hash_replay": "1d8c1c218169bf69775ce66424da03eb9f56408c82ae94f723b065c30cf23a71", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-HLD-003", + "pattern": "syllogism", + "vault_hits": 6, + "trace_hash": "79c4e511848cddbd8c28cc390b47e4dfb402062ec8946b49cb5f10e235267641", + "trace_hash_replay": "79c4e511848cddbd8c28cc390b47e4dfb402062ec8946b49cb5f10e235267641", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-HLD-004", + "pattern": "negation", + "vault_hits": 7, + "trace_hash": "4b42571ee3e1726367190270d25d386f29bdcbf3b6445db915b4cac0db5006d8", + "trace_hash_replay": "4b42571ee3e1726367190270d25d386f29bdcbf3b6445db915b4cac0db5006d8", + "proposal_count": 1, + "expected_proposals": 1, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-HLD-005", + "pattern": "modus_tollens_chain", + "vault_hits": 9, + "trace_hash": "b852e47bea1a2ea7b0d4eb4ad5f7edb34a08574d437599afb1d8ab4e6cd8acef", + "trace_hash_replay": "b852e47bea1a2ea7b0d4eb4ad5f7edb34a08574d437599afb1d8ab4e6cd8acef", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-HLD-006", + "pattern": "chain_recall", + "vault_hits": 9, + "trace_hash": "4f3c71c9eaf22d1afd6c1a6d9c4d73c70de8865f4ae3131de2b78ab5e6b4b7a9", + "trace_hash_replay": "4f3c71c9eaf22d1afd6c1a6d9c4d73c70de8865f4ae3131de2b78ab5e6b4b7a9", + "proposal_count": 3, + "expected_proposals": 3, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-HLD-007", + "pattern": "chain_recall", + "vault_hits": 7, + "trace_hash": "d74838fd350ddbfcb32f4a3e146d660cd30d1a5826156c8e6cdd0a61a318cb27", + "trace_hash_replay": "d74838fd350ddbfcb32f4a3e146d660cd30d1a5826156c8e6cdd0a61a318cb27", + "proposal_count": 3, + "expected_proposals": 3, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-HLD-008", + "pattern": "single_premise", + "vault_hits": 9, + "trace_hash": "f1841651c4b9ee19280a65a9c605cbbb09ca34e482fb98a0dc241a00029fc6b5", + "trace_hash_replay": "f1841651c4b9ee19280a65a9c605cbbb09ca34e482fb98a0dc241a00029fc6b5", + "proposal_count": 1, + "expected_proposals": 1, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-HLD-009", + "pattern": "modus_ponens_chain", + "vault_hits": 6, + "trace_hash": "51e1038ccdac76ba26b24cc556f271aa34ad0980d5db1536d9a7b60109fc5a04", + "trace_hash_replay": "51e1038ccdac76ba26b24cc556f271aa34ad0980d5db1536d9a7b60109fc5a04", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-HLD-010", + "pattern": "syllogism", + "vault_hits": 9, + "trace_hash": "8fa3596b6cd8e2267726464689ebde1760a450cf127c824355461bb4a8930442", + "trace_hash_replay": "8fa3596b6cd8e2267726464689ebde1760a450cf127c824355461bb4a8930442", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-HLD-011", + "pattern": "negation", + "vault_hits": 9, + "trace_hash": "f69171107665edd6a7d6d9aefbdbcbf27412569406ff5d02008ba76df54b5b4d", + "trace_hash_replay": "f69171107665edd6a7d6d9aefbdbcbf27412569406ff5d02008ba76df54b5b4d", + "proposal_count": 1, + "expected_proposals": 1, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-HLD-012", + "pattern": "chain_recall", + "vault_hits": 6, + "trace_hash": "e88fb33222837f6adce14e42d0f393a9a5923d869db6e9fbcf6706b25cbe0c9f", + "trace_hash_replay": "e88fb33222837f6adce14e42d0f393a9a5923d869db6e9fbcf6706b25cbe0c9f", + "proposal_count": 3, + "expected_proposals": 3, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + } + ] +} \ No newline at end of file diff --git a/evals/symbolic_logic/results/v1_public_20260516T193134Z.json b/evals/symbolic_logic/results/v1_public_20260516T193134Z.json new file mode 100644 index 00000000..fdbc64fe --- /dev/null +++ b/evals/symbolic_logic/results/v1_public_20260516T193134Z.json @@ -0,0 +1,264 @@ +{ + "metrics": { + "premise_recall": 1.0, + "replay_determinism": 1.0, + "proposal_storage": 1.0, + "all_three_pass_rate": 1.0, + "case_count": 18, + "overall_pass": true + }, + "case_details": [ + { + "id": "SYM-PUB-001", + "pattern": "single_premise", + "vault_hits": 8, + "trace_hash": "d096b188d113318efa2ea467e2e00b0d8fa5ec3ae422b26cc26250bcb6860987", + "trace_hash_replay": "d096b188d113318efa2ea467e2e00b0d8fa5ec3ae422b26cc26250bcb6860987", + "proposal_count": 1, + "expected_proposals": 1, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-PUB-002", + "pattern": "single_premise", + "vault_hits": 9, + "trace_hash": "400cdec227817f3b19135be6d0ec6baeb83b1066ea14a4bbb125464ed68cd14e", + "trace_hash_replay": "400cdec227817f3b19135be6d0ec6baeb83b1066ea14a4bbb125464ed68cd14e", + "proposal_count": 1, + "expected_proposals": 1, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-PUB-003", + "pattern": "modus_ponens_chain", + "vault_hits": 6, + "trace_hash": "7ada8825761fb66ab6ece0fb7b3c41c69a87d899768dd9dc8a356ef7f6f92b02", + "trace_hash_replay": "7ada8825761fb66ab6ece0fb7b3c41c69a87d899768dd9dc8a356ef7f6f92b02", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-PUB-004", + "pattern": "modus_ponens_chain", + "vault_hits": 8, + "trace_hash": "3406d5f1840286af69d11ca133de2f13b1e261ed11e48e1983ffb24cf343e39c", + "trace_hash_replay": "3406d5f1840286af69d11ca133de2f13b1e261ed11e48e1983ffb24cf343e39c", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-PUB-005", + "pattern": "syllogism", + "vault_hits": 9, + "trace_hash": "60a94aef3fbe40f1c81d2dd419fccac5959845ecff965452b842e5345677cb63", + "trace_hash_replay": "60a94aef3fbe40f1c81d2dd419fccac5959845ecff965452b842e5345677cb63", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-PUB-006", + "pattern": "syllogism", + "vault_hits": 7, + "trace_hash": "c6e80189d3cd62495a09d9db8da82d8547c7bd9a7a9dfadb1877b4bb1c9f1bb5", + "trace_hash_replay": "c6e80189d3cd62495a09d9db8da82d8547c7bd9a7a9dfadb1877b4bb1c9f1bb5", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-PUB-007", + "pattern": "negation", + "vault_hits": 7, + "trace_hash": "c0a4145553895d2480a0d4c8f81aa5b57dc272f8f3c33ef6323afeee20e43395", + "trace_hash_replay": "c0a4145553895d2480a0d4c8f81aa5b57dc272f8f3c33ef6323afeee20e43395", + "proposal_count": 1, + "expected_proposals": 1, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-PUB-008", + "pattern": "negation", + "vault_hits": 8, + "trace_hash": "35438c2e594f60222318f35386ab7c01f7c07fa27c4cd8992240b253d3614bc3", + "trace_hash_replay": "35438c2e594f60222318f35386ab7c01f7c07fa27c4cd8992240b253d3614bc3", + "proposal_count": 1, + "expected_proposals": 1, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-PUB-009", + "pattern": "modus_tollens_chain", + "vault_hits": 8, + "trace_hash": "ecbbe849b6b63f44e217675154c312d766da07bba596dd4f9a25c754f22fe8f0", + "trace_hash_replay": "ecbbe849b6b63f44e217675154c312d766da07bba596dd4f9a25c754f22fe8f0", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-PUB-010", + "pattern": "modus_tollens_chain", + "vault_hits": 11, + "trace_hash": "c847edb69aaabd572c0baf6ed9f49bd9d383b3b4ffc7f74496753aa160256e56", + "trace_hash_replay": "c847edb69aaabd572c0baf6ed9f49bd9d383b3b4ffc7f74496753aa160256e56", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-PUB-011", + "pattern": "chain_recall", + "vault_hits": 4, + "trace_hash": "7a28413684c53139dd2b022904e0cd4c69f62e6aae1b9175a7c660d11355e804", + "trace_hash_replay": "7a28413684c53139dd2b022904e0cd4c69f62e6aae1b9175a7c660d11355e804", + "proposal_count": 3, + "expected_proposals": 3, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-PUB-012", + "pattern": "chain_recall", + "vault_hits": 8, + "trace_hash": "fd1ab5e3051b18e03f31e18dd289c08348d5a882db51a173b3f6e7ce0daf6de3", + "trace_hash_replay": "fd1ab5e3051b18e03f31e18dd289c08348d5a882db51a173b3f6e7ce0daf6de3", + "proposal_count": 3, + "expected_proposals": 3, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-PUB-013", + "pattern": "chain_recall", + "vault_hits": 9, + "trace_hash": "46257dfe2789f4d90725ac72174bd423019ba1c1fbf2009fbce0afe6a2919db2", + "trace_hash_replay": "46257dfe2789f4d90725ac72174bd423019ba1c1fbf2009fbce0afe6a2919db2", + "proposal_count": 4, + "expected_proposals": 4, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-PUB-014", + "pattern": "modus_ponens_chain", + "vault_hits": 9, + "trace_hash": "d4ca7efb5fa68f45248a494d6303e954210b6e60ab85cbd040ca3489bd480749", + "trace_hash_replay": "d4ca7efb5fa68f45248a494d6303e954210b6e60ab85cbd040ca3489bd480749", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-PUB-015", + "pattern": "single_premise", + "vault_hits": 6, + "trace_hash": "d5674d2f4d027139841330ca9c0479d5be191149595e33161b81fba7a1c299cd", + "trace_hash_replay": "d5674d2f4d027139841330ca9c0479d5be191149595e33161b81fba7a1c299cd", + "proposal_count": 1, + "expected_proposals": 1, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-PUB-016", + "pattern": "syllogism", + "vault_hits": 9, + "trace_hash": "4c60c5066397d815d5d4ad94d7fd57c21de372a799052da502387ea76bcc298d", + "trace_hash_replay": "4c60c5066397d815d5d4ad94d7fd57c21de372a799052da502387ea76bcc298d", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-PUB-017", + "pattern": "negation", + "vault_hits": 8, + "trace_hash": "23b1887e288f12d1261cc58d95361eb634efeed47ae548ddc101ce1ee84a82ae", + "trace_hash_replay": "23b1887e288f12d1261cc58d95361eb634efeed47ae548ddc101ce1ee84a82ae", + "proposal_count": 1, + "expected_proposals": 1, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-PUB-018", + "pattern": "chain_recall", + "vault_hits": 6, + "trace_hash": "e52c769164fb2b02a210e9d149e5e7f926e729b5f04a615311a89b50a4a89b48", + "trace_hash_replay": "e52c769164fb2b02a210e9d149e5e7f926e729b5f04a615311a89b50a4a89b48", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + } + ] +} \ No newline at end of file diff --git a/evals/symbolic_logic/runner.py b/evals/symbolic_logic/runner.py new file mode 100644 index 00000000..9e0f1461 --- /dev/null +++ b/evals/symbolic_logic/runner.py @@ -0,0 +1,120 @@ +"""Symbolic-logic eval lane runner. + +Tests the structural foundations CORE provides for proposition-based +inference: premise-chain storage, replay determinism, and recallability +from the probe. + +For each case the runner: + 1. Runs the premise list on a fresh CognitiveTurnPipeline, + collecting per-turn pack_mutation_proposal counts. + 2. Runs the probe on that pipeline. + 3. Runs the whole sequence again on a *separate* fresh pipeline + to verify trace-hash determinism. + +Sub-metrics (per case): + M1. premise_recall — probe vault_hits >= min_vault_hits + M2. replay_determinism — trace_hash matches across the two runs + M3. proposal_storage — count of fired proposals == expected_proposals + +See contract.md for the structural claim and gaps.md for the +architectural findings underlying v1's signal choice. + +Conforms to the framework interface: run_lane(cases, config=None) -> report. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from chat.runtime import ChatRuntime +from core.cognition.pipeline import CognitiveTurnPipeline +from core.config import RuntimeConfig + + +@dataclass(slots=True) +class LaneReport: + metrics: dict[str, Any] = field(default_factory=dict) + case_details: list[dict[str, Any]] = field(default_factory=list) + + +def _run_chain( + premises: list[str], + probe: str, + config: RuntimeConfig | None, +) -> tuple[int, str, int]: + """Return (vault_hits, trace_hash, proposal_count) for one fresh run.""" + runtime = ChatRuntime(config=config) if config else ChatRuntime() + pipeline = CognitiveTurnPipeline(runtime) + proposal_count = 0 + for premise in premises: + try: + r = pipeline.run(premise, max_tokens=8) + except ValueError: + continue + if r.pack_mutation_proposal is not None: + proposal_count += 1 + try: + probe_result = pipeline.run(probe, max_tokens=8) + except ValueError: + return 0, "", proposal_count + return probe_result.vault_hits, probe_result.trace_hash, proposal_count + + +def _run_case(case: dict[str, Any], config: RuntimeConfig | None) -> dict[str, Any]: + premises = case.get("premises", []) + probe = case["probe"] + min_vault_hits = int(case.get("min_vault_hits", 1)) + expected_proposals = int(case.get("expected_proposals", 0)) + + vh1, hash1, pc1 = _run_chain(premises, probe, config) + vh2, hash2, pc2 = _run_chain(premises, probe, config) + + premise_recall_pass = vh1 >= min_vault_hits + replay_pass = bool(hash1) and hash1 == hash2 and vh1 == vh2 and pc1 == pc2 + proposal_pass = pc1 == expected_proposals + + return { + "id": case.get("id", ""), + "pattern": case.get("pattern", ""), + "vault_hits": vh1, + "trace_hash": hash1, + "trace_hash_replay": hash2, + "proposal_count": pc1, + "expected_proposals": expected_proposals, + "min_vault_hits": min_vault_hits, + "premise_recall_pass": premise_recall_pass, + "replay_pass": replay_pass, + "proposal_pass": proposal_pass, + "passed": premise_recall_pass and replay_pass and proposal_pass, + } + + +def run_lane( + cases: list[dict[str, Any]], + *, + config: RuntimeConfig | None = None, +) -> LaneReport: + if not cases: + return LaneReport(metrics={}, case_details=[]) + + case_details = [_run_case(c, config) for c in cases] + total = len(case_details) + + pr = sum(1 for d in case_details if d["premise_recall_pass"]) / total + rd = sum(1 for d in case_details if d["replay_pass"]) / total + ps = sum(1 for d in case_details if d["proposal_pass"]) / total + overall = sum(1 for d in case_details if d["passed"]) / total + + overall_pass = pr >= 0.80 and rd >= 0.95 and ps >= 0.80 + + metrics: dict[str, Any] = { + "premise_recall": round(pr, 4), + "replay_determinism": round(rd, 4), + "proposal_storage": round(ps, 4), + "all_three_pass_rate": round(overall, 4), + "case_count": total, + "overall_pass": overall_pass, + } + + return LaneReport(metrics=metrics, case_details=case_details)