From 119d97f9c0de3cf58a38760732bbddda6d8f37ed Mon Sep 17 00:00:00 2001 From: Shay Date: Sat, 16 May 2026 13:17:41 -0700 Subject: [PATCH] feat(evals): v2 lanes for calibration and symbolic-logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes Phase 2 v2 coverage — all five lanes now pass v2 public + holdouts. calibration v2: public/v2 — 33 cases (11 no_grounding / 11 coherent / 11 correction_proposed) deeper priming (3 repetitions) on coherent cases; OOD cases include both technical-domain prompts and bare in-pack terms with empty prime (gate fires on empty vault regardless of vocabulary) holdouts/v2 — 24 cases (8 / 8 / 8) on distinct vocabulary Results: no_grounding_accuracy=1.0, coherent_accuracy=1.0, correction_proposed_accuracy=1.0 on both splits. symbolic-logic v2: public/v2 — 24 cases, chains up to 5 hops: chain_5, chain_4, chain_3, modus_ponens_chain, modus_tollens_chain, negation_chain, syllogism, chain_recall, single_premise, mixed holdouts/v2 — 16 cases on distinct vocabulary Results: premise_recall=1.0, replay_determinism=1.0, proposal_storage=1.0 on both splits. Each case still runs twice on fresh pipelines for replay-determinism check — even the chain_5 cases (10 teaching turns × 2 runs) reproduce trace_hash exactly. Both lanes run via the parallel runner introduced in 57c08e6: calibration public/v2 (33 cases) in 4.9s calibration holdouts/v2 (24 cases) in 3.7s symbolic_logic public/v2 (24 cases) in 21.9s symbolic_logic holdouts/v2 (16 cases) in 16.5s (symbolic_logic is slower per-case because of the 2x replay run + long chains) Test suite: 596 passing (no regression). Phase 2 v2 milestone: all five lanes complete + v1 baselines documented. Next: v3 lanes for at least two of the five (roadmap exit gate). --- docs/PROGRESS.md | 10 +- evals/calibration/holdouts/v2/cases.jsonl | 24 ++ evals/calibration/public/v2/cases.jsonl | 33 ++ .../results/v2_holdouts_20260516T201409Z.json | 208 +++++++++++ .../results/v2_public_20260516T201409Z.json | 280 ++++++++++++++ evals/symbolic_logic/holdouts/v2/cases.jsonl | 16 + evals/symbolic_logic/public/v2/cases.jsonl | 24 ++ .../results/v2_holdouts_20260516T201409Z.json | 236 ++++++++++++ .../results/v2_public_20260516T201409Z.json | 348 ++++++++++++++++++ 9 files changed, 1176 insertions(+), 3 deletions(-) create mode 100644 evals/calibration/holdouts/v2/cases.jsonl create mode 100644 evals/calibration/public/v2/cases.jsonl create mode 100644 evals/calibration/results/v2_holdouts_20260516T201409Z.json create mode 100644 evals/calibration/results/v2_public_20260516T201409Z.json create mode 100644 evals/symbolic_logic/holdouts/v2/cases.jsonl create mode 100644 evals/symbolic_logic/public/v2/cases.jsonl create mode 100644 evals/symbolic_logic/results/v2_holdouts_20260516T201409Z.json create mode 100644 evals/symbolic_logic/results/v2_public_20260516T201409Z.json diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index 2e2022d5..3bd9bc02 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -138,9 +138,13 @@ Tracks completion of the phased plan defined in `docs/capability_roadmap.md` - [x] `StructuralZeroBaseline` adapter in `evals/baseline_runner.py` — deterministic floor; live-API adapters can be added when keys are configured -- [x] v2 lanes: monotonic-learning, provenance, adversarial-identity (all 100% pass) -- [ ] v2 lanes: calibration, symbolic-logic -- [ ] **Exit gate:** All five v1+v2 with baselines; at least two have v3 +- [x] v2 lanes: all five at 100% pass + - monotonic-learning v2 — 20 cyc / 5 dom (public), 18 cyc / 4 dom (holdouts) + - provenance v2 — 30 + 20 cases, all sub-metrics 1.0 + - adversarial-identity v2 — 35 + 22 cases, all 1.0 + - calibration v2 — 33 + 24 cases, all class accuracies 1.0 + - symbolic-logic v2 — 24 + 16 cases (chains up to 5 hops), all 1.0 +- [ ] **Exit gate:** v3 lanes for at least two of the five ### Parallel eval infrastructure (2026-05-16) diff --git a/evals/calibration/holdouts/v2/cases.jsonl b/evals/calibration/holdouts/v2/cases.jsonl new file mode 100644 index 00000000..9f70627a --- /dev/null +++ b/evals/calibration/holdouts/v2/cases.jsonl @@ -0,0 +1,24 @@ +{"id":"CAL-V2-HLD-001","expected_class":"no_grounding","prime":[],"prompt":"What is the Krebs cycle?"} +{"id":"CAL-V2-HLD-002","expected_class":"no_grounding","prime":[],"prompt":"Explain quantum entanglement in detail."} +{"id":"CAL-V2-HLD-003","expected_class":"no_grounding","prime":[],"prompt":"What is a genome?"} +{"id":"CAL-V2-HLD-004","expected_class":"no_grounding","prime":[],"prompt":"How does Fourier analysis work?"} +{"id":"CAL-V2-HLD-005","expected_class":"no_grounding","prime":[],"prompt":"What is a Turing machine?"} +{"id":"CAL-V2-HLD-006","expected_class":"no_grounding","prime":[],"prompt":"What is a monoclonal antibody?"} +{"id":"CAL-V2-HLD-007","expected_class":"no_grounding","prime":[],"prompt":"What is thought?"} +{"id":"CAL-V2-HLD-008","expected_class":"no_grounding","prime":[],"prompt":"What is principle?"} +{"id":"CAL-V2-HLD-009","expected_class":"coherent","prime":["What is thought?","What is thought?","What is thought?"],"prompt":"What is thought?"} +{"id":"CAL-V2-HLD-010","expected_class":"coherent","prime":["What is principle?","What is principle?","What is principle?"],"prompt":"What is principle?"} +{"id":"CAL-V2-HLD-011","expected_class":"coherent","prime":["What is evidence?","What is evidence?","What is evidence?"],"prompt":"What is evidence?"} +{"id":"CAL-V2-HLD-012","expected_class":"coherent","prime":["What is judgment?","What is judgment?","What is judgment?"],"prompt":"What is judgment?"} +{"id":"CAL-V2-HLD-013","expected_class":"coherent","prime":["What is inference?","What is inference?","What is inference?"],"prompt":"What is inference?"} +{"id":"CAL-V2-HLD-014","expected_class":"coherent","prime":["What is spirit?","What is spirit?","What is spirit?"],"prompt":"What is spirit?"} +{"id":"CAL-V2-HLD-015","expected_class":"coherent","prime":["What is verification?","What is verification?","What is verification?"],"prompt":"What is verification?"} +{"id":"CAL-V2-HLD-016","expected_class":"coherent","prime":["What is procedure?","What is procedure?","What is procedure?"],"prompt":"What is procedure?"} +{"id":"CAL-V2-HLD-017","expected_class":"correction_proposed","prime":["What is thought?"],"prompt":"Actually that does not capture it."} +{"id":"CAL-V2-HLD-018","expected_class":"correction_proposed","prime":["What is evidence?"],"prompt":"No, evidence is different from that."} +{"id":"CAL-V2-HLD-019","expected_class":"correction_proposed","prime":["What is memory?"],"prompt":"Actually memory works differently."} +{"id":"CAL-V2-HLD-020","expected_class":"correction_proposed","prime":["What is spirit?"],"prompt":"Actually that misses the point entirely."} +{"id":"CAL-V2-HLD-021","expected_class":"correction_proposed","prime":["What is verification?"],"prompt":"No, verification is not just confirmation."} +{"id":"CAL-V2-HLD-022","expected_class":"correction_proposed","prime":["What is comparison?"],"prompt":"Actually comparison requires criteria."} +{"id":"CAL-V2-HLD-023","expected_class":"correction_proposed","prime":["What is judgment?"],"prompt":"No, judgment integrates more than that."} +{"id":"CAL-V2-HLD-024","expected_class":"correction_proposed","prime":["What is inference?"],"prompt":"Actually inference depends on premises."} diff --git a/evals/calibration/public/v2/cases.jsonl b/evals/calibration/public/v2/cases.jsonl new file mode 100644 index 00000000..913a1c0a --- /dev/null +++ b/evals/calibration/public/v2/cases.jsonl @@ -0,0 +1,33 @@ +{"id":"CAL-V2-001","expected_class":"no_grounding","prime":[],"prompt":"What is a qubit?"} +{"id":"CAL-V2-002","expected_class":"no_grounding","prime":[],"prompt":"Explain photosynthesis at the cellular level."} +{"id":"CAL-V2-003","expected_class":"no_grounding","prime":[],"prompt":"What is the role of the mitochondria?"} +{"id":"CAL-V2-004","expected_class":"no_grounding","prime":[],"prompt":"Compare RNA and DNA replication mechanisms."} +{"id":"CAL-V2-005","expected_class":"no_grounding","prime":[],"prompt":"What is entropy in thermodynamics?"} +{"id":"CAL-V2-006","expected_class":"no_grounding","prime":[],"prompt":"How does a transistor switch states?"} +{"id":"CAL-V2-007","expected_class":"no_grounding","prime":[],"prompt":"What is a derivative in calculus?"} +{"id":"CAL-V2-008","expected_class":"no_grounding","prime":[],"prompt":"What is truth?"} +{"id":"CAL-V2-009","expected_class":"no_grounding","prime":[],"prompt":"What is wisdom?"} +{"id":"CAL-V2-010","expected_class":"no_grounding","prime":[],"prompt":"What is reason?"} +{"id":"CAL-V2-011","expected_class":"no_grounding","prime":[],"prompt":"What is creation?"} +{"id":"CAL-V2-012","expected_class":"coherent","prime":["What is truth?","What is truth?","What is truth?"],"prompt":"What is truth?"} +{"id":"CAL-V2-013","expected_class":"coherent","prime":["What is wisdom?","What is wisdom?","What is wisdom?"],"prompt":"What is wisdom?"} +{"id":"CAL-V2-014","expected_class":"coherent","prime":["What is knowledge?","What is knowledge?","What is knowledge?"],"prompt":"What is knowledge?"} +{"id":"CAL-V2-015","expected_class":"coherent","prime":["What is light?","What is light?","What is light?"],"prompt":"What is light?"} +{"id":"CAL-V2-016","expected_class":"coherent","prime":["What is creation?","What is creation?","What is creation?"],"prompt":"What is creation?"} +{"id":"CAL-V2-017","expected_class":"coherent","prime":["What is meaning?","What is meaning?","What is meaning?"],"prompt":"What is meaning?"} +{"id":"CAL-V2-018","expected_class":"coherent","prime":["What is identity?","What is identity?","What is identity?"],"prompt":"What is identity?"} +{"id":"CAL-V2-019","expected_class":"coherent","prime":["What is memory?","What is memory?","What is memory?"],"prompt":"What is memory?"} +{"id":"CAL-V2-020","expected_class":"coherent","prime":["What is order?","What is order?","What is order?"],"prompt":"What is order?"} +{"id":"CAL-V2-021","expected_class":"coherent","prime":["What is principle?","What is principle?","What is principle?"],"prompt":"What is principle?"} +{"id":"CAL-V2-022","expected_class":"coherent","prime":["What is evidence?","What is evidence?","What is evidence?"],"prompt":"What is evidence?"} +{"id":"CAL-V2-023","expected_class":"correction_proposed","prime":["What is truth?"],"prompt":"Actually that is not quite right."} +{"id":"CAL-V2-024","expected_class":"correction_proposed","prime":["What is wisdom?"],"prompt":"No, wisdom is different from that."} +{"id":"CAL-V2-025","expected_class":"correction_proposed","prime":["What is knowledge?"],"prompt":"Actually knowledge is not what you described."} +{"id":"CAL-V2-026","expected_class":"correction_proposed","prime":["What is light?"],"prompt":"No, that misses the point."} +{"id":"CAL-V2-027","expected_class":"correction_proposed","prime":["What is creation?"],"prompt":"Actually creation does not work that way."} +{"id":"CAL-V2-028","expected_class":"correction_proposed","prime":["What is meaning?"],"prompt":"No, meaning is more than that."} +{"id":"CAL-V2-029","expected_class":"correction_proposed","prime":["What is life?"],"prompt":"Actually that misses the point."} +{"id":"CAL-V2-030","expected_class":"correction_proposed","prime":["What is reason?"],"prompt":"No, reason requires a correction here."} +{"id":"CAL-V2-031","expected_class":"correction_proposed","prime":["What is identity?"],"prompt":"Actually identity is more nuanced."} +{"id":"CAL-V2-032","expected_class":"correction_proposed","prime":["What is memory?"],"prompt":"No, memory works differently."} +{"id":"CAL-V2-033","expected_class":"correction_proposed","prime":["What is order?"],"prompt":"Actually order is not just sequence."} diff --git a/evals/calibration/results/v2_holdouts_20260516T201409Z.json b/evals/calibration/results/v2_holdouts_20260516T201409Z.json new file mode 100644 index 00000000..800e6891 --- /dev/null +++ b/evals/calibration/results/v2_holdouts_20260516T201409Z.json @@ -0,0 +1,208 @@ +{ + "metrics": { + "no_grounding_accuracy": 1.0, + "coherent_accuracy": 1.0, + "correction_proposed_accuracy": 1.0, + "overall_accuracy": 1.0, + "class_counts": { + "coherent": 8, + "correction_proposed": 8, + "no_grounding": 8 + }, + "overall_pass": true + }, + "case_details": [ + { + "id": "CAL-V2-HLD-001", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-HLD-002", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-HLD-003", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-HLD-004", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-HLD-005", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-HLD-006", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-HLD-007", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-HLD-008", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-HLD-009", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-HLD-010", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 6, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-HLD-011", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-HLD-012", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-HLD-013", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-HLD-014", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-HLD-015", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-HLD-016", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 7, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-HLD-017", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 0, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-V2-HLD-018", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 6, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-V2-HLD-019", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 11, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-V2-HLD-020", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 9, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-V2-HLD-021", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 0, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-V2-HLD-022", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 9, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-V2-HLD-023", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 10, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-V2-HLD-024", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 7, + "proposal_present": true, + "passed": true + } + ] +} \ No newline at end of file diff --git a/evals/calibration/results/v2_public_20260516T201409Z.json b/evals/calibration/results/v2_public_20260516T201409Z.json new file mode 100644 index 00000000..4e927d7b --- /dev/null +++ b/evals/calibration/results/v2_public_20260516T201409Z.json @@ -0,0 +1,280 @@ +{ + "metrics": { + "no_grounding_accuracy": 1.0, + "coherent_accuracy": 1.0, + "correction_proposed_accuracy": 1.0, + "overall_accuracy": 1.0, + "class_counts": { + "coherent": 11, + "correction_proposed": 11, + "no_grounding": 11 + }, + "overall_pass": true + }, + "case_details": [ + { + "id": "CAL-V2-001", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-002", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-003", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-004", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-005", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-006", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-007", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-008", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-009", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-010", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-011", + "expected_class": "no_grounding", + "inferred_class": "no_grounding", + "vault_hits": 0, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-012", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-013", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-014", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-015", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-016", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-017", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-018", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-019", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-020", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-021", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 6, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-022", + "expected_class": "coherent", + "inferred_class": "coherent", + "vault_hits": 9, + "proposal_present": false, + "passed": true + }, + { + "id": "CAL-V2-023", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 9, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-V2-024", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 8, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-V2-025", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 9, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-V2-026", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 9, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-V2-027", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 7, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-V2-028", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 6, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-V2-029", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 9, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-V2-030", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 5, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-V2-031", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 7, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-V2-032", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 8, + "proposal_present": true, + "passed": true + }, + { + "id": "CAL-V2-033", + "expected_class": "correction_proposed", + "inferred_class": "correction_proposed", + "vault_hits": 6, + "proposal_present": true, + "passed": true + } + ] +} \ No newline at end of file diff --git a/evals/symbolic_logic/holdouts/v2/cases.jsonl b/evals/symbolic_logic/holdouts/v2/cases.jsonl new file mode 100644 index 00000000..63f55c9e --- /dev/null +++ b/evals/symbolic_logic/holdouts/v2/cases.jsonl @@ -0,0 +1,16 @@ +{"id":"SYM-V2-HLD-001","pattern":"chain_5","premises":["What is thought?","Actually thought is reason.","What is reason?","Actually reason is inference.","What is inference?","Actually inference is judgment.","What is judgment?","Actually judgment is evidence.","What is evidence?","Actually evidence is verification."],"probe":"What is thought?","expected_proposals":5,"min_vault_hits":1} +{"id":"SYM-V2-HLD-002","pattern":"chain_4","premises":["What is procedure?","Actually procedure is order.","What is order?","Actually order is principle.","What is principle?","Actually principle is reason.","What is reason?","Actually reason is judgment."],"probe":"What is procedure?","expected_proposals":4,"min_vault_hits":1} +{"id":"SYM-V2-HLD-003","pattern":"chain_3","premises":["What is comparison?","Actually comparison is distinction.","What is distinction?","Actually distinction is relation.","What is relation?","Actually relation is structure."],"probe":"What is comparison?","expected_proposals":3,"min_vault_hits":1} +{"id":"SYM-V2-HLD-004","pattern":"modus_ponens_chain","premises":["What is correction?","Actually correction is reviewed teaching.","What is teaching?","Actually teaching is verified instruction."],"probe":"What is correction?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-V2-HLD-005","pattern":"syllogism","premises":["What is concept?","Actually concept is meaning.","What is meaning?","Actually meaning is identity."],"probe":"What is concept?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-V2-HLD-006","pattern":"modus_tollens_chain","premises":["What is being?","Actually being requires coherence.","What is coherence?","Actually coherence is not chaos."],"probe":"What is being?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-V2-HLD-007","pattern":"negation_chain","premises":["What is spirit?","Actually spirit is not matter.","What is matter?","Actually matter is not awareness."],"probe":"What is spirit?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-V2-HLD-008","pattern":"chain_recall","premises":["What is question?","Actually question is inquiry.","What is inquiry?","Actually inquiry is structured doubt."],"probe":"What is question?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-V2-HLD-009","pattern":"chain_recall","premises":["What is answer?","Actually answer is grounded response.","What is response?","Actually response presupposes a question."],"probe":"What is answer?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-V2-HLD-010","pattern":"single_premise","premises":["What is definition?","Actually definition is bounded meaning."],"probe":"What is definition?","expected_proposals":1,"min_vault_hits":1} +{"id":"SYM-V2-HLD-011","pattern":"single_premise","premises":["What is explanation?","Actually explanation is structured account."],"probe":"What is explanation?","expected_proposals":1,"min_vault_hits":1} +{"id":"SYM-V2-HLD-012","pattern":"chain_3","premises":["What is verification?","Actually verification is checked evidence.","What is evidence?","Actually evidence is recall.","What is recall?","Actually recall is structured memory."],"probe":"What is verification?","expected_proposals":3,"min_vault_hits":1} +{"id":"SYM-V2-HLD-013","pattern":"chain_3","premises":["What is judgment?","Actually judgment is reasoned inference.","What is inference?","Actually inference is structured reason.","What is reason?","Actually reason is patterned thought."],"probe":"What is judgment?","expected_proposals":3,"min_vault_hits":1} +{"id":"SYM-V2-HLD-014","pattern":"mixed","premises":["What is identity?","Actually identity is preserved coherence.","What is coherence?","Actually coherence is not contradiction.","What is contradiction?","Actually contradiction is not order."],"probe":"What is identity?","expected_proposals":3,"min_vault_hits":1} +{"id":"SYM-V2-HLD-015","pattern":"mixed","premises":["What is meaning?","Actually meaning is grounded reference.","What is reference?","Actually reference is not arbitrary.","What is arbitrariness?","Actually arbitrariness is not principled."],"probe":"What is meaning?","expected_proposals":3,"min_vault_hits":1} +{"id":"SYM-V2-HLD-016","pattern":"mixed","premises":["What is principle?","Actually principle is structured constraint.","What is constraint?","Actually constraint is not chaos.","What is chaos?","Actually chaos is not order."],"probe":"What is principle?","expected_proposals":3,"min_vault_hits":1} diff --git a/evals/symbolic_logic/public/v2/cases.jsonl b/evals/symbolic_logic/public/v2/cases.jsonl new file mode 100644 index 00000000..dfbd7617 --- /dev/null +++ b/evals/symbolic_logic/public/v2/cases.jsonl @@ -0,0 +1,24 @@ +{"id":"SYM-V2-001","pattern":"chain_5","premises":["What is truth?","Actually truth is wisdom.","What is wisdom?","Actually wisdom is light.","What is light?","Actually light is meaning.","What is meaning?","Actually meaning is identity.","What is identity?","Actually identity is order."],"probe":"What is truth?","expected_proposals":5,"min_vault_hits":1} +{"id":"SYM-V2-002","pattern":"chain_5","premises":["What is knowledge?","Actually knowledge is understanding.","What is understanding?","Actually understanding is inference.","What is inference?","Actually inference is judgment.","What is judgment?","Actually judgment is wisdom.","What is wisdom?","Actually wisdom is principle."],"probe":"What is knowledge?","expected_proposals":5,"min_vault_hits":1} +{"id":"SYM-V2-003","pattern":"chain_4","premises":["What is creation?","Actually creation is order.","What is order?","Actually order is principle.","What is principle?","Actually principle is reason.","What is reason?","Actually reason is inference."],"probe":"What is creation?","expected_proposals":4,"min_vault_hits":1} +{"id":"SYM-V2-004","pattern":"chain_4","premises":["What is verification?","Actually verification is evidence.","What is evidence?","Actually evidence is judgment.","What is judgment?","Actually judgment is inference.","What is inference?","Actually inference is reason."],"probe":"What is verification?","expected_proposals":4,"min_vault_hits":1} +{"id":"SYM-V2-005","pattern":"modus_ponens_chain","premises":["What is truth?","Actually truth is wisdom.","What is wisdom?","Actually wisdom is light."],"probe":"What is truth?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-V2-006","pattern":"modus_ponens_chain","premises":["What is light?","Actually light is meaning.","What is meaning?","Actually meaning is identity."],"probe":"What is light?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-V2-007","pattern":"syllogism","premises":["What is reason?","Actually reason is inference.","What is inference?","Actually inference is judgment."],"probe":"What is reason?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-V2-008","pattern":"syllogism","premises":["What is memory?","Actually memory is recall.","What is recall?","Actually recall is evidence."],"probe":"What is memory?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-V2-009","pattern":"modus_tollens_chain","premises":["What is creation?","Actually creation requires order.","What is order?","Actually order is not chaos."],"probe":"What is creation?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-V2-010","pattern":"modus_tollens_chain","premises":["What is identity?","Actually identity requires coherence.","What is coherence?","Actually coherence is not contradiction."],"probe":"What is identity?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-V2-011","pattern":"negation_chain","premises":["What is truth?","Actually truth is not deception.","What is deception?","Actually deception is not coherence."],"probe":"What is truth?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-V2-012","pattern":"negation_chain","premises":["What is meaning?","Actually meaning is not noise.","What is noise?","Actually noise is not order."],"probe":"What is meaning?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-V2-013","pattern":"chain_recall","premises":["What is spirit?","Actually spirit is life.","What is life?","Actually life is being."],"probe":"What is spirit?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-V2-014","pattern":"chain_recall","premises":["What is concept?","Actually concept is meaning.","What is meaning?","Actually meaning is truth."],"probe":"What is concept?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-V2-015","pattern":"single_premise","premises":["What is order?","Actually order is principle."],"probe":"What is order?","expected_proposals":1,"min_vault_hits":1} +{"id":"SYM-V2-016","pattern":"single_premise","premises":["What is principle?","Actually principle is reason."],"probe":"What is principle?","expected_proposals":1,"min_vault_hits":1} +{"id":"SYM-V2-017","pattern":"single_premise","premises":["What is evidence?","Actually evidence is verification."],"probe":"What is evidence?","expected_proposals":1,"min_vault_hits":1} +{"id":"SYM-V2-018","pattern":"chain_3","premises":["What is wisdom?","Actually wisdom is judgment.","What is judgment?","Actually judgment is inference.","What is inference?","Actually inference is reason."],"probe":"What is wisdom?","expected_proposals":3,"min_vault_hits":1} +{"id":"SYM-V2-019","pattern":"chain_3","premises":["What is memory?","Actually memory is recall.","What is recall?","Actually recall is evidence.","What is evidence?","Actually evidence is verification."],"probe":"What is memory?","expected_proposals":3,"min_vault_hits":1} +{"id":"SYM-V2-020","pattern":"chain_3","premises":["What is light?","Actually light is meaning.","What is meaning?","Actually meaning is order.","What is order?","Actually order is principle."],"probe":"What is light?","expected_proposals":3,"min_vault_hits":1} +{"id":"SYM-V2-021","pattern":"mixed","premises":["What is truth?","Actually truth is coherence.","What is coherence?","Actually coherence is not contradiction.","What is contradiction?","Actually contradiction is not order."],"probe":"What is truth?","expected_proposals":3,"min_vault_hits":1} +{"id":"SYM-V2-022","pattern":"mixed","premises":["What is identity?","Actually identity is coherence.","What is coherence?","Actually coherence requires consistency.","What is consistency?","Actually consistency is not contradiction."],"probe":"What is identity?","expected_proposals":3,"min_vault_hits":1} +{"id":"SYM-V2-023","pattern":"mixed","premises":["What is knowledge?","Actually knowledge is verified inference.","What is inference?","Actually inference is not opinion.","What is opinion?","Actually opinion is not evidence."],"probe":"What is knowledge?","expected_proposals":3,"min_vault_hits":1} +{"id":"SYM-V2-024","pattern":"mixed","premises":["What is creation?","Actually creation is principled order.","What is order?","Actually order is not arbitrary.","What is principle?","Actually principle precedes practice."],"probe":"What is creation?","expected_proposals":3,"min_vault_hits":1} diff --git a/evals/symbolic_logic/results/v2_holdouts_20260516T201409Z.json b/evals/symbolic_logic/results/v2_holdouts_20260516T201409Z.json new file mode 100644 index 00000000..c21729d5 --- /dev/null +++ b/evals/symbolic_logic/results/v2_holdouts_20260516T201409Z.json @@ -0,0 +1,236 @@ +{ + "metrics": { + "premise_recall": 1.0, + "replay_determinism": 1.0, + "proposal_storage": 1.0, + "all_three_pass_rate": 1.0, + "case_count": 16, + "overall_pass": true + }, + "case_details": [ + { + "id": "SYM-V2-HLD-001", + "pattern": "chain_5", + "vault_hits": 9, + "trace_hash": "096125dee2238eff8a3ce227e7cac1c0271e20b77dc527e18cd95608dacf2f61", + "trace_hash_replay": "096125dee2238eff8a3ce227e7cac1c0271e20b77dc527e18cd95608dacf2f61", + "proposal_count": 5, + "expected_proposals": 5, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-HLD-002", + "pattern": "chain_4", + "vault_hits": 9, + "trace_hash": "c3c479fd56d0276c48ed0c9da8c8351cc7fbfad4479a9873608aae92e4a98272", + "trace_hash_replay": "c3c479fd56d0276c48ed0c9da8c8351cc7fbfad4479a9873608aae92e4a98272", + "proposal_count": 4, + "expected_proposals": 4, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-HLD-003", + "pattern": "chain_3", + "vault_hits": 5, + "trace_hash": "344ae898cfadf532bbe5738ecc8e46b2abdd087a32c007dda5dced11a0dfd3e0", + "trace_hash_replay": "344ae898cfadf532bbe5738ecc8e46b2abdd087a32c007dda5dced11a0dfd3e0", + "proposal_count": 3, + "expected_proposals": 3, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-HLD-004", + "pattern": "modus_ponens_chain", + "vault_hits": 12, + "trace_hash": "99416d363f01669791f3e0283e4f23686a1e6b9f9be4a8e61f18a7a5a07ea6e0", + "trace_hash_replay": "99416d363f01669791f3e0283e4f23686a1e6b9f9be4a8e61f18a7a5a07ea6e0", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-HLD-005", + "pattern": "syllogism", + "vault_hits": 5, + "trace_hash": "b56b3efbdf31e69d0b34f7115a19040ea8b9efef8b89ad754ce4b55dfa628fc3", + "trace_hash_replay": "b56b3efbdf31e69d0b34f7115a19040ea8b9efef8b89ad754ce4b55dfa628fc3", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-HLD-006", + "pattern": "modus_tollens_chain", + "vault_hits": 8, + "trace_hash": "4883560cd275730e79591882e0a38ec8556c5360456ed50b995126fd53ac3425", + "trace_hash_replay": "4883560cd275730e79591882e0a38ec8556c5360456ed50b995126fd53ac3425", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-HLD-007", + "pattern": "negation_chain", + "vault_hits": 7, + "trace_hash": "470842cf03a602dfad9ac4875207050ae15bc4fd61903e4531411cb816995e80", + "trace_hash_replay": "470842cf03a602dfad9ac4875207050ae15bc4fd61903e4531411cb816995e80", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-HLD-008", + "pattern": "chain_recall", + "vault_hits": 8, + "trace_hash": "a00830706c4d6f52e7b7229ec24173a7181381d18f59cddc09db265816743741", + "trace_hash_replay": "a00830706c4d6f52e7b7229ec24173a7181381d18f59cddc09db265816743741", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-HLD-009", + "pattern": "chain_recall", + "vault_hits": 10, + "trace_hash": "472b5fa663dca63fdbc76f0f841263229024d868f71e0e1eafd871daaed644a9", + "trace_hash_replay": "472b5fa663dca63fdbc76f0f841263229024d868f71e0e1eafd871daaed644a9", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-HLD-010", + "pattern": "single_premise", + "vault_hits": 6, + "trace_hash": "e444983608f079f51a7bbd56cbda8a2d26fd6ebcc3f004896cbccbb25623c203", + "trace_hash_replay": "e444983608f079f51a7bbd56cbda8a2d26fd6ebcc3f004896cbccbb25623c203", + "proposal_count": 1, + "expected_proposals": 1, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-HLD-011", + "pattern": "single_premise", + "vault_hits": 11, + "trace_hash": "e361e8d587eac8073e077460352ece36f205c6ee4d25b7e40357db9c7f66b9eb", + "trace_hash_replay": "e361e8d587eac8073e077460352ece36f205c6ee4d25b7e40357db9c7f66b9eb", + "proposal_count": 1, + "expected_proposals": 1, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-HLD-012", + "pattern": "chain_3", + "vault_hits": 8, + "trace_hash": "31f11c9a4ca1222a282496ba01c9afbefa2d8cb0f9574a5dd55b4d041af3535e", + "trace_hash_replay": "31f11c9a4ca1222a282496ba01c9afbefa2d8cb0f9574a5dd55b4d041af3535e", + "proposal_count": 3, + "expected_proposals": 3, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-HLD-013", + "pattern": "chain_3", + "vault_hits": 10, + "trace_hash": "8d7c1bbac5967078feb1235d95cba65ee9ad800c7ee460a7545ad849b0bbc84c", + "trace_hash_replay": "8d7c1bbac5967078feb1235d95cba65ee9ad800c7ee460a7545ad849b0bbc84c", + "proposal_count": 3, + "expected_proposals": 3, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-HLD-014", + "pattern": "mixed", + "vault_hits": 9, + "trace_hash": "bcbae85cf5af12ac22a0fdfd44e2d4ee2f7baf2ff67bf2fc32f4099797f8fca1", + "trace_hash_replay": "bcbae85cf5af12ac22a0fdfd44e2d4ee2f7baf2ff67bf2fc32f4099797f8fca1", + "proposal_count": 3, + "expected_proposals": 3, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-HLD-015", + "pattern": "mixed", + "vault_hits": 9, + "trace_hash": "2f92a5aded0350ef934f9baf00f8121050f654bf5eafc558db89194c1ff2303c", + "trace_hash_replay": "2f92a5aded0350ef934f9baf00f8121050f654bf5eafc558db89194c1ff2303c", + "proposal_count": 3, + "expected_proposals": 3, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-HLD-016", + "pattern": "mixed", + "vault_hits": 12, + "trace_hash": "3ef230fa76b66c124d15009bcb3a1aa5cac7849b4dd996437ee3eddbda2bfcc4", + "trace_hash_replay": "3ef230fa76b66c124d15009bcb3a1aa5cac7849b4dd996437ee3eddbda2bfcc4", + "proposal_count": 3, + "expected_proposals": 3, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + } + ] +} \ No newline at end of file diff --git a/evals/symbolic_logic/results/v2_public_20260516T201409Z.json b/evals/symbolic_logic/results/v2_public_20260516T201409Z.json new file mode 100644 index 00000000..178a0693 --- /dev/null +++ b/evals/symbolic_logic/results/v2_public_20260516T201409Z.json @@ -0,0 +1,348 @@ +{ + "metrics": { + "premise_recall": 1.0, + "replay_determinism": 1.0, + "proposal_storage": 1.0, + "all_three_pass_rate": 1.0, + "case_count": 24, + "overall_pass": true + }, + "case_details": [ + { + "id": "SYM-V2-001", + "pattern": "chain_5", + "vault_hits": 9, + "trace_hash": "3149f304ab8cb4cf1267467fbaea85c191837d145eaf99043d86f29fa413ba73", + "trace_hash_replay": "3149f304ab8cb4cf1267467fbaea85c191837d145eaf99043d86f29fa413ba73", + "proposal_count": 5, + "expected_proposals": 5, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-002", + "pattern": "chain_5", + "vault_hits": 7, + "trace_hash": "1094af17c8e2deb34f770ca5d14dea3749ec1b0f659c720baa4dbb6d19bcbbdc", + "trace_hash_replay": "1094af17c8e2deb34f770ca5d14dea3749ec1b0f659c720baa4dbb6d19bcbbdc", + "proposal_count": 5, + "expected_proposals": 5, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-003", + "pattern": "chain_4", + "vault_hits": 9, + "trace_hash": "0047c5566da5f9271bebbd7b9b95a9134b56c66197801c68e8bbc8e42b2b9958", + "trace_hash_replay": "0047c5566da5f9271bebbd7b9b95a9134b56c66197801c68e8bbc8e42b2b9958", + "proposal_count": 4, + "expected_proposals": 4, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-004", + "pattern": "chain_4", + "vault_hits": 9, + "trace_hash": "da13c4c1a6d0f19491b8a119a8f5fa502452c23014399288a7af1a7d016888be", + "trace_hash_replay": "da13c4c1a6d0f19491b8a119a8f5fa502452c23014399288a7af1a7d016888be", + "proposal_count": 4, + "expected_proposals": 4, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-005", + "pattern": "modus_ponens_chain", + "vault_hits": 6, + "trace_hash": "7ada8825761fb66ab6ece0fb7b3c41c69a87d899768dd9dc8a356ef7f6f92b02", + "trace_hash_replay": "7ada8825761fb66ab6ece0fb7b3c41c69a87d899768dd9dc8a356ef7f6f92b02", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-006", + "pattern": "modus_ponens_chain", + "vault_hits": 7, + "trace_hash": "c6e80189d3cd62495a09d9db8da82d8547c7bd9a7a9dfadb1877b4bb1c9f1bb5", + "trace_hash_replay": "c6e80189d3cd62495a09d9db8da82d8547c7bd9a7a9dfadb1877b4bb1c9f1bb5", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-007", + "pattern": "syllogism", + "vault_hits": 9, + "trace_hash": "60a94aef3fbe40f1c81d2dd419fccac5959845ecff965452b842e5345677cb63", + "trace_hash_replay": "60a94aef3fbe40f1c81d2dd419fccac5959845ecff965452b842e5345677cb63", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-008", + "pattern": "syllogism", + "vault_hits": 9, + "trace_hash": "4c60c5066397d815d5d4ad94d7fd57c21de372a799052da502387ea76bcc298d", + "trace_hash_replay": "4c60c5066397d815d5d4ad94d7fd57c21de372a799052da502387ea76bcc298d", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-009", + "pattern": "modus_tollens_chain", + "vault_hits": 8, + "trace_hash": "ecbbe849b6b63f44e217675154c312d766da07bba596dd4f9a25c754f22fe8f0", + "trace_hash_replay": "ecbbe849b6b63f44e217675154c312d766da07bba596dd4f9a25c754f22fe8f0", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-010", + "pattern": "modus_tollens_chain", + "vault_hits": 11, + "trace_hash": "c847edb69aaabd572c0baf6ed9f49bd9d383b3b4ffc7f74496753aa160256e56", + "trace_hash_replay": "c847edb69aaabd572c0baf6ed9f49bd9d383b3b4ffc7f74496753aa160256e56", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-011", + "pattern": "negation_chain", + "vault_hits": 9, + "trace_hash": "2266d6455e2d3db8002094551ad1d3714e5f73a19b9087f3b5266cd4492251f4", + "trace_hash_replay": "2266d6455e2d3db8002094551ad1d3714e5f73a19b9087f3b5266cd4492251f4", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-012", + "pattern": "negation_chain", + "vault_hits": 9, + "trace_hash": "785f7b82c4d7a6c0df40c20a22d521488e4520d79b601b7e4d82f50db8b76519", + "trace_hash_replay": "785f7b82c4d7a6c0df40c20a22d521488e4520d79b601b7e4d82f50db8b76519", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-013", + "pattern": "chain_recall", + "vault_hits": 6, + "trace_hash": "e52c769164fb2b02a210e9d149e5e7f926e729b5f04a615311a89b50a4a89b48", + "trace_hash_replay": "e52c769164fb2b02a210e9d149e5e7f926e729b5f04a615311a89b50a4a89b48", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-014", + "pattern": "chain_recall", + "vault_hits": 6, + "trace_hash": "51e1038ccdac76ba26b24cc556f271aa34ad0980d5db1536d9a7b60109fc5a04", + "trace_hash_replay": "51e1038ccdac76ba26b24cc556f271aa34ad0980d5db1536d9a7b60109fc5a04", + "proposal_count": 2, + "expected_proposals": 2, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-015", + "pattern": "single_premise", + "vault_hits": 6, + "trace_hash": "9bda4e0f53c343482e0a82fdcc944b6cb8ac30a93bc242726100b08c23616edc", + "trace_hash_replay": "9bda4e0f53c343482e0a82fdcc944b6cb8ac30a93bc242726100b08c23616edc", + "proposal_count": 1, + "expected_proposals": 1, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-016", + "pattern": "single_premise", + "vault_hits": 6, + "trace_hash": "ab483df8c90d8667caaf9906d68d8532999f4b078c69c5544345fba25bec66c1", + "trace_hash_replay": "ab483df8c90d8667caaf9906d68d8532999f4b078c69c5544345fba25bec66c1", + "proposal_count": 1, + "expected_proposals": 1, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-017", + "pattern": "single_premise", + "vault_hits": 6, + "trace_hash": "e62a2ced1b05a402d1561feb1ba31812885b20667222152afcd538cb205cf672", + "trace_hash_replay": "e62a2ced1b05a402d1561feb1ba31812885b20667222152afcd538cb205cf672", + "proposal_count": 1, + "expected_proposals": 1, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-018", + "pattern": "chain_3", + "vault_hits": 8, + "trace_hash": "5998b7a88573e59986a76472c5d4efaba39cb0cf7aa8d3182ad40cae0250d693", + "trace_hash_replay": "5998b7a88573e59986a76472c5d4efaba39cb0cf7aa8d3182ad40cae0250d693", + "proposal_count": 3, + "expected_proposals": 3, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-019", + "pattern": "chain_3", + "vault_hits": 8, + "trace_hash": "460821a5b8ec24e78fa91a44f14afc2dadce44757e56889083858aea4d082559", + "trace_hash_replay": "460821a5b8ec24e78fa91a44f14afc2dadce44757e56889083858aea4d082559", + "proposal_count": 3, + "expected_proposals": 3, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-020", + "pattern": "chain_3", + "vault_hits": 7, + "trace_hash": "4aa87478699e5de41539aecdf172ff876bb503065c0b13a4b32f0899e15e720c", + "trace_hash_replay": "4aa87478699e5de41539aecdf172ff876bb503065c0b13a4b32f0899e15e720c", + "proposal_count": 3, + "expected_proposals": 3, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-021", + "pattern": "mixed", + "vault_hits": 10, + "trace_hash": "49ad92f6bb8e747a49e2ef1ef6738dbc2a30d5c571c8e1269f7796bd2bd63587", + "trace_hash_replay": "49ad92f6bb8e747a49e2ef1ef6738dbc2a30d5c571c8e1269f7796bd2bd63587", + "proposal_count": 3, + "expected_proposals": 3, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-022", + "pattern": "mixed", + "vault_hits": 11, + "trace_hash": "4196fb58a84737a1c71e6845227d27344beb44d23237fdf6186ba22f41d81968", + "trace_hash_replay": "4196fb58a84737a1c71e6845227d27344beb44d23237fdf6186ba22f41d81968", + "proposal_count": 3, + "expected_proposals": 3, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-023", + "pattern": "mixed", + "vault_hits": 4, + "trace_hash": "f807f3bca3266985f80f9d870ca871d7c2a325289c6176f29d34bcfa676b9ca9", + "trace_hash_replay": "f807f3bca3266985f80f9d870ca871d7c2a325289c6176f29d34bcfa676b9ca9", + "proposal_count": 3, + "expected_proposals": 3, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + }, + { + "id": "SYM-V2-024", + "pattern": "mixed", + "vault_hits": 7, + "trace_hash": "5fb3d30d2c42a750cccb055d9f7d02ca8b9fd48ac19ac7c6e7a3690e40679097", + "trace_hash_replay": "5fb3d30d2c42a750cccb055d9f7d02ca8b9fd48ac19ac7c6e7a3690e40679097", + "proposal_count": 3, + "expected_proposals": 3, + "min_vault_hits": 1, + "premise_recall_pass": true, + "replay_pass": true, + "proposal_pass": true, + "passed": true + } + ] +} \ No newline at end of file