From a0e9833851d94ba24aa86f11be25696db2ba53f9 Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 22 May 2026 16:26:01 -0700 Subject: [PATCH] feat: ADR-0122 systems_software audit-passed deferred (lane-shape mismatch) --- ...-systems-software-audit-passed-deferred.md | 95 +++++ docs/decisions/README.md | 3 + .../results/v1_holdout_20260522T232242Z.json | 232 +++++++++++ .../results/v1_public_20260522T232205Z.json | 376 ++++++++++++++++++ .../holdouts/v1/cases_plaintext.jsonl | 12 + .../results/v1_holdout_20260522T232327Z.json | 184 +++++++++ .../results/v1_public_20260522T232200Z.json | 268 +++++++++++++ tests/test_adr_0122_deferral.py | 74 ++++ 8 files changed, 1244 insertions(+) create mode 100644 docs/decisions/ADR-0122-systems-software-audit-passed-deferred.md create mode 100644 evals/inference_closure/results/v1_holdout_20260522T232242Z.json create mode 100644 evals/inference_closure/results/v1_public_20260522T232205Z.json create mode 100644 evals/symbolic_logic/holdouts/v1/cases_plaintext.jsonl create mode 100644 evals/symbolic_logic/results/v1_holdout_20260522T232327Z.json create mode 100644 evals/symbolic_logic/results/v1_public_20260522T232200Z.json create mode 100644 tests/test_adr_0122_deferral.py diff --git a/docs/decisions/ADR-0122-systems-software-audit-passed-deferred.md b/docs/decisions/ADR-0122-systems-software-audit-passed-deferred.md new file mode 100644 index 00000000..0899216a --- /dev/null +++ b/docs/decisions/ADR-0122-systems-software-audit-passed-deferred.md @@ -0,0 +1,95 @@ +# ADR-0122 — `systems_software` Audit-Passed Promotion: Deferred + +**Status:** Accepted (decision: defer promotion) +**Date:** 2026-05-22 +**Author:** CORE agents + reviewers +**Depends on:** ADR-0101, ADR-0106, ADR-0109, ADR-0110, ADR-0111, ADR-0113 + +--- + +## Context + +ADR-0101 ratified `systems_software` as `reasoning-capable`. +ADR-0110 and ADR-0111 then demonstrated two successful `audit-passed` +promotions (`mathematics_logic`, `physics`) under the ADR-0106 + ADR-0109 +contract shape (signed digest + replay determinism + typed refusal + exact recall). + +ADR-0122 is the attempted third promotion on `systems_software`. + +--- + +## Attempt + +`systems_software` attaches three eval lanes via its pack manifest: + +- `symbolic_logic` +- `inference_closure` +- `fabrication_control` + +For ADR-0122 we re-ran all three lanes on both `public` and `holdout` splits. +`inference_closure` and `fabrication_control` both pass with their expected +contract shapes (`inference_shape`, `refusal_shape`). + +--- + +## Blocker + +`symbolic_logic` lane produces inference-shape metrics +(`all_three_pass_rate` / `overall_pass` / `replay_determinism`) but +`LANE_SHAPE_REGISTRY` dispatches it to `_check_accuracy_shape` +(`symbolic_logic` → `symbolic_logic_shape`, checker expects +`accuracy` ≥ 0.95 or `passed/total` fallback). + +Current `symbolic_logic` result payloads do not include `accuracy` +or `passed/total`, so the gate fails closed with: + +`lane 'symbolic_logic' missing accuracy (and no passed/total fallback)`. + +This is a real contract-shape mismatch. Either: + +1. The lane runner must change to emit accuracy-shape output. +2. `LANE_SHAPE_REGISTRY` must be amended (e.g. map `symbolic_logic` to + `inference_shape`, or add a dedicated checker that consumes its + inference-style metric schema). + +Both are contract/lane-shape amendments and are out of scope for ADR-0122. + +--- + +## Decision + +Defer `systems_software` audit-passed promotion until the named blocker +is resolved by a future ADR amendment to ADR-0109 lane-shape mapping/checking. + +No claim is signed; no `audit_passed_claims` entry is added for +`systems_software`. + +--- + +## Invariants + +### `adr_0122_no_silent_promotion` + +`systems_software` remains at `status="reasoning-capable"` with +`predicates.audit_passed == false`. + +### `adr_0122_blocker_is_explicit` + +Evaluating the gate with `systems_software` lanes and current +`symbolic_logic` outputs fails specifically on the accuracy-shape mismatch, +not on an opaque or unrelated error. + +### `adr_0122_other_promotions_unchanged` + +`mathematics_logic` and `physics` remain `audit-passed`; ADR-0122 does not +modify or weaken their claims. + +--- + +## Consequences + +- This is the second honest contract refusal after ADR-0107. +- Gate credibility increases: promotions are recorded only when evidence and + contract shape align. +- Next move is explicit: lane-shape amendment ADR, then re-attempt promotion. + diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 0ad12f29..4082f087 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -38,6 +38,7 @@ ADRs record significant architectural decisions: what was decided, why, what alt | [ADR-0114a](ADR-0114a-anti-overfitting-proof-obligations.md) | Anti-Overfitting Proof Obligations for `expert` Promotion | Accepted (2026-05-22) | | [ADR-0115](ADR-0115-math-problem-parser-and-graph.md) | Math Problem Parser and Typed Proposition Graph | Phase 1.1+1.2+1.3 Accepted (2026-05-22) | | [ADR-0116](ADR-0116-deterministic-solver.md) | Deterministic Solver (`MathProblemGraph` → `SolutionTrace`) | Accepted (2026-05-22) | +| [ADR-0122](ADR-0122-systems-software-audit-passed-deferred.md) | `systems_software` Audit-Passed Promotion: Deferred | Accepted (2026-05-22) | --- @@ -94,6 +95,8 @@ Sequencing per ADR-0108. Listed in priority order: ADR-0080 (Contemplation Loop, Phase 1), ADR-0110 (math audit-passed), and ADR-0111 (physics audit-passed) have all landed — `mathematics_logic` and `physics` are at `audit_passed=true`; the contemplation loop emits read-only `SPECULATIVE` findings from `frontier_compare` reports. The remaining two ratified domains (`systems_software`, `hebrew_greek_textual_reasoning`) need their own promotion ADRs. +ADR-0122 attempted `systems_software` promotion and deferred honestly on a lane-shape contract mismatch (`symbolic_logic` output shape vs registered checker), becoming the second worked refusal after ADR-0107. + ### Open candidate directions (no ADR yet) - **Multi-reviewer holdout governance and threshold signing.** ADR-0105 seals holdout payloads with a single recipient identity; multi-reviewer governance remains future work. diff --git a/evals/inference_closure/results/v1_holdout_20260522T232242Z.json b/evals/inference_closure/results/v1_holdout_20260522T232242Z.json new file mode 100644 index 00000000..2184e6cc --- /dev/null +++ b/evals/inference_closure/results/v1_holdout_20260522T232242Z.json @@ -0,0 +1,232 @@ +{ + "cases": [ + { + "derived_recall_pass": true, + "entailment_tokens": [ + "reality" + ], + "expected_proposals": 2, + "id": "INF-V1-HLD-001", + "passed": true, + "pattern": "transitive_is", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "c7adfb910bd4aa7869f0c9fa6ed6e27b79d66e9429da4d7bfbe8d4d87bacfdb9", + "trace_hash_replay": "c7adfb910bd4aa7869f0c9fa6ed6e27b79d66e9429da4d7bfbe8d4d87bacfdb9", + "vault_hit": false, + "vault_hits": 6 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "order" + ], + "expected_proposals": 2, + "id": "INF-V1-HLD-002", + "passed": true, + "pattern": "transitive_is", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "8ff570c7e5ad365712e57b8ca40dc777a44c633742ded157804ce28962af99ab", + "trace_hash_replay": "8ff570c7e5ad365712e57b8ca40dc777a44c633742ded157804ce28962af99ab", + "vault_hit": false, + "vault_hits": 9 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "change" + ], + "expected_proposals": 2, + "id": "INF-V1-HLD-003", + "passed": true, + "pattern": "transitive_is", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "3a4a05863fb7baf7fafc303d59c70152ba17ef24a19b3707648268e2fea5e38b", + "trace_hash_replay": "3a4a05863fb7baf7fafc303d59c70152ba17ef24a19b3707648268e2fea5e38b", + "vault_hit": false, + "vault_hits": 8 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "explanation" + ], + "expected_proposals": 2, + "id": "INF-V1-HLD-004", + "passed": true, + "pattern": "transitive_precedes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "faf59ddd31dfad5409f00642ce687515d7a81914dd2ee152fc1d90d2eeaeaa78", + "trace_hash_replay": "faf59ddd31dfad5409f00642ce687515d7a81914dd2ee152fc1d90d2eeaeaa78", + "vault_hit": false, + "vault_hits": 10 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "naming" + ], + "expected_proposals": 2, + "id": "INF-V1-HLD-005", + "passed": true, + "pattern": "transitive_precedes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "5b06de51142d66f00004820f8cdff458ed7baf0d8fd3b1ea5fa626dfc7e896a5", + "trace_hash_replay": "5b06de51142d66f00004820f8cdff458ed7baf0d8fd3b1ea5fa626dfc7e896a5", + "vault_hit": false, + "vault_hits": 9 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "mastery" + ], + "expected_proposals": 2, + "id": "INF-V1-HLD-006", + "passed": true, + "pattern": "transitive_precedes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "231e802b5be2853e52e5e45a23ffb8e47afb07eb1dcec2154ab34fc0e912020e", + "trace_hash_replay": "231e802b5be2853e52e5e45a23ffb8e47afb07eb1dcec2154ab34fc0e912020e", + "vault_hit": false, + "vault_hits": 6 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "conclusion" + ], + "expected_proposals": 2, + "id": "INF-V1-HLD-007", + "passed": true, + "pattern": "transitive_grounds", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "7be02885ebed84313290470406ac5a15abe7bdbf190e27c6b819c2ede3585025", + "trace_hash_replay": "7be02885ebed84313290470406ac5a15abe7bdbf190e27c6b819c2ede3585025", + "vault_hit": false, + "vault_hits": 7 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "action" + ], + "expected_proposals": 2, + "id": "INF-V1-HLD-008", + "passed": true, + "pattern": "transitive_grounds", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "2c44988ee246b0cd1bc9cb6aaf23b3e923bc39a2f820a4b43b47d5a7fac51d34", + "trace_hash_replay": "2c44988ee246b0cd1bc9cb6aaf23b3e923bc39a2f820a4b43b47d5a7fac51d34", + "vault_hit": false, + "vault_hits": 8 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "definition" + ], + "expected_proposals": 2, + "id": "INF-V1-HLD-009", + "passed": true, + "pattern": "transitive_causes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "adbd2fccb9acbdf90a017077dcc60eec88acba3bcbba380781e4047477a7bac4", + "trace_hash_replay": "adbd2fccb9acbdf90a017077dcc60eec88acba3bcbba380781e4047477a7bac4", + "vault_hit": false, + "vault_hits": 8 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "learning" + ], + "expected_proposals": 2, + "id": "INF-V1-HLD-010", + "passed": true, + "pattern": "transitive_causes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "bc5f0d3a57080410d20fe6e1e8f722e885d2617b4bd045363afdeefda98e53f2", + "trace_hash_replay": "bc5f0d3a57080410d20fe6e1e8f722e885d2617b4bd045363afdeefda98e53f2", + "vault_hit": false, + "vault_hits": 9 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "inquiry" + ], + "expected_proposals": 2, + "id": "INF-V1-HLD-011", + "passed": true, + "pattern": "transitive_belongs_to", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "3ba3808c263232ce07f7e04a981cd90855b4626b6dc7572746865bd0a244dbfc", + "trace_hash_replay": "3ba3808c263232ce07f7e04a981cd90855b4626b6dc7572746865bd0a244dbfc", + "vault_hit": false, + "vault_hits": 9 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "ground" + ], + "expected_proposals": 2, + "id": "INF-V1-HLD-012", + "passed": true, + "pattern": "transitive_belongs_to", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "13d496b471863abf94ba48b7006ddba57cec3985dec559f514ab645c039ec05c", + "trace_hash_replay": "13d496b471863abf94ba48b7006ddba57cec3985dec559f514ab645c039ec05c", + "vault_hit": false, + "vault_hits": 9 + } + ], + "lane": "inference_closure", + "metrics": { + "all_pass_rate": 1.0, + "case_count": 12, + "derived_recall_rate": 1.0, + "overall_pass": true, + "premises_stored_rate": 1.0, + "replay_determinism": 1.0 + }, + "split": "holdout", + "timestamp": "2026-05-22T23:22:42.349984+00:00", + "version": "v1" +} diff --git a/evals/inference_closure/results/v1_public_20260522T232205Z.json b/evals/inference_closure/results/v1_public_20260522T232205Z.json new file mode 100644 index 00000000..ef10f0c0 --- /dev/null +++ b/evals/inference_closure/results/v1_public_20260522T232205Z.json @@ -0,0 +1,376 @@ +{ + "cases": [ + { + "derived_recall_pass": true, + "entailment_tokens": [ + "truth" + ], + "expected_proposals": 2, + "id": "INF-V1-001", + "passed": true, + "pattern": "transitive_is", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "e228ea6d2d138d03d7b2111cb98cbea6ba68fb0a7bc05b02831a19dd669007a8", + "trace_hash_replay": "e228ea6d2d138d03d7b2111cb98cbea6ba68fb0a7bc05b02831a19dd669007a8", + "vault_hit": false, + "vault_hits": 9 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "principle" + ], + "expected_proposals": 2, + "id": "INF-V1-002", + "passed": true, + "pattern": "transitive_is", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "9e80eacaa26d08d54cf537c0fee0d238a93eeb97aeb40e8a0e27bdaef673b3f7", + "trace_hash_replay": "9e80eacaa26d08d54cf537c0fee0d238a93eeb97aeb40e8a0e27bdaef673b3f7", + "vault_hit": false, + "vault_hits": 9 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "wisdom" + ], + "expected_proposals": 2, + "id": "INF-V1-003", + "passed": true, + "pattern": "transitive_is", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "c55631835496b7bdea5dd88b3a07e9c2bfd266baf9bddc982618e309223b5219", + "trace_hash_replay": "c55631835496b7bdea5dd88b3a07e9c2bfd266baf9bddc982618e309223b5219", + "vault_hit": false, + "vault_hits": 8 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "structure" + ], + "expected_proposals": 2, + "id": "INF-V1-004", + "passed": true, + "pattern": "transitive_is", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "478e8aa0bd4a108b568be00f79de90db0bf50d57f78776f5d16625ba5bafd731", + "trace_hash_replay": "478e8aa0bd4a108b568be00f79de90db0bf50d57f78776f5d16625ba5bafd731", + "vault_hit": false, + "vault_hits": 7 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "meaning" + ], + "expected_proposals": 2, + "id": "INF-V1-005", + "passed": true, + "pattern": "transitive_precedes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "ed7232b60c363a369a10400b3ce264d6a84905795b9b197f5b45ddccddf30be5", + "trace_hash_replay": "ed7232b60c363a369a10400b3ce264d6a84905795b9b197f5b45ddccddf30be5", + "vault_hit": false, + "vault_hits": 7 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "recall" + ], + "expected_proposals": 2, + "id": "INF-V1-006", + "passed": true, + "pattern": "transitive_precedes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "3895f56f601b5604a8069b2ef4d97cc039e044e46ed7da1698f71670544c61dd", + "trace_hash_replay": "3895f56f601b5604a8069b2ef4d97cc039e044e46ed7da1698f71670544c61dd", + "vault_hit": false, + "vault_hits": 6 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "recognition" + ], + "expected_proposals": 2, + "id": "INF-V1-007", + "passed": true, + "pattern": "transitive_precedes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "556b62484be30ef6e3d8f757e5f3226ea5c2f2cf84ea8cf46a4c44aae61cd3dd", + "trace_hash_replay": "556b62484be30ef6e3d8f757e5f3226ea5c2f2cf84ea8cf46a4c44aae61cd3dd", + "vault_hit": false, + "vault_hits": 7 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "judgment" + ], + "expected_proposals": 2, + "id": "INF-V1-008", + "passed": true, + "pattern": "transitive_precedes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "649f63f06ffcb16c9f3157a67b72bd3a849e083a9c607d81265334ab66eec9df", + "trace_hash_replay": "649f63f06ffcb16c9f3157a67b72bd3a849e083a9c607d81265334ab66eec9df", + "vault_hit": false, + "vault_hits": 7 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "judgment" + ], + "expected_proposals": 2, + "id": "INF-V1-009", + "passed": true, + "pattern": "transitive_grounds", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "8b3e9a44665362ac059ff08c1806b6037ba46479934fe6e8e625d9194fc2850a", + "trace_hash_replay": "8b3e9a44665362ac059ff08c1806b6037ba46479934fe6e8e625d9194fc2850a", + "vault_hit": false, + "vault_hits": 8 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "inference" + ], + "expected_proposals": 2, + "id": "INF-V1-010", + "passed": true, + "pattern": "transitive_grounds", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "2411f6c620a83b791f9ce531c2cc2e2ab99232dc4106f3d675a722d031d26de8", + "trace_hash_replay": "2411f6c620a83b791f9ce531c2cc2e2ab99232dc4106f3d675a722d031d26de8", + "vault_hit": false, + "vault_hits": 6 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "confidence" + ], + "expected_proposals": 2, + "id": "INF-V1-011", + "passed": true, + "pattern": "transitive_grounds", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "e78cf2211296722f5818acc51e0e27b1bf7078a73126101452fa4e3236f74261", + "trace_hash_replay": "e78cf2211296722f5818acc51e0e27b1bf7078a73126101452fa4e3236f74261", + "vault_hit": false, + "vault_hits": 11 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "meaning" + ], + "expected_proposals": 2, + "id": "INF-V1-012", + "passed": true, + "pattern": "transitive_grounds", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "a054ef36b7d692a6a36655f09d4437b9d375b4f1e8dd07a913be50bec63b8f0f", + "trace_hash_replay": "a054ef36b7d692a6a36655f09d4437b9d375b4f1e8dd07a913be50bec63b8f0f", + "vault_hit": false, + "vault_hits": 8 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "recognition" + ], + "expected_proposals": 2, + "id": "INF-V1-013", + "passed": true, + "pattern": "transitive_causes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "ad9ce5470b3ad041e312986a73bdbab5f08f287c53b82786faa691bec17e5699", + "trace_hash_replay": "ad9ce5470b3ad041e312986a73bdbab5f08f287c53b82786faa691bec17e5699", + "vault_hit": false, + "vault_hits": 10 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "thought" + ], + "expected_proposals": 2, + "id": "INF-V1-014", + "passed": true, + "pattern": "transitive_causes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "dddc38be0922e26c67d8315d2a90b5f50225577b10dc0b223b3d02345c8e482a", + "trace_hash_replay": "dddc38be0922e26c67d8315d2a90b5f50225577b10dc0b223b3d02345c8e482a", + "vault_hit": false, + "vault_hits": 9 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "decision" + ], + "expected_proposals": 2, + "id": "INF-V1-015", + "passed": true, + "pattern": "transitive_causes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "7e2387159559363b524d0dd5ee84a51f32c15a3029cd13e1af443014a70511ed", + "trace_hash_replay": "7e2387159559363b524d0dd5ee84a51f32c15a3029cd13e1af443014a70511ed", + "vault_hit": false, + "vault_hits": 9 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "coherence" + ], + "expected_proposals": 2, + "id": "INF-V1-016", + "passed": true, + "pattern": "transitive_causes", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "bbc40a3e50910c23271a86adb4a56b8207a652f6bf3ef54ad715410d2d825376", + "trace_hash_replay": "bbc40a3e50910c23271a86adb4a56b8207a652f6bf3ef54ad715410d2d825376", + "vault_hit": false, + "vault_hits": 8 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "thought" + ], + "expected_proposals": 2, + "id": "INF-V1-017", + "passed": true, + "pattern": "transitive_belongs_to", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "566823c524d52ca919dbf65367e8ce11a7b21288ec8e609f68c91da285fd17a2", + "trace_hash_replay": "566823c524d52ca919dbf65367e8ce11a7b21288ec8e609f68c91da285fd17a2", + "vault_hit": false, + "vault_hits": 9 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "truth" + ], + "expected_proposals": 2, + "id": "INF-V1-018", + "passed": true, + "pattern": "transitive_belongs_to", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "6f7d7900846ce601176022d6feb8bafeedc18065c4ef42c8296f6ee451c5435a", + "trace_hash_replay": "6f7d7900846ce601176022d6feb8bafeedc18065c4ef42c8296f6ee451c5435a", + "vault_hit": false, + "vault_hits": 9 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "cognition" + ], + "expected_proposals": 2, + "id": "INF-V1-019", + "passed": true, + "pattern": "transitive_belongs_to", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "15f87deb2fdc497c6bcab3343a6d781850728b725904623d34518391b503a893", + "trace_hash_replay": "15f87deb2fdc497c6bcab3343a6d781850728b725904623d34518391b503a893", + "vault_hit": false, + "vault_hits": 7 + }, + { + "derived_recall_pass": true, + "entailment_tokens": [ + "truth" + ], + "expected_proposals": 2, + "id": "INF-V1-020", + "passed": true, + "pattern": "transitive_belongs_to", + "premise_proposal_count": 2, + "premises_stored_pass": true, + "replay_pass": true, + "surface_hit": true, + "trace_hash": "4f9a081207336549958a1d0c7105ce20d0a045a09842f182e6335f050c6c9eb4", + "trace_hash_replay": "4f9a081207336549958a1d0c7105ce20d0a045a09842f182e6335f050c6c9eb4", + "vault_hit": false, + "vault_hits": 9 + } + ], + "lane": "inference_closure", + "metrics": { + "all_pass_rate": 1.0, + "case_count": 20, + "derived_recall_rate": 1.0, + "overall_pass": true, + "premises_stored_rate": 1.0, + "replay_determinism": 1.0 + }, + "split": "public", + "timestamp": "2026-05-22T23:22:05.854118+00:00", + "version": "v1" +} diff --git a/evals/symbolic_logic/holdouts/v1/cases_plaintext.jsonl b/evals/symbolic_logic/holdouts/v1/cases_plaintext.jsonl new file mode 100644 index 00000000..517d3bd0 --- /dev/null +++ b/evals/symbolic_logic/holdouts/v1/cases_plaintext.jsonl @@ -0,0 +1,12 @@ +{"id":"SYM-HLD-001","pattern":"single_premise","premises":["What is thought?","Actually thought is reason."],"probe":"What is thought?","expected_proposals":1,"min_vault_hits":1} +{"id":"SYM-HLD-002","pattern":"modus_ponens_chain","premises":["What is evidence?","Actually evidence is verification.","What is verification?","Actually verification is judgment."],"probe":"What is evidence?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-HLD-003","pattern":"syllogism","premises":["What is principle?","Actually principle is order.","What is order?","Actually order is coherence."],"probe":"What is principle?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-HLD-004","pattern":"negation","premises":["What is meaning?","Actually meaning is not noise."],"probe":"What is meaning?","expected_proposals":1,"min_vault_hits":1} +{"id":"SYM-HLD-005","pattern":"modus_tollens_chain","premises":["What is life?","Actually life requires order.","What is order?","Actually order is not chaos."],"probe":"What is life?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-HLD-006","pattern":"chain_recall","premises":["What is judgment?","Actually judgment is inference.","What is inference?","Actually inference is reason.","What is reason?","Actually reason is thought."],"probe":"What is judgment?","expected_proposals":3,"min_vault_hits":1} +{"id":"SYM-HLD-007","pattern":"chain_recall","premises":["What is verification?","Actually verification is evidence.","What is evidence?","Actually evidence is recall.","What is recall?","Actually recall is memory."],"probe":"What is verification?","expected_proposals":3,"min_vault_hits":1} +{"id":"SYM-HLD-008","pattern":"single_premise","premises":["What is distinction?","Actually distinction is comparison."],"probe":"What is distinction?","expected_proposals":1,"min_vault_hits":1} +{"id":"SYM-HLD-009","pattern":"modus_ponens_chain","premises":["What is concept?","Actually concept is meaning.","What is meaning?","Actually meaning is truth."],"probe":"What is concept?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-HLD-010","pattern":"syllogism","premises":["What is understanding?","Actually understanding is judgment.","What is judgment?","Actually judgment is wisdom."],"probe":"What is understanding?","expected_proposals":2,"min_vault_hits":1} +{"id":"SYM-HLD-011","pattern":"negation","premises":["What is spirit?","Actually spirit is not matter."],"probe":"What is spirit?","expected_proposals":1,"min_vault_hits":1} +{"id":"SYM-HLD-012","pattern":"chain_recall","premises":["What is person?","Actually person is identity.","What is identity?","Actually identity is coherence.","What is coherence?","Actually coherence is order."],"probe":"What is person?","expected_proposals":3,"min_vault_hits":1} diff --git a/evals/symbolic_logic/results/v1_holdout_20260522T232327Z.json b/evals/symbolic_logic/results/v1_holdout_20260522T232327Z.json new file mode 100644 index 00000000..c6101b60 --- /dev/null +++ b/evals/symbolic_logic/results/v1_holdout_20260522T232327Z.json @@ -0,0 +1,184 @@ +{ + "cases": [ + { + "expected_proposals": 1, + "id": "SYM-HLD-001", + "min_vault_hits": 1, + "passed": true, + "pattern": "single_premise", + "premise_recall_pass": true, + "proposal_count": 1, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "80224fd99afb276e1cb4587534ef48e11e448fd8fc674c08df7271b7a7711532", + "trace_hash_replay": "80224fd99afb276e1cb4587534ef48e11e448fd8fc674c08df7271b7a7711532", + "vault_hits": 7 + }, + { + "expected_proposals": 2, + "id": "SYM-HLD-002", + "min_vault_hits": 1, + "passed": true, + "pattern": "modus_ponens_chain", + "premise_recall_pass": true, + "proposal_count": 2, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "16890086eb090b02afded85d0042086546e95de19bc92a641c7f119845016826", + "trace_hash_replay": "16890086eb090b02afded85d0042086546e95de19bc92a641c7f119845016826", + "vault_hits": 8 + }, + { + "expected_proposals": 2, + "id": "SYM-HLD-003", + "min_vault_hits": 1, + "passed": true, + "pattern": "syllogism", + "premise_recall_pass": true, + "proposal_count": 2, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "96739aafd272ffa6514aab20a9c4f5d88a61c50c1efe5fba915411d7385813b3", + "trace_hash_replay": "96739aafd272ffa6514aab20a9c4f5d88a61c50c1efe5fba915411d7385813b3", + "vault_hits": 6 + }, + { + "expected_proposals": 1, + "id": "SYM-HLD-004", + "min_vault_hits": 1, + "passed": true, + "pattern": "negation", + "premise_recall_pass": true, + "proposal_count": 1, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "c3420ea6d2fdabd317a8b02c4b5e207e006f83f285bcff2b04b4d1be911e231c", + "trace_hash_replay": "c3420ea6d2fdabd317a8b02c4b5e207e006f83f285bcff2b04b4d1be911e231c", + "vault_hits": 7 + }, + { + "expected_proposals": 2, + "id": "SYM-HLD-005", + "min_vault_hits": 1, + "passed": true, + "pattern": "modus_tollens_chain", + "premise_recall_pass": true, + "proposal_count": 2, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "67dc98b2f100d4fdafbf845d7b7cf51fcf5fbc5750a96a532674d1cbe78bf5ac", + "trace_hash_replay": "67dc98b2f100d4fdafbf845d7b7cf51fcf5fbc5750a96a532674d1cbe78bf5ac", + "vault_hits": 9 + }, + { + "expected_proposals": 3, + "id": "SYM-HLD-006", + "min_vault_hits": 1, + "passed": true, + "pattern": "chain_recall", + "premise_recall_pass": true, + "proposal_count": 3, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "ae8e1bd81c0d65225a414e143c112fc888d3d0298e9dd926e6ef6a498545a650", + "trace_hash_replay": "ae8e1bd81c0d65225a414e143c112fc888d3d0298e9dd926e6ef6a498545a650", + "vault_hits": 9 + }, + { + "expected_proposals": 3, + "id": "SYM-HLD-007", + "min_vault_hits": 1, + "passed": true, + "pattern": "chain_recall", + "premise_recall_pass": true, + "proposal_count": 3, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "a72d0e9670c82078b383c87f01181b51fe2d73b9ea954e4dd5b2b391ad4a17d3", + "trace_hash_replay": "a72d0e9670c82078b383c87f01181b51fe2d73b9ea954e4dd5b2b391ad4a17d3", + "vault_hits": 7 + }, + { + "expected_proposals": 1, + "id": "SYM-HLD-008", + "min_vault_hits": 1, + "passed": true, + "pattern": "single_premise", + "premise_recall_pass": true, + "proposal_count": 1, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "8152b48b4249e6bd7c87effb8dab81ea0f31cda4586193f17180e95bb92a2038", + "trace_hash_replay": "8152b48b4249e6bd7c87effb8dab81ea0f31cda4586193f17180e95bb92a2038", + "vault_hits": 9 + }, + { + "expected_proposals": 2, + "id": "SYM-HLD-009", + "min_vault_hits": 1, + "passed": true, + "pattern": "modus_ponens_chain", + "premise_recall_pass": true, + "proposal_count": 2, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "5006702acd91365e9dc94f96e76290b91d8390570e1db8f7d51591f59760c55c", + "trace_hash_replay": "5006702acd91365e9dc94f96e76290b91d8390570e1db8f7d51591f59760c55c", + "vault_hits": 6 + }, + { + "expected_proposals": 2, + "id": "SYM-HLD-010", + "min_vault_hits": 1, + "passed": true, + "pattern": "syllogism", + "premise_recall_pass": true, + "proposal_count": 2, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "76fcba154a23d8e5ea8404d1e77aebce985a7a332dbfcd5cb3c92571e0e00f2a", + "trace_hash_replay": "76fcba154a23d8e5ea8404d1e77aebce985a7a332dbfcd5cb3c92571e0e00f2a", + "vault_hits": 9 + }, + { + "expected_proposals": 1, + "id": "SYM-HLD-011", + "min_vault_hits": 1, + "passed": true, + "pattern": "negation", + "premise_recall_pass": true, + "proposal_count": 1, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "264d4f457950adfdf70800267cc52eb73228f5450d2485b2ae2381cff882676f", + "trace_hash_replay": "264d4f457950adfdf70800267cc52eb73228f5450d2485b2ae2381cff882676f", + "vault_hits": 9 + }, + { + "expected_proposals": 3, + "id": "SYM-HLD-012", + "min_vault_hits": 1, + "passed": true, + "pattern": "chain_recall", + "premise_recall_pass": true, + "proposal_count": 3, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "97c7c298a9ccd424ea3219229ca0773d771b7742ae13c735b411cab3b0f1c776", + "trace_hash_replay": "97c7c298a9ccd424ea3219229ca0773d771b7742ae13c735b411cab3b0f1c776", + "vault_hits": 6 + } + ], + "lane": "symbolic_logic", + "metrics": { + "all_three_pass_rate": 1.0, + "case_count": 12, + "overall_pass": true, + "premise_recall": 1.0, + "proposal_storage": 1.0, + "replay_determinism": 1.0 + }, + "split": "holdout", + "timestamp": "2026-05-22T23:23:27.759978+00:00", + "version": "v1" +} diff --git a/evals/symbolic_logic/results/v1_public_20260522T232200Z.json b/evals/symbolic_logic/results/v1_public_20260522T232200Z.json new file mode 100644 index 00000000..22320223 --- /dev/null +++ b/evals/symbolic_logic/results/v1_public_20260522T232200Z.json @@ -0,0 +1,268 @@ +{ + "cases": [ + { + "expected_proposals": 1, + "id": "SYM-PUB-001", + "min_vault_hits": 1, + "passed": true, + "pattern": "single_premise", + "premise_recall_pass": true, + "proposal_count": 1, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "55e2104ccc41356d36a049376948693631d1390ab0fa0f055652c012e8b722f6", + "trace_hash_replay": "55e2104ccc41356d36a049376948693631d1390ab0fa0f055652c012e8b722f6", + "vault_hits": 8 + }, + { + "expected_proposals": 1, + "id": "SYM-PUB-002", + "min_vault_hits": 1, + "passed": true, + "pattern": "single_premise", + "premise_recall_pass": true, + "proposal_count": 1, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "5cb9e8a7ff412e3a4543ff682ec59ca744005a4bf6e64568d071df18c237d017", + "trace_hash_replay": "5cb9e8a7ff412e3a4543ff682ec59ca744005a4bf6e64568d071df18c237d017", + "vault_hits": 9 + }, + { + "expected_proposals": 2, + "id": "SYM-PUB-003", + "min_vault_hits": 1, + "passed": true, + "pattern": "modus_ponens_chain", + "premise_recall_pass": true, + "proposal_count": 2, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "b3ed54d19e9b28b3f9e68ead8f3e4091f20d69b5d6ca5337cc01c7f386cace7c", + "trace_hash_replay": "b3ed54d19e9b28b3f9e68ead8f3e4091f20d69b5d6ca5337cc01c7f386cace7c", + "vault_hits": 6 + }, + { + "expected_proposals": 2, + "id": "SYM-PUB-004", + "min_vault_hits": 1, + "passed": true, + "pattern": "modus_ponens_chain", + "premise_recall_pass": true, + "proposal_count": 2, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "eb96dcaa7ae701fa02b45385c5603f9d2efa82733c1ef2a68f2ff9ce663e0a54", + "trace_hash_replay": "eb96dcaa7ae701fa02b45385c5603f9d2efa82733c1ef2a68f2ff9ce663e0a54", + "vault_hits": 8 + }, + { + "expected_proposals": 2, + "id": "SYM-PUB-005", + "min_vault_hits": 1, + "passed": true, + "pattern": "syllogism", + "premise_recall_pass": true, + "proposal_count": 2, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "261703a0da24e4f210a72577d5b5aeddbb3dd4cf5c695636a2cb5ac429006f7f", + "trace_hash_replay": "261703a0da24e4f210a72577d5b5aeddbb3dd4cf5c695636a2cb5ac429006f7f", + "vault_hits": 9 + }, + { + "expected_proposals": 2, + "id": "SYM-PUB-006", + "min_vault_hits": 1, + "passed": true, + "pattern": "syllogism", + "premise_recall_pass": true, + "proposal_count": 2, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "ec246139e93fd02652471d0b164106ddcefc870b2d575c003b586962d47c2efe", + "trace_hash_replay": "ec246139e93fd02652471d0b164106ddcefc870b2d575c003b586962d47c2efe", + "vault_hits": 7 + }, + { + "expected_proposals": 1, + "id": "SYM-PUB-007", + "min_vault_hits": 1, + "passed": true, + "pattern": "negation", + "premise_recall_pass": true, + "proposal_count": 1, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "2f1a134b0318008f0703f7c727d8d4fa23a8230852dfea76d2b2d8e59ad0fc7e", + "trace_hash_replay": "2f1a134b0318008f0703f7c727d8d4fa23a8230852dfea76d2b2d8e59ad0fc7e", + "vault_hits": 7 + }, + { + "expected_proposals": 1, + "id": "SYM-PUB-008", + "min_vault_hits": 1, + "passed": true, + "pattern": "negation", + "premise_recall_pass": true, + "proposal_count": 1, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "36f57fb9014e01dd3bcb7a71ebe90f868d688b357f098ab6cbc76bb04105dccb", + "trace_hash_replay": "36f57fb9014e01dd3bcb7a71ebe90f868d688b357f098ab6cbc76bb04105dccb", + "vault_hits": 8 + }, + { + "expected_proposals": 2, + "id": "SYM-PUB-009", + "min_vault_hits": 1, + "passed": true, + "pattern": "modus_tollens_chain", + "premise_recall_pass": true, + "proposal_count": 2, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "a5660b9b6fc02cd31cbc29bf32cbee7e26a7e36c210ff83368423818ded63db8", + "trace_hash_replay": "a5660b9b6fc02cd31cbc29bf32cbee7e26a7e36c210ff83368423818ded63db8", + "vault_hits": 8 + }, + { + "expected_proposals": 2, + "id": "SYM-PUB-010", + "min_vault_hits": 1, + "passed": true, + "pattern": "modus_tollens_chain", + "premise_recall_pass": true, + "proposal_count": 2, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "5094014c1059555827099e5db78bd1c8d364c8b5e3347a1a4b15182a1709f2dd", + "trace_hash_replay": "5094014c1059555827099e5db78bd1c8d364c8b5e3347a1a4b15182a1709f2dd", + "vault_hits": 11 + }, + { + "expected_proposals": 3, + "id": "SYM-PUB-011", + "min_vault_hits": 1, + "passed": true, + "pattern": "chain_recall", + "premise_recall_pass": true, + "proposal_count": 3, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "4cc1af73e8ff207b3a3474174241f792b35e04fad70cc1ea6b036360526d7f1c", + "trace_hash_replay": "4cc1af73e8ff207b3a3474174241f792b35e04fad70cc1ea6b036360526d7f1c", + "vault_hits": 4 + }, + { + "expected_proposals": 3, + "id": "SYM-PUB-012", + "min_vault_hits": 1, + "passed": true, + "pattern": "chain_recall", + "premise_recall_pass": true, + "proposal_count": 3, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "a94af500f39a835e9f5bcf9398c0ce8dc0410a4148d6a03eb8a8d80f1fe3f852", + "trace_hash_replay": "a94af500f39a835e9f5bcf9398c0ce8dc0410a4148d6a03eb8a8d80f1fe3f852", + "vault_hits": 8 + }, + { + "expected_proposals": 4, + "id": "SYM-PUB-013", + "min_vault_hits": 1, + "passed": true, + "pattern": "chain_recall", + "premise_recall_pass": true, + "proposal_count": 4, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "1b32f72a20f2e74f490218b127e588c6241e8957f2c15a480472061c5c945df0", + "trace_hash_replay": "1b32f72a20f2e74f490218b127e588c6241e8957f2c15a480472061c5c945df0", + "vault_hits": 9 + }, + { + "expected_proposals": 2, + "id": "SYM-PUB-014", + "min_vault_hits": 1, + "passed": true, + "pattern": "modus_ponens_chain", + "premise_recall_pass": true, + "proposal_count": 2, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "9e80eacaa26d08d54cf537c0fee0d238a93eeb97aeb40e8a0e27bdaef673b3f7", + "trace_hash_replay": "9e80eacaa26d08d54cf537c0fee0d238a93eeb97aeb40e8a0e27bdaef673b3f7", + "vault_hits": 9 + }, + { + "expected_proposals": 1, + "id": "SYM-PUB-015", + "min_vault_hits": 1, + "passed": true, + "pattern": "single_premise", + "premise_recall_pass": true, + "proposal_count": 1, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "f8d6254665eaadd6272196260394cbf53231d368e3f6dd520ebba7f8a814851d", + "trace_hash_replay": "f8d6254665eaadd6272196260394cbf53231d368e3f6dd520ebba7f8a814851d", + "vault_hits": 6 + }, + { + "expected_proposals": 2, + "id": "SYM-PUB-016", + "min_vault_hits": 1, + "passed": true, + "pattern": "syllogism", + "premise_recall_pass": true, + "proposal_count": 2, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "8f10e619ac4162b6077b3577565efb5c7921b1e8bac386e3e510742bbe4591e3", + "trace_hash_replay": "8f10e619ac4162b6077b3577565efb5c7921b1e8bac386e3e510742bbe4591e3", + "vault_hits": 9 + }, + { + "expected_proposals": 1, + "id": "SYM-PUB-017", + "min_vault_hits": 1, + "passed": true, + "pattern": "negation", + "premise_recall_pass": true, + "proposal_count": 1, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "39be6151a82543bdaf8cb4cccbee70b1bd969d6fb914ec26410d1c278a977ebe", + "trace_hash_replay": "39be6151a82543bdaf8cb4cccbee70b1bd969d6fb914ec26410d1c278a977ebe", + "vault_hits": 8 + }, + { + "expected_proposals": 2, + "id": "SYM-PUB-018", + "min_vault_hits": 1, + "passed": true, + "pattern": "chain_recall", + "premise_recall_pass": true, + "proposal_count": 2, + "proposal_pass": true, + "replay_pass": true, + "trace_hash": "cef95573b770e69ba726799bbd3925608053c0d29cc6b081e65d9f685adeeb85", + "trace_hash_replay": "cef95573b770e69ba726799bbd3925608053c0d29cc6b081e65d9f685adeeb85", + "vault_hits": 6 + } + ], + "lane": "symbolic_logic", + "metrics": { + "all_three_pass_rate": 1.0, + "case_count": 18, + "overall_pass": true, + "premise_recall": 1.0, + "proposal_storage": 1.0, + "replay_determinism": 1.0 + }, + "split": "public", + "timestamp": "2026-05-22T23:22:00.948314+00:00", + "version": "v1" +} diff --git a/tests/test_adr_0122_deferral.py b/tests/test_adr_0122_deferral.py new file mode 100644 index 00000000..8d105596 --- /dev/null +++ b/tests/test_adr_0122_deferral.py @@ -0,0 +1,74 @@ +"""ADR-0122 — systems_software audit-passed promotion deferred. + +Pins the deferral invariants: + +1. systems_software stays reasoning-capable with audit_passed=false. +2. The blocker is the named symbolic_logic shape mismatch. +""" + +from __future__ import annotations + +from core.capability.expert_demo import evaluate_expert_demo, materialise_lane_results +from core.capability.reporting import _latest_eval_result, ledger_report +from core.capability.reviewers import ExpertDemoClaim, Reviewer, ReviewerRegistry + + +_SYSTEMS_LANES = ("symbolic_logic", "inference_closure", "fabrication_control") + + +def _fetch(lane: str, split: str) -> dict[str, object]: + payload = _latest_eval_result(lane, "v1", split) + metrics = dict(payload.get("metrics", {}) or {}) + if "by_class" not in metrics and "by_class" in payload: + metrics["by_class"] = payload["by_class"] + return metrics + + +def _systems_row() -> dict: + rows = {row["domain"]: row for row in ledger_report()["domains"]} + return rows["systems_software"] + + +class TestAdr0122Deferral: + def test_systems_software_stays_reasoning_capable(self) -> None: + row = _systems_row() + assert row["status"] == "reasoning-capable" + assert row["predicates"]["reasoning_capable"] is True + assert row["predicates"]["audit_passed"] is False + + def test_named_blocker_matches_gate_reason(self) -> None: + reviewer = Reviewer( + reviewer_id="shay-j", + display_name="Joshua Shay", + role="primary", + domains=("*",), + review_scope=("pack", "proposal", "chain", "eval"), + provenance="adr-0092:bootstrap:2026-05-21", + ) + claim = ExpertDemoClaim( + domain_id="systems_software", + evidence_lanes=_SYSTEMS_LANES, + evidence_revision="adr-0122:reviewed:2026-05-22", + signed_by="shay-j", + claim_digest="0" * 64, + ) + registry = ReviewerRegistry( + schema_version=1, + reviewers=(reviewer,), + expert_demo_claims=(claim,), + ) + lane_results = materialise_lane_results(_SYSTEMS_LANES, fetch_split=_fetch) + + verdict = evaluate_expert_demo( + domain_id="systems_software", + reasoning_capable=True, + registry=registry, + domain_lanes=_SYSTEMS_LANES, + lane_results=lane_results, + ) + + assert verdict.passed is False + assert ( + verdict.reason + == "lane 'symbolic_logic' missing accuracy (and no passed/total fallback) (split=public)" + )