From 51c6852b0cb0b609e10d8d0fe7286a45bc092bd6 Mon Sep 17 00:00:00 2001 From: Shay Date: Fri, 5 Jun 2026 09:08:23 -0700 Subject: [PATCH] test(l10): add independent-gold adversarial logic fixtures --- evals/deductive_logic/fixtures/l10_adversarial.jsonl | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 evals/deductive_logic/fixtures/l10_adversarial.jsonl diff --git a/evals/deductive_logic/fixtures/l10_adversarial.jsonl b/evals/deductive_logic/fixtures/l10_adversarial.jsonl new file mode 100644 index 00000000..d604fc70 --- /dev/null +++ b/evals/deductive_logic/fixtures/l10_adversarial.jsonl @@ -0,0 +1,8 @@ +{"case_id":"l10-adv-001-vacuous-universal-no-witness","category":"vacuous_truth","finite_domain":["raven_a","raven_b"],"gold":{"epistemic_state":"UNDETERMINED","label":"refuse","reason":"Universal over an empty guarded range does not ground silver(raven_a).","wrong_value":0},"premises":[{"guard":{"args":["x"],"pred":"unicorn"},"implies":{"args":["x"],"pred":"silver"},"op":"forall_guarded","var":"x"},{"args":["raven_a"],"op":"closed_world_negative","pred":"unicorn"},{"args":["raven_b"],"op":"closed_world_negative","pred":"unicorn"}],"query":{"args":["raven_a"],"pred":"silver"},"symbolic_resolution":["range(unicorn) = \u2205 under finite_domain","forall_guarded is vacuously satisfied","query subject lacks explicit unicorn witness","no admissible entailment; refuse"]} +{"case_id":"l10-adv-002-contradiction-no-explosion","category":"contradictory_premises","finite_domain":["invoice_1"],"gold":{"epistemic_state":"CONTRADICTED","label":"refuse","reason":"The same entity is asserted paid and not_paid; no explosion from inconsistent premises.","wrong_value":0},"premises":[{"args":["invoice_1"],"op":"fact","pred":"paid"},{"args":["invoice_1"],"op":"fact","pred":"not_paid"},{"guard":{"args":["x"],"pred":"paid"},"implies":{"args":["x"],"pred":"has_receipt"},"op":"forall_guarded","var":"x"}],"query":{"args":["invoice_1"],"pred":"has_receipt"},"symbolic_resolution":["detect paid(invoice_1) \u2227 not_paid(invoice_1)","mark premise set inconsistent","block ex-falso entailment","refuse"]} +{"case_id":"l10-adv-003-range-restriction-blocks-link","category":"range_restriction","finite_domain":["alice","bob"],"gold":{"epistemic_state":"VERIFIED","label":"false","reason":"Alice is grounded in team_red only; the team_blue guard is not satisfied.","wrong_value":0},"premises":[{"args":["alice","team_red"],"op":"fact","pred":"member"},{"guard":{"args":["x","team_blue"],"pred":"member"},"implies":{"args":["x","vault_blue"],"pred":"can_access"},"op":"forall_guarded","var":"x"}],"query":{"args":["alice","vault_blue"],"pred":"can_access"},"symbolic_resolution":["member(alice, team_blue) absent from finite facts","guard fails for alice","implication cannot fire","query is false under finite closure"]} +{"case_id":"l10-adv-004-grounded-universal-entails","category":"grounded_entailment","finite_domain":["sam"],"gold":{"epistemic_state":"VERIFIED","label":"true","reason":"Sam is explicitly grounded and satisfies the guarded universal range.","wrong_value":0},"premises":[{"args":["sam"],"op":"fact","pred":"finite_entity"},{"args":["sam"],"op":"fact","pred":"student"},{"guard":{"args":["x"],"pred":"student"},"implies":{"args":["x"],"pred":"has_id"},"op":"forall_guarded","var":"x"}],"query":{"args":["sam"],"pred":"has_id"},"symbolic_resolution":["finite_entity(sam)","student(sam) satisfies guard","forall_guarded fires","entails has_id(sam)"]} +{"case_id":"l10-adv-005-existential-without-witness","category":"existential_grounding","finite_domain":["box_1","box_2"],"gold":{"epistemic_state":"UNDETERMINED","label":"refuse","reason":"An unbound existential does not identify box_1 as the witness.","wrong_value":0},"premises":[{"body":{"args":["x"],"pred":"contains_key"},"op":"exists_unbound","var":"x"}],"query":{"args":["box_1"],"pred":"contains_key"},"symbolic_resolution":["exists x contains_key(x) has no witness binding","box_1 is not independently bound","query would require hallucinated witness link","refuse"]} +{"case_id":"l10-adv-006-contradictory-rule-heads","category":"contradictory_premises","finite_domain":["node_a"],"gold":{"epistemic_state":"CONTRADICTED","label":"refuse","reason":"The same grounded guard entails incompatible heads; runtime must not choose one.","wrong_value":0},"premises":[{"args":["node_a"],"op":"fact","pred":"sensor_green"},{"guard":{"args":["x"],"pred":"sensor_green"},"implies":{"args":["x"],"pred":"safe"},"op":"forall_guarded","var":"x"},{"guard":{"args":["x"],"pred":"sensor_green"},"implies":{"args":["x"],"pred":"unsafe"},"op":"forall_guarded","var":"x"}],"query":{"args":["node_a"],"pred":"safe"},"symbolic_resolution":["sensor_green(node_a) grounds both rules","derive safe(node_a) and unsafe(node_a)","detect incompatible heads","refuse"]} +{"case_id":"l10-adv-007-negated-ground-false","category":"finite_closure","finite_domain":["door_1"],"gold":{"epistemic_state":"VERIFIED","label":"false","reason":"locked and unlocked are declared incompatible for the same grounded entity.","wrong_value":0},"premises":[{"args":["door_1"],"op":"fact","pred":"finite_entity"},{"args":["door_1"],"op":"fact","pred":"locked"}],"query":{"args":["door_1"],"pred":"unlocked"},"symbolic_resolution":["finite_entity(door_1)","locked(door_1)","incompatible(locked, unlocked)","unlocked(door_1) is false"]} +{"case_id":"l10-adv-008-out-of-domain-entity","category":"range_restriction","finite_domain":["asset_1"],"gold":{"epistemic_state":"SCOPE_BOUNDARY","label":"refuse","reason":"asset_1 is grounded, but not in the account range required by the rule.","wrong_value":0},"premises":[{"args":["asset_1"],"op":"fact","pred":"asset"},{"guard":{"args":["x"],"pred":"account"},"implies":{"args":["x"],"pred":"reconciled"},"op":"forall_guarded","var":"x"}],"query":{"args":["asset_1"],"pred":"reconciled"},"symbolic_resolution":["asset(asset_1) only","account(asset_1) absent","range-restricted rule cannot fire","refuse instead of hallucinating account(asset_1)"]}