diff --git a/evals/relational_operator_ablation/__init__.py b/evals/relational_operator_ablation/__init__.py new file mode 100644 index 00000000..4a6a5613 --- /dev/null +++ b/evals/relational_operator_ablation/__init__.py @@ -0,0 +1 @@ +"""Deterministic Relational Operator Ablation eval lane (fraction_decrease).""" diff --git a/evals/relational_operator_ablation/v1/__init__.py b/evals/relational_operator_ablation/v1/__init__.py new file mode 100644 index 00000000..03fe0361 --- /dev/null +++ b/evals/relational_operator_ablation/v1/__init__.py @@ -0,0 +1 @@ +"""Sealed v1 fixture + runner for relational operator ablation.""" diff --git a/evals/relational_operator_ablation/v1/cases.jsonl b/evals/relational_operator_ablation/v1/cases.jsonl new file mode 100644 index 00000000..5e5b56ad --- /dev/null +++ b/evals/relational_operator_ablation/v1/cases.jsonl @@ -0,0 +1,8 @@ +{"case_id": "roa-v1-0001", "problem": "In one hour, Addison mountain's temperature will decrease to 3/4 of its temperature. If the current temperature of the mountain is 84 degrees, what will the temperature decrease by?", "expected_answer": 21.0, "expected_unit": "degrees", "expected_outcome": "correct", "tags": ["fraction_decrease", "gold", "train_sample_0005"], "notes": "Canonical decrease-to-fraction delta; gold=21."} +{"case_id": "roa-v1-0002", "problem": "In two hours, Cedar peak's temperature will decrease to 2/3 of its temperature. If the current temperature of the peak is 60 degrees, what will the temperature decrease by?", "expected_answer": 20.0, "expected_unit": "degrees", "expected_outcome": "correct", "tags": ["fraction_decrease", "gold", "sibling"], "notes": "Sibling shape: 60*(1-2/3)=20."} +{"case_id": "roa-v1-0003", "problem": "In one hour, the lake's temperature will decrease to 3/4 of its temperature. If the current temperature of the lake is 80 degrees, what will the temperature be?", "expected_answer": null, "expected_unit": "", "expected_outcome": "refused", "tags": ["adversarial", "final_value", "invalid"], "notes": "Final-value question must refuse delta organ (not guess 60)."} +{"case_id": "roa-v1-0004", "problem": "Yun had 20 paperclips initially, but then lost 12. Marion has 1/4 more than what Yun currently has, plus 7. How many paperclips does Marion have?", "expected_answer": null, "expected_unit": "", "expected_outcome": "refused", "tags": ["adversarial", "affine_confuser", "invalid"], "notes": "Percent/fraction affine confuser; fraction_decrease must refuse."} +{"case_id": "roa-v1-0005", "problem": "A school has 100 students. Half of the students are girls, the other half are boys. 20% of the girls have dogs at home and 10% of the boys have dogs at home. How many students own dogs?", "expected_answer": null, "expected_unit": "", "expected_outcome": "refused", "tags": ["adversarial", "percent_partition", "invalid"], "notes": "Percent partition is out of family scope for this ablation; must refuse on fraction_decrease conditions."} +{"case_id": "roa-v1-0006", "problem": "In one hour, Addison mountain's temperature will decrease to 3/4 of its temperature. If the current temperature of the mountain is 84 degrees and the cabin is 40 degrees, what will the temperature decrease by?", "expected_answer": null, "expected_unit": "", "expected_outcome": "refused", "tags": ["adversarial", "distractor_quantity", "invalid"], "notes": "Multiple nearby temperatures; organ must refuse rather than pick one."} +{"case_id": "roa-v1-0007", "problem": "The mountain will decrease by 3/4 of its temperature. Current temperature is 84 degrees. What will the temperature decrease by?", "expected_answer": null, "expected_unit": "", "expected_outcome": "refused", "tags": ["adversarial", "percent_change_vs_of", "invalid"], "notes": "decrease by fraction language (not decrease to); refuse."} +{"case_id": "roa-v1-0008", "problem": "In one hour, Addison mountain's temperature will decrease to 3/4 of its temperature and then to 1/2 of its temperature. If the current temperature of the mountain is 84 degrees, what will the temperature decrease by?", "expected_answer": null, "expected_unit": "", "expected_outcome": "refused", "tags": ["adversarial", "multi_fraction_hazard", "invalid"], "notes": "Multiple slash-fractions hazard; organ must refuse rather than pick one."} diff --git a/evals/relational_operator_ablation/v1/report.json b/evals/relational_operator_ablation/v1/report.json new file mode 100644 index 00000000..8c0c6d95 --- /dev/null +++ b/evals/relational_operator_ablation/v1/report.json @@ -0,0 +1,1727 @@ +{ + "conditions": [ + "baseline", + "operator", + "depth", + "metadata_only", + "invalid" + ], + "counts": { + "baseline": { + "correct": 2, + "correct_rate": 0.25, + "coverage": 2, + "coverage_rate": 0.25, + "n": 8, + "refused": 6, + "refused_rate": 0.75, + "wrong": 0, + "wrong_rate": 0.0 + }, + "depth": { + "correct": 2, + "correct_rate": 0.25, + "coverage": 2, + "coverage_rate": 0.25, + "n": 8, + "refused": 6, + "refused_rate": 0.75, + "wrong": 0, + "wrong_rate": 0.0 + }, + "invalid": { + "correct": 0, + "correct_rate": 0.0, + "coverage": 0, + "coverage_rate": 0.0, + "n": 6, + "refused": 6, + "refused_rate": 1.0, + "wrong": 0, + "wrong_rate": 0.0 + }, + "metadata_only": { + "correct": 2, + "correct_rate": 0.25, + "coverage": 2, + "coverage_rate": 0.25, + "n": 8, + "refused": 6, + "refused_rate": 0.75, + "wrong": 0, + "wrong_rate": 0.0 + }, + "operator": { + "correct": 2, + "correct_rate": 0.25, + "coverage": 2, + "coverage_rate": 0.25, + "n": 8, + "refused": 6, + "refused_rate": 0.75, + "wrong": 0, + "wrong_rate": 0.0 + } + }, + "determinism": [ + { + "case_id": "roa-v1-0001", + "operator_repeat_identical": true + }, + { + "case_id": "roa-v1-0002", + "operator_repeat_identical": true + }, + { + "case_id": "roa-v1-0003", + "operator_repeat_identical": true + }, + { + "case_id": "roa-v1-0004", + "operator_repeat_identical": true + }, + { + "case_id": "roa-v1-0005", + "operator_repeat_identical": true + }, + { + "case_id": "roa-v1-0006", + "operator_repeat_identical": true + }, + { + "case_id": "roa-v1-0007", + "operator_repeat_identical": true + }, + { + "case_id": "roa-v1-0008", + "operator_repeat_identical": true + } + ], + "family": "proportional_change.decrease_to_fraction", + "findings": { + "baseline_wrong_zero": true, + "depth_executable_inactive_equals_operator": true, + "metadata_only_inert": true, + "operator_deterministic": true, + "operator_solves_gold_fraction_cases": true, + "operator_wrong_zero": true, + "scientific_note": "Depth executable mapping is intentionally inactive on English-only inputs (anti-circularity). Metadata-only he-root labels must not change answers. Operator geometric path and baseline scalar path should agree when both commit." + }, + "inert_checks": [ + { + "baseline_eq_operator_when_both_commit": true, + "case_id": "roa-v1-0001", + "metadata_has_root_note_when_runnable": true, + "operator_eq_depth": true, + "operator_eq_metadata": true + }, + { + "baseline_eq_operator_when_both_commit": true, + "case_id": "roa-v1-0002", + "metadata_has_root_note_when_runnable": true, + "operator_eq_depth": true, + "operator_eq_metadata": true + }, + { + "baseline_eq_operator_when_both_commit": null, + "case_id": "roa-v1-0003", + "metadata_has_root_note_when_runnable": null, + "operator_eq_depth": true, + "operator_eq_metadata": true + }, + { + "baseline_eq_operator_when_both_commit": null, + "case_id": "roa-v1-0004", + "metadata_has_root_note_when_runnable": null, + "operator_eq_depth": true, + "operator_eq_metadata": true + }, + { + "baseline_eq_operator_when_both_commit": null, + "case_id": "roa-v1-0005", + "metadata_has_root_note_when_runnable": null, + "operator_eq_depth": true, + "operator_eq_metadata": true + }, + { + "baseline_eq_operator_when_both_commit": null, + "case_id": "roa-v1-0006", + "metadata_has_root_note_when_runnable": null, + "operator_eq_depth": true, + "operator_eq_metadata": true + }, + { + "baseline_eq_operator_when_both_commit": null, + "case_id": "roa-v1-0007", + "metadata_has_root_note_when_runnable": null, + "operator_eq_depth": true, + "operator_eq_metadata": true + }, + { + "baseline_eq_operator_when_both_commit": null, + "case_id": "roa-v1-0008", + "metadata_has_root_note_when_runnable": true, + "operator_eq_depth": true, + "operator_eq_metadata": true + } + ], + "lane": "relational_operator_ablation/v1", + "limitations": [ + "n=8 sealed cases; not a full GSM8K claim.", + "Depth-as-executable is unproven for English arithmetic; recorded inactive.", + "percent_partition dual-track is out of family scope for this slice.", + "Chat CognitiveTurnPipeline still passes contract_assessment=None." + ], + "organ": "fraction_decrease", + "per_case": [ + { + "baseline": { + "answer": 21.0, + "answer_unit": "degrees", + "case_id": "roa-v1-0001", + "condition": "baseline", + "depth": null, + "expected_answer": 21.0, + "explanation_has_root_note": false, + "frame_runnable": true, + "operator_bindings": 0, + "outcome": "correct", + "readback": { + "base_surface": "mention-0003", + "explanation": "all fraction-decrease roles and delta target obligations are grounded", + "has_versor_binding": true, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [ + [ + "base_quantity", + "mention-0003" + ], + [ + "scale", + "mention-0001" + ], + [ + "state_entity", + "mention-0000" + ], + [ + "transition", + "span:49:60" + ], + [ + "unit", + "mention-0005" + ] + ], + "scale_surface": "3/4", + "versor_error": 2.220446049250313e-16 + }, + "refusal_reason": "" + }, + "case_id": "roa-v1-0001", + "depth": { + "answer": 21.000000000007148, + "answer_unit": "degrees", + "case_id": "roa-v1-0001", + "condition": "depth", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "depth_executable.fraction_decrease.v0_absent", + "reason": "No authored executable depth\u2192relation mapping for English surface; anti-circularity forbids inventing Greek case / Hebrew binyan from English.", + "source_pack_ids": [], + "source_record_ids": [], + "validity": "inactive_no_mapping" + }, + "expected_answer": 21.0, + "explanation_has_root_note": false, + "frame_runnable": true, + "operator_bindings": 1, + "outcome": "correct", + "readback": { + "base_surface": "mention-0003", + "explanation": "all fraction-decrease roles and delta target obligations are grounded", + "has_versor_binding": true, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [ + [ + "base_quantity", + "mention-0003" + ], + [ + "scale", + "mention-0001" + ], + [ + "state_entity", + "mention-0000" + ], + [ + "transition", + "span:49:60" + ], + [ + "unit", + "mention-0005" + ] + ], + "scale_surface": "3/4", + "versor_error": 2.220446049250313e-16 + }, + "refusal_reason": "" + }, + "invalid": { + "case_id": "roa-v1-0001", + "condition": "invalid", + "note": "invalid condition reserved for refuse-gold/adversarial tags", + "outcome": "skipped_not_adversarial" + }, + "metadata_only": { + "answer": 21.000000000007148, + "answer_unit": "degrees", + "case_id": "roa-v1-0001", + "condition": "metadata_only", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "depth_metadata.enrich_assessments_with_depth.v1", + "reason": "Depth labels decorate ContractAssessment.explanation only; not executable.", + "source_pack_ids": [ + "synthetic_metadata_only" + ], + "source_record_ids": [ + "p0" + ], + "validity": "metadata_only" + }, + "expected_answer": 21.0, + "explanation_has_root_note": true, + "frame_runnable": true, + "operator_bindings": 1, + "outcome": "correct", + "readback": { + "base_surface": "mention-0003", + "explanation": "all fraction-decrease roles and delta target obligations are grounded [root:\u05d0-\u05de-\u05e0]", + "has_versor_binding": true, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [ + [ + "base_quantity", + "mention-0003" + ], + [ + "scale", + "mention-0001" + ], + [ + "state_entity", + "mention-0000" + ], + [ + "transition", + "span:49:60" + ], + [ + "unit", + "mention-0005" + ] + ], + "scale_surface": "3/4", + "versor_error": 2.220446049250313e-16 + }, + "refusal_reason": "" + }, + "operator": { + "answer": 21.000000000007148, + "answer_unit": "degrees", + "case_id": "roa-v1-0001", + "condition": "operator", + "depth": null, + "expected_answer": 21.0, + "explanation_has_root_note": false, + "frame_runnable": true, + "operator_bindings": 1, + "outcome": "correct", + "readback": { + "base_surface": "mention-0003", + "explanation": "all fraction-decrease roles and delta target obligations are grounded", + "has_versor_binding": true, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [ + [ + "base_quantity", + "mention-0003" + ], + [ + "scale", + "mention-0001" + ], + [ + "state_entity", + "mention-0000" + ], + [ + "transition", + "span:49:60" + ], + [ + "unit", + "mention-0005" + ] + ], + "scale_surface": "3/4", + "versor_error": 2.220446049250313e-16 + }, + "refusal_reason": "" + }, + "tags": [ + "fraction_decrease", + "gold", + "train_sample_0005" + ] + }, + { + "baseline": { + "answer": 20.0, + "answer_unit": "degrees", + "case_id": "roa-v1-0002", + "condition": "baseline", + "depth": null, + "expected_answer": 20.0, + "explanation_has_root_note": false, + "frame_runnable": true, + "operator_bindings": 0, + "outcome": "correct", + "readback": { + "base_surface": "mention-0003", + "explanation": "all fraction-decrease roles and delta target obligations are grounded", + "has_versor_binding": true, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [ + [ + "base_quantity", + "mention-0003" + ], + [ + "scale", + "mention-0001" + ], + [ + "state_entity", + "mention-0000" + ], + [ + "transition", + "span:44:55" + ], + [ + "unit", + "mention-0005" + ] + ], + "scale_surface": "2/3", + "versor_error": 0.0 + }, + "refusal_reason": "" + }, + "case_id": "roa-v1-0002", + "depth": { + "answer": 20.00000000000606, + "answer_unit": "degrees", + "case_id": "roa-v1-0002", + "condition": "depth", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "depth_executable.fraction_decrease.v0_absent", + "reason": "No authored executable depth\u2192relation mapping for English surface; anti-circularity forbids inventing Greek case / Hebrew binyan from English.", + "source_pack_ids": [], + "source_record_ids": [], + "validity": "inactive_no_mapping" + }, + "expected_answer": 20.0, + "explanation_has_root_note": false, + "frame_runnable": true, + "operator_bindings": 1, + "outcome": "correct", + "readback": { + "base_surface": "mention-0003", + "explanation": "all fraction-decrease roles and delta target obligations are grounded", + "has_versor_binding": true, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [ + [ + "base_quantity", + "mention-0003" + ], + [ + "scale", + "mention-0001" + ], + [ + "state_entity", + "mention-0000" + ], + [ + "transition", + "span:44:55" + ], + [ + "unit", + "mention-0005" + ] + ], + "scale_surface": "2/3", + "versor_error": 0.0 + }, + "refusal_reason": "" + }, + "invalid": { + "case_id": "roa-v1-0002", + "condition": "invalid", + "note": "invalid condition reserved for refuse-gold/adversarial tags", + "outcome": "skipped_not_adversarial" + }, + "metadata_only": { + "answer": 20.00000000000606, + "answer_unit": "degrees", + "case_id": "roa-v1-0002", + "condition": "metadata_only", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "depth_metadata.enrich_assessments_with_depth.v1", + "reason": "Depth labels decorate ContractAssessment.explanation only; not executable.", + "source_pack_ids": [ + "synthetic_metadata_only" + ], + "source_record_ids": [ + "p0" + ], + "validity": "metadata_only" + }, + "expected_answer": 20.0, + "explanation_has_root_note": true, + "frame_runnable": true, + "operator_bindings": 1, + "outcome": "correct", + "readback": { + "base_surface": "mention-0003", + "explanation": "all fraction-decrease roles and delta target obligations are grounded [root:\u05d0-\u05de-\u05e0]", + "has_versor_binding": true, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [ + [ + "base_quantity", + "mention-0003" + ], + [ + "scale", + "mention-0001" + ], + [ + "state_entity", + "mention-0000" + ], + [ + "transition", + "span:44:55" + ], + [ + "unit", + "mention-0005" + ] + ], + "scale_surface": "2/3", + "versor_error": 0.0 + }, + "refusal_reason": "" + }, + "operator": { + "answer": 20.00000000000606, + "answer_unit": "degrees", + "case_id": "roa-v1-0002", + "condition": "operator", + "depth": null, + "expected_answer": 20.0, + "explanation_has_root_note": false, + "frame_runnable": true, + "operator_bindings": 1, + "outcome": "correct", + "readback": { + "base_surface": "mention-0003", + "explanation": "all fraction-decrease roles and delta target obligations are grounded", + "has_versor_binding": true, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [ + [ + "base_quantity", + "mention-0003" + ], + [ + "scale", + "mention-0001" + ], + [ + "state_entity", + "mention-0000" + ], + [ + "transition", + "span:44:55" + ], + [ + "unit", + "mention-0005" + ] + ], + "scale_surface": "2/3", + "versor_error": 0.0 + }, + "refusal_reason": "" + }, + "tags": [ + "fraction_decrease", + "gold", + "sibling" + ] + }, + { + "baseline": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0003", + "condition": "baseline", + "depth": null, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": "mention-0003", + "explanation": "diagnostic candidate is not runnable: delta_decrease_target_unbound", + "has_versor_binding": false, + "missing_bindings": [ + "delta_decrease_target_unbound" + ], + "relation_type": "decrease_to_fraction", + "roles": [ + [ + "base_quantity", + "mention-0003" + ], + [ + "scale", + "mention-0001" + ], + [ + "state_entity", + "mention-0000" + ], + [ + "transition", + "span:41:52" + ], + [ + "unit", + "mention-0005" + ] + ], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "diagnostic candidate is not runnable: delta_decrease_target_unbound" + }, + "case_id": "roa-v1-0003", + "depth": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0003", + "condition": "depth", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "depth_executable.fraction_decrease.v0_absent", + "reason": "No authored executable depth\u2192relation mapping for English surface; anti-circularity forbids inventing Greek case / Hebrew binyan from English.", + "source_pack_ids": [], + "source_record_ids": [], + "validity": "inactive_no_mapping" + }, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 1, + "outcome": "refused", + "readback": { + "base_surface": "mention-0003", + "explanation": "Assessed via geometric algebra from frame-grounded scale", + "has_versor_binding": true, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [ + [ + "base_quantity", + "mention-0003" + ], + [ + "scale", + "mention-0001" + ], + [ + "state_entity", + "mention-0000" + ], + [ + "transition", + "span:41:52" + ], + [ + "unit", + "mention-0005" + ] + ], + "scale_surface": "3/4", + "versor_error": 2.220446049250313e-16 + }, + "refusal_reason": "diagnostic candidate is not runnable: delta_decrease_target_unbound" + }, + "invalid": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0003", + "condition": "invalid", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "invalid.control", + "reason": "Adversarial/invalid control case; refuse preferred over wrong.", + "source_pack_ids": [], + "source_record_ids": [], + "validity": "invalid" + }, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 1, + "outcome": "refused", + "readback": { + "base_surface": "mention-0003", + "explanation": "Assessed via geometric algebra from frame-grounded scale", + "has_versor_binding": true, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [ + [ + "base_quantity", + "mention-0003" + ], + [ + "scale", + "mention-0001" + ], + [ + "state_entity", + "mention-0000" + ], + [ + "transition", + "span:41:52" + ], + [ + "unit", + "mention-0005" + ] + ], + "scale_surface": "3/4", + "versor_error": 2.220446049250313e-16 + }, + "refusal_reason": "diagnostic candidate is not runnable: delta_decrease_target_unbound" + }, + "metadata_only": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0003", + "condition": "metadata_only", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "depth_metadata.enrich_assessments_with_depth.v1", + "reason": "Depth labels decorate ContractAssessment.explanation only; not executable.", + "source_pack_ids": [ + "synthetic_metadata_only" + ], + "source_record_ids": [ + "p0" + ], + "validity": "metadata_only" + }, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": "mention-0003", + "explanation": "diagnostic candidate is not runnable: delta_decrease_target_unbound", + "has_versor_binding": false, + "missing_bindings": [ + "delta_decrease_target_unbound" + ], + "relation_type": "decrease_to_fraction", + "roles": [ + [ + "base_quantity", + "mention-0003" + ], + [ + "scale", + "mention-0001" + ], + [ + "state_entity", + "mention-0000" + ], + [ + "transition", + "span:41:52" + ], + [ + "unit", + "mention-0005" + ] + ], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "diagnostic candidate is not runnable: delta_decrease_target_unbound" + }, + "operator": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0003", + "condition": "operator", + "depth": null, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 1, + "outcome": "refused", + "readback": { + "base_surface": "mention-0003", + "explanation": "Assessed via geometric algebra from frame-grounded scale", + "has_versor_binding": true, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [ + [ + "base_quantity", + "mention-0003" + ], + [ + "scale", + "mention-0001" + ], + [ + "state_entity", + "mention-0000" + ], + [ + "transition", + "span:41:52" + ], + [ + "unit", + "mention-0005" + ] + ], + "scale_surface": "3/4", + "versor_error": 2.220446049250313e-16 + }, + "refusal_reason": "diagnostic candidate is not runnable: delta_decrease_target_unbound" + }, + "tags": [ + "adversarial", + "final_value", + "invalid" + ] + }, + { + "baseline": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0004", + "condition": "baseline", + "depth": null, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": null, + "explanation": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, delta_decrease_target_required, unbound_base_quantity", + "has_versor_binding": false, + "missing_bindings": [ + "decrease_relation_ambiguous", + "base_quantity_unbound", + "scale_unbound", + "state_entity_unbound", + "delta_decrease_target_required" + ], + "relation_type": "decrease_to_fraction", + "roles": [], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, delta_decrease_target_required, unbound_base_quantity" + }, + "case_id": "roa-v1-0004", + "depth": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0004", + "condition": "depth", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "depth_executable.fraction_decrease.v0_absent", + "reason": "No authored executable depth\u2192relation mapping for English surface; anti-circularity forbids inventing Greek case / Hebrew binyan from English.", + "source_pack_ids": [], + "source_record_ids": [], + "validity": "inactive_no_mapping" + }, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": null, + "explanation": "no_fraction_decrease_assessment", + "has_versor_binding": false, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, delta_decrease_target_required, unbound_base_quantity" + }, + "invalid": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0004", + "condition": "invalid", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "invalid.control", + "reason": "Adversarial/invalid control case; refuse preferred over wrong.", + "source_pack_ids": [], + "source_record_ids": [], + "validity": "invalid" + }, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": null, + "explanation": "no_fraction_decrease_assessment", + "has_versor_binding": false, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, delta_decrease_target_required, unbound_base_quantity" + }, + "metadata_only": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0004", + "condition": "metadata_only", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "depth_metadata.enrich_assessments_with_depth.v1", + "reason": "Depth labels decorate ContractAssessment.explanation only; not executable.", + "source_pack_ids": [ + "synthetic_metadata_only" + ], + "source_record_ids": [ + "p0" + ], + "validity": "metadata_only" + }, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": null, + "explanation": "no_fraction_decrease_in_assess_contracts", + "has_versor_binding": false, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "operator_refused" + }, + "operator": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0004", + "condition": "operator", + "depth": null, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": null, + "explanation": "no_fraction_decrease_assessment", + "has_versor_binding": false, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, delta_decrease_target_required, unbound_base_quantity" + }, + "tags": [ + "adversarial", + "affine_confuser", + "invalid" + ] + }, + { + "baseline": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0005", + "condition": "baseline", + "depth": null, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": null, + "explanation": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, delta_decrease_target_required, unbound_base_quantity", + "has_versor_binding": false, + "missing_bindings": [ + "decrease_relation_ambiguous", + "base_quantity_unbound", + "scale_unbound", + "state_entity_unbound", + "delta_decrease_target_required" + ], + "relation_type": "decrease_to_fraction", + "roles": [], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, delta_decrease_target_required, unbound_base_quantity" + }, + "case_id": "roa-v1-0005", + "depth": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0005", + "condition": "depth", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "depth_executable.fraction_decrease.v0_absent", + "reason": "No authored executable depth\u2192relation mapping for English surface; anti-circularity forbids inventing Greek case / Hebrew binyan from English.", + "source_pack_ids": [], + "source_record_ids": [], + "validity": "inactive_no_mapping" + }, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": null, + "explanation": "no_fraction_decrease_assessment", + "has_versor_binding": false, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, delta_decrease_target_required, unbound_base_quantity" + }, + "invalid": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0005", + "condition": "invalid", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "invalid.control", + "reason": "Adversarial/invalid control case; refuse preferred over wrong.", + "source_pack_ids": [], + "source_record_ids": [], + "validity": "invalid" + }, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": null, + "explanation": "no_fraction_decrease_assessment", + "has_versor_binding": false, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, delta_decrease_target_required, unbound_base_quantity" + }, + "metadata_only": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0005", + "condition": "metadata_only", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "depth_metadata.enrich_assessments_with_depth.v1", + "reason": "Depth labels decorate ContractAssessment.explanation only; not executable.", + "source_pack_ids": [ + "synthetic_metadata_only" + ], + "source_record_ids": [ + "p0" + ], + "validity": "metadata_only" + }, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": null, + "explanation": "no_fraction_decrease_in_assess_contracts", + "has_versor_binding": false, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "operator_refused" + }, + "operator": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0005", + "condition": "operator", + "depth": null, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": null, + "explanation": "no_fraction_decrease_assessment", + "has_versor_binding": false, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, delta_decrease_target_required, unbound_base_quantity" + }, + "tags": [ + "adversarial", + "percent_partition", + "invalid" + ] + }, + { + "baseline": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0006", + "condition": "baseline", + "depth": null, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": null, + "explanation": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, unbound_base_quantity", + "has_versor_binding": false, + "missing_bindings": [ + "decrease_relation_ambiguous", + "base_quantity_unbound", + "scale_unbound", + "state_entity_unbound" + ], + "relation_type": "decrease_to_fraction", + "roles": [], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, unbound_base_quantity" + }, + "case_id": "roa-v1-0006", + "depth": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0006", + "condition": "depth", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "depth_executable.fraction_decrease.v0_absent", + "reason": "No authored executable depth\u2192relation mapping for English surface; anti-circularity forbids inventing Greek case / Hebrew binyan from English.", + "source_pack_ids": [], + "source_record_ids": [], + "validity": "inactive_no_mapping" + }, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": null, + "explanation": "Assessed via geometric algebra from frame-grounded scale", + "has_versor_binding": false, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, unbound_base_quantity" + }, + "invalid": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0006", + "condition": "invalid", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "invalid.control", + "reason": "Adversarial/invalid control case; refuse preferred over wrong.", + "source_pack_ids": [], + "source_record_ids": [], + "validity": "invalid" + }, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": null, + "explanation": "Assessed via geometric algebra from frame-grounded scale", + "has_versor_binding": false, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, unbound_base_quantity" + }, + "metadata_only": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0006", + "condition": "metadata_only", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "depth_metadata.enrich_assessments_with_depth.v1", + "reason": "Depth labels decorate ContractAssessment.explanation only; not executable.", + "source_pack_ids": [ + "synthetic_metadata_only" + ], + "source_record_ids": [ + "p0" + ], + "validity": "metadata_only" + }, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": null, + "explanation": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, unbound_base_quantity", + "has_versor_binding": false, + "missing_bindings": [ + "decrease_relation_ambiguous", + "base_quantity_unbound", + "scale_unbound", + "state_entity_unbound" + ], + "relation_type": "decrease_to_fraction", + "roles": [], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, unbound_base_quantity" + }, + "operator": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0006", + "condition": "operator", + "depth": null, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": null, + "explanation": "Assessed via geometric algebra from frame-grounded scale", + "has_versor_binding": false, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, unbound_base_quantity" + }, + "tags": [ + "adversarial", + "distractor_quantity", + "invalid" + ] + }, + { + "baseline": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0007", + "condition": "baseline", + "depth": null, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": null, + "explanation": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, delta_decrease_target_unbound, unbound_base_quantity", + "has_versor_binding": false, + "missing_bindings": [ + "decrease_relation_ambiguous", + "base_quantity_unbound", + "scale_unbound", + "state_entity_unbound", + "delta_decrease_target_unbound" + ], + "relation_type": "decrease_to_fraction", + "roles": [], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, delta_decrease_target_unbound, unbound_base_quantity" + }, + "case_id": "roa-v1-0007", + "depth": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0007", + "condition": "depth", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "depth_executable.fraction_decrease.v0_absent", + "reason": "No authored executable depth\u2192relation mapping for English surface; anti-circularity forbids inventing Greek case / Hebrew binyan from English.", + "source_pack_ids": [], + "source_record_ids": [], + "validity": "inactive_no_mapping" + }, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": null, + "explanation": "no_fraction_decrease_assessment", + "has_versor_binding": false, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, delta_decrease_target_unbound, unbound_base_quantity" + }, + "invalid": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0007", + "condition": "invalid", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "invalid.control", + "reason": "Adversarial/invalid control case; refuse preferred over wrong.", + "source_pack_ids": [], + "source_record_ids": [], + "validity": "invalid" + }, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": null, + "explanation": "no_fraction_decrease_assessment", + "has_versor_binding": false, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, delta_decrease_target_unbound, unbound_base_quantity" + }, + "metadata_only": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0007", + "condition": "metadata_only", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "depth_metadata.enrich_assessments_with_depth.v1", + "reason": "Depth labels decorate ContractAssessment.explanation only; not executable.", + "source_pack_ids": [ + "synthetic_metadata_only" + ], + "source_record_ids": [ + "p0" + ], + "validity": "metadata_only" + }, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": null, + "explanation": "no_fraction_decrease_in_assess_contracts", + "has_versor_binding": false, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "operator_refused" + }, + "operator": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0007", + "condition": "operator", + "depth": null, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": false, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": null, + "explanation": "no_fraction_decrease_assessment", + "has_versor_binding": false, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [], + "scale_surface": null, + "versor_error": null + }, + "refusal_reason": "diagnostic candidate is not runnable: decrease_relation_ambiguous, base_quantity_unbound, scale_unbound, state_entity_unbound, delta_decrease_target_unbound, unbound_base_quantity" + }, + "tags": [ + "adversarial", + "percent_change_vs_of", + "invalid" + ] + }, + { + "baseline": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0008", + "condition": "baseline", + "depth": null, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": true, + "operator_bindings": 0, + "outcome": "refused", + "readback": { + "base_surface": "mention-0005", + "explanation": "all fraction-decrease roles and delta target obligations are grounded", + "has_versor_binding": true, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [ + [ + "base_quantity", + "mention-0005" + ], + [ + "scale", + "mention-0001" + ], + [ + "state_entity", + "mention-0000" + ], + [ + "transition", + "span:49:60" + ], + [ + "unit", + "mention-0007" + ] + ], + "scale_surface": "3/4", + "versor_error": 2.220446049250313e-16 + }, + "refusal_reason": "baseline_hazard_surface" + }, + "case_id": "roa-v1-0008", + "depth": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0008", + "condition": "depth", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "depth_executable.fraction_decrease.v0_absent", + "reason": "No authored executable depth\u2192relation mapping for English surface; anti-circularity forbids inventing Greek case / Hebrew binyan from English.", + "source_pack_ids": [], + "source_record_ids": [], + "validity": "inactive_no_mapping" + }, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": true, + "operator_bindings": 1, + "outcome": "refused", + "readback": { + "base_surface": "mention-0005", + "explanation": "all fraction-decrease roles and delta target obligations are grounded", + "has_versor_binding": true, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [ + [ + "base_quantity", + "mention-0005" + ], + [ + "scale", + "mention-0001" + ], + [ + "state_entity", + "mention-0000" + ], + [ + "transition", + "span:49:60" + ], + [ + "unit", + "mention-0007" + ] + ], + "scale_surface": "3/4", + "versor_error": 2.220446049250313e-16 + }, + "refusal_reason": "operator_refused" + }, + "invalid": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0008", + "condition": "invalid", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "invalid.control", + "reason": "Adversarial/invalid control case; refuse preferred over wrong.", + "source_pack_ids": [], + "source_record_ids": [], + "validity": "invalid" + }, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": true, + "operator_bindings": 1, + "outcome": "refused", + "readback": { + "base_surface": "mention-0005", + "explanation": "all fraction-decrease roles and delta target obligations are grounded", + "has_versor_binding": true, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [ + [ + "base_quantity", + "mention-0005" + ], + [ + "scale", + "mention-0001" + ], + [ + "state_entity", + "mention-0000" + ], + [ + "transition", + "span:49:60" + ], + [ + "unit", + "mention-0007" + ] + ], + "scale_surface": "3/4", + "versor_error": 2.220446049250313e-16 + }, + "refusal_reason": "operator_refused" + }, + "metadata_only": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0008", + "condition": "metadata_only", + "depth": { + "evidence_spans": [], + "mapping_rule_id": "depth_metadata.enrich_assessments_with_depth.v1", + "reason": "Depth labels decorate ContractAssessment.explanation only; not executable.", + "source_pack_ids": [ + "synthetic_metadata_only" + ], + "source_record_ids": [ + "p0" + ], + "validity": "metadata_only" + }, + "expected_answer": null, + "explanation_has_root_note": true, + "frame_runnable": true, + "operator_bindings": 1, + "outcome": "refused", + "readback": { + "base_surface": "mention-0005", + "explanation": "all fraction-decrease roles and delta target obligations are grounded [root:\u05d0-\u05de-\u05e0]", + "has_versor_binding": true, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [ + [ + "base_quantity", + "mention-0005" + ], + [ + "scale", + "mention-0001" + ], + [ + "state_entity", + "mention-0000" + ], + [ + "transition", + "span:49:60" + ], + [ + "unit", + "mention-0007" + ] + ], + "scale_surface": "3/4", + "versor_error": 2.220446049250313e-16 + }, + "refusal_reason": "all fraction-decrease roles and delta target obligations are grounded [root:\u05d0-\u05de-\u05e0]" + }, + "operator": { + "answer": null, + "answer_unit": null, + "case_id": "roa-v1-0008", + "condition": "operator", + "depth": null, + "expected_answer": null, + "explanation_has_root_note": false, + "frame_runnable": true, + "operator_bindings": 1, + "outcome": "refused", + "readback": { + "base_surface": "mention-0005", + "explanation": "all fraction-decrease roles and delta target obligations are grounded", + "has_versor_binding": true, + "missing_bindings": [], + "relation_type": "decrease_to_fraction", + "roles": [ + [ + "base_quantity", + "mention-0005" + ], + [ + "scale", + "mention-0001" + ], + [ + "state_entity", + "mention-0000" + ], + [ + "transition", + "span:49:60" + ], + [ + "unit", + "mention-0007" + ] + ], + "scale_surface": "3/4", + "versor_error": 2.220446049250313e-16 + }, + "refusal_reason": "operator_refused" + }, + "tags": [ + "adversarial", + "multi_fraction_hazard", + "invalid" + ] + } + ], + "report_sha256": "53df6d17df934f0c44f5d1063b16259f6d9db0b59834b7ca20f895eb5e5719c7", + "sample_size": 8, + "schema_version": 1 +} diff --git a/evals/relational_operator_ablation/v1/runner.py b/evals/relational_operator_ablation/v1/runner.py new file mode 100644 index 00000000..1ef26120 --- /dev/null +++ b/evals/relational_operator_ablation/v1/runner.py @@ -0,0 +1,251 @@ +"""Sealed measurement runner for Deterministic Relational Operator Ablation v1. + +Conditions: baseline | operator | depth | metadata_only | invalid (adversarial cases). + +Run: + PYTHONPATH=. python -m evals.relational_operator_ablation.v1.runner + +Writes ``report.json`` beside this module (override via CORE_ROA_REPORT_PATH). +""" + +from __future__ import annotations + +import hashlib +import json +import os +import sys +from collections import Counter +from pathlib import Path +from typing import Any + +from generate.relational_operator_ablation import ( + AblationCase, + ConditionName, + answers_match, + run_all_conditions, + run_condition, +) + +_HERE = Path(__file__).resolve().parent +_CASES = _HERE / "cases.jsonl" +_REPORT = Path( + os.environ.get("CORE_ROA_REPORT_PATH", str(_HERE / "report.json")) +) +_SCHEMA_VERSION = 1 +_LANE = "relational_operator_ablation/v1" +_CONDITIONS: tuple[ConditionName, ...] = ( + "baseline", + "operator", + "depth", + "metadata_only", + "invalid", +) + + +def _load_cases(path: Path = _CASES) -> list[AblationCase]: + cases: list[AblationCase] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + raw = json.loads(line) + cases.append( + AblationCase( + case_id=raw["case_id"], + problem=raw["problem"], + expected_answer=raw.get("expected_answer"), + expected_unit=raw.get("expected_unit") or "", + expected_outcome=raw["expected_outcome"], + tags=tuple(raw.get("tags") or ()), + notes=raw.get("notes") or "", + ) + ) + return cases + + +def _empty_counts() -> dict[str, int]: + return {"correct": 0, "wrong": 0, "refused": 0, "n": 0, "coverage": 0} + + +def build_report(cases: list[AblationCase]) -> dict[str, Any]: + per_condition: dict[str, dict[str, int]] = { + c: _empty_counts() for c in _CONDITIONS + } + per_case: list[dict[str, Any]] = [] + inert_checks: list[dict[str, Any]] = [] + determinism: list[dict[str, Any]] = [] + + for case in cases: + row: dict[str, Any] = {"case_id": case.case_id, "tags": list(case.tags)} + results = run_all_conditions(case) + + # Always score primary four conditions. + for name in ("baseline", "operator", "depth", "metadata_only"): + result = results[name] + bucket = per_condition[name] + bucket["n"] += 1 + bucket[result.outcome] += 1 + if result.answer is not None: + bucket["coverage"] += 1 + row[name] = result.as_json() + + # Invalid condition only for adversarial/refuse-gold cases. + if "invalid" in results: + result = results["invalid"] + bucket = per_condition["invalid"] + bucket["n"] += 1 + bucket[result.outcome] += 1 + if result.answer is not None: + bucket["coverage"] += 1 + row["invalid"] = result.as_json() + else: + row["invalid"] = { + "condition": "invalid", + "case_id": case.case_id, + "outcome": "skipped_not_adversarial", + "note": "invalid condition reserved for refuse-gold/adversarial tags", + } + + # Metadata-only must match operator answers (inert depth labels). + op = results["operator"] + meta = results["metadata_only"] + depth = results["depth"] + base = results["baseline"] + inert_checks.append( + { + "case_id": case.case_id, + "operator_eq_metadata": answers_match(op, meta), + "operator_eq_depth": answers_match(op, depth), + "baseline_eq_operator_when_both_commit": ( + answers_match(base, op) + if base.answer is not None and op.answer is not None + else None + ), + "metadata_has_root_note_when_runnable": meta.explanation_has_root_note + if meta.frame_runnable + else None, + } + ) + + # Determinism: re-run operator twice. + again = run_condition("operator", case) + determinism.append( + { + "case_id": case.case_id, + "operator_repeat_identical": answers_match(op, again) + and op.refusal_reason == again.refusal_reason, + } + ) + per_case.append(row) + + # Coverage fraction + for name, bucket in per_condition.items(): + n = bucket["n"] or 1 + bucket["coverage_rate"] = bucket["coverage"] / n + bucket["correct_rate"] = bucket["correct"] / n + bucket["wrong_rate"] = bucket["wrong"] / n + bucket["refused_rate"] = bucket["refused"] / n + + inert_ok = all(c["operator_eq_metadata"] for c in inert_checks) + depth_inactive_ok = all(c["operator_eq_depth"] for c in inert_checks) + det_ok = all(d["operator_repeat_identical"] for d in determinism) + no_wrong_on_operator = per_condition["operator"]["wrong"] == 0 + no_wrong_on_baseline = per_condition["baseline"]["wrong"] == 0 + + findings = { + "metadata_only_inert": inert_ok, + "depth_executable_inactive_equals_operator": depth_inactive_ok, + "operator_deterministic": det_ok, + "operator_wrong_zero": no_wrong_on_operator, + "baseline_wrong_zero": no_wrong_on_baseline, + "operator_solves_gold_fraction_cases": per_condition["operator"]["correct"] + >= 2, + "scientific_note": ( + "Depth executable mapping is intentionally inactive on English-only " + "inputs (anti-circularity). Metadata-only he-root labels must not " + "change answers. Operator geometric path and baseline scalar path " + "should agree when both commit." + ), + } + + body = { + "schema_version": _SCHEMA_VERSION, + "lane": _LANE, + "family": "proportional_change.decrease_to_fraction", + "organ": "fraction_decrease", + "sample_size": len(cases), + "conditions": list(_CONDITIONS), + "counts": per_condition, + "findings": findings, + "inert_checks": inert_checks, + "determinism": determinism, + "per_case": per_case, + "limitations": [ + "n=8 sealed cases; not a full GSM8K claim.", + "Depth-as-executable is unproven for English arithmetic; recorded inactive.", + "percent_partition dual-track is out of family scope for this slice.", + "Chat CognitiveTurnPipeline still passes contract_assessment=None.", + ], + } + # Content-addressed digest of counts+findings+case outcomes (stable). + digest_src = json.dumps( + { + "counts": per_condition, + "findings": findings, + "outcomes": [ + { + "id": row["case_id"], + "baseline": row["baseline"]["outcome"], + "operator": row["operator"]["outcome"], + "depth": row["depth"]["outcome"], + "metadata_only": row["metadata_only"]["outcome"], + } + for row in per_case + ], + }, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + body["report_sha256"] = hashlib.sha256(digest_src).hexdigest() + return body + + +def write_report(report: dict[str, Any], path: Path = _REPORT) -> None: + path.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def main() -> int: + cases = _load_cases() + report = build_report(cases) + write_report(report) + findings = report["findings"] + ok = ( + findings["metadata_only_inert"] + and findings["depth_executable_inactive_equals_operator"] + and findings["operator_deterministic"] + and findings["operator_wrong_zero"] + and findings["baseline_wrong_zero"] + and findings["operator_solves_gold_fraction_cases"] + ) + print( + json.dumps( + { + "lane": _LANE, + "sample_size": report["sample_size"], + "counts": report["counts"], + "findings": findings, + "report_sha256": report["report_sha256"], + "passed": ok, + }, + indent=2, + sort_keys=True, + ) + ) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main())