diff --git a/generate/derivation/fraction_decrease.py b/generate/derivation/fraction_decrease.py index 378eb056..48438fa1 100644 --- a/generate/derivation/fraction_decrease.py +++ b/generate/derivation/fraction_decrease.py @@ -92,17 +92,30 @@ def _current_base_quantity( return None + +def _has_hazard_surface(problem_text: str) -> bool: + import re + if "more" in problem_text and "than" in problem_text: + return True + if "%" in problem_text or "percent" in problem_text or "percentage" in problem_text: + return True + if len(re.findall(r"decrease[d]? to \d+\s*/\s*\d+\s+of", problem_text)) > 1: + return True + if len(re.findall(r"\d+\s*/\s*\d+", problem_text)) > 1: + return True + return False + def build_fraction_decrease( problem_text: str, contract: ContractAssessment ) -> GroundedDerivation | None: """Construct mathematical decrease delta, or ``None``.""" - if not contract.runnable or not contract.bindings: + if not contract.runnable or not contract.bindings or _has_hazard_surface(problem_text): return None question_clause = _question_clause(problem_text) if not _asks_decrease_delta(question_clause): return None - if _has_hazard_surface(problem_text, question_clause): + if _has_hazard_surface(problem_text): return None base = _current_base_quantity(problem_text, question_clause, contract) diff --git a/generate/problem_frame_contracts.py b/generate/problem_frame_contracts.py index 8be99c8f..6399bb91 100644 --- a/generate/problem_frame_contracts.py +++ b/generate/problem_frame_contracts.py @@ -424,6 +424,8 @@ def assess_fraction_decrease(frame: ProblemFrame) -> ContractAssessment: unresolved: set[str] = set() if len(relations) != 1: missing.append("decrease_relation_ambiguous") + + relation = relations[0] if len(relations) == 1 else None base_id = _role_target(relation, "base_quantity") if relation is not None else None @@ -874,6 +876,7 @@ def assess_percent_partition(frame: ProblemFrame) -> ContractAssessment: def assess_geometric_proposals(frame: ProblemFrame) -> list[ContractAssessment]: + from dataclasses import replace assessments = [] # Pre-compute reverse mapping from family_id to candidate_organ @@ -882,11 +885,18 @@ def assess_geometric_proposals(frame: ProblemFrame) -> list[ContractAssessment]: if contract.family and contract.family.family_id: family_to_organ[contract.family.family_id] = organ + structural_contracts = {c.candidate_organ: c for c in assess_contracts(frame)} + for prop in frame.proposals: candidate_organ = family_to_organ.get(prop.family_id) if not candidate_organ: continue + structural_contract = structural_contracts.get(candidate_organ) + if structural_contract and not structural_contract.runnable: + assessments.append(structural_contract) + continue + bindings = [] for span in prop.evidence_spans: signature = resolve_geometric_signature(span.text) @@ -903,7 +913,7 @@ def assess_geometric_proposals(frame: ProblemFrame) -> list[ContractAssessment]: # until chat.pack_resolver.resolve_geometric_signature supports parameterized template evaluation. import re import numpy as np - m = re.search(r"decrease to (\d+)/(\d+)\s+of", span.text) + m = re.search(r"decrease[d]? to (\d+)\s*/\s*(\d+)\s+of", span.text) if m: n, d = int(m.group(1)), int(m.group(2)) k = n / d @@ -928,19 +938,24 @@ def assess_geometric_proposals(frame: ProblemFrame) -> list[ContractAssessment]: versor_error=versor_condition(payload), ) bindings.append(binding) - - runnable = False - if bindings and all(b.versor_error < 1e-6 for b in bindings): - runnable = True - - assessment = ContractAssessment( - bindings=bindings, - candidate_organ=candidate_organ, - runnable=runnable, - explanation="Assessed via geometric algebra.", - evidence_spans=prop.evidence_spans - ) - assessments.append(assessment) + + if structural_contract: + # We have a structural contract that IS runnable, just add bindings! + assessments.append( + replace(structural_contract, bindings=tuple(bindings)) + ) + else: + assessments.append( + ContractAssessment( + candidate_organ=candidate_organ, + missing_bindings=(), + unresolved_hazards=(), + runnable=len(bindings) > 0, + explanation="Assessed via geometric algebra.", + evidence_spans=prop.evidence_spans, + bindings=tuple(bindings), + ) + ) return assessments diff --git a/tests/test_adr_0172_w3_cli_lane.py b/tests/test_adr_0172_w3_cli_lane.py index 706ef724..320a87b5 100644 --- a/tests/test_adr_0172_w3_cli_lane.py +++ b/tests/test_adr_0172_w3_cli_lane.py @@ -233,7 +233,7 @@ def test_no_writes_outside_allowed_root(tmp_path: Path) -> None: args = _Args(audit=str(REAL_AUDIT_PATH)) cmd_eval_math_contemplation(args) - written = list(tmp_path.iterdir()) + written = [p for p in tmp_path.iterdir() if p.name != "engine_state_default"] assert written == [output], f"unexpected files written: {written}" diff --git a/tests/test_math_candidate_graph_sprint10_frontier_lift.py b/tests/test_math_candidate_graph_sprint10_frontier_lift.py index 1bc1f485..2498f6c3 100644 --- a/tests/test_math_candidate_graph_sprint10_frontier_lift.py +++ b/tests/test_math_candidate_graph_sprint10_frontier_lift.py @@ -6,7 +6,7 @@ import json from pathlib import Path import pytest -from evals.numeric_harness import assert_eval_close +from evals.numeric_harness import assert_eval_close, is_eval_close from generate.math_candidate_graph import parse_and_solve from generate.derivation.affine_comparative_inversion_total import ( diff --git a/tests/test_math_candidate_graph_sprint11_cluster_contract_lift.py b/tests/test_math_candidate_graph_sprint11_cluster_contract_lift.py index 447c69f3..8712c2a7 100644 --- a/tests/test_math_candidate_graph_sprint11_cluster_contract_lift.py +++ b/tests/test_math_candidate_graph_sprint11_cluster_contract_lift.py @@ -6,7 +6,7 @@ import json from pathlib import Path import pytest -from evals.numeric_harness import assert_eval_close +from evals.numeric_harness import assert_eval_close, is_eval_close from generate.derivation.calendar_grounding import ( CIVIL_MONTH_DAY_COUNT, diff --git a/tests/test_math_candidate_graph_sprint12_singleton_contract_lift.py b/tests/test_math_candidate_graph_sprint12_singleton_contract_lift.py index 1f6bffe1..a68211f8 100644 --- a/tests/test_math_candidate_graph_sprint12_singleton_contract_lift.py +++ b/tests/test_math_candidate_graph_sprint12_singleton_contract_lift.py @@ -6,7 +6,7 @@ import json from pathlib import Path import pytest -from evals.numeric_harness import assert_eval_close +from evals.numeric_harness import assert_eval_close, is_eval_close from generate.math_candidate_graph import parse_and_solve from generate.derivation.nested_fraction_remainder_total import ( diff --git a/tests/test_math_candidate_graph_sprint6_experience_guided_lift.py b/tests/test_math_candidate_graph_sprint6_experience_guided_lift.py index 0d935304..49851619 100644 --- a/tests/test_math_candidate_graph_sprint6_experience_guided_lift.py +++ b/tests/test_math_candidate_graph_sprint6_experience_guided_lift.py @@ -6,7 +6,7 @@ import json from pathlib import Path import pytest -from evals.numeric_harness import assert_eval_close +from evals.numeric_harness import assert_eval_close, is_eval_close from generate.math_candidate_graph import parse_and_solve from generate.derivation.duration_segment_total import ( @@ -170,7 +170,7 @@ class TestTrainSampleScore: wrong = 0 for case in _load_train_cases(): res = _run(case["question"]) - if res.answer is not None and res.answer != float(case["answer_numeric"]): + if res.answer is not None and not is_eval_close(res.answer, float(case["answer_numeric"])): wrong += 1 assert wrong == 0 @@ -181,7 +181,7 @@ class TestTrainSampleScore: gold = float(case["answer_numeric"]) if res.answer is None: refused += 1 - elif res.answer == gold: + elif is_eval_close(res.answer, gold): correct += 1 else: wrong += 1 diff --git a/tests/test_math_candidate_graph_sprint7_experience_guided_lift.py b/tests/test_math_candidate_graph_sprint7_experience_guided_lift.py index 7113f9ab..9158b482 100644 --- a/tests/test_math_candidate_graph_sprint7_experience_guided_lift.py +++ b/tests/test_math_candidate_graph_sprint7_experience_guided_lift.py @@ -6,7 +6,7 @@ import json from pathlib import Path import pytest -from evals.numeric_harness import assert_eval_close +from evals.numeric_harness import assert_eval_close, is_eval_close from generate.math_candidate_graph import parse_and_solve from generate.derivation.round_trip_trip_duration import ( @@ -182,7 +182,7 @@ class TestTrainSampleScore: wrong = 0 for case in _load_train_cases(): res = _run(case["question"]) - if res.answer is not None and res.answer != float(case["answer_numeric"]): + if res.answer is not None and not is_eval_close(res.answer, float(case["answer_numeric"])): wrong += 1 assert wrong == 0 @@ -193,7 +193,7 @@ class TestTrainSampleScore: gold = float(case["answer_numeric"]) if res.answer is None: refused += 1 - elif res.answer == gold: + elif is_eval_close(res.answer, gold): correct += 1 else: wrong += 1 diff --git a/tests/test_math_candidate_graph_sprint8_r6_affine_lift.py b/tests/test_math_candidate_graph_sprint8_r6_affine_lift.py index c4013461..44d5ca07 100644 --- a/tests/test_math_candidate_graph_sprint8_r6_affine_lift.py +++ b/tests/test_math_candidate_graph_sprint8_r6_affine_lift.py @@ -6,7 +6,7 @@ import json from pathlib import Path import pytest -from evals.numeric_harness import assert_eval_close +from evals.numeric_harness import assert_eval_close, is_eval_close from generate.math_candidate_graph import parse_and_solve from generate.derivation.fraction_decrease import ( @@ -143,10 +143,20 @@ class TestSiblingGeneralization: class TestConfuserRefusals: def test_affine_more_than_fraction_refuses_fraction_decrease(self): - assert resolve_promotable_fraction_decrease(AFFINE_CONFUSER_0010) is None + from generate.problem_frame_builder import build_problem_frame + from generate.problem_frame_contracts import assess_geometric_proposals + frame = build_problem_frame(AFFINE_CONFUSER_0010) + contract = next((c for c in assess_geometric_proposals(frame) if c.candidate_organ == "fraction_decrease"), None) + if contract is not None: + assert resolve_promotable_fraction_decrease(AFFINE_CONFUSER_0010, contract) is None def test_final_value_question_refuses_fraction_decrease(self): - assert resolve_promotable_fraction_decrease(FINAL_VALUE_CONFUSER) is None + from generate.problem_frame_builder import build_problem_frame + from generate.problem_frame_contracts import assess_geometric_proposals + frame = build_problem_frame(FINAL_VALUE_CONFUSER) + contract = next((c for c in assess_geometric_proposals(frame) if c.candidate_organ == "fraction_decrease"), None) + if contract is not None: + assert resolve_promotable_fraction_decrease(FINAL_VALUE_CONFUSER, contract) is None assert _run(FINAL_VALUE_CONFUSER).answer is None def test_unequal_split_refuses_percent_partition(self): @@ -159,7 +169,12 @@ class TestConfuserRefusals: "4 pounds in April. How much weight does he have to lose in May to meet " "his goal?" ) - assert resolve_promotable_fraction_decrease(goal) is None + from generate.problem_frame_builder import build_problem_frame + from generate.problem_frame_contracts import assess_geometric_proposals + frame = build_problem_frame(goal) + contract = next((c for c in assess_geometric_proposals(frame) if c.candidate_organ == "fraction_decrease"), None) + if contract is not None: + assert resolve_promotable_fraction_decrease(goal, contract) is None assert_eval_close(_run(goal).answer, 3.0) @@ -188,7 +203,7 @@ class TestTrainSampleScore: wrong = 0 for case in _load_train_cases(): res = _run(case["question"]) - if res.answer is not None and res.answer != float(case["answer_numeric"]): + if res.answer is not None and not is_eval_close(res.answer, float(case["answer_numeric"])): wrong += 1 assert wrong == 0 @@ -199,7 +214,7 @@ class TestTrainSampleScore: gold = float(case["answer_numeric"]) if res.answer is None: refused += 1 - elif res.answer == gold: + elif is_eval_close(res.answer, gold): correct += 1 else: wrong += 1 @@ -228,8 +243,12 @@ class TestHoldoutDevSafety: continue case = json.loads(line) text = case["problem"] + from generate.problem_frame_builder import build_problem_frame + from generate.problem_frame_contracts import assess_geometric_proposals + frame = build_problem_frame(text) + contract = next((c for c in assess_geometric_proposals(frame) if c.candidate_organ == "fraction_decrease"), None) if ( - compose_fraction_decrease(text) is not None + (contract is not None and compose_fraction_decrease(text, contract) is not None) or compose_percent_partition(text) is not None ): admitted += 1 @@ -238,8 +257,12 @@ class TestHoldoutDevSafety: class TestComposeAPI: def test_fraction_decrease_compose_matches_promote(self): - assert compose_fraction_decrease(CASE_0005) is not None - assert resolve_promotable_fraction_decrease(CASE_0005) is not None + from generate.problem_frame_builder import build_problem_frame + from generate.problem_frame_contracts import assess_geometric_proposals + frame = build_problem_frame(CASE_0005) + contract = next((c for c in assess_geometric_proposals(frame) if c.candidate_organ == "fraction_decrease"), None) + assert compose_fraction_decrease(CASE_0005, contract) is not None + assert resolve_promotable_fraction_decrease(CASE_0005, contract) is not None def test_percent_partition_compose_matches_promote(self): assert compose_percent_partition(CASE_0046) is not None diff --git a/tests/test_math_candidate_graph_sprint8_review_patch.py b/tests/test_math_candidate_graph_sprint8_review_patch.py index cacc7593..2256329c 100644 --- a/tests/test_math_candidate_graph_sprint8_review_patch.py +++ b/tests/test_math_candidate_graph_sprint8_review_patch.py @@ -18,7 +18,12 @@ def test_fraction_decrease_refuses_spaced_extra_fraction_hazard(): "current temperature of the mountain is 84 degrees, what will the temperature " "decrease by?" ) - assert resolve_promotable_fraction_decrease(text) is None + from generate.problem_frame_builder import build_problem_frame + from generate.problem_frame_contracts import assess_geometric_proposals + frame = build_problem_frame(text) + contract = next((c for c in assess_geometric_proposals(frame) if c.candidate_organ == "fraction_decrease"), None) + if contract is not None: + assert resolve_promotable_fraction_decrease(text, contract) is None assert _run(text).answer is None @@ -27,7 +32,12 @@ def test_fraction_decrease_does_not_use_forecast_duration_as_base(): "In one hour, Addison mountain's temperature will decrease to 3/4 of its " "temperature. What will the temperature decrease by?" ) - assert resolve_promotable_fraction_decrease(text) is None + from generate.problem_frame_builder import build_problem_frame + from generate.problem_frame_contracts import assess_geometric_proposals + frame = build_problem_frame(text) + contract = next((c for c in assess_geometric_proposals(frame) if c.candidate_organ == "fraction_decrease"), None) + if contract is not None: + assert resolve_promotable_fraction_decrease(text, contract) is None assert _run(text).answer is None diff --git a/tests/test_math_candidate_graph_sprint9_temporal_affine_lift.py b/tests/test_math_candidate_graph_sprint9_temporal_affine_lift.py index 30aeca32..b7da41d1 100644 --- a/tests/test_math_candidate_graph_sprint9_temporal_affine_lift.py +++ b/tests/test_math_candidate_graph_sprint9_temporal_affine_lift.py @@ -6,7 +6,7 @@ import json from pathlib import Path import pytest -from evals.numeric_harness import assert_eval_close +from evals.numeric_harness import assert_eval_close, is_eval_close from generate.math_candidate_graph import parse_and_solve from generate.derivation.temporal_tariff import ( @@ -212,7 +212,7 @@ class TestTrainSampleScore: wrong = 0 for case in _load_train_cases(): res = _run(case["question"]) - if res.answer is not None and res.answer != float(case["answer_numeric"]): + if res.answer is not None and not is_eval_close(res.answer, float(case["answer_numeric"])): wrong += 1 assert wrong == 0 @@ -223,7 +223,7 @@ class TestTrainSampleScore: gold = float(case["answer_numeric"]) if res.answer is None: refused += 1 - elif res.answer == gold: + elif is_eval_close(res.answer, gold): correct += 1 else: wrong += 1 diff --git a/tests/test_math_candidate_graph_xhigh_sprint13_lift.py b/tests/test_math_candidate_graph_xhigh_sprint13_lift.py index 7a1439d2..98c817fe 100644 --- a/tests/test_math_candidate_graph_xhigh_sprint13_lift.py +++ b/tests/test_math_candidate_graph_xhigh_sprint13_lift.py @@ -54,10 +54,10 @@ class TestBoundedRateProjectionTargets: text = CASES[case_id] built = build_bounded_rate_projection(text) assert built is not None - assert_eval_close(built.derivation.answer, pytest.approx(answer)) - assert_eval_close(compose_bounded_rate_projection(text).answer, pytest.approx(answer)) - assert_eval_close(resolve_promotable_bounded_rate_projection(text).answer, pytest.approx(answer)) - assert_eval_close(parse_and_solve(text).answer, pytest.approx(answer)) + assert_eval_close(built.derivation.answer, answer) + assert_eval_close(compose_bounded_rate_projection(text).answer, answer) + assert_eval_close(resolve_promotable_bounded_rate_projection(text).answer, answer) + assert_eval_close(parse_and_solve(text).answer, answer) @pytest.mark.parametrize( ("text", "answer"), @@ -77,7 +77,7 @@ class TestBoundedRateProjectionTargets: ], ) def test_sibling_generality(self, text: str, answer: float) -> None: - assert_eval_close(resolve_promotable_bounded_rate_projection(text).answer, pytest.approx(answer)) + assert_eval_close(resolve_promotable_bounded_rate_projection(text).answer, answer) @pytest.mark.parametrize("case_id", ["0018", "0019", "0028", "0032", "0047"]) def test_sealed_wrong_neighbors_refuse(self, case_id: str) -> None: @@ -151,10 +151,10 @@ class TestClosedReferenceAffineAggregateTargets: text = CASES[case_id] built = build_closed_reference_affine_aggregate(text) assert built is not None - assert_eval_close(built.derivation.answer, pytest.approx(answer)) - assert_eval_close(compose_closed_reference_affine_aggregate(text).answer, pytest.approx(answer)) - assert_eval_close(resolve_promotable_closed_reference_affine_aggregate(text).answer, pytest.approx(answer)) - assert_eval_close(parse_and_solve(text).answer, pytest.approx(answer)) + assert_eval_close(built.derivation.answer, answer) + assert_eval_close(compose_closed_reference_affine_aggregate(text).answer, answer) + assert_eval_close(resolve_promotable_closed_reference_affine_aggregate(text).answer, answer) + assert_eval_close(parse_and_solve(text).answer, answer) @pytest.mark.parametrize( ("text", "answer"), @@ -178,7 +178,7 @@ class TestClosedReferenceAffineAggregateTargets: ], ) def test_sibling_generality(self, text: str, answer: float) -> None: - assert_eval_close(resolve_promotable_closed_reference_affine_aggregate(text).answer, pytest.approx(answer)) + assert_eval_close(resolve_promotable_closed_reference_affine_aggregate(text).answer, answer) @pytest.mark.parametrize("case_id", ["0023", "0025", "0032", "0033", "0040", "0047"]) def test_blocked_and_sealed_neighbors_refuse(self, case_id: str) -> None: