From 80753e13de30e3ea1b1001a0e5b92de372d69fcf Mon Sep 17 00:00:00 2001 From: Shay Date: Thu, 18 Jun 2026 10:57:36 -0700 Subject: [PATCH] fix(gsm8k): harden sprint 9 temporal and affine gates (#822) Patch post-merge wrong=0 hazards from #820 review without reverting Sprint 9 organs: bind affine/temporal answers to asked subjects and body actors, anchor overtime thresholds to per-shift/day surfaces, fix affine completeness double-counting, narrow percent_partition group skip, and expand temporal question/rental verb recognition. --- generate/derivation/affine_fraction_delta.py | 46 +++++- generate/derivation/percent_partition.py | 2 +- generate/derivation/temporal_tariff.py | 107 ++++++++++++- ...idate_graph_sprint9_postmerge_hardening.py | 144 ++++++++++++++++++ 4 files changed, 291 insertions(+), 8 deletions(-) create mode 100644 tests/test_math_candidate_graph_sprint9_postmerge_hardening.py diff --git a/generate/derivation/affine_fraction_delta.py b/generate/derivation/affine_fraction_delta.py index b1e756e5..a449125c 100644 --- a/generate/derivation/affine_fraction_delta.py +++ b/generate/derivation/affine_fraction_delta.py @@ -33,6 +33,7 @@ from collections import Counter from generate.derivation.clauses import segment_clauses from generate.derivation.extract import extract_quantities from generate.derivation.model import GroundedDerivation, Quantity, Step +from generate.derivation.state.bind import PRONOUNS, leading_subject_token from generate.derivation.target import _question_clause from generate.derivation.verify import Resolution, SelfVerification from generate.math_candidate_parser import _init_mutation_admitted @@ -51,6 +52,10 @@ _GOAL_INTENT: Final[frozenset[str]] = frozenset( {"want", "wants", "wanted", "need", "needs", "hoping", "hopes", "plans", "aims", "goal"} ) _POSSESSION_CUES: Final[frozenset[str]] = frozenset({"has", "have", "had"}) +_QUESTION_POSSESSION_RE: Final[re.Pattern[str]] = re.compile( + r"how\s+many\b.+?\bdoes\s+(\w+)\s+have\b", + re.IGNORECASE, +) def _asks_subject_total(question_clause: str) -> bool: @@ -65,6 +70,43 @@ def _affine_match(problem_text: str) -> re.Match[str] | None: return matches[0] +def _affine_clause(problem_text: str) -> str | None: + match = _affine_match(problem_text) + if match is None: + return None + for clause in segment_clauses(problem_text): + if match.group(0) in clause: + return clause + return None + + +def _affine_subject(problem_text: str) -> str | None: + clause = _affine_clause(problem_text) + if clause is None: + return None + subject = leading_subject_token(clause) + return subject.lower() if subject is not None else None + + +def _question_possession_subject(question_clause: str) -> str | None: + match = _QUESTION_POSSESSION_RE.search(question_clause) + if match is None: + return None + return match.group(1).lower() + + +def _question_subject_matches_affine( + problem_text: str, question_clause: str +) -> bool: + affine_subject = _affine_subject(problem_text) + asked_subject = _question_possession_subject(question_clause) + if affine_subject is None or asked_subject is None: + return False + if asked_subject in PRONOUNS: + return False + return asked_subject == affine_subject + + def _has_hazard_surface(problem_text: str, question_clause: str) -> bool: if _DECREASE_TO_FRACTION_RE.search(problem_text): return True @@ -118,6 +160,8 @@ def build_affine_fraction_delta(problem_text: str) -> GroundedDerivation | None: question_clause = _question_clause(problem_text) if not _asks_subject_total(question_clause): return None + if not _question_subject_matches_affine(problem_text, question_clause): + return None if _has_hazard_surface(problem_text, question_clause): return None @@ -202,8 +246,6 @@ def _self_verifies_affine_fraction_delta( + [step.operand.source_token for step in derivation.steps] ) if match is not None: - used[match.group(4)] += 1 - used[f"{match.group(1)}/{match.group(2)}"] += 1 used[match.group(1)] += 1 used[match.group(2)] += 1 reference_candidate = _reference_candidate(problem_text, match.group(3)) diff --git a/generate/derivation/percent_partition.py b/generate/derivation/percent_partition.py index 016c30dd..c7c5875b 100644 --- a/generate/derivation/percent_partition.py +++ b/generate/derivation/percent_partition.py @@ -82,7 +82,7 @@ def _total_population(problem_text: str, question_clause: str) -> Quantity | Non lowered = clause.lower() if "half" in lowered or _PERCENT_OF_GROUP_RE.search(clause): continue - if "group" in lowered: + if "group" in lowered and "group of" not in lowered: continue quantities = [q for q in extract_quantities(clause) if q.unit] if len(quantities) == 1: diff --git a/generate/derivation/temporal_tariff.py b/generate/derivation/temporal_tariff.py index 275a83c3..f9d79d74 100644 --- a/generate/derivation/temporal_tariff.py +++ b/generate/derivation/temporal_tariff.py @@ -31,6 +31,7 @@ from collections import Counter from generate.derivation.clauses import segment_clauses from generate.derivation.extract import extract_quantities from generate.derivation.model import GroundedDerivation, Quantity, Step +from generate.derivation.state.bind import PRONOUNS, leading_subject_token from generate.derivation.target import _question_clause from generate.derivation.verify import Resolution, SelfVerification from generate.math_roundtrip import _token_in, _tokens, _value_grounds @@ -40,10 +41,25 @@ _HOURLY_RATE_RE: Final[re.Pattern[str]] = re.compile( r"\$\s*(\d+(?:\.\d+)?)\s*(?:an\s+hour|/hour|per\s+hour)", re.IGNORECASE, ) -_THRESHOLD_HOURS_RE: Final[re.Pattern[str]] = re.compile( - r"more\s+than\s+(\d+)\s+hours?", +_THRESHOLD_SHIFT_RE: Final[re.Pattern[str]] = re.compile( + r"more\s+than\s+(\d+)\s+hours?\s+(?:a|an|each|per)\s+(?:day|shift|workday)", re.IGNORECASE, ) +_THRESHOLD_BAD_ANCHOR_RE: Final[re.Pattern[str]] = re.compile( + r"more\s+than\s+\d+\s+hours?\s+(?:per|a|an|each|for)\s+(?:week|month|year|total)", + re.IGNORECASE, +) +_RENT_VERBS: Final[frozenset[str]] = frozenset({"rent", "rents", "rented", "renting"}) +_QUESTION_MONEY_SUBJECT_RE: Final[tuple[re.Pattern[str], ...]] = ( + re.compile( + r"how\s+much(?:\s+money)?\s+does\s+(\w+)\s+(?:make|earn)", + re.IGNORECASE, + ), + re.compile(r"how\s+much\s+will\s+it\s+cost\s+(\w+)", re.IGNORECASE), + re.compile(r"what\s+does\s+it\s+cost\s+(\w+)", re.IGNORECASE), + re.compile(r"how\s+much\s+does\s+(\w+)\s+pay", re.IGNORECASE), + re.compile(r"what\s+is\s+the\s+total\s+cost(?:\s+for\s+(\w+))?", re.IGNORECASE), +) _BUNDLE_TARIFF_RE: Final[re.Pattern[str]] = re.compile( r"\$\s*(\d+(?:\.\d+)?)\s+per\s+day\s+or\s+\$\s*(\d+(?:\.\d+)?)\s+for\s+(\d+)\s+days?", re.IGNORECASE, @@ -71,11 +87,86 @@ def _asks_money_total(question_clause: str) -> bool: tokens = _tokens(question_clause) if "money" in tokens and bool(_EARN_VERBS & tokens): return True - if "how" in tokens and "much" in tokens and bool(_COST_VERBS & tokens): + if "how" in tokens and "much" in tokens and bool((_COST_VERBS | _EARN_VERBS) & tokens): + return True + if "what" in tokens and bool({"cost", "costs", "charge", "charges", "price", "pay", "paid"} & tokens): return True return False +def _question_money_subject(question_clause: str) -> str | None: + for pattern in _QUESTION_MONEY_SUBJECT_RE: + match = pattern.search(question_clause) + if match is None: + continue + subject = match.group(1) + if subject is None: + continue + return subject.lower() + return None + + +def _subjects_match(body_actor: str | None, asked_subject: str | None) -> bool: + if body_actor is None or asked_subject is None: + return False + if asked_subject in PRONOUNS: + return True + return asked_subject == body_actor.lower() + + +def _overtime_worker(problem_text: str, question_clause: str) -> str | None: + for clause in segment_clauses(problem_text): + if clause == question_clause: + continue + if _HOURLY_RATE_RE.search(clause) or re.search( + r"makes\s+\$\s*\d", clause, re.IGNORECASE + ): + subject = leading_subject_token(clause) + if subject is not None: + return subject.lower() + for clause in segment_clauses(problem_text): + if clause == question_clause: + continue + tokens = _tokens(clause) + if "hours" not in tokens and "hour" not in tokens: + continue + if "every" not in tokens and "per" not in tokens and "each" not in tokens: + continue + subject = leading_subject_token(clause) + if subject is not None and subject.lower() not in {"if", "when"}: + return subject.lower() + return None + + +def _bundle_renter(problem_text: str, question_clause: str) -> str | None: + for clause in segment_clauses(problem_text): + if clause == question_clause: + continue + tokens = _tokens(clause) + if not (_RENT_VERBS & tokens): + continue + subject = leading_subject_token(clause) + if subject is not None: + return subject.lower() + return None + + +def _question_target_matches_actor( + problem_text: str, question_clause: str, *, pattern: str +) -> bool: + asked = _question_money_subject(question_clause) + if pattern == "overtime": + body_actor = _overtime_worker(problem_text, question_clause) + else: + body_actor = _bundle_renter(problem_text, question_clause) + if body_actor is None: + return False + if asked is None: + tokens = _tokens(question_clause) + return "what" in tokens and "total" in tokens and "cost" in tokens + return _subjects_match(body_actor, asked) + + def _body_text(problem_text: str) -> str: question_clause = _question_clause(problem_text) return problem_text.replace(question_clause, "").strip() @@ -113,7 +204,9 @@ def _hourly_rate(problem_text: str) -> Quantity | None: def _threshold_hours(problem_text: str) -> Quantity | None: - match = _THRESHOLD_HOURS_RE.search(problem_text) + if _THRESHOLD_BAD_ANCHOR_RE.search(problem_text): + return None + match = _THRESHOLD_SHIFT_RE.search(problem_text) if match is None: return None token = match.group(1) @@ -167,7 +260,7 @@ def _rental_days(problem_text: str, question_clause: str) -> Quantity | None: if clause == question_clause: continue tokens = _tokens(clause) - if "rent" not in tokens and "rents" not in tokens: + if not (_RENT_VERBS & tokens): continue quantities = [ q @@ -201,6 +294,8 @@ def build_overtime_shift_earnings(problem_text: str) -> GroundedDerivation | Non question_clause = _question_clause(problem_text) if not _asks_money_total(question_clause): return None + if not _question_target_matches_actor(problem_text, question_clause, pattern="overtime"): + return None if _has_hazard_surface(problem_text, question_clause): return None if "overtime" not in _tokens(problem_text): @@ -243,6 +338,8 @@ def build_bundle_overflow_tariff(problem_text: str) -> GroundedDerivation | None question_clause = _question_clause(problem_text) if not _asks_money_total(question_clause): return None + if not _question_target_matches_actor(problem_text, question_clause, pattern="bundle"): + return None if _has_hazard_surface(problem_text, question_clause): return None if "overtime" in _tokens(problem_text): diff --git a/tests/test_math_candidate_graph_sprint9_postmerge_hardening.py b/tests/test_math_candidate_graph_sprint9_postmerge_hardening.py new file mode 100644 index 00000000..72ebc2e8 --- /dev/null +++ b/tests/test_math_candidate_graph_sprint9_postmerge_hardening.py @@ -0,0 +1,144 @@ +"""Post-merge wrong=0 hardening for Sprint 9 Gates A2m/A2n (#820).""" + +from __future__ import annotations + +from generate.math_candidate_graph import parse_and_solve +from generate.derivation.affine_fraction_delta import ( + compose_affine_fraction_delta, + resolve_promotable_affine_fraction_delta, +) +from generate.derivation.temporal_tariff import ( + compose_bundle_overflow_tariff, + compose_overtime_shift_earnings, + compose_temporal_tariff, + resolve_promotable_temporal_tariff, +) +from generate.derivation.percent_partition import ( + compose_percent_partition, + resolve_promotable_percent_partition, +) + +_AFFINE_BODY = ( + "Yun had 20 paperclips initially, but then lost 12. Marion has 1/4 more than what " + "Yun currently has, plus 7." +) + +_OVERTIME_BODY = ( + "Tina makes $18.00 an hour. If she works more than 8 hours per shift, she is " + "eligible for overtime, which is paid by your hourly wage + 1/2 your hourly wage. " + "If she works 10 hours every day for 5 days," +) + +_BUNDLE_BODY = ( + "Jason has a carriage house that he rents out. He's charging $50.00 per day or " + "$500.00 for 14 days. Eric wants to rent the house for 20 days." +) + +_PERCENT_GROUP_OF_100 = ( + "A group of 100 students. Half of the students are girls, the other half are boys. " + "20% of the girls have dogs and 10% of the boys have dogs. " + "How many students own dogs?" +) + +_PERCENT_SUBGROUP_CONFUSER = ( + "Half of the students are girls, the other half are boys. The girls group has 50 students. " + "20% of the girls have dogs and 10% of the boys have dogs. How many students own dogs?" +) + + +def _run(text: str): + return parse_and_solve(text, sealed=False) + + +class TestAffineSubjectBinding: + def test_question_subject_mismatch_refuses(self): + text = f"{_AFFINE_BODY} How many paperclips does Alice have?" + assert resolve_promotable_affine_fraction_delta(text) is None + assert _run(text).answer is None + + def test_question_reference_subject_refuses(self): + text = f"{_AFFINE_BODY} How many paperclips does Yun have?" + assert resolve_promotable_affine_fraction_delta(text) is None + assert _run(text).answer is None + + +class TestAffineCompletenessGuard: + def test_duplicate_offset_distractor_refuses(self): + text = ( + "Yun had 20 paperclips initially, but then lost 12. There are 7 boxes on the shelf. " + "Marion has 1/4 more than what Yun currently has, plus 7. " + "How many paperclips does Marion have?" + ) + assert compose_affine_fraction_delta(text) is None + assert _run(text).answer is None + + +class TestTemporalActorBinding: + def test_overtime_question_actor_mismatch_refuses(self): + text = f"{_OVERTIME_BODY} how much money does Bob make?" + assert resolve_promotable_temporal_tariff(text) is None + assert _run(text).answer is None + + def test_bundle_question_renter_mismatch_refuses(self): + text = f"{_BUNDLE_BODY} What does it cost Alice?" + assert resolve_promotable_temporal_tariff(text) is None + assert _run(text).answer is None + + +class TestTemporalThresholdAnchoring: + def test_non_shift_threshold_refuses(self): + text = ( + "Tina makes $18.00 an hour. If she works more than 8 hours per week, she is " + "eligible for overtime, which is paid by your hourly wage + 1/2 your hourly wage. " + "If she works 10 hours every day for 5 days, how much money does she make?" + ) + assert compose_overtime_shift_earnings(text) is None + assert _run(text).answer is None + + +class TestTemporalRentalVerbVariants: + def test_rented_still_admits(self): + text = ( + "Jason has a carriage house that he rents out. He's charging $50.00 per day or " + "$500.00 for 14 days. Eric rented the house for 20 days. " + "How much will it cost him?" + ) + res = compose_bundle_overflow_tariff(text) + assert res is not None + assert res.answer == 800.0 + + def test_renting_still_admits(self): + text = ( + "Jason has a carriage house that he rents out. He's charging $50.00 per day or " + "$500.00 for 14 days. Eric is renting the house for 20 days. " + "How much will it cost him?" + ) + res = compose_bundle_overflow_tariff(text) + assert res is not None + assert res.answer == 800.0 + + +class TestTemporalQuestionRobustness: + def test_how_much_does_she_earn_admits(self): + text = f"{_OVERTIME_BODY} how much does she earn?" + res = compose_overtime_shift_earnings(text) + assert res is not None + assert res.answer == 990.0 + + def test_what_is_total_cost_admits(self): + text = f"{_BUNDLE_BODY} What is the total cost?" + res = compose_bundle_overflow_tariff(text) + assert res is not None + assert res.answer == 800.0 + + +class TestPercentPartitionGroupOf: + def test_group_of_total_still_admits(self): + res = compose_percent_partition(_PERCENT_GROUP_OF_100) + assert res is not None + assert res.answer == 15.0 + assert _run(_PERCENT_GROUP_OF_100).answer == 15.0 + + def test_subgroup_group_still_refuses(self): + assert resolve_promotable_percent_partition(_PERCENT_SUBGROUP_CONFUSER) is None + assert _run(_PERCENT_SUBGROUP_CONFUSER).answer is None \ No newline at end of file