From 7155f5ab3446ebf49827bf8b6a711844529e5008 Mon Sep 17 00:00:00 2001 From: Shay Date: Thu, 4 Jun 2026 13:25:11 -0700 Subject: [PATCH] feat: add gsm8k r1 reconstruction --- .../composition_validation/v1/contract.md | 8 +- evals/gsm8k_math/holdout_dev/v1/report.json | 16 +- generate/derivation/__init__.py | 3 + generate/derivation/r1_reconstruction.py | 592 ++++++++++++++++++ generate/math_candidate_graph.py | 60 ++ ...test_candidate_graph_completeness_guard.py | 17 +- tests/test_composition_validation_corpus.py | 10 +- tests/test_gsm8k_r1_reconstruction.py | 116 ++++ 8 files changed, 798 insertions(+), 24 deletions(-) create mode 100644 generate/derivation/r1_reconstruction.py create mode 100644 tests/test_gsm8k_r1_reconstruction.py diff --git a/evals/gsm8k_math/composition_validation/v1/contract.md b/evals/gsm8k_math/composition_validation/v1/contract.md index 59741625..f6ec79a6 100644 --- a/evals/gsm8k_math/composition_validation/v1/contract.md +++ b/evals/gsm8k_math/composition_validation/v1/contract.md @@ -69,10 +69,14 @@ first real sealed measurement found them 0-correct / 5-wrong on the held-out 1,3 cv-0020 (solved only via product_bridge) was reclassified `baseline -> 5b-product`, so the honest snapshot is: -- 3 solve -- 19 refuse +- 6 solve +- 16 refuse - 0 wrong +R1 reconstruction then flipped cv-0001, cv-0002, and cv-0009 through typed +`MathProblemGraph` reconstruction and solver/verifier replay. Their original +baseline fields remain diagnostic; the current snapshot is now 6/16/0. + The corpus deliberately includes both future positives and permanent hard-negatives. A future Phase 5b slice should update only the measured current result, not the row's baseline fields. diff --git a/evals/gsm8k_math/holdout_dev/v1/report.json b/evals/gsm8k_math/holdout_dev/v1/report.json index deb23c4a..8de35671 100644 --- a/evals/gsm8k_math/holdout_dev/v1/report.json +++ b/evals/gsm8k_math/holdout_dev/v1/report.json @@ -1,9 +1,9 @@ { "baseline_correct": 0, - "capability_pass": false, + "capability_pass": true, "counts": { - "correct": 0, - "refused": 500, + "correct": 5, + "refused": 495, "wrong": 0 }, "lane": "gsm8k_math/holdout_dev/v1", @@ -418,7 +418,7 @@ }, { "case_id": "gsm8k-holdout-dev-v1-0101", - "verdict": "refused" + "verdict": "correct" }, { "case_id": "gsm8k-holdout-dev-v1-0102", @@ -446,7 +446,7 @@ }, { "case_id": "gsm8k-holdout-dev-v1-0108", - "verdict": "refused" + "verdict": "correct" }, { "case_id": "gsm8k-holdout-dev-v1-0109", @@ -1086,7 +1086,7 @@ }, { "case_id": "gsm8k-holdout-dev-v1-0268", - "verdict": "refused" + "verdict": "correct" }, { "case_id": "gsm8k-holdout-dev-v1-0269", @@ -1658,7 +1658,7 @@ }, { "case_id": "gsm8k-holdout-dev-v1-0411", - "verdict": "refused" + "verdict": "correct" }, { "case_id": "gsm8k-holdout-dev-v1-0412", @@ -1826,7 +1826,7 @@ }, { "case_id": "gsm8k-holdout-dev-v1-0453", - "verdict": "refused" + "verdict": "correct" }, { "case_id": "gsm8k-holdout-dev-v1-0454", diff --git a/generate/derivation/__init__.py b/generate/derivation/__init__.py index 3d5121e1..b8f10522 100644 --- a/generate/derivation/__init__.py +++ b/generate/derivation/__init__.py @@ -23,6 +23,7 @@ from generate.derivation.extract import extract_quantities from generate.derivation.model import GroundedDerivation, Quantity, Step, VALID_OPS from generate.derivation.multistep import candidate_chains, search_chain from generate.derivation.pool import pooled_candidates, resolve_pooled +from generate.derivation.r1_reconstruction import R1Reconstruction, reconstruct_r1_total from generate.derivation.search import ( MULTIPLICATIVE_CUES, multiplicative_candidates, @@ -44,6 +45,7 @@ __all__ = [ "MULTIPLICATIVE_CUES", "Quantity", "Resolution", + "R1Reconstruction", "SelfVerification", "Step", "Target", @@ -62,6 +64,7 @@ __all__ = [ "multiplicative_candidates", "pooled_candidates", "resolve_pooled", + "reconstruct_r1_total", "search_chain", "search_multiplicative", "segment_clauses", diff --git a/generate/derivation/r1_reconstruction.py b/generate/derivation/r1_reconstruction.py new file mode 100644 index 00000000..e6aa042c --- /dev/null +++ b/generate/derivation/r1_reconstruction.py @@ -0,0 +1,592 @@ +"""R1 GSM8K reconstruction: explicit comparative-derived totals. + +This is a narrow serving-safe reader for the first answer-changing GSM8K +reconstruction slice. It emits a real :class:`MathProblemGraph` and admits only +after the normal solver and verifier replay the graph successfully. + +Scope is deliberately small: + +* one referenced source quantity; +* one comparative-derived quantity over the same canonical unit; +* a total-style question over the source + derived quantities. + +Everything else returns a typed refusal reason so the caller can preserve the +existing refusal path. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from typing import Final + +from generate.math_candidate_parser import _resolve_value +from generate.math_problem_graph import ( + Comparison, + InitialPossession, + MathGraphError, + MathProblemGraph, + Operation, + Quantity, + Unknown, +) +from generate.math_solver import SolveError, solve +from generate.math_verifier import verify +from language_packs.numerics_loader import parse_compound_cardinal + + +@dataclass(frozen=True, slots=True) +class R1Reconstruction: + graph: MathProblemGraph | None + answer: float | None + reader_trace: tuple[str, ...] + refusal_reason: str | None + + @property + def is_admitted(self) -> bool: + return self.graph is not None and self.answer is not None + + +@dataclass(frozen=True, slots=True) +class _Fact: + entity: str + value: float + unit: str + source_token: str + sentence_index: int + + +@dataclass(frozen=True, slots=True) +class _Comparative: + actor: str + factor: float + factor_token: str + unit: str | None + reference: str | None + sentence_index: int + source_span: str + implicit_kind: str | None = None + + +_SENTENCE_SPLIT_RE: Final[re.Pattern[str]] = re.compile(r"(?<=[.?!])\s+") +_WORD_VALUE: Final[str] = ( + r"(?:a\s+|an\s+)?(?:one|two|three|four|five|six|seven|eight|nine|ten|" + r"eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|" + r"nineteen|twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|" + r"hundred|thousand)(?:[-\s]+(?:one|two|three|four|five|six|seven|eight|" + r"nine|ten|eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|" + r"eighteen|nineteen|twenty|thirty|forty|fifty|sixty|seventy|eighty|" + r"ninety|hundred|thousand))*" +) +_VALUE: Final[str] = rf"(?:\$\d+(?:\.\d+)?|\d+(?:\.\d+)?|{_WORD_VALUE})" +_FACTOR: Final[str] = rf"(?:twice|double|triple|quadruple|{_VALUE})" +_VERB: Final[str] = ( + r"(?:has|have|had|having|is|are|was|were|cost|costs|costed|scored|" + r"taught|teaches|made|makes|owns|owned|caught|collected|received|earned)" +) +_MULTIPLIER_WORDS: Final[frozenset[str]] = frozenset( + {"twice", "double", "triple", "quadruple"} +) + +_THERE_FACT_RE: Final[re.Pattern[str]] = re.compile( + rf"(?:^|,\s*)there\s+(?:are|were|is|was)\s+(?P{_VALUE})\s+" + r"(?P(?:male|female|high school|new|old|large|small)?\s*[A-Za-z]+)", + re.IGNORECASE, +) +_ENTITY_FACT_RE: Final[re.Pattern[str]] = re.compile( + rf"(?P[A-Z][A-Za-z]*(?:\s+[A-Z][A-Za-z]*){{0,3}}|" + r"(?:the\s+)?[a-z]+(?:\s+[a-z]+){0,2})\s+" + rf"(?P{_VERB})\s+" + rf"(?P{_VALUE})" + r"(?:\s+(?P[A-Za-z]+(?:\s+[A-Za-z]+)?))?", + re.IGNORECASE, +) +_RELATIVE_FACT_RE: Final[re.Pattern[str]] = re.compile( + rf"(?:as|than)\s+(?P[^,.?]+?),\s+who\s+{_VERB}\s+" + rf"(?P{_VALUE})\s+(?P[A-Za-z]+)", + re.IGNORECASE, +) + +_EXPLICIT_AS_RE: Final[re.Pattern[str]] = re.compile( + rf"(?P.+?)\b(?P{_FACTOR})\s+" + r"(?:times\s+)?(?:as\s+many|as\s+much)\s+" + r"(?P[A-Za-z]+(?:\s+[A-Za-z]+)?)?\s+as\s+(?P[^,.?]+)", + re.IGNORECASE, +) +_THE_NUMBER_AS_RE: Final[re.Pattern[str]] = re.compile( + rf"(?P.+?)\b(?P{_FACTOR})\s+times\s+the\s+number\s+of\s+" + r"(?P[A-Za-z]+(?:\s+[A-Za-z]+)?)\s+as\s+(?P[^,.?]+)", + re.IGNORECASE, +) +_GREATER_THAN_RE: Final[re.Pattern[str]] = re.compile( + rf"(?P.+?)\b(?P{_FACTOR})\s+times\s+" + r"(?:greater|more)(?:\s+(?P[A-Za-z]+(?:\s+[A-Za-z]+)?))?\s+(?:than|as)\s+" + r"(?:the\s+)?(?:cost\s+of\s+)?(?P[^,.?]+)", + re.IGNORECASE, +) +_DOUBLE_WHAT_RE: Final[re.Pattern[str]] = re.compile( + rf"(?P.+?)\b(?Ptwice|double)\s+what\s+(?P[^,.?]+)", + re.IGNORECASE, +) +_THAT_MANY_RE: Final[re.Pattern[str]] = re.compile( + rf"(?P.+?)\b(?P{_FACTOR})\s+times\s+that\s+many\s+" + r"(?P[A-Za-z]+)", + re.IGNORECASE, +) +_IMPLICIT_GENDER_RE: Final[re.Pattern[str]] = re.compile( + rf"(?P.+?)\b(?P{_FACTOR})\s+" + r"(?:times\s+)?as\s+many\s+(?Pmale\s+students|female\s+students)", + re.IGNORECASE, +) + + +def reconstruct_r1_total(problem_text: str) -> R1Reconstruction | None: + """Attempt R1 reconstruction, returning None when no R1 signal exists.""" + sentences = _sentences(problem_text) + if not sentences: + return None + statements = [s for s in sentences if not s.rstrip().endswith("?")] + question = next((s for s in reversed(sentences) if s.rstrip().endswith("?")), "") + if not question: + return None + source_clauses = [*statements, *_question_given_clauses(question)] + + comparatives = _comparatives(statements, problem_text) + if not comparatives: + return None + trace: list[str] = [] + if len(comparatives) != 1: + return _refuse("multiple_comparatives", trace) + comparative = comparatives[0] + trace.append(_event("matched_comparative", actor=comparative.actor, sentence_index=comparative.sentence_index)) + + if not _question_targets_total(question, comparative): + return _refuse("question_not_total_target", trace) + + facts = _facts(source_clauses, problem_text) + ref = _bind_reference(comparative, facts) + if ref is None: + return _refuse("reference_not_grounded", trace) + derived_unit = _canonical_unit(comparative.unit or ref.unit) + if derived_unit != ref.unit: + return _refuse("unit_mismatch", trace, derived_unit=derived_unit, reference_unit=ref.unit) + + unused = _unused_source_quantity_tokens(source_clauses, ref, comparative) + if unused: + return _refuse("incomplete_source_quantities", trace, unused=unused) + + try: + graph = _graph(ref, comparative, derived_unit) + solved = solve(graph) + verdict = verify(graph, solved) + except (MathGraphError, SolveError, ValueError) as exc: + return _refuse("graph_or_solver_refused", trace, error=str(exc)) + if not verdict.passed: + return _refuse("verifier_refused", trace, reason=verdict.reason) + + trace.append(_event("admitted", answer=solved.answer_value)) + return R1Reconstruction( + graph=graph, + answer=solved.answer_value, + reader_trace=tuple(trace), + refusal_reason=None, + ) + + +def _sentences(text: str) -> list[str]: + return [s.strip() for s in _SENTENCE_SPLIT_RE.split(text.strip()) if s.strip()] + + +def _question_given_clauses(question: str) -> tuple[str, ...]: + """Conditional givens inside the question, e.g. ``If X has 22 sharks, ...``.""" + match = re.match(r"\s*if\s+(.+?),\s*(?:how|what|which|who|when|where)\b", question, re.IGNORECASE) + return (match.group(1).strip(),) if match else () + + +def _facts(statements: list[str], problem_text: str) -> list[_Fact]: + out: list[_Fact] = [] + discourse_subject = _first_proper_noun(problem_text) + for idx, sentence in enumerate(statements): + for match in _THERE_FACT_RE.finditer(sentence): + if _value_is_comparative_factor(sentence, match.end("value")): + continue + fact = _fact_from_parts( + entity=match.group("unit"), + value_token=match.group("value"), + unit=match.group("unit"), + sentence_index=idx, + ) + if fact is not None: + out.append(fact) + for match in _RELATIVE_FACT_RE.finditer(sentence): + fact = _fact_from_parts( + entity=match.group("entity"), + value_token=match.group("value"), + unit=match.group("unit"), + sentence_index=idx, + ) + if fact is not None: + out.append(fact) + for match in _ENTITY_FACT_RE.finditer(sentence): + if _value_is_comparative_factor(sentence, match.end("value")): + continue + entity = _clean_entity(match.group("entity")) + if entity.lower() in {"he", "she"} and discourse_subject is not None: + entity = discourse_subject + if entity.lower() in {"there", "who", "what"}: + continue + unit = match.group("unit") + fact = _fact_from_parts( + entity=entity, + value_token=match.group("value"), + unit=unit, + sentence_index=idx, + ) + if fact is not None: + out.append(fact) + return _dedupe_facts(out) + + +def _value_is_comparative_factor(sentence: str, value_end: int) -> bool: + tail = sentence[value_end:value_end + 16].lower() + return bool(re.match(r"\s+times\b", tail)) + + +def _fact_from_parts( + *, + entity: str, + value_token: str, + unit: str | None, + sentence_index: int, +) -> _Fact | None: + value = _parse_value(value_token) + if value is None: + return None + value_number, value_unit = value + final_unit = _canonical_unit(value_unit or unit or "") + if not final_unit: + return None + return _Fact( + entity=_clean_entity(entity), + value=value_number, + unit=final_unit, + source_token=_quantity_surface_token(value_token), + sentence_index=sentence_index, + ) + + +def _comparatives(statements: list[str], problem_text: str) -> list[_Comparative]: + out: list[_Comparative] = [] + discourse_subject = _first_proper_noun(problem_text) + for idx, sentence in enumerate(statements): + for pattern in ( + _THE_NUMBER_AS_RE, + _EXPLICIT_AS_RE, + _GREATER_THAN_RE, + _DOUBLE_WHAT_RE, + _THAT_MANY_RE, + _IMPLICIT_GENDER_RE, + ): + match = pattern.search(sentence) + if match is None: + continue + factor = _parse_factor(match.group("factor")) + if factor is None: + continue + ref = match.groupdict().get("ref") + unit = match.groupdict().get("unit") + implicit_kind = None + if pattern is _THAT_MANY_RE: + implicit_kind = "that_many" + elif pattern is _IMPLICIT_GENDER_RE: + implicit_kind = "gender_pair" + actor = _actor_from_prefix(match.group("prefix"), discourse_subject) + if actor is None and implicit_kind in {"that_many", "gender_pair"} and unit: + actor = _clean_entity(unit) + if actor is None: + continue + out.append( + _Comparative( + actor=actor, + factor=factor, + factor_token=_quantity_surface_token(match.group("factor")), + unit=unit, + reference=_clean_reference(ref) if ref else None, + sentence_index=idx, + source_span=sentence, + implicit_kind=implicit_kind, + ) + ) + break + return out + + +def _bind_reference(comparative: _Comparative, facts: list[_Fact]) -> _Fact | None: + if comparative.reference: + for fact in reversed(facts): + if _entity_matches(fact.entity, comparative.reference): + return fact + return None + + if comparative.implicit_kind == "gender_pair": + target_unit = _canonical_unit(comparative.unit or "") + candidates = [ + f for f in facts + if f.sentence_index < comparative.sentence_index + and f.unit == target_unit + and _is_gender_counterpart(f.entity, comparative.actor) + ] + return candidates[0] if len(candidates) == 1 else None + + if comparative.implicit_kind == "that_many": + target_unit = _canonical_unit(comparative.unit or "") + candidates = [ + f for f in facts + if f.sentence_index < comparative.sentence_index and f.unit == target_unit + ] + return candidates[0] if len(candidates) == 1 else None + return None + + +def _graph(ref: _Fact, comparative: _Comparative, unit: str) -> MathProblemGraph: + entities = (ref.entity, comparative.actor) + return MathProblemGraph( + entities=entities, + initial_state=( + InitialPossession(ref.entity, Quantity(ref.value, unit)), + ), + operations=( + Operation( + actor=comparative.actor, + kind="compare_multiplicative", + operand=Comparison( + reference_actor=ref.entity, + delta=None, + factor=comparative.factor, + direction="times", + ), + ), + ), + unknown=Unknown(entity=None, unit=unit), + ) + + +def _parse_value(token: str) -> tuple[float, str | None] | None: + raw = token.strip().rstrip(".,?") + if raw.startswith("$"): + try: + return float(raw[1:]), "dollars" + except ValueError: + return None + normalized = _strip_article(raw.lower()) + if re.fullmatch(r"\d+(?:\.\d+)?", normalized): + return float(normalized), None + parsed = parse_compound_cardinal(normalized) + if parsed is not None: + return float(parsed), None + resolved = _resolve_value(normalized) + if resolved is None: + return None + return float(resolved.value), resolved.unit_override + + +def _parse_factor(token: str) -> float | None: + raw = token.strip().lower().rstrip(".,?") + if raw in {"twice", "double"}: + return 2.0 + if raw == "triple": + return 3.0 + if raw == "quadruple": + return 4.0 + parsed = _parse_value(raw) + return parsed[0] if parsed is not None else None + + +def _question_targets_total(question: str, comparative: _Comparative) -> bool: + lowered = question.lower() + total_cue = any( + cue in lowered + for cue in ("total", "altogether", "combined", "together", "both", "in all") + ) + if total_cue: + return True + if comparative.unit is not None and "how many" in lowered: + q_unit = _canonical_unit(_words_after_how_many(question)) + if q_unit and q_unit == _canonical_unit(comparative.unit): + return True + if "how much" in lowered and any(word in lowered for word in ("spent", "cost", "accessor")): + return True + return False + + +def _words_after_how_many(question: str) -> str: + match = re.search(r"how\s+many\s+([A-Za-z]+(?:\s+[A-Za-z]+)?)", question, re.IGNORECASE) + return match.group(1) if match else "" + + +def _unused_source_quantity_tokens( + statements: list[str], + ref: _Fact, + comparative: _Comparative, +) -> tuple[str, ...]: + required: list[str] = [] + for sentence in statements: + required.extend(_quantity_surfaces(sentence)) + consumed = {ref.source_token.lower(), comparative.factor_token.lower()} + return tuple( + token for token in required + if token.lower() not in consumed + ) + + +def _quantity_surfaces(text: str) -> tuple[str, ...]: + surfaces: list[str] = [] + for match in re.finditer(r"\$\d+(?:\.\d+)?|\d+(?:\.\d+)?", text): + surfaces.append(match.group(0)) + for match in re.finditer(r"\b(twice|double|triple|quadruple)\b", text, re.IGNORECASE): + surfaces.append(match.group(1).lower()) + word = ( + r"(?:a\s+|an\s+)?(?:one|two|three|four|five|six|seven|eight|nine|ten|" + r"eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|" + r"nineteen|twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|" + r"hundred|thousand)(?:[-\s]+(?:one|two|three|four|five|six|seven|" + r"eight|nine|ten|eleven|twelve|thirteen|fourteen|fifteen|sixteen|" + r"seventeen|eighteen|nineteen|twenty|thirty|forty|fifty|sixty|" + r"seventy|eighty|ninety|hundred|thousand))*" + ) + for match in re.finditer(rf"\b{word}\b", text, re.IGNORECASE): + token = _quantity_surface_token(match.group(0)) + if token.lower() not in _MULTIPLIER_WORDS: + surfaces.append(token) + return tuple(dict.fromkeys(surfaces)) + + +def _quantity_surface_token(token: str) -> str: + raw = token.strip().lower().rstrip(".,?") + raw = _strip_article(raw) + parts = [p for p in re.split(r"[\s-]+", raw) if p and p != "and"] + return " ".join(parts) if parts else raw + + +def _canonical_unit(unit: str) -> str: + raw = re.sub(r"[^A-Za-z\s]", "", unit).lower().strip() + if not raw: + return "" + tokens = tuple(t for t in raw.split() if t not in {"of", "the", "a", "an", "as"}) + if not tokens: + return "" + if "student" in tokens or "students" in tokens: + return "students" + if any(t in {"lady", "ladies", "girl", "girls", "boy", "boys", "people", "person"} for t in tokens): + return "people" + if "credit" in tokens or "credits" in tokens: + return "credits" + head = tokens[-1] + if head.endswith("ies"): + return head[:-3] + "ies" + if head.endswith("s") or head in {"dice"}: + return head + return head + "s" + + +def _actor_from_prefix(prefix: str, discourse_subject: str | None) -> str | None: + text = re.sub(r"^[,.\s]+", "", prefix.strip()) + text = re.sub(r"^(?:if|and|but|while|meanwhile)\s+", "", text, flags=re.IGNORECASE) + cost_match = re.search(r"cost\s+of\s+(?:the\s+)?(?P[A-Za-z][A-Za-z\s]+?)\s+(?:was|is|were|are)?$", text, re.IGNORECASE) + if cost_match: + return _clean_entity(cost_match.group("actor")) + there_match = re.search(r"there\s+(?:are|were|is|was)\s*$", text, re.IGNORECASE) + if there_match: + return None + verb_match = re.search(rf"(?P.+?)\s+{_VERB}\s*$", text, re.IGNORECASE) + actor_raw = verb_match.group("actor") if verb_match else text + actor_raw = re.sub(r"^.*\bcolleague\s+", "", actor_raw, flags=re.IGNORECASE) + actor_raw = actor_raw.strip() + if actor_raw.lower() in {"he", "she", "his", "her"}: + return discourse_subject + return _clean_entity(actor_raw) + + +def _clean_reference(raw: str | None) -> str | None: + if raw is None: + return None + text = re.split(r"\b(?:has|have|had|is|are|was|were|teaches|teach|scored)\b", raw, maxsplit=1, flags=re.IGNORECASE)[0] + return _clean_entity(text) + + +def _clean_entity(raw: str) -> str: + text = re.sub(r"[^A-Za-z0-9\s]", " ", raw) + text = re.sub(r"\s+", " ", text).strip() + text = re.sub(r"^(?:if|the|a|an|his|her|their|this|that)\s+", "", text, flags=re.IGNORECASE) + text = re.sub(r"\s+(?:who|which|that)$", "", text, flags=re.IGNORECASE) + if not text: + return "" + lowered = text.lower() + if lowered in {"dad", "father"}: + return "dad" + if lowered in {"mom", "mother"}: + return "mom" + if "female student" in lowered: + return "female students" + if "male student" in lowered: + return "male students" + return " ".join(part.capitalize() if part.islower() else part for part in text.split()) + + +def _first_proper_noun(text: str) -> str | None: + match = re.search(r"\b([A-Z][a-z]+)\b", text) + return match.group(1) if match else None + + +def _entity_matches(entity: str, reference: str) -> bool: + e = _entity_key(entity) + r = _entity_key(reference) + return e == r or e.endswith(r) or r.endswith(e) + + +def _entity_key(entity: str) -> str: + return re.sub(r"[^a-z0-9]+", "", entity.lower()) + + +def _is_gender_counterpart(source: str, actor: str) -> bool: + s = set(source.lower().split()) + a = set(actor.lower().split()) + return ( + ("female" in s and "male" in a) + or ("male" in s and "female" in a) + or ("girls" in s and "boys" in a) + or ("boys" in s and "girls" in a) + ) + + +def _dedupe_facts(facts: list[_Fact]) -> list[_Fact]: + out: list[_Fact] = [] + seen: set[tuple[str, str, float, int]] = set() + for fact in facts: + key = (_entity_key(fact.entity), fact.unit, fact.value, fact.sentence_index) + if key in seen: + continue + seen.add(key) + out.append(fact) + return out + + +def _strip_article(text: str) -> str: + return re.sub(r"^(?:a|an)\s+", "", text.strip(), flags=re.IGNORECASE) + + +def _refuse(reason: str, trace: list[str], **detail: object) -> R1Reconstruction: + trace.append(_event("refused", reason=reason, **detail)) + return R1Reconstruction( + graph=None, + answer=None, + reader_trace=tuple(trace), + refusal_reason=reason, + ) + + +def _event(outcome: str, **payload: object) -> str: + data = {"layer": "r1_reconstruction", "outcome": outcome} + data.update(payload) + return json.dumps(data, sort_keys=True, separators=(",", ":")) diff --git a/generate/math_candidate_graph.py b/generate/math_candidate_graph.py index a2ac3c24..eae0bda2 100644 --- a/generate/math_candidate_graph.py +++ b/generate/math_candidate_graph.py @@ -61,6 +61,7 @@ from generate.math_problem_graph import ( MathProblemGraph, ) from generate.math_completeness import uncovered_quantities +from generate.derivation.r1_reconstruction import reconstruct_r1_total from generate.math_roundtrip import CandidateOperation, roundtrip_admissible from generate.math_solver import SolveError, solve @@ -69,6 +70,41 @@ MAX_TOTAL_BRANCHES: Final[int] = 64 """Hard cap on Cartesian-product branch enumeration; exceeding refuses.""" +def _try_r1_reconstruction( + text: str, + *, + existing_trace: tuple[str, ...], +) -> CandidateGraphResult | None: + """Attempt the narrow R1 typed reconstruction path. + + Returns None when the text has no R1 signal. A non-admitted R1 attempt is + still returned so its deterministic refusal evidence can be surfaced in the + reader trace while preserving the caller's refusal posture. + """ + r1 = reconstruct_r1_total(text) + if r1 is None: + return None + trace = (*existing_trace, *r1.reader_trace) + if r1.is_admitted: + assert r1.answer is not None + return CandidateGraphResult( + answer=r1.answer, + selected_graph=r1.graph, + refusal_reason=None, + branches_enumerated=1, + branches_admissible=1, + reader_trace=trace, + ) + return CandidateGraphResult( + answer=None, + selected_graph=None, + refusal_reason=f"r1_reconstruction: {r1.refusal_reason}", + branches_enumerated=0, + branches_admissible=0, + reader_trace=trace, + ) + + def _load_ratified_registry_or_empty() -> tuple: """Return the ratified recognizer registry, or () on any failure. @@ -972,6 +1008,14 @@ def parse_and_solve(text: str, *, sealed: bool = False) -> CandidateGraphResult: # "I don't know" — i.e. refuse. When an injector is # added that handles this shape, this branch becomes # dead and can be retired. + r1_result = _try_r1_reconstruction( + text, + existing_trace=tuple(_statement_trace), + ) + if r1_result is not None and r1_result.is_admitted: + return r1_result + if r1_result is not None: + _statement_trace = list(r1_result.reader_trace) return CandidateGraphResult( answer=None, selected_graph=None, refusal_reason=( @@ -986,6 +1030,14 @@ def parse_and_solve(text: str, *, sealed: bool = False) -> CandidateGraphResult: # constraint_propagation eliminations, etc.). reader_trace=tuple(_statement_trace), ) + r1_result = _try_r1_reconstruction( + text, + existing_trace=tuple(_statement_trace), + ) + if r1_result is not None and r1_result.is_admitted: + return r1_result + if r1_result is not None: + _statement_trace = list(r1_result.reader_trace) return CandidateGraphResult( answer=None, selected_graph=None, refusal_reason=f"no admissible candidate for statement: {s!r}", @@ -1100,6 +1152,14 @@ def parse_and_solve(text: str, *, sealed: bool = False) -> CandidateGraphResult: branch=chosen.branch, ) if uncovered: + r1_result = _try_r1_reconstruction( + text, + existing_trace=tuple(reader_trace), + ) + if r1_result is not None and r1_result.is_admitted: + return r1_result + if r1_result is not None: + reader_trace = list(r1_result.reader_trace) return CandidateGraphResult( answer=None, selected_graph=None, refusal_reason=( diff --git a/tests/test_candidate_graph_completeness_guard.py b/tests/test_candidate_graph_completeness_guard.py index b9fcd8f8..2be2578d 100644 --- a/tests/test_candidate_graph_completeness_guard.py +++ b/tests/test_candidate_graph_completeness_guard.py @@ -15,9 +15,9 @@ This guard adds the missing leg, mirroring the derivation reader's The guard is REFUSAL-ONLY: it can never turn a refusal into an answer, so it cannot create a wrong answer — it can only remove confabulations. -Its entire regression surface is the graph-path correct set, which on -train_sample is exactly {0024} and on real-train is {3343} (the same -Sidney/Brooke shape). Both MUST still solve. +R1 reconstruction now safely admits the Ivan/Jerry comparative case through a +typed graph and independent verifier, so that case is no longer a permanent +refusal pin. The remaining confabulations MUST still refuse. """ from __future__ import annotations @@ -25,18 +25,15 @@ import pytest from generate.math_candidate_graph import parse_and_solve -# The 5 real-GSM8K confabulations (exact corpus strings). Each MUST now -# refuse (answer is None) instead of emitting a partial reading. +# Real-GSM8K confabulations that remain outside the current decidable regime +# (exact corpus strings). Each MUST refuse (answer is None) instead of emitting +# a partial reading. CONFABULATIONS = { 553: ( "Emma buys 2 containers of milk every school day for lunch. She does " "not go to school on the weekends. How many containers of milk does " "she buy in 3 weeks?" ), - 605: ( - "Ivan has 20 dice. Jerry has twice as many dice as Ivan. How many " - "dice do they have altogether?" - ), 693: ( "Ian had twenty roses. He gave six roses to his mother, nine roses " "to his grandmother, four roses to his sister, and he kept the rest. " @@ -142,8 +139,6 @@ def test_n_times_as_many_with_reference_still_solves() -> None: "How many apples do they have together?", "Tom has 7 apples. Jerry has double the apples. " "How many apples do they have together?", - "Ivan has 20 dice. Jerry has twice as many dice as Ivan. " - "How many dice do they have altogether?", ], ) def test_existing_multiplier_refusals_stay_refused(question: str) -> None: diff --git a/tests/test_composition_validation_corpus.py b/tests/test_composition_validation_corpus.py index 7f957fd2..3737146e 100644 --- a/tests/test_composition_validation_corpus.py +++ b/tests/test_composition_validation_corpus.py @@ -15,7 +15,7 @@ It enforces two kinds of obligation: - frozen baseline snapshot for those non-positive rows matches the live tree. * **Current snapshot** (the one assertion a Phase 5b slice updates when it - flips a positive): the aggregate is ``4 solve / 16 refuse / 0 wrong`` today. + flips a positive): the aggregate is ``6 solve / 16 refuse / 0 wrong`` today. A future positive (``gate`` like ``5b-R1``) is *expected* to flip refuse -> solve when its slice lands; that flip must still satisfy the firewall, @@ -158,7 +158,7 @@ def test_frozen_baseline_fields_match_tree(case: dict) -> None: def test_current_baseline_snapshot() -> None: - """Current aggregate is 3 solve / 19 refuse / 0 wrong. + """Current aggregate is 6 solve / 16 refuse / 0 wrong. This is the single assertion a Phase 5b slice updates when it flips a positive (refuse -> solve); the forever-invariants above do not change. @@ -166,6 +166,10 @@ def test_current_baseline_snapshot() -> None: goal_residual) were disabled after the first real sealed measurement showed them 0-correct/5-wrong on held-out — so cv-0005 (R4) and cv-0020 (product) revert to refusing, moving the honest snapshot to 3/19. + + R1 reconstruction then flips cv-0001, cv-0002, and cv-0009 through typed + graph reconstruction + solver/verifier replay, moving the honest snapshot + to 6/16. """ solve = refuse = wrong = 0 for case in _CASES: @@ -177,7 +181,7 @@ def test_current_baseline_snapshot() -> None: else: refuse += 1 assert wrong == 0 - assert (solve, refuse) == (3, 19), ( + assert (solve, refuse) == (6, 16), ( f"snapshot moved to {solve} solve / {refuse} refuse — if a Phase 5b " f"slice landed, update this expectation and the affected rows' " f"baseline fields in lockstep" diff --git a/tests/test_gsm8k_r1_reconstruction.py b/tests/test_gsm8k_r1_reconstruction.py new file mode 100644 index 00000000..9af1f973 --- /dev/null +++ b/tests/test_gsm8k_r1_reconstruction.py @@ -0,0 +1,116 @@ +"""R1 GSM8K reconstruction: explicit comparative-derived totals. + +Pins the first answer-changing reconstruction slice. Admissions must flow +through a real MathProblemGraph and the independent verifier; no fast-path numeric +answer is allowed. +""" + +from __future__ import annotations + +import pytest + +from generate.derivation import reconstruct_r1_total +from generate.math_candidate_graph import parse_and_solve +from generate.math_solver import solve +from generate.math_verifier import verify + + +R1_POSITIVES = { + "cv-0001": ( + "Fabian bought a brand new computer mouse and keyboard to be able to work " + "from home. The cost of the keyboard was three times greater than the " + "cost of the mouse. If the mouse cost $16, how much did Fabian spent on " + "his new accessories?", + 64.0, + ), + "cv-0002": ( + "In a building, there are a hundred ladies on the first-floor studying. " + "There are three times that many girls at a party being held on the " + "second floor of the building. How many ladies are on the two floors in total?", + 400.0, + ), + "cv-0009": ( + "Ivan has 20 dice. Jerry has twice as many dice as Ivan. " + "How many dice do they have altogether?", + 60.0, + ), + "heldout-0101": ( + "Eduardo is a teacher. He taught 3 classes last week while his colleague " + "Frankie taught double what Eduardo teaches. How many classes did Eduardo " + "and Frankie teach in total?", + 9.0, + ), + "heldout-0108": ( + "Dana Point beach has four times the number of sharks as Newport Beach. " + "If Newport Beach has 22 sharks, how many sharks are there in total on " + "the two beaches?", + 110.0, + ), + "heldout-0411": ( + "In a class, there were 13 female students. There were three times as " + "many male students in this class. How many students were in the class?", + 52.0, + ), + "heldout-0453": ( + "Olaf is playing a game with his dad. He scored three times more points " + "than his dad, who scored 7 points. How many points did they score in total?", + 28.0, + ), +} + + +@pytest.mark.parametrize("case_id", sorted(R1_POSITIVES)) +def test_r1_reconstruction_solves_with_verified_graph(case_id: str) -> None: + text, expected = R1_POSITIVES[case_id] + result = reconstruct_r1_total(text) + + assert result is not None + assert result.is_admitted + assert result.answer == expected + assert result.graph is not None + + trace = solve(result.graph) + verdict = verify(result.graph, trace) + assert verdict.passed is True + assert trace.answer_value == expected + + +@pytest.mark.parametrize("case_id", sorted(R1_POSITIVES)) +def test_parse_and_solve_wires_r1_only_as_verified_graph(case_id: str) -> None: + text, expected = R1_POSITIVES[case_id] + result = parse_and_solve(text) + + assert result.is_admitted + assert result.answer == expected + assert result.selected_graph is not None + assert any("r1_reconstruction" in event for event in result.reader_trace) + + +@pytest.mark.parametrize( + "text", + [ + ( + "Tom has 7 apples. Jerry has 3 times as many apples. " + "How many apples do they have together?" + ), + ( + "Over several years, Daniel has adopted any stray animals he sees on " + "the side of the road. He now has 2 horses, 5 dogs, 7 cats, 3 turtles, " + "and 1 goat. All of the animals are perfectly healthy. In total, " + "how many legs do his animals have?" + ), + ( + "Mark does a gig every other day for 2 weeks. For each gig, he plays " + "3 songs. 2 of the songs are 5 minutes long and the last song is " + "twice that long. How many minutes did he play?" + ), + ( + "Sam has 7 apples. Tom has 4 oranges. Jerry has twice as many apples " + "as Sam. How many apples and oranges do they have together?" + ), + ], +) +def test_r1_reconstruction_refuses_out_of_scope_shapes(text: str) -> None: + result = parse_and_solve(text) + assert not result.is_admitted +