feat: add gsm8k r1 reconstruction
This commit is contained in:
Shay
parent
fc20d01832
commit
7155f5ab34
8 changed files with 798 additions and 24 deletions
|
|
@ -69,10 +69,14 @@ first real sealed measurement found them 0-correct / 5-wrong on the held-out 1,3
|
|||
cv-0020 (solved only via product_bridge) was reclassified `baseline -> 5b-product`,
|
||||
so the honest snapshot is:
|
||||
|
||||
- 3 solve
|
||||
- 19 refuse
|
||||
- 6 solve
|
||||
- 16 refuse
|
||||
- 0 wrong
|
||||
|
||||
R1 reconstruction then flipped cv-0001, cv-0002, and cv-0009 through typed
|
||||
`MathProblemGraph` reconstruction and solver/verifier replay. Their original
|
||||
baseline fields remain diagnostic; the current snapshot is now 6/16/0.
|
||||
|
||||
The corpus deliberately includes both future positives and permanent
|
||||
hard-negatives. A future Phase 5b slice should update only the measured current
|
||||
result, not the row's baseline fields.
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
{
|
||||
"baseline_correct": 0,
|
||||
"capability_pass": false,
|
||||
"capability_pass": true,
|
||||
"counts": {
|
||||
"correct": 0,
|
||||
"refused": 500,
|
||||
"correct": 5,
|
||||
"refused": 495,
|
||||
"wrong": 0
|
||||
},
|
||||
"lane": "gsm8k_math/holdout_dev/v1",
|
||||
|
|
@ -418,7 +418,7 @@
|
|||
},
|
||||
{
|
||||
"case_id": "gsm8k-holdout-dev-v1-0101",
|
||||
"verdict": "refused"
|
||||
"verdict": "correct"
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-holdout-dev-v1-0102",
|
||||
|
|
@ -446,7 +446,7 @@
|
|||
},
|
||||
{
|
||||
"case_id": "gsm8k-holdout-dev-v1-0108",
|
||||
"verdict": "refused"
|
||||
"verdict": "correct"
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-holdout-dev-v1-0109",
|
||||
|
|
@ -1086,7 +1086,7 @@
|
|||
},
|
||||
{
|
||||
"case_id": "gsm8k-holdout-dev-v1-0268",
|
||||
"verdict": "refused"
|
||||
"verdict": "correct"
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-holdout-dev-v1-0269",
|
||||
|
|
@ -1658,7 +1658,7 @@
|
|||
},
|
||||
{
|
||||
"case_id": "gsm8k-holdout-dev-v1-0411",
|
||||
"verdict": "refused"
|
||||
"verdict": "correct"
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-holdout-dev-v1-0412",
|
||||
|
|
@ -1826,7 +1826,7 @@
|
|||
},
|
||||
{
|
||||
"case_id": "gsm8k-holdout-dev-v1-0453",
|
||||
"verdict": "refused"
|
||||
"verdict": "correct"
|
||||
},
|
||||
{
|
||||
"case_id": "gsm8k-holdout-dev-v1-0454",
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from generate.derivation.extract import extract_quantities
|
|||
from generate.derivation.model import GroundedDerivation, Quantity, Step, VALID_OPS
|
||||
from generate.derivation.multistep import candidate_chains, search_chain
|
||||
from generate.derivation.pool import pooled_candidates, resolve_pooled
|
||||
from generate.derivation.r1_reconstruction import R1Reconstruction, reconstruct_r1_total
|
||||
from generate.derivation.search import (
|
||||
MULTIPLICATIVE_CUES,
|
||||
multiplicative_candidates,
|
||||
|
|
@ -44,6 +45,7 @@ __all__ = [
|
|||
"MULTIPLICATIVE_CUES",
|
||||
"Quantity",
|
||||
"Resolution",
|
||||
"R1Reconstruction",
|
||||
"SelfVerification",
|
||||
"Step",
|
||||
"Target",
|
||||
|
|
@ -62,6 +64,7 @@ __all__ = [
|
|||
"multiplicative_candidates",
|
||||
"pooled_candidates",
|
||||
"resolve_pooled",
|
||||
"reconstruct_r1_total",
|
||||
"search_chain",
|
||||
"search_multiplicative",
|
||||
"segment_clauses",
|
||||
|
|
|
|||
592
generate/derivation/r1_reconstruction.py
Normal file
592
generate/derivation/r1_reconstruction.py
Normal file
|
|
@ -0,0 +1,592 @@
|
|||
"""R1 GSM8K reconstruction: explicit comparative-derived totals.
|
||||
|
||||
This is a narrow serving-safe reader for the first answer-changing GSM8K
|
||||
reconstruction slice. It emits a real :class:`MathProblemGraph` and admits only
|
||||
after the normal solver and verifier replay the graph successfully.
|
||||
|
||||
Scope is deliberately small:
|
||||
|
||||
* one referenced source quantity;
|
||||
* one comparative-derived quantity over the same canonical unit;
|
||||
* a total-style question over the source + derived quantities.
|
||||
|
||||
Everything else returns a typed refusal reason so the caller can preserve the
|
||||
existing refusal path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
from generate.math_candidate_parser import _resolve_value
|
||||
from generate.math_problem_graph import (
|
||||
Comparison,
|
||||
InitialPossession,
|
||||
MathGraphError,
|
||||
MathProblemGraph,
|
||||
Operation,
|
||||
Quantity,
|
||||
Unknown,
|
||||
)
|
||||
from generate.math_solver import SolveError, solve
|
||||
from generate.math_verifier import verify
|
||||
from language_packs.numerics_loader import parse_compound_cardinal
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class R1Reconstruction:
|
||||
graph: MathProblemGraph | None
|
||||
answer: float | None
|
||||
reader_trace: tuple[str, ...]
|
||||
refusal_reason: str | None
|
||||
|
||||
@property
|
||||
def is_admitted(self) -> bool:
|
||||
return self.graph is not None and self.answer is not None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Fact:
|
||||
entity: str
|
||||
value: float
|
||||
unit: str
|
||||
source_token: str
|
||||
sentence_index: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Comparative:
|
||||
actor: str
|
||||
factor: float
|
||||
factor_token: str
|
||||
unit: str | None
|
||||
reference: str | None
|
||||
sentence_index: int
|
||||
source_span: str
|
||||
implicit_kind: str | None = None
|
||||
|
||||
|
||||
_SENTENCE_SPLIT_RE: Final[re.Pattern[str]] = re.compile(r"(?<=[.?!])\s+")
|
||||
_WORD_VALUE: Final[str] = (
|
||||
r"(?:a\s+|an\s+)?(?:one|two|three|four|five|six|seven|eight|nine|ten|"
|
||||
r"eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|"
|
||||
r"nineteen|twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|"
|
||||
r"hundred|thousand)(?:[-\s]+(?:one|two|three|four|five|six|seven|eight|"
|
||||
r"nine|ten|eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|"
|
||||
r"eighteen|nineteen|twenty|thirty|forty|fifty|sixty|seventy|eighty|"
|
||||
r"ninety|hundred|thousand))*"
|
||||
)
|
||||
_VALUE: Final[str] = rf"(?:\$\d+(?:\.\d+)?|\d+(?:\.\d+)?|{_WORD_VALUE})"
|
||||
_FACTOR: Final[str] = rf"(?:twice|double|triple|quadruple|{_VALUE})"
|
||||
_VERB: Final[str] = (
|
||||
r"(?:has|have|had|having|is|are|was|were|cost|costs|costed|scored|"
|
||||
r"taught|teaches|made|makes|owns|owned|caught|collected|received|earned)"
|
||||
)
|
||||
_MULTIPLIER_WORDS: Final[frozenset[str]] = frozenset(
|
||||
{"twice", "double", "triple", "quadruple"}
|
||||
)
|
||||
|
||||
_THERE_FACT_RE: Final[re.Pattern[str]] = re.compile(
|
||||
rf"(?:^|,\s*)there\s+(?:are|were|is|was)\s+(?P<value>{_VALUE})\s+"
|
||||
r"(?P<unit>(?:male|female|high school|new|old|large|small)?\s*[A-Za-z]+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_ENTITY_FACT_RE: Final[re.Pattern[str]] = re.compile(
|
||||
rf"(?P<entity>[A-Z][A-Za-z]*(?:\s+[A-Z][A-Za-z]*){{0,3}}|"
|
||||
r"(?:the\s+)?[a-z]+(?:\s+[a-z]+){0,2})\s+"
|
||||
rf"(?P<verb>{_VERB})\s+"
|
||||
rf"(?P<value>{_VALUE})"
|
||||
r"(?:\s+(?P<unit>[A-Za-z]+(?:\s+[A-Za-z]+)?))?",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_RELATIVE_FACT_RE: Final[re.Pattern[str]] = re.compile(
|
||||
rf"(?:as|than)\s+(?P<entity>[^,.?]+?),\s+who\s+{_VERB}\s+"
|
||||
rf"(?P<value>{_VALUE})\s+(?P<unit>[A-Za-z]+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_EXPLICIT_AS_RE: Final[re.Pattern[str]] = re.compile(
|
||||
rf"(?P<prefix>.+?)\b(?P<factor>{_FACTOR})\s+"
|
||||
r"(?:times\s+)?(?:as\s+many|as\s+much)\s+"
|
||||
r"(?P<unit>[A-Za-z]+(?:\s+[A-Za-z]+)?)?\s+as\s+(?P<ref>[^,.?]+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_THE_NUMBER_AS_RE: Final[re.Pattern[str]] = re.compile(
|
||||
rf"(?P<prefix>.+?)\b(?P<factor>{_FACTOR})\s+times\s+the\s+number\s+of\s+"
|
||||
r"(?P<unit>[A-Za-z]+(?:\s+[A-Za-z]+)?)\s+as\s+(?P<ref>[^,.?]+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_GREATER_THAN_RE: Final[re.Pattern[str]] = re.compile(
|
||||
rf"(?P<prefix>.+?)\b(?P<factor>{_FACTOR})\s+times\s+"
|
||||
r"(?:greater|more)(?:\s+(?P<unit>[A-Za-z]+(?:\s+[A-Za-z]+)?))?\s+(?:than|as)\s+"
|
||||
r"(?:the\s+)?(?:cost\s+of\s+)?(?P<ref>[^,.?]+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_DOUBLE_WHAT_RE: Final[re.Pattern[str]] = re.compile(
|
||||
rf"(?P<prefix>.+?)\b(?P<factor>twice|double)\s+what\s+(?P<ref>[^,.?]+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_THAT_MANY_RE: Final[re.Pattern[str]] = re.compile(
|
||||
rf"(?P<prefix>.+?)\b(?P<factor>{_FACTOR})\s+times\s+that\s+many\s+"
|
||||
r"(?P<unit>[A-Za-z]+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_IMPLICIT_GENDER_RE: Final[re.Pattern[str]] = re.compile(
|
||||
rf"(?P<prefix>.+?)\b(?P<factor>{_FACTOR})\s+"
|
||||
r"(?:times\s+)?as\s+many\s+(?P<unit>male\s+students|female\s+students)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def reconstruct_r1_total(problem_text: str) -> R1Reconstruction | None:
|
||||
"""Attempt R1 reconstruction, returning None when no R1 signal exists."""
|
||||
sentences = _sentences(problem_text)
|
||||
if not sentences:
|
||||
return None
|
||||
statements = [s for s in sentences if not s.rstrip().endswith("?")]
|
||||
question = next((s for s in reversed(sentences) if s.rstrip().endswith("?")), "")
|
||||
if not question:
|
||||
return None
|
||||
source_clauses = [*statements, *_question_given_clauses(question)]
|
||||
|
||||
comparatives = _comparatives(statements, problem_text)
|
||||
if not comparatives:
|
||||
return None
|
||||
trace: list[str] = []
|
||||
if len(comparatives) != 1:
|
||||
return _refuse("multiple_comparatives", trace)
|
||||
comparative = comparatives[0]
|
||||
trace.append(_event("matched_comparative", actor=comparative.actor, sentence_index=comparative.sentence_index))
|
||||
|
||||
if not _question_targets_total(question, comparative):
|
||||
return _refuse("question_not_total_target", trace)
|
||||
|
||||
facts = _facts(source_clauses, problem_text)
|
||||
ref = _bind_reference(comparative, facts)
|
||||
if ref is None:
|
||||
return _refuse("reference_not_grounded", trace)
|
||||
derived_unit = _canonical_unit(comparative.unit or ref.unit)
|
||||
if derived_unit != ref.unit:
|
||||
return _refuse("unit_mismatch", trace, derived_unit=derived_unit, reference_unit=ref.unit)
|
||||
|
||||
unused = _unused_source_quantity_tokens(source_clauses, ref, comparative)
|
||||
if unused:
|
||||
return _refuse("incomplete_source_quantities", trace, unused=unused)
|
||||
|
||||
try:
|
||||
graph = _graph(ref, comparative, derived_unit)
|
||||
solved = solve(graph)
|
||||
verdict = verify(graph, solved)
|
||||
except (MathGraphError, SolveError, ValueError) as exc:
|
||||
return _refuse("graph_or_solver_refused", trace, error=str(exc))
|
||||
if not verdict.passed:
|
||||
return _refuse("verifier_refused", trace, reason=verdict.reason)
|
||||
|
||||
trace.append(_event("admitted", answer=solved.answer_value))
|
||||
return R1Reconstruction(
|
||||
graph=graph,
|
||||
answer=solved.answer_value,
|
||||
reader_trace=tuple(trace),
|
||||
refusal_reason=None,
|
||||
)
|
||||
|
||||
|
||||
def _sentences(text: str) -> list[str]:
|
||||
return [s.strip() for s in _SENTENCE_SPLIT_RE.split(text.strip()) if s.strip()]
|
||||
|
||||
|
||||
def _question_given_clauses(question: str) -> tuple[str, ...]:
|
||||
"""Conditional givens inside the question, e.g. ``If X has 22 sharks, ...``."""
|
||||
match = re.match(r"\s*if\s+(.+?),\s*(?:how|what|which|who|when|where)\b", question, re.IGNORECASE)
|
||||
return (match.group(1).strip(),) if match else ()
|
||||
|
||||
|
||||
def _facts(statements: list[str], problem_text: str) -> list[_Fact]:
|
||||
out: list[_Fact] = []
|
||||
discourse_subject = _first_proper_noun(problem_text)
|
||||
for idx, sentence in enumerate(statements):
|
||||
for match in _THERE_FACT_RE.finditer(sentence):
|
||||
if _value_is_comparative_factor(sentence, match.end("value")):
|
||||
continue
|
||||
fact = _fact_from_parts(
|
||||
entity=match.group("unit"),
|
||||
value_token=match.group("value"),
|
||||
unit=match.group("unit"),
|
||||
sentence_index=idx,
|
||||
)
|
||||
if fact is not None:
|
||||
out.append(fact)
|
||||
for match in _RELATIVE_FACT_RE.finditer(sentence):
|
||||
fact = _fact_from_parts(
|
||||
entity=match.group("entity"),
|
||||
value_token=match.group("value"),
|
||||
unit=match.group("unit"),
|
||||
sentence_index=idx,
|
||||
)
|
||||
if fact is not None:
|
||||
out.append(fact)
|
||||
for match in _ENTITY_FACT_RE.finditer(sentence):
|
||||
if _value_is_comparative_factor(sentence, match.end("value")):
|
||||
continue
|
||||
entity = _clean_entity(match.group("entity"))
|
||||
if entity.lower() in {"he", "she"} and discourse_subject is not None:
|
||||
entity = discourse_subject
|
||||
if entity.lower() in {"there", "who", "what"}:
|
||||
continue
|
||||
unit = match.group("unit")
|
||||
fact = _fact_from_parts(
|
||||
entity=entity,
|
||||
value_token=match.group("value"),
|
||||
unit=unit,
|
||||
sentence_index=idx,
|
||||
)
|
||||
if fact is not None:
|
||||
out.append(fact)
|
||||
return _dedupe_facts(out)
|
||||
|
||||
|
||||
def _value_is_comparative_factor(sentence: str, value_end: int) -> bool:
|
||||
tail = sentence[value_end:value_end + 16].lower()
|
||||
return bool(re.match(r"\s+times\b", tail))
|
||||
|
||||
|
||||
def _fact_from_parts(
|
||||
*,
|
||||
entity: str,
|
||||
value_token: str,
|
||||
unit: str | None,
|
||||
sentence_index: int,
|
||||
) -> _Fact | None:
|
||||
value = _parse_value(value_token)
|
||||
if value is None:
|
||||
return None
|
||||
value_number, value_unit = value
|
||||
final_unit = _canonical_unit(value_unit or unit or "")
|
||||
if not final_unit:
|
||||
return None
|
||||
return _Fact(
|
||||
entity=_clean_entity(entity),
|
||||
value=value_number,
|
||||
unit=final_unit,
|
||||
source_token=_quantity_surface_token(value_token),
|
||||
sentence_index=sentence_index,
|
||||
)
|
||||
|
||||
|
||||
def _comparatives(statements: list[str], problem_text: str) -> list[_Comparative]:
|
||||
out: list[_Comparative] = []
|
||||
discourse_subject = _first_proper_noun(problem_text)
|
||||
for idx, sentence in enumerate(statements):
|
||||
for pattern in (
|
||||
_THE_NUMBER_AS_RE,
|
||||
_EXPLICIT_AS_RE,
|
||||
_GREATER_THAN_RE,
|
||||
_DOUBLE_WHAT_RE,
|
||||
_THAT_MANY_RE,
|
||||
_IMPLICIT_GENDER_RE,
|
||||
):
|
||||
match = pattern.search(sentence)
|
||||
if match is None:
|
||||
continue
|
||||
factor = _parse_factor(match.group("factor"))
|
||||
if factor is None:
|
||||
continue
|
||||
ref = match.groupdict().get("ref")
|
||||
unit = match.groupdict().get("unit")
|
||||
implicit_kind = None
|
||||
if pattern is _THAT_MANY_RE:
|
||||
implicit_kind = "that_many"
|
||||
elif pattern is _IMPLICIT_GENDER_RE:
|
||||
implicit_kind = "gender_pair"
|
||||
actor = _actor_from_prefix(match.group("prefix"), discourse_subject)
|
||||
if actor is None and implicit_kind in {"that_many", "gender_pair"} and unit:
|
||||
actor = _clean_entity(unit)
|
||||
if actor is None:
|
||||
continue
|
||||
out.append(
|
||||
_Comparative(
|
||||
actor=actor,
|
||||
factor=factor,
|
||||
factor_token=_quantity_surface_token(match.group("factor")),
|
||||
unit=unit,
|
||||
reference=_clean_reference(ref) if ref else None,
|
||||
sentence_index=idx,
|
||||
source_span=sentence,
|
||||
implicit_kind=implicit_kind,
|
||||
)
|
||||
)
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _bind_reference(comparative: _Comparative, facts: list[_Fact]) -> _Fact | None:
|
||||
if comparative.reference:
|
||||
for fact in reversed(facts):
|
||||
if _entity_matches(fact.entity, comparative.reference):
|
||||
return fact
|
||||
return None
|
||||
|
||||
if comparative.implicit_kind == "gender_pair":
|
||||
target_unit = _canonical_unit(comparative.unit or "")
|
||||
candidates = [
|
||||
f for f in facts
|
||||
if f.sentence_index < comparative.sentence_index
|
||||
and f.unit == target_unit
|
||||
and _is_gender_counterpart(f.entity, comparative.actor)
|
||||
]
|
||||
return candidates[0] if len(candidates) == 1 else None
|
||||
|
||||
if comparative.implicit_kind == "that_many":
|
||||
target_unit = _canonical_unit(comparative.unit or "")
|
||||
candidates = [
|
||||
f for f in facts
|
||||
if f.sentence_index < comparative.sentence_index and f.unit == target_unit
|
||||
]
|
||||
return candidates[0] if len(candidates) == 1 else None
|
||||
return None
|
||||
|
||||
|
||||
def _graph(ref: _Fact, comparative: _Comparative, unit: str) -> MathProblemGraph:
|
||||
entities = (ref.entity, comparative.actor)
|
||||
return MathProblemGraph(
|
||||
entities=entities,
|
||||
initial_state=(
|
||||
InitialPossession(ref.entity, Quantity(ref.value, unit)),
|
||||
),
|
||||
operations=(
|
||||
Operation(
|
||||
actor=comparative.actor,
|
||||
kind="compare_multiplicative",
|
||||
operand=Comparison(
|
||||
reference_actor=ref.entity,
|
||||
delta=None,
|
||||
factor=comparative.factor,
|
||||
direction="times",
|
||||
),
|
||||
),
|
||||
),
|
||||
unknown=Unknown(entity=None, unit=unit),
|
||||
)
|
||||
|
||||
|
||||
def _parse_value(token: str) -> tuple[float, str | None] | None:
|
||||
raw = token.strip().rstrip(".,?")
|
||||
if raw.startswith("$"):
|
||||
try:
|
||||
return float(raw[1:]), "dollars"
|
||||
except ValueError:
|
||||
return None
|
||||
normalized = _strip_article(raw.lower())
|
||||
if re.fullmatch(r"\d+(?:\.\d+)?", normalized):
|
||||
return float(normalized), None
|
||||
parsed = parse_compound_cardinal(normalized)
|
||||
if parsed is not None:
|
||||
return float(parsed), None
|
||||
resolved = _resolve_value(normalized)
|
||||
if resolved is None:
|
||||
return None
|
||||
return float(resolved.value), resolved.unit_override
|
||||
|
||||
|
||||
def _parse_factor(token: str) -> float | None:
|
||||
raw = token.strip().lower().rstrip(".,?")
|
||||
if raw in {"twice", "double"}:
|
||||
return 2.0
|
||||
if raw == "triple":
|
||||
return 3.0
|
||||
if raw == "quadruple":
|
||||
return 4.0
|
||||
parsed = _parse_value(raw)
|
||||
return parsed[0] if parsed is not None else None
|
||||
|
||||
|
||||
def _question_targets_total(question: str, comparative: _Comparative) -> bool:
|
||||
lowered = question.lower()
|
||||
total_cue = any(
|
||||
cue in lowered
|
||||
for cue in ("total", "altogether", "combined", "together", "both", "in all")
|
||||
)
|
||||
if total_cue:
|
||||
return True
|
||||
if comparative.unit is not None and "how many" in lowered:
|
||||
q_unit = _canonical_unit(_words_after_how_many(question))
|
||||
if q_unit and q_unit == _canonical_unit(comparative.unit):
|
||||
return True
|
||||
if "how much" in lowered and any(word in lowered for word in ("spent", "cost", "accessor")):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _words_after_how_many(question: str) -> str:
|
||||
match = re.search(r"how\s+many\s+([A-Za-z]+(?:\s+[A-Za-z]+)?)", question, re.IGNORECASE)
|
||||
return match.group(1) if match else ""
|
||||
|
||||
|
||||
def _unused_source_quantity_tokens(
|
||||
statements: list[str],
|
||||
ref: _Fact,
|
||||
comparative: _Comparative,
|
||||
) -> tuple[str, ...]:
|
||||
required: list[str] = []
|
||||
for sentence in statements:
|
||||
required.extend(_quantity_surfaces(sentence))
|
||||
consumed = {ref.source_token.lower(), comparative.factor_token.lower()}
|
||||
return tuple(
|
||||
token for token in required
|
||||
if token.lower() not in consumed
|
||||
)
|
||||
|
||||
|
||||
def _quantity_surfaces(text: str) -> tuple[str, ...]:
|
||||
surfaces: list[str] = []
|
||||
for match in re.finditer(r"\$\d+(?:\.\d+)?|\d+(?:\.\d+)?", text):
|
||||
surfaces.append(match.group(0))
|
||||
for match in re.finditer(r"\b(twice|double|triple|quadruple)\b", text, re.IGNORECASE):
|
||||
surfaces.append(match.group(1).lower())
|
||||
word = (
|
||||
r"(?:a\s+|an\s+)?(?:one|two|three|four|five|six|seven|eight|nine|ten|"
|
||||
r"eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|"
|
||||
r"nineteen|twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|"
|
||||
r"hundred|thousand)(?:[-\s]+(?:one|two|three|four|five|six|seven|"
|
||||
r"eight|nine|ten|eleven|twelve|thirteen|fourteen|fifteen|sixteen|"
|
||||
r"seventeen|eighteen|nineteen|twenty|thirty|forty|fifty|sixty|"
|
||||
r"seventy|eighty|ninety|hundred|thousand))*"
|
||||
)
|
||||
for match in re.finditer(rf"\b{word}\b", text, re.IGNORECASE):
|
||||
token = _quantity_surface_token(match.group(0))
|
||||
if token.lower() not in _MULTIPLIER_WORDS:
|
||||
surfaces.append(token)
|
||||
return tuple(dict.fromkeys(surfaces))
|
||||
|
||||
|
||||
def _quantity_surface_token(token: str) -> str:
|
||||
raw = token.strip().lower().rstrip(".,?")
|
||||
raw = _strip_article(raw)
|
||||
parts = [p for p in re.split(r"[\s-]+", raw) if p and p != "and"]
|
||||
return " ".join(parts) if parts else raw
|
||||
|
||||
|
||||
def _canonical_unit(unit: str) -> str:
|
||||
raw = re.sub(r"[^A-Za-z\s]", "", unit).lower().strip()
|
||||
if not raw:
|
||||
return ""
|
||||
tokens = tuple(t for t in raw.split() if t not in {"of", "the", "a", "an", "as"})
|
||||
if not tokens:
|
||||
return ""
|
||||
if "student" in tokens or "students" in tokens:
|
||||
return "students"
|
||||
if any(t in {"lady", "ladies", "girl", "girls", "boy", "boys", "people", "person"} for t in tokens):
|
||||
return "people"
|
||||
if "credit" in tokens or "credits" in tokens:
|
||||
return "credits"
|
||||
head = tokens[-1]
|
||||
if head.endswith("ies"):
|
||||
return head[:-3] + "ies"
|
||||
if head.endswith("s") or head in {"dice"}:
|
||||
return head
|
||||
return head + "s"
|
||||
|
||||
|
||||
def _actor_from_prefix(prefix: str, discourse_subject: str | None) -> str | None:
|
||||
text = re.sub(r"^[,.\s]+", "", prefix.strip())
|
||||
text = re.sub(r"^(?:if|and|but|while|meanwhile)\s+", "", text, flags=re.IGNORECASE)
|
||||
cost_match = re.search(r"cost\s+of\s+(?:the\s+)?(?P<actor>[A-Za-z][A-Za-z\s]+?)\s+(?:was|is|were|are)?$", text, re.IGNORECASE)
|
||||
if cost_match:
|
||||
return _clean_entity(cost_match.group("actor"))
|
||||
there_match = re.search(r"there\s+(?:are|were|is|was)\s*$", text, re.IGNORECASE)
|
||||
if there_match:
|
||||
return None
|
||||
verb_match = re.search(rf"(?P<actor>.+?)\s+{_VERB}\s*$", text, re.IGNORECASE)
|
||||
actor_raw = verb_match.group("actor") if verb_match else text
|
||||
actor_raw = re.sub(r"^.*\bcolleague\s+", "", actor_raw, flags=re.IGNORECASE)
|
||||
actor_raw = actor_raw.strip()
|
||||
if actor_raw.lower() in {"he", "she", "his", "her"}:
|
||||
return discourse_subject
|
||||
return _clean_entity(actor_raw)
|
||||
|
||||
|
||||
def _clean_reference(raw: str | None) -> str | None:
|
||||
if raw is None:
|
||||
return None
|
||||
text = re.split(r"\b(?:has|have|had|is|are|was|were|teaches|teach|scored)\b", raw, maxsplit=1, flags=re.IGNORECASE)[0]
|
||||
return _clean_entity(text)
|
||||
|
||||
|
||||
def _clean_entity(raw: str) -> str:
|
||||
text = re.sub(r"[^A-Za-z0-9\s]", " ", raw)
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
text = re.sub(r"^(?:if|the|a|an|his|her|their|this|that)\s+", "", text, flags=re.IGNORECASE)
|
||||
text = re.sub(r"\s+(?:who|which|that)$", "", text, flags=re.IGNORECASE)
|
||||
if not text:
|
||||
return ""
|
||||
lowered = text.lower()
|
||||
if lowered in {"dad", "father"}:
|
||||
return "dad"
|
||||
if lowered in {"mom", "mother"}:
|
||||
return "mom"
|
||||
if "female student" in lowered:
|
||||
return "female students"
|
||||
if "male student" in lowered:
|
||||
return "male students"
|
||||
return " ".join(part.capitalize() if part.islower() else part for part in text.split())
|
||||
|
||||
|
||||
def _first_proper_noun(text: str) -> str | None:
|
||||
match = re.search(r"\b([A-Z][a-z]+)\b", text)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def _entity_matches(entity: str, reference: str) -> bool:
|
||||
e = _entity_key(entity)
|
||||
r = _entity_key(reference)
|
||||
return e == r or e.endswith(r) or r.endswith(e)
|
||||
|
||||
|
||||
def _entity_key(entity: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", "", entity.lower())
|
||||
|
||||
|
||||
def _is_gender_counterpart(source: str, actor: str) -> bool:
|
||||
s = set(source.lower().split())
|
||||
a = set(actor.lower().split())
|
||||
return (
|
||||
("female" in s and "male" in a)
|
||||
or ("male" in s and "female" in a)
|
||||
or ("girls" in s and "boys" in a)
|
||||
or ("boys" in s and "girls" in a)
|
||||
)
|
||||
|
||||
|
||||
def _dedupe_facts(facts: list[_Fact]) -> list[_Fact]:
|
||||
out: list[_Fact] = []
|
||||
seen: set[tuple[str, str, float, int]] = set()
|
||||
for fact in facts:
|
||||
key = (_entity_key(fact.entity), fact.unit, fact.value, fact.sentence_index)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(fact)
|
||||
return out
|
||||
|
||||
|
||||
def _strip_article(text: str) -> str:
|
||||
return re.sub(r"^(?:a|an)\s+", "", text.strip(), flags=re.IGNORECASE)
|
||||
|
||||
|
||||
def _refuse(reason: str, trace: list[str], **detail: object) -> R1Reconstruction:
|
||||
trace.append(_event("refused", reason=reason, **detail))
|
||||
return R1Reconstruction(
|
||||
graph=None,
|
||||
answer=None,
|
||||
reader_trace=tuple(trace),
|
||||
refusal_reason=reason,
|
||||
)
|
||||
|
||||
|
||||
def _event(outcome: str, **payload: object) -> str:
|
||||
data = {"layer": "r1_reconstruction", "outcome": outcome}
|
||||
data.update(payload)
|
||||
return json.dumps(data, sort_keys=True, separators=(",", ":"))
|
||||
|
|
@ -61,6 +61,7 @@ from generate.math_problem_graph import (
|
|||
MathProblemGraph,
|
||||
)
|
||||
from generate.math_completeness import uncovered_quantities
|
||||
from generate.derivation.r1_reconstruction import reconstruct_r1_total
|
||||
from generate.math_roundtrip import CandidateOperation, roundtrip_admissible
|
||||
from generate.math_solver import SolveError, solve
|
||||
|
||||
|
|
@ -69,6 +70,41 @@ MAX_TOTAL_BRANCHES: Final[int] = 64
|
|||
"""Hard cap on Cartesian-product branch enumeration; exceeding refuses."""
|
||||
|
||||
|
||||
def _try_r1_reconstruction(
|
||||
text: str,
|
||||
*,
|
||||
existing_trace: tuple[str, ...],
|
||||
) -> CandidateGraphResult | None:
|
||||
"""Attempt the narrow R1 typed reconstruction path.
|
||||
|
||||
Returns None when the text has no R1 signal. A non-admitted R1 attempt is
|
||||
still returned so its deterministic refusal evidence can be surfaced in the
|
||||
reader trace while preserving the caller's refusal posture.
|
||||
"""
|
||||
r1 = reconstruct_r1_total(text)
|
||||
if r1 is None:
|
||||
return None
|
||||
trace = (*existing_trace, *r1.reader_trace)
|
||||
if r1.is_admitted:
|
||||
assert r1.answer is not None
|
||||
return CandidateGraphResult(
|
||||
answer=r1.answer,
|
||||
selected_graph=r1.graph,
|
||||
refusal_reason=None,
|
||||
branches_enumerated=1,
|
||||
branches_admissible=1,
|
||||
reader_trace=trace,
|
||||
)
|
||||
return CandidateGraphResult(
|
||||
answer=None,
|
||||
selected_graph=None,
|
||||
refusal_reason=f"r1_reconstruction: {r1.refusal_reason}",
|
||||
branches_enumerated=0,
|
||||
branches_admissible=0,
|
||||
reader_trace=trace,
|
||||
)
|
||||
|
||||
|
||||
def _load_ratified_registry_or_empty() -> tuple:
|
||||
"""Return the ratified recognizer registry, or () on any failure.
|
||||
|
||||
|
|
@ -972,6 +1008,14 @@ def parse_and_solve(text: str, *, sealed: bool = False) -> CandidateGraphResult:
|
|||
# "I don't know" — i.e. refuse. When an injector is
|
||||
# added that handles this shape, this branch becomes
|
||||
# dead and can be retired.
|
||||
r1_result = _try_r1_reconstruction(
|
||||
text,
|
||||
existing_trace=tuple(_statement_trace),
|
||||
)
|
||||
if r1_result is not None and r1_result.is_admitted:
|
||||
return r1_result
|
||||
if r1_result is not None:
|
||||
_statement_trace = list(r1_result.reader_trace)
|
||||
return CandidateGraphResult(
|
||||
answer=None, selected_graph=None,
|
||||
refusal_reason=(
|
||||
|
|
@ -986,6 +1030,14 @@ def parse_and_solve(text: str, *, sealed: bool = False) -> CandidateGraphResult:
|
|||
# constraint_propagation eliminations, etc.).
|
||||
reader_trace=tuple(_statement_trace),
|
||||
)
|
||||
r1_result = _try_r1_reconstruction(
|
||||
text,
|
||||
existing_trace=tuple(_statement_trace),
|
||||
)
|
||||
if r1_result is not None and r1_result.is_admitted:
|
||||
return r1_result
|
||||
if r1_result is not None:
|
||||
_statement_trace = list(r1_result.reader_trace)
|
||||
return CandidateGraphResult(
|
||||
answer=None, selected_graph=None,
|
||||
refusal_reason=f"no admissible candidate for statement: {s!r}",
|
||||
|
|
@ -1100,6 +1152,14 @@ def parse_and_solve(text: str, *, sealed: bool = False) -> CandidateGraphResult:
|
|||
branch=chosen.branch,
|
||||
)
|
||||
if uncovered:
|
||||
r1_result = _try_r1_reconstruction(
|
||||
text,
|
||||
existing_trace=tuple(reader_trace),
|
||||
)
|
||||
if r1_result is not None and r1_result.is_admitted:
|
||||
return r1_result
|
||||
if r1_result is not None:
|
||||
reader_trace = list(r1_result.reader_trace)
|
||||
return CandidateGraphResult(
|
||||
answer=None, selected_graph=None,
|
||||
refusal_reason=(
|
||||
|
|
|
|||
|
|
@ -15,9 +15,9 @@ This guard adds the missing leg, mirroring the derivation reader's
|
|||
|
||||
The guard is REFUSAL-ONLY: it can never turn a refusal into an answer,
|
||||
so it cannot create a wrong answer — it can only remove confabulations.
|
||||
Its entire regression surface is the graph-path correct set, which on
|
||||
train_sample is exactly {0024} and on real-train is {3343} (the same
|
||||
Sidney/Brooke shape). Both MUST still solve.
|
||||
R1 reconstruction now safely admits the Ivan/Jerry comparative case through a
|
||||
typed graph and independent verifier, so that case is no longer a permanent
|
||||
refusal pin. The remaining confabulations MUST still refuse.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
|
|
@ -25,18 +25,15 @@ import pytest
|
|||
|
||||
from generate.math_candidate_graph import parse_and_solve
|
||||
|
||||
# The 5 real-GSM8K confabulations (exact corpus strings). Each MUST now
|
||||
# refuse (answer is None) instead of emitting a partial reading.
|
||||
# Real-GSM8K confabulations that remain outside the current decidable regime
|
||||
# (exact corpus strings). Each MUST refuse (answer is None) instead of emitting
|
||||
# a partial reading.
|
||||
CONFABULATIONS = {
|
||||
553: (
|
||||
"Emma buys 2 containers of milk every school day for lunch. She does "
|
||||
"not go to school on the weekends. How many containers of milk does "
|
||||
"she buy in 3 weeks?"
|
||||
),
|
||||
605: (
|
||||
"Ivan has 20 dice. Jerry has twice as many dice as Ivan. How many "
|
||||
"dice do they have altogether?"
|
||||
),
|
||||
693: (
|
||||
"Ian had twenty roses. He gave six roses to his mother, nine roses "
|
||||
"to his grandmother, four roses to his sister, and he kept the rest. "
|
||||
|
|
@ -142,8 +139,6 @@ def test_n_times_as_many_with_reference_still_solves() -> None:
|
|||
"How many apples do they have together?",
|
||||
"Tom has 7 apples. Jerry has double the apples. "
|
||||
"How many apples do they have together?",
|
||||
"Ivan has 20 dice. Jerry has twice as many dice as Ivan. "
|
||||
"How many dice do they have altogether?",
|
||||
],
|
||||
)
|
||||
def test_existing_multiplier_refusals_stay_refused(question: str) -> None:
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ It enforces two kinds of obligation:
|
|||
- frozen baseline snapshot for those non-positive rows matches the live tree.
|
||||
|
||||
* **Current snapshot** (the one assertion a Phase 5b slice updates when it
|
||||
flips a positive): the aggregate is ``4 solve / 16 refuse / 0 wrong`` today.
|
||||
flips a positive): the aggregate is ``6 solve / 16 refuse / 0 wrong`` today.
|
||||
|
||||
A future positive (``gate`` like ``5b-R1``) is *expected* to flip
|
||||
refuse -> solve when its slice lands; that flip must still satisfy the firewall,
|
||||
|
|
@ -158,7 +158,7 @@ def test_frozen_baseline_fields_match_tree(case: dict) -> None:
|
|||
|
||||
|
||||
def test_current_baseline_snapshot() -> None:
|
||||
"""Current aggregate is 3 solve / 19 refuse / 0 wrong.
|
||||
"""Current aggregate is 6 solve / 16 refuse / 0 wrong.
|
||||
|
||||
This is the single assertion a Phase 5b slice updates when it flips a
|
||||
positive (refuse -> solve); the forever-invariants above do not change.
|
||||
|
|
@ -166,6 +166,10 @@ def test_current_baseline_snapshot() -> None:
|
|||
goal_residual) were disabled after the first real sealed measurement showed
|
||||
them 0-correct/5-wrong on held-out — so cv-0005 (R4) and cv-0020 (product)
|
||||
revert to refusing, moving the honest snapshot to 3/19.
|
||||
|
||||
R1 reconstruction then flips cv-0001, cv-0002, and cv-0009 through typed
|
||||
graph reconstruction + solver/verifier replay, moving the honest snapshot
|
||||
to 6/16.
|
||||
"""
|
||||
solve = refuse = wrong = 0
|
||||
for case in _CASES:
|
||||
|
|
@ -177,7 +181,7 @@ def test_current_baseline_snapshot() -> None:
|
|||
else:
|
||||
refuse += 1
|
||||
assert wrong == 0
|
||||
assert (solve, refuse) == (3, 19), (
|
||||
assert (solve, refuse) == (6, 16), (
|
||||
f"snapshot moved to {solve} solve / {refuse} refuse — if a Phase 5b "
|
||||
f"slice landed, update this expectation and the affected rows' "
|
||||
f"baseline fields in lockstep"
|
||||
|
|
|
|||
116
tests/test_gsm8k_r1_reconstruction.py
Normal file
116
tests/test_gsm8k_r1_reconstruction.py
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
"""R1 GSM8K reconstruction: explicit comparative-derived totals.
|
||||
|
||||
Pins the first answer-changing reconstruction slice. Admissions must flow
|
||||
through a real MathProblemGraph and the independent verifier; no fast-path numeric
|
||||
answer is allowed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from generate.derivation import reconstruct_r1_total
|
||||
from generate.math_candidate_graph import parse_and_solve
|
||||
from generate.math_solver import solve
|
||||
from generate.math_verifier import verify
|
||||
|
||||
|
||||
R1_POSITIVES = {
|
||||
"cv-0001": (
|
||||
"Fabian bought a brand new computer mouse and keyboard to be able to work "
|
||||
"from home. The cost of the keyboard was three times greater than the "
|
||||
"cost of the mouse. If the mouse cost $16, how much did Fabian spent on "
|
||||
"his new accessories?",
|
||||
64.0,
|
||||
),
|
||||
"cv-0002": (
|
||||
"In a building, there are a hundred ladies on the first-floor studying. "
|
||||
"There are three times that many girls at a party being held on the "
|
||||
"second floor of the building. How many ladies are on the two floors in total?",
|
||||
400.0,
|
||||
),
|
||||
"cv-0009": (
|
||||
"Ivan has 20 dice. Jerry has twice as many dice as Ivan. "
|
||||
"How many dice do they have altogether?",
|
||||
60.0,
|
||||
),
|
||||
"heldout-0101": (
|
||||
"Eduardo is a teacher. He taught 3 classes last week while his colleague "
|
||||
"Frankie taught double what Eduardo teaches. How many classes did Eduardo "
|
||||
"and Frankie teach in total?",
|
||||
9.0,
|
||||
),
|
||||
"heldout-0108": (
|
||||
"Dana Point beach has four times the number of sharks as Newport Beach. "
|
||||
"If Newport Beach has 22 sharks, how many sharks are there in total on "
|
||||
"the two beaches?",
|
||||
110.0,
|
||||
),
|
||||
"heldout-0411": (
|
||||
"In a class, there were 13 female students. There were three times as "
|
||||
"many male students in this class. How many students were in the class?",
|
||||
52.0,
|
||||
),
|
||||
"heldout-0453": (
|
||||
"Olaf is playing a game with his dad. He scored three times more points "
|
||||
"than his dad, who scored 7 points. How many points did they score in total?",
|
||||
28.0,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case_id", sorted(R1_POSITIVES))
|
||||
def test_r1_reconstruction_solves_with_verified_graph(case_id: str) -> None:
|
||||
text, expected = R1_POSITIVES[case_id]
|
||||
result = reconstruct_r1_total(text)
|
||||
|
||||
assert result is not None
|
||||
assert result.is_admitted
|
||||
assert result.answer == expected
|
||||
assert result.graph is not None
|
||||
|
||||
trace = solve(result.graph)
|
||||
verdict = verify(result.graph, trace)
|
||||
assert verdict.passed is True
|
||||
assert trace.answer_value == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case_id", sorted(R1_POSITIVES))
|
||||
def test_parse_and_solve_wires_r1_only_as_verified_graph(case_id: str) -> None:
|
||||
text, expected = R1_POSITIVES[case_id]
|
||||
result = parse_and_solve(text)
|
||||
|
||||
assert result.is_admitted
|
||||
assert result.answer == expected
|
||||
assert result.selected_graph is not None
|
||||
assert any("r1_reconstruction" in event for event in result.reader_trace)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
(
|
||||
"Tom has 7 apples. Jerry has 3 times as many apples. "
|
||||
"How many apples do they have together?"
|
||||
),
|
||||
(
|
||||
"Over several years, Daniel has adopted any stray animals he sees on "
|
||||
"the side of the road. He now has 2 horses, 5 dogs, 7 cats, 3 turtles, "
|
||||
"and 1 goat. All of the animals are perfectly healthy. In total, "
|
||||
"how many legs do his animals have?"
|
||||
),
|
||||
(
|
||||
"Mark does a gig every other day for 2 weeks. For each gig, he plays "
|
||||
"3 songs. 2 of the songs are 5 minutes long and the last song is "
|
||||
"twice that long. How many minutes did he play?"
|
||||
),
|
||||
(
|
||||
"Sam has 7 apples. Tom has 4 oranges. Jerry has twice as many apples "
|
||||
"as Sam. How many apples and oranges do they have together?"
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_r1_reconstruction_refuses_out_of_scope_shapes(text: str) -> None:
|
||||
result = parse_and_solve(text)
|
||||
assert not result.is_admitted
|
||||
|
||||
Loading…
Reference in a new issue